diff --git a/CHANGELOG.md b/CHANGELOG.md index 6734b2874fd92..74f5db1f851d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +## 14.4.0 (Upcoming) + +While Teleport 14 is discontinued and out-of-support, some users are still running it. +To help updating to a supported version, we are issuing exceptional v14 releases to +backport Managed Updates v2. + +Managed Updates v2 offer a smoother upgrade path for existing users and should reduce +the cost and pain of updating Teleport agents in large deployments. + +### Automatic Updates + +14.4 introduces a new automatic update mechanism for system administrators to control which Teleport version their +agents are running. You can now configure the agent update schedule and desired agent version via the `autoupdate_config` +and `autoupdate_version` resources. + +Updates are performed by the new `teleport-update` binary. +This new system is package manager-agnostic and opt-in. Existing agents won't be automatically enrolled, you can enroll +existing 14.4+ agents by running `teleport-update enable`. + +`teleport-update` will become the new standard way of installing Teleport as it always picks the appropriate Teleport +edition (Community vs Enterprise), the cluster's desired version, and the correct Teleport variant (e.g. FIPS-compliant +cryptography). + +You can find more information about the feature in [our documentation](). + +### Package layout changes + +Starting with 14.4.0, the Teleport DEB and RPM packages, notably used by the `apt`, `yum`, `dnf` and `zypper` package +managers, will place the Teleport binaries in `/opt/teleport` instead of `/usr/local/bin`. + +The binaries will be symlinked to their previous location, no change should be required in your scripts or systemd units. + +This change allows us to do automatic updates without conflicting with the package manager. + ## 14.3.36 (02/13/25) ### Security Fixes diff --git a/Makefile b/Makefile index f36b95d6e91ff..8457ceb94a0be 100644 --- a/Makefile +++ b/Makefile @@ -48,9 +48,12 @@ GO_LDFLAGS ?= -w -s $(KUBECTL_SETVERSION) ifeq ("$(TELEPORT_DEBUG)","true") BUILDFLAGS ?= $(ADDFLAGS) -gcflags=all="-N -l" BUILDFLAGS_TBOT ?= $(ADDFLAGS) -gcflags=all="-N -l" +BUILDFLAGS_TELEPORT_UPDATE ?= $(ADDFLAGS) -gcflags=all="-N -l" else BUILDFLAGS ?= $(ADDFLAGS) -ldflags '$(GO_LDFLAGS)' -trimpath BUILDFLAGS_TBOT ?= $(ADDFLAGS) -ldflags '$(GO_LDFLAGS)' -trimpath +# teleport-update builds with disabled cgo, buildmode=pie is not required. +BUILDFLAGS_TELEPORT_UPDATE ?= $(ADDFLAGS) -ldflags '$(GO_LDFLAGS)' -trimpath endif GO_ENV_OS := $(shell go env GOOS) @@ -203,7 +206,8 @@ endif # On Windows only build tsh. On all other platforms build teleport, tctl, # and tsh. -BINS_default = teleport tctl tsh tbot +BINS_default = teleport tctl tsh tbot teleport-update +BINS_darwin = teleport tctl tsh tbot BINS_windows = tsh BINS = $(or $(BINS_$(OS)),$(BINS_default)) BINARIES = $(addprefix $(BUILDDIR)/,$(BINS)) @@ -273,6 +277,8 @@ endif CGOFLAG = CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ BUILDFLAGS = $(ADDFLAGS) -ldflags '-w -s $(KUBECTL_SETVERSION)' -trimpath -buildmode=exe BUILDFLAGS_TBOT = $(ADDFLAGS) -ldflags '-w -s $(KUBECTL_SETVERSION)' -trimpath +# teleport-update builds with disabled cgo, buildmode=pie is not required. +BUILDFLAGS_TELEPORT_UPDATE = $(ADDFLAGS) -ldflags '-w -s $(KUBECTL_SETVERSION)' -trimpath endif ifeq ("$(OS)","darwin") @@ -347,6 +353,10 @@ else GOOS=$(OS) GOARCH=$(ARCH) CGO_ENABLED=0 go build -tags "$(FIPS_TAG)" -o $(BUILDDIR)/tbot $(BUILDFLAGS_TBOT) ./tool/tbot endif +.PHONY: $(BUILDDIR)/teleport-update +$(BUILDDIR)/teleport-update: + GOOS=$(OS) GOARCH=$(ARCH) CGO_ENABLED=0 go build -o $(BUILDDIR)/teleport-update $(BUILDFLAGS_TELEPORT_UPDATE) ./tool/teleport-update + # # BPF support (IF ENABLED) # Requires a recent version of clang and libbpf installed. @@ -1446,10 +1456,11 @@ goinstall: .PHONY: install install: build @echo "\n** Make sure to run 'make install' as root! **\n" - cp -f $(BUILDDIR)/tctl $(BINDIR)/ - cp -f $(BUILDDIR)/tsh $(BINDIR)/ - cp -f $(BUILDDIR)/tbot $(BINDIR)/ - cp -f $(BUILDDIR)/teleport $(BINDIR)/ + cp -f $(BUILDDIR)/tctl $(BINDIR)/ + cp -f $(BUILDDIR)/tsh $(BINDIR)/ + cp -f $(BUILDDIR)/tbot $(BINDIR)/ + cp -f $(BUILDDIR)/teleport $(BINDIR)/ + cp -f $(BUILDDIR)/teleport-update $(BINDIR)/ mkdir -p $(DATADIR) # Docker image build. Always build the binaries themselves within docker (see diff --git a/api/client/client.go b/api/client/client.go index 8c13833c2b8f9..38b45ceec7eb4 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -2825,6 +2825,59 @@ func (c *Client) DeleteAutoUpdateVersion(ctx context.Context) error { return trace.Wrap(err) } +// CreateAutoUpdateAgentRollout creates AutoUpdateAgentRollout resource. +func (c *Client) CreateAutoUpdateAgentRollout(ctx context.Context, rollout *autoupdatev1pb.AutoUpdateAgentRollout) (*autoupdatev1pb.AutoUpdateAgentRollout, error) { + client := autoupdatev1pb.NewAutoUpdateServiceClient(c.conn) + resp, err := client.CreateAutoUpdateAgentRollout(ctx, &autoupdatev1pb.CreateAutoUpdateAgentRolloutRequest{ + Rollout: rollout, + }) + if err != nil { + return nil, trace.Wrap(err) + } + return resp, nil +} + +// GetAutoUpdateAgentRollout gets AutoUpdateAgentRollout resource. +func (c *Client) GetAutoUpdateAgentRollout(ctx context.Context) (*autoupdatev1pb.AutoUpdateAgentRollout, error) { + client := autoupdatev1pb.NewAutoUpdateServiceClient(c.conn) + resp, err := client.GetAutoUpdateAgentRollout(ctx, &autoupdatev1pb.GetAutoUpdateAgentRolloutRequest{}) + if err != nil { + return nil, trace.Wrap(err) + } + return resp, nil +} + +// UpdateAutoUpdateAgentRollout updates AutoUpdateAgentRollout resource. +func (c *Client) UpdateAutoUpdateAgentRollout(ctx context.Context, rollout *autoupdatev1pb.AutoUpdateAgentRollout) (*autoupdatev1pb.AutoUpdateAgentRollout, error) { + client := autoupdatev1pb.NewAutoUpdateServiceClient(c.conn) + resp, err := client.UpdateAutoUpdateAgentRollout(ctx, &autoupdatev1pb.UpdateAutoUpdateAgentRolloutRequest{ + Rollout: rollout, + }) + if err != nil { + return nil, trace.Wrap(err) + } + return resp, nil +} + +// UpsertAutoUpdateAgentRollout updates or creates AutoUpdateAgentRollout resource. +func (c *Client) UpsertAutoUpdateAgentRollout(ctx context.Context, rollout *autoupdatev1pb.AutoUpdateAgentRollout) (*autoupdatev1pb.AutoUpdateAgentRollout, error) { + client := autoupdatev1pb.NewAutoUpdateServiceClient(c.conn) + resp, err := client.UpsertAutoUpdateAgentRollout(ctx, &autoupdatev1pb.UpsertAutoUpdateAgentRolloutRequest{ + Rollout: rollout, + }) + if err != nil { + return nil, trace.Wrap(err) + } + return resp, nil +} + +// DeleteAutoUpdateAgentRollout deletes AutoUpdateAgentRollout resource. +func (c *Client) DeleteAutoUpdateAgentRollout(ctx context.Context) error { + client := autoupdatev1pb.NewAutoUpdateServiceClient(c.conn) + _, err := client.DeleteAutoUpdateAgentRollout(ctx, &autoupdatev1pb.DeleteAutoUpdateAgentRolloutRequest{}) + return trace.Wrap(err) +} + // GetClusterAccessGraphConfig retrieves the Cluster Access Graph configuration from Auth server. func (c *Client) GetClusterAccessGraphConfig(ctx context.Context) (*clusterconfigpb.AccessGraphConfig, error) { rsp, err := c.ClusterConfigClient().GetClusterAccessGraphConfig(ctx, &clusterconfigpb.GetClusterAccessGraphConfigRequest{}) diff --git a/api/client/events.go b/api/client/events.go index 98fa50defe09d..fd49a3b3231df 100644 --- a/api/client/events.go +++ b/api/client/events.go @@ -65,6 +65,12 @@ func EventToGRPC(in types.Event) (*proto.Event, error) { out.Resource = &proto.Event_AutoUpdateVersion{ AutoUpdateVersion: r, } + case *autoupdate.AutoUpdateAgentRollout: + out.Resource = &proto.Event_AutoUpdateAgentRollout{ + AutoUpdateAgentRollout: r, + } + default: + return nil, trace.BadParameter("resource type %T is not supported", r) } case *types.ResourceHeader: out.Resource = &proto.Event_ResourceHeader{ @@ -485,6 +491,9 @@ func EventFromGRPC(in *proto.Event) (*types.Event, error) { } else if r := in.GetAutoUpdateVersion(); r != nil { out.Resource = types.Resource153ToLegacy(r) return &out, nil + } else if r := in.GetAutoUpdateAgentRollout(); r != nil { + out.Resource = types.Resource153ToLegacy(r) + return &out, nil } else { return nil, trace.BadParameter("received unsupported resource %T", in.Resource) } diff --git a/api/client/proto/event.pb.go b/api/client/proto/event.pb.go index a21853b1cf667..1855475f91013 100644 --- a/api/client/proto/event.pb.go +++ b/api/client/proto/event.pb.go @@ -160,6 +160,7 @@ type Event struct { // *Event_KubernetesWaitingContainer // *Event_AutoUpdateConfig // *Event_AutoUpdateVersion + // *Event_AutoUpdateAgentRollout Resource isEvent_Resource `protobuf_oneof:"Resource"` } @@ -580,6 +581,13 @@ func (x *Event) GetAutoUpdateVersion() *v15.AutoUpdateVersion { return nil } +func (x *Event) GetAutoUpdateAgentRollout() *v15.AutoUpdateAgentRollout { + if x, ok := x.GetResource().(*Event_AutoUpdateAgentRollout); ok { + return x.AutoUpdateAgentRollout + } + return nil +} + type isEvent_Resource interface { isEvent_Resource() } @@ -852,6 +860,11 @@ type Event_AutoUpdateVersion struct { AutoUpdateVersion *v15.AutoUpdateVersion `protobuf:"bytes,65,opt,name=AutoUpdateVersion,proto3,oneof"` } +type Event_AutoUpdateAgentRollout struct { + // AutoUpdateVersion is a resource for controlling the autoupdate agent rollout. + AutoUpdateAgentRollout *v15.AutoUpdateAgentRollout `protobuf:"bytes,71,opt,name=AutoUpdateAgentRollout,proto3,oneof"` +} + func (*Event_ResourceHeader) isEvent_Resource() {} func (*Event_CertAuthority) isEvent_Resource() {} @@ -958,6 +971,8 @@ func (*Event_AutoUpdateConfig) isEvent_Resource() {} func (*Event_AutoUpdateVersion) isEvent_Resource() {} +func (*Event_AutoUpdateAgentRollout) isEvent_Resource() {} + var File_teleport_legacy_client_proto_event_proto protoreflect.FileDescriptor var file_teleport_legacy_client_proto_event_proto_rawDesc = []byte{ @@ -984,7 +999,7 @@ var file_teleport_legacy_client_proto_event_proto_rawDesc = []byte{ 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x75, 0x73, 0x65, 0x72, 0x6c, 0x6f, 0x67, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x75, 0x73, 0x65, 0x72, 0x6c, 0x6f, 0x67, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x22, 0xbc, 0x1c, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x24, 0x0a, 0x04, 0x54, 0x79, + 0x22, 0xd7, 0x1d, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x24, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x3f, 0x0a, 0x0e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x65, 0x61, 0x64, @@ -1209,16 +1224,26 @@ var file_teleport_legacy_client_proto_event_proto_rawDesc = []byte{ 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x11, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x42, 0x0a, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4a, - 0x04, 0x08, 0x07, 0x10, 0x08, 0x4a, 0x04, 0x08, 0x31, 0x10, 0x32, 0x52, 0x12, 0x45, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x41, 0x75, 0x64, 0x69, 0x74, 0x2a, - 0x2a, 0x0a, 0x09, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x08, 0x0a, 0x04, - 0x49, 0x4e, 0x49, 0x54, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x50, 0x55, 0x54, 0x10, 0x01, 0x12, - 0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x42, 0x34, 0x5a, 0x32, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72, 0x61, 0x76, 0x69, 0x74, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, - 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x69, 0x6f, 0x6e, 0x12, 0x68, 0x0a, 0x16, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x18, 0x47, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, + 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, + 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, + 0x6f, 0x75, 0x74, 0x48, 0x00, 0x52, 0x16, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x42, 0x0a, 0x0a, + 0x08, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4a, 0x04, 0x08, 0x07, 0x10, 0x08, 0x4a, + 0x04, 0x08, 0x31, 0x10, 0x32, 0x4a, 0x04, 0x08, 0x3f, 0x10, 0x40, 0x4a, 0x04, 0x08, 0x44, 0x10, + 0x45, 0x52, 0x12, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x43, 0x6c, 0x6f, 0x75, 0x64, + 0x41, 0x75, 0x64, 0x69, 0x74, 0x52, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x63, 0x48, 0x6f, 0x73, + 0x74, 0x55, 0x73, 0x65, 0x72, 0x52, 0x13, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x50, 0x6c, 0x61, 0x6e, 0x2a, 0x2a, 0x0a, 0x09, 0x4f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x49, 0x54, 0x10, + 0x00, 0x12, 0x07, 0x0a, 0x03, 0x50, 0x55, 0x54, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x45, + 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72, 0x61, 0x76, 0x69, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x61, 0x6c, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x70, 0x69, 0x2f, + 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1288,6 +1313,7 @@ var file_teleport_legacy_client_proto_event_proto_goTypes = []interface{}{ (*v14.KubernetesWaitingContainer)(nil), // 49: teleport.kubewaitingcontainer.v1.KubernetesWaitingContainer (*v15.AutoUpdateConfig)(nil), // 50: teleport.autoupdate.v1.AutoUpdateConfig (*v15.AutoUpdateVersion)(nil), // 51: teleport.autoupdate.v1.AutoUpdateVersion + (*v15.AutoUpdateAgentRollout)(nil), // 52: teleport.autoupdate.v1.AutoUpdateAgentRollout } var file_teleport_legacy_client_proto_event_proto_depIdxs = []int32{ 0, // 0: proto.Event.Type:type_name -> proto.Operation @@ -1344,11 +1370,12 @@ var file_teleport_legacy_client_proto_event_proto_depIdxs = []int32{ 49, // 51: proto.Event.KubernetesWaitingContainer:type_name -> teleport.kubewaitingcontainer.v1.KubernetesWaitingContainer 50, // 52: proto.Event.AutoUpdateConfig:type_name -> teleport.autoupdate.v1.AutoUpdateConfig 51, // 53: proto.Event.AutoUpdateVersion:type_name -> teleport.autoupdate.v1.AutoUpdateVersion - 54, // [54:54] is the sub-list for method output_type - 54, // [54:54] is the sub-list for method input_type - 54, // [54:54] is the sub-list for extension type_name - 54, // [54:54] is the sub-list for extension extendee - 0, // [0:54] is the sub-list for field type_name + 52, // 54: proto.Event.AutoUpdateAgentRollout:type_name -> teleport.autoupdate.v1.AutoUpdateAgentRollout + 55, // [55:55] is the sub-list for method output_type + 55, // [55:55] is the sub-list for method input_type + 55, // [55:55] is the sub-list for extension type_name + 55, // [55:55] is the sub-list for extension extendee + 0, // [0:55] is the sub-list for field type_name } func init() { file_teleport_legacy_client_proto_event_proto_init() } @@ -1424,6 +1451,7 @@ func file_teleport_legacy_client_proto_event_proto_init() { (*Event_KubernetesWaitingContainer)(nil), (*Event_AutoUpdateConfig)(nil), (*Event_AutoUpdateVersion)(nil), + (*Event_AutoUpdateAgentRollout)(nil), } type x struct{} out := protoimpl.TypeBuilder{ diff --git a/api/client/webclient/webclient.go b/api/client/webclient/webclient.go index cdf94bc88b6a0..62d12299ab584 100644 --- a/api/client/webclient/webclient.go +++ b/api/client/webclient/webclient.go @@ -47,6 +47,22 @@ import ( "github.com/gravitational/teleport/api/utils/keys" ) +const ( + // AgentUpdateGroupParameter is the parameter used to specify the updater + // group when doing a Ping() or Find() query. + // The proxy server will modulate the auto_update part of the PingResponse + // based on the specified group. e.g. some groups might need to update + // before others. + AgentUpdateGroupParameter = "group" + + // AgentUpdateIDParameter is the parameter used to specify the updater + // ID during a Ping() or Find() query. + // The proxy server will modulate the auto_update part of the PingResponse + // based on the specified update ID. e.g. canary hosts might need to update + // before others. + AgentUpdateIDParameter = "update_id" +) + // Config specifies information when building requests with the // webclient. type Config struct { @@ -68,6 +84,12 @@ type Config struct { Timeout time.Duration // TraceProvider is used to retrieve a Tracer for creating spans TraceProvider oteltrace.TracerProvider + // UpdateGroup is used to vary the webapi response based on the + // client's Managed Update group. + UpdateGroup string + // UpdateID is used to vary the webapi response based on the + // client's Managed Update ID. + UpdateID string } // CheckAndSetDefaults checks and sets defaults @@ -165,12 +187,28 @@ func Find(cfg *Config) (*PingResponse, error) { } defer clt.CloseIdleConnections() + return findWithClient(cfg, clt) +} + +func findWithClient(cfg *Config, clt *http.Client) (*PingResponse, error) { ctx, span := cfg.TraceProvider.Tracer("webclient").Start(cfg.Context, "webclient/Find") defer span.End() - endpoint := fmt.Sprintf("https://%s/webapi/find", cfg.ProxyAddr) + endpoint := &url.URL{ + Scheme: "https", + Host: cfg.ProxyAddr, + Path: "/webapi/find", + } + query := url.Values{} + if cfg.UpdateGroup != "" { + query[AgentUpdateGroupParameter] = []string{cfg.UpdateGroup} + } + if cfg.UpdateID != "" { + query[AgentUpdateIDParameter] = []string{cfg.UpdateID} + } + endpoint.RawQuery = query.Encode() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil) if err != nil { return nil, trace.Wrap(err) } @@ -201,15 +239,31 @@ func Ping(cfg *Config) (*PingResponse, error) { } defer clt.CloseIdleConnections() + return pingWithClient(cfg, clt) +} + +func pingWithClient(cfg *Config, clt *http.Client) (*PingResponse, error) { ctx, span := cfg.TraceProvider.Tracer("webclient").Start(cfg.Context, "webclient/Ping") defer span.End() - endpoint := fmt.Sprintf("https://%s/webapi/ping", cfg.ProxyAddr) + endpoint := &url.URL{ + Scheme: "https", + Host: cfg.ProxyAddr, + Path: "/webapi/ping", + } + query := url.Values{} + if cfg.UpdateGroup != "" { + query[AgentUpdateGroupParameter] = []string{cfg.UpdateGroup} + } + if cfg.UpdateID != "" { + query[AgentUpdateIDParameter] = []string{cfg.UpdateID} + } + endpoint.RawQuery = query.Encode() if cfg.ConnectorName != "" { - endpoint = fmt.Sprintf("%s/%s", endpoint, cfg.ConnectorName) + endpoint = endpoint.JoinPath(cfg.ConnectorName) } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil) if err != nil { return nil, trace.Wrap(err) } @@ -245,6 +299,7 @@ func Ping(cfg *Config) (*PingResponse, error) { return pr, nil } +// GetMOTD retrieves the Message Of The Day from the web proxy. func GetMOTD(cfg *Config) (*MotD, error) { clt, err := newWebClient(cfg) if err != nil { @@ -252,6 +307,10 @@ func GetMOTD(cfg *Config) (*MotD, error) { } defer clt.CloseIdleConnections() + return getMOTDWithClient(cfg, clt) +} + +func getMOTDWithClient(cfg *Config, clt *http.Client) (*MotD, error) { ctx, span := cfg.TraceProvider.Tracer("webclient").Start(cfg.Context, "webclient/GetMOTD") defer span.End() @@ -280,6 +339,60 @@ func GetMOTD(cfg *Config) (*MotD, error) { return motd, nil } +// NewReusableClient creates a reusable webproxy client. If you need to do a single call, +// use the webclient.Ping or webclient.Find functions instead. +func NewReusableClient(cfg *Config) (*ReusableClient, error) { + // no need to check and set config defaults, this happens in newWebClient + client, err := newWebClient(cfg) + if err != nil { + return nil, trace.Wrap(err, "building new web client") + } + + return &ReusableClient{ + client: client, + config: cfg, + }, nil +} + +// ReusableClient is a webproxy client that allows the caller to make multiple calls +// without having to buildi a new HTTP client each time. +// Before retiring the client, you must make sure no calls are still in-flight, then call +// ReusableClient.CloseIdleConnections(). +type ReusableClient struct { + client *http.Client + config *Config +} + +// Find fetches discovery data by connecting to the given web proxy address. +// It is designed to fetch proxy public addresses without any inefficiencies. +func (c *ReusableClient) Find() (*PingResponse, error) { + return findWithClient(c.config, c.client) +} + +// Ping serves two purposes. The first is to validate the HTTP endpoint of a +// Teleport proxy. This leads to better user experience: users get connection +// errors before being asked for passwords. The second is to return the form +// of authentication that the server supports. This also leads to better user +// experience: users only get prompted for the type of authentication the server supports. +func (c *ReusableClient) Ping() (*PingResponse, error) { + return pingWithClient(c.config, c.client) +} + +// GetMOTD retrieves the Message Of The Day from the web proxy. +func (c *ReusableClient) GetMOTD() (*MotD, error) { + return getMOTDWithClient(c.config, c.client) +} + +// CloseIdleConnections closes any connections on its [Transport] which +// were previously connected from previous requests but are now +// sitting idle in a "keep-alive" state. It does not interrupt any +// connections currently in use. +// +// This must be run before retiring the ReusableClient. +func (c *ReusableClient) CloseIdleConnections() { + c.client.CloseIdleConnections() +} + // MotD holds data about the current message of the day. type MotD struct { Text string @@ -304,6 +417,10 @@ type PingResponse struct { // reserved: license_warnings ([]string) // AutomaticUpgrades describes whether agents should automatically upgrade. AutomaticUpgrades bool `json:"automatic_upgrades"` + // Edition represents the Teleport edition. Possible values are "oss", "ent", and "community". + Edition string `json:"edition"` + // FIPS represents if Teleport is using FIPS-compliant cryptography. + FIPS bool `json:"fips"` } // PingErrorResponse contains the error from /webapi/ping. @@ -337,6 +454,12 @@ type AutoUpdateSettings struct { ToolsVersion string `json:"tools_version"` // ToolsAutoUpdate indicates if the requesting tools client should be updated. ToolsAutoUpdate bool `json:"tools_auto_update"` + // AgentVersion defines the version of teleport that agents enrolled into autoupdates should run. + AgentVersion string `json:"agent_version"` + // AgentAutoUpdate indicates if the requesting agent should attempt to update now. + AgentAutoUpdate bool `json:"agent_auto_update"` + // AgentUpdateJitterSeconds defines the jitter time an agent should wait before updating. + AgentUpdateJitterSeconds int `json:"agent_update_jitter_seconds"` } // KubeProxySettings is kubernetes proxy settings diff --git a/api/gen/proto/go/teleport/autoupdate/v1/autoupdate.pb.go b/api/gen/proto/go/teleport/autoupdate/v1/autoupdate.pb.go index ffc36aeb70163..8432ae441191b 100644 --- a/api/gen/proto/go/teleport/autoupdate/v1/autoupdate.pb.go +++ b/api/gen/proto/go/teleport/autoupdate/v1/autoupdate.pb.go @@ -24,6 +24,8 @@ import ( v1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/header/v1" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" ) @@ -35,6 +37,135 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// AutoUpdateAgentGroupState represents the agent group state. This state controls whether the agents from this group +// should install the start version, the target version, and if they should update immediately or wait. +type AutoUpdateAgentGroupState int32 + +const ( + // AUTO_UPDATE_AGENT_GROUP_STATE_UNSPECIFIED state + AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSPECIFIED AutoUpdateAgentGroupState = 0 + // AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED represents that the group update has not been started yet. + AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED AutoUpdateAgentGroupState = 1 + // AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE represents that the group is actively getting updated. + // New agents should run v2, existing agents are instructed to update to v2. + AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE AutoUpdateAgentGroupState = 2 + // AUTO_UPDATE_AGENT_GROUP_STATE_DONE represents that the group has been updated. New agents should run v2. + AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE AutoUpdateAgentGroupState = 3 + // AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK represents that the group has been rolled back. + // New agents should run v1, existing agents should update to v1. + AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK AutoUpdateAgentGroupState = 4 +) + +// Enum value maps for AutoUpdateAgentGroupState. +var ( + AutoUpdateAgentGroupState_name = map[int32]string{ + 0: "AUTO_UPDATE_AGENT_GROUP_STATE_UNSPECIFIED", + 1: "AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED", + 2: "AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE", + 3: "AUTO_UPDATE_AGENT_GROUP_STATE_DONE", + 4: "AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK", + } + AutoUpdateAgentGroupState_value = map[string]int32{ + "AUTO_UPDATE_AGENT_GROUP_STATE_UNSPECIFIED": 0, + "AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED": 1, + "AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE": 2, + "AUTO_UPDATE_AGENT_GROUP_STATE_DONE": 3, + "AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK": 4, + } +) + +func (x AutoUpdateAgentGroupState) Enum() *AutoUpdateAgentGroupState { + p := new(AutoUpdateAgentGroupState) + *p = x + return p +} + +func (x AutoUpdateAgentGroupState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (AutoUpdateAgentGroupState) Descriptor() protoreflect.EnumDescriptor { + return file_teleport_autoupdate_v1_autoupdate_proto_enumTypes[0].Descriptor() +} + +func (AutoUpdateAgentGroupState) Type() protoreflect.EnumType { + return &file_teleport_autoupdate_v1_autoupdate_proto_enumTypes[0] +} + +func (x AutoUpdateAgentGroupState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use AutoUpdateAgentGroupState.Descriptor instead. +func (AutoUpdateAgentGroupState) EnumDescriptor() ([]byte, []int) { + return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{0} +} + +// AutoUpdateAgentRolloutState represents the rollout state. This tells if Teleport started updating agents from the +// start version to the target version, if the update is done, still in progress +// or if the rollout was manually reverted. +type AutoUpdateAgentRolloutState int32 + +const ( + // AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSPECIFIED state + AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSPECIFIED AutoUpdateAgentRolloutState = 0 + // AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSTARTED represents that no group in the rollout has been started yet. + AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSTARTED AutoUpdateAgentRolloutState = 1 + // AUTO_UPDATE_AGENT_ROLLOUT_STATE_ACTIVE represents that at least one group of the rollout has started. + // If every group is finished, the state will be AUTO_UPDATE_AGENT_ROLLOUT_STATE_DONE. + AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_ACTIVE AutoUpdateAgentRolloutState = 2 + // AUTO_UPDATE_AGENT_ROLLOUT_STATE_DONE represents that every group is in the DONE state, or has been in the done + // state (groups might become active again in time-based strategy). + AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_DONE AutoUpdateAgentRolloutState = 3 + // AUTO_UPDATE_AGENT_ROLLOUT_STATE_ROLLEDBACK represents that at least one group is in the rolledback state. + AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_ROLLEDBACK AutoUpdateAgentRolloutState = 4 +) + +// Enum value maps for AutoUpdateAgentRolloutState. +var ( + AutoUpdateAgentRolloutState_name = map[int32]string{ + 0: "AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSPECIFIED", + 1: "AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSTARTED", + 2: "AUTO_UPDATE_AGENT_ROLLOUT_STATE_ACTIVE", + 3: "AUTO_UPDATE_AGENT_ROLLOUT_STATE_DONE", + 4: "AUTO_UPDATE_AGENT_ROLLOUT_STATE_ROLLEDBACK", + } + AutoUpdateAgentRolloutState_value = map[string]int32{ + "AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSPECIFIED": 0, + "AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSTARTED": 1, + "AUTO_UPDATE_AGENT_ROLLOUT_STATE_ACTIVE": 2, + "AUTO_UPDATE_AGENT_ROLLOUT_STATE_DONE": 3, + "AUTO_UPDATE_AGENT_ROLLOUT_STATE_ROLLEDBACK": 4, + } +) + +func (x AutoUpdateAgentRolloutState) Enum() *AutoUpdateAgentRolloutState { + p := new(AutoUpdateAgentRolloutState) + *p = x + return p +} + +func (x AutoUpdateAgentRolloutState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (AutoUpdateAgentRolloutState) Descriptor() protoreflect.EnumDescriptor { + return file_teleport_autoupdate_v1_autoupdate_proto_enumTypes[1].Descriptor() +} + +func (AutoUpdateAgentRolloutState) Type() protoreflect.EnumType { + return &file_teleport_autoupdate_v1_autoupdate_proto_enumTypes[1] +} + +func (x AutoUpdateAgentRolloutState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use AutoUpdateAgentRolloutState.Descriptor instead. +func (AutoUpdateAgentRolloutState) EnumDescriptor() ([]byte, []int) { + return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{1} +} + // AutoUpdateConfig is a config singleton used to configure cluster // autoupdate settings. type AutoUpdateConfig struct { @@ -122,7 +253,8 @@ type AutoUpdateConfigSpec struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Tools *AutoUpdateConfigSpecTools `protobuf:"bytes,2,opt,name=tools,proto3" json:"tools,omitempty"` + Tools *AutoUpdateConfigSpecTools `protobuf:"bytes,2,opt,name=tools,proto3" json:"tools,omitempty"` + Agents *AutoUpdateConfigSpecAgents `protobuf:"bytes,3,opt,name=agents,proto3" json:"agents,omitempty"` } func (x *AutoUpdateConfigSpec) Reset() { @@ -164,6 +296,13 @@ func (x *AutoUpdateConfigSpec) GetTools() *AutoUpdateConfigSpecTools { return nil } +func (x *AutoUpdateConfigSpec) GetAgents() *AutoUpdateConfigSpecAgents { + if x != nil { + return x.Agents + } + return nil +} + // AutoUpdateConfigSpecTools encodes the parameters for client tools auto updates. type AutoUpdateConfigSpecTools struct { state protoimpl.MessageState @@ -213,6 +352,210 @@ func (x *AutoUpdateConfigSpecTools) GetMode() string { return "" } +// AutoUpdateConfigSpecAgents encodes the parameters of automatic agent updates. +type AutoUpdateConfigSpecAgents struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // mode specifies whether agent autoupdates are enabled, disabled, or paused. + Mode string `protobuf:"bytes,1,opt,name=mode,proto3" json:"mode,omitempty"` + // strategy to use for updating the agents. + Strategy string `protobuf:"bytes,2,opt,name=strategy,proto3" json:"strategy,omitempty"` + // maintenance_window_duration is the maintenance window duration. This can only be set if `strategy` is "time-based". + // Once the window is over, the group transitions to the done state. Existing agents won't be updated until the next + // maintenance window. + MaintenanceWindowDuration *durationpb.Duration `protobuf:"bytes,3,opt,name=maintenance_window_duration,json=maintenanceWindowDuration,proto3" json:"maintenance_window_duration,omitempty"` + // schedules specifies schedules for updates of grouped agents. + Schedules *AgentAutoUpdateSchedules `protobuf:"bytes,6,opt,name=schedules,proto3" json:"schedules,omitempty"` +} + +func (x *AutoUpdateConfigSpecAgents) Reset() { + *x = AutoUpdateConfigSpecAgents{} + if protoimpl.UnsafeEnabled { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AutoUpdateConfigSpecAgents) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AutoUpdateConfigSpecAgents) ProtoMessage() {} + +func (x *AutoUpdateConfigSpecAgents) ProtoReflect() protoreflect.Message { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AutoUpdateConfigSpecAgents.ProtoReflect.Descriptor instead. +func (*AutoUpdateConfigSpecAgents) Descriptor() ([]byte, []int) { + return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{3} +} + +func (x *AutoUpdateConfigSpecAgents) GetMode() string { + if x != nil { + return x.Mode + } + return "" +} + +func (x *AutoUpdateConfigSpecAgents) GetStrategy() string { + if x != nil { + return x.Strategy + } + return "" +} + +func (x *AutoUpdateConfigSpecAgents) GetMaintenanceWindowDuration() *durationpb.Duration { + if x != nil { + return x.MaintenanceWindowDuration + } + return nil +} + +func (x *AutoUpdateConfigSpecAgents) GetSchedules() *AgentAutoUpdateSchedules { + if x != nil { + return x.Schedules + } + return nil +} + +// AgentAutoUpdateSchedules specifies update scheduled for grouped agents. +type AgentAutoUpdateSchedules struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // regular schedules for non-critical versions. + Regular []*AgentAutoUpdateGroup `protobuf:"bytes,1,rep,name=regular,proto3" json:"regular,omitempty"` +} + +func (x *AgentAutoUpdateSchedules) Reset() { + *x = AgentAutoUpdateSchedules{} + if protoimpl.UnsafeEnabled { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AgentAutoUpdateSchedules) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AgentAutoUpdateSchedules) ProtoMessage() {} + +func (x *AgentAutoUpdateSchedules) ProtoReflect() protoreflect.Message { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AgentAutoUpdateSchedules.ProtoReflect.Descriptor instead. +func (*AgentAutoUpdateSchedules) Descriptor() ([]byte, []int) { + return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{4} +} + +func (x *AgentAutoUpdateSchedules) GetRegular() []*AgentAutoUpdateGroup { + if x != nil { + return x.Regular + } + return nil +} + +// AgentAutoUpdateGroup specifies the update schedule for a group of agents. +type AgentAutoUpdateGroup struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // name of the group + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // days when the update can run. Supported values are "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" and "*" + Days []string `protobuf:"bytes,2,rep,name=days,proto3" json:"days,omitempty"` + // start_hour to initiate update + StartHour int32 `protobuf:"varint,3,opt,name=start_hour,json=startHour,proto3" json:"start_hour,omitempty"` + // wait_hours after last group succeeds before this group can run. This can only be used when the strategy is "halt-on-failure". + // This field must be positive. + WaitHours int32 `protobuf:"varint,5,opt,name=wait_hours,json=waitHours,proto3" json:"wait_hours,omitempty"` +} + +func (x *AgentAutoUpdateGroup) Reset() { + *x = AgentAutoUpdateGroup{} + if protoimpl.UnsafeEnabled { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AgentAutoUpdateGroup) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AgentAutoUpdateGroup) ProtoMessage() {} + +func (x *AgentAutoUpdateGroup) ProtoReflect() protoreflect.Message { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AgentAutoUpdateGroup.ProtoReflect.Descriptor instead. +func (*AgentAutoUpdateGroup) Descriptor() ([]byte, []int) { + return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{5} +} + +func (x *AgentAutoUpdateGroup) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *AgentAutoUpdateGroup) GetDays() []string { + if x != nil { + return x.Days + } + return nil +} + +func (x *AgentAutoUpdateGroup) GetStartHour() int32 { + if x != nil { + return x.StartHour + } + return 0 +} + +func (x *AgentAutoUpdateGroup) GetWaitHours() int32 { + if x != nil { + return x.WaitHours + } + return 0 +} + // AutoUpdateVersion is a resource singleton with version required for // tools autoupdate. type AutoUpdateVersion struct { @@ -230,7 +573,7 @@ type AutoUpdateVersion struct { func (x *AutoUpdateVersion) Reset() { *x = AutoUpdateVersion{} if protoimpl.UnsafeEnabled { - mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[3] + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -243,7 +586,7 @@ func (x *AutoUpdateVersion) String() string { func (*AutoUpdateVersion) ProtoMessage() {} func (x *AutoUpdateVersion) ProtoReflect() protoreflect.Message { - mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[3] + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -256,7 +599,7 @@ func (x *AutoUpdateVersion) ProtoReflect() protoreflect.Message { // Deprecated: Use AutoUpdateVersion.ProtoReflect.Descriptor instead. func (*AutoUpdateVersion) Descriptor() ([]byte, []int) { - return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{3} + return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{6} } func (x *AutoUpdateVersion) GetKind() string { @@ -300,13 +643,14 @@ type AutoUpdateVersionSpec struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Tools *AutoUpdateVersionSpecTools `protobuf:"bytes,2,opt,name=tools,proto3" json:"tools,omitempty"` + Tools *AutoUpdateVersionSpecTools `protobuf:"bytes,2,opt,name=tools,proto3" json:"tools,omitempty"` + Agents *AutoUpdateVersionSpecAgents `protobuf:"bytes,3,opt,name=agents,proto3" json:"agents,omitempty"` } func (x *AutoUpdateVersionSpec) Reset() { *x = AutoUpdateVersionSpec{} if protoimpl.UnsafeEnabled { - mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[4] + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -319,7 +663,7 @@ func (x *AutoUpdateVersionSpec) String() string { func (*AutoUpdateVersionSpec) ProtoMessage() {} func (x *AutoUpdateVersionSpec) ProtoReflect() protoreflect.Message { - mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[4] + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -332,7 +676,7 @@ func (x *AutoUpdateVersionSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use AutoUpdateVersionSpec.ProtoReflect.Descriptor instead. func (*AutoUpdateVersionSpec) Descriptor() ([]byte, []int) { - return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{4} + return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{7} } func (x *AutoUpdateVersionSpec) GetTools() *AutoUpdateVersionSpecTools { @@ -342,6 +686,13 @@ func (x *AutoUpdateVersionSpec) GetTools() *AutoUpdateVersionSpecTools { return nil } +func (x *AutoUpdateVersionSpec) GetAgents() *AutoUpdateVersionSpecAgents { + if x != nil { + return x.Agents + } + return nil +} + // AutoUpdateVersionSpecTools encodes the parameters for client tools auto updates. type AutoUpdateVersionSpecTools struct { state protoimpl.MessageState @@ -356,7 +707,7 @@ type AutoUpdateVersionSpecTools struct { func (x *AutoUpdateVersionSpecTools) Reset() { *x = AutoUpdateVersionSpecTools{} if protoimpl.UnsafeEnabled { - mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[5] + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -369,7 +720,7 @@ func (x *AutoUpdateVersionSpecTools) String() string { func (*AutoUpdateVersionSpecTools) ProtoMessage() {} func (x *AutoUpdateVersionSpecTools) ProtoReflect() protoreflect.Message { - mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[5] + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -382,7 +733,7 @@ func (x *AutoUpdateVersionSpecTools) ProtoReflect() protoreflect.Message { // Deprecated: Use AutoUpdateVersionSpecTools.ProtoReflect.Descriptor instead. func (*AutoUpdateVersionSpecTools) Descriptor() ([]byte, []int) { - return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{5} + return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{8} } func (x *AutoUpdateVersionSpecTools) GetTargetVersion() string { @@ -392,118 +743,795 @@ func (x *AutoUpdateVersionSpecTools) GetTargetVersion() string { return "" } -var File_teleport_autoupdate_v1_autoupdate_proto protoreflect.FileDescriptor +// AutoUpdateVersionSpecAgents is the spec for the autoupdate version. +type AutoUpdateVersionSpecAgents struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -var file_teleport_autoupdate_v1_autoupdate_proto_rawDesc = []byte{ - 0x0a, 0x27, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x75, 0x74, 0x6f, 0x75, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x16, 0x74, 0x65, 0x6c, 0x65, 0x70, - 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, - 0x31, 0x1a, 0x21, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x68, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x2f, 0x76, 0x31, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xd7, 0x01, 0x0a, 0x10, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, 0x6e, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x19, 0x0a, - 0x08, 0x73, 0x75, 0x62, 0x5f, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x73, 0x75, 0x62, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x38, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, - 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x40, 0x0a, 0x04, - 0x73, 0x70, 0x65, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x22, 0x77, - 0x0a, 0x14, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x12, 0x47, 0x0a, 0x05, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, - 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, - 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, - 0x70, 0x65, 0x63, 0x54, 0x6f, 0x6f, 0x6c, 0x73, 0x52, 0x05, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x4a, - 0x04, 0x08, 0x01, 0x10, 0x02, 0x52, 0x10, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x5f, 0x61, 0x75, 0x74, - 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x22, 0x2f, 0x0a, 0x19, 0x41, 0x75, 0x74, 0x6f, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x54, - 0x6f, 0x6f, 0x6c, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x22, 0xd9, 0x01, 0x0a, 0x11, 0x41, 0x75, 0x74, - 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, - 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, - 0x6e, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x75, 0x62, 0x5f, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x75, 0x62, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x18, 0x0a, - 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x38, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x65, 0x6c, 0x65, - 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x12, 0x41, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x2d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x52, 0x04, - 0x73, 0x70, 0x65, 0x63, 0x22, 0x76, 0x0a, 0x15, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x12, 0x48, 0x0a, - 0x05, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x74, - 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x54, 0x6f, 0x6f, 0x6c, 0x73, - 0x52, 0x05, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x52, 0x0d, 0x74, - 0x6f, 0x6f, 0x6c, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x43, 0x0a, 0x1a, - 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x53, 0x70, 0x65, 0x63, 0x54, 0x6f, 0x6f, 0x6c, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0d, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x42, 0x56, 0x5a, 0x54, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x67, 0x72, 0x61, 0x76, 0x69, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x2f, 0x74, 0x65, - 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, - 0x2f, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2f, 0x76, 0x31, 0x3b, 0x61, - 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + // start_version is the version to update from. + StartVersion string `protobuf:"bytes,1,opt,name=start_version,json=startVersion,proto3" json:"start_version,omitempty"` + // target_version is the version to update to. + TargetVersion string `protobuf:"bytes,2,opt,name=target_version,json=targetVersion,proto3" json:"target_version,omitempty"` + // schedule to use for the rollout + Schedule string `protobuf:"bytes,3,opt,name=schedule,proto3" json:"schedule,omitempty"` + // autoupdate_mode to use for the rollout + Mode string `protobuf:"bytes,4,opt,name=mode,proto3" json:"mode,omitempty"` } -var ( - file_teleport_autoupdate_v1_autoupdate_proto_rawDescOnce sync.Once - file_teleport_autoupdate_v1_autoupdate_proto_rawDescData = file_teleport_autoupdate_v1_autoupdate_proto_rawDesc -) +func (x *AutoUpdateVersionSpecAgents) Reset() { + *x = AutoUpdateVersionSpecAgents{} + if protoimpl.UnsafeEnabled { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} -func file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP() []byte { - file_teleport_autoupdate_v1_autoupdate_proto_rawDescOnce.Do(func() { - file_teleport_autoupdate_v1_autoupdate_proto_rawDescData = protoimpl.X.CompressGZIP(file_teleport_autoupdate_v1_autoupdate_proto_rawDescData) - }) - return file_teleport_autoupdate_v1_autoupdate_proto_rawDescData +func (x *AutoUpdateVersionSpecAgents) String() string { + return protoimpl.X.MessageStringOf(x) } -var file_teleport_autoupdate_v1_autoupdate_proto_msgTypes = make([]protoimpl.MessageInfo, 6) -var file_teleport_autoupdate_v1_autoupdate_proto_goTypes = []interface{}{ - (*AutoUpdateConfig)(nil), // 0: teleport.autoupdate.v1.AutoUpdateConfig - (*AutoUpdateConfigSpec)(nil), // 1: teleport.autoupdate.v1.AutoUpdateConfigSpec - (*AutoUpdateConfigSpecTools)(nil), // 2: teleport.autoupdate.v1.AutoUpdateConfigSpecTools - (*AutoUpdateVersion)(nil), // 3: teleport.autoupdate.v1.AutoUpdateVersion - (*AutoUpdateVersionSpec)(nil), // 4: teleport.autoupdate.v1.AutoUpdateVersionSpec - (*AutoUpdateVersionSpecTools)(nil), // 5: teleport.autoupdate.v1.AutoUpdateVersionSpecTools - (*v1.Metadata)(nil), // 6: teleport.header.v1.Metadata +func (*AutoUpdateVersionSpecAgents) ProtoMessage() {} + +func (x *AutoUpdateVersionSpecAgents) ProtoReflect() protoreflect.Message { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var file_teleport_autoupdate_v1_autoupdate_proto_depIdxs = []int32{ - 6, // 0: teleport.autoupdate.v1.AutoUpdateConfig.metadata:type_name -> teleport.header.v1.Metadata - 1, // 1: teleport.autoupdate.v1.AutoUpdateConfig.spec:type_name -> teleport.autoupdate.v1.AutoUpdateConfigSpec - 2, // 2: teleport.autoupdate.v1.AutoUpdateConfigSpec.tools:type_name -> teleport.autoupdate.v1.AutoUpdateConfigSpecTools - 6, // 3: teleport.autoupdate.v1.AutoUpdateVersion.metadata:type_name -> teleport.header.v1.Metadata - 4, // 4: teleport.autoupdate.v1.AutoUpdateVersion.spec:type_name -> teleport.autoupdate.v1.AutoUpdateVersionSpec - 5, // 5: teleport.autoupdate.v1.AutoUpdateVersionSpec.tools:type_name -> teleport.autoupdate.v1.AutoUpdateVersionSpecTools - 6, // [6:6] is the sub-list for method output_type - 6, // [6:6] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name + +// Deprecated: Use AutoUpdateVersionSpecAgents.ProtoReflect.Descriptor instead. +func (*AutoUpdateVersionSpecAgents) Descriptor() ([]byte, []int) { + return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{9} } -func init() { file_teleport_autoupdate_v1_autoupdate_proto_init() } -func file_teleport_autoupdate_v1_autoupdate_proto_init() { - if File_teleport_autoupdate_v1_autoupdate_proto != nil { - return +func (x *AutoUpdateVersionSpecAgents) GetStartVersion() string { + if x != nil { + return x.StartVersion } - if !protoimpl.UnsafeEnabled { - file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AutoUpdateConfig); i { + return "" +} + +func (x *AutoUpdateVersionSpecAgents) GetTargetVersion() string { + if x != nil { + return x.TargetVersion + } + return "" +} + +func (x *AutoUpdateVersionSpecAgents) GetSchedule() string { + if x != nil { + return x.Schedule + } + return "" +} + +func (x *AutoUpdateVersionSpecAgents) GetMode() string { + if x != nil { + return x.Mode + } + return "" +} + +// AutoUpdateAgentRollout is the resource the Teleport Auth Service uses to track and control the rollout of a new +// agent version. This resource is written by the automatic agent update controller in the Teleport Auth Service +// and read by the Teleport Proxy Service. +type AutoUpdateAgentRollout struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Kind string `protobuf:"bytes,1,opt,name=kind,proto3" json:"kind,omitempty"` + SubKind string `protobuf:"bytes,2,opt,name=sub_kind,json=subKind,proto3" json:"sub_kind,omitempty"` + Version string `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` + Metadata *v1.Metadata `protobuf:"bytes,4,opt,name=metadata,proto3" json:"metadata,omitempty"` + Spec *AutoUpdateAgentRolloutSpec `protobuf:"bytes,5,opt,name=spec,proto3" json:"spec,omitempty"` + Status *AutoUpdateAgentRolloutStatus `protobuf:"bytes,6,opt,name=status,proto3" json:"status,omitempty"` +} + +func (x *AutoUpdateAgentRollout) Reset() { + *x = AutoUpdateAgentRollout{} + if protoimpl.UnsafeEnabled { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AutoUpdateAgentRollout) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AutoUpdateAgentRollout) ProtoMessage() {} + +func (x *AutoUpdateAgentRollout) ProtoReflect() protoreflect.Message { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AutoUpdateAgentRollout.ProtoReflect.Descriptor instead. +func (*AutoUpdateAgentRollout) Descriptor() ([]byte, []int) { + return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{10} +} + +func (x *AutoUpdateAgentRollout) GetKind() string { + if x != nil { + return x.Kind + } + return "" +} + +func (x *AutoUpdateAgentRollout) GetSubKind() string { + if x != nil { + return x.SubKind + } + return "" +} + +func (x *AutoUpdateAgentRollout) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *AutoUpdateAgentRollout) GetMetadata() *v1.Metadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *AutoUpdateAgentRollout) GetSpec() *AutoUpdateAgentRolloutSpec { + if x != nil { + return x.Spec + } + return nil +} + +func (x *AutoUpdateAgentRollout) GetStatus() *AutoUpdateAgentRolloutStatus { + if x != nil { + return x.Status + } + return nil +} + +// AutoUpdateAgentRolloutSpec describes the desired agent rollout. +// This is built by merging the user-provided AutoUpdateConfigSpecAgents and the operator-provided +// AutoUpdateVersionSpecAgents. +type AutoUpdateAgentRolloutSpec struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // start_version is the version to update from. + StartVersion string `protobuf:"bytes,1,opt,name=start_version,json=startVersion,proto3" json:"start_version,omitempty"` + // target_version is the version to update to. + TargetVersion string `protobuf:"bytes,2,opt,name=target_version,json=targetVersion,proto3" json:"target_version,omitempty"` + // schedule to use for the rollout. Supported values are "regular" and "immediate". + // - "regular" follows the regular group schedule + // - "immediate" updates all the agents immediately + Schedule string `protobuf:"bytes,3,opt,name=schedule,proto3" json:"schedule,omitempty"` + // autoupdate_mode to use for the rollout. Supported modes are: + // - "enabled": Teleport will update existing agents. + // - "disabled": Teleport will not update existing agents. + // - "suspended": Teleport will temporarily stop updating existing agents. + AutoupdateMode string `protobuf:"bytes,4,opt,name=autoupdate_mode,json=autoupdateMode,proto3" json:"autoupdate_mode,omitempty"` + // strategy to use for updating the agents. Supported strategies are: + // - "time-based": agents update as soon as their maintenance window starts. There is no dependency between groups. + // This strategy allows Teleport users to setup reliable follow-the-sun updates and enforce the maintenance window + // more strictly. A group finishes its update at the end of the maintenance window, regardless of the new version + // adoption rate. Agents that missed the maintenance window will not attempt to update until the next maintenance + // window. + // - "halt-on-failure": the update proceeds from the first group to the last group, ensuring that each group + // successfully updates before allowing the next group to proceed. This is the strategy that offers the best + // availability. A group finishes its update once most of its agents are running the correct version. Agents that + // missed the group update will try to catch back as soon as possible. + Strategy string `protobuf:"bytes,5,opt,name=strategy,proto3" json:"strategy,omitempty"` + // maintenance_window_duration is the maintenance window duration. This can only be set if `strategy` is "time-based". + // Once the window is over, the group transitions to the done state. Existing agents won't be updated until the next + // maintenance window. + MaintenanceWindowDuration *durationpb.Duration `protobuf:"bytes,6,opt,name=maintenance_window_duration,json=maintenanceWindowDuration,proto3" json:"maintenance_window_duration,omitempty"` +} + +func (x *AutoUpdateAgentRolloutSpec) Reset() { + *x = AutoUpdateAgentRolloutSpec{} + if protoimpl.UnsafeEnabled { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AutoUpdateAgentRolloutSpec) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AutoUpdateAgentRolloutSpec) ProtoMessage() {} + +func (x *AutoUpdateAgentRolloutSpec) ProtoReflect() protoreflect.Message { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AutoUpdateAgentRolloutSpec.ProtoReflect.Descriptor instead. +func (*AutoUpdateAgentRolloutSpec) Descriptor() ([]byte, []int) { + return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{11} +} + +func (x *AutoUpdateAgentRolloutSpec) GetStartVersion() string { + if x != nil { + return x.StartVersion + } + return "" +} + +func (x *AutoUpdateAgentRolloutSpec) GetTargetVersion() string { + if x != nil { + return x.TargetVersion + } + return "" +} + +func (x *AutoUpdateAgentRolloutSpec) GetSchedule() string { + if x != nil { + return x.Schedule + } + return "" +} + +func (x *AutoUpdateAgentRolloutSpec) GetAutoupdateMode() string { + if x != nil { + return x.AutoupdateMode + } + return "" +} + +func (x *AutoUpdateAgentRolloutSpec) GetStrategy() string { + if x != nil { + return x.Strategy + } + return "" +} + +func (x *AutoUpdateAgentRolloutSpec) GetMaintenanceWindowDuration() *durationpb.Duration { + if x != nil { + return x.MaintenanceWindowDuration + } + return nil +} + +// AutoUpdateAgentRolloutStatus tracks the current agent rollout status. +// The status is reset if any spec field changes except the mode. +type AutoUpdateAgentRolloutStatus struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Groups []*AutoUpdateAgentRolloutStatusGroup `protobuf:"bytes,1,rep,name=groups,proto3" json:"groups,omitempty"` + State AutoUpdateAgentRolloutState `protobuf:"varint,2,opt,name=state,proto3,enum=teleport.autoupdate.v1.AutoUpdateAgentRolloutState" json:"state,omitempty"` + // The start time is set when the rollout is created or reset. Usually this is caused by a version change. + // The timestamp allows the controller to detect that the rollout just changed. + // The controller will not start any group that should have been active before the start_time to avoid a double-update + // effect. + // For example, a group updates every day between 13:00 and 14:00. If the target version changes to 13:30, the group + // will not start updating to the new version directly. The controller sees that the group theoretical start time is + // before the rollout start time and the maintenance window belongs to the previous rollout. + // When the timestamp is nil, the controller will ignore the start time and check and allow groups to activate. + StartTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` + // Time override is an optional timestamp making the autoupdate_agent_rollout controller use a specific time instead + // of the system clock when evaluating time-based criteria. This field is used for testing and troubleshooting + // purposes. + TimeOverride *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=time_override,json=timeOverride,proto3" json:"time_override,omitempty"` +} + +func (x *AutoUpdateAgentRolloutStatus) Reset() { + *x = AutoUpdateAgentRolloutStatus{} + if protoimpl.UnsafeEnabled { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AutoUpdateAgentRolloutStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AutoUpdateAgentRolloutStatus) ProtoMessage() {} + +func (x *AutoUpdateAgentRolloutStatus) ProtoReflect() protoreflect.Message { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AutoUpdateAgentRolloutStatus.ProtoReflect.Descriptor instead. +func (*AutoUpdateAgentRolloutStatus) Descriptor() ([]byte, []int) { + return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{12} +} + +func (x *AutoUpdateAgentRolloutStatus) GetGroups() []*AutoUpdateAgentRolloutStatusGroup { + if x != nil { + return x.Groups + } + return nil +} + +func (x *AutoUpdateAgentRolloutStatus) GetState() AutoUpdateAgentRolloutState { + if x != nil { + return x.State + } + return AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSPECIFIED +} + +func (x *AutoUpdateAgentRolloutStatus) GetStartTime() *timestamppb.Timestamp { + if x != nil { + return x.StartTime + } + return nil +} + +func (x *AutoUpdateAgentRolloutStatus) GetTimeOverride() *timestamppb.Timestamp { + if x != nil { + return x.TimeOverride + } + return nil +} + +// AutoUpdateAgentRolloutStatusGroup tracks the current agent rollout status of a specific group. +type AutoUpdateAgentRolloutStatusGroup struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // name of the group + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // start_time of the rollout + StartTime *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` + // state is the current state of the rollout. + State AutoUpdateAgentGroupState `protobuf:"varint,3,opt,name=state,proto3,enum=teleport.autoupdate.v1.AutoUpdateAgentGroupState" json:"state,omitempty"` + // last_update_time is the time of the previous update for this group. + LastUpdateTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=last_update_time,json=lastUpdateTime,proto3" json:"last_update_time,omitempty"` + // last_update_reason is the trigger for the last update + LastUpdateReason string `protobuf:"bytes,5,opt,name=last_update_reason,json=lastUpdateReason,proto3" json:"last_update_reason,omitempty"` + // config_days when the update can run. Supported values are "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" and "*" + ConfigDays []string `protobuf:"bytes,6,rep,name=config_days,json=configDays,proto3" json:"config_days,omitempty"` + // config_start_hour to initiate update + ConfigStartHour int32 `protobuf:"varint,7,opt,name=config_start_hour,json=configStartHour,proto3" json:"config_start_hour,omitempty"` + // config_wait_hours after last group succeeds before this group can run. This can only be used when the strategy is "halt-on-failure". + // This field must be positive. + ConfigWaitHours int32 `protobuf:"varint,9,opt,name=config_wait_hours,json=configWaitHours,proto3" json:"config_wait_hours,omitempty"` +} + +func (x *AutoUpdateAgentRolloutStatusGroup) Reset() { + *x = AutoUpdateAgentRolloutStatusGroup{} + if protoimpl.UnsafeEnabled { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AutoUpdateAgentRolloutStatusGroup) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AutoUpdateAgentRolloutStatusGroup) ProtoMessage() {} + +func (x *AutoUpdateAgentRolloutStatusGroup) ProtoReflect() protoreflect.Message { + mi := &file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AutoUpdateAgentRolloutStatusGroup.ProtoReflect.Descriptor instead. +func (*AutoUpdateAgentRolloutStatusGroup) Descriptor() ([]byte, []int) { + return file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP(), []int{13} +} + +func (x *AutoUpdateAgentRolloutStatusGroup) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *AutoUpdateAgentRolloutStatusGroup) GetStartTime() *timestamppb.Timestamp { + if x != nil { + return x.StartTime + } + return nil +} + +func (x *AutoUpdateAgentRolloutStatusGroup) GetState() AutoUpdateAgentGroupState { + if x != nil { + return x.State + } + return AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSPECIFIED +} + +func (x *AutoUpdateAgentRolloutStatusGroup) GetLastUpdateTime() *timestamppb.Timestamp { + if x != nil { + return x.LastUpdateTime + } + return nil +} + +func (x *AutoUpdateAgentRolloutStatusGroup) GetLastUpdateReason() string { + if x != nil { + return x.LastUpdateReason + } + return "" +} + +func (x *AutoUpdateAgentRolloutStatusGroup) GetConfigDays() []string { + if x != nil { + return x.ConfigDays + } + return nil +} + +func (x *AutoUpdateAgentRolloutStatusGroup) GetConfigStartHour() int32 { + if x != nil { + return x.ConfigStartHour + } + return 0 +} + +func (x *AutoUpdateAgentRolloutStatusGroup) GetConfigWaitHours() int32 { + if x != nil { + return x.ConfigWaitHours + } + return 0 +} + +var File_teleport_autoupdate_v1_autoupdate_proto protoreflect.FileDescriptor + +var file_teleport_autoupdate_v1_autoupdate_proto_rawDesc = []byte{ + 0x0a, 0x27, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x75, 0x74, 0x6f, 0x75, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x16, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, + 0x31, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x1a, 0x21, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x68, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x2f, 0x76, 0x31, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xd7, 0x01, 0x0a, 0x10, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, + 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x19, + 0x0a, 0x08, 0x73, 0x75, 0x62, 0x5f, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x73, 0x75, 0x62, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x38, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, + 0x2e, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x40, 0x0a, + 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x74, 0x65, + 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x22, + 0xc3, 0x01, 0x0a, 0x14, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x12, 0x47, 0x0a, 0x05, 0x74, 0x6f, 0x6f, 0x6c, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, + 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x53, 0x70, 0x65, 0x63, 0x54, 0x6f, 0x6f, 0x6c, 0x73, 0x52, 0x05, 0x74, 0x6f, 0x6f, 0x6c, + 0x73, 0x12, 0x4a, 0x0a, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x32, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, + 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x41, + 0x67, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x4a, 0x04, 0x08, + 0x01, 0x10, 0x02, 0x52, 0x10, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x5f, 0x61, 0x75, 0x74, 0x6f, 0x75, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x22, 0x2f, 0x0a, 0x19, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x54, 0x6f, 0x6f, + 0x6c, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x22, 0x8e, 0x02, 0x0a, 0x1a, 0x41, 0x75, 0x74, 0x6f, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x70, 0x65, 0x63, 0x41, + 0x67, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x74, 0x72, + 0x61, 0x74, 0x65, 0x67, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x74, 0x72, + 0x61, 0x74, 0x65, 0x67, 0x79, 0x12, 0x59, 0x0a, 0x1b, 0x6d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, + 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x5f, 0x64, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x19, 0x6d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, + 0x63, 0x65, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x4e, 0x0a, 0x09, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, + 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x67, 0x65, + 0x6e, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x63, 0x68, 0x65, + 0x64, 0x75, 0x6c, 0x65, 0x73, 0x52, 0x09, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x73, + 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x52, 0x0f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x63, + 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x22, 0x62, 0x0a, 0x18, 0x41, 0x67, 0x65, 0x6e, 0x74, + 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, + 0x6c, 0x65, 0x73, 0x12, 0x46, 0x0a, 0x07, 0x72, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, + 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x67, + 0x65, 0x6e, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x47, 0x72, 0x6f, + 0x75, 0x70, 0x52, 0x07, 0x72, 0x65, 0x67, 0x75, 0x6c, 0x61, 0x72, 0x22, 0x8d, 0x01, 0x0a, 0x14, + 0x41, 0x67, 0x65, 0x6e, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x47, + 0x72, 0x6f, 0x75, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x79, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x64, 0x61, 0x79, 0x73, 0x12, 0x1d, 0x0a, 0x0a, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x68, 0x6f, 0x75, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x48, 0x6f, 0x75, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x77, + 0x61, 0x69, 0x74, 0x5f, 0x68, 0x6f, 0x75, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x09, 0x77, 0x61, 0x69, 0x74, 0x48, 0x6f, 0x75, 0x72, 0x73, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, + 0x52, 0x09, 0x77, 0x61, 0x69, 0x74, 0x5f, 0x64, 0x61, 0x79, 0x73, 0x22, 0xd9, 0x01, 0x0a, 0x11, + 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x75, 0x62, 0x5f, 0x6b, 0x69, 0x6e, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x75, 0x62, 0x4b, 0x69, 0x6e, 0x64, + 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x38, 0x0a, 0x08, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x76, + 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x12, 0x41, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, + 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, + 0x63, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x22, 0xc3, 0x01, 0x0a, 0x15, 0x41, 0x75, 0x74, 0x6f, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, + 0x63, 0x12, 0x48, 0x0a, 0x05, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x32, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, + 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x54, + 0x6f, 0x6f, 0x6c, 0x73, 0x52, 0x05, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x12, 0x4b, 0x0a, 0x06, 0x61, + 0x67, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x74, 0x65, + 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x73, + 0x52, 0x06, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x73, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x52, 0x0d, + 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x43, 0x0a, + 0x1a, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x54, 0x6f, 0x6f, 0x6c, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x22, 0x99, 0x01, 0x0a, 0x1b, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x41, 0x67, 0x65, 0x6e, + 0x74, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x74, 0x61, 0x72, 0x74, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0d, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1a, + 0x0a, 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, + 0x64, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x22, 0xb1, + 0x02, 0x0a, 0x16, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, + 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, 0x6e, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x19, 0x0a, + 0x08, 0x73, 0x75, 0x62, 0x5f, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x73, 0x75, 0x62, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x38, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, + 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x46, 0x0a, 0x04, + 0x73, 0x70, 0x65, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x74, 0x65, 0x6c, + 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, + 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x53, 0x70, 0x65, 0x63, 0x52, 0x04, + 0x73, 0x70, 0x65, 0x63, 0x12, 0x4c, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, + 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, + 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, + 0x6c, 0x6f, 0x75, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x22, 0xa4, 0x02, 0x0a, 0x1a, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x53, 0x70, 0x65, + 0x63, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x74, 0x61, 0x72, 0x74, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a, + 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x61, 0x75, 0x74, + 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x6f, + 0x64, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x12, 0x59, + 0x0a, 0x1b, 0x6d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x77, 0x69, + 0x6e, 0x64, 0x6f, 0x77, 0x5f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x19, + 0x6d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x57, 0x69, 0x6e, 0x64, 0x6f, + 0x77, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xb8, 0x02, 0x0a, 0x1c, 0x41, 0x75, + 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, + 0x6c, 0x6f, 0x75, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x51, 0x0a, 0x06, 0x67, 0x72, + 0x6f, 0x75, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x74, 0x65, 0x6c, + 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, + 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, 0x06, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x12, 0x49, 0x0a, + 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x33, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, + 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, + 0x69, 0x6d, 0x65, 0x12, 0x3f, 0x0a, 0x0d, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x22, 0xc0, 0x03, 0x0a, 0x21, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x39, + 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x47, 0x0a, 0x05, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x31, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, + 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x12, 0x44, 0x0a, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x6c, 0x61, 0x73, 0x74, + 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x5f, 0x64, 0x61, 0x79, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x44, 0x61, 0x79, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x68, 0x6f, 0x75, 0x72, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x72, 0x74, 0x48, + 0x6f, 0x75, 0x72, 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x77, 0x61, + 0x69, 0x74, 0x5f, 0x68, 0x6f, 0x75, 0x72, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x57, 0x61, 0x69, 0x74, 0x48, 0x6f, 0x75, 0x72, 0x73, 0x4a, + 0x04, 0x08, 0x08, 0x10, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x77, 0x61, + 0x69, 0x74, 0x5f, 0x64, 0x61, 0x79, 0x73, 0x2a, 0xf7, 0x01, 0x0a, 0x19, 0x41, 0x75, 0x74, 0x6f, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2d, 0x0a, 0x29, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, + 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, + 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x2b, 0x0a, 0x27, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, + 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x54, 0x41, 0x52, 0x54, 0x45, 0x44, 0x10, + 0x01, 0x12, 0x28, 0x0a, 0x24, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, + 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, 0x54, 0x41, + 0x54, 0x45, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x26, 0x0a, 0x22, 0x41, + 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, + 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x4f, 0x4e, + 0x45, 0x10, 0x03, 0x12, 0x2c, 0x0a, 0x28, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, + 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x45, 0x44, 0x42, 0x41, 0x43, 0x4b, 0x10, + 0x04, 0x2a, 0x83, 0x02, 0x0a, 0x1b, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x12, 0x2f, 0x0a, 0x2b, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, + 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x4f, 0x55, 0x54, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x2d, 0x0a, 0x29, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, + 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x4f, 0x55, 0x54, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x54, 0x41, 0x52, 0x54, 0x45, 0x44, 0x10, + 0x01, 0x12, 0x2a, 0x0a, 0x26, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, + 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x4f, 0x55, 0x54, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x28, 0x0a, + 0x24, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, + 0x4e, 0x54, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x4f, 0x55, 0x54, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, + 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x03, 0x12, 0x2e, 0x0a, 0x2a, 0x41, 0x55, 0x54, 0x4f, 0x5f, + 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x52, 0x4f, 0x4c, + 0x4c, 0x4f, 0x55, 0x54, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x45, + 0x44, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x04, 0x42, 0x56, 0x5a, 0x54, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72, 0x61, 0x76, 0x69, 0x74, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x61, 0x6c, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x70, 0x69, + 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x74, 0x65, + 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_teleport_autoupdate_v1_autoupdate_proto_rawDescOnce sync.Once + file_teleport_autoupdate_v1_autoupdate_proto_rawDescData = file_teleport_autoupdate_v1_autoupdate_proto_rawDesc +) + +func file_teleport_autoupdate_v1_autoupdate_proto_rawDescGZIP() []byte { + file_teleport_autoupdate_v1_autoupdate_proto_rawDescOnce.Do(func() { + file_teleport_autoupdate_v1_autoupdate_proto_rawDescData = protoimpl.X.CompressGZIP(file_teleport_autoupdate_v1_autoupdate_proto_rawDescData) + }) + return file_teleport_autoupdate_v1_autoupdate_proto_rawDescData +} + +var file_teleport_autoupdate_v1_autoupdate_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_teleport_autoupdate_v1_autoupdate_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_teleport_autoupdate_v1_autoupdate_proto_goTypes = []interface{}{ + (AutoUpdateAgentGroupState)(0), // 0: teleport.autoupdate.v1.AutoUpdateAgentGroupState + (AutoUpdateAgentRolloutState)(0), // 1: teleport.autoupdate.v1.AutoUpdateAgentRolloutState + (*AutoUpdateConfig)(nil), // 2: teleport.autoupdate.v1.AutoUpdateConfig + (*AutoUpdateConfigSpec)(nil), // 3: teleport.autoupdate.v1.AutoUpdateConfigSpec + (*AutoUpdateConfigSpecTools)(nil), // 4: teleport.autoupdate.v1.AutoUpdateConfigSpecTools + (*AutoUpdateConfigSpecAgents)(nil), // 5: teleport.autoupdate.v1.AutoUpdateConfigSpecAgents + (*AgentAutoUpdateSchedules)(nil), // 6: teleport.autoupdate.v1.AgentAutoUpdateSchedules + (*AgentAutoUpdateGroup)(nil), // 7: teleport.autoupdate.v1.AgentAutoUpdateGroup + (*AutoUpdateVersion)(nil), // 8: teleport.autoupdate.v1.AutoUpdateVersion + (*AutoUpdateVersionSpec)(nil), // 9: teleport.autoupdate.v1.AutoUpdateVersionSpec + (*AutoUpdateVersionSpecTools)(nil), // 10: teleport.autoupdate.v1.AutoUpdateVersionSpecTools + (*AutoUpdateVersionSpecAgents)(nil), // 11: teleport.autoupdate.v1.AutoUpdateVersionSpecAgents + (*AutoUpdateAgentRollout)(nil), // 12: teleport.autoupdate.v1.AutoUpdateAgentRollout + (*AutoUpdateAgentRolloutSpec)(nil), // 13: teleport.autoupdate.v1.AutoUpdateAgentRolloutSpec + (*AutoUpdateAgentRolloutStatus)(nil), // 14: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatus + (*AutoUpdateAgentRolloutStatusGroup)(nil), // 15: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatusGroup + (*v1.Metadata)(nil), // 16: teleport.header.v1.Metadata + (*durationpb.Duration)(nil), // 17: google.protobuf.Duration + (*timestamppb.Timestamp)(nil), // 18: google.protobuf.Timestamp +} +var file_teleport_autoupdate_v1_autoupdate_proto_depIdxs = []int32{ + 16, // 0: teleport.autoupdate.v1.AutoUpdateConfig.metadata:type_name -> teleport.header.v1.Metadata + 3, // 1: teleport.autoupdate.v1.AutoUpdateConfig.spec:type_name -> teleport.autoupdate.v1.AutoUpdateConfigSpec + 4, // 2: teleport.autoupdate.v1.AutoUpdateConfigSpec.tools:type_name -> teleport.autoupdate.v1.AutoUpdateConfigSpecTools + 5, // 3: teleport.autoupdate.v1.AutoUpdateConfigSpec.agents:type_name -> teleport.autoupdate.v1.AutoUpdateConfigSpecAgents + 17, // 4: teleport.autoupdate.v1.AutoUpdateConfigSpecAgents.maintenance_window_duration:type_name -> google.protobuf.Duration + 6, // 5: teleport.autoupdate.v1.AutoUpdateConfigSpecAgents.schedules:type_name -> teleport.autoupdate.v1.AgentAutoUpdateSchedules + 7, // 6: teleport.autoupdate.v1.AgentAutoUpdateSchedules.regular:type_name -> teleport.autoupdate.v1.AgentAutoUpdateGroup + 16, // 7: teleport.autoupdate.v1.AutoUpdateVersion.metadata:type_name -> teleport.header.v1.Metadata + 9, // 8: teleport.autoupdate.v1.AutoUpdateVersion.spec:type_name -> teleport.autoupdate.v1.AutoUpdateVersionSpec + 10, // 9: teleport.autoupdate.v1.AutoUpdateVersionSpec.tools:type_name -> teleport.autoupdate.v1.AutoUpdateVersionSpecTools + 11, // 10: teleport.autoupdate.v1.AutoUpdateVersionSpec.agents:type_name -> teleport.autoupdate.v1.AutoUpdateVersionSpecAgents + 16, // 11: teleport.autoupdate.v1.AutoUpdateAgentRollout.metadata:type_name -> teleport.header.v1.Metadata + 13, // 12: teleport.autoupdate.v1.AutoUpdateAgentRollout.spec:type_name -> teleport.autoupdate.v1.AutoUpdateAgentRolloutSpec + 14, // 13: teleport.autoupdate.v1.AutoUpdateAgentRollout.status:type_name -> teleport.autoupdate.v1.AutoUpdateAgentRolloutStatus + 17, // 14: teleport.autoupdate.v1.AutoUpdateAgentRolloutSpec.maintenance_window_duration:type_name -> google.protobuf.Duration + 15, // 15: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatus.groups:type_name -> teleport.autoupdate.v1.AutoUpdateAgentRolloutStatusGroup + 1, // 16: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatus.state:type_name -> teleport.autoupdate.v1.AutoUpdateAgentRolloutState + 18, // 17: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatus.start_time:type_name -> google.protobuf.Timestamp + 18, // 18: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatus.time_override:type_name -> google.protobuf.Timestamp + 18, // 19: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatusGroup.start_time:type_name -> google.protobuf.Timestamp + 0, // 20: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatusGroup.state:type_name -> teleport.autoupdate.v1.AutoUpdateAgentGroupState + 18, // 21: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatusGroup.last_update_time:type_name -> google.protobuf.Timestamp + 22, // [22:22] is the sub-list for method output_type + 22, // [22:22] is the sub-list for method input_type + 22, // [22:22] is the sub-list for extension type_name + 22, // [22:22] is the sub-list for extension extendee + 0, // [0:22] is the sub-list for field type_name +} + +func init() { file_teleport_autoupdate_v1_autoupdate_proto_init() } +func file_teleport_autoupdate_v1_autoupdate_proto_init() { + if File_teleport_autoupdate_v1_autoupdate_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AutoUpdateConfig); i { case 0: return &v.state case 1: @@ -539,7 +1567,7 @@ func file_teleport_autoupdate_v1_autoupdate_proto_init() { } } file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AutoUpdateVersion); i { + switch v := v.(*AutoUpdateConfigSpecAgents); i { case 0: return &v.state case 1: @@ -551,7 +1579,7 @@ func file_teleport_autoupdate_v1_autoupdate_proto_init() { } } file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AutoUpdateVersionSpec); i { + switch v := v.(*AgentAutoUpdateSchedules); i { case 0: return &v.state case 1: @@ -563,6 +1591,42 @@ func file_teleport_autoupdate_v1_autoupdate_proto_init() { } } file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AgentAutoUpdateGroup); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AutoUpdateVersion); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AutoUpdateVersionSpec); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*AutoUpdateVersionSpecTools); i { case 0: return &v.state @@ -574,19 +1638,80 @@ func file_teleport_autoupdate_v1_autoupdate_proto_init() { return nil } } + file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AutoUpdateVersionSpecAgents); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AutoUpdateAgentRollout); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AutoUpdateAgentRolloutSpec); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AutoUpdateAgentRolloutStatus); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_teleport_autoupdate_v1_autoupdate_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AutoUpdateAgentRolloutStatusGroup); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_teleport_autoupdate_v1_autoupdate_proto_rawDesc, - NumEnums: 0, - NumMessages: 6, + NumEnums: 2, + NumMessages: 14, NumExtensions: 0, NumServices: 0, }, GoTypes: file_teleport_autoupdate_v1_autoupdate_proto_goTypes, DependencyIndexes: file_teleport_autoupdate_v1_autoupdate_proto_depIdxs, + EnumInfos: file_teleport_autoupdate_v1_autoupdate_proto_enumTypes, MessageInfos: file_teleport_autoupdate_v1_autoupdate_proto_msgTypes, }.Build() File_teleport_autoupdate_v1_autoupdate_proto = out.File diff --git a/api/gen/proto/go/teleport/autoupdate/v1/autoupdate_service.pb.go b/api/gen/proto/go/teleport/autoupdate/v1/autoupdate_service.pb.go index 92d3898b1e75f..3ce99c99616a3 100644 --- a/api/gen/proto/go/teleport/autoupdate/v1/autoupdate_service.pb.go +++ b/api/gen/proto/go/teleport/autoupdate/v1/autoupdate_service.pb.go @@ -479,6 +479,228 @@ func (*DeleteAutoUpdateVersionRequest) Descriptor() ([]byte, []int) { return file_teleport_autoupdate_v1_autoupdate_service_proto_rawDescGZIP(), []int{9} } +// Request for GetAutoUpdateAgentRollout. +type GetAutoUpdateAgentRolloutRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *GetAutoUpdateAgentRolloutRequest) Reset() { + *x = GetAutoUpdateAgentRolloutRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetAutoUpdateAgentRolloutRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetAutoUpdateAgentRolloutRequest) ProtoMessage() {} + +func (x *GetAutoUpdateAgentRolloutRequest) ProtoReflect() protoreflect.Message { + mi := &file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetAutoUpdateAgentRolloutRequest.ProtoReflect.Descriptor instead. +func (*GetAutoUpdateAgentRolloutRequest) Descriptor() ([]byte, []int) { + return file_teleport_autoupdate_v1_autoupdate_service_proto_rawDescGZIP(), []int{10} +} + +// Request for CreateAutoUpdateAgentRollout. +type CreateAutoUpdateAgentRolloutRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Rollout *AutoUpdateAgentRollout `protobuf:"bytes,1,opt,name=rollout,proto3" json:"rollout,omitempty"` +} + +func (x *CreateAutoUpdateAgentRolloutRequest) Reset() { + *x = CreateAutoUpdateAgentRolloutRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CreateAutoUpdateAgentRolloutRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateAutoUpdateAgentRolloutRequest) ProtoMessage() {} + +func (x *CreateAutoUpdateAgentRolloutRequest) ProtoReflect() protoreflect.Message { + mi := &file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateAutoUpdateAgentRolloutRequest.ProtoReflect.Descriptor instead. +func (*CreateAutoUpdateAgentRolloutRequest) Descriptor() ([]byte, []int) { + return file_teleport_autoupdate_v1_autoupdate_service_proto_rawDescGZIP(), []int{11} +} + +func (x *CreateAutoUpdateAgentRolloutRequest) GetRollout() *AutoUpdateAgentRollout { + if x != nil { + return x.Rollout + } + return nil +} + +// Request for UpdateAutoUpdateConfig. +type UpdateAutoUpdateAgentRolloutRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Rollout *AutoUpdateAgentRollout `protobuf:"bytes,1,opt,name=rollout,proto3" json:"rollout,omitempty"` +} + +func (x *UpdateAutoUpdateAgentRolloutRequest) Reset() { + *x = UpdateAutoUpdateAgentRolloutRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *UpdateAutoUpdateAgentRolloutRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpdateAutoUpdateAgentRolloutRequest) ProtoMessage() {} + +func (x *UpdateAutoUpdateAgentRolloutRequest) ProtoReflect() protoreflect.Message { + mi := &file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpdateAutoUpdateAgentRolloutRequest.ProtoReflect.Descriptor instead. +func (*UpdateAutoUpdateAgentRolloutRequest) Descriptor() ([]byte, []int) { + return file_teleport_autoupdate_v1_autoupdate_service_proto_rawDescGZIP(), []int{12} +} + +func (x *UpdateAutoUpdateAgentRolloutRequest) GetRollout() *AutoUpdateAgentRollout { + if x != nil { + return x.Rollout + } + return nil +} + +// Request for UpsertAutoUpdateAgentRollout. +type UpsertAutoUpdateAgentRolloutRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Rollout *AutoUpdateAgentRollout `protobuf:"bytes,1,opt,name=rollout,proto3" json:"rollout,omitempty"` +} + +func (x *UpsertAutoUpdateAgentRolloutRequest) Reset() { + *x = UpsertAutoUpdateAgentRolloutRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *UpsertAutoUpdateAgentRolloutRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpsertAutoUpdateAgentRolloutRequest) ProtoMessage() {} + +func (x *UpsertAutoUpdateAgentRolloutRequest) ProtoReflect() protoreflect.Message { + mi := &file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpsertAutoUpdateAgentRolloutRequest.ProtoReflect.Descriptor instead. +func (*UpsertAutoUpdateAgentRolloutRequest) Descriptor() ([]byte, []int) { + return file_teleport_autoupdate_v1_autoupdate_service_proto_rawDescGZIP(), []int{13} +} + +func (x *UpsertAutoUpdateAgentRolloutRequest) GetRollout() *AutoUpdateAgentRollout { + if x != nil { + return x.Rollout + } + return nil +} + +// Request for DeleteAutoUpdateAgentRollout. +type DeleteAutoUpdateAgentRolloutRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *DeleteAutoUpdateAgentRolloutRequest) Reset() { + *x = DeleteAutoUpdateAgentRolloutRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DeleteAutoUpdateAgentRolloutRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteAutoUpdateAgentRolloutRequest) ProtoMessage() {} + +func (x *DeleteAutoUpdateAgentRolloutRequest) ProtoReflect() protoreflect.Message { + mi := &file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteAutoUpdateAgentRolloutRequest.ProtoReflect.Descriptor instead. +func (*DeleteAutoUpdateAgentRolloutRequest) Descriptor() ([]byte, []int) { + return file_teleport_autoupdate_v1_autoupdate_service_proto_rawDescGZIP(), []int{14} +} + var File_teleport_autoupdate_v1_autoupdate_service_proto protoreflect.FileDescriptor var file_teleport_autoupdate_v1_autoupdate_service_proto_rawDesc = []byte{ @@ -536,89 +758,158 @@ var file_teleport_autoupdate_v1_autoupdate_service_proto_rawDesc = []byte{ 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x20, 0x0a, 0x1e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x32, 0xbf, 0x09, - 0x0a, 0x11, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x12, 0x73, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x32, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, - 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x79, 0x0a, 0x16, 0x43, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x35, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, - 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x22, 0x0a, + 0x20, 0x47, 0x65, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, + 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x22, 0x6f, 0x0a, 0x23, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x48, 0x0a, 0x07, 0x72, 0x6f, 0x6c, 0x6c, + 0x6f, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x12, 0x79, 0x0a, 0x16, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, - 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x35, 0x2e, + 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, + 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x52, 0x07, 0x72, 0x6f, 0x6c, 0x6c, 0x6f, + 0x75, 0x74, 0x22, 0x6f, 0x0a, 0x23, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, + 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x48, 0x0a, 0x07, 0x72, 0x6f, 0x6c, + 0x6c, 0x6f, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x74, 0x65, 0x6c, + 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, + 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x52, 0x07, 0x72, 0x6f, 0x6c, 0x6c, + 0x6f, 0x75, 0x74, 0x22, 0x6f, 0x0a, 0x23, 0x55, 0x70, 0x73, 0x65, 0x72, 0x74, 0x41, 0x75, 0x74, + 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, + 0x6f, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x48, 0x0a, 0x07, 0x72, 0x6f, + 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x74, 0x65, + 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, + 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x52, 0x07, 0x72, 0x6f, 0x6c, + 0x6c, 0x6f, 0x75, 0x74, 0x22, 0x25, 0x0a, 0x23, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x41, 0x75, + 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, + 0x6c, 0x6f, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x32, 0xe6, 0x0e, 0x0a, 0x11, + 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x12, 0x73, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x32, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x74, + 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x79, 0x0a, 0x16, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x12, 0x35, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, + 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, + 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x12, 0x79, 0x0a, 0x16, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x35, 0x2e, 0x74, 0x65, + 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, + 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x79, 0x0a, 0x16, + 0x55, 0x70, 0x73, 0x65, 0x72, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x35, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, + 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x55, 0x70, 0x73, 0x65, 0x72, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, - 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, - 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, - 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x79, - 0x0a, 0x16, 0x55, 0x70, 0x73, 0x65, 0x72, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x35, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x67, 0x0a, 0x16, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x12, 0x35, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, + 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x12, 0x76, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x33, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x55, 0x70, 0x73, 0x65, 0x72, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x28, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x67, 0x0a, 0x16, 0x44, 0x65, 0x6c, - 0x65, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x12, 0x35, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, - 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x6c, - 0x65, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x12, 0x76, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x33, 0x2e, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x7c, 0x0a, 0x17, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x36, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, - 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, + 0x31, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x7c, 0x0a, 0x17, 0x55, 0x70, 0x64, 0x61, + 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x7c, 0x0a, 0x17, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x36, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, - 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, + 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x7c, 0x0a, 0x17, 0x55, 0x70, 0x73, 0x65, 0x72, 0x74, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x7c, 0x0a, 0x17, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x36, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, - 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x73, 0x65, 0x72, - 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x69, 0x0a, 0x17, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x41, 0x75, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x7c, 0x0a, 0x17, 0x55, 0x70, 0x73, 0x65, 0x72, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x36, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x41, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x73, 0x65, 0x72, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, - 0x56, 0x5a, 0x54, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72, - 0x61, 0x76, 0x69, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x2f, 0x74, 0x65, 0x6c, 0x65, - 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, - 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x75, 0x74, - 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, + 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x69, 0x0a, 0x17, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x36, 0x2e, + 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x41, 0x75, 0x74, + 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x85, 0x01, + 0x0a, 0x19, 0x47, 0x65, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, + 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x12, 0x38, 0x2e, 0x74, 0x65, + 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, + 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, + 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, + 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x12, 0x8b, 0x01, 0x0a, 0x1c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, + 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x12, 0x3b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, + 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, + 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, + 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, + 0x6f, 0x75, 0x74, 0x12, 0x8b, 0x01, 0x0a, 0x1c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x75, + 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, + 0x6c, 0x6f, 0x75, 0x74, 0x12, 0x3b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, + 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, + 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x2e, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, + 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, + 0x74, 0x12, 0x8b, 0x01, 0x0a, 0x1c, 0x55, 0x70, 0x73, 0x65, 0x72, 0x74, 0x41, 0x75, 0x74, 0x6f, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, + 0x75, 0x74, 0x12, 0x3b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, + 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x73, 0x65, + 0x72, 0x74, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, + 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x2e, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x12, + 0x73, 0x0a, 0x1c, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x12, + 0x3b, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x41, + 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, + 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x42, 0x56, 0x5a, 0x54, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x67, 0x72, 0x61, 0x76, 0x69, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, + 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x65, + 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2f, 0x76, + 0x31, 0x3b, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -633,54 +924,73 @@ func file_teleport_autoupdate_v1_autoupdate_service_proto_rawDescGZIP() []byte { return file_teleport_autoupdate_v1_autoupdate_service_proto_rawDescData } -var file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_teleport_autoupdate_v1_autoupdate_service_proto_goTypes = []interface{}{ - (*GetAutoUpdateConfigRequest)(nil), // 0: teleport.autoupdate.v1.GetAutoUpdateConfigRequest - (*CreateAutoUpdateConfigRequest)(nil), // 1: teleport.autoupdate.v1.CreateAutoUpdateConfigRequest - (*UpdateAutoUpdateConfigRequest)(nil), // 2: teleport.autoupdate.v1.UpdateAutoUpdateConfigRequest - (*UpsertAutoUpdateConfigRequest)(nil), // 3: teleport.autoupdate.v1.UpsertAutoUpdateConfigRequest - (*DeleteAutoUpdateConfigRequest)(nil), // 4: teleport.autoupdate.v1.DeleteAutoUpdateConfigRequest - (*GetAutoUpdateVersionRequest)(nil), // 5: teleport.autoupdate.v1.GetAutoUpdateVersionRequest - (*CreateAutoUpdateVersionRequest)(nil), // 6: teleport.autoupdate.v1.CreateAutoUpdateVersionRequest - (*UpdateAutoUpdateVersionRequest)(nil), // 7: teleport.autoupdate.v1.UpdateAutoUpdateVersionRequest - (*UpsertAutoUpdateVersionRequest)(nil), // 8: teleport.autoupdate.v1.UpsertAutoUpdateVersionRequest - (*DeleteAutoUpdateVersionRequest)(nil), // 9: teleport.autoupdate.v1.DeleteAutoUpdateVersionRequest - (*AutoUpdateConfig)(nil), // 10: teleport.autoupdate.v1.AutoUpdateConfig - (*AutoUpdateVersion)(nil), // 11: teleport.autoupdate.v1.AutoUpdateVersion - (*emptypb.Empty)(nil), // 12: google.protobuf.Empty + (*GetAutoUpdateConfigRequest)(nil), // 0: teleport.autoupdate.v1.GetAutoUpdateConfigRequest + (*CreateAutoUpdateConfigRequest)(nil), // 1: teleport.autoupdate.v1.CreateAutoUpdateConfigRequest + (*UpdateAutoUpdateConfigRequest)(nil), // 2: teleport.autoupdate.v1.UpdateAutoUpdateConfigRequest + (*UpsertAutoUpdateConfigRequest)(nil), // 3: teleport.autoupdate.v1.UpsertAutoUpdateConfigRequest + (*DeleteAutoUpdateConfigRequest)(nil), // 4: teleport.autoupdate.v1.DeleteAutoUpdateConfigRequest + (*GetAutoUpdateVersionRequest)(nil), // 5: teleport.autoupdate.v1.GetAutoUpdateVersionRequest + (*CreateAutoUpdateVersionRequest)(nil), // 6: teleport.autoupdate.v1.CreateAutoUpdateVersionRequest + (*UpdateAutoUpdateVersionRequest)(nil), // 7: teleport.autoupdate.v1.UpdateAutoUpdateVersionRequest + (*UpsertAutoUpdateVersionRequest)(nil), // 8: teleport.autoupdate.v1.UpsertAutoUpdateVersionRequest + (*DeleteAutoUpdateVersionRequest)(nil), // 9: teleport.autoupdate.v1.DeleteAutoUpdateVersionRequest + (*GetAutoUpdateAgentRolloutRequest)(nil), // 10: teleport.autoupdate.v1.GetAutoUpdateAgentRolloutRequest + (*CreateAutoUpdateAgentRolloutRequest)(nil), // 11: teleport.autoupdate.v1.CreateAutoUpdateAgentRolloutRequest + (*UpdateAutoUpdateAgentRolloutRequest)(nil), // 12: teleport.autoupdate.v1.UpdateAutoUpdateAgentRolloutRequest + (*UpsertAutoUpdateAgentRolloutRequest)(nil), // 13: teleport.autoupdate.v1.UpsertAutoUpdateAgentRolloutRequest + (*DeleteAutoUpdateAgentRolloutRequest)(nil), // 14: teleport.autoupdate.v1.DeleteAutoUpdateAgentRolloutRequest + (*AutoUpdateConfig)(nil), // 15: teleport.autoupdate.v1.AutoUpdateConfig + (*AutoUpdateVersion)(nil), // 16: teleport.autoupdate.v1.AutoUpdateVersion + (*AutoUpdateAgentRollout)(nil), // 17: teleport.autoupdate.v1.AutoUpdateAgentRollout + (*emptypb.Empty)(nil), // 18: google.protobuf.Empty } var file_teleport_autoupdate_v1_autoupdate_service_proto_depIdxs = []int32{ - 10, // 0: teleport.autoupdate.v1.CreateAutoUpdateConfigRequest.config:type_name -> teleport.autoupdate.v1.AutoUpdateConfig - 10, // 1: teleport.autoupdate.v1.UpdateAutoUpdateConfigRequest.config:type_name -> teleport.autoupdate.v1.AutoUpdateConfig - 10, // 2: teleport.autoupdate.v1.UpsertAutoUpdateConfigRequest.config:type_name -> teleport.autoupdate.v1.AutoUpdateConfig - 11, // 3: teleport.autoupdate.v1.CreateAutoUpdateVersionRequest.version:type_name -> teleport.autoupdate.v1.AutoUpdateVersion - 11, // 4: teleport.autoupdate.v1.UpdateAutoUpdateVersionRequest.version:type_name -> teleport.autoupdate.v1.AutoUpdateVersion - 11, // 5: teleport.autoupdate.v1.UpsertAutoUpdateVersionRequest.version:type_name -> teleport.autoupdate.v1.AutoUpdateVersion - 0, // 6: teleport.autoupdate.v1.AutoUpdateService.GetAutoUpdateConfig:input_type -> teleport.autoupdate.v1.GetAutoUpdateConfigRequest - 1, // 7: teleport.autoupdate.v1.AutoUpdateService.CreateAutoUpdateConfig:input_type -> teleport.autoupdate.v1.CreateAutoUpdateConfigRequest - 2, // 8: teleport.autoupdate.v1.AutoUpdateService.UpdateAutoUpdateConfig:input_type -> teleport.autoupdate.v1.UpdateAutoUpdateConfigRequest - 3, // 9: teleport.autoupdate.v1.AutoUpdateService.UpsertAutoUpdateConfig:input_type -> teleport.autoupdate.v1.UpsertAutoUpdateConfigRequest - 4, // 10: teleport.autoupdate.v1.AutoUpdateService.DeleteAutoUpdateConfig:input_type -> teleport.autoupdate.v1.DeleteAutoUpdateConfigRequest - 5, // 11: teleport.autoupdate.v1.AutoUpdateService.GetAutoUpdateVersion:input_type -> teleport.autoupdate.v1.GetAutoUpdateVersionRequest - 6, // 12: teleport.autoupdate.v1.AutoUpdateService.CreateAutoUpdateVersion:input_type -> teleport.autoupdate.v1.CreateAutoUpdateVersionRequest - 7, // 13: teleport.autoupdate.v1.AutoUpdateService.UpdateAutoUpdateVersion:input_type -> teleport.autoupdate.v1.UpdateAutoUpdateVersionRequest - 8, // 14: teleport.autoupdate.v1.AutoUpdateService.UpsertAutoUpdateVersion:input_type -> teleport.autoupdate.v1.UpsertAutoUpdateVersionRequest - 9, // 15: teleport.autoupdate.v1.AutoUpdateService.DeleteAutoUpdateVersion:input_type -> teleport.autoupdate.v1.DeleteAutoUpdateVersionRequest - 10, // 16: teleport.autoupdate.v1.AutoUpdateService.GetAutoUpdateConfig:output_type -> teleport.autoupdate.v1.AutoUpdateConfig - 10, // 17: teleport.autoupdate.v1.AutoUpdateService.CreateAutoUpdateConfig:output_type -> teleport.autoupdate.v1.AutoUpdateConfig - 10, // 18: teleport.autoupdate.v1.AutoUpdateService.UpdateAutoUpdateConfig:output_type -> teleport.autoupdate.v1.AutoUpdateConfig - 10, // 19: teleport.autoupdate.v1.AutoUpdateService.UpsertAutoUpdateConfig:output_type -> teleport.autoupdate.v1.AutoUpdateConfig - 12, // 20: teleport.autoupdate.v1.AutoUpdateService.DeleteAutoUpdateConfig:output_type -> google.protobuf.Empty - 11, // 21: teleport.autoupdate.v1.AutoUpdateService.GetAutoUpdateVersion:output_type -> teleport.autoupdate.v1.AutoUpdateVersion - 11, // 22: teleport.autoupdate.v1.AutoUpdateService.CreateAutoUpdateVersion:output_type -> teleport.autoupdate.v1.AutoUpdateVersion - 11, // 23: teleport.autoupdate.v1.AutoUpdateService.UpdateAutoUpdateVersion:output_type -> teleport.autoupdate.v1.AutoUpdateVersion - 11, // 24: teleport.autoupdate.v1.AutoUpdateService.UpsertAutoUpdateVersion:output_type -> teleport.autoupdate.v1.AutoUpdateVersion - 12, // 25: teleport.autoupdate.v1.AutoUpdateService.DeleteAutoUpdateVersion:output_type -> google.protobuf.Empty - 16, // [16:26] is the sub-list for method output_type - 6, // [6:16] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name + 15, // 0: teleport.autoupdate.v1.CreateAutoUpdateConfigRequest.config:type_name -> teleport.autoupdate.v1.AutoUpdateConfig + 15, // 1: teleport.autoupdate.v1.UpdateAutoUpdateConfigRequest.config:type_name -> teleport.autoupdate.v1.AutoUpdateConfig + 15, // 2: teleport.autoupdate.v1.UpsertAutoUpdateConfigRequest.config:type_name -> teleport.autoupdate.v1.AutoUpdateConfig + 16, // 3: teleport.autoupdate.v1.CreateAutoUpdateVersionRequest.version:type_name -> teleport.autoupdate.v1.AutoUpdateVersion + 16, // 4: teleport.autoupdate.v1.UpdateAutoUpdateVersionRequest.version:type_name -> teleport.autoupdate.v1.AutoUpdateVersion + 16, // 5: teleport.autoupdate.v1.UpsertAutoUpdateVersionRequest.version:type_name -> teleport.autoupdate.v1.AutoUpdateVersion + 17, // 6: teleport.autoupdate.v1.CreateAutoUpdateAgentRolloutRequest.rollout:type_name -> teleport.autoupdate.v1.AutoUpdateAgentRollout + 17, // 7: teleport.autoupdate.v1.UpdateAutoUpdateAgentRolloutRequest.rollout:type_name -> teleport.autoupdate.v1.AutoUpdateAgentRollout + 17, // 8: teleport.autoupdate.v1.UpsertAutoUpdateAgentRolloutRequest.rollout:type_name -> teleport.autoupdate.v1.AutoUpdateAgentRollout + 0, // 9: teleport.autoupdate.v1.AutoUpdateService.GetAutoUpdateConfig:input_type -> teleport.autoupdate.v1.GetAutoUpdateConfigRequest + 1, // 10: teleport.autoupdate.v1.AutoUpdateService.CreateAutoUpdateConfig:input_type -> teleport.autoupdate.v1.CreateAutoUpdateConfigRequest + 2, // 11: teleport.autoupdate.v1.AutoUpdateService.UpdateAutoUpdateConfig:input_type -> teleport.autoupdate.v1.UpdateAutoUpdateConfigRequest + 3, // 12: teleport.autoupdate.v1.AutoUpdateService.UpsertAutoUpdateConfig:input_type -> teleport.autoupdate.v1.UpsertAutoUpdateConfigRequest + 4, // 13: teleport.autoupdate.v1.AutoUpdateService.DeleteAutoUpdateConfig:input_type -> teleport.autoupdate.v1.DeleteAutoUpdateConfigRequest + 5, // 14: teleport.autoupdate.v1.AutoUpdateService.GetAutoUpdateVersion:input_type -> teleport.autoupdate.v1.GetAutoUpdateVersionRequest + 6, // 15: teleport.autoupdate.v1.AutoUpdateService.CreateAutoUpdateVersion:input_type -> teleport.autoupdate.v1.CreateAutoUpdateVersionRequest + 7, // 16: teleport.autoupdate.v1.AutoUpdateService.UpdateAutoUpdateVersion:input_type -> teleport.autoupdate.v1.UpdateAutoUpdateVersionRequest + 8, // 17: teleport.autoupdate.v1.AutoUpdateService.UpsertAutoUpdateVersion:input_type -> teleport.autoupdate.v1.UpsertAutoUpdateVersionRequest + 9, // 18: teleport.autoupdate.v1.AutoUpdateService.DeleteAutoUpdateVersion:input_type -> teleport.autoupdate.v1.DeleteAutoUpdateVersionRequest + 10, // 19: teleport.autoupdate.v1.AutoUpdateService.GetAutoUpdateAgentRollout:input_type -> teleport.autoupdate.v1.GetAutoUpdateAgentRolloutRequest + 11, // 20: teleport.autoupdate.v1.AutoUpdateService.CreateAutoUpdateAgentRollout:input_type -> teleport.autoupdate.v1.CreateAutoUpdateAgentRolloutRequest + 12, // 21: teleport.autoupdate.v1.AutoUpdateService.UpdateAutoUpdateAgentRollout:input_type -> teleport.autoupdate.v1.UpdateAutoUpdateAgentRolloutRequest + 13, // 22: teleport.autoupdate.v1.AutoUpdateService.UpsertAutoUpdateAgentRollout:input_type -> teleport.autoupdate.v1.UpsertAutoUpdateAgentRolloutRequest + 14, // 23: teleport.autoupdate.v1.AutoUpdateService.DeleteAutoUpdateAgentRollout:input_type -> teleport.autoupdate.v1.DeleteAutoUpdateAgentRolloutRequest + 15, // 24: teleport.autoupdate.v1.AutoUpdateService.GetAutoUpdateConfig:output_type -> teleport.autoupdate.v1.AutoUpdateConfig + 15, // 25: teleport.autoupdate.v1.AutoUpdateService.CreateAutoUpdateConfig:output_type -> teleport.autoupdate.v1.AutoUpdateConfig + 15, // 26: teleport.autoupdate.v1.AutoUpdateService.UpdateAutoUpdateConfig:output_type -> teleport.autoupdate.v1.AutoUpdateConfig + 15, // 27: teleport.autoupdate.v1.AutoUpdateService.UpsertAutoUpdateConfig:output_type -> teleport.autoupdate.v1.AutoUpdateConfig + 18, // 28: teleport.autoupdate.v1.AutoUpdateService.DeleteAutoUpdateConfig:output_type -> google.protobuf.Empty + 16, // 29: teleport.autoupdate.v1.AutoUpdateService.GetAutoUpdateVersion:output_type -> teleport.autoupdate.v1.AutoUpdateVersion + 16, // 30: teleport.autoupdate.v1.AutoUpdateService.CreateAutoUpdateVersion:output_type -> teleport.autoupdate.v1.AutoUpdateVersion + 16, // 31: teleport.autoupdate.v1.AutoUpdateService.UpdateAutoUpdateVersion:output_type -> teleport.autoupdate.v1.AutoUpdateVersion + 16, // 32: teleport.autoupdate.v1.AutoUpdateService.UpsertAutoUpdateVersion:output_type -> teleport.autoupdate.v1.AutoUpdateVersion + 18, // 33: teleport.autoupdate.v1.AutoUpdateService.DeleteAutoUpdateVersion:output_type -> google.protobuf.Empty + 17, // 34: teleport.autoupdate.v1.AutoUpdateService.GetAutoUpdateAgentRollout:output_type -> teleport.autoupdate.v1.AutoUpdateAgentRollout + 17, // 35: teleport.autoupdate.v1.AutoUpdateService.CreateAutoUpdateAgentRollout:output_type -> teleport.autoupdate.v1.AutoUpdateAgentRollout + 17, // 36: teleport.autoupdate.v1.AutoUpdateService.UpdateAutoUpdateAgentRollout:output_type -> teleport.autoupdate.v1.AutoUpdateAgentRollout + 17, // 37: teleport.autoupdate.v1.AutoUpdateService.UpsertAutoUpdateAgentRollout:output_type -> teleport.autoupdate.v1.AutoUpdateAgentRollout + 18, // 38: teleport.autoupdate.v1.AutoUpdateService.DeleteAutoUpdateAgentRollout:output_type -> google.protobuf.Empty + 24, // [24:39] is the sub-list for method output_type + 9, // [9:24] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_teleport_autoupdate_v1_autoupdate_service_proto_init() } @@ -810,6 +1120,66 @@ func file_teleport_autoupdate_v1_autoupdate_service_proto_init() { return nil } } + file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetAutoUpdateAgentRolloutRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CreateAutoUpdateAgentRolloutRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*UpdateAutoUpdateAgentRolloutRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*UpsertAutoUpdateAgentRolloutRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_teleport_autoupdate_v1_autoupdate_service_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DeleteAutoUpdateAgentRolloutRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -817,7 +1187,7 @@ func file_teleport_autoupdate_v1_autoupdate_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_teleport_autoupdate_v1_autoupdate_service_proto_rawDesc, NumEnums: 0, - NumMessages: 10, + NumMessages: 15, NumExtensions: 0, NumServices: 1, }, diff --git a/api/gen/proto/go/teleport/autoupdate/v1/autoupdate_service_grpc.pb.go b/api/gen/proto/go/teleport/autoupdate/v1/autoupdate_service_grpc.pb.go index aee73687b3b3f..5fcb16b46f243 100644 --- a/api/gen/proto/go/teleport/autoupdate/v1/autoupdate_service_grpc.pb.go +++ b/api/gen/proto/go/teleport/autoupdate/v1/autoupdate_service_grpc.pb.go @@ -34,16 +34,21 @@ import ( const _ = grpc.SupportPackageIsVersion7 const ( - AutoUpdateService_GetAutoUpdateConfig_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/GetAutoUpdateConfig" - AutoUpdateService_CreateAutoUpdateConfig_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/CreateAutoUpdateConfig" - AutoUpdateService_UpdateAutoUpdateConfig_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/UpdateAutoUpdateConfig" - AutoUpdateService_UpsertAutoUpdateConfig_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/UpsertAutoUpdateConfig" - AutoUpdateService_DeleteAutoUpdateConfig_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/DeleteAutoUpdateConfig" - AutoUpdateService_GetAutoUpdateVersion_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/GetAutoUpdateVersion" - AutoUpdateService_CreateAutoUpdateVersion_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/CreateAutoUpdateVersion" - AutoUpdateService_UpdateAutoUpdateVersion_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/UpdateAutoUpdateVersion" - AutoUpdateService_UpsertAutoUpdateVersion_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/UpsertAutoUpdateVersion" - AutoUpdateService_DeleteAutoUpdateVersion_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/DeleteAutoUpdateVersion" + AutoUpdateService_GetAutoUpdateConfig_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/GetAutoUpdateConfig" + AutoUpdateService_CreateAutoUpdateConfig_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/CreateAutoUpdateConfig" + AutoUpdateService_UpdateAutoUpdateConfig_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/UpdateAutoUpdateConfig" + AutoUpdateService_UpsertAutoUpdateConfig_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/UpsertAutoUpdateConfig" + AutoUpdateService_DeleteAutoUpdateConfig_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/DeleteAutoUpdateConfig" + AutoUpdateService_GetAutoUpdateVersion_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/GetAutoUpdateVersion" + AutoUpdateService_CreateAutoUpdateVersion_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/CreateAutoUpdateVersion" + AutoUpdateService_UpdateAutoUpdateVersion_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/UpdateAutoUpdateVersion" + AutoUpdateService_UpsertAutoUpdateVersion_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/UpsertAutoUpdateVersion" + AutoUpdateService_DeleteAutoUpdateVersion_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/DeleteAutoUpdateVersion" + AutoUpdateService_GetAutoUpdateAgentRollout_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/GetAutoUpdateAgentRollout" + AutoUpdateService_CreateAutoUpdateAgentRollout_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/CreateAutoUpdateAgentRollout" + AutoUpdateService_UpdateAutoUpdateAgentRollout_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/UpdateAutoUpdateAgentRollout" + AutoUpdateService_UpsertAutoUpdateAgentRollout_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/UpsertAutoUpdateAgentRollout" + AutoUpdateService_DeleteAutoUpdateAgentRollout_FullMethodName = "/teleport.autoupdate.v1.AutoUpdateService/DeleteAutoUpdateAgentRollout" ) // AutoUpdateServiceClient is the client API for AutoUpdateService service. @@ -70,6 +75,16 @@ type AutoUpdateServiceClient interface { UpsertAutoUpdateVersion(ctx context.Context, in *UpsertAutoUpdateVersionRequest, opts ...grpc.CallOption) (*AutoUpdateVersion, error) // DeleteAutoUpdateVersion hard deletes the specified AutoUpdateVersionRequest. DeleteAutoUpdateVersion(ctx context.Context, in *DeleteAutoUpdateVersionRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + // GetAutoUpdateVersion gets the current autoupdate version singleton. + GetAutoUpdateAgentRollout(ctx context.Context, in *GetAutoUpdateAgentRolloutRequest, opts ...grpc.CallOption) (*AutoUpdateAgentRollout, error) + // CreateAutoUpdateAgentRollout creates a new AutoUpdateAgentRollout. + CreateAutoUpdateAgentRollout(ctx context.Context, in *CreateAutoUpdateAgentRolloutRequest, opts ...grpc.CallOption) (*AutoUpdateAgentRollout, error) + // UpdateAutoUpdateAgentRollout updates AutoUpdateAgentRollout singleton. + UpdateAutoUpdateAgentRollout(ctx context.Context, in *UpdateAutoUpdateAgentRolloutRequest, opts ...grpc.CallOption) (*AutoUpdateAgentRollout, error) + // UpsertAutoUpdateAgentRollout creates a new AutoUpdateAgentRollout or replaces an existing AutoUpdateAgentRollout. + UpsertAutoUpdateAgentRollout(ctx context.Context, in *UpsertAutoUpdateAgentRolloutRequest, opts ...grpc.CallOption) (*AutoUpdateAgentRollout, error) + // DeleteAutoUpdateAgentRollout hard deletes the specified AutoUpdateAgentRolloutRequest. + DeleteAutoUpdateAgentRollout(ctx context.Context, in *DeleteAutoUpdateAgentRolloutRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) } type autoUpdateServiceClient struct { @@ -170,6 +185,51 @@ func (c *autoUpdateServiceClient) DeleteAutoUpdateVersion(ctx context.Context, i return out, nil } +func (c *autoUpdateServiceClient) GetAutoUpdateAgentRollout(ctx context.Context, in *GetAutoUpdateAgentRolloutRequest, opts ...grpc.CallOption) (*AutoUpdateAgentRollout, error) { + out := new(AutoUpdateAgentRollout) + err := c.cc.Invoke(ctx, AutoUpdateService_GetAutoUpdateAgentRollout_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *autoUpdateServiceClient) CreateAutoUpdateAgentRollout(ctx context.Context, in *CreateAutoUpdateAgentRolloutRequest, opts ...grpc.CallOption) (*AutoUpdateAgentRollout, error) { + out := new(AutoUpdateAgentRollout) + err := c.cc.Invoke(ctx, AutoUpdateService_CreateAutoUpdateAgentRollout_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *autoUpdateServiceClient) UpdateAutoUpdateAgentRollout(ctx context.Context, in *UpdateAutoUpdateAgentRolloutRequest, opts ...grpc.CallOption) (*AutoUpdateAgentRollout, error) { + out := new(AutoUpdateAgentRollout) + err := c.cc.Invoke(ctx, AutoUpdateService_UpdateAutoUpdateAgentRollout_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *autoUpdateServiceClient) UpsertAutoUpdateAgentRollout(ctx context.Context, in *UpsertAutoUpdateAgentRolloutRequest, opts ...grpc.CallOption) (*AutoUpdateAgentRollout, error) { + out := new(AutoUpdateAgentRollout) + err := c.cc.Invoke(ctx, AutoUpdateService_UpsertAutoUpdateAgentRollout_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *autoUpdateServiceClient) DeleteAutoUpdateAgentRollout(ctx context.Context, in *DeleteAutoUpdateAgentRolloutRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, AutoUpdateService_DeleteAutoUpdateAgentRollout_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // AutoUpdateServiceServer is the server API for AutoUpdateService service. // All implementations must embed UnimplementedAutoUpdateServiceServer // for forward compatibility @@ -194,6 +254,16 @@ type AutoUpdateServiceServer interface { UpsertAutoUpdateVersion(context.Context, *UpsertAutoUpdateVersionRequest) (*AutoUpdateVersion, error) // DeleteAutoUpdateVersion hard deletes the specified AutoUpdateVersionRequest. DeleteAutoUpdateVersion(context.Context, *DeleteAutoUpdateVersionRequest) (*emptypb.Empty, error) + // GetAutoUpdateVersion gets the current autoupdate version singleton. + GetAutoUpdateAgentRollout(context.Context, *GetAutoUpdateAgentRolloutRequest) (*AutoUpdateAgentRollout, error) + // CreateAutoUpdateAgentRollout creates a new AutoUpdateAgentRollout. + CreateAutoUpdateAgentRollout(context.Context, *CreateAutoUpdateAgentRolloutRequest) (*AutoUpdateAgentRollout, error) + // UpdateAutoUpdateAgentRollout updates AutoUpdateAgentRollout singleton. + UpdateAutoUpdateAgentRollout(context.Context, *UpdateAutoUpdateAgentRolloutRequest) (*AutoUpdateAgentRollout, error) + // UpsertAutoUpdateAgentRollout creates a new AutoUpdateAgentRollout or replaces an existing AutoUpdateAgentRollout. + UpsertAutoUpdateAgentRollout(context.Context, *UpsertAutoUpdateAgentRolloutRequest) (*AutoUpdateAgentRollout, error) + // DeleteAutoUpdateAgentRollout hard deletes the specified AutoUpdateAgentRolloutRequest. + DeleteAutoUpdateAgentRollout(context.Context, *DeleteAutoUpdateAgentRolloutRequest) (*emptypb.Empty, error) mustEmbedUnimplementedAutoUpdateServiceServer() } @@ -231,6 +301,21 @@ func (UnimplementedAutoUpdateServiceServer) UpsertAutoUpdateVersion(context.Cont func (UnimplementedAutoUpdateServiceServer) DeleteAutoUpdateVersion(context.Context, *DeleteAutoUpdateVersionRequest) (*emptypb.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method DeleteAutoUpdateVersion not implemented") } +func (UnimplementedAutoUpdateServiceServer) GetAutoUpdateAgentRollout(context.Context, *GetAutoUpdateAgentRolloutRequest) (*AutoUpdateAgentRollout, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetAutoUpdateAgentRollout not implemented") +} +func (UnimplementedAutoUpdateServiceServer) CreateAutoUpdateAgentRollout(context.Context, *CreateAutoUpdateAgentRolloutRequest) (*AutoUpdateAgentRollout, error) { + return nil, status.Errorf(codes.Unimplemented, "method CreateAutoUpdateAgentRollout not implemented") +} +func (UnimplementedAutoUpdateServiceServer) UpdateAutoUpdateAgentRollout(context.Context, *UpdateAutoUpdateAgentRolloutRequest) (*AutoUpdateAgentRollout, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateAutoUpdateAgentRollout not implemented") +} +func (UnimplementedAutoUpdateServiceServer) UpsertAutoUpdateAgentRollout(context.Context, *UpsertAutoUpdateAgentRolloutRequest) (*AutoUpdateAgentRollout, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpsertAutoUpdateAgentRollout not implemented") +} +func (UnimplementedAutoUpdateServiceServer) DeleteAutoUpdateAgentRollout(context.Context, *DeleteAutoUpdateAgentRolloutRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteAutoUpdateAgentRollout not implemented") +} func (UnimplementedAutoUpdateServiceServer) mustEmbedUnimplementedAutoUpdateServiceServer() {} // UnsafeAutoUpdateServiceServer may be embedded to opt out of forward compatibility for this service. @@ -424,6 +509,96 @@ func _AutoUpdateService_DeleteAutoUpdateVersion_Handler(srv interface{}, ctx con return interceptor(ctx, in, info, handler) } +func _AutoUpdateService_GetAutoUpdateAgentRollout_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetAutoUpdateAgentRolloutRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AutoUpdateServiceServer).GetAutoUpdateAgentRollout(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AutoUpdateService_GetAutoUpdateAgentRollout_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AutoUpdateServiceServer).GetAutoUpdateAgentRollout(ctx, req.(*GetAutoUpdateAgentRolloutRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AutoUpdateService_CreateAutoUpdateAgentRollout_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateAutoUpdateAgentRolloutRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AutoUpdateServiceServer).CreateAutoUpdateAgentRollout(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AutoUpdateService_CreateAutoUpdateAgentRollout_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AutoUpdateServiceServer).CreateAutoUpdateAgentRollout(ctx, req.(*CreateAutoUpdateAgentRolloutRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AutoUpdateService_UpdateAutoUpdateAgentRollout_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UpdateAutoUpdateAgentRolloutRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AutoUpdateServiceServer).UpdateAutoUpdateAgentRollout(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AutoUpdateService_UpdateAutoUpdateAgentRollout_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AutoUpdateServiceServer).UpdateAutoUpdateAgentRollout(ctx, req.(*UpdateAutoUpdateAgentRolloutRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AutoUpdateService_UpsertAutoUpdateAgentRollout_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UpsertAutoUpdateAgentRolloutRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AutoUpdateServiceServer).UpsertAutoUpdateAgentRollout(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AutoUpdateService_UpsertAutoUpdateAgentRollout_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AutoUpdateServiceServer).UpsertAutoUpdateAgentRollout(ctx, req.(*UpsertAutoUpdateAgentRolloutRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AutoUpdateService_DeleteAutoUpdateAgentRollout_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteAutoUpdateAgentRolloutRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AutoUpdateServiceServer).DeleteAutoUpdateAgentRollout(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AutoUpdateService_DeleteAutoUpdateAgentRollout_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AutoUpdateServiceServer).DeleteAutoUpdateAgentRollout(ctx, req.(*DeleteAutoUpdateAgentRolloutRequest)) + } + return interceptor(ctx, in, info, handler) +} + // AutoUpdateService_ServiceDesc is the grpc.ServiceDesc for AutoUpdateService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -471,6 +646,26 @@ var AutoUpdateService_ServiceDesc = grpc.ServiceDesc{ MethodName: "DeleteAutoUpdateVersion", Handler: _AutoUpdateService_DeleteAutoUpdateVersion_Handler, }, + { + MethodName: "GetAutoUpdateAgentRollout", + Handler: _AutoUpdateService_GetAutoUpdateAgentRollout_Handler, + }, + { + MethodName: "CreateAutoUpdateAgentRollout", + Handler: _AutoUpdateService_CreateAutoUpdateAgentRollout_Handler, + }, + { + MethodName: "UpdateAutoUpdateAgentRollout", + Handler: _AutoUpdateService_UpdateAutoUpdateAgentRollout_Handler, + }, + { + MethodName: "UpsertAutoUpdateAgentRollout", + Handler: _AutoUpdateService_UpsertAutoUpdateAgentRollout_Handler, + }, + { + MethodName: "DeleteAutoUpdateAgentRollout", + Handler: _AutoUpdateService_DeleteAutoUpdateAgentRollout_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "teleport/autoupdate/v1/autoupdate_service.proto", diff --git a/api/proto/teleport/autoupdate/v1/autoupdate.proto b/api/proto/teleport/autoupdate/v1/autoupdate.proto index b4e557549b316..73f6d440f998e 100644 --- a/api/proto/teleport/autoupdate/v1/autoupdate.proto +++ b/api/proto/teleport/autoupdate/v1/autoupdate.proto @@ -16,6 +16,8 @@ syntax = "proto3"; package teleport.autoupdate.v1; +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; import "teleport/header/v1/metadata.proto"; option go_package = "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1;autoupdate"; @@ -36,6 +38,7 @@ message AutoUpdateConfigSpec { reserved 1; reserved "tools_autoupdate"; // ToolsAutoupdate is replaced by tools.mode. AutoUpdateConfigSpecTools tools = 2; + AutoUpdateConfigSpecAgents agents = 3; } // AutoUpdateConfigSpecTools encodes the parameters for client tools auto updates. @@ -44,6 +47,44 @@ message AutoUpdateConfigSpecTools { string mode = 1; } +// AutoUpdateConfigSpecAgents encodes the parameters of automatic agent updates. +message AutoUpdateConfigSpecAgents { + reserved 5; + reserved "agent_schedules"; + // mode specifies whether agent autoupdates are enabled, disabled, or paused. + string mode = 1; + // strategy to use for updating the agents. + string strategy = 2; + // maintenance_window_duration is the maintenance window duration. This can only be set if `strategy` is "time-based". + // Once the window is over, the group transitions to the done state. Existing agents won't be updated until the next + // maintenance window. + google.protobuf.Duration maintenance_window_duration = 3; + // schedules specifies schedules for updates of grouped agents. + AgentAutoUpdateSchedules schedules = 6; +} + +// AgentAutoUpdateSchedules specifies update scheduled for grouped agents. +message AgentAutoUpdateSchedules { + // regular schedules for non-critical versions. + repeated AgentAutoUpdateGroup regular = 1; +} + +// AgentAutoUpdateGroup specifies the update schedule for a group of agents. +message AgentAutoUpdateGroup { + reserved 4; + reserved "wait_days"; + + // name of the group + string name = 1; + // days when the update can run. Supported values are "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" and "*" + repeated string days = 2; + // start_hour to initiate update + int32 start_hour = 3; + // wait_hours after last group succeeds before this group can run. This can only be used when the strategy is "halt-on-failure". + // This field must be positive. + int32 wait_hours = 5; +} + // AutoUpdateVersion is a resource singleton with version required for // tools autoupdate. message AutoUpdateVersion { @@ -60,6 +101,7 @@ message AutoUpdateVersionSpec { reserved 1; reserved "tools_version"; // ToolsVersion is replaced by tools.target_version. AutoUpdateVersionSpecTools tools = 2; + AutoUpdateVersionSpecAgents agents = 3; } // AutoUpdateVersionSpecTools encodes the parameters for client tools auto updates. @@ -68,3 +110,141 @@ message AutoUpdateVersionSpecTools { // Client tools after connection to the cluster going to be updated to this version automatically. string target_version = 1; } + +// AutoUpdateVersionSpecAgents is the spec for the autoupdate version. +message AutoUpdateVersionSpecAgents { + // start_version is the version to update from. + string start_version = 1; + // target_version is the version to update to. + string target_version = 2; + // schedule to use for the rollout + string schedule = 3; + // autoupdate_mode to use for the rollout + string mode = 4; +} + +// AutoUpdateAgentRollout is the resource the Teleport Auth Service uses to track and control the rollout of a new +// agent version. This resource is written by the automatic agent update controller in the Teleport Auth Service +// and read by the Teleport Proxy Service. +message AutoUpdateAgentRollout { + string kind = 1; + string sub_kind = 2; + string version = 3; + teleport.header.v1.Metadata metadata = 4; + AutoUpdateAgentRolloutSpec spec = 5; + AutoUpdateAgentRolloutStatus status = 6; +} + +// AutoUpdateAgentRolloutSpec describes the desired agent rollout. +// This is built by merging the user-provided AutoUpdateConfigSpecAgents and the operator-provided +// AutoUpdateVersionSpecAgents. +message AutoUpdateAgentRolloutSpec { + // start_version is the version to update from. + string start_version = 1; + // target_version is the version to update to. + string target_version = 2; + // schedule to use for the rollout. Supported values are "regular" and "immediate". + // - "regular" follows the regular group schedule + // - "immediate" updates all the agents immediately + string schedule = 3; + // autoupdate_mode to use for the rollout. Supported modes are: + // - "enabled": Teleport will update existing agents. + // - "disabled": Teleport will not update existing agents. + // - "suspended": Teleport will temporarily stop updating existing agents. + string autoupdate_mode = 4; + // strategy to use for updating the agents. Supported strategies are: + // - "time-based": agents update as soon as their maintenance window starts. There is no dependency between groups. + // This strategy allows Teleport users to setup reliable follow-the-sun updates and enforce the maintenance window + // more strictly. A group finishes its update at the end of the maintenance window, regardless of the new version + // adoption rate. Agents that missed the maintenance window will not attempt to update until the next maintenance + // window. + // - "halt-on-failure": the update proceeds from the first group to the last group, ensuring that each group + // successfully updates before allowing the next group to proceed. This is the strategy that offers the best + // availability. A group finishes its update once most of its agents are running the correct version. Agents that + // missed the group update will try to catch back as soon as possible. + string strategy = 5; + // maintenance_window_duration is the maintenance window duration. This can only be set if `strategy` is "time-based". + // Once the window is over, the group transitions to the done state. Existing agents won't be updated until the next + // maintenance window. + google.protobuf.Duration maintenance_window_duration = 6; +} + +// AutoUpdateAgentRolloutStatus tracks the current agent rollout status. +// The status is reset if any spec field changes except the mode. +message AutoUpdateAgentRolloutStatus { + repeated AutoUpdateAgentRolloutStatusGroup groups = 1; + AutoUpdateAgentRolloutState state = 2; + // The start time is set when the rollout is created or reset. Usually this is caused by a version change. + // The timestamp allows the controller to detect that the rollout just changed. + // The controller will not start any group that should have been active before the start_time to avoid a double-update + // effect. + // For example, a group updates every day between 13:00 and 14:00. If the target version changes to 13:30, the group + // will not start updating to the new version directly. The controller sees that the group theoretical start time is + // before the rollout start time and the maintenance window belongs to the previous rollout. + // When the timestamp is nil, the controller will ignore the start time and check and allow groups to activate. + google.protobuf.Timestamp start_time = 3; + + // Time override is an optional timestamp making the autoupdate_agent_rollout controller use a specific time instead + // of the system clock when evaluating time-based criteria. This field is used for testing and troubleshooting + // purposes. + google.protobuf.Timestamp time_override = 4; +} + +// AutoUpdateAgentRolloutStatusGroup tracks the current agent rollout status of a specific group. +message AutoUpdateAgentRolloutStatusGroup { + reserved 8; + reserved "config_wait_days"; + + // name of the group + string name = 1; + // start_time of the rollout + google.protobuf.Timestamp start_time = 2; + // state is the current state of the rollout. + AutoUpdateAgentGroupState state = 3; + // last_update_time is the time of the previous update for this group. + google.protobuf.Timestamp last_update_time = 4; + // last_update_reason is the trigger for the last update + string last_update_reason = 5; + // config_days when the update can run. Supported values are "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" and "*" + repeated string config_days = 6; + // config_start_hour to initiate update + int32 config_start_hour = 7; + // config_wait_hours after last group succeeds before this group can run. This can only be used when the strategy is "halt-on-failure". + // This field must be positive. + int32 config_wait_hours = 9; +} + +// AutoUpdateAgentGroupState represents the agent group state. This state controls whether the agents from this group +// should install the start version, the target version, and if they should update immediately or wait. +enum AutoUpdateAgentGroupState { + // AUTO_UPDATE_AGENT_GROUP_STATE_UNSPECIFIED state + AUTO_UPDATE_AGENT_GROUP_STATE_UNSPECIFIED = 0; + // AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED represents that the group update has not been started yet. + AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED = 1; + // AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE represents that the group is actively getting updated. + // New agents should run v2, existing agents are instructed to update to v2. + AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE = 2; + // AUTO_UPDATE_AGENT_GROUP_STATE_DONE represents that the group has been updated. New agents should run v2. + AUTO_UPDATE_AGENT_GROUP_STATE_DONE = 3; + // AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK represents that the group has been rolled back. + // New agents should run v1, existing agents should update to v1. + AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK = 4; +} + +// AutoUpdateAgentRolloutState represents the rollout state. This tells if Teleport started updating agents from the +// start version to the target version, if the update is done, still in progress +// or if the rollout was manually reverted. +enum AutoUpdateAgentRolloutState { + // AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSPECIFIED state + AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSPECIFIED = 0; + // AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSTARTED represents that no group in the rollout has been started yet. + AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSTARTED = 1; + // AUTO_UPDATE_AGENT_ROLLOUT_STATE_ACTIVE represents that at least one group of the rollout has started. + // If every group is finished, the state will be AUTO_UPDATE_AGENT_ROLLOUT_STATE_DONE. + AUTO_UPDATE_AGENT_ROLLOUT_STATE_ACTIVE = 2; + // AUTO_UPDATE_AGENT_ROLLOUT_STATE_DONE represents that every group is in the DONE state, or has been in the done + // state (groups might become active again in time-based strategy). + AUTO_UPDATE_AGENT_ROLLOUT_STATE_DONE = 3; + // AUTO_UPDATE_AGENT_ROLLOUT_STATE_ROLLEDBACK represents that at least one group is in the rolledback state. + AUTO_UPDATE_AGENT_ROLLOUT_STATE_ROLLEDBACK = 4; +} diff --git a/api/proto/teleport/autoupdate/v1/autoupdate_service.proto b/api/proto/teleport/autoupdate/v1/autoupdate_service.proto index efd045306d63e..4191d09f83101 100644 --- a/api/proto/teleport/autoupdate/v1/autoupdate_service.proto +++ b/api/proto/teleport/autoupdate/v1/autoupdate_service.proto @@ -52,6 +52,21 @@ service AutoUpdateService { // DeleteAutoUpdateVersion hard deletes the specified AutoUpdateVersionRequest. rpc DeleteAutoUpdateVersion(DeleteAutoUpdateVersionRequest) returns (google.protobuf.Empty); + + // GetAutoUpdateVersion gets the current autoupdate version singleton. + rpc GetAutoUpdateAgentRollout(GetAutoUpdateAgentRolloutRequest) returns (AutoUpdateAgentRollout); + + // CreateAutoUpdateAgentRollout creates a new AutoUpdateAgentRollout. + rpc CreateAutoUpdateAgentRollout(CreateAutoUpdateAgentRolloutRequest) returns (AutoUpdateAgentRollout); + + // UpdateAutoUpdateAgentRollout updates AutoUpdateAgentRollout singleton. + rpc UpdateAutoUpdateAgentRollout(UpdateAutoUpdateAgentRolloutRequest) returns (AutoUpdateAgentRollout); + + // UpsertAutoUpdateAgentRollout creates a new AutoUpdateAgentRollout or replaces an existing AutoUpdateAgentRollout. + rpc UpsertAutoUpdateAgentRollout(UpsertAutoUpdateAgentRolloutRequest) returns (AutoUpdateAgentRollout); + + // DeleteAutoUpdateAgentRollout hard deletes the specified AutoUpdateAgentRolloutRequest. + rpc DeleteAutoUpdateAgentRollout(DeleteAutoUpdateAgentRolloutRequest) returns (google.protobuf.Empty); } // Request for GetAutoUpdateConfig. @@ -95,3 +110,24 @@ message UpsertAutoUpdateVersionRequest { // Request for DeleteAutoUpdateVersion. message DeleteAutoUpdateVersionRequest {} + +// Request for GetAutoUpdateAgentRollout. +message GetAutoUpdateAgentRolloutRequest {} + +// Request for CreateAutoUpdateAgentRollout. +message CreateAutoUpdateAgentRolloutRequest { + AutoUpdateAgentRollout rollout = 1; +} + +// Request for UpdateAutoUpdateConfig. +message UpdateAutoUpdateAgentRolloutRequest { + AutoUpdateAgentRollout rollout = 1; +} + +// Request for UpsertAutoUpdateAgentRollout. +message UpsertAutoUpdateAgentRolloutRequest { + AutoUpdateAgentRollout rollout = 1; +} + +// Request for DeleteAutoUpdateAgentRollout. +message DeleteAutoUpdateAgentRolloutRequest {} diff --git a/api/proto/teleport/legacy/client/proto/event.proto b/api/proto/teleport/legacy/client/proto/event.proto index 88bc008871e77..8cb4ab073eece 100644 --- a/api/proto/teleport/legacy/client/proto/event.proto +++ b/api/proto/teleport/legacy/client/proto/event.proto @@ -41,7 +41,11 @@ enum Operation { message Event { reserved 7; reserved 49; + reserved 63; + reserved 68; reserved "ExternalCloudAudit"; + reserved "StaticHostUser"; + reserved "AutoUpdateAgentPlan"; // Operation identifies operation Operation Type = 1; @@ -156,5 +160,7 @@ message Event { teleport.autoupdate.v1.AutoUpdateConfig AutoUpdateConfig = 64; // AutoUpdateVersion is a resource for autoupdate version. teleport.autoupdate.v1.AutoUpdateVersion AutoUpdateVersion = 65; + // AutoUpdateVersion is a resource for controlling the autoupdate agent rollout. + teleport.autoupdate.v1.AutoUpdateAgentRollout AutoUpdateAgentRollout = 71; } } diff --git a/api/proto/teleport/legacy/types/events/events.proto b/api/proto/teleport/legacy/types/events/events.proto index 43cdde01e42f1..57909691fab76 100644 --- a/api/proto/teleport/legacy/types/events/events.proto +++ b/api/proto/teleport/legacy/types/events/events.proto @@ -3806,6 +3806,8 @@ message AzureOIDCIntegrationMetadata { // OneOf is a union of one of audit events submitted to the auth service message OneOf { // Event is one of the audit events + reserved 185, 186, 187; + reserved "AutoUpdateAgentPlanCreate", "AutoUpdateAgentPlanUpdate", "AutoUpdateAgentPlanDelete"; oneof Event { events.UserLogin UserLogin = 1; events.UserCreate UserCreate = 2; diff --git a/api/types/autoupdate/config.go b/api/types/autoupdate/config.go index d61c35eccf0c2..ad79765895c0d 100644 --- a/api/types/autoupdate/config.go +++ b/api/types/autoupdate/config.go @@ -1,24 +1,24 @@ /* - * Teleport - * Copyright (C) 2024 Gravitational, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package autoupdate import ( + "time" + "github.com/gravitational/trace" "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" @@ -26,13 +26,6 @@ import ( "github.com/gravitational/teleport/api/types" ) -const ( - // ToolsUpdateModeEnabled enables client tools automatic updates. - ToolsUpdateModeEnabled = "enabled" - // ToolsUpdateModeDisabled disables client tools automatic updates. - ToolsUpdateModeDisabled = "disabled" -) - // NewAutoUpdateConfig creates a new auto update configuration resource. func NewAutoUpdateConfig(spec *autoupdate.AutoUpdateConfigSpec) (*autoupdate.AutoUpdateConfig, error) { config := &autoupdate.AutoUpdateConfig{ @@ -66,10 +59,61 @@ func ValidateAutoUpdateConfig(c *autoupdate.AutoUpdateConfig) error { return trace.BadParameter("Spec is nil") } if c.Spec.Tools != nil { - if c.Spec.Tools.Mode != ToolsUpdateModeDisabled && c.Spec.Tools.Mode != ToolsUpdateModeEnabled { - return trace.BadParameter("ToolsMode is not valid") + if err := checkToolsMode(c.Spec.Tools.Mode); err != nil { + return trace.Wrap(err, "validating spec.tools.mode") } } + if c.Spec.Agents != nil { + if err := checkAgentsMode(c.Spec.Agents.Mode); err != nil { + return trace.Wrap(err, "validating spec.agents.mode") + } + if err := checkAgentsStrategy(c.Spec.Agents.Strategy); err != nil { + return trace.Wrap(err, "validating spec.agents.strategy") + } + + windowDuration := c.Spec.Agents.MaintenanceWindowDuration.AsDuration() + if c.Spec.Agents.Strategy == AgentsStrategyHaltOnError && windowDuration != 0 { + return trace.BadParameter("spec.agents.maintenance_window_duration must be zero when the strategy is %q", c.Spec.Agents.Strategy) + } + if c.Spec.Agents.Strategy == AgentsStrategyTimeBased && windowDuration < 10*time.Minute { + return trace.BadParameter("spec.agents.maintenance_window_duration must be greater than 10 minutes when the strategy is %q", c.Spec.Agents.Strategy) + } + if err := checkAgentSchedules(c); err != nil { + return trace.Wrap(err, "validating spec.agents.schedules") + } + } + + return nil +} + +func checkAgentSchedules(c *autoupdate.AutoUpdateConfig) error { + // Validate groups + groups := c.Spec.Agents.GetSchedules().GetRegular() + seenGroups := make(map[string]int, len(groups)) + for i, group := range groups { + if group.Name == "" { + return trace.BadParameter("spec.agents.schedules.regular[%d].name should not be empty", i) + } + if _, err := types.ParseWeekdays(group.Days); err != nil { + return trace.Wrap(err, "validating spec.agents.schedules.regular[%d].days", i) + } + if group.WaitHours < 0 { + return trace.BadParameter("spec.agents.schedules.regular[%d].wait_hours cannot be negative", i) + } + if group.StartHour > 23 || group.StartHour < 0 { + return trace.BadParameter("spec.agents.schedules.regular[%d].start_hour must be between 0 and 23", i) + } + if c.Spec.Agents.Strategy == AgentsStrategyTimeBased && group.WaitHours != 0 { + return trace.BadParameter("spec.agents.schedules.regular[%d].wait_hours must be zero when strategy is %s", i, AgentsStrategyTimeBased) + } + if c.Spec.Agents.Strategy == AgentsStrategyHaltOnError && i == 0 && group.WaitHours != 0 { + return trace.BadParameter("spec.agents.schedules.regular[0].wait_hours must be zero as it's the first group") + } + if conflictingGroup, ok := seenGroups[group.Name]; ok { + return trace.BadParameter("spec.agents.schedules.regular contains groups with the same name %q at indices %d and %d", group.Name, conflictingGroup, i) + } + seenGroups[group.Name] = i + } return nil } diff --git a/api/types/autoupdate/config_test.go b/api/types/autoupdate/config_test.go index 443d6f246fa56..0981dd7e681c1 100644 --- a/api/types/autoupdate/config_test.go +++ b/api/types/autoupdate/config_test.go @@ -1,29 +1,29 @@ /* - * Teleport - * Copyright (C) 2024 Gravitational, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package autoupdate import ( "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/durationpb" "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" headerv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/header/v1" @@ -32,6 +32,7 @@ import ( // TestNewAutoUpdateConfig verifies validation for AutoUpdateConfig resource. func TestNewAutoUpdateConfig(t *testing.T) { + t.Parallel() tests := []struct { name string spec *autoupdate.AutoUpdateConfigSpec @@ -99,7 +100,121 @@ func TestNewAutoUpdateConfig(t *testing.T) { }, }, assertErr: func(t *testing.T, err error, a ...any) { - require.ErrorContains(t, err, "ToolsMode is not valid") + require.ErrorContains(t, err, "unsupported tools mode: \"invalid-mode\"") + }, + }, + { + name: "invalid agents mode", + spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: "invalid-mode", + Strategy: AgentsStrategyHaltOnError, + }, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.ErrorContains(t, err, "unsupported agents mode: \"invalid-mode\"") + }, + }, + { + name: "invalid agents strategy", + spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: "invalid-strategy", + }, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.ErrorContains(t, err, "unsupported agents strategy: \"invalid-strategy\"") + }, + }, + { + name: "invalid agents non-nil maintenance window with halt-on-error", + spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + MaintenanceWindowDuration: durationpb.New(time.Hour), + }, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.ErrorContains(t, err, "maintenance_window_duration must be zero") + }, + }, + { + name: "invalid agents nil maintenance window with time-based strategy", + spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyTimeBased, + }, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.ErrorContains(t, err, "maintenance_window_duration must be greater than 10 minutes") + }, + }, + { + name: "invalid agents short maintenance window", + spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyTimeBased, + MaintenanceWindowDuration: durationpb.New(time.Minute), + }, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.ErrorContains(t, err, "maintenance_window_duration must be greater than 10 minutes") + }, + }, + { + name: "success agents autoupdate halt-on-failure", + spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + }, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.NoError(t, err) + }, + want: &autoupdate.AutoUpdateConfig{ + Kind: types.KindAutoUpdateConfig, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateConfig, + }, + Spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + }, + }, + }, + }, + { + name: "success agents autoupdate time-based", + spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyTimeBased, + MaintenanceWindowDuration: durationpb.New(time.Hour), + }, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.NoError(t, err) + }, + want: &autoupdate.AutoUpdateConfig{ + Kind: types.KindAutoUpdateConfig, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateConfig, + }, + Spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyTimeBased, + MaintenanceWindowDuration: durationpb.New(time.Hour), + }, + }, }, }, } @@ -111,3 +226,250 @@ func TestNewAutoUpdateConfig(t *testing.T) { }) } } + +func TestValidateAutoUpdateConfig(t *testing.T) { + t.Parallel() + tests := []struct { + name string + config *autoupdate.AutoUpdateConfig + assertErr require.ErrorAssertionFunc + }{ + { + name: "valid time-based rollout with groups", + config: &autoupdate.AutoUpdateConfig{ + Kind: types.KindAutoUpdateConfig, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateConfig, + }, + Spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyTimeBased, + MaintenanceWindowDuration: durationpb.New(time.Hour), + Schedules: &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + {Name: "g1", Days: []string{"*"}, WaitHours: 0}, + {Name: "g2", Days: []string{"*"}, WaitHours: 0}, + }, + }, + }, + }, + }, + assertErr: require.NoError, + }, + { + name: "valid halt-on-error config with groups", + config: &autoupdate.AutoUpdateConfig{ + Kind: types.KindAutoUpdateConfig, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateConfig, + }, + Spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + Schedules: &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + {Name: "g1", Days: []string{"*"}, WaitHours: 0}, + {Name: "g2", Days: []string{"*"}, WaitHours: 1}, + }, + }, + }, + }, + }, + assertErr: require.NoError, + }, + { + name: "group with negative wait days", + config: &autoupdate.AutoUpdateConfig{ + Kind: types.KindAutoUpdateConfig, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateConfig, + }, + Spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + Schedules: &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + {Name: "g1", Days: []string{"*"}, WaitHours: 0}, + {Name: "g2", Days: []string{"*"}, WaitHours: -1}, + }, + }, + }, + }, + }, + assertErr: require.Error, + }, + { + name: "group with invalid week days", + config: &autoupdate.AutoUpdateConfig{ + Kind: types.KindAutoUpdateConfig, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateConfig, + }, + Spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + Schedules: &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + {Name: "g1", Days: []string{"*"}, WaitHours: 0}, + {Name: "g2", Days: []string{"frurfday"}, WaitHours: 1}, + }, + }, + }, + }, + }, + assertErr: require.Error, + }, + { + name: "group with no week days", + config: &autoupdate.AutoUpdateConfig{ + Kind: types.KindAutoUpdateConfig, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateConfig, + }, + Spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + Schedules: &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + {Name: "g1", Days: []string{"*"}, WaitHours: 0}, + {Name: "g2", WaitHours: 1}, + }, + }, + }, + }, + }, + assertErr: require.Error, + }, + { + name: "group with empty name", + config: &autoupdate.AutoUpdateConfig{ + Kind: types.KindAutoUpdateConfig, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateConfig, + }, + Spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + Schedules: &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + {Name: "g1", Days: []string{"*"}, WaitHours: 0}, + {Name: "", Days: []string{"*"}, WaitHours: 1}, + }, + }, + }, + }, + }, + assertErr: require.Error, + }, + { + name: "first group with non zero wait days", + config: &autoupdate.AutoUpdateConfig{ + Kind: types.KindAutoUpdateConfig, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateConfig, + }, + Spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + Schedules: &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + {Name: "g1", Days: []string{"*"}, WaitHours: 1}, + {Name: "g2", Days: []string{"*"}, WaitHours: 0}, + }, + }, + }, + }, + }, + assertErr: require.Error, + }, + { + name: "group with non zero wait days on a time-based config", + config: &autoupdate.AutoUpdateConfig{ + Kind: types.KindAutoUpdateConfig, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateConfig, + }, + Spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyTimeBased, + Schedules: &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + {Name: "g1", Days: []string{"*"}, WaitHours: 0}, + {Name: "g2", Days: []string{"*"}, WaitHours: 1}, + }, + }, + }, + }, + }, + assertErr: require.Error, + }, + { + name: "group with impossible start hour", + config: &autoupdate.AutoUpdateConfig{ + Kind: types.KindAutoUpdateConfig, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateConfig, + }, + Spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + Schedules: &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + {Name: "g1", Days: []string{"*"}, WaitHours: 0}, + {Name: "dark hour", Days: []string{"*"}, StartHour: 24}, + }, + }, + }, + }, + }, + assertErr: require.Error, + }, + { + name: "groups with same names", + config: &autoupdate.AutoUpdateConfig{ + Kind: types.KindAutoUpdateConfig, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateConfig, + }, + Spec: &autoupdate.AutoUpdateConfigSpec{ + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + Schedules: &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + {Name: "g1", Days: []string{"*"}, WaitHours: 0}, + {Name: "g1", Days: []string{"*"}, WaitHours: 0}, + }, + }, + }, + }, + }, + assertErr: require.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateAutoUpdateConfig(tt.config) + tt.assertErr(t, err) + }) + } +} diff --git a/api/types/autoupdate/constants.go b/api/types/autoupdate/constants.go new file mode 100644 index 0000000000000..deed5168fb21f --- /dev/null +++ b/api/types/autoupdate/constants.go @@ -0,0 +1,46 @@ +/* +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoupdate + +const ( + // ToolsUpdateModeEnabled enables client tools automatic updates. + ToolsUpdateModeEnabled = "enabled" + // ToolsUpdateModeDisabled disables client tools automatic updates. + ToolsUpdateModeDisabled = "disabled" + + // AgentsUpdateModeEnabled enabled agent automatic updates. + AgentsUpdateModeEnabled = "enabled" + // AgentsUpdateModeDisabled disables agent automatic updates. + AgentsUpdateModeDisabled = "disabled" + // AgentsUpdateModeSuspended temporarily suspends agent automatic updates. + AgentsUpdateModeSuspended = "suspended" + + // AgentsScheduleRegular is the regular agent update schedule. + AgentsScheduleRegular = "regular" + // AgentsScheduleImmediate is the immediate agent update schedule. + // Every agent must update immediately if it's not already running the target version. + // This can be used to recover agents in case of major incident or actively exploited vulnerability. + AgentsScheduleImmediate = "immediate" + + // AgentsStrategyHaltOnError is the agent update strategy that updates groups sequentially + // according to their order in the schedule. The previous groups must succeed. + AgentsStrategyHaltOnError = "halt-on-error" + // AgentsStrategyTimeBased is the agent update strategy that updates groups solely based on their + // maintenance window. There is no dependency between groups. Agents won't be instructed to update + // if the window is over. + AgentsStrategyTimeBased = "time-based" +) diff --git a/api/types/autoupdate/rollout.go b/api/types/autoupdate/rollout.go new file mode 100644 index 0000000000000..111c9a65e0095 --- /dev/null +++ b/api/types/autoupdate/rollout.go @@ -0,0 +1,103 @@ +/* +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoupdate + +import ( + "github.com/gravitational/trace" + + "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + headerv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/header/v1" + "github.com/gravitational/teleport/api/types" +) + +// NewAutoUpdateAgentRollout creates a new auto update version resource. +func NewAutoUpdateAgentRollout(spec *autoupdate.AutoUpdateAgentRolloutSpec) (*autoupdate.AutoUpdateAgentRollout, error) { + rollout := &autoupdate.AutoUpdateAgentRollout{ + Kind: types.KindAutoUpdateAgentRollout, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateAgentRollout, + }, + Spec: spec, + } + if err := ValidateAutoUpdateAgentRollout(rollout); err != nil { + return nil, trace.Wrap(err) + } + + return rollout, nil +} + +// ValidateAutoUpdateAgentRollout checks that required parameters are set +// for the specified AutoUpdateAgentRollout. +func ValidateAutoUpdateAgentRollout(v *autoupdate.AutoUpdateAgentRollout) error { + if v == nil { + return trace.BadParameter("AutoUpdateAgentRollout is nil") + } + if v.Metadata == nil { + return trace.BadParameter("Metadata is nil") + } + if v.Metadata.Name != types.MetaNameAutoUpdateAgentRollout { + return trace.BadParameter("Name is not valid") + } + if v.Spec == nil { + return trace.BadParameter("Spec is nil") + } + if err := checkVersion(v.Spec.StartVersion); err != nil { + return trace.Wrap(err, "validating spec.start_version") + } + if err := checkVersion(v.Spec.TargetVersion); err != nil { + return trace.Wrap(err, "validating spec.target_version") + } + if err := checkAgentsMode(v.Spec.AutoupdateMode); err != nil { + return trace.Wrap(err, "validating spec.autoupdate_mode") + } + if err := checkScheduleName(v.Spec.Schedule); err != nil { + return trace.Wrap(err, "validating spec.schedule") + } + if err := checkAgentsStrategy(v.Spec.Strategy); err != nil { + return trace.Wrap(err, "validating spec.strategy") + } + + groups := v.GetStatus().GetGroups() + seenGroups := make(map[string]int, len(groups)) + for i, group := range groups { + if group.Name == "" { + return trace.BadParameter("status.groups[%d].name is empty", i) + } + if _, err := types.ParseWeekdays(group.ConfigDays); err != nil { + return trace.BadParameter("status.groups[%d].config_days is invalid", i) + } + if group.ConfigStartHour > 23 || group.ConfigStartHour < 0 { + return trace.BadParameter("spec.agents.schedules.regular[%d].start_hour must be less than or equal to 23", i) + } + if group.ConfigWaitHours < 0 { + return trace.BadParameter("status.schedules.groups[%d].config_wait_hours cannot be negative", i) + } + if v.Spec.Strategy == AgentsStrategyTimeBased && group.ConfigWaitHours != 0 { + return trace.BadParameter("status.schedules.groups[%d].config_wait_hours must be zero when strategy is %s", i, AgentsStrategyTimeBased) + } + if v.Spec.Strategy == AgentsStrategyHaltOnError && i == 0 && group.ConfigWaitHours != 0 { + return trace.BadParameter("status.schedules.groups[0].config_wait_hours must be zero as it's the first group") + } + if conflictingGroup, ok := seenGroups[group.Name]; ok { + return trace.BadParameter("spec.agents.schedules.regular contains groups with the same name %q at indices %d and %d", group.Name, conflictingGroup, i) + } + seenGroups[group.Name] = i + } + + return nil +} diff --git a/api/types/autoupdate/rollout_test.go b/api/types/autoupdate/rollout_test.go new file mode 100644 index 0000000000000..d95ba9ef890fd --- /dev/null +++ b/api/types/autoupdate/rollout_test.go @@ -0,0 +1,359 @@ +/* +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoupdate + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/testing/protocmp" + + "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + headerv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/header/v1" + "github.com/gravitational/teleport/api/types" +) + +// TestNewAutoUpdateConfig verifies validation for AutoUpdateConfig resource. +func TestNewAutoUpdateAgentRollout(t *testing.T) { + t.Parallel() + tests := []struct { + name string + spec *autoupdate.AutoUpdateAgentRolloutSpec + want *autoupdate.AutoUpdateAgentRollout + assertErr func(*testing.T, error, ...any) + }{ + { + name: "success valid rollout", + spec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "2.3.4-dev", + Schedule: AgentsScheduleImmediate, + AutoupdateMode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.NoError(t, err) + }, + want: &autoupdate.AutoUpdateAgentRollout{ + Kind: types.KindAutoUpdateAgentRollout, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateAgentRollout, + }, + Spec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "2.3.4-dev", + Schedule: AgentsScheduleImmediate, + AutoupdateMode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + }, + }, + }, + { + name: "missing spec", + spec: nil, + assertErr: func(t *testing.T, err error, a ...any) { + require.ErrorContains(t, err, "Spec is nil") + }, + }, + { + name: "missing start version", + spec: &autoupdate.AutoUpdateAgentRolloutSpec{ + TargetVersion: "2.3.4-dev", + Schedule: AgentsScheduleImmediate, + AutoupdateMode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.ErrorContains(t, err, "start_version\n\tversion is unset") + }, + }, + { + name: "invalid target version", + spec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "2-3-4", + Schedule: AgentsScheduleImmediate, + AutoupdateMode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.ErrorContains(t, err, "target_version\n\tversion \"2-3-4\" is not a valid semantic version") + }, + }, + { + name: "invalid autoupdate mode", + spec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "2.3.4-dev", + Schedule: AgentsScheduleImmediate, + AutoupdateMode: "invalid-mode", + Strategy: AgentsStrategyHaltOnError, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.ErrorContains(t, err, "unsupported agents mode: \"invalid-mode\"") + }, + }, + { + name: "invalid schedule name", + spec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "2.3.4-dev", + Schedule: "invalid-schedule", + AutoupdateMode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.ErrorContains(t, err, "unsupported schedule type: \"invalid-schedule\"") + }, + }, + { + name: "invalid strategy", + spec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "2.3.4-dev", + Schedule: AgentsScheduleImmediate, + AutoupdateMode: AgentsUpdateModeEnabled, + Strategy: "invalid-strategy", + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.ErrorContains(t, err, "unsupported agents strategy: \"invalid-strategy\"") + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewAutoUpdateAgentRollout(tt.spec) + tt.assertErr(t, err) + require.Empty(t, cmp.Diff(got, tt.want, protocmp.Transform())) + }) + } +} + +var ( + timeBasedRolloutSpec = autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "2.3.4-dev", + Schedule: AgentsScheduleRegular, + AutoupdateMode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyTimeBased, + } + haltOnErrorRolloutSpec = autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "2.3.4-dev", + Schedule: AgentsScheduleRegular, + AutoupdateMode: AgentsUpdateModeEnabled, + Strategy: AgentsStrategyHaltOnError, + } +) + +func TestValidateAutoUpdateAgentRollout(t *testing.T) { + t.Parallel() + tests := []struct { + name string + rollout *autoupdate.AutoUpdateAgentRollout + assertErr require.ErrorAssertionFunc + }{ + { + name: "valid time-based rollout with groups", + rollout: &autoupdate.AutoUpdateAgentRollout{ + Kind: types.KindAutoUpdateAgentRollout, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateAgentRollout, + }, + Spec: &timeBasedRolloutSpec, + Status: &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + {Name: "g1", ConfigDays: []string{"*"}}, + {Name: "g2", ConfigDays: []string{"*"}}, + }, + }, + }, + assertErr: require.NoError, + }, + { + name: "valid halt-on-error rollout with groups", + rollout: &autoupdate.AutoUpdateAgentRollout{ + Kind: types.KindAutoUpdateAgentRollout, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateAgentRollout, + }, + Spec: &haltOnErrorRolloutSpec, + Status: &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + {Name: "g1", ConfigDays: []string{"*"}}, + {Name: "g2", ConfigDays: []string{"*"}, ConfigWaitHours: 1}, + }, + }, + }, + assertErr: require.NoError, + }, + { + name: "group with negative wait days", + rollout: &autoupdate.AutoUpdateAgentRollout{ + Kind: types.KindAutoUpdateAgentRollout, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateAgentRollout, + }, + Spec: &haltOnErrorRolloutSpec, + Status: &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + {Name: "g1", ConfigDays: []string{"*"}}, + {Name: "g2", ConfigDays: []string{"*"}, ConfigWaitHours: -1}, + }, + }, + }, + assertErr: require.Error, + }, + { + name: "group with invalid week days", + rollout: &autoupdate.AutoUpdateAgentRollout{ + Kind: types.KindAutoUpdateAgentRollout, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateAgentRollout, + }, + Spec: &haltOnErrorRolloutSpec, + Status: &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + {Name: "g1", ConfigDays: []string{"*"}}, + {Name: "g2", ConfigDays: []string{"frurfday"}, ConfigWaitHours: 1}, + }, + }, + }, + assertErr: require.Error, + }, + { + name: "group with no week days", + rollout: &autoupdate.AutoUpdateAgentRollout{ + Kind: types.KindAutoUpdateAgentRollout, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateAgentRollout, + }, + Spec: &haltOnErrorRolloutSpec, + Status: &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + {Name: "g1", ConfigDays: []string{"*"}}, + {Name: "g2", ConfigDays: []string{}, ConfigWaitHours: 1}, + }, + }, + }, + assertErr: require.Error, + }, + { + name: "group with empty name", + rollout: &autoupdate.AutoUpdateAgentRollout{ + Kind: types.KindAutoUpdateAgentRollout, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateAgentRollout, + }, + Spec: &haltOnErrorRolloutSpec, + Status: &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + {Name: "g1", ConfigDays: []string{"*"}}, + {Name: "", ConfigDays: []string{"*"}, ConfigWaitHours: 1}, + }, + }, + }, + assertErr: require.Error, + }, + { + name: "first group with non zero wait days", + rollout: &autoupdate.AutoUpdateAgentRollout{ + Kind: types.KindAutoUpdateAgentRollout, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateAgentRollout, + }, + Spec: &haltOnErrorRolloutSpec, + Status: &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + {Name: "g1", ConfigDays: []string{"*"}, ConfigWaitHours: 1}, + {Name: "g2", ConfigDays: []string{"*"}}, + }, + }, + }, + assertErr: require.Error, + }, + { + name: "group with non zero wait days on a time-based rollout", + rollout: &autoupdate.AutoUpdateAgentRollout{ + Kind: types.KindAutoUpdateAgentRollout, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateAgentRollout, + }, + Spec: &timeBasedRolloutSpec, + Status: &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + {Name: "g1", ConfigDays: []string{"*"}}, + {Name: "g2", ConfigDays: []string{"*"}, ConfigWaitHours: 1}, + }, + }, + }, + assertErr: require.Error, + }, + { + name: "group with impossible start hour", + rollout: &autoupdate.AutoUpdateAgentRollout{ + Kind: types.KindAutoUpdateAgentRollout, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateAgentRollout, + }, + Spec: &haltOnErrorRolloutSpec, + Status: &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + {Name: "g1", ConfigDays: []string{"*"}}, + {Name: "dark hour", ConfigDays: []string{"*"}, ConfigStartHour: 24}, + }, + }, + }, + assertErr: require.Error, + }, + { + name: "group with same name", + rollout: &autoupdate.AutoUpdateAgentRollout{ + Kind: types.KindAutoUpdateAgentRollout, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateAgentRollout, + }, + Spec: &haltOnErrorRolloutSpec, + Status: &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + {Name: "g1", ConfigDays: []string{"*"}}, + {Name: "g1", ConfigDays: []string{"*"}}, + }, + }, + }, + assertErr: require.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateAutoUpdateAgentRollout(tt.rollout) + tt.assertErr(t, err) + }) + } +} diff --git a/api/types/autoupdate/utils.go b/api/types/autoupdate/utils.go new file mode 100644 index 0000000000000..7fdcf3d612903 --- /dev/null +++ b/api/types/autoupdate/utils.go @@ -0,0 +1,68 @@ +/* +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package autoupdate + +import ( + "github.com/coreos/go-semver/semver" + "github.com/gravitational/trace" +) + +func checkVersion(version string) error { + if version == "" { + return trace.BadParameter("version is unset") + } + if _, err := semver.NewVersion(version); err != nil { + return trace.BadParameter("version %q is not a valid semantic version", version) + } + return nil +} + +func checkAgentsMode(mode string) error { + switch mode { + case AgentsUpdateModeEnabled, AgentsUpdateModeDisabled, AgentsUpdateModeSuspended: + return nil + default: + return trace.BadParameter("unsupported agents mode: %q", mode) + } +} + +func checkToolsMode(mode string) error { + switch mode { + case ToolsUpdateModeEnabled, ToolsUpdateModeDisabled: + return nil + default: + return trace.BadParameter("unsupported tools mode: %q", mode) + } +} + +func checkScheduleName(schedule string) error { + switch schedule { + case AgentsScheduleImmediate, AgentsScheduleRegular: + return nil + default: + return trace.BadParameter("unsupported schedule type: %q", schedule) + } +} + +func checkAgentsStrategy(strategy string) error { + switch strategy { + case AgentsStrategyHaltOnError, AgentsStrategyTimeBased: + return nil + default: + return trace.BadParameter("unsupported agents strategy: %q", strategy) + } +} diff --git a/api/types/autoupdate/version.go b/api/types/autoupdate/version.go index ad2d12f265949..4bfe14c53fc1f 100644 --- a/api/types/autoupdate/version.go +++ b/api/types/autoupdate/version.go @@ -1,25 +1,22 @@ /* - * Teleport - * Copyright (C) 2024 Gravitational, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package autoupdate import ( - "github.com/coreos/go-semver/semver" "github.com/gravitational/trace" "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" @@ -61,11 +58,22 @@ func ValidateAutoUpdateVersion(v *autoupdate.AutoUpdateVersion) error { } if v.Spec.Tools != nil { - if v.Spec.Tools.TargetVersion == "" { - return trace.BadParameter("TargetVersion is unset") + if err := checkVersion(v.Spec.Tools.TargetVersion); err != nil { + return trace.Wrap(err, "validating spec.tools.target_version") + } + } + if v.Spec.Agents != nil { + if err := checkVersion(v.Spec.Agents.StartVersion); err != nil { + return trace.Wrap(err, "validating spec.agents.start_version") + } + if err := checkVersion(v.Spec.Agents.TargetVersion); err != nil { + return trace.Wrap(err, "validating spec.agents.target_version") + } + if err := checkAgentsMode(v.Spec.Agents.Mode); err != nil { + return trace.Wrap(err, "validating spec.agents.mode") } - if _, err := semver.NewVersion(v.Spec.Tools.TargetVersion); err != nil { - return trace.BadParameter("TargetVersion is not a valid semantic version") + if err := checkScheduleName(v.Spec.Agents.Schedule); err != nil { + return trace.Wrap(err, "validating spec.agents.schedule") } } diff --git a/api/types/autoupdate/version_test.go b/api/types/autoupdate/version_test.go index 70790a204b219..793d7d6a2a145 100644 --- a/api/types/autoupdate/version_test.go +++ b/api/types/autoupdate/version_test.go @@ -1,20 +1,18 @@ /* - * Teleport - * Copyright (C) 2024 Gravitational, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package autoupdate @@ -69,7 +67,7 @@ func TestNewAutoUpdateVersion(t *testing.T) { }, }, assertErr: func(t *testing.T, err error, a ...any) { - require.ErrorContains(t, err, "TargetVersion is unset") + require.ErrorContains(t, err, "target_version\n\tversion is unset") }, }, { @@ -80,7 +78,7 @@ func TestNewAutoUpdateVersion(t *testing.T) { }, }, assertErr: func(t *testing.T, err error, a ...any) { - require.ErrorContains(t, err, "TargetVersion is not a valid semantic version") + require.ErrorContains(t, err, "target_version\n\tversion \"17-0-0\" is not a valid semantic version") }, }, { @@ -90,6 +88,91 @@ func TestNewAutoUpdateVersion(t *testing.T) { require.ErrorContains(t, err, "Spec is nil") }, }, + { + name: "success agents autoupdate version", + spec: &autoupdate.AutoUpdateVersionSpec{ + Agents: &autoupdate.AutoUpdateVersionSpecAgents{ + StartVersion: "1.2.3-dev.1", + TargetVersion: "1.2.3-dev.2", + Schedule: AgentsScheduleImmediate, + Mode: AgentsUpdateModeEnabled, + }, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.NoError(t, err) + }, + want: &autoupdate.AutoUpdateVersion{ + Kind: types.KindAutoUpdateVersion, + Version: types.V1, + Metadata: &headerv1.Metadata{ + Name: types.MetaNameAutoUpdateVersion, + }, + Spec: &autoupdate.AutoUpdateVersionSpec{ + Agents: &autoupdate.AutoUpdateVersionSpecAgents{ + StartVersion: "1.2.3-dev.1", + TargetVersion: "1.2.3-dev.2", + Schedule: AgentsScheduleImmediate, + Mode: AgentsUpdateModeEnabled, + }, + }, + }, + }, + { + name: "invalid empty agents start version", + spec: &autoupdate.AutoUpdateVersionSpec{ + Agents: &autoupdate.AutoUpdateVersionSpecAgents{ + StartVersion: "", + TargetVersion: "1.2.3", + Mode: AgentsUpdateModeEnabled, + Schedule: AgentsScheduleImmediate, + }, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.ErrorContains(t, err, "start_version\n\tversion is unset") + }, + }, + { + name: "invalid empty agents target version", + spec: &autoupdate.AutoUpdateVersionSpec{ + Agents: &autoupdate.AutoUpdateVersionSpecAgents{ + StartVersion: "1.2.3-dev", + TargetVersion: "", + Mode: AgentsUpdateModeEnabled, + Schedule: AgentsScheduleImmediate, + }, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.ErrorContains(t, err, "target_version\n\tversion is unset") + }, + }, + { + name: "invalid semantic agents start version", + spec: &autoupdate.AutoUpdateVersionSpec{ + Agents: &autoupdate.AutoUpdateVersionSpecAgents{ + StartVersion: "17-0-0", + TargetVersion: "1.2.3", + Mode: AgentsUpdateModeEnabled, + Schedule: AgentsScheduleImmediate, + }, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.ErrorContains(t, err, "start_version\n\tversion \"17-0-0\" is not a valid semantic version") + }, + }, + { + name: "invalid semantic agents target version", + spec: &autoupdate.AutoUpdateVersionSpec{ + Agents: &autoupdate.AutoUpdateVersionSpecAgents{ + StartVersion: "1.2.3", + TargetVersion: "17-0-0", + Mode: AgentsUpdateModeEnabled, + Schedule: AgentsScheduleImmediate, + }, + }, + assertErr: func(t *testing.T, err error, a ...any) { + require.ErrorContains(t, err, "target_version\n\tversion \"17-0-0\" is not a valid semantic version") + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/api/types/constants.go b/api/types/constants.go index 66ea9bbb614a2..d4d6cdcc66954 100644 --- a/api/types/constants.go +++ b/api/types/constants.go @@ -310,12 +310,18 @@ const ( // KindAutoUpdateVersion is the resource with autoupdate versions. KindAutoUpdateVersion = "autoupdate_version" + // KindAutoUpdateAgentRollout is the resource that controls and tracks agent rollouts. + KindAutoUpdateAgentRollout = "autoupdate_agent_rollout" + // MetaNameAutoUpdateConfig is the name of a configuration resource for autoupdate config. MetaNameAutoUpdateConfig = "autoupdate-config" // MetaNameAutoUpdateVersion is the name of a resource for autoupdate version. MetaNameAutoUpdateVersion = "autoupdate-version" + // MetaNameAutoUpdateAgentRollout is the name of the autoupdate agent rollout resource. + MetaNameAutoUpdateAgentRollout = "autoupdate-agent-rollout" + // KindClusterAuditConfig is the resource that holds cluster audit configuration. KindClusterAuditConfig = "cluster_audit_config" diff --git a/api/types/events/events.pb.go b/api/types/events/events.pb.go index 4eb292b80fe1f..d164e57556fee 100644 --- a/api/types/events/events.pb.go +++ b/api/types/events/events.pb.go @@ -6550,8 +6550,6 @@ var xxx_messageInfo_AzureOIDCIntegrationMetadata proto.InternalMessageInfo // OneOf is a union of one of audit events submitted to the auth service type OneOf struct { - // Event is one of the audit events - // // Types that are valid to be assigned to Event: // *OneOf_UserLogin // *OneOf_UserCreate @@ -12397,879 +12395,882 @@ func init() { } var fileDescriptor_007ba1c3d6266d56 = []byte{ - // 13941 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0xbd, 0x7b, 0x70, 0x24, 0xc7, - 0x79, 0x18, 0x8e, 0x7d, 0xe0, 0xd5, 0x78, 0x2d, 0xfa, 0x5e, 0x43, 0xf0, 0xee, 0x96, 0x1c, 0x4a, - 0xc7, 0x3b, 0xea, 0x88, 0x13, 0xef, 0x4e, 0xa4, 0x48, 0x91, 0x22, 0x17, 0x58, 0xe0, 0xb0, 0x3c, - 0x3c, 0x96, 0xb3, 0xb8, 0x3b, 0x52, 0x0f, 0xae, 0x07, 0x3b, 0x7d, 0xc0, 0x10, 0xbb, 0x33, 0xab, - 0x99, 0xd9, 0xc3, 0x81, 0xbf, 0x5f, 0x12, 0xcb, 0xf1, 0x33, 0x91, 0x54, 0x2a, 0xa7, 0x52, 0x76, - 0x2a, 0xa9, 0x8a, 0x1f, 0xe5, 0xc4, 0x71, 0xd9, 0x96, 0xed, 0xa4, 0x6c, 0xcb, 0x8e, 0x2b, 0x76, - 0xe4, 0x54, 0xe8, 0x28, 0x49, 0xd9, 0x4e, 0xca, 0x95, 0x4a, 0x1c, 0xc8, 0x51, 0xca, 0xf9, 0x03, - 0x95, 0x54, 0x39, 0x15, 0x55, 0xac, 0x38, 0x4e, 0x2a, 0xd5, 0x5f, 0xf7, 0xcc, 0x74, 0xcf, 0x63, - 0xf1, 0xa4, 0x41, 0xe8, 0xf0, 0xcf, 0x1d, 0xf6, 0xfb, 0xbe, 0xfe, 0xba, 0xe7, 0xeb, 0xaf, 0xdf, - 0xdf, 0x03, 0x5d, 0xf1, 0x48, 0x93, 0xb4, 0x6d, 0xc7, 0xbb, 0xd6, 0x24, 0xab, 0x7a, 0x63, 0xf3, - 0x9a, 0xb7, 0xd9, 0x26, 0xee, 0x35, 0xf2, 0x80, 0x58, 0x9e, 0xff, 0xdf, 0x64, 0xdb, 0xb1, 0x3d, - 0x1b, 0xf7, 0xb1, 0x5f, 0x13, 0xa7, 0x57, 0xed, 0x55, 0x1b, 0x40, 0xd7, 0xe8, 0x5f, 0x0c, 0x3b, - 0x71, 0x7e, 0xd5, 0xb6, 0x57, 0x9b, 0xe4, 0x1a, 0xfc, 0x5a, 0xe9, 0xdc, 0xbf, 0xe6, 0x7a, 0x4e, - 0xa7, 0xe1, 0x71, 0x6c, 0x31, 0x8a, 0xf5, 0xcc, 0x16, 0x71, 0x3d, 0xbd, 0xd5, 0xe6, 0x04, 0x17, - 0xa3, 0x04, 0x1b, 0x8e, 0xde, 0x6e, 0x13, 0x87, 0x57, 0x3e, 0xf1, 0x64, 0x72, 0x3b, 0xe1, 0x5f, - 0x4e, 0xf2, 0x6c, 0x32, 0x89, 0xcf, 0x28, 0xc2, 0x51, 0xfd, 0xe1, 0x2c, 0x1a, 0x58, 0x20, 0x9e, - 0x6e, 0xe8, 0x9e, 0x8e, 0xcf, 0xa3, 0xde, 0x8a, 0x65, 0x90, 0x87, 0x4a, 0xe6, 0x89, 0xcc, 0xe5, - 0xdc, 0x54, 0xdf, 0xf6, 0x56, 0x31, 0x4b, 0x4c, 0x8d, 0x01, 0xf1, 0x05, 0x94, 0x5f, 0xde, 0x6c, - 0x13, 0x25, 0xfb, 0x44, 0xe6, 0xf2, 0xe0, 0xd4, 0xe0, 0xf6, 0x56, 0xb1, 0x17, 0x64, 0xa1, 0x01, - 0x18, 0x3f, 0x89, 0xb2, 0x95, 0xb2, 0x92, 0x03, 0xe4, 0xf8, 0xf6, 0x56, 0x71, 0xa4, 0x63, 0x1a, - 0x57, 0xed, 0x96, 0xe9, 0x91, 0x56, 0xdb, 0xdb, 0xd4, 0xb2, 0x95, 0x32, 0xbe, 0x84, 0xf2, 0xd3, - 0xb6, 0x41, 0x94, 0x3c, 0x10, 0xe1, 0xed, 0xad, 0xe2, 0x68, 0xc3, 0x36, 0x88, 0x40, 0x05, 0x78, - 0xfc, 0x1a, 0xca, 0x2f, 0x9b, 0x2d, 0xa2, 0xf4, 0x3e, 0x91, 0xb9, 0x3c, 0x74, 0x7d, 0x62, 0x92, - 0x49, 0x65, 0xd2, 0x97, 0xca, 0xe4, 0xb2, 0x2f, 0xb6, 0xa9, 0xc2, 0x7b, 0x5b, 0xc5, 0x9e, 0xed, - 0xad, 0x62, 0x9e, 0x4a, 0xf2, 0xcb, 0xdf, 0x28, 0x66, 0x34, 0x28, 0x89, 0x5f, 0x46, 0x43, 0xd3, - 0xcd, 0x8e, 0xeb, 0x11, 0x67, 0x51, 0x6f, 0x11, 0xa5, 0x0f, 0x2a, 0x9c, 0xd8, 0xde, 0x2a, 0x9e, - 0x6d, 0x30, 0x70, 0xdd, 0xd2, 0x5b, 0x62, 0xc5, 0x22, 0xb9, 0xfa, 0xab, 0x19, 0x34, 0x56, 0x23, - 0xae, 0x6b, 0xda, 0x56, 0x20, 0x9b, 0x0f, 0xa3, 0x41, 0x0e, 0xaa, 0x94, 0x41, 0x3e, 0x83, 0x53, - 0xfd, 0xdb, 0x5b, 0xc5, 0x9c, 0x6b, 0x1a, 0x5a, 0x88, 0xc1, 0x1f, 0x45, 0xfd, 0xf7, 0x4c, 0x6f, - 0x6d, 0x61, 0xb6, 0xc4, 0xe5, 0x74, 0x76, 0x7b, 0xab, 0x88, 0x37, 0x4c, 0x6f, 0xad, 0xde, 0xba, - 0xaf, 0x0b, 0x15, 0xfa, 0x64, 0x78, 0x1e, 0x15, 0xaa, 0x8e, 0xf9, 0x40, 0xf7, 0xc8, 0x6d, 0xb2, - 0x59, 0xb5, 0x9b, 0x66, 0x63, 0x93, 0x4b, 0xf1, 0x89, 0xed, 0xad, 0xe2, 0xf9, 0x36, 0xc3, 0xd5, - 0xd7, 0xc9, 0x66, 0xbd, 0x0d, 0x58, 0x81, 0x49, 0xac, 0xa4, 0xfa, 0xb5, 0x5e, 0x34, 0x7c, 0xc7, - 0x25, 0x4e, 0xd0, 0xee, 0x4b, 0x28, 0x4f, 0x7f, 0xf3, 0x26, 0x83, 0xcc, 0x3b, 0x2e, 0x71, 0x44, - 0x99, 0x53, 0x3c, 0xbe, 0x82, 0x7a, 0xe7, 0xed, 0x55, 0xd3, 0xe2, 0xcd, 0x3e, 0xb5, 0xbd, 0x55, - 0x1c, 0x6b, 0x52, 0x80, 0x40, 0xc9, 0x28, 0xf0, 0x27, 0xd1, 0x70, 0xa5, 0x45, 0x75, 0xc8, 0xb6, - 0x74, 0xcf, 0x76, 0x78, 0x6b, 0x41, 0xba, 0xa6, 0x00, 0x17, 0x0a, 0x4a, 0xf4, 0xf8, 0x25, 0x84, - 0x4a, 0xf7, 0x6a, 0x9a, 0xdd, 0x24, 0x25, 0x6d, 0x91, 0x2b, 0x03, 0x94, 0xd6, 0x37, 0xdc, 0xba, - 0x63, 0x37, 0x49, 0x5d, 0x77, 0xc4, 0x6a, 0x05, 0x6a, 0x3c, 0x83, 0x46, 0x4b, 0x8d, 0x06, 0x71, - 0x5d, 0x8d, 0x7c, 0xae, 0x43, 0x5c, 0xcf, 0x55, 0x7a, 0x9f, 0xc8, 0x5d, 0x1e, 0x9c, 0xba, 0xb0, - 0xbd, 0x55, 0x7c, 0x4c, 0x07, 0x4c, 0xdd, 0xe1, 0x28, 0x81, 0x45, 0xa4, 0x10, 0x9e, 0x42, 0x23, - 0xa5, 0x77, 0x3b, 0x0e, 0xa9, 0x18, 0xc4, 0xf2, 0x4c, 0x6f, 0x93, 0x6b, 0xc8, 0xf9, 0xed, 0xad, - 0xa2, 0xa2, 0x53, 0x44, 0xdd, 0xe4, 0x18, 0x81, 0x89, 0x5c, 0x04, 0x2f, 0xa1, 0xf1, 0x5b, 0xd3, - 0xd5, 0x1a, 0x71, 0x1e, 0x98, 0x0d, 0x52, 0x6a, 0x34, 0xec, 0x8e, 0xe5, 0x29, 0xfd, 0xc0, 0xe7, - 0xc9, 0xed, 0xad, 0xe2, 0x85, 0xd5, 0x46, 0xbb, 0xee, 0x32, 0x6c, 0x5d, 0x67, 0x68, 0x81, 0x59, - 0xbc, 0x2c, 0xfe, 0x14, 0x1a, 0x59, 0x76, 0xa8, 0x16, 0x1a, 0x65, 0x42, 0xe1, 0xca, 0x00, 0xe8, - 0xff, 0xd9, 0x49, 0x3e, 0x01, 0x31, 0xa8, 0xdf, 0xb3, 0xac, 0xb1, 0x1e, 0x2b, 0x50, 0x37, 0x00, - 0x27, 0x36, 0x56, 0x62, 0x85, 0x09, 0x52, 0xe8, 0xc7, 0x9b, 0x0e, 0x31, 0x62, 0xda, 0x36, 0x08, - 0x6d, 0xbe, 0xb2, 0xbd, 0x55, 0xfc, 0xb0, 0xc3, 0x69, 0xea, 0x5d, 0xd5, 0x2e, 0x95, 0x15, 0x9e, - 0x41, 0x03, 0x54, 0x9b, 0x6e, 0x9b, 0x96, 0xa1, 0xa0, 0x27, 0x32, 0x97, 0x47, 0xaf, 0x17, 0xfc, - 0xd6, 0xfb, 0xf0, 0xa9, 0x73, 0xdb, 0x5b, 0xc5, 0x53, 0x54, 0x07, 0xeb, 0xeb, 0xa6, 0x25, 0x4e, - 0x11, 0x41, 0x51, 0xf5, 0x4f, 0xf3, 0x68, 0x94, 0x0a, 0x47, 0xd0, 0xe3, 0x12, 0x1d, 0x92, 0x14, - 0x42, 0x47, 0xa8, 0xdb, 0xd6, 0x1b, 0x84, 0xab, 0x34, 0xb0, 0xb3, 0x7c, 0xa0, 0xc0, 0x2e, 0x4a, - 0x8f, 0xaf, 0xa0, 0x01, 0x06, 0xaa, 0x94, 0xb9, 0x96, 0x8f, 0x6c, 0x6f, 0x15, 0x07, 0x5d, 0x80, - 0xd5, 0x4d, 0x43, 0x0b, 0xd0, 0x54, 0xcd, 0xd8, 0xdf, 0x73, 0xb6, 0xeb, 0x51, 0xe6, 0x5c, 0xc9, - 0x41, 0xcd, 0x78, 0x81, 0x35, 0x8e, 0x12, 0xd5, 0x4c, 0x2e, 0x84, 0x5f, 0x44, 0x88, 0x41, 0x4a, - 0x86, 0xe1, 0x70, 0x4d, 0x7f, 0x6c, 0x7b, 0xab, 0x78, 0x86, 0xb3, 0xd0, 0x0d, 0x43, 0x1c, 0x26, - 0x02, 0x31, 0x6e, 0xa1, 0x61, 0xf6, 0x6b, 0x5e, 0x5f, 0x21, 0x4d, 0xa6, 0xe6, 0x43, 0xd7, 0x2f, - 0xfb, 0xd2, 0x94, 0xa5, 0x33, 0x29, 0x92, 0xce, 0x58, 0x9e, 0xb3, 0x39, 0x55, 0xe4, 0x33, 0xe3, - 0x39, 0x5e, 0x55, 0x13, 0x70, 0xe2, 0x98, 0x14, 0xcb, 0xd0, 0x09, 0x73, 0xd6, 0x76, 0x36, 0x74, - 0xc7, 0x20, 0xc6, 0xd4, 0xa6, 0x38, 0x61, 0xde, 0xf7, 0xc1, 0xf5, 0x15, 0x51, 0x07, 0x44, 0x72, - 0x3c, 0x8d, 0x46, 0x18, 0xb7, 0x5a, 0x67, 0x05, 0xfa, 0xbe, 0x3f, 0x26, 0x2d, 0xb7, 0xb3, 0x12, - 0xed, 0x6f, 0xb9, 0x0c, 0x1d, 0x93, 0x0c, 0x70, 0x97, 0x38, 0x74, 0x36, 0x05, 0xf5, 0xe7, 0x63, - 0x92, 0x33, 0x79, 0xc0, 0x30, 0x71, 0x1e, 0xbc, 0xc8, 0xc4, 0xab, 0x68, 0x3c, 0x26, 0x0a, 0x5c, - 0x40, 0xb9, 0x75, 0xb2, 0xc9, 0xd4, 0x45, 0xa3, 0x7f, 0xe2, 0xd3, 0xa8, 0xf7, 0x81, 0xde, 0xec, - 0xf0, 0xb5, 0x4c, 0x63, 0x3f, 0x5e, 0xca, 0x7e, 0x3c, 0x43, 0xa7, 0x7e, 0x3c, 0x6d, 0x5b, 0x16, - 0x69, 0x78, 0xe2, 0xec, 0xff, 0x3c, 0x1a, 0x9c, 0xb7, 0x1b, 0x7a, 0x13, 0xfa, 0x91, 0xe9, 0x9d, - 0xb2, 0xbd, 0x55, 0x3c, 0x4d, 0x3b, 0x70, 0xb2, 0x49, 0x31, 0x42, 0x9b, 0x42, 0x52, 0xaa, 0x00, - 0x1a, 0x69, 0xd9, 0x1e, 0x81, 0x82, 0xd9, 0x50, 0x01, 0xa0, 0xa0, 0x03, 0x28, 0x51, 0x01, 0x42, - 0x62, 0x7c, 0x0d, 0x0d, 0x54, 0xe9, 0x82, 0xd7, 0xb0, 0x9b, 0x5c, 0xf9, 0x60, 0x4e, 0x86, 0x45, - 0x50, 0x1c, 0x34, 0x3e, 0x91, 0x3a, 0x87, 0x46, 0xa7, 0x9b, 0x26, 0xb1, 0x3c, 0xb1, 0xd5, 0x74, - 0x48, 0x95, 0x56, 0x89, 0xe5, 0x89, 0xad, 0x86, 0xc1, 0xa7, 0x53, 0xa8, 0xd8, 0xea, 0x80, 0x54, - 0xfd, 0xd7, 0x39, 0xf4, 0xd8, 0xed, 0xce, 0x0a, 0x71, 0x2c, 0xe2, 0x11, 0x97, 0xaf, 0x8c, 0x01, - 0xd7, 0x45, 0x34, 0x1e, 0x43, 0x72, 0xee, 0xb0, 0x62, 0xad, 0x07, 0xc8, 0x3a, 0x5f, 0x6c, 0xc5, - 0x69, 0x2f, 0x56, 0x14, 0xcf, 0xa1, 0xb1, 0x10, 0x48, 0x1b, 0xe1, 0x2a, 0x59, 0x98, 0xd3, 0x2f, - 0x6e, 0x6f, 0x15, 0x27, 0x04, 0x6e, 0xb4, 0xd9, 0xa2, 0x06, 0x47, 0x8b, 0xe1, 0xdb, 0xa8, 0x10, - 0x82, 0x6e, 0x39, 0x76, 0xa7, 0xed, 0x2a, 0x39, 0x60, 0x55, 0xdc, 0xde, 0x2a, 0x3e, 0x2e, 0xb0, - 0x5a, 0x05, 0xa4, 0xb8, 0x92, 0x46, 0x0b, 0xe2, 0xef, 0xcd, 0x88, 0xdc, 0xf8, 0x28, 0xcc, 0xc3, - 0x28, 0x7c, 0xc1, 0x1f, 0x85, 0xa9, 0x42, 0x9a, 0x8c, 0x96, 0xe4, 0x83, 0x32, 0xd2, 0x8c, 0xd8, - 0xa0, 0x8c, 0xd5, 0x38, 0x31, 0x8d, 0xce, 0x24, 0xf2, 0xda, 0x93, 0x56, 0xff, 0x71, 0x4e, 0xe4, - 0x52, 0xb5, 0x8d, 0xa0, 0x33, 0x97, 0xc4, 0xce, 0xac, 0xda, 0x06, 0x6c, 0x97, 0x32, 0xe1, 0x22, - 0x26, 0x34, 0xb6, 0x6d, 0x1b, 0xd1, 0x5d, 0x53, 0xbc, 0x2c, 0x7e, 0x1b, 0x9d, 0x8d, 0x01, 0xd9, - 0x74, 0xcd, 0xb4, 0xff, 0xd2, 0xf6, 0x56, 0x51, 0x4d, 0xe0, 0x1a, 0x9d, 0xbd, 0x53, 0xb8, 0x60, - 0x1d, 0x9d, 0x13, 0xa4, 0x6e, 0x5b, 0x9e, 0x6e, 0x5a, 0x7c, 0x97, 0xc7, 0x46, 0xc9, 0xd3, 0xdb, - 0x5b, 0xc5, 0xa7, 0x44, 0x1d, 0xf4, 0x69, 0xa2, 0x8d, 0x4f, 0xe3, 0x83, 0x0d, 0xa4, 0x24, 0xa0, - 0x2a, 0x2d, 0x7d, 0xd5, 0xdf, 0xba, 0x5e, 0xde, 0xde, 0x2a, 0x7e, 0x28, 0xb1, 0x0e, 0x93, 0x52, - 0x89, 0x4b, 0x65, 0x1a, 0x27, 0xac, 0x21, 0x1c, 0xe2, 0x16, 0x6d, 0x83, 0xc0, 0x37, 0xf4, 0x02, - 0x7f, 0x75, 0x7b, 0xab, 0x78, 0x51, 0xe0, 0x6f, 0xd9, 0x06, 0x89, 0x36, 0x3f, 0xa1, 0xb4, 0xfa, - 0xab, 0x39, 0x74, 0xb1, 0x56, 0x5a, 0x98, 0xaf, 0x18, 0xfe, 0xde, 0xa2, 0xea, 0xd8, 0x0f, 0x4c, - 0x43, 0x18, 0xbd, 0x2b, 0xe8, 0x5c, 0x04, 0x35, 0x03, 0xdb, 0x99, 0x60, 0x57, 0x0b, 0xdf, 0xe6, - 0xef, 0x5b, 0xda, 0x9c, 0xa6, 0xce, 0xf6, 0x3c, 0x75, 0x69, 0x4b, 0x9f, 0xc6, 0x88, 0xf6, 0x51, - 0x04, 0x55, 0x5b, 0xb3, 0x1d, 0xaf, 0xd1, 0xf1, 0xb8, 0x12, 0x40, 0x1f, 0xc5, 0xea, 0x70, 0x39, - 0x51, 0x97, 0x2a, 0x7c, 0x3e, 0xf8, 0x87, 0x32, 0xa8, 0x50, 0xf2, 0x3c, 0xc7, 0x5c, 0xe9, 0x78, - 0x64, 0x41, 0x6f, 0xb7, 0x4d, 0x6b, 0x15, 0xc6, 0xfa, 0xd0, 0xf5, 0x97, 0x83, 0x35, 0xb2, 0xab, - 0x24, 0x26, 0xa3, 0xc5, 0x85, 0x21, 0xaa, 0xfb, 0xa8, 0x7a, 0x8b, 0xe1, 0xc4, 0x21, 0x1a, 0x2d, - 0x47, 0x87, 0x68, 0x22, 0xaf, 0x3d, 0x0d, 0xd1, 0x1f, 0xce, 0xa1, 0xf3, 0x4b, 0xeb, 0x9e, 0xae, - 0x11, 0xd7, 0xee, 0x38, 0x0d, 0xe2, 0xde, 0x69, 0x1b, 0xba, 0x47, 0xc2, 0x91, 0x5a, 0x44, 0xbd, - 0x25, 0xc3, 0x20, 0x06, 0xb0, 0xeb, 0x65, 0xe7, 0x2f, 0x9d, 0x02, 0x34, 0x06, 0xc7, 0x1f, 0x46, - 0xfd, 0xbc, 0x0c, 0x70, 0xef, 0x9d, 0x1a, 0xda, 0xde, 0x2a, 0xf6, 0x77, 0x18, 0x48, 0xf3, 0x71, - 0x94, 0xac, 0x4c, 0x9a, 0x84, 0x92, 0xe5, 0x42, 0x32, 0x83, 0x81, 0x34, 0x1f, 0x87, 0xdf, 0x40, - 0xa3, 0xc0, 0x36, 0x68, 0x0f, 0x9f, 0xfb, 0x4e, 0xfb, 0xd2, 0x15, 0x1b, 0xcb, 0x96, 0x26, 0x68, - 0x4d, 0xdd, 0xf1, 0x0b, 0x68, 0x11, 0x06, 0xf8, 0x1e, 0x2a, 0xf0, 0x46, 0x84, 0x4c, 0x7b, 0xbb, - 0x30, 0x3d, 0xb3, 0xbd, 0x55, 0x1c, 0xe7, 0xed, 0x17, 0xd8, 0xc6, 0x98, 0x50, 0xc6, 0xbc, 0xd9, - 0x21, 0xe3, 0xbe, 0x9d, 0x18, 0xf3, 0x2f, 0x16, 0x19, 0x47, 0x99, 0xa8, 0x6f, 0xa1, 0x61, 0xb1, - 0x20, 0x3e, 0x0b, 0x67, 0x5c, 0x36, 0x4e, 0xe0, 0x74, 0x6c, 0x1a, 0x70, 0xb0, 0x7d, 0x0e, 0x0d, - 0x95, 0x89, 0xdb, 0x70, 0xcc, 0x36, 0xdd, 0x35, 0x70, 0x25, 0x1f, 0xdb, 0xde, 0x2a, 0x0e, 0x19, - 0x21, 0x58, 0x13, 0x69, 0xd4, 0xff, 0x99, 0x41, 0x67, 0x29, 0xef, 0x92, 0xeb, 0x9a, 0xab, 0x56, - 0x4b, 0x5c, 0xb6, 0xaf, 0xa2, 0xbe, 0x1a, 0xd4, 0xc7, 0x6b, 0x3a, 0xbd, 0xbd, 0x55, 0x2c, 0xb0, - 0x16, 0x08, 0x7a, 0xc8, 0x69, 0x82, 0x03, 0x5e, 0x76, 0x87, 0x03, 0x1e, 0xdd, 0xd2, 0x7a, 0xba, - 0xe3, 0x99, 0xd6, 0x6a, 0xcd, 0xd3, 0xbd, 0x8e, 0x2b, 0x6d, 0x69, 0x39, 0xa6, 0xee, 0x02, 0x4a, - 0xda, 0xd2, 0x4a, 0x85, 0xf0, 0xab, 0x68, 0x78, 0xc6, 0x32, 0x42, 0x26, 0x6c, 0x42, 0x7c, 0x9c, - 0xee, 0x34, 0x09, 0xc0, 0xe3, 0x2c, 0xa4, 0x02, 0xea, 0xcf, 0x65, 0x90, 0xc2, 0x4e, 0x63, 0xf3, - 0xa6, 0xeb, 0x2d, 0x90, 0xd6, 0x8a, 0x30, 0x3b, 0xcd, 0xfa, 0xc7, 0x3b, 0x8a, 0x13, 0xd6, 0x22, - 0xd8, 0x0a, 0xf0, 0xe3, 0x5d, 0xd3, 0x74, 0xbd, 0xe8, 0x64, 0x18, 0x29, 0x85, 0x2b, 0xa8, 0x9f, - 0x71, 0x66, 0x7b, 0x89, 0xa1, 0xeb, 0x8a, 0xaf, 0x08, 0xd1, 0xaa, 0x99, 0x32, 0xb4, 0x18, 0xb1, - 0x78, 0x3e, 0xe7, 0xe5, 0xd5, 0x5f, 0xc8, 0xa2, 0x42, 0xb4, 0x10, 0xbe, 0x87, 0x06, 0x5e, 0xb7, - 0x4d, 0x8b, 0x18, 0x4b, 0x16, 0xb4, 0xb0, 0xfb, 0x2d, 0x85, 0xbf, 0x17, 0x3f, 0xf5, 0x0e, 0x94, - 0xa9, 0x8b, 0x3b, 0x58, 0xb8, 0xb4, 0x08, 0x98, 0xe1, 0x4f, 0xa1, 0x41, 0xba, 0x07, 0x7c, 0x00, - 0x9c, 0xb3, 0x3b, 0x72, 0x7e, 0x82, 0x73, 0x3e, 0xed, 0xb0, 0x42, 0x71, 0xd6, 0x21, 0x3b, 0xaa, - 0x57, 0x1a, 0xd1, 0x5d, 0xdb, 0xe2, 0x3d, 0x0f, 0x7a, 0xe5, 0x00, 0x44, 0xd4, 0x2b, 0x46, 0x43, - 0xb7, 0xae, 0xec, 0x63, 0xa1, 0x1b, 0x84, 0xb3, 0x0b, 0x93, 0x55, 0xb4, 0x07, 0x04, 0x62, 0xf5, - 0xfb, 0xb3, 0xe8, 0xd9, 0x50, 0x64, 0x1a, 0x79, 0x60, 0x92, 0x0d, 0x2e, 0xce, 0x35, 0xb3, 0xcd, - 0x0f, 0x8f, 0x54, 0xe5, 0xdd, 0xe9, 0x35, 0xdd, 0x5a, 0x25, 0x06, 0xbe, 0x82, 0x7a, 0xe9, 0x09, - 0xdf, 0x55, 0x32, 0xb0, 0x5d, 0x83, 0xe9, 0xc4, 0xa1, 0x00, 0xf1, 0xf6, 0x01, 0x28, 0xb0, 0x8d, - 0xfa, 0x96, 0x1d, 0xdd, 0xf4, 0xfc, 0x9e, 0x2d, 0xc5, 0x7b, 0x76, 0x17, 0x35, 0x4e, 0x32, 0x1e, - 0x6c, 0xce, 0x07, 0x41, 0x78, 0x00, 0x10, 0x05, 0xc1, 0x48, 0x26, 0x5e, 0x44, 0x43, 0x02, 0xf1, - 0x9e, 0x26, 0xf5, 0xaf, 0xe6, 0x45, 0x5d, 0xf7, 0x9b, 0xc5, 0x75, 0xfd, 0x1a, 0xd5, 0x51, 0xd7, - 0xa5, 0xbb, 0x0a, 0xa6, 0xe4, 0x5c, 0x13, 0x01, 0x24, 0x6b, 0x22, 0x80, 0xf0, 0x0d, 0x34, 0xc0, - 0x58, 0x04, 0xe7, 0x57, 0x38, 0xfb, 0x3a, 0x00, 0x93, 0x97, 0xe6, 0x80, 0x10, 0xff, 0x4c, 0x06, - 0x5d, 0xe8, 0x2a, 0x09, 0x50, 0x86, 0xa1, 0xeb, 0x1f, 0xdb, 0x97, 0x18, 0xa7, 0x9e, 0xdd, 0xde, - 0x2a, 0x5e, 0x69, 0x05, 0x24, 0x75, 0x47, 0xa0, 0xa9, 0x37, 0x18, 0x91, 0xd0, 0xae, 0xee, 0x4d, - 0xa1, 0x9b, 0x47, 0x56, 0xe9, 0x2c, 0xdc, 0xe1, 0x58, 0x8d, 0x4d, 0xbf, 0x91, 0xf9, 0x70, 0xf3, - 0xc8, 0xbf, 0xf7, 0xbe, 0x4f, 0x92, 0x50, 0x4d, 0x0a, 0x17, 0xdc, 0x40, 0xe7, 0x18, 0xa6, 0xac, - 0x6f, 0x2e, 0xdd, 0x5f, 0xb0, 0x2d, 0x6f, 0xcd, 0xaf, 0xa0, 0x57, 0xbc, 0x04, 0x81, 0x0a, 0x0c, - 0x7d, 0xb3, 0x6e, 0xdf, 0xaf, 0xb7, 0x28, 0x55, 0x42, 0x1d, 0x69, 0x9c, 0xe8, 0x44, 0xcb, 0xc7, - 0x9c, 0x3f, 0x05, 0xf5, 0x85, 0x57, 0x54, 0xfe, 0x38, 0x8d, 0x4f, 0x38, 0x91, 0x42, 0x6a, 0x05, - 0x0d, 0xcf, 0xdb, 0x8d, 0xf5, 0x40, 0x5d, 0x5e, 0x44, 0x7d, 0xcb, 0xba, 0xb3, 0x4a, 0x3c, 0x90, - 0xc5, 0xd0, 0xf5, 0xf1, 0x49, 0x76, 0xed, 0x4b, 0x89, 0x18, 0x62, 0x6a, 0x94, 0xcf, 0x06, 0x7d, - 0x1e, 0xfc, 0xd6, 0x78, 0x01, 0xf5, 0x1b, 0xbd, 0x68, 0x98, 0x5f, 0x51, 0xc2, 0x6c, 0x8e, 0x5f, - 0x0a, 0x2f, 0x7d, 0xf9, 0xf4, 0x15, 0x5c, 0xd3, 0x04, 0xd7, 0x4b, 0xc3, 0x94, 0xd9, 0xef, 0x6e, - 0x15, 0x33, 0xdb, 0x5b, 0xc5, 0x1e, 0x6d, 0x40, 0x38, 0x54, 0x86, 0xeb, 0x8d, 0xb0, 0xc0, 0x8a, - 0x97, 0x8e, 0x91, 0xb2, 0x6c, 0xfd, 0x79, 0x15, 0xf5, 0xf3, 0x36, 0x70, 0x8d, 0x3b, 0x17, 0xde, - 0x65, 0x48, 0x57, 0xad, 0x91, 0xd2, 0x7e, 0x29, 0xfc, 0x32, 0xea, 0x63, 0x67, 0x7b, 0x2e, 0x80, - 0xb3, 0xc9, 0x77, 0x21, 0x91, 0xe2, 0xbc, 0x0c, 0x9e, 0x43, 0x28, 0x3c, 0xd7, 0x07, 0x37, 0xcb, - 0x9c, 0x43, 0xfc, 0xc4, 0x1f, 0xe1, 0x22, 0x94, 0xc5, 0xcf, 0xa3, 0xe1, 0x65, 0xe2, 0xb4, 0x4c, - 0x4b, 0x6f, 0xd6, 0xcc, 0x77, 0xfd, 0xcb, 0x65, 0x58, 0x78, 0x5d, 0xf3, 0x5d, 0x71, 0xe4, 0x4a, - 0x74, 0xf8, 0xb3, 0x49, 0xe7, 0xe6, 0x7e, 0x68, 0xc8, 0x93, 0x3b, 0x1e, 0x28, 0x23, 0xed, 0x49, - 0x38, 0x46, 0xbf, 0x81, 0x46, 0xa4, 0x23, 0x13, 0xbf, 0x3d, 0xbc, 0x10, 0x67, 0x2d, 0x9c, 0xff, - 0x22, 0x6c, 0x65, 0x0e, 0x54, 0x93, 0x2b, 0x96, 0xe9, 0x99, 0x7a, 0x73, 0xda, 0x6e, 0xb5, 0x74, - 0xcb, 0x50, 0x06, 0x43, 0x4d, 0x36, 0x19, 0xa6, 0xde, 0x60, 0x28, 0x51, 0x93, 0xe5, 0x42, 0xf4, - 0x58, 0xce, 0xfb, 0x50, 0x23, 0x0d, 0xdb, 0xa1, 0x7b, 0x01, 0xb8, 0x1c, 0xe4, 0xc7, 0x72, 0x97, - 0xe1, 0xea, 0x8e, 0x8f, 0x14, 0x37, 0xdb, 0xd1, 0x82, 0xaf, 0xe7, 0x07, 0x86, 0x0a, 0xc3, 0xd1, - 0xfb, 0x5c, 0xf5, 0x1f, 0xe4, 0xd0, 0x10, 0x27, 0xa5, 0x4b, 0xe9, 0x89, 0x82, 0x1f, 0x44, 0xc1, - 0x13, 0x15, 0xb5, 0xef, 0xb0, 0x14, 0x55, 0xfd, 0x42, 0x36, 0x98, 0x8d, 0xaa, 0x8e, 0x69, 0x1d, - 0x6c, 0x36, 0xba, 0x84, 0xd0, 0xf4, 0x5a, 0xc7, 0x5a, 0x67, 0xef, 0x56, 0xd9, 0xf0, 0xdd, 0xaa, - 0x61, 0x6a, 0x02, 0x06, 0x5f, 0x40, 0xf9, 0x32, 0xe5, 0x4f, 0x7b, 0x66, 0x78, 0x6a, 0xf0, 0x3d, - 0xc6, 0x29, 0xf3, 0xac, 0x06, 0x60, 0x7a, 0xb8, 0x9a, 0xda, 0xf4, 0x08, 0xdb, 0xce, 0xe6, 0xd8, - 0xe1, 0x6a, 0x85, 0x02, 0x34, 0x06, 0xc7, 0x37, 0xd1, 0x78, 0x99, 0x34, 0xf5, 0xcd, 0x05, 0xb3, - 0xd9, 0x34, 0x5d, 0xd2, 0xb0, 0x2d, 0xc3, 0x05, 0x21, 0xf3, 0xea, 0x5a, 0xae, 0x16, 0x27, 0xc0, - 0x2a, 0xea, 0x5b, 0xba, 0x7f, 0xdf, 0x25, 0x1e, 0x88, 0x2f, 0x37, 0x85, 0xe8, 0xe4, 0x6c, 0x03, - 0x44, 0xe3, 0x18, 0xf5, 0x2b, 0x19, 0x7a, 0x7a, 0x71, 0xd7, 0x3d, 0xbb, 0x1d, 0x68, 0xf9, 0x81, - 0x44, 0x72, 0x25, 0xdc, 0x57, 0x64, 0xe1, 0x6b, 0xc7, 0xf8, 0xd7, 0xf6, 0xf3, 0xbd, 0x45, 0xb8, - 0xa3, 0x48, 0xfc, 0xaa, 0xdc, 0x0e, 0x5f, 0xa5, 0xfe, 0x49, 0x16, 0x9d, 0xe3, 0x2d, 0x9e, 0x6e, - 0x9a, 0xed, 0x15, 0x5b, 0x77, 0x0c, 0x8d, 0x34, 0x88, 0xf9, 0x80, 0x1c, 0xcf, 0x81, 0x27, 0x0f, - 0x9d, 0xfc, 0x01, 0x86, 0xce, 0x75, 0x38, 0x08, 0x52, 0xc9, 0xc0, 0x85, 0x2f, 0xdb, 0x54, 0x14, - 0xb6, 0xb7, 0x8a, 0xc3, 0x06, 0x03, 0xc3, 0x95, 0xbf, 0x26, 0x12, 0x51, 0x25, 0x99, 0x27, 0xd6, - 0xaa, 0xb7, 0x06, 0x4a, 0xd2, 0xcb, 0x94, 0xa4, 0x09, 0x10, 0x8d, 0x63, 0xd4, 0xff, 0x96, 0x45, - 0xa7, 0xa3, 0x22, 0xaf, 0x11, 0xcb, 0x38, 0x91, 0xf7, 0xfb, 0x23, 0xef, 0x6f, 0xe5, 0xd0, 0xe3, - 0xbc, 0x4c, 0x6d, 0x4d, 0x77, 0x88, 0x51, 0x36, 0x1d, 0xd2, 0xf0, 0x6c, 0x67, 0xf3, 0x18, 0x6f, - 0xa0, 0x0e, 0x4f, 0xec, 0x37, 0x51, 0x1f, 0x3f, 0xfe, 0xb3, 0x75, 0x66, 0x34, 0x68, 0x09, 0x40, - 0x63, 0x2b, 0x14, 0xbb, 0x3a, 0x88, 0x74, 0x56, 0xdf, 0x6e, 0x3a, 0xeb, 0xe3, 0x68, 0x24, 0x10, - 0x3d, 0x1c, 0x44, 0xfb, 0xc3, 0xdd, 0x96, 0xe1, 0x23, 0xe0, 0x2c, 0xaa, 0xc9, 0x84, 0x50, 0x9b, - 0x0f, 0xa8, 0x94, 0x61, 0x37, 0x34, 0xc2, 0x6b, 0x0b, 0xca, 0x99, 0x86, 0x26, 0x12, 0xa9, 0x5b, - 0x79, 0x34, 0x91, 0xdc, 0xed, 0x1a, 0xd1, 0x8d, 0x93, 0x5e, 0xff, 0x8e, 0xec, 0x75, 0xfc, 0x24, - 0xca, 0x57, 0x75, 0x6f, 0x8d, 0xbf, 0x83, 0xc3, 0x9b, 0xf0, 0x7d, 0xb3, 0x49, 0xea, 0x6d, 0xdd, - 0x5b, 0xd3, 0x00, 0x25, 0xcc, 0x19, 0x08, 0x38, 0x26, 0xcc, 0x19, 0xc2, 0x62, 0x3f, 0xf4, 0x44, - 0xe6, 0x72, 0x3e, 0x71, 0xb1, 0xff, 0x46, 0x3e, 0x6d, 0x5e, 0xb9, 0xe7, 0x98, 0x1e, 0x39, 0xd1, - 0xb0, 0x13, 0x0d, 0x3b, 0xa0, 0x86, 0xfd, 0x7e, 0x16, 0x8d, 0x04, 0x87, 0xa6, 0x77, 0x48, 0xe3, - 0x68, 0xd6, 0xaa, 0xf0, 0x28, 0x93, 0x3b, 0xf0, 0x51, 0xe6, 0x20, 0x0a, 0xa5, 0x06, 0x57, 0x9e, - 0x6c, 0x6b, 0x00, 0x12, 0x63, 0x57, 0x9e, 0xc1, 0x45, 0xe7, 0x93, 0xa8, 0x7f, 0x41, 0x7f, 0x68, - 0xb6, 0x3a, 0x2d, 0xbe, 0x4b, 0x07, 0xbb, 0xae, 0x96, 0xfe, 0x50, 0xf3, 0xe1, 0xea, 0xbf, 0xcd, - 0xa0, 0x51, 0x2e, 0x54, 0xce, 0xfc, 0x40, 0x52, 0x0d, 0xa5, 0x93, 0x3d, 0xb0, 0x74, 0x72, 0xfb, - 0x97, 0x8e, 0xfa, 0x77, 0x72, 0x48, 0x99, 0x35, 0x9b, 0x64, 0xd9, 0xd1, 0x2d, 0xf7, 0x3e, 0x71, - 0xf8, 0x71, 0x7a, 0x86, 0xb2, 0x3a, 0xd0, 0x07, 0x0a, 0x53, 0x4a, 0x76, 0x5f, 0x53, 0xca, 0x47, - 0xd0, 0x20, 0x6f, 0x4c, 0x60, 0x53, 0x08, 0xa3, 0xc6, 0xf1, 0x81, 0x5a, 0x88, 0xa7, 0xc4, 0xa5, - 0x76, 0xdb, 0xb1, 0x1f, 0x10, 0x87, 0xbd, 0x52, 0x71, 0x62, 0xdd, 0x07, 0x6a, 0x21, 0x5e, 0xe0, - 0x4c, 0xfc, 0xfd, 0xa2, 0xc8, 0x99, 0x38, 0x5a, 0x88, 0xc7, 0x97, 0xd1, 0xc0, 0xbc, 0xdd, 0xd0, - 0x41, 0xd0, 0x6c, 0x5a, 0x19, 0xde, 0xde, 0x2a, 0x0e, 0x34, 0x39, 0x4c, 0x0b, 0xb0, 0x94, 0xb2, - 0x6c, 0x6f, 0x58, 0x4d, 0x5b, 0x67, 0xc6, 0x2f, 0x03, 0x8c, 0xd2, 0xe0, 0x30, 0x2d, 0xc0, 0x52, - 0x4a, 0x2a, 0x73, 0x30, 0x2a, 0x1a, 0x08, 0x79, 0xde, 0xe7, 0x30, 0x2d, 0xc0, 0xaa, 0x5f, 0xc9, - 0x53, 0xed, 0x75, 0xcd, 0x77, 0x1f, 0xf9, 0x75, 0x21, 0x1c, 0x30, 0xbd, 0xfb, 0x18, 0x30, 0x8f, - 0xcc, 0x85, 0x9d, 0xfa, 0xa7, 0xfd, 0x08, 0x71, 0xe9, 0xcf, 0x9c, 0x1c, 0x0e, 0x0f, 0xa6, 0x35, - 0x65, 0x34, 0x3e, 0x63, 0xad, 0xe9, 0x56, 0x83, 0x18, 0xe1, 0xb5, 0x65, 0x1f, 0x0c, 0x6d, 0xb0, - 0xe9, 0x25, 0x1c, 0x19, 0xde, 0x5b, 0x6a, 0xf1, 0x02, 0xf8, 0x39, 0x34, 0x54, 0xb1, 0x3c, 0xe2, - 0xe8, 0x0d, 0xcf, 0x7c, 0x40, 0xf8, 0xd4, 0x00, 0x2f, 0xc3, 0x66, 0x08, 0xd6, 0x44, 0x1a, 0x7c, - 0x13, 0x0d, 0x57, 0x75, 0xc7, 0x33, 0x1b, 0x66, 0x5b, 0xb7, 0x3c, 0x57, 0x19, 0x80, 0x19, 0x0d, - 0x76, 0x18, 0x6d, 0x01, 0xae, 0x49, 0x54, 0xf8, 0xb3, 0x68, 0x10, 0x8e, 0xa6, 0x60, 0x38, 0x3d, - 0xb8, 0xe3, 0xc3, 0xe1, 0x53, 0xa1, 0x79, 0x20, 0xbb, 0x7d, 0x85, 0x17, 0xe0, 0xe8, 0xdb, 0x61, - 0xc0, 0x11, 0xbf, 0x89, 0xfa, 0x67, 0x2c, 0x03, 0x98, 0xa3, 0x1d, 0x99, 0xab, 0x9c, 0xf9, 0xd9, - 0x90, 0xb9, 0xdd, 0x8e, 0xf0, 0xf6, 0xd9, 0x25, 0x8f, 0xb2, 0xa1, 0xf7, 0x6f, 0x94, 0x0d, 0xbf, - 0x0f, 0xd7, 0xe2, 0x23, 0x87, 0x75, 0x2d, 0x3e, 0xba, 0xcf, 0x6b, 0x71, 0xf5, 0x5d, 0x34, 0x34, - 0x55, 0x9d, 0x0d, 0x46, 0xef, 0x63, 0x28, 0x57, 0xe5, 0x96, 0x0a, 0x79, 0xb6, 0x9f, 0x69, 0x9b, - 0x86, 0x46, 0x61, 0xf8, 0x0a, 0x1a, 0x98, 0x06, 0xf3, 0x37, 0xfe, 0x8a, 0x98, 0x67, 0xeb, 0x5f, - 0x03, 0x60, 0x60, 0x05, 0xeb, 0xa3, 0xf1, 0x87, 0x51, 0x7f, 0xd5, 0xb1, 0x57, 0x1d, 0xbd, 0xc5, - 0xd7, 0x60, 0x30, 0x15, 0x69, 0x33, 0x90, 0xe6, 0xe3, 0xd4, 0xbf, 0x91, 0xf1, 0xb7, 0xed, 0xb4, - 0x44, 0xad, 0x03, 0x57, 0xf3, 0x50, 0xf7, 0x00, 0x2b, 0xe1, 0x32, 0x90, 0xe6, 0xe3, 0xf0, 0x15, - 0xd4, 0x3b, 0xe3, 0x38, 0xb6, 0x23, 0x1a, 0x9b, 0x13, 0x0a, 0x10, 0x9f, 0x7b, 0x81, 0x02, 0xbf, - 0x80, 0x86, 0xd8, 0x9c, 0xc3, 0x6e, 0x34, 0x73, 0xdd, 0x5e, 0x4a, 0x45, 0x4a, 0xf5, 0x6b, 0x39, - 0x61, 0xcf, 0xc6, 0x24, 0xfe, 0x08, 0xbe, 0x0a, 0xdc, 0x40, 0xb9, 0xa9, 0xea, 0x2c, 0x9f, 0x00, - 0x4f, 0xf9, 0x45, 0x05, 0x55, 0x89, 0x94, 0xa3, 0xd4, 0xf8, 0x3c, 0xca, 0x57, 0xa9, 0xfa, 0xf4, - 0x81, 0x7a, 0x0c, 0x6c, 0x6f, 0x15, 0xf3, 0x6d, 0xaa, 0x3f, 0x00, 0x05, 0x2c, 0x3d, 0xcc, 0xb0, - 0x13, 0x13, 0xc3, 0x86, 0xe7, 0x98, 0xf3, 0x28, 0x5f, 0x72, 0x56, 0x1f, 0xf0, 0x59, 0x0b, 0xb0, - 0xba, 0xb3, 0xfa, 0x40, 0x03, 0x28, 0xbe, 0x86, 0x90, 0x46, 0xbc, 0x8e, 0x63, 0x81, 0x1f, 0xc8, - 0x20, 0xdc, 0xbf, 0xc1, 0x6c, 0xe8, 0x00, 0xb4, 0xde, 0xb0, 0x0d, 0xa2, 0x09, 0x24, 0xea, 0x4f, - 0x85, 0x0f, 0x3b, 0x65, 0xd3, 0x5d, 0x3f, 0xe9, 0xc2, 0x3d, 0x74, 0xa1, 0xce, 0xaf, 0x38, 0xe3, - 0x9d, 0x54, 0x44, 0xbd, 0xb3, 0x4d, 0x7d, 0xd5, 0x85, 0x3e, 0xe4, 0xb6, 0x64, 0xf7, 0x29, 0x40, - 0x63, 0xf0, 0x48, 0x3f, 0x0d, 0xec, 0xdc, 0x4f, 0x3f, 0xd2, 0x1b, 0x8c, 0xb6, 0x45, 0xe2, 0x6d, - 0xd8, 0xce, 0x49, 0x57, 0xed, 0xb6, 0xab, 0x2e, 0xa1, 0xfe, 0x9a, 0xd3, 0x10, 0xae, 0x2e, 0xe0, - 0x3c, 0xe0, 0x3a, 0x0d, 0x76, 0x6d, 0xe1, 0x23, 0x29, 0x5d, 0xd9, 0xf5, 0x80, 0xae, 0x3f, 0xa4, - 0x33, 0x5c, 0x8f, 0xd3, 0x71, 0x24, 0xa7, 0xab, 0xda, 0x8e, 0xc7, 0x3b, 0x2e, 0xa0, 0x6b, 0xdb, - 0x8e, 0xa7, 0xf9, 0x48, 0xfc, 0x11, 0x84, 0x96, 0xa7, 0xab, 0xbe, 0xb1, 0xfd, 0x60, 0x68, 0x0b, - 0xc8, 0xad, 0xec, 0x35, 0x01, 0x8d, 0x97, 0xd1, 0xe0, 0x52, 0x9b, 0x38, 0xec, 0x28, 0xc4, 0x3c, - 0x3b, 0x9e, 0x8e, 0x88, 0x96, 0xf7, 0xfb, 0x24, 0xff, 0x3f, 0x20, 0x67, 0xeb, 0x8b, 0xed, 0xff, - 0xd4, 0x42, 0x46, 0xf8, 0x05, 0xd4, 0x57, 0x62, 0xfb, 0xbc, 0x21, 0x60, 0x19, 0x88, 0x0c, 0x8e, - 0xa0, 0x0c, 0xc5, 0xce, 0xec, 0x3a, 0xfc, 0xad, 0x71, 0x72, 0xf5, 0x0a, 0x2a, 0x44, 0xab, 0xc1, - 0x43, 0xa8, 0x7f, 0x7a, 0x69, 0x71, 0x71, 0x66, 0x7a, 0xb9, 0xd0, 0x83, 0x07, 0x50, 0xbe, 0x36, - 0xb3, 0x58, 0x2e, 0x64, 0xd4, 0x9f, 0x15, 0x66, 0x10, 0xaa, 0x5a, 0x27, 0x4f, 0xc3, 0x07, 0x7a, - 0x6f, 0x29, 0xc0, 0x7b, 0x28, 0xdc, 0x18, 0xb4, 0x4c, 0xcf, 0x23, 0x06, 0x5f, 0x25, 0xe0, 0xbd, - 0xd0, 0x7b, 0xa8, 0xc5, 0xf0, 0xf8, 0x2a, 0x1a, 0x01, 0x18, 0x7f, 0x22, 0x64, 0xe7, 0x63, 0x5e, - 0xc0, 0x79, 0xa8, 0xc9, 0x48, 0xf5, 0xeb, 0xe1, 0xeb, 0xf0, 0x3c, 0xd1, 0x8f, 0xeb, 0x8b, 0xe2, - 0x07, 0xa4, 0xbf, 0xd4, 0x3f, 0xcf, 0x33, 0x17, 0x10, 0xe6, 0xb8, 0x77, 0x14, 0xa2, 0x0c, 0xaf, - 0x74, 0x73, 0x7b, 0xb8, 0xd2, 0xbd, 0x8a, 0xfa, 0x16, 0x88, 0xb7, 0x66, 0xfb, 0x86, 0x5f, 0x60, - 0xa1, 0xd7, 0x02, 0x88, 0x68, 0xa1, 0xc7, 0x68, 0xf0, 0x3a, 0xc2, 0xbe, 0x57, 0x5e, 0x60, 0x88, - 0xed, 0x5f, 0x21, 0x9f, 0x8b, 0x9d, 0x53, 0x6a, 0xe0, 0x92, 0x0b, 0x36, 0xf6, 0xa7, 0x03, 0x43, - 0x6f, 0xc1, 0x12, 0xeb, 0xcf, 0xb6, 0x8a, 0x7d, 0x8c, 0x46, 0x4b, 0x60, 0x8b, 0xdf, 0x40, 0x83, - 0x0b, 0xb3, 0x25, 0xee, 0xa1, 0xc7, 0xac, 0x22, 0x1e, 0x0b, 0xa4, 0xe8, 0x23, 0x02, 0x91, 0x80, - 0xbf, 0x4d, 0xeb, 0xbe, 0x1e, 0x77, 0xd0, 0x0b, 0xb9, 0x50, 0x6d, 0x61, 0x9e, 0x3b, 0xfc, 0x76, - 0x21, 0xd0, 0x16, 0xd9, 0x9f, 0x27, 0x2a, 0x2b, 0x86, 0x8d, 0x68, 0xcb, 0xc0, 0x01, 0x46, 0xf7, - 0x12, 0x1a, 0x2f, 0xb5, 0xdb, 0x4d, 0x93, 0x18, 0xa0, 0x2f, 0x5a, 0xa7, 0x49, 0x5c, 0x6e, 0xf2, - 0x03, 0xce, 0x20, 0x3a, 0x43, 0xd6, 0xc1, 0x2f, 0xb4, 0xee, 0x74, 0x64, 0xfb, 0xcc, 0x78, 0x59, - 0xf5, 0xbf, 0x66, 0x50, 0xc1, 0x37, 0x9e, 0x16, 0x3d, 0x52, 0x05, 0xcb, 0x5e, 0xb8, 0x86, 0x89, - 0xd8, 0x92, 0x02, 0x1e, 0xd7, 0x50, 0xff, 0xcc, 0xc3, 0xb6, 0xe9, 0x10, 0x77, 0x17, 0x86, 0xb0, - 0x17, 0xf8, 0x91, 0x73, 0x9c, 0xb0, 0x22, 0xb1, 0xd3, 0x26, 0x03, 0x83, 0x4b, 0x14, 0x33, 0x1f, - 0x9f, 0xf2, 0xdd, 0x6c, 0x99, 0x4b, 0x14, 0x37, 0x33, 0x97, 0x7c, 0xdc, 0x42, 0x52, 0xfc, 0x14, - 0xca, 0x2d, 0x2f, 0xcf, 0x73, 0x6d, 0x04, 0xf7, 0x66, 0xcf, 0x13, 0x7d, 0xbe, 0x28, 0x56, 0xfd, - 0xc3, 0x2c, 0x42, 0x54, 0xe9, 0xa7, 0x1d, 0xa2, 0x1f, 0xd1, 0x63, 0xce, 0x14, 0x1a, 0xf0, 0x05, - 0xce, 0x07, 0x5c, 0x60, 0xf9, 0x1c, 0xed, 0x88, 0x68, 0xdd, 0x81, 0x95, 0x7b, 0xd1, 0x37, 0xc6, - 0x65, 0x77, 0xa9, 0xb0, 0x3b, 0x04, 0x63, 0x5c, 0xdf, 0x04, 0xf7, 0x23, 0x68, 0x90, 0x6b, 0x8d, - 0x2d, 0xdd, 0xa1, 0x36, 0x7c, 0xa0, 0x16, 0xe2, 0x23, 0xea, 0xd9, 0x77, 0x80, 0xc9, 0xec, 0x4b, - 0x5c, 0xbc, 0xcc, 0x4c, 0xff, 0xd8, 0x8a, 0xf7, 0xd0, 0x2e, 0xb8, 0xd4, 0xdf, 0xcf, 0x20, 0x4c, - 0x9b, 0x55, 0xd5, 0x5d, 0x77, 0xc3, 0x76, 0x0c, 0x66, 0x81, 0x7a, 0x24, 0x82, 0x39, 0xbc, 0x47, - 0x89, 0xaf, 0x0d, 0xa0, 0x53, 0x92, 0x75, 0xdf, 0x31, 0x1f, 0x4d, 0x57, 0xe4, 0xd1, 0xd4, 0xcd, - 0xb4, 0xfd, 0x43, 0xe2, 0xab, 0x47, 0xaf, 0xe4, 0x65, 0x22, 0x3c, 0x77, 0x3c, 0x8b, 0x86, 0xf9, - 0x0f, 0xba, 0x58, 0xfa, 0xd7, 0xd9, 0x30, 0x4a, 0x5d, 0x0a, 0xd0, 0x24, 0x34, 0xfe, 0x18, 0x1a, - 0xa4, 0x03, 0x66, 0x15, 0x5c, 0xf5, 0xfb, 0x43, 0xb3, 0x71, 0xc3, 0x07, 0x8a, 0x13, 0x5e, 0x40, - 0x29, 0x38, 0x0b, 0x0c, 0xec, 0xc2, 0x59, 0xe0, 0x6d, 0x34, 0x54, 0xb2, 0x2c, 0xdb, 0x83, 0x9d, - 0xb8, 0xcb, 0xef, 0x1f, 0x53, 0x97, 0xde, 0xa7, 0xc0, 0x03, 0x36, 0xa4, 0x4f, 0x5c, 0x7b, 0x45, - 0x86, 0xf8, 0xba, 0x6f, 0xfa, 0x4e, 0x1c, 0x6e, 0x3a, 0x0a, 0x77, 0xb0, 0x0e, 0x87, 0xc5, 0x2d, - 0xdf, 0xa1, 0xf3, 0x46, 0xaa, 0x8e, 0xdd, 0xb6, 0x5d, 0x62, 0x30, 0x41, 0x0d, 0x85, 0xfe, 0xc4, - 0x6d, 0x8e, 0x00, 0x67, 0x15, 0xc9, 0x6d, 0x5e, 0x2a, 0x82, 0xef, 0xa3, 0xd3, 0xfe, 0x6b, 0x50, - 0xe0, 0x16, 0x54, 0x29, 0xbb, 0xca, 0x30, 0xb8, 0x1e, 0xe0, 0xa8, 0x32, 0x54, 0xca, 0x53, 0x17, - 0xfd, 0xbb, 0x4f, 0xdf, 0xaf, 0xa8, 0x6e, 0x1a, 0x62, 0x57, 0x27, 0xf2, 0xc3, 0xdf, 0x85, 0x86, - 0x16, 0xf4, 0x87, 0xe5, 0x0e, 0x3f, 0x60, 0x8d, 0xec, 0xfe, 0x8a, 0xb5, 0xa5, 0x3f, 0xac, 0x1b, - 0xbc, 0x5c, 0x64, 0xd1, 0x13, 0x59, 0xe2, 0x3a, 0x3a, 0x5b, 0x75, 0xec, 0x96, 0xed, 0x11, 0x23, - 0xe2, 0x61, 0x33, 0x16, 0xba, 0xe4, 0xb5, 0x39, 0x45, 0xbd, 0x8b, 0xab, 0x4d, 0x0a, 0x1b, 0xdc, - 0x42, 0x63, 0x25, 0xd7, 0xed, 0xb4, 0x48, 0x78, 0x0d, 0x5d, 0xd8, 0xf1, 0x33, 0x9e, 0xe6, 0xa6, - 0x89, 0x8f, 0xeb, 0x50, 0x94, 0xdd, 0x42, 0xd7, 0x3d, 0x53, 0xac, 0x11, 0xbe, 0x25, 0xca, 0xfb, - 0xf5, 0xfc, 0xc0, 0x68, 0x61, 0x4c, 0x3b, 0x17, 0x6f, 0xcc, 0xb2, 0xe9, 0x35, 0x89, 0xfa, 0x1b, - 0x19, 0x84, 0x42, 0x01, 0xe3, 0x67, 0xe5, 0x78, 0x20, 0x99, 0xf0, 0x36, 0x93, 0xbb, 0x28, 0x4b, - 0x01, 0x40, 0xf0, 0x79, 0x94, 0x07, 0x37, 0xf6, 0x6c, 0x78, 0x7b, 0xb2, 0x6e, 0x5a, 0x86, 0x06, - 0x50, 0x8a, 0x15, 0xfc, 0x4d, 0x01, 0x0b, 0x2f, 0x77, 0x6c, 0xdb, 0x52, 0x46, 0x63, 0xb5, 0xce, - 0x8a, 0x5f, 0xb7, 0xe0, 0x3c, 0x03, 0xde, 0xf4, 0x6e, 0x67, 0x25, 0xf0, 0x38, 0x93, 0x62, 0x15, - 0xc8, 0x45, 0xd4, 0xaf, 0x64, 0x22, 0xb3, 0xe0, 0x11, 0x2e, 0x7a, 0x1f, 0x8a, 0x3f, 0xc6, 0xc6, - 0xa7, 0x25, 0xf5, 0xef, 0x66, 0xd1, 0x50, 0xd5, 0x76, 0x3c, 0x1e, 0x17, 0xe0, 0x78, 0xaf, 0x42, - 0xc2, 0xb1, 0x25, 0xbf, 0x87, 0x63, 0xcb, 0x79, 0x94, 0x17, 0xec, 0x10, 0xd9, 0xe5, 0xa7, 0x61, - 0x38, 0x1a, 0x40, 0xd5, 0xef, 0xce, 0x22, 0xf4, 0xe6, 0x73, 0xcf, 0x3d, 0xc2, 0x02, 0x52, 0xff, - 0x76, 0x06, 0x8d, 0xf1, 0xdb, 0x78, 0x21, 0xb2, 0x4e, 0xbf, 0xff, 0x8e, 0x22, 0x8e, 0x4b, 0x06, - 0xd2, 0x7c, 0x1c, 0x5d, 0x02, 0x66, 0x1e, 0x9a, 0x1e, 0x5c, 0x48, 0x0a, 0xa1, 0x75, 0x08, 0x87, - 0x89, 0x4b, 0x80, 0x4f, 0x87, 0x9f, 0xf5, 0xdf, 0x19, 0x72, 0xe1, 0xba, 0x47, 0x0b, 0xcc, 0x24, - 0xbe, 0x35, 0xa8, 0xbf, 0x9c, 0x47, 0xf9, 0x99, 0x87, 0xa4, 0x71, 0xcc, 0xbb, 0x46, 0xb8, 0xbd, - 0xc8, 0x1f, 0xf0, 0xf6, 0x62, 0x3f, 0x0f, 0xa7, 0xaf, 0x86, 0xfd, 0xd9, 0x27, 0x57, 0x1f, 0xe9, - 0xf9, 0x68, 0xf5, 0x7e, 0x4f, 0x1f, 0xbf, 0x77, 0xf7, 0x7f, 0x96, 0x43, 0xb9, 0xda, 0x74, 0xf5, - 0x44, 0x6f, 0x8e, 0x54, 0x6f, 0xba, 0x3f, 0x4c, 0xa9, 0xc1, 0x5d, 0xf3, 0x40, 0x68, 0x0a, 0x16, - 0xb9, 0x56, 0xfe, 0x56, 0x0e, 0x8d, 0xd6, 0x66, 0x97, 0xab, 0xc2, 0x75, 0xcf, 0x6d, 0x66, 0xae, - 0x03, 0x86, 0x23, 0xac, 0x4b, 0xcf, 0xc7, 0xf6, 0x33, 0x77, 0x2a, 0x96, 0xf7, 0xfc, 0xcd, 0xbb, - 0x7a, 0xb3, 0x43, 0xe0, 0x6e, 0x80, 0x19, 0xf7, 0xb9, 0xe6, 0xbb, 0xe4, 0xc7, 0xc0, 0xbb, 0xd7, - 0x67, 0x80, 0x3f, 0x81, 0x72, 0x77, 0xf8, 0xb3, 0x6b, 0x1a, 0x9f, 0x1b, 0xd7, 0x19, 0x1f, 0x3a, - 0x09, 0xe6, 0x3a, 0xa6, 0x01, 0x1c, 0x68, 0x29, 0x5a, 0xf8, 0x16, 0x5f, 0x80, 0x77, 0x55, 0x78, - 0xd5, 0x2f, 0x7c, 0xab, 0x52, 0xc6, 0x35, 0x34, 0x54, 0x25, 0x4e, 0xcb, 0x84, 0x8e, 0xf2, 0xe7, - 0xec, 0xee, 0x4c, 0xe8, 0x49, 0x65, 0xa8, 0x1d, 0x16, 0x02, 0x66, 0x22, 0x17, 0xfc, 0x16, 0x42, - 0x6c, 0x8f, 0xb2, 0xcb, 0x68, 0x6d, 0x17, 0x60, 0xdf, 0xcf, 0xb6, 0x96, 0x09, 0x7b, 0x3c, 0x81, - 0x19, 0x5e, 0x47, 0x85, 0x05, 0xdb, 0x30, 0xef, 0x9b, 0xcc, 0xbe, 0x0a, 0x2a, 0xe8, 0xdb, 0xd9, - 0xaa, 0x81, 0x6e, 0x25, 0x5b, 0x42, 0xb9, 0xa4, 0x6a, 0x62, 0x8c, 0xd5, 0x7f, 0xd2, 0x8b, 0xf2, - 0xb4, 0xdb, 0x4f, 0xc6, 0xef, 0x41, 0xc6, 0x6f, 0x09, 0x15, 0xee, 0xd9, 0xce, 0xba, 0x69, 0xad, - 0x06, 0xa6, 0xaf, 0xfc, 0x6c, 0x0a, 0xcf, 0xf5, 0x1b, 0x0c, 0x57, 0x0f, 0xac, 0x64, 0xb5, 0x18, - 0xf9, 0x0e, 0x23, 0xf8, 0x45, 0x84, 0x98, 0x43, 0x2b, 0xd0, 0x0c, 0x84, 0x1e, 0xe9, 0xcc, 0xdd, - 0x15, 0xac, 0x69, 0x45, 0x8f, 0xf4, 0x90, 0x98, 0x1e, 0xc2, 0xd9, 0x83, 0xe7, 0x20, 0x18, 0xd7, - 0xc2, 0x21, 0x1c, 0x1e, 0x3c, 0xc5, 0x4d, 0x00, 0x7b, 0xfa, 0xac, 0x22, 0x24, 0x5c, 0x22, 0xa3, - 0x88, 0x20, 0xa4, 0xc9, 0x81, 0xc7, 0x80, 0x4a, 0xb8, 0x43, 0xd6, 0x04, 0x1e, 0xf8, 0xf9, 0xc8, - 0x2b, 0x17, 0x96, 0xb8, 0xa5, 0x3e, 0x72, 0x85, 0x56, 0x12, 0xc3, 0x3b, 0x59, 0x49, 0xa8, 0x5f, - 0xc8, 0xa2, 0xc1, 0x5a, 0x67, 0xc5, 0xdd, 0x74, 0x3d, 0xd2, 0x3a, 0xe6, 0x6a, 0xec, 0x1f, 0xaf, - 0xf2, 0x89, 0xc7, 0xab, 0xa7, 0x7c, 0xa1, 0x08, 0xf7, 0x8e, 0xc1, 0x96, 0xce, 0x17, 0xc7, 0x2f, - 0x65, 0x51, 0x81, 0xdd, 0x8e, 0x97, 0x4d, 0xb7, 0x71, 0x08, 0x16, 0xbb, 0x47, 0x2f, 0x95, 0x83, - 0xbd, 0x28, 0xed, 0xc2, 0x0e, 0x5a, 0xfd, 0x7c, 0x16, 0x0d, 0x95, 0x3a, 0xde, 0x5a, 0xc9, 0x03, - 0xdd, 0x7a, 0x24, 0xcf, 0x27, 0xbf, 0x93, 0x41, 0x63, 0xb4, 0x21, 0xcb, 0xf6, 0x3a, 0xb1, 0x0e, - 0xe1, 0xe2, 0x51, 0xbc, 0x40, 0xcc, 0xee, 0xf3, 0x02, 0xd1, 0x97, 0x65, 0x6e, 0x6f, 0xb2, 0x84, - 0xeb, 0x72, 0xcd, 0x6e, 0x92, 0xe3, 0xfd, 0x19, 0x87, 0x78, 0x5d, 0xee, 0x0b, 0xe4, 0x10, 0xae, - 0x52, 0xbe, 0x33, 0x04, 0xf2, 0xa3, 0x59, 0x74, 0x9a, 0x87, 0x09, 0xe5, 0x87, 0xa3, 0x13, 0x5d, - 0x49, 0x15, 0xcd, 0x89, 0xd6, 0x70, 0xd1, 0xfc, 0x74, 0x0e, 0x9d, 0x86, 0x60, 0x6a, 0x74, 0xcf, - 0xf8, 0x1d, 0x30, 0x51, 0xe2, 0x86, 0xfc, 0x42, 0xb3, 0x90, 0xf0, 0x42, 0xf3, 0x67, 0x5b, 0xc5, - 0xe7, 0x57, 0x4d, 0x6f, 0xad, 0xb3, 0x32, 0xd9, 0xb0, 0x5b, 0xd7, 0x56, 0x1d, 0xfd, 0x81, 0xc9, - 0xde, 0x26, 0xf4, 0xe6, 0xb5, 0x20, 0xe2, 0xb6, 0xde, 0x36, 0x79, 0x2c, 0xee, 0x1a, 0x6c, 0xc4, - 0x28, 0x57, 0xff, 0x6d, 0xc7, 0x45, 0xe8, 0x75, 0xdb, 0xb4, 0xb8, 0x55, 0x03, 0x5b, 0x85, 0x6b, - 0x74, 0xf3, 0xfa, 0x8e, 0x6d, 0x5a, 0xf5, 0xa8, 0x69, 0xc3, 0x5e, 0xeb, 0x0b, 0x59, 0x6b, 0x42, - 0x35, 0xea, 0xbf, 0xc9, 0xa0, 0xc7, 0x64, 0x2d, 0xfe, 0x4e, 0x58, 0xd8, 0xfe, 0x56, 0x16, 0x9d, - 0xb9, 0x05, 0xc2, 0x09, 0x5e, 0x99, 0x4f, 0xe6, 0x2d, 0x3e, 0x38, 0x13, 0x64, 0x73, 0x32, 0x71, - 0x71, 0xd9, 0xfc, 0xab, 0x0c, 0x3a, 0xb5, 0x54, 0x29, 0x4f, 0x7f, 0x87, 0x68, 0x4d, 0xfc, 0x7b, - 0x8e, 0x77, 0x4f, 0xc3, 0xf7, 0xd4, 0x4a, 0x0b, 0xf3, 0xdf, 0x49, 0xfd, 0x23, 0x7d, 0xcf, 0x31, - 0xef, 0x9f, 0xdf, 0xee, 0x43, 0x43, 0xb7, 0x3b, 0x2b, 0x84, 0xbf, 0xf9, 0x3d, 0xd2, 0x07, 0xea, - 0xeb, 0x68, 0x88, 0x8b, 0x01, 0x2e, 0xa3, 0x84, 0xc0, 0x23, 0xdc, 0x91, 0x94, 0xf9, 0x76, 0x8b, - 0x44, 0xf8, 0x3c, 0xca, 0xdf, 0x25, 0xce, 0x8a, 0x68, 0x93, 0xff, 0x80, 0x38, 0x2b, 0x1a, 0x40, - 0xf1, 0x7c, 0x68, 0x2a, 0x57, 0xaa, 0x56, 0x20, 0x08, 0x35, 0xbf, 0x07, 0x83, 0xa8, 0xda, 0x81, - 0x39, 0x81, 0xde, 0x36, 0x59, 0xf8, 0x6a, 0xd1, 0x1f, 0x28, 0x5a, 0x12, 0x2f, 0xa2, 0x71, 0xf1, - 0x3d, 0x99, 0x45, 0x60, 0x1e, 0x48, 0x60, 0x97, 0x14, 0x7b, 0x39, 0x5e, 0x14, 0xbf, 0x8a, 0x86, - 0x7d, 0x20, 0xbc, 0x8c, 0x0f, 0x86, 0x61, 0x3f, 0x03, 0x56, 0x91, 0xf0, 0xee, 0x52, 0x01, 0x91, - 0x01, 0xdc, 0xee, 0xa0, 0x04, 0x06, 0x11, 0x4b, 0x03, 0xa9, 0x00, 0xfe, 0x18, 0x30, 0x68, 0xdb, - 0x96, 0x4b, 0xe0, 0x0d, 0x70, 0x08, 0x0c, 0xd6, 0xc1, 0x14, 0xcf, 0xe1, 0x70, 0xe6, 0x96, 0x20, - 0x91, 0xe1, 0x25, 0x84, 0xc2, 0xb7, 0x1a, 0xee, 0xfc, 0xb5, 0xe7, 0x57, 0x24, 0x81, 0x85, 0x78, - 0xcb, 0x3a, 0xb2, 0x9f, 0x5b, 0x56, 0xf5, 0xf7, 0xb2, 0x68, 0xa8, 0xd4, 0x6e, 0x07, 0x43, 0xe1, - 0x59, 0xd4, 0x57, 0x6a, 0xb7, 0xef, 0x68, 0x15, 0x31, 0x0c, 0xa4, 0xde, 0x6e, 0xd7, 0x3b, 0x8e, - 0x29, 0x9a, 0xda, 0x30, 0x22, 0x3c, 0x8d, 0x46, 0x4a, 0xed, 0x76, 0xb5, 0xb3, 0xd2, 0x34, 0x1b, - 0x42, 0x54, 0x79, 0x96, 0x00, 0xa3, 0xdd, 0xae, 0xb7, 0x01, 0x13, 0x4d, 0x2d, 0x20, 0x97, 0xc1, - 0x6f, 0x83, 0xcb, 0x34, 0x0f, 0x6a, 0xce, 0xc2, 0x26, 0xab, 0x41, 0x00, 0xc8, 0xb0, 0x6d, 0x93, + // 13986 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0xbd, 0x7b, 0x70, 0x1c, 0xc9, + 0x79, 0x18, 0x8e, 0x7d, 0x60, 0xb1, 0x68, 0x3c, 0xb8, 0x68, 0xbe, 0xe6, 0x78, 0x24, 0xf7, 0x6e, + 0x4e, 0xa2, 0xc8, 0x13, 0x0f, 0xd4, 0x91, 0xd4, 0x9d, 0xee, 0x74, 0xa7, 0xbb, 0xc5, 0x8b, 0x58, + 0x12, 0x8f, 0xbd, 0x59, 0x90, 0xd4, 0xe9, 0x71, 0xeb, 0xc1, 0x4e, 0x13, 0x98, 0xc3, 0xee, 0xcc, + 0x6a, 0x66, 0x96, 0x20, 0xee, 0xf7, 0x4b, 0x62, 0x39, 0x7e, 0x26, 0x92, 0x4a, 0x65, 0x57, 0xca, + 0x4e, 0x25, 0x55, 0xf1, 0xa3, 0x9c, 0x38, 0x2e, 0xdb, 0xb2, 0x9d, 0x94, 0x6d, 0x49, 0x76, 0xc5, + 0x8e, 0x9c, 0x8a, 0x1c, 0x25, 0x29, 0xdb, 0x49, 0xb9, 0x52, 0x89, 0x03, 0x39, 0x4a, 0x39, 0x7f, + 0xa0, 0x92, 0x2a, 0xa7, 0xa2, 0x8a, 0x1d, 0xc7, 0x49, 0xa5, 0xfa, 0xeb, 0x9e, 0x99, 0xee, 0x79, + 0x2c, 0x9e, 0x67, 0x1c, 0x44, 0xfc, 0x43, 0x62, 0xbf, 0xef, 0xeb, 0xaf, 0x7b, 0xbe, 0xfe, 0xfa, + 0xfd, 0x3d, 0xd0, 0x15, 0x8f, 0xb4, 0x48, 0xc7, 0x76, 0xbc, 0x6b, 0x2d, 0xb2, 0xa2, 0x37, 0x37, + 0xae, 0x79, 0x1b, 0x1d, 0xe2, 0x5e, 0x23, 0x0f, 0x89, 0xe5, 0xf9, 0xff, 0x8d, 0x77, 0x1c, 0xdb, + 0xb3, 0x71, 0x81, 0xfd, 0x3a, 0x77, 0x6a, 0xc5, 0x5e, 0xb1, 0x01, 0x74, 0x8d, 0xfe, 0xc5, 0xb0, + 0xe7, 0xce, 0xaf, 0xd8, 0xf6, 0x4a, 0x8b, 0x5c, 0x83, 0x5f, 0xcb, 0xdd, 0x07, 0xd7, 0x5c, 0xcf, + 0xe9, 0x36, 0x3d, 0x8e, 0x2d, 0x47, 0xb1, 0x9e, 0xd9, 0x26, 0xae, 0xa7, 0xb7, 0x3b, 0x9c, 0xe0, + 0x62, 0x94, 0x60, 0xdd, 0xd1, 0x3b, 0x1d, 0xe2, 0xf0, 0xca, 0xcf, 0x3d, 0x9d, 0xdc, 0x4e, 0xf8, + 0x97, 0x93, 0x3c, 0x97, 0x4c, 0xe2, 0x33, 0x8a, 0x70, 0x54, 0x7f, 0x38, 0x8b, 0x8a, 0xf3, 0xc4, + 0xd3, 0x0d, 0xdd, 0xd3, 0xf1, 0x79, 0xd4, 0x5f, 0xb5, 0x0c, 0xf2, 0x48, 0xc9, 0x3c, 0x95, 0xb9, + 0x9c, 0x9b, 0x28, 0x6c, 0x6d, 0x96, 0xb3, 0xc4, 0xd4, 0x18, 0x10, 0x5f, 0x40, 0xf9, 0xa5, 0x8d, + 0x0e, 0x51, 0xb2, 0x4f, 0x65, 0x2e, 0x0f, 0x4e, 0x0c, 0x6e, 0x6d, 0x96, 0xfb, 0x41, 0x16, 0x1a, + 0x80, 0xf1, 0xd3, 0x28, 0x5b, 0x9d, 0x52, 0x72, 0x80, 0x1c, 0xdb, 0xda, 0x2c, 0x8f, 0x74, 0x4d, + 0xe3, 0xaa, 0xdd, 0x36, 0x3d, 0xd2, 0xee, 0x78, 0x1b, 0x5a, 0xb6, 0x3a, 0x85, 0x2f, 0xa1, 0xfc, + 0xa4, 0x6d, 0x10, 0x25, 0x0f, 0x44, 0x78, 0x6b, 0xb3, 0x3c, 0xda, 0xb4, 0x0d, 0x22, 0x50, 0x01, + 0x1e, 0xbf, 0x8e, 0xf2, 0x4b, 0x66, 0x9b, 0x28, 0xfd, 0x4f, 0x65, 0x2e, 0x0f, 0x5d, 0x3f, 0x37, + 0xce, 0xa4, 0x32, 0xee, 0x4b, 0x65, 0x7c, 0xc9, 0x17, 0xdb, 0x44, 0xe9, 0xeb, 0x9b, 0xe5, 0xbe, + 0xad, 0xcd, 0x72, 0x9e, 0x4a, 0xf2, 0x8b, 0xdf, 0x2c, 0x67, 0x34, 0x28, 0x89, 0x5f, 0x41, 0x43, + 0x93, 0xad, 0xae, 0xeb, 0x11, 0x67, 0x41, 0x6f, 0x13, 0xa5, 0x00, 0x15, 0x9e, 0xdb, 0xda, 0x2c, + 0x9f, 0x69, 0x32, 0x70, 0xc3, 0xd2, 0xdb, 0x62, 0xc5, 0x22, 0xb9, 0xfa, 0x6b, 0x19, 0x74, 0xa2, + 0x4e, 0x5c, 0xd7, 0xb4, 0xad, 0x40, 0x36, 0xef, 0x47, 0x83, 0x1c, 0x54, 0x9d, 0x02, 0xf9, 0x0c, + 0x4e, 0x0c, 0x6c, 0x6d, 0x96, 0x73, 0xae, 0x69, 0x68, 0x21, 0x06, 0x7f, 0x08, 0x0d, 0xdc, 0x37, + 0xbd, 0xd5, 0xf9, 0x99, 0x0a, 0x97, 0xd3, 0x99, 0xad, 0xcd, 0x32, 0x5e, 0x37, 0xbd, 0xd5, 0x46, + 0xfb, 0x81, 0x2e, 0x54, 0xe8, 0x93, 0xe1, 0x39, 0x54, 0xaa, 0x39, 0xe6, 0x43, 0xdd, 0x23, 0x77, + 0xc8, 0x46, 0xcd, 0x6e, 0x99, 0xcd, 0x0d, 0x2e, 0xc5, 0xa7, 0xb6, 0x36, 0xcb, 0xe7, 0x3b, 0x0c, + 0xd7, 0x58, 0x23, 0x1b, 0x8d, 0x0e, 0x60, 0x05, 0x26, 0xb1, 0x92, 0xea, 0xd7, 0xfa, 0xd1, 0xf0, + 0x5d, 0x97, 0x38, 0x41, 0xbb, 0x2f, 0xa1, 0x3c, 0xfd, 0xcd, 0x9b, 0x0c, 0x32, 0xef, 0xba, 0xc4, + 0x11, 0x65, 0x4e, 0xf1, 0xf8, 0x0a, 0xea, 0x9f, 0xb3, 0x57, 0x4c, 0x8b, 0x37, 0xfb, 0xe4, 0xd6, + 0x66, 0xf9, 0x44, 0x8b, 0x02, 0x04, 0x4a, 0x46, 0x81, 0x3f, 0x86, 0x86, 0xab, 0x6d, 0xaa, 0x43, + 0xb6, 0xa5, 0x7b, 0xb6, 0xc3, 0x5b, 0x0b, 0xd2, 0x35, 0x05, 0xb8, 0x50, 0x50, 0xa2, 0xc7, 0x2f, + 0x23, 0x54, 0xb9, 0x5f, 0xd7, 0xec, 0x16, 0xa9, 0x68, 0x0b, 0x5c, 0x19, 0xa0, 0xb4, 0xbe, 0xee, + 0x36, 0x1c, 0xbb, 0x45, 0x1a, 0xba, 0x23, 0x56, 0x2b, 0x50, 0xe3, 0x69, 0x34, 0x5a, 0x69, 0x36, + 0x89, 0xeb, 0x6a, 0xe4, 0x33, 0x5d, 0xe2, 0x7a, 0xae, 0xd2, 0xff, 0x54, 0xee, 0xf2, 0xe0, 0xc4, + 0x85, 0xad, 0xcd, 0xf2, 0x13, 0x3a, 0x60, 0x1a, 0x0e, 0x47, 0x09, 0x2c, 0x22, 0x85, 0xf0, 0x04, + 0x1a, 0xa9, 0xbc, 0xd3, 0x75, 0x48, 0xd5, 0x20, 0x96, 0x67, 0x7a, 0x1b, 0x5c, 0x43, 0xce, 0x6f, + 0x6d, 0x96, 0x15, 0x9d, 0x22, 0x1a, 0x26, 0xc7, 0x08, 0x4c, 0xe4, 0x22, 0x78, 0x11, 0x8d, 0xdd, + 0x9a, 0xac, 0xd5, 0x89, 0xf3, 0xd0, 0x6c, 0x92, 0x4a, 0xb3, 0x69, 0x77, 0x2d, 0x4f, 0x19, 0x00, + 0x3e, 0x4f, 0x6f, 0x6d, 0x96, 0x2f, 0xac, 0x34, 0x3b, 0x0d, 0x97, 0x61, 0x1b, 0x3a, 0x43, 0x0b, + 0xcc, 0xe2, 0x65, 0xf1, 0x27, 0xd0, 0xc8, 0x92, 0x43, 0xb5, 0xd0, 0x98, 0x22, 0x14, 0xae, 0x14, + 0x41, 0xff, 0xcf, 0x8c, 0xf3, 0x09, 0x88, 0x41, 0xfd, 0x9e, 0x65, 0x8d, 0xf5, 0x58, 0x81, 0x86, + 0x01, 0x38, 0xb1, 0xb1, 0x12, 0x2b, 0x4c, 0x90, 0x42, 0x3f, 0xde, 0x74, 0x88, 0x11, 0xd3, 0xb6, + 0x41, 0x68, 0xf3, 0x95, 0xad, 0xcd, 0xf2, 0xfb, 0x1d, 0x4e, 0xd3, 0xe8, 0xa9, 0x76, 0xa9, 0xac, + 0xf0, 0x34, 0x2a, 0x52, 0x6d, 0xba, 0x63, 0x5a, 0x86, 0x82, 0x9e, 0xca, 0x5c, 0x1e, 0xbd, 0x5e, + 0xf2, 0x5b, 0xef, 0xc3, 0x27, 0xce, 0x6e, 0x6d, 0x96, 0x4f, 0x52, 0x1d, 0x6c, 0xac, 0x99, 0x96, + 0x38, 0x45, 0x04, 0x45, 0xd5, 0x3f, 0xcd, 0xa3, 0x51, 0x2a, 0x1c, 0x41, 0x8f, 0x2b, 0x74, 0x48, + 0x52, 0x08, 0x1d, 0xa1, 0x6e, 0x47, 0x6f, 0x12, 0xae, 0xd2, 0xc0, 0xce, 0xf2, 0x81, 0x02, 0xbb, + 0x28, 0x3d, 0xbe, 0x82, 0x8a, 0x0c, 0x54, 0x9d, 0xe2, 0x5a, 0x3e, 0xb2, 0xb5, 0x59, 0x1e, 0x74, + 0x01, 0xd6, 0x30, 0x0d, 0x2d, 0x40, 0x53, 0x35, 0x63, 0x7f, 0xcf, 0xda, 0xae, 0x47, 0x99, 0x73, + 0x25, 0x07, 0x35, 0xe3, 0x05, 0x56, 0x39, 0x4a, 0x54, 0x33, 0xb9, 0x10, 0x7e, 0x09, 0x21, 0x06, + 0xa9, 0x18, 0x86, 0xc3, 0x35, 0xfd, 0x89, 0xad, 0xcd, 0xf2, 0x69, 0xce, 0x42, 0x37, 0x0c, 0x71, + 0x98, 0x08, 0xc4, 0xb8, 0x8d, 0x86, 0xd9, 0xaf, 0x39, 0x7d, 0x99, 0xb4, 0x98, 0x9a, 0x0f, 0x5d, + 0xbf, 0xec, 0x4b, 0x53, 0x96, 0xce, 0xb8, 0x48, 0x3a, 0x6d, 0x79, 0xce, 0xc6, 0x44, 0x99, 0xcf, + 0x8c, 0x67, 0x79, 0x55, 0x2d, 0xc0, 0x89, 0x63, 0x52, 0x2c, 0x43, 0x27, 0xcc, 0x19, 0xdb, 0x59, + 0xd7, 0x1d, 0x83, 0x18, 0x13, 0x1b, 0xe2, 0x84, 0xf9, 0xc0, 0x07, 0x37, 0x96, 0x45, 0x1d, 0x10, + 0xc9, 0xf1, 0x24, 0x1a, 0x61, 0xdc, 0xea, 0xdd, 0x65, 0xe8, 0xfb, 0x81, 0x98, 0xb4, 0xdc, 0xee, + 0x72, 0xb4, 0xbf, 0xe5, 0x32, 0x74, 0x4c, 0x32, 0xc0, 0x3d, 0xe2, 0xd0, 0xd9, 0x14, 0xd4, 0x9f, + 0x8f, 0x49, 0xce, 0xe4, 0x21, 0xc3, 0xc4, 0x79, 0xf0, 0x22, 0xe7, 0x5e, 0x43, 0x63, 0x31, 0x51, + 0xe0, 0x12, 0xca, 0xad, 0x91, 0x0d, 0xa6, 0x2e, 0x1a, 0xfd, 0x13, 0x9f, 0x42, 0xfd, 0x0f, 0xf5, + 0x56, 0x97, 0xaf, 0x65, 0x1a, 0xfb, 0xf1, 0x72, 0xf6, 0x23, 0x19, 0x3a, 0xf5, 0xe3, 0x49, 0xdb, + 0xb2, 0x48, 0xd3, 0x13, 0x67, 0xff, 0x17, 0xd0, 0xe0, 0x9c, 0xdd, 0xd4, 0x5b, 0xd0, 0x8f, 0x4c, + 0xef, 0x94, 0xad, 0xcd, 0xf2, 0x29, 0xda, 0x81, 0xe3, 0x2d, 0x8a, 0x11, 0xda, 0x14, 0x92, 0x52, + 0x05, 0xd0, 0x48, 0xdb, 0xf6, 0x08, 0x14, 0xcc, 0x86, 0x0a, 0x00, 0x05, 0x1d, 0x40, 0x89, 0x0a, + 0x10, 0x12, 0xe3, 0x6b, 0xa8, 0x58, 0xa3, 0x0b, 0x5e, 0xd3, 0x6e, 0x71, 0xe5, 0x83, 0x39, 0x19, + 0x16, 0x41, 0x71, 0xd0, 0xf8, 0x44, 0xea, 0x2c, 0x1a, 0x9d, 0x6c, 0x99, 0xc4, 0xf2, 0xc4, 0x56, + 0xd3, 0x21, 0x55, 0x59, 0x21, 0x96, 0x27, 0xb6, 0x1a, 0x06, 0x9f, 0x4e, 0xa1, 0x62, 0xab, 0x03, + 0x52, 0xf5, 0x5f, 0xe7, 0xd0, 0x13, 0x77, 0xba, 0xcb, 0xc4, 0xb1, 0x88, 0x47, 0x5c, 0xbe, 0x32, + 0x06, 0x5c, 0x17, 0xd0, 0x58, 0x0c, 0xc9, 0xb9, 0xc3, 0x8a, 0xb5, 0x16, 0x20, 0x1b, 0x7c, 0xb1, + 0x15, 0xa7, 0xbd, 0x58, 0x51, 0x3c, 0x8b, 0x4e, 0x84, 0x40, 0xda, 0x08, 0x57, 0xc9, 0xc2, 0x9c, + 0x7e, 0x71, 0x6b, 0xb3, 0x7c, 0x4e, 0xe0, 0x46, 0x9b, 0x2d, 0x6a, 0x70, 0xb4, 0x18, 0xbe, 0x83, + 0x4a, 0x21, 0xe8, 0x96, 0x63, 0x77, 0x3b, 0xae, 0x92, 0x03, 0x56, 0xe5, 0xad, 0xcd, 0xf2, 0x93, + 0x02, 0xab, 0x15, 0x40, 0x8a, 0x2b, 0x69, 0xb4, 0x20, 0xfe, 0xde, 0x8c, 0xc8, 0x8d, 0x8f, 0xc2, + 0x3c, 0x8c, 0xc2, 0x17, 0xfd, 0x51, 0x98, 0x2a, 0xa4, 0xf1, 0x68, 0x49, 0x3e, 0x28, 0x23, 0xcd, + 0x88, 0x0d, 0xca, 0x58, 0x8d, 0xe7, 0x26, 0xd1, 0xe9, 0x44, 0x5e, 0xbb, 0xd2, 0xea, 0x3f, 0xce, + 0x89, 0x5c, 0x6a, 0xb6, 0x11, 0x74, 0xe6, 0xa2, 0xd8, 0x99, 0x35, 0xdb, 0x80, 0xed, 0x52, 0x26, + 0x5c, 0xc4, 0x84, 0xc6, 0x76, 0x6c, 0x23, 0xba, 0x6b, 0x8a, 0x97, 0xc5, 0x6f, 0xa1, 0x33, 0x31, + 0x20, 0x9b, 0xae, 0x99, 0xf6, 0x5f, 0xda, 0xda, 0x2c, 0xab, 0x09, 0x5c, 0xa3, 0xb3, 0x77, 0x0a, + 0x17, 0xac, 0xa3, 0xb3, 0x82, 0xd4, 0x6d, 0xcb, 0xd3, 0x4d, 0x8b, 0xef, 0xf2, 0xd8, 0x28, 0xf9, + 0xc0, 0xd6, 0x66, 0xf9, 0x19, 0x51, 0x07, 0x7d, 0x9a, 0x68, 0xe3, 0xd3, 0xf8, 0x60, 0x03, 0x29, + 0x09, 0xa8, 0x6a, 0x5b, 0x5f, 0xf1, 0xb7, 0xae, 0x97, 0xb7, 0x36, 0xcb, 0xef, 0x4b, 0xac, 0xc3, + 0xa4, 0x54, 0xe2, 0x52, 0x99, 0xc6, 0x09, 0x6b, 0x08, 0x87, 0xb8, 0x05, 0xdb, 0x20, 0xf0, 0x0d, + 0xfd, 0xc0, 0x5f, 0xdd, 0xda, 0x2c, 0x5f, 0x14, 0xf8, 0x5b, 0xb6, 0x41, 0xa2, 0xcd, 0x4f, 0x28, + 0xad, 0xfe, 0x5a, 0x0e, 0x5d, 0xac, 0x57, 0xe6, 0xe7, 0xaa, 0x86, 0xbf, 0xb7, 0xa8, 0x39, 0xf6, + 0x43, 0xd3, 0x10, 0x46, 0xef, 0x32, 0x3a, 0x1b, 0x41, 0x4d, 0xc3, 0x76, 0x26, 0xd8, 0xd5, 0xc2, + 0xb7, 0xf9, 0xfb, 0x96, 0x0e, 0xa7, 0x69, 0xb0, 0x3d, 0x4f, 0x43, 0xda, 0xd2, 0xa7, 0x31, 0xa2, + 0x7d, 0x14, 0x41, 0xd5, 0x57, 0x6d, 0xc7, 0x6b, 0x76, 0x3d, 0xae, 0x04, 0xd0, 0x47, 0xb1, 0x3a, + 0x5c, 0x4e, 0xd4, 0xa3, 0x0a, 0x9f, 0x0f, 0xfe, 0xa1, 0x0c, 0x2a, 0x55, 0x3c, 0xcf, 0x31, 0x97, + 0xbb, 0x1e, 0x99, 0xd7, 0x3b, 0x1d, 0xd3, 0x5a, 0x81, 0xb1, 0x3e, 0x74, 0xfd, 0x95, 0x60, 0x8d, + 0xec, 0x29, 0x89, 0xf1, 0x68, 0x71, 0x61, 0x88, 0xea, 0x3e, 0xaa, 0xd1, 0x66, 0x38, 0x71, 0x88, + 0x46, 0xcb, 0xd1, 0x21, 0x9a, 0xc8, 0x6b, 0x57, 0x43, 0xf4, 0x87, 0x73, 0xe8, 0xfc, 0xe2, 0x9a, + 0xa7, 0x6b, 0xc4, 0xb5, 0xbb, 0x4e, 0x93, 0xb8, 0x77, 0x3b, 0x86, 0xee, 0x91, 0x70, 0xa4, 0x96, + 0x51, 0x7f, 0xc5, 0x30, 0x88, 0x01, 0xec, 0xfa, 0xd9, 0xf9, 0x4b, 0xa7, 0x00, 0x8d, 0xc1, 0xf1, + 0xfb, 0xd1, 0x00, 0x2f, 0x03, 0xdc, 0xfb, 0x27, 0x86, 0xb6, 0x36, 0xcb, 0x03, 0x5d, 0x06, 0xd2, + 0x7c, 0x1c, 0x25, 0x9b, 0x22, 0x2d, 0x42, 0xc9, 0x72, 0x21, 0x99, 0xc1, 0x40, 0x9a, 0x8f, 0xc3, + 0x6f, 0xa0, 0x51, 0x60, 0x1b, 0xb4, 0x87, 0xcf, 0x7d, 0xa7, 0x7c, 0xe9, 0x8a, 0x8d, 0x65, 0x4b, + 0x13, 0xb4, 0xa6, 0xe1, 0xf8, 0x05, 0xb4, 0x08, 0x03, 0x7c, 0x1f, 0x95, 0x78, 0x23, 0x42, 0xa6, + 0xfd, 0x3d, 0x98, 0x9e, 0xde, 0xda, 0x2c, 0x8f, 0xf1, 0xf6, 0x0b, 0x6c, 0x63, 0x4c, 0x28, 0x63, + 0xde, 0xec, 0x90, 0x71, 0x61, 0x3b, 0xc6, 0xfc, 0x8b, 0x45, 0xc6, 0x51, 0x26, 0xea, 0x9b, 0x68, + 0x58, 0x2c, 0x88, 0xcf, 0xc0, 0x19, 0x97, 0x8d, 0x13, 0x38, 0x1d, 0x9b, 0x06, 0x1c, 0x6c, 0x9f, + 0x47, 0x43, 0x53, 0xc4, 0x6d, 0x3a, 0x66, 0x87, 0xee, 0x1a, 0xb8, 0x92, 0x9f, 0xd8, 0xda, 0x2c, + 0x0f, 0x19, 0x21, 0x58, 0x13, 0x69, 0xd4, 0xff, 0x99, 0x41, 0x67, 0x28, 0xef, 0x8a, 0xeb, 0x9a, + 0x2b, 0x56, 0x5b, 0x5c, 0xb6, 0xaf, 0xa2, 0x42, 0x1d, 0xea, 0xe3, 0x35, 0x9d, 0xda, 0xda, 0x2c, + 0x97, 0x58, 0x0b, 0x04, 0x3d, 0xe4, 0x34, 0xc1, 0x01, 0x2f, 0xbb, 0xcd, 0x01, 0x8f, 0x6e, 0x69, + 0x3d, 0xdd, 0xf1, 0x4c, 0x6b, 0xa5, 0xee, 0xe9, 0x5e, 0xd7, 0x95, 0xb6, 0xb4, 0x1c, 0xd3, 0x70, + 0x01, 0x25, 0x6d, 0x69, 0xa5, 0x42, 0xf8, 0x35, 0x34, 0x3c, 0x6d, 0x19, 0x21, 0x13, 0x36, 0x21, + 0x3e, 0x49, 0x77, 0x9a, 0x04, 0xe0, 0x71, 0x16, 0x52, 0x01, 0xf5, 0xe7, 0x33, 0x48, 0x61, 0xa7, + 0xb1, 0x39, 0xd3, 0xf5, 0xe6, 0x49, 0x7b, 0x59, 0x98, 0x9d, 0x66, 0xfc, 0xe3, 0x1d, 0xc5, 0x09, + 0x6b, 0x11, 0x6c, 0x05, 0xf8, 0xf1, 0xae, 0x65, 0xba, 0x5e, 0x74, 0x32, 0x8c, 0x94, 0xc2, 0x55, + 0x34, 0xc0, 0x38, 0xb3, 0xbd, 0xc4, 0xd0, 0x75, 0xc5, 0x57, 0x84, 0x68, 0xd5, 0x4c, 0x19, 0xda, + 0x8c, 0x58, 0x3c, 0x9f, 0xf3, 0xf2, 0xea, 0x2f, 0x66, 0x51, 0x29, 0x5a, 0x08, 0xdf, 0x47, 0xc5, + 0xdb, 0xb6, 0x69, 0x11, 0x63, 0xd1, 0x82, 0x16, 0xf6, 0xbe, 0xa5, 0xf0, 0xf7, 0xe2, 0x27, 0xdf, + 0x86, 0x32, 0x0d, 0x71, 0x07, 0x0b, 0x97, 0x16, 0x01, 0x33, 0xfc, 0x09, 0x34, 0x48, 0xf7, 0x80, + 0x0f, 0x81, 0x73, 0x76, 0x5b, 0xce, 0x4f, 0x71, 0xce, 0xa7, 0x1c, 0x56, 0x28, 0xce, 0x3a, 0x64, + 0x47, 0xf5, 0x4a, 0x23, 0xba, 0x6b, 0x5b, 0xbc, 0xe7, 0x41, 0xaf, 0x1c, 0x80, 0x88, 0x7a, 0xc5, + 0x68, 0xe8, 0xd6, 0x95, 0x7d, 0x2c, 0x74, 0x83, 0x70, 0x76, 0x61, 0xb2, 0x8a, 0xf6, 0x80, 0x40, + 0xac, 0x7e, 0x7f, 0x16, 0x3d, 0x17, 0x8a, 0x4c, 0x23, 0x0f, 0x4d, 0xb2, 0xce, 0xc5, 0xb9, 0x6a, + 0x76, 0xf8, 0xe1, 0x91, 0xaa, 0xbc, 0x3b, 0xb9, 0xaa, 0x5b, 0x2b, 0xc4, 0xc0, 0x57, 0x50, 0x3f, + 0x3d, 0xe1, 0xbb, 0x4a, 0x06, 0xb6, 0x6b, 0x30, 0x9d, 0x38, 0x14, 0x20, 0xde, 0x3e, 0x00, 0x05, + 0xb6, 0x51, 0x61, 0xc9, 0xd1, 0x4d, 0xcf, 0xef, 0xd9, 0x4a, 0xbc, 0x67, 0x77, 0x50, 0xe3, 0x38, + 0xe3, 0xc1, 0xe6, 0x7c, 0x10, 0x84, 0x07, 0x00, 0x51, 0x10, 0x8c, 0xe4, 0xdc, 0x4b, 0x68, 0x48, + 0x20, 0xde, 0xd5, 0xa4, 0xfe, 0xe5, 0xbc, 0xa8, 0xeb, 0x7e, 0xb3, 0xb8, 0xae, 0x5f, 0xa3, 0x3a, + 0xea, 0xba, 0x74, 0x57, 0xc1, 0x94, 0x9c, 0x6b, 0x22, 0x80, 0x64, 0x4d, 0x04, 0x10, 0xbe, 0x81, + 0x8a, 0x8c, 0x45, 0x70, 0x7e, 0x85, 0xb3, 0xaf, 0x03, 0x30, 0x79, 0x69, 0x0e, 0x08, 0xf1, 0xcf, + 0x66, 0xd0, 0x85, 0x9e, 0x92, 0x00, 0x65, 0x18, 0xba, 0xfe, 0xe1, 0x3d, 0x89, 0x71, 0xe2, 0xb9, + 0xad, 0xcd, 0xf2, 0x95, 0x76, 0x40, 0xd2, 0x70, 0x04, 0x9a, 0x46, 0x93, 0x11, 0x09, 0xed, 0xea, + 0xdd, 0x14, 0xba, 0x79, 0x64, 0x95, 0xce, 0xc0, 0x1d, 0x8e, 0xd5, 0xdc, 0xf0, 0x1b, 0x99, 0x0f, + 0x37, 0x8f, 0xfc, 0x7b, 0x1f, 0xf8, 0x24, 0x09, 0xd5, 0xa4, 0x70, 0xc1, 0x4d, 0x74, 0x96, 0x61, + 0xa6, 0xf4, 0x8d, 0xc5, 0x07, 0xf3, 0xb6, 0xe5, 0xad, 0xfa, 0x15, 0xf4, 0x8b, 0x97, 0x20, 0x50, + 0x81, 0xa1, 0x6f, 0x34, 0xec, 0x07, 0x8d, 0x36, 0xa5, 0x4a, 0xa8, 0x23, 0x8d, 0x13, 0x9d, 0x68, + 0xf9, 0x98, 0xf3, 0xa7, 0xa0, 0x42, 0x78, 0x45, 0xe5, 0x8f, 0xd3, 0xf8, 0x84, 0x13, 0x29, 0xa4, + 0x56, 0xd1, 0xf0, 0x9c, 0xdd, 0x5c, 0x0b, 0xd4, 0xe5, 0x25, 0x54, 0x58, 0xd2, 0x9d, 0x15, 0xe2, + 0x81, 0x2c, 0x86, 0xae, 0x8f, 0x8d, 0xb3, 0x6b, 0x5f, 0x4a, 0xc4, 0x10, 0x13, 0xa3, 0x7c, 0x36, + 0x28, 0x78, 0xf0, 0x5b, 0xe3, 0x05, 0xd4, 0x6f, 0xf6, 0xa3, 0x61, 0x7e, 0x45, 0x09, 0xb3, 0x39, + 0x7e, 0x39, 0xbc, 0xf4, 0xe5, 0xd3, 0x57, 0x70, 0x4d, 0x13, 0x5c, 0x2f, 0x0d, 0x53, 0x66, 0xbf, + 0xbb, 0x59, 0xce, 0x6c, 0x6d, 0x96, 0xfb, 0xb4, 0xa2, 0x70, 0xa8, 0x0c, 0xd7, 0x1b, 0x61, 0x81, + 0x15, 0x2f, 0x1d, 0x23, 0x65, 0xd9, 0xfa, 0xf3, 0x1a, 0x1a, 0xe0, 0x6d, 0xe0, 0x1a, 0x77, 0x36, + 0xbc, 0xcb, 0x90, 0xae, 0x5a, 0x23, 0xa5, 0xfd, 0x52, 0xf8, 0x15, 0x54, 0x60, 0x67, 0x7b, 0x2e, + 0x80, 0x33, 0xc9, 0x77, 0x21, 0x91, 0xe2, 0xbc, 0x0c, 0x9e, 0x45, 0x28, 0x3c, 0xd7, 0x07, 0x37, + 0xcb, 0x9c, 0x43, 0xfc, 0xc4, 0x1f, 0xe1, 0x22, 0x94, 0xc5, 0x2f, 0xa0, 0xe1, 0x25, 0xe2, 0xb4, + 0x4d, 0x4b, 0x6f, 0xd5, 0xcd, 0x77, 0xfc, 0xcb, 0x65, 0x58, 0x78, 0x5d, 0xf3, 0x1d, 0x71, 0xe4, + 0x4a, 0x74, 0xf8, 0xd3, 0x49, 0xe7, 0xe6, 0x01, 0x68, 0xc8, 0xd3, 0xdb, 0x1e, 0x28, 0x23, 0xed, + 0x49, 0x38, 0x46, 0xbf, 0x81, 0x46, 0xa4, 0x23, 0x13, 0xbf, 0x3d, 0xbc, 0x10, 0x67, 0x2d, 0x9c, + 0xff, 0x22, 0x6c, 0x65, 0x0e, 0x54, 0x93, 0xab, 0x96, 0xe9, 0x99, 0x7a, 0x6b, 0xd2, 0x6e, 0xb7, + 0x75, 0xcb, 0x50, 0x06, 0x43, 0x4d, 0x36, 0x19, 0xa6, 0xd1, 0x64, 0x28, 0x51, 0x93, 0xe5, 0x42, + 0xf4, 0x58, 0xce, 0xfb, 0x50, 0x23, 0x4d, 0xdb, 0xa1, 0x7b, 0x01, 0xb8, 0x1c, 0xe4, 0xc7, 0x72, + 0x97, 0xe1, 0x1a, 0x8e, 0x8f, 0x14, 0x37, 0xdb, 0xd1, 0x82, 0xb7, 0xf3, 0xc5, 0xa1, 0xd2, 0x70, + 0xf4, 0x3e, 0x57, 0xfd, 0x87, 0x39, 0x34, 0xc4, 0x49, 0xe9, 0x52, 0x7a, 0xac, 0xe0, 0xfb, 0x51, + 0xf0, 0x44, 0x45, 0x2d, 0x1c, 0x94, 0xa2, 0xaa, 0x9f, 0xcb, 0x06, 0xb3, 0x51, 0xcd, 0x31, 0xad, + 0xfd, 0xcd, 0x46, 0x97, 0x10, 0x9a, 0x5c, 0xed, 0x5a, 0x6b, 0xec, 0xdd, 0x2a, 0x1b, 0xbe, 0x5b, + 0x35, 0x4d, 0x4d, 0xc0, 0xe0, 0x0b, 0x28, 0x3f, 0x45, 0xf9, 0xd3, 0x9e, 0x19, 0x9e, 0x18, 0xfc, + 0x3a, 0xe3, 0x94, 0x79, 0x4e, 0x03, 0x30, 0x3d, 0x5c, 0x4d, 0x6c, 0x78, 0x84, 0x6d, 0x67, 0x73, + 0xec, 0x70, 0xb5, 0x4c, 0x01, 0x1a, 0x83, 0xe3, 0x9b, 0x68, 0x6c, 0x8a, 0xb4, 0xf4, 0x8d, 0x79, + 0xb3, 0xd5, 0x32, 0x5d, 0xd2, 0xb4, 0x2d, 0xc3, 0x05, 0x21, 0xf3, 0xea, 0xda, 0xae, 0x16, 0x27, + 0xc0, 0x2a, 0x2a, 0x2c, 0x3e, 0x78, 0xe0, 0x12, 0x0f, 0xc4, 0x97, 0x9b, 0x40, 0x74, 0x72, 0xb6, + 0x01, 0xa2, 0x71, 0x8c, 0xfa, 0xa5, 0x0c, 0x3d, 0xbd, 0xb8, 0x6b, 0x9e, 0xdd, 0x09, 0xb4, 0x7c, + 0x5f, 0x22, 0xb9, 0x12, 0xee, 0x2b, 0xb2, 0xf0, 0xb5, 0x27, 0xf8, 0xd7, 0x0e, 0xf0, 0xbd, 0x45, + 0xb8, 0xa3, 0x48, 0xfc, 0xaa, 0xdc, 0x36, 0x5f, 0xa5, 0xfe, 0x49, 0x16, 0x9d, 0xe5, 0x2d, 0x9e, + 0x6c, 0x99, 0x9d, 0x65, 0x5b, 0x77, 0x0c, 0x8d, 0x34, 0x89, 0xf9, 0x90, 0x1c, 0xcd, 0x81, 0x27, + 0x0f, 0x9d, 0xfc, 0x3e, 0x86, 0xce, 0x75, 0x38, 0x08, 0x52, 0xc9, 0xc0, 0x85, 0x2f, 0xdb, 0x54, + 0x94, 0xb6, 0x36, 0xcb, 0xc3, 0x06, 0x03, 0xc3, 0x95, 0xbf, 0x26, 0x12, 0x51, 0x25, 0x99, 0x23, + 0xd6, 0x8a, 0xb7, 0x0a, 0x4a, 0xd2, 0xcf, 0x94, 0xa4, 0x05, 0x10, 0x8d, 0x63, 0xd4, 0xff, 0x96, + 0x45, 0xa7, 0xa2, 0x22, 0xaf, 0x13, 0xcb, 0x38, 0x96, 0xf7, 0xbb, 0x23, 0xef, 0x6f, 0xe7, 0xd0, + 0x93, 0xbc, 0x4c, 0x7d, 0x55, 0x77, 0x88, 0x31, 0x65, 0x3a, 0xa4, 0xe9, 0xd9, 0xce, 0xc6, 0x11, + 0xde, 0x40, 0x1d, 0x9c, 0xd8, 0x6f, 0xa2, 0x02, 0x3f, 0xfe, 0xb3, 0x75, 0x66, 0x34, 0x68, 0x09, + 0x40, 0x63, 0x2b, 0x14, 0xbb, 0x3a, 0x88, 0x74, 0x56, 0x61, 0x27, 0x9d, 0xf5, 0x11, 0x34, 0x12, + 0x88, 0x1e, 0x0e, 0xa2, 0x03, 0xe1, 0x6e, 0xcb, 0xf0, 0x11, 0x70, 0x16, 0xd5, 0x64, 0x42, 0xa8, + 0xcd, 0x07, 0x54, 0xa7, 0x60, 0x37, 0x34, 0xc2, 0x6b, 0x0b, 0xca, 0x99, 0x86, 0x26, 0x12, 0xa9, + 0x9b, 0x79, 0x74, 0x2e, 0xb9, 0xdb, 0x35, 0xa2, 0x1b, 0xc7, 0xbd, 0xfe, 0x1d, 0xd9, 0xeb, 0xf8, + 0x69, 0x94, 0xaf, 0xe9, 0xde, 0x2a, 0x7f, 0x07, 0x87, 0x37, 0xe1, 0x07, 0x66, 0x8b, 0x34, 0x3a, + 0xba, 0xb7, 0xaa, 0x01, 0x4a, 0x98, 0x33, 0x10, 0x70, 0x4c, 0x98, 0x33, 0x84, 0xc5, 0x7e, 0xe8, + 0xa9, 0xcc, 0xe5, 0x7c, 0xe2, 0x62, 0xff, 0xcd, 0x7c, 0xda, 0xbc, 0x72, 0xdf, 0x31, 0x3d, 0x72, + 0xac, 0x61, 0xc7, 0x1a, 0xb6, 0x4f, 0x0d, 0xfb, 0xfd, 0x2c, 0x1a, 0x09, 0x0e, 0x4d, 0x6f, 0x93, + 0xe6, 0xe1, 0xac, 0x55, 0xe1, 0x51, 0x26, 0xb7, 0xef, 0xa3, 0xcc, 0x7e, 0x14, 0x4a, 0x0d, 0xae, + 0x3c, 0xd9, 0xd6, 0x00, 0x24, 0xc6, 0xae, 0x3c, 0x83, 0x8b, 0xce, 0xa7, 0xd1, 0xc0, 0xbc, 0xfe, + 0xc8, 0x6c, 0x77, 0xdb, 0x7c, 0x97, 0x0e, 0x76, 0x5d, 0x6d, 0xfd, 0x91, 0xe6, 0xc3, 0xd5, 0x7f, + 0x9b, 0x41, 0xa3, 0x5c, 0xa8, 0x9c, 0xf9, 0xbe, 0xa4, 0x1a, 0x4a, 0x27, 0xbb, 0x6f, 0xe9, 0xe4, + 0xf6, 0x2e, 0x1d, 0xf5, 0xef, 0xe6, 0x90, 0x32, 0x63, 0xb6, 0xc8, 0x92, 0xa3, 0x5b, 0xee, 0x03, + 0xe2, 0xf0, 0xe3, 0xf4, 0x34, 0x65, 0xb5, 0xaf, 0x0f, 0x14, 0xa6, 0x94, 0xec, 0x9e, 0xa6, 0x94, + 0x0f, 0xa2, 0x41, 0xde, 0x98, 0xc0, 0xa6, 0x10, 0x46, 0x8d, 0xe3, 0x03, 0xb5, 0x10, 0x4f, 0x89, + 0x2b, 0x9d, 0x8e, 0x63, 0x3f, 0x24, 0x0e, 0x7b, 0xa5, 0xe2, 0xc4, 0xba, 0x0f, 0xd4, 0x42, 0xbc, + 0xc0, 0x99, 0xf8, 0xfb, 0x45, 0x91, 0x33, 0x71, 0xb4, 0x10, 0x8f, 0x2f, 0xa3, 0xe2, 0x9c, 0xdd, + 0xd4, 0x41, 0xd0, 0x6c, 0x5a, 0x19, 0xde, 0xda, 0x2c, 0x17, 0x5b, 0x1c, 0xa6, 0x05, 0x58, 0x4a, + 0x39, 0x65, 0xaf, 0x5b, 0x2d, 0x5b, 0x67, 0xc6, 0x2f, 0x45, 0x46, 0x69, 0x70, 0x98, 0x16, 0x60, + 0x29, 0x25, 0x95, 0x39, 0x18, 0x15, 0x15, 0x43, 0x9e, 0x0f, 0x38, 0x4c, 0x0b, 0xb0, 0xea, 0x97, + 0xf2, 0x54, 0x7b, 0x5d, 0xf3, 0x9d, 0xc7, 0x7e, 0x5d, 0x08, 0x07, 0x4c, 0xff, 0x1e, 0x06, 0xcc, + 0x63, 0x73, 0x61, 0xa7, 0xfe, 0xe9, 0x00, 0x42, 0x5c, 0xfa, 0xd3, 0xc7, 0x87, 0xc3, 0xfd, 0x69, + 0xcd, 0x14, 0x1a, 0x9b, 0xb6, 0x56, 0x75, 0xab, 0x49, 0x8c, 0xf0, 0xda, 0xb2, 0x00, 0x43, 0x1b, + 0x6c, 0x7a, 0x09, 0x47, 0x86, 0xf7, 0x96, 0x5a, 0xbc, 0x00, 0x7e, 0x1e, 0x0d, 0x55, 0x2d, 0x8f, + 0x38, 0x7a, 0xd3, 0x33, 0x1f, 0x12, 0x3e, 0x35, 0xc0, 0xcb, 0xb0, 0x19, 0x82, 0x35, 0x91, 0x06, + 0xdf, 0x44, 0xc3, 0x35, 0xdd, 0xf1, 0xcc, 0xa6, 0xd9, 0xd1, 0x2d, 0xcf, 0x55, 0x8a, 0x30, 0xa3, + 0xc1, 0x0e, 0xa3, 0x23, 0xc0, 0x35, 0x89, 0x0a, 0x7f, 0x1a, 0x0d, 0xc2, 0xd1, 0x14, 0x0c, 0xa7, + 0x07, 0xb7, 0x7d, 0x38, 0x7c, 0x26, 0x34, 0x0f, 0x64, 0xb7, 0xaf, 0xf0, 0x02, 0x1c, 0x7d, 0x3b, + 0x0c, 0x38, 0xe2, 0x8f, 0xa3, 0x81, 0x69, 0xcb, 0x00, 0xe6, 0x68, 0x5b, 0xe6, 0x2a, 0x67, 0x7e, + 0x26, 0x64, 0x6e, 0x77, 0x22, 0xbc, 0x7d, 0x76, 0xc9, 0xa3, 0x6c, 0xe8, 0xdd, 0x1b, 0x65, 0xc3, + 0xef, 0xc2, 0xb5, 0xf8, 0xc8, 0x41, 0x5d, 0x8b, 0x8f, 0xee, 0xf1, 0x5a, 0x5c, 0x7d, 0x07, 0x0d, + 0x4d, 0xd4, 0x66, 0x82, 0xd1, 0xfb, 0x04, 0xca, 0xd5, 0xb8, 0xa5, 0x42, 0x9e, 0xed, 0x67, 0x3a, + 0xa6, 0xa1, 0x51, 0x18, 0xbe, 0x82, 0x8a, 0x93, 0x60, 0xfe, 0xc6, 0x5f, 0x11, 0xf3, 0x6c, 0xfd, + 0x6b, 0x02, 0x0c, 0xac, 0x60, 0x7d, 0x34, 0x7e, 0x3f, 0x1a, 0xa8, 0x39, 0xf6, 0x8a, 0xa3, 0xb7, + 0xf9, 0x1a, 0x0c, 0xa6, 0x22, 0x1d, 0x06, 0xd2, 0x7c, 0x9c, 0xfa, 0x23, 0x19, 0x7f, 0xdb, 0x4e, + 0x4b, 0xd4, 0xbb, 0x70, 0x35, 0x0f, 0x75, 0x17, 0x59, 0x09, 0x97, 0x81, 0x34, 0x1f, 0x87, 0xaf, + 0xa0, 0xfe, 0x69, 0xc7, 0xb1, 0x1d, 0xd1, 0xd8, 0x9c, 0x50, 0x80, 0xf8, 0xdc, 0x0b, 0x14, 0xf8, + 0x45, 0x34, 0xc4, 0xe6, 0x1c, 0x76, 0xa3, 0x99, 0xeb, 0xf5, 0x52, 0x2a, 0x52, 0xaa, 0x5f, 0xcb, + 0x09, 0x7b, 0x36, 0x26, 0xf1, 0xc7, 0xf0, 0x55, 0xe0, 0x06, 0xca, 0x4d, 0xd4, 0x66, 0xf8, 0x04, + 0x78, 0xd2, 0x2f, 0x2a, 0xa8, 0x4a, 0xa4, 0x1c, 0xa5, 0xc6, 0xe7, 0x51, 0xbe, 0x46, 0xd5, 0xa7, + 0x00, 0xea, 0x51, 0xdc, 0xda, 0x2c, 0xe7, 0x3b, 0x54, 0x7f, 0x00, 0x0a, 0x58, 0x7a, 0x98, 0x61, + 0x27, 0x26, 0x86, 0x0d, 0xcf, 0x31, 0xe7, 0x51, 0xbe, 0xe2, 0xac, 0x3c, 0xe4, 0xb3, 0x16, 0x60, + 0x75, 0x67, 0xe5, 0xa1, 0x06, 0x50, 0x7c, 0x0d, 0x21, 0x8d, 0x78, 0x5d, 0xc7, 0x02, 0x3f, 0x90, + 0x41, 0xb8, 0x7f, 0x83, 0xd9, 0xd0, 0x01, 0x68, 0xa3, 0x69, 0x1b, 0x44, 0x13, 0x48, 0xd4, 0x9f, + 0x0e, 0x1f, 0x76, 0xa6, 0x4c, 0x77, 0xed, 0xb8, 0x0b, 0x77, 0xd1, 0x85, 0x3a, 0xbf, 0xe2, 0x8c, + 0x77, 0x52, 0x19, 0xf5, 0xcf, 0xb4, 0xf4, 0x15, 0x17, 0xfa, 0x90, 0xdb, 0x92, 0x3d, 0xa0, 0x00, + 0x8d, 0xc1, 0x23, 0xfd, 0x54, 0xdc, 0xbe, 0x9f, 0x7e, 0xb4, 0x3f, 0x18, 0x6d, 0x0b, 0xc4, 0x5b, + 0xb7, 0x9d, 0xe3, 0xae, 0xda, 0x69, 0x57, 0x5d, 0x42, 0x03, 0x75, 0xa7, 0x29, 0x5c, 0x5d, 0xc0, + 0x79, 0xc0, 0x75, 0x9a, 0xec, 0xda, 0xc2, 0x47, 0x52, 0xba, 0x29, 0xd7, 0x03, 0xba, 0x81, 0x90, + 0xce, 0x70, 0x3d, 0x4e, 0xc7, 0x91, 0x9c, 0xae, 0x66, 0x3b, 0x1e, 0xef, 0xb8, 0x80, 0xae, 0x63, + 0x3b, 0x9e, 0xe6, 0x23, 0xf1, 0x07, 0x11, 0x5a, 0x9a, 0xac, 0xf9, 0xc6, 0xf6, 0x83, 0xa1, 0x2d, + 0x20, 0xb7, 0xb2, 0xd7, 0x04, 0x34, 0x5e, 0x42, 0x83, 0x8b, 0x1d, 0xe2, 0xb0, 0xa3, 0x10, 0xf3, + 0xec, 0xf8, 0x40, 0x44, 0xb4, 0xbc, 0xdf, 0xc7, 0xf9, 0xff, 0x01, 0x39, 0x5b, 0x5f, 0x6c, 0xff, + 0xa7, 0x16, 0x32, 0xc2, 0x2f, 0xa2, 0x42, 0x85, 0xed, 0xf3, 0x86, 0x80, 0x65, 0x20, 0x32, 0x38, + 0x82, 0x32, 0x14, 0x3b, 0xb3, 0xeb, 0xf0, 0xb7, 0xc6, 0xc9, 0xd5, 0x2b, 0xa8, 0x14, 0xad, 0x06, + 0x0f, 0xa1, 0x81, 0xc9, 0xc5, 0x85, 0x85, 0xe9, 0xc9, 0xa5, 0x52, 0x1f, 0x2e, 0xa2, 0x7c, 0x7d, + 0x7a, 0x61, 0xaa, 0x94, 0x51, 0x7f, 0x4e, 0x98, 0x41, 0xa8, 0x6a, 0x1d, 0x3f, 0x0d, 0xef, 0xeb, + 0xbd, 0xa5, 0x04, 0xef, 0xa1, 0x70, 0x63, 0xd0, 0x36, 0x3d, 0x8f, 0x18, 0x7c, 0x95, 0x80, 0xf7, + 0x42, 0xef, 0x91, 0x16, 0xc3, 0xe3, 0xab, 0x68, 0x04, 0x60, 0xfc, 0x89, 0x90, 0x9d, 0x8f, 0x79, + 0x01, 0xe7, 0x91, 0x26, 0x23, 0xd5, 0x6f, 0x84, 0xaf, 0xc3, 0x73, 0x44, 0x3f, 0xaa, 0x2f, 0x8a, + 0xef, 0x91, 0xfe, 0x52, 0xff, 0x22, 0xcf, 0x5c, 0x40, 0x98, 0xe3, 0xde, 0x61, 0x88, 0x32, 0xbc, + 0xd2, 0xcd, 0xed, 0xe2, 0x4a, 0xf7, 0x2a, 0x2a, 0xcc, 0x13, 0x6f, 0xd5, 0xf6, 0x0d, 0xbf, 0xc0, + 0x42, 0xaf, 0x0d, 0x10, 0xd1, 0x42, 0x8f, 0xd1, 0xe0, 0x35, 0x84, 0x7d, 0xaf, 0xbc, 0xc0, 0x10, + 0xdb, 0xbf, 0x42, 0x3e, 0x1b, 0x3b, 0xa7, 0xd4, 0xc1, 0x25, 0x17, 0x6c, 0xec, 0x4f, 0x05, 0x86, + 0xde, 0x82, 0x25, 0xd6, 0x9f, 0x6f, 0x96, 0x0b, 0x8c, 0x46, 0x4b, 0x60, 0x8b, 0xdf, 0x40, 0x83, + 0xf3, 0x33, 0x15, 0xee, 0xa1, 0xc7, 0xac, 0x22, 0x9e, 0x08, 0xa4, 0xe8, 0x23, 0x02, 0x91, 0x80, + 0xbf, 0x4d, 0xfb, 0x81, 0x1e, 0x77, 0xd0, 0x0b, 0xb9, 0x50, 0x6d, 0x61, 0x9e, 0x3b, 0xfc, 0x76, + 0x21, 0xd0, 0x16, 0xd9, 0x9f, 0x27, 0x2a, 0x2b, 0x86, 0x8d, 0x68, 0x4b, 0x71, 0x1f, 0xa3, 0x7b, + 0x11, 0x8d, 0x55, 0x3a, 0x9d, 0x96, 0x49, 0x0c, 0xd0, 0x17, 0xad, 0xdb, 0x22, 0x2e, 0x37, 0xf9, + 0x01, 0x67, 0x10, 0x9d, 0x21, 0x1b, 0xe0, 0x17, 0xda, 0x70, 0xba, 0xb2, 0x7d, 0x66, 0xbc, 0xac, + 0xfa, 0x5f, 0x33, 0xa8, 0xe4, 0x1b, 0x4f, 0x8b, 0x1e, 0xa9, 0x82, 0x65, 0x2f, 0x5c, 0xc3, 0x44, + 0x6c, 0x49, 0x01, 0x8f, 0xeb, 0x68, 0x60, 0xfa, 0x51, 0xc7, 0x74, 0x88, 0xbb, 0x03, 0x43, 0xd8, + 0x0b, 0xfc, 0xc8, 0x39, 0x46, 0x58, 0x91, 0xd8, 0x69, 0x93, 0x81, 0xc1, 0x25, 0x8a, 0x99, 0x8f, + 0x4f, 0xf8, 0x6e, 0xb6, 0xcc, 0x25, 0x8a, 0x9b, 0x99, 0x4b, 0x3e, 0x6e, 0x21, 0x29, 0x7e, 0x06, + 0xe5, 0x96, 0x96, 0xe6, 0xb8, 0x36, 0x82, 0x7b, 0xb3, 0xe7, 0x89, 0x3e, 0x5f, 0x14, 0xab, 0xfe, + 0x61, 0x16, 0x21, 0xaa, 0xf4, 0x93, 0x0e, 0xd1, 0x0f, 0xe9, 0x31, 0x67, 0x02, 0x15, 0x7d, 0x81, + 0xf3, 0x01, 0x17, 0x58, 0x3e, 0x47, 0x3b, 0x22, 0x5a, 0x77, 0x60, 0xe5, 0x5e, 0xf6, 0x8d, 0x71, + 0xd9, 0x5d, 0x2a, 0xec, 0x0e, 0xc1, 0x18, 0xd7, 0x37, 0xc1, 0xfd, 0x20, 0x1a, 0xe4, 0x5a, 0x63, + 0x4b, 0x77, 0xa8, 0x4d, 0x1f, 0xa8, 0x85, 0xf8, 0x88, 0x7a, 0x16, 0xf6, 0x31, 0x99, 0x7d, 0x81, + 0x8b, 0x97, 0x99, 0xe9, 0x1f, 0x59, 0xf1, 0x1e, 0xd8, 0x05, 0x97, 0xfa, 0xfb, 0x19, 0x84, 0x69, + 0xb3, 0x6a, 0xba, 0xeb, 0xae, 0xdb, 0x8e, 0xc1, 0x2c, 0x50, 0x0f, 0x45, 0x30, 0x07, 0xf7, 0x28, + 0xf1, 0xb5, 0x22, 0x3a, 0x29, 0x59, 0xf7, 0x1d, 0xf1, 0xd1, 0x74, 0x45, 0x1e, 0x4d, 0xbd, 0x4c, + 0xdb, 0xdf, 0x27, 0xbe, 0x7a, 0xf4, 0x4b, 0x5e, 0x26, 0xc2, 0x73, 0xc7, 0x73, 0x68, 0x98, 0xff, + 0xa0, 0x8b, 0xa5, 0x7f, 0x9d, 0x0d, 0xa3, 0xd4, 0xa5, 0x00, 0x4d, 0x42, 0xe3, 0x0f, 0xa3, 0x41, + 0x3a, 0x60, 0x56, 0xc0, 0x55, 0x7f, 0x20, 0x34, 0x1b, 0x37, 0x7c, 0xa0, 0x38, 0xe1, 0x05, 0x94, + 0x82, 0xb3, 0x40, 0x71, 0x07, 0xce, 0x02, 0x6f, 0xa1, 0xa1, 0x8a, 0x65, 0xd9, 0x1e, 0xec, 0xc4, + 0x5d, 0x7e, 0xff, 0x98, 0xba, 0xf4, 0x3e, 0x03, 0x1e, 0xb0, 0x21, 0x7d, 0xe2, 0xda, 0x2b, 0x32, + 0xc4, 0xd7, 0x7d, 0xd3, 0x77, 0xe2, 0x70, 0xd3, 0x51, 0xb8, 0x83, 0x75, 0x38, 0x2c, 0x6e, 0xf9, + 0x0e, 0x9d, 0x37, 0x52, 0x73, 0xec, 0x8e, 0xed, 0x12, 0x83, 0x09, 0x6a, 0x28, 0xf4, 0x27, 0xee, + 0x70, 0x04, 0x38, 0xab, 0x48, 0x6e, 0xf3, 0x52, 0x11, 0xfc, 0x00, 0x9d, 0xf2, 0x5f, 0x83, 0x02, + 0xb7, 0xa0, 0xea, 0x94, 0xab, 0x0c, 0x83, 0xeb, 0x01, 0x8e, 0x2a, 0x43, 0x75, 0x6a, 0xe2, 0xa2, + 0x7f, 0xf7, 0xe9, 0xfb, 0x15, 0x35, 0x4c, 0x43, 0xec, 0xea, 0x44, 0x7e, 0xf8, 0xbb, 0xd0, 0xd0, + 0xbc, 0xfe, 0x68, 0xaa, 0xcb, 0x0f, 0x58, 0x23, 0x3b, 0xbf, 0x62, 0x6d, 0xeb, 0x8f, 0x1a, 0x06, + 0x2f, 0x17, 0x59, 0xf4, 0x44, 0x96, 0xb8, 0x81, 0xce, 0xd4, 0x1c, 0xbb, 0x6d, 0x7b, 0xc4, 0x88, + 0x78, 0xd8, 0x9c, 0x08, 0x5d, 0xf2, 0x3a, 0x9c, 0xa2, 0xd1, 0xc3, 0xd5, 0x26, 0x85, 0x0d, 0x6e, + 0xa3, 0x13, 0x15, 0xd7, 0xed, 0xb6, 0x49, 0x78, 0x0d, 0x5d, 0xda, 0xf6, 0x33, 0x3e, 0xc0, 0x4d, + 0x13, 0x9f, 0xd4, 0xa1, 0x28, 0xbb, 0x85, 0x6e, 0x78, 0xa6, 0x58, 0x23, 0x7c, 0x4b, 0x94, 0xf7, + 0xed, 0x7c, 0x71, 0xb4, 0x74, 0x42, 0x3b, 0x1b, 0x6f, 0xcc, 0x92, 0xe9, 0xb5, 0x88, 0xfa, 0x1b, + 0x19, 0x84, 0x42, 0x01, 0xe3, 0xe7, 0xe4, 0x78, 0x20, 0x99, 0xf0, 0x36, 0x93, 0xbb, 0x28, 0x4b, + 0x01, 0x40, 0xf0, 0x79, 0x94, 0x07, 0x37, 0xf6, 0x6c, 0x78, 0x7b, 0xb2, 0x66, 0x5a, 0x86, 0x06, + 0x50, 0x8a, 0x15, 0xfc, 0x4d, 0x01, 0x0b, 0x2f, 0x77, 0x6c, 0xdb, 0x32, 0x85, 0x4e, 0xd4, 0xbb, + 0xcb, 0x7e, 0xdd, 0x82, 0xf3, 0x0c, 0x78, 0xd3, 0xbb, 0xdd, 0xe5, 0xc0, 0xe3, 0x4c, 0x8a, 0x55, + 0x20, 0x17, 0x51, 0xbf, 0x94, 0x89, 0xcc, 0x82, 0x87, 0xb8, 0xe8, 0xbd, 0x2f, 0xfe, 0x18, 0x1b, + 0x9f, 0x96, 0xd4, 0xbf, 0x97, 0x45, 0x43, 0x35, 0xdb, 0xf1, 0x78, 0x5c, 0x80, 0xa3, 0xbd, 0x0a, + 0x09, 0xc7, 0x96, 0xfc, 0x2e, 0x8e, 0x2d, 0xe7, 0x51, 0x5e, 0xb0, 0x43, 0x64, 0x97, 0x9f, 0x86, + 0xe1, 0x68, 0x00, 0x55, 0xbf, 0x3b, 0x8b, 0xd0, 0xc7, 0x9f, 0x7f, 0xfe, 0x31, 0x16, 0x90, 0xfa, + 0x77, 0x32, 0xe8, 0x04, 0xbf, 0x8d, 0x17, 0x22, 0xeb, 0x0c, 0xf8, 0xef, 0x28, 0xe2, 0xb8, 0x64, + 0x20, 0xcd, 0xc7, 0xd1, 0x25, 0x60, 0xfa, 0x91, 0xe9, 0xc1, 0x85, 0xa4, 0x10, 0x5a, 0x87, 0x70, + 0x98, 0xb8, 0x04, 0xf8, 0x74, 0xf8, 0x39, 0xff, 0x9d, 0x21, 0x17, 0xae, 0x7b, 0xb4, 0xc0, 0x74, + 0xe2, 0x5b, 0x83, 0xfa, 0x2b, 0x79, 0x94, 0x9f, 0x7e, 0x44, 0x9a, 0x47, 0xbc, 0x6b, 0x84, 0xdb, + 0x8b, 0xfc, 0x3e, 0x6f, 0x2f, 0xf6, 0xf2, 0x70, 0xfa, 0x5a, 0xd8, 0x9f, 0x05, 0xb9, 0xfa, 0x48, + 0xcf, 0x47, 0xab, 0xf7, 0x7b, 0xfa, 0xe8, 0xbd, 0xbb, 0xff, 0xb3, 0x1c, 0xca, 0xd5, 0x27, 0x6b, + 0xc7, 0x7a, 0x73, 0xa8, 0x7a, 0xd3, 0xfb, 0x61, 0x4a, 0x0d, 0xee, 0x9a, 0x8b, 0xa1, 0x29, 0x58, + 0xe4, 0x5a, 0xf9, 0xdb, 0x39, 0x34, 0x5a, 0x9f, 0x59, 0xaa, 0x09, 0xd7, 0x3d, 0x77, 0x98, 0xb9, + 0x0e, 0x18, 0x8e, 0xb0, 0x2e, 0x3d, 0x1f, 0xdb, 0xcf, 0xdc, 0xad, 0x5a, 0xde, 0x0b, 0x37, 0xef, + 0xe9, 0xad, 0x2e, 0x81, 0xbb, 0x01, 0x66, 0xdc, 0xe7, 0x9a, 0xef, 0x90, 0x1f, 0x07, 0xef, 0x5e, + 0x9f, 0x01, 0xfe, 0x28, 0xca, 0xdd, 0xe5, 0xcf, 0xae, 0x69, 0x7c, 0x6e, 0x5c, 0x67, 0x7c, 0xe8, + 0x24, 0x98, 0xeb, 0x9a, 0x06, 0x70, 0xa0, 0xa5, 0x68, 0xe1, 0x5b, 0x7c, 0x01, 0xde, 0x51, 0xe1, + 0x15, 0xbf, 0xf0, 0xad, 0xea, 0x14, 0xae, 0xa3, 0xa1, 0x1a, 0x71, 0xda, 0x26, 0x74, 0x94, 0x3f, + 0x67, 0xf7, 0x66, 0x42, 0x4f, 0x2a, 0x43, 0x9d, 0xb0, 0x10, 0x30, 0x13, 0xb9, 0xe0, 0x37, 0x11, + 0x62, 0x7b, 0x94, 0x1d, 0x46, 0x6b, 0xbb, 0x00, 0xfb, 0x7e, 0xb6, 0xb5, 0x4c, 0xd8, 0xe3, 0x09, + 0xcc, 0xf0, 0x1a, 0x2a, 0xcd, 0xdb, 0x86, 0xf9, 0xc0, 0x64, 0xf6, 0x55, 0x50, 0x41, 0x61, 0x7b, + 0xab, 0x06, 0xba, 0x95, 0x6c, 0x0b, 0xe5, 0x92, 0xaa, 0x89, 0x31, 0x56, 0xff, 0x49, 0x3f, 0xca, + 0xd3, 0x6e, 0x3f, 0x1e, 0xbf, 0xfb, 0x19, 0xbf, 0x15, 0x54, 0xba, 0x6f, 0x3b, 0x6b, 0xa6, 0xb5, + 0x12, 0x98, 0xbe, 0xf2, 0xb3, 0x29, 0x3c, 0xd7, 0xaf, 0x33, 0x5c, 0x23, 0xb0, 0x92, 0xd5, 0x62, + 0xe4, 0xdb, 0x8c, 0xe0, 0x97, 0x10, 0x62, 0x0e, 0xad, 0x40, 0x53, 0x0c, 0x3d, 0xd2, 0x99, 0xbb, + 0x2b, 0x58, 0xd3, 0x8a, 0x1e, 0xe9, 0x21, 0x31, 0x3d, 0x84, 0xb3, 0x07, 0xcf, 0x41, 0x30, 0xae, + 0x85, 0x43, 0x38, 0x3c, 0x78, 0x8a, 0x9b, 0x00, 0xf6, 0xf4, 0x59, 0x43, 0x48, 0xb8, 0x44, 0x46, + 0x11, 0x41, 0x48, 0x93, 0x03, 0x8f, 0x01, 0x95, 0x70, 0x87, 0xac, 0x09, 0x3c, 0xf0, 0x0b, 0x91, + 0x57, 0x2e, 0x2c, 0x71, 0x4b, 0x7d, 0xe4, 0x0a, 0xad, 0x24, 0x86, 0xb7, 0xb3, 0x92, 0x50, 0x3f, + 0x97, 0x45, 0x83, 0xf5, 0xee, 0xb2, 0xbb, 0xe1, 0x7a, 0xa4, 0x7d, 0xc4, 0xd5, 0xd8, 0x3f, 0x5e, + 0xe5, 0x13, 0x8f, 0x57, 0xcf, 0xf8, 0x42, 0x11, 0xee, 0x1d, 0x83, 0x2d, 0x9d, 0x2f, 0x8e, 0x5f, + 0xce, 0xa2, 0x12, 0xbb, 0x1d, 0x9f, 0x32, 0xdd, 0xe6, 0x01, 0x58, 0xec, 0x1e, 0xbe, 0x54, 0xf6, + 0xf7, 0xa2, 0xb4, 0x03, 0x3b, 0x68, 0xf5, 0xb3, 0x59, 0x34, 0x54, 0xe9, 0x7a, 0xab, 0x15, 0x0f, + 0x74, 0xeb, 0xb1, 0x3c, 0x9f, 0xfc, 0x4e, 0x06, 0x9d, 0xa0, 0x0d, 0x59, 0xb2, 0xd7, 0x88, 0x75, + 0x00, 0x17, 0x8f, 0xe2, 0x05, 0x62, 0x76, 0x8f, 0x17, 0x88, 0xbe, 0x2c, 0x73, 0xbb, 0x93, 0x25, + 0x5c, 0x97, 0x6b, 0x76, 0x8b, 0x1c, 0xed, 0xcf, 0x38, 0xc0, 0xeb, 0x72, 0x5f, 0x20, 0x07, 0x70, + 0x95, 0xf2, 0x9d, 0x21, 0x90, 0x1f, 0xcb, 0xa2, 0x53, 0x3c, 0x4c, 0x28, 0x3f, 0x1c, 0x1d, 0xeb, + 0x4a, 0xaa, 0x68, 0x8e, 0xb5, 0x86, 0x8b, 0xe6, 0x67, 0x72, 0xe8, 0x14, 0x04, 0x53, 0xa3, 0x7b, + 0xc6, 0xef, 0x80, 0x89, 0x12, 0x37, 0xe5, 0x17, 0x9a, 0xf9, 0x84, 0x17, 0x9a, 0x3f, 0xdf, 0x2c, + 0xbf, 0xb0, 0x62, 0x7a, 0xab, 0xdd, 0xe5, 0xf1, 0xa6, 0xdd, 0xbe, 0xb6, 0xe2, 0xe8, 0x0f, 0x4d, + 0xf6, 0x36, 0xa1, 0xb7, 0xae, 0x05, 0x11, 0xb7, 0xf5, 0x8e, 0xc9, 0x63, 0x71, 0xd7, 0x61, 0x23, + 0x46, 0xb9, 0xfa, 0x6f, 0x3b, 0x2e, 0x42, 0xb7, 0x6d, 0xd3, 0xe2, 0x56, 0x0d, 0x6c, 0x15, 0xae, + 0xd3, 0xcd, 0xeb, 0xdb, 0xb6, 0x69, 0x35, 0xa2, 0xa6, 0x0d, 0xbb, 0xad, 0x2f, 0x64, 0xad, 0x09, + 0xd5, 0xa8, 0xff, 0x26, 0x83, 0x9e, 0x90, 0xb5, 0xf8, 0x3b, 0x61, 0x61, 0xfb, 0xdb, 0x59, 0x74, + 0xfa, 0x16, 0x08, 0x27, 0x78, 0x65, 0x3e, 0x9e, 0xb7, 0xf8, 0xe0, 0x4c, 0x90, 0xcd, 0xf1, 0xc4, + 0xc5, 0x65, 0xf3, 0xaf, 0x32, 0xe8, 0xe4, 0x62, 0x75, 0x6a, 0xf2, 0x3b, 0x44, 0x6b, 0xe2, 0xdf, + 0x73, 0xb4, 0x7b, 0x1a, 0xbe, 0xa7, 0x5e, 0x99, 0x9f, 0xfb, 0x4e, 0xea, 0x1f, 0xe9, 0x7b, 0x8e, + 0x78, 0xff, 0xfc, 0x76, 0x01, 0x0d, 0xdd, 0xe9, 0x2e, 0x13, 0xfe, 0xe6, 0xf7, 0x58, 0x1f, 0xa8, + 0xaf, 0xa3, 0x21, 0x2e, 0x06, 0xb8, 0x8c, 0x12, 0x02, 0x8f, 0x70, 0x47, 0x52, 0xe6, 0xdb, 0x2d, + 0x12, 0xe1, 0xf3, 0x28, 0x7f, 0x8f, 0x38, 0xcb, 0xa2, 0x4d, 0xfe, 0x43, 0xe2, 0x2c, 0x6b, 0x00, + 0xc5, 0x73, 0xa1, 0xa9, 0x5c, 0xa5, 0x56, 0x85, 0x20, 0xd4, 0xfc, 0x1e, 0x0c, 0xa2, 0x6a, 0x07, + 0xe6, 0x04, 0x7a, 0xc7, 0x64, 0xe1, 0xab, 0x45, 0x7f, 0xa0, 0x68, 0x49, 0xbc, 0x80, 0xc6, 0xc4, + 0xf7, 0x64, 0x16, 0x81, 0xb9, 0x98, 0xc0, 0x2e, 0x29, 0xf6, 0x72, 0xbc, 0x28, 0x7e, 0x0d, 0x0d, + 0xfb, 0x40, 0x78, 0x19, 0x1f, 0x0c, 0xc3, 0x7e, 0x06, 0xac, 0x22, 0xe1, 0xdd, 0xa5, 0x02, 0x22, + 0x03, 0xb8, 0xdd, 0x41, 0x09, 0x0c, 0x22, 0x96, 0x06, 0x52, 0x01, 0xfc, 0x61, 0x60, 0xd0, 0xb1, + 0x2d, 0x97, 0xc0, 0x1b, 0xe0, 0x10, 0x18, 0xac, 0x83, 0x29, 0x9e, 0xc3, 0xe1, 0xcc, 0x2d, 0x41, + 0x22, 0xc3, 0x8b, 0x08, 0x85, 0x6f, 0x35, 0xdc, 0xf9, 0x6b, 0xd7, 0xaf, 0x48, 0x02, 0x0b, 0xf1, + 0x96, 0x75, 0x64, 0x2f, 0xb7, 0xac, 0xea, 0xef, 0x65, 0xd1, 0x50, 0xa5, 0xd3, 0x09, 0x86, 0xc2, + 0x73, 0xa8, 0x50, 0xe9, 0x74, 0xee, 0x6a, 0x55, 0x31, 0x0c, 0xa4, 0xde, 0xe9, 0x34, 0xba, 0x8e, + 0x29, 0x9a, 0xda, 0x30, 0x22, 0x3c, 0x89, 0x46, 0x2a, 0x9d, 0x4e, 0xad, 0xbb, 0xdc, 0x32, 0x9b, + 0x42, 0x54, 0x79, 0x96, 0x00, 0xa3, 0xd3, 0x69, 0x74, 0x00, 0x13, 0x4d, 0x2d, 0x20, 0x97, 0xc1, + 0x6f, 0x81, 0xcb, 0x34, 0x0f, 0x6a, 0xce, 0xc2, 0x26, 0xab, 0x41, 0x00, 0xc8, 0xb0, 0x6d, 0xe3, 0x01, 0x11, 0x0b, 0x94, 0x79, 0xde, 0x0f, 0x37, 0x4a, 0x2b, 0x8a, 0x05, 0x2f, 0x0f, 0x59, 0xe2, - 0x8f, 0xa2, 0xfe, 0x52, 0xbb, 0x2d, 0x5c, 0xe3, 0xc1, 0x5b, 0x2d, 0x2d, 0x15, 0xe9, 0x63, 0x9f, - 0x6c, 0xe2, 0x65, 0x34, 0x2a, 0x57, 0xb6, 0xa7, 0x40, 0x9b, 0xdf, 0xce, 0xc0, 0x07, 0x1d, 0x73, - 0x53, 0xb1, 0x1b, 0x28, 0x57, 0x6a, 0xb7, 0xf9, 0x7c, 0x74, 0x2a, 0xa1, 0x3f, 0xa2, 0xee, 0x23, - 0xa5, 0x76, 0xdb, 0xff, 0x74, 0x66, 0xaa, 0xfa, 0x68, 0x7d, 0xfa, 0xd7, 0xd8, 0xa7, 0x1f, 0x6f, - 0x7b, 0x50, 0xf5, 0x97, 0x73, 0x68, 0xac, 0xd4, 0x6e, 0x9f, 0x04, 0xe8, 0x3c, 0x2c, 0x27, 0x95, - 0xe7, 0x10, 0x12, 0xa6, 0xc7, 0xfe, 0xc0, 0x64, 0x7b, 0x48, 0x98, 0x1a, 0x95, 0x8c, 0x26, 0x10, - 0xf9, 0xea, 0x37, 0xb0, 0x27, 0xf5, 0xfb, 0x7c, 0x0e, 0xa6, 0xe2, 0xe3, 0xee, 0x70, 0xff, 0x41, - 0xe9, 0x36, 0xde, 0x07, 0x7d, 0x7b, 0xea, 0x83, 0xdf, 0x92, 0x06, 0x0f, 0x04, 0x7c, 0x3c, 0xe9, - 0x85, 0xde, 0x03, 0x6d, 0x8b, 0x47, 0x45, 0x61, 0x72, 0x2f, 0x60, 0x3f, 0x08, 0x3d, 0xf7, 0x49, - 0x6f, 0x50, 0x54, 0xdd, 0x34, 0xb4, 0x08, 0xad, 0xdf, 0x87, 0xfd, 0x7b, 0xea, 0xc3, 0xad, 0x2c, - 0xf8, 0x9d, 0x04, 0x3e, 0xed, 0x07, 0x3f, 0x5d, 0x5c, 0x43, 0x88, 0x3d, 0xe8, 0x04, 0xd6, 0x62, - 0x23, 0xcc, 0x7d, 0x95, 0xc5, 0xa6, 0xe7, 0xee, 0xab, 0x21, 0x49, 0xf0, 0xf0, 0x9c, 0x4b, 0x7c, - 0x78, 0xbe, 0x82, 0x06, 0x34, 0x7d, 0xe3, 0x8d, 0x0e, 0x71, 0x36, 0xf9, 0x76, 0x86, 0x85, 0x8c, - 0xd1, 0x37, 0xea, 0x9f, 0xa3, 0x40, 0x2d, 0x40, 0x63, 0x35, 0x70, 0x5c, 0x12, 0x1e, 0xda, 0xd8, - 0xed, 0x5e, 0xe0, 0xae, 0xb4, 0x1f, 0x45, 0xc7, 0x2f, 0xa1, 0x5c, 0xe9, 0x5e, 0x8d, 0x4b, 0x36, - 0xe8, 0xda, 0xd2, 0xbd, 0x1a, 0x97, 0x57, 0x6a, 0xd9, 0x7b, 0x35, 0xf5, 0xf3, 0x59, 0x84, 0xe3, - 0x94, 0xf8, 0x79, 0x34, 0x08, 0xd0, 0x55, 0xaa, 0x33, 0x62, 0x52, 0xa3, 0x0d, 0xb7, 0xee, 0x00, - 0x54, 0xda, 0xdc, 0xf9, 0xa4, 0xf8, 0x45, 0xc8, 0xdf, 0xc6, 0xd3, 0x6a, 0x48, 0x49, 0x8d, 0x36, - 0x5c, 0x3f, 0xe3, 0x59, 0x24, 0x7d, 0x1b, 0x27, 0x86, 0x7d, 0xe1, 0xbd, 0xda, 0x9c, 0xed, 0x7a, - 0x5c, 0xd4, 0x6c, 0x5f, 0xb8, 0xe1, 0x42, 0x36, 0x2d, 0x69, 0x5f, 0xc8, 0xc8, 0x20, 0x23, 0xc0, - 0xbd, 0x1a, 0xb3, 0xfe, 0x35, 0x34, 0xbb, 0xe9, 0x6f, 0x28, 0x59, 0x46, 0x80, 0x0d, 0xb7, 0xce, - 0x2c, 0x87, 0x0d, 0x48, 0x1c, 0x27, 0x65, 0x04, 0x90, 0x4a, 0xa9, 0x5f, 0x1c, 0x40, 0x85, 0xb2, - 0xee, 0xe9, 0x2b, 0xba, 0x4b, 0x84, 0xd3, 0xf4, 0x98, 0x0f, 0xf3, 0x3f, 0x47, 0x90, 0x83, 0xb1, - 0x92, 0xf0, 0x35, 0xd1, 0x02, 0xf8, 0x13, 0x21, 0xdf, 0x20, 0x5f, 0x93, 0x98, 0x00, 0x62, 0xa5, - 0xde, 0xe6, 0x60, 0x2d, 0x46, 0x88, 0xaf, 0xa2, 0x21, 0x1f, 0x46, 0x0f, 0x00, 0xb9, 0x50, 0x67, - 0x8c, 0x15, 0xba, 0xff, 0xd7, 0x44, 0x34, 0x7e, 0x11, 0x0d, 0xfb, 0x3f, 0x85, 0xad, 0x35, 0xcb, - 0x66, 0xb1, 0x12, 0x3b, 0x3d, 0x89, 0xa4, 0x62, 0x51, 0x98, 0xdf, 0x7a, 0xa5, 0xa2, 0x91, 0x84, - 0x11, 0x12, 0x29, 0xfe, 0x1c, 0x1a, 0xf5, 0x7f, 0xf3, 0x03, 0x03, 0xcb, 0xad, 0x71, 0x35, 0xc8, - 0x4b, 0x17, 0x11, 0xeb, 0xa4, 0x4c, 0xce, 0x8e, 0x0e, 0x8f, 0xfb, 0x39, 0x10, 0x8c, 0x95, 0xf8, - 0xc9, 0x21, 0x52, 0x01, 0xae, 0xa0, 0x71, 0x1f, 0x12, 0x6a, 0x68, 0x7f, 0x78, 0x62, 0x34, 0x56, - 0xea, 0x89, 0x4a, 0x1a, 0x2f, 0x85, 0x9b, 0xe8, 0xbc, 0x04, 0x34, 0xdc, 0x35, 0xf3, 0xbe, 0xc7, - 0x8f, 0x7b, 0x3c, 0x7e, 0x1b, 0x4f, 0x7a, 0x13, 0x70, 0x65, 0x34, 0x7e, 0xf6, 0x2a, 0x39, 0xb2, - 0x7e, 0x57, 0x6e, 0xb8, 0x86, 0x4e, 0xfb, 0xf8, 0x5b, 0xd3, 0xd5, 0xaa, 0x63, 0xbf, 0x43, 0x1a, - 0x5e, 0xa5, 0xcc, 0x8f, 0xcb, 0x10, 0xd7, 0xc3, 0x58, 0xa9, 0xaf, 0x36, 0xda, 0x54, 0x29, 0x28, - 0x4e, 0x66, 0x9e, 0x58, 0x18, 0xdf, 0x45, 0x67, 0x04, 0x78, 0xc5, 0x72, 0x3d, 0xdd, 0x6a, 0x90, - 0x4a, 0x99, 0x9f, 0xa1, 0xe1, 0x3c, 0xcf, 0xb9, 0x9a, 0x1c, 0x29, 0xb3, 0x4d, 0x2e, 0x8e, 0x5f, - 0x46, 0x23, 0x3e, 0x82, 0xbd, 0x7f, 0x0c, 0xc1, 0xfb, 0x07, 0x0c, 0x49, 0x63, 0xa5, 0x1e, 0x75, - 0x52, 0x91, 0x89, 0x45, 0x8d, 0x82, 0xb4, 0xa0, 0xc3, 0x92, 0x46, 0x79, 0x9b, 0xed, 0x44, 0x65, - 0x84, 0x54, 0xa1, 0xaf, 0x86, 0x1a, 0xb5, 0xe4, 0x98, 0xab, 0x26, 0x3b, 0x49, 0xfb, 0x7e, 0x29, - 0x2b, 0x75, 0x1b, 0x80, 0x49, 0xfa, 0xc1, 0xc8, 0x27, 0x4a, 0xe8, 0x54, 0x82, 0x8e, 0xed, 0xe9, - 0xc4, 0xf8, 0x85, 0x6c, 0xd8, 0x88, 0x63, 0x7e, 0x6c, 0x9c, 0x42, 0x03, 0xfe, 0x97, 0xf0, 0xcd, - 0x83, 0x92, 0x36, 0x34, 0xa3, 0x3c, 0x7c, 0xbc, 0x24, 0x8e, 0x63, 0x7e, 0x94, 0x3c, 0x0c, 0x71, - 0xbc, 0x97, 0x09, 0xc5, 0x71, 0xcc, 0x8f, 0x97, 0x7f, 0x3d, 0x1f, 0xce, 0x49, 0x27, 0x67, 0xcc, - 0xc3, 0xda, 0x26, 0x87, 0xe6, 0x45, 0x7d, 0x7b, 0xf0, 0x0f, 0x11, 0x55, 0xb3, 0x7f, 0x7f, 0xaa, - 0x89, 0x5f, 0x46, 0x43, 0x55, 0xdb, 0xf5, 0x56, 0x1d, 0xe2, 0x56, 0x83, 0xf8, 0xa3, 0xe0, 0x5b, - 0xd4, 0xe6, 0xe0, 0x7a, 0x5b, 0x9a, 0xfd, 0x45, 0x72, 0xf5, 0x0f, 0x72, 0x31, 0x6d, 0x60, 0x1b, - 0xd7, 0x63, 0xa9, 0x0d, 0x87, 0x30, 0xd4, 0xf1, 0xf5, 0x70, 0x15, 0x64, 0x3b, 0xfc, 0x5e, 0x21, - 0xb8, 0xca, 0x0a, 0xdf, 0xe0, 0xcb, 0x24, 0xf8, 0xd3, 0xe8, 0x9c, 0x04, 0xa8, 0xea, 0x8e, 0xde, - 0x22, 0x5e, 0x98, 0xeb, 0x05, 0xdc, 0xe5, 0xfd, 0xd2, 0xf5, 0x76, 0x80, 0x16, 0xf3, 0xc7, 0xa4, - 0x70, 0x10, 0x54, 0xab, 0x7f, 0x0f, 0x96, 0x6b, 0xff, 0x39, 0x8b, 0x46, 0x82, 0x8e, 0xd6, 0x1d, - 0x97, 0x3c, 0xba, 0x3d, 0xfa, 0x71, 0x34, 0x02, 0xde, 0x9b, 0x2d, 0x62, 0x79, 0x42, 0x52, 0x45, - 0x16, 0xf0, 0xd1, 0x47, 0xf0, 0xd8, 0xbe, 0x12, 0x21, 0x2e, 0xa2, 0x5e, 0xa6, 0x03, 0x82, 0x4f, - 0x2d, 0x53, 0x00, 0x06, 0x57, 0x7f, 0x22, 0x87, 0x86, 0x7d, 0x29, 0x4f, 0x99, 0xc7, 0xf5, 0xc6, - 0xe7, 0x68, 0x85, 0x7c, 0x0d, 0xa1, 0xaa, 0xed, 0x78, 0x7a, 0x53, 0x48, 0xcd, 0x0e, 0x47, 0xa5, - 0x36, 0x40, 0x59, 0x19, 0x81, 0x04, 0x4f, 0x22, 0x24, 0x0c, 0xb0, 0x7e, 0x18, 0x60, 0xa3, 0xdb, - 0x5b, 0x45, 0x14, 0x8e, 0x2b, 0x4d, 0xa0, 0x50, 0x7f, 0x3d, 0x8b, 0xc6, 0xfc, 0x4e, 0x9a, 0x79, - 0x48, 0x1a, 0x1d, 0xef, 0x11, 0x1e, 0x0c, 0xb2, 0xb4, 0x7b, 0x77, 0x94, 0xb6, 0xfa, 0x3f, 0x84, - 0x89, 0x64, 0xba, 0x69, 0x9f, 0x4c, 0x24, 0x7f, 0x11, 0x3a, 0xae, 0x7e, 0x6f, 0x0e, 0x9d, 0xf6, - 0xa5, 0x3e, 0xdb, 0xb1, 0x60, 0x93, 0x31, 0xad, 0x37, 0x9b, 0x8f, 0xf2, 0xba, 0x3c, 0xe4, 0x0b, - 0x62, 0x89, 0x87, 0x43, 0xe0, 0x71, 0xd6, 0xef, 0x73, 0x70, 0xdd, 0x36, 0x0d, 0x4d, 0x24, 0xc2, - 0xaf, 0xa2, 0x61, 0xff, 0x67, 0xc9, 0x59, 0xf5, 0x17, 0x63, 0xb8, 0x32, 0x08, 0x0a, 0xe9, 0x8e, - 0xe4, 0xf5, 0x21, 0x15, 0x50, 0xff, 0x4b, 0x1f, 0x9a, 0xb8, 0x67, 0x5a, 0x86, 0xbd, 0xe1, 0xfa, - 0x61, 0xfa, 0x8f, 0xfd, 0x96, 0xf9, 0xa8, 0xc3, 0xf3, 0xbf, 0x81, 0xce, 0x44, 0x45, 0xea, 0x04, - 0xc1, 0x93, 0x78, 0xef, 0x6c, 0x30, 0x82, 0xba, 0x1f, 0xb0, 0x9f, 0xdf, 0xbb, 0x69, 0xc9, 0x25, - 0xa3, 0x11, 0xff, 0xfb, 0x77, 0x13, 0xf1, 0xff, 0x19, 0xd4, 0x57, 0xb6, 0x5b, 0xba, 0xe9, 0xfb, - 0xff, 0xc1, 0x28, 0x0e, 0xea, 0x05, 0x8c, 0xc6, 0x29, 0x28, 0x7f, 0x5e, 0x31, 0x74, 0xd9, 0x60, - 0xc8, 0xdf, 0x2f, 0xd0, 0x71, 0x89, 0xa3, 0x89, 0x44, 0xd8, 0x46, 0x23, 0xbc, 0x3a, 0x7e, 0x4b, - 0x86, 0xe0, 0x96, 0x2c, 0xc8, 0xab, 0x98, 0xae, 0x56, 0x93, 0x52, 0x39, 0x76, 0x5d, 0xc6, 0x12, - 0x11, 0xf0, 0x8f, 0x61, 0xf7, 0x65, 0x9a, 0xcc, 0x5f, 0x10, 0x02, 0x4c, 0x32, 0x43, 0x71, 0x21, - 0xc0, 0x2c, 0x23, 0x12, 0xe1, 0x19, 0x34, 0x5e, 0x6a, 0x36, 0xed, 0x8d, 0x20, 0x4a, 0x11, 0x55, - 0x89, 0x61, 0x88, 0xd4, 0x0a, 0x97, 0x2f, 0x3a, 0x45, 0xc2, 0xc7, 0xd5, 0x1b, 0x1c, 0xad, 0xc5, - 0x4b, 0x4c, 0xbc, 0x86, 0x70, 0xbc, 0xcd, 0x7b, 0xba, 0x7e, 0xf9, 0x62, 0x16, 0xe1, 0xc8, 0x39, - 0x64, 0xe6, 0x11, 0xde, 0x4e, 0xa9, 0x3f, 0x9f, 0x41, 0xe3, 0xb1, 0xe8, 0x61, 0xf8, 0x06, 0x42, - 0x0c, 0x22, 0x44, 0xad, 0x00, 0x37, 0xb0, 0x30, 0xa2, 0x18, 0x5f, 0x4a, 0x42, 0x32, 0x7c, 0x0d, - 0x0d, 0xb0, 0x5f, 0x41, 0x9a, 0xd0, 0x68, 0x91, 0x4e, 0xc7, 0x34, 0xb4, 0x80, 0x28, 0xac, 0x05, - 0xee, 0xf1, 0x72, 0x89, 0x45, 0xbc, 0xcd, 0x76, 0x50, 0x0b, 0x25, 0xa3, 0x1d, 0x38, 0x1c, 0x34, - 0xb8, 0x64, 0x1c, 0x55, 0xd7, 0xf5, 0xf1, 0x40, 0x6c, 0xb9, 0x9d, 0x02, 0xb1, 0x45, 0xe6, 0x26, - 0x1e, 0x79, 0xed, 0xf0, 0x8c, 0x4b, 0xbf, 0x9c, 0x45, 0x63, 0x41, 0xad, 0x47, 0x78, 0x65, 0xf4, - 0x01, 0x12, 0xc9, 0x97, 0x32, 0x48, 0x99, 0x32, 0x9b, 0x4d, 0xd3, 0x5a, 0xad, 0x58, 0xf7, 0x6d, - 0xa7, 0x05, 0x93, 0xc7, 0xd1, 0xdd, 0x2e, 0xaa, 0x3f, 0x90, 0x41, 0xe3, 0xbc, 0x41, 0xd3, 0xba, - 0x63, 0x1c, 0xdd, 0xb5, 0x6f, 0xb4, 0x25, 0x47, 0xa7, 0x2f, 0xea, 0x57, 0xb3, 0x08, 0xcd, 0xdb, - 0x8d, 0xf5, 0x63, 0x6e, 0x41, 0xff, 0x89, 0x9d, 0xb3, 0xe3, 0x16, 0xe4, 0xec, 0xb8, 0x4a, 0xc6, - 0xcf, 0x8f, 0x4b, 0x2b, 0xa5, 0x74, 0x7c, 0x57, 0x13, 0x54, 0x2a, 0xa6, 0xdf, 0x65, 0x95, 0x6e, - 0x6f, 0x15, 0xf3, 0x4d, 0xbb, 0xb1, 0xae, 0x01, 0xbd, 0xfa, 0xe7, 0x19, 0x26, 0xbb, 0x63, 0x6e, - 0x61, 0xef, 0x7f, 0x7e, 0x7e, 0x8f, 0x9f, 0xff, 0xd7, 0x32, 0xe8, 0xb4, 0x46, 0x1a, 0xf6, 0x03, - 0xe2, 0x6c, 0x4e, 0xdb, 0x06, 0xb9, 0x45, 0x2c, 0xe2, 0x1c, 0xd5, 0x88, 0xfa, 0xc7, 0x10, 0x6a, - 0x32, 0x6c, 0xcc, 0x1d, 0x97, 0x18, 0xc7, 0x27, 0xe0, 0xa9, 0xfa, 0x8f, 0xfa, 0x91, 0x92, 0xb8, - 0x43, 0x3c, 0xb6, 0xbb, 0xa2, 0xd4, 0x6d, 0x7f, 0xfe, 0xb0, 0xb6, 0xfd, 0xbd, 0x7b, 0xdb, 0xf6, - 0xf7, 0xed, 0x75, 0xdb, 0xdf, 0xbf, 0x9b, 0x6d, 0x7f, 0x2b, 0xba, 0xed, 0x1f, 0x80, 0x6d, 0xff, - 0x8d, 0xae, 0xdb, 0xfe, 0x19, 0xcb, 0xd8, 0xe7, 0xa6, 0xff, 0xd8, 0xa6, 0xf9, 0xd8, 0xcf, 0x69, - 0xe5, 0x32, 0x9d, 0x14, 0x1b, 0xb6, 0x63, 0x10, 0x83, 0x1f, 0x52, 0xe0, 0x56, 0xde, 0xe1, 0x30, - 0x2d, 0xc0, 0xc6, 0x72, 0xa6, 0x8c, 0xec, 0x26, 0x67, 0xca, 0x21, 0x1c, 0x63, 0xbe, 0x90, 0x45, - 0xe3, 0xd3, 0xc4, 0xf1, 0x58, 0x3c, 0x91, 0xc3, 0x78, 0x48, 0x2e, 0xa1, 0x31, 0x81, 0x21, 0xec, - 0xc8, 0x85, 0x5c, 0xff, 0x0d, 0xe2, 0x78, 0xd1, 0xb7, 0xf5, 0x28, 0x3d, 0xad, 0xde, 0x8f, 0x5b, - 0xcc, 0xc7, 0x6e, 0x50, 0xbd, 0x0f, 0x67, 0x82, 0x34, 0xf9, 0x2f, 0x2d, 0xa0, 0x17, 0x42, 0x11, - 0xe7, 0xf7, 0x1e, 0x8a, 0x58, 0xfd, 0xd9, 0x0c, 0xba, 0xa4, 0x11, 0x8b, 0x6c, 0xe8, 0x2b, 0x4d, - 0x22, 0x34, 0x8b, 0xaf, 0x0c, 0x74, 0xd6, 0x30, 0xdd, 0x96, 0xee, 0x35, 0xd6, 0x0e, 0x24, 0xa3, - 0x59, 0x34, 0x2c, 0xce, 0x5f, 0x7b, 0x98, 0xdb, 0xa4, 0x72, 0xea, 0xaf, 0xe5, 0x50, 0xff, 0x94, - 0xed, 0x1d, 0x38, 0x63, 0x78, 0x38, 0xe5, 0x67, 0xf7, 0x70, 0x2f, 0xf2, 0x51, 0xa8, 0x5c, 0x88, - 0x24, 0x08, 0x86, 0x17, 0x2b, 0x76, 0x2c, 0xe2, 0xa2, 0x4f, 0xb6, 0xc7, 0xa8, 0xd8, 0xcf, 0xa3, + 0x0f, 0xa1, 0x81, 0x4a, 0xa7, 0x23, 0x5c, 0xe3, 0xc1, 0x5b, 0x2d, 0x2d, 0x15, 0xe9, 0x63, 0x9f, + 0xec, 0xdc, 0x2b, 0x68, 0x54, 0xae, 0x6c, 0x57, 0x81, 0x36, 0xff, 0x2c, 0x03, 0x1f, 0x74, 0xc4, + 0x4d, 0xc5, 0x6e, 0xa0, 0x5c, 0xa5, 0xd3, 0xe1, 0xf3, 0xd1, 0xc9, 0x84, 0xfe, 0x88, 0xba, 0x8f, + 0x54, 0x3a, 0x1d, 0xff, 0xd3, 0x99, 0xa9, 0xea, 0xe3, 0xf5, 0xe9, 0x5f, 0x63, 0x9f, 0x7e, 0xb4, + 0xed, 0x41, 0xd5, 0x5f, 0xc9, 0xa1, 0x13, 0x95, 0x4e, 0xe7, 0x38, 0x40, 0xe7, 0x41, 0x39, 0xa9, + 0x3c, 0x8f, 0x90, 0x30, 0x3d, 0x0e, 0x04, 0x26, 0xdb, 0x43, 0xc2, 0xd4, 0xa8, 0x64, 0x34, 0x81, + 0xc8, 0x57, 0xbf, 0xe2, 0xae, 0xd4, 0xef, 0xb3, 0x39, 0x98, 0x8a, 0x8f, 0xba, 0xc3, 0xfd, 0x7b, + 0xa5, 0xdb, 0x78, 0x1f, 0x14, 0x76, 0xd5, 0x07, 0xbf, 0x25, 0x0d, 0x1e, 0x08, 0xf8, 0x78, 0xdc, + 0x0b, 0xfd, 0xfb, 0xda, 0x16, 0x8f, 0x8a, 0xc2, 0xe4, 0x5e, 0xc0, 0x7e, 0x10, 0x7a, 0xee, 0x93, + 0xde, 0xa4, 0xa8, 0x86, 0x69, 0x68, 0x11, 0x5a, 0xbf, 0x0f, 0x07, 0x76, 0xd5, 0x87, 0x9b, 0x59, + 0xf0, 0x3b, 0x09, 0x7c, 0xda, 0xf7, 0x7f, 0xba, 0xb8, 0x86, 0x10, 0x7b, 0xd0, 0x09, 0xac, 0xc5, + 0x46, 0x98, 0xfb, 0x2a, 0x8b, 0x4d, 0xcf, 0xdd, 0x57, 0x43, 0x92, 0xe0, 0xe1, 0x39, 0x97, 0xf8, + 0xf0, 0x7c, 0x05, 0x15, 0x35, 0x7d, 0xfd, 0x8d, 0x2e, 0x71, 0x36, 0xf8, 0x76, 0x86, 0x85, 0x8c, + 0xd1, 0xd7, 0x1b, 0x9f, 0xa1, 0x40, 0x2d, 0x40, 0x63, 0x35, 0x70, 0x5c, 0x12, 0x1e, 0xda, 0xd8, + 0xed, 0x5e, 0xe0, 0xae, 0xb4, 0x17, 0x45, 0xc7, 0x2f, 0xa3, 0x5c, 0xe5, 0x7e, 0x9d, 0x4b, 0x36, + 0xe8, 0xda, 0xca, 0xfd, 0x3a, 0x97, 0x57, 0x6a, 0xd9, 0xfb, 0x75, 0xf5, 0xb3, 0x59, 0x84, 0xe3, + 0x94, 0xf8, 0x05, 0x34, 0x08, 0xd0, 0x15, 0xaa, 0x33, 0x62, 0x52, 0xa3, 0x75, 0xb7, 0xe1, 0x00, + 0x54, 0xda, 0xdc, 0xf9, 0xa4, 0xf8, 0x25, 0xc8, 0xdf, 0xc6, 0xd3, 0x6a, 0x48, 0x49, 0x8d, 0xd6, + 0x5d, 0x3f, 0xe3, 0x59, 0x24, 0x7d, 0x1b, 0x27, 0x86, 0x7d, 0xe1, 0xfd, 0xfa, 0xac, 0xed, 0x7a, + 0x5c, 0xd4, 0x6c, 0x5f, 0xb8, 0xee, 0x42, 0x36, 0x2d, 0x69, 0x5f, 0xc8, 0xc8, 0x20, 0x23, 0xc0, + 0xfd, 0x3a, 0xb3, 0xfe, 0x35, 0x34, 0xbb, 0xe5, 0x6f, 0x28, 0x59, 0x46, 0x80, 0x75, 0xb7, 0xc1, + 0x2c, 0x87, 0x0d, 0x48, 0x1c, 0x27, 0x65, 0x04, 0x90, 0x4a, 0xa9, 0x9f, 0x2f, 0xa2, 0xd2, 0x94, + 0xee, 0xe9, 0xcb, 0xba, 0x4b, 0x84, 0xd3, 0xf4, 0x09, 0x1f, 0xe6, 0x7f, 0x8e, 0x20, 0x07, 0x63, + 0x39, 0xe1, 0x6b, 0xa2, 0x05, 0xf0, 0x47, 0x43, 0xbe, 0x41, 0xbe, 0x26, 0x31, 0x01, 0xc4, 0x72, + 0xa3, 0xc3, 0xc1, 0x5a, 0x8c, 0x10, 0x5f, 0x45, 0x43, 0x3e, 0x8c, 0x1e, 0x00, 0x72, 0xa1, 0xce, + 0x18, 0xcb, 0x74, 0xff, 0xaf, 0x89, 0x68, 0xfc, 0x12, 0x1a, 0xf6, 0x7f, 0x0a, 0x5b, 0x6b, 0x96, + 0xcd, 0x62, 0x39, 0x76, 0x7a, 0x12, 0x49, 0xc5, 0xa2, 0x30, 0xbf, 0xf5, 0x4b, 0x45, 0x23, 0x09, + 0x23, 0x24, 0x52, 0xfc, 0x19, 0x34, 0xea, 0xff, 0xe6, 0x07, 0x06, 0x96, 0x5b, 0xe3, 0x6a, 0x90, + 0x97, 0x2e, 0x22, 0xd6, 0x71, 0x99, 0x9c, 0x1d, 0x1d, 0x9e, 0xf4, 0x73, 0x20, 0x18, 0xcb, 0xf1, + 0x93, 0x43, 0xa4, 0x02, 0x5c, 0x45, 0x63, 0x3e, 0x24, 0xd4, 0xd0, 0x81, 0xf0, 0xc4, 0x68, 0x2c, + 0x37, 0x12, 0x95, 0x34, 0x5e, 0x0a, 0xb7, 0xd0, 0x79, 0x09, 0x68, 0xb8, 0xab, 0xe6, 0x03, 0x8f, + 0x1f, 0xf7, 0x78, 0xfc, 0x36, 0x9e, 0xf4, 0x26, 0xe0, 0xca, 0x68, 0xfc, 0xec, 0x55, 0x72, 0x64, + 0xfd, 0x9e, 0xdc, 0x70, 0x1d, 0x9d, 0xf2, 0xf1, 0xb7, 0x26, 0x6b, 0x35, 0xc7, 0x7e, 0x9b, 0x34, + 0xbd, 0xea, 0x14, 0x3f, 0x2e, 0x43, 0x5c, 0x0f, 0x63, 0xb9, 0xb1, 0xd2, 0xec, 0x50, 0xa5, 0xa0, + 0x38, 0x99, 0x79, 0x62, 0x61, 0x7c, 0x0f, 0x9d, 0x16, 0xe0, 0x55, 0xcb, 0xf5, 0x74, 0xab, 0x49, + 0xaa, 0x53, 0xfc, 0x0c, 0x0d, 0xe7, 0x79, 0xce, 0xd5, 0xe4, 0x48, 0x99, 0x6d, 0x72, 0x71, 0xfc, + 0x0a, 0x1a, 0xf1, 0x11, 0xec, 0xfd, 0x63, 0x08, 0xde, 0x3f, 0x60, 0x48, 0x1a, 0xcb, 0x8d, 0xa8, + 0x93, 0x8a, 0x4c, 0x2c, 0x6a, 0x14, 0xa4, 0x05, 0x1d, 0x96, 0x34, 0xca, 0xdb, 0xe8, 0x24, 0x2a, + 0x23, 0xa4, 0x0a, 0x7d, 0x2d, 0xd4, 0xa8, 0x45, 0xc7, 0x5c, 0x31, 0xd9, 0x49, 0xda, 0xf7, 0x4b, + 0x59, 0x6e, 0xd8, 0x00, 0x4c, 0xd2, 0x0f, 0x46, 0x7e, 0xae, 0x82, 0x4e, 0x26, 0xe8, 0xd8, 0xae, + 0x4e, 0x8c, 0x9f, 0xcb, 0x86, 0x8d, 0x38, 0xe2, 0xc7, 0xc6, 0x09, 0x54, 0xf4, 0xbf, 0x84, 0x6f, + 0x1e, 0x94, 0xb4, 0xa1, 0x19, 0xe5, 0xe1, 0xe3, 0x25, 0x71, 0x1c, 0xf1, 0xa3, 0xe4, 0x41, 0x88, + 0xe3, 0xeb, 0x99, 0x50, 0x1c, 0x47, 0xfc, 0x78, 0xf9, 0x37, 0xf3, 0xe1, 0x9c, 0x74, 0x7c, 0xc6, + 0x3c, 0xa8, 0x6d, 0x72, 0x68, 0x5e, 0x54, 0xd8, 0x85, 0x7f, 0x88, 0xa8, 0x9a, 0x03, 0x7b, 0x53, + 0x4d, 0xfc, 0x0a, 0x1a, 0xaa, 0xd9, 0xae, 0xb7, 0xe2, 0x10, 0xb7, 0x16, 0xc4, 0x1f, 0x05, 0xdf, + 0xa2, 0x0e, 0x07, 0x37, 0x3a, 0xd2, 0xec, 0x2f, 0x92, 0xab, 0x7f, 0x90, 0x8b, 0x69, 0x03, 0xdb, + 0xb8, 0x1e, 0x49, 0x6d, 0x38, 0x80, 0xa1, 0x8e, 0xaf, 0x87, 0xab, 0x20, 0xdb, 0xe1, 0xf7, 0x0b, + 0xc1, 0x55, 0x96, 0xf9, 0x06, 0x5f, 0x26, 0xc1, 0x9f, 0x44, 0x67, 0x25, 0x40, 0x4d, 0x77, 0xf4, + 0x36, 0xf1, 0xc2, 0x5c, 0x2f, 0xe0, 0x2e, 0xef, 0x97, 0x6e, 0x74, 0x02, 0xb4, 0x98, 0x3f, 0x26, + 0x85, 0x83, 0xa0, 0x5a, 0x03, 0xbb, 0xb0, 0x5c, 0xfb, 0xcf, 0x59, 0x34, 0x12, 0x74, 0xb4, 0xee, + 0xb8, 0xe4, 0xf1, 0xed, 0xd1, 0x8f, 0xa0, 0x11, 0xf0, 0xde, 0x6c, 0x13, 0xcb, 0x13, 0x92, 0x2a, + 0xb2, 0x80, 0x8f, 0x3e, 0x82, 0xc7, 0xf6, 0x95, 0x08, 0x71, 0x19, 0xf5, 0x33, 0x1d, 0x10, 0x7c, + 0x6a, 0x99, 0x02, 0x30, 0xb8, 0xfa, 0x93, 0x39, 0x34, 0xec, 0x4b, 0x79, 0xc2, 0x3c, 0xaa, 0x37, + 0x3e, 0x87, 0x2b, 0xe4, 0x6b, 0x08, 0xd5, 0x6c, 0xc7, 0xd3, 0x5b, 0x42, 0x6a, 0x76, 0x38, 0x2a, + 0x75, 0x00, 0xca, 0xca, 0x08, 0x24, 0x78, 0x1c, 0x21, 0x61, 0x80, 0x0d, 0xc0, 0x00, 0x1b, 0xdd, + 0xda, 0x2c, 0xa3, 0x70, 0x5c, 0x69, 0x02, 0x85, 0xfa, 0xd5, 0x2c, 0x3a, 0xe1, 0x77, 0xd2, 0xf4, + 0x23, 0xd2, 0xec, 0x7a, 0x8f, 0xf1, 0x60, 0x90, 0xa5, 0xdd, 0xbf, 0xad, 0xb4, 0xd5, 0xff, 0x21, + 0x4c, 0x24, 0x93, 0x2d, 0xfb, 0x78, 0x22, 0xf9, 0xcb, 0xd0, 0x71, 0xf5, 0x7b, 0x73, 0xe8, 0x94, + 0x2f, 0xf5, 0x99, 0xae, 0x05, 0x9b, 0x8c, 0x49, 0xbd, 0xd5, 0x7a, 0x9c, 0xd7, 0xe5, 0x21, 0x5f, + 0x10, 0x8b, 0x3c, 0x1c, 0x02, 0x8f, 0xb3, 0xfe, 0x80, 0x83, 0x1b, 0xb6, 0x69, 0x68, 0x22, 0x11, + 0x7e, 0x0d, 0x0d, 0xfb, 0x3f, 0x2b, 0xce, 0x8a, 0xbf, 0x18, 0xc3, 0x95, 0x41, 0x50, 0x48, 0x77, + 0x24, 0xaf, 0x0f, 0xa9, 0x80, 0xfa, 0x5f, 0x0a, 0xe8, 0xdc, 0x7d, 0xd3, 0x32, 0xec, 0x75, 0xd7, + 0x0f, 0xd3, 0x7f, 0xe4, 0xb7, 0xcc, 0x87, 0x1d, 0x9e, 0xff, 0x0d, 0x74, 0x3a, 0x2a, 0x52, 0x27, + 0x08, 0x9e, 0xc4, 0x7b, 0x67, 0x9d, 0x11, 0x34, 0xfc, 0x80, 0xfd, 0xfc, 0xde, 0x4d, 0x4b, 0x2e, + 0x19, 0x8d, 0xf8, 0x3f, 0xb0, 0x93, 0x88, 0xff, 0xcf, 0xa2, 0xc2, 0x94, 0xdd, 0xd6, 0x4d, 0xdf, + 0xff, 0x0f, 0x46, 0x71, 0x50, 0x2f, 0x60, 0x34, 0x4e, 0x41, 0xf9, 0xf3, 0x8a, 0xa1, 0xcb, 0x06, + 0x43, 0xfe, 0x7e, 0x81, 0xae, 0x4b, 0x1c, 0x4d, 0x24, 0xc2, 0x36, 0x1a, 0xe1, 0xd5, 0xf1, 0x5b, + 0x32, 0x04, 0xb7, 0x64, 0x41, 0x5e, 0xc5, 0x74, 0xb5, 0x1a, 0x97, 0xca, 0xb1, 0xeb, 0x32, 0x96, + 0x88, 0x80, 0x7f, 0x0c, 0xbb, 0x2f, 0xd3, 0x64, 0xfe, 0x82, 0x10, 0x60, 0x92, 0x19, 0x8a, 0x0b, + 0x01, 0x66, 0x19, 0x91, 0x08, 0x4f, 0xa3, 0xb1, 0x4a, 0xab, 0x65, 0xaf, 0x07, 0x51, 0x8a, 0xa8, + 0x4a, 0x0c, 0x43, 0xa4, 0x56, 0xb8, 0x7c, 0xd1, 0x29, 0x12, 0x3e, 0xae, 0xd1, 0xe4, 0x68, 0x2d, + 0x5e, 0xe2, 0xdc, 0xeb, 0x08, 0xc7, 0xdb, 0xbc, 0xab, 0xeb, 0x97, 0xcf, 0x67, 0x11, 0x8e, 0x9c, + 0x43, 0xa6, 0x1f, 0xe3, 0xed, 0x94, 0xfa, 0x0b, 0x19, 0x34, 0x16, 0x8b, 0x1e, 0x86, 0x6f, 0x20, + 0xc4, 0x20, 0x42, 0xd4, 0x0a, 0x70, 0x03, 0x0b, 0x23, 0x8a, 0xf1, 0xa5, 0x24, 0x24, 0xc3, 0xd7, + 0x50, 0x91, 0xfd, 0x0a, 0xd2, 0x84, 0x46, 0x8b, 0x74, 0xbb, 0xa6, 0xa1, 0x05, 0x44, 0x61, 0x2d, + 0x70, 0x8f, 0x97, 0x4b, 0x2c, 0xe2, 0x6d, 0x74, 0x82, 0x5a, 0x28, 0x19, 0xed, 0xc0, 0xe1, 0xa0, + 0xc1, 0x15, 0xe3, 0xb0, 0xba, 0xae, 0xc0, 0x03, 0xb1, 0xe5, 0xb6, 0x0b, 0xc4, 0x16, 0x99, 0x9b, + 0x78, 0xe4, 0xb5, 0x83, 0x33, 0x2e, 0xfd, 0x62, 0x16, 0x9d, 0x08, 0x6a, 0x3d, 0xc4, 0x2b, 0xa3, + 0xf7, 0x90, 0x48, 0xbe, 0x90, 0x41, 0xca, 0x84, 0xd9, 0x6a, 0x99, 0xd6, 0x4a, 0xd5, 0x7a, 0x60, + 0x3b, 0x6d, 0x98, 0x3c, 0x0e, 0xef, 0x76, 0x51, 0xfd, 0x81, 0x0c, 0x1a, 0xe3, 0x0d, 0x9a, 0xd4, + 0x1d, 0xe3, 0xf0, 0xae, 0x7d, 0xa3, 0x2d, 0x39, 0x3c, 0x7d, 0x51, 0xbf, 0x9c, 0x45, 0x68, 0xce, + 0x6e, 0xae, 0x1d, 0x71, 0x0b, 0xfa, 0x8f, 0x6e, 0x9f, 0x1d, 0xb7, 0x24, 0x67, 0xc7, 0x55, 0x32, + 0x7e, 0x7e, 0x5c, 0x5a, 0x29, 0xa5, 0xe3, 0xbb, 0x9a, 0xa0, 0x52, 0x31, 0xfd, 0x2e, 0xab, 0x74, + 0x6b, 0xb3, 0x9c, 0x6f, 0xd9, 0xcd, 0x35, 0x0d, 0xe8, 0xd5, 0xbf, 0xc8, 0x30, 0xd9, 0x1d, 0x71, + 0x0b, 0x7b, 0xff, 0xf3, 0xf3, 0xbb, 0xfc, 0xfc, 0xbf, 0x91, 0x41, 0xa7, 0x34, 0xd2, 0xb4, 0x1f, + 0x12, 0x67, 0x63, 0xd2, 0x36, 0xc8, 0x2d, 0x62, 0x11, 0xe7, 0xb0, 0x46, 0xd4, 0xaf, 0x43, 0xa8, + 0xc9, 0xb0, 0x31, 0x77, 0x5d, 0x62, 0x1c, 0x9d, 0x80, 0xa7, 0xea, 0x3f, 0x1e, 0x40, 0x4a, 0xe2, + 0x0e, 0xf1, 0xc8, 0xee, 0x8a, 0x52, 0xb7, 0xfd, 0xf9, 0x83, 0xda, 0xf6, 0xf7, 0xef, 0x6e, 0xdb, + 0x5f, 0xd8, 0xed, 0xb6, 0x7f, 0x60, 0x27, 0xdb, 0xfe, 0x76, 0x74, 0xdb, 0x5f, 0x84, 0x6d, 0xff, + 0x8d, 0x9e, 0xdb, 0xfe, 0x69, 0xcb, 0xd8, 0xe3, 0xa6, 0xff, 0xc8, 0xa6, 0xf9, 0xd8, 0xcb, 0x69, + 0xe5, 0x32, 0x9d, 0x14, 0x9b, 0xb6, 0x63, 0x10, 0x83, 0x1f, 0x52, 0xe0, 0x56, 0xde, 0xe1, 0x30, + 0x2d, 0xc0, 0xc6, 0x72, 0xa6, 0x8c, 0xec, 0x24, 0x67, 0xca, 0x01, 0x1c, 0x63, 0x3e, 0x97, 0x45, + 0x63, 0x93, 0xc4, 0xf1, 0x58, 0x3c, 0x91, 0x83, 0x78, 0x48, 0xae, 0xa0, 0x13, 0x02, 0x43, 0xd8, + 0x91, 0x0b, 0xb9, 0xfe, 0x9b, 0xc4, 0xf1, 0xa2, 0x6f, 0xeb, 0x51, 0x7a, 0x5a, 0xbd, 0x1f, 0xb7, + 0x98, 0x8f, 0xdd, 0xa0, 0x7a, 0x1f, 0xce, 0x04, 0x69, 0xf2, 0x5f, 0x5a, 0x40, 0x2f, 0x84, 0x22, + 0xce, 0xef, 0x3e, 0x14, 0xb1, 0xfa, 0x73, 0x19, 0x74, 0x49, 0x23, 0x16, 0x59, 0xd7, 0x97, 0x5b, + 0x44, 0x68, 0x16, 0x5f, 0x19, 0xe8, 0xac, 0x61, 0xba, 0x6d, 0xdd, 0x6b, 0xae, 0xee, 0x4b, 0x46, + 0x33, 0x68, 0x58, 0x9c, 0xbf, 0x76, 0x31, 0xb7, 0x49, 0xe5, 0xd4, 0xaf, 0xe4, 0xd0, 0xc0, 0x84, + 0xed, 0xed, 0x3b, 0x63, 0x78, 0x38, 0xe5, 0x67, 0x77, 0x71, 0x2f, 0xf2, 0x21, 0xa8, 0x5c, 0x88, + 0x24, 0x08, 0x86, 0x17, 0xcb, 0x76, 0x2c, 0xe2, 0xa2, 0x4f, 0xb6, 0xcb, 0xa8, 0xd8, 0x2f, 0xa0, 0x41, 0xf0, 0xf6, 0x14, 0x6e, 0x2e, 0xc1, 0xac, 0xc9, 0xa3, 0xc0, 0x68, 0x1d, 0x21, 0x29, 0xfe, - 0xb4, 0x14, 0x00, 0xa5, 0xef, 0xe0, 0x51, 0xb4, 0xc5, 0x58, 0x28, 0x87, 0x16, 0xac, 0x5a, 0xfd, - 0x56, 0x1e, 0x0d, 0xfb, 0xc6, 0x2c, 0x47, 0xd4, 0x83, 0xcf, 0xa2, 0xbe, 0x39, 0x5b, 0x88, 0x8a, - 0x08, 0xc6, 0x2f, 0x6b, 0xb6, 0x1b, 0xb1, 0xea, 0xe1, 0x44, 0xf8, 0x06, 0x1a, 0x58, 0xb4, 0x0d, - 0xd1, 0x74, 0x0b, 0xc6, 0xb4, 0x65, 0x1b, 0x31, 0xd7, 0x97, 0x80, 0x10, 0x5f, 0x42, 0x79, 0xb0, - 0x7a, 0x13, 0xae, 0x9e, 0x23, 0x96, 0x6e, 0x80, 0x17, 0x74, 0xa3, 0x6f, 0xaf, 0xba, 0xd1, 0xbf, - 0x5f, 0xdd, 0x18, 0x38, 0x5c, 0xdd, 0x78, 0x0b, 0x0d, 0x43, 0x4d, 0x7e, 0xd4, 0xef, 0x9d, 0x97, - 0xb7, 0xc7, 0xf8, 0x0a, 0x34, 0xc2, 0xda, 0xcd, 0x63, 0x7f, 0xc3, 0xc2, 0x23, 0xb1, 0x8a, 0xa8, - 0x1d, 0x3a, 0x80, 0xda, 0xfd, 0x41, 0x06, 0xf5, 0xdf, 0xb1, 0xd6, 0x2d, 0x7b, 0xe3, 0x60, 0x1a, - 0x77, 0x03, 0x0d, 0x71, 0x36, 0xc2, 0x1c, 0x0f, 0xde, 0x4c, 0x1d, 0x06, 0xae, 0x03, 0x27, 0x4d, - 0xa4, 0xc2, 0x2f, 0x07, 0x85, 0xc0, 0xb0, 0x35, 0x17, 0xc6, 0x15, 0xf5, 0x0b, 0x35, 0xe4, 0x50, - 0x88, 0x22, 0x39, 0x3e, 0xcf, 0x73, 0xe0, 0x0b, 0x81, 0x75, 0x68, 0x53, 0x58, 0x0a, 0x7c, 0xf5, - 0x5f, 0x66, 0xd1, 0x68, 0xe4, 0xfa, 0xe9, 0x19, 0x34, 0xc8, 0xaf, 0x7f, 0x4c, 0x3f, 0x36, 0x23, - 0x18, 0xbe, 0x06, 0x40, 0x6d, 0x80, 0xfd, 0x59, 0x31, 0xf0, 0x27, 0x51, 0xbf, 0xed, 0xc2, 0xd2, - 0x04, 0xdf, 0x32, 0x1a, 0x0e, 0xa1, 0xa5, 0x1a, 0x6d, 0x3b, 0x1b, 0x1c, 0x9c, 0x44, 0xd4, 0x48, - 0xdb, 0x85, 0x4f, 0xbb, 0x89, 0x06, 0x75, 0xd7, 0x25, 0x5e, 0xdd, 0xd3, 0x57, 0xc5, 0x70, 0x8d, - 0x01, 0x50, 0x1c, 0x1d, 0x00, 0x5c, 0xd6, 0x57, 0xf1, 0x6b, 0x68, 0xa4, 0xe1, 0x10, 0x58, 0xbc, - 0xf4, 0x26, 0x6d, 0xa5, 0xb0, 0xb9, 0x94, 0x10, 0xe2, 0x8d, 0x7f, 0x88, 0xa8, 0x18, 0xf8, 0x2e, - 0x1a, 0xe1, 0x9f, 0xc3, 0xac, 0xce, 0x60, 0xa0, 0x8d, 0x86, 0x8b, 0x09, 0x13, 0x09, 0xb3, 0x3b, - 0xe3, 0xc6, 0x87, 0x22, 0xb9, 0xc8, 0xd7, 0x10, 0x48, 0xd5, 0xaf, 0x67, 0xe8, 0x86, 0x87, 0x02, - 0x82, 0x74, 0x9a, 0xad, 0x3d, 0xea, 0x4a, 0x2b, 0x8c, 0x98, 0xdf, 0xe7, 0x76, 0x99, 0x9d, 0x34, - 0x8e, 0xc5, 0x93, 0xa8, 0xcf, 0x10, 0xef, 0x7e, 0xce, 0xca, 0x1f, 0xe1, 0xd7, 0xa3, 0x71, 0x2a, - 0x7c, 0x19, 0xe5, 0xe9, 0x86, 0x36, 0x7a, 0xf0, 0x13, 0xd7, 0x48, 0x0d, 0x28, 0xd4, 0xef, 0xce, - 0xa2, 0x61, 0xe1, 0x6b, 0xae, 0x1f, 0xe8, 0x73, 0x5e, 0xda, 0x5d, 0x33, 0xb9, 0x1d, 0x2c, 0xc0, - 0x82, 0x26, 0xdf, 0x0c, 0x44, 0xb1, 0xab, 0x27, 0x08, 0x2e, 0x98, 0xe7, 0xf9, 0x87, 0xf6, 0xed, - 0xfe, 0x10, 0x44, 0xe9, 0x5f, 0xcf, 0x0f, 0x64, 0x0b, 0xb9, 0xd7, 0xf3, 0x03, 0xf9, 0x42, 0x2f, - 0x78, 0xd6, 0x43, 0x34, 0x29, 0x76, 0xc2, 0xb4, 0xee, 0x9b, 0xab, 0xc7, 0xdc, 0x6e, 0xf0, 0x70, - 0xa3, 0x0e, 0x44, 0x64, 0x73, 0xcc, 0x8d, 0x08, 0xdf, 0x57, 0xd9, 0x9c, 0x24, 0x30, 0xe0, 0xb2, - 0xf9, 0x77, 0x19, 0xa4, 0x24, 0xca, 0xa6, 0x74, 0x44, 0x2f, 0xdf, 0x87, 0x97, 0xc6, 0xe0, 0x9b, - 0x59, 0x34, 0x5e, 0xb1, 0x3c, 0xb2, 0xca, 0xce, 0x3d, 0xc7, 0x7c, 0xaa, 0xb8, 0xcd, 0xd2, 0x98, - 0xf2, 0x8f, 0xe1, 0x7d, 0xfe, 0x78, 0x70, 0xaa, 0x0c, 0x51, 0x29, 0x9c, 0xc4, 0xd2, 0x87, 0x98, - 0xde, 0x28, 0x22, 0xe4, 0x63, 0x3e, 0xe7, 0x1c, 0x0f, 0x21, 0x1f, 0xf3, 0xc9, 0xeb, 0x03, 0x2a, - 0xe4, 0xff, 0x9e, 0x41, 0xa7, 0x12, 0x2a, 0x87, 0xe4, 0x80, 0x9d, 0x15, 0x08, 0xb9, 0x90, 0x11, - 0x92, 0x03, 0x76, 0x56, 0x20, 0xda, 0x82, 0xe6, 0x23, 0xf1, 0x32, 0x38, 0x56, 0x2d, 0x55, 0xca, - 0xd3, 0x5c, 0xaa, 0xaa, 0xe0, 0x22, 0x46, 0xc1, 0x49, 0x5f, 0x16, 0x38, 0x5f, 0xd9, 0xa6, 0xd1, - 0x88, 0x38, 0x5f, 0xd1, 0x32, 0xf8, 0x33, 0x68, 0xb0, 0xf4, 0x6e, 0xc7, 0x21, 0xc0, 0x97, 0x49, - 0xfc, 0x43, 0x01, 0x5f, 0x1f, 0x91, 0xc4, 0x99, 0xf9, 0x91, 0x51, 0x8a, 0x28, 0xef, 0x90, 0xa1, - 0xfa, 0xc5, 0x0c, 0x9a, 0x48, 0x6f, 0x1d, 0xfe, 0x28, 0xea, 0xa7, 0x27, 0xdb, 0x92, 0xb6, 0xc8, - 0x3f, 0x9d, 0xa5, 0xfc, 0xb0, 0x9b, 0xa4, 0xae, 0x3b, 0xe2, 0xc6, 0xdb, 0x27, 0xc3, 0xaf, 0xa0, - 0xa1, 0x8a, 0xeb, 0x76, 0x88, 0x53, 0xbb, 0x71, 0x47, 0xab, 0xf0, 0x33, 0x15, 0xec, 0xd9, 0x4d, - 0x00, 0xd7, 0xdd, 0x1b, 0x91, 0xa0, 0x0a, 0x22, 0xbd, 0xfa, 0x83, 0x19, 0x74, 0xbe, 0xdb, 0x57, - 0xd1, 0x03, 0xfc, 0x32, 0xb1, 0x74, 0xcb, 0xe3, 0xa9, 0x75, 0xf9, 0x11, 0xc5, 0x03, 0x98, 0x7c, - 0xc8, 0x08, 0x08, 0x69, 0x21, 0x76, 0x3b, 0x16, 0x3c, 0xc7, 0xb3, 0x9b, 0x3c, 0x80, 0x45, 0x0a, - 0xf9, 0x84, 0xea, 0xcf, 0xbc, 0x89, 0x7a, 0x97, 0x2c, 0xb2, 0x74, 0x1f, 0x3f, 0x27, 0x24, 0x70, - 0xe3, 0x03, 0x6d, 0x5c, 0x1c, 0x30, 0x80, 0x98, 0xeb, 0xd1, 0x84, 0x34, 0x6f, 0x37, 0xc5, 0x2c, - 0x54, 0x5c, 0x1d, 0xb0, 0x58, 0x86, 0x61, 0xe6, 0x7a, 0x34, 0x31, 0x5b, 0xd5, 0x4d, 0x31, 0xb9, - 0x12, 0xef, 0x6c, 0xa9, 0x14, 0xc3, 0xf8, 0xa5, 0xf8, 0x34, 0x30, 0x9f, 0x94, 0x81, 0x28, 0xba, - 0x27, 0x88, 0x53, 0xcc, 0xf5, 0x68, 0xc9, 0x99, 0x8b, 0x86, 0x45, 0xc3, 0x98, 0xe8, 0x83, 0x9c, - 0x88, 0x9b, 0xeb, 0xd1, 0x24, 0x5a, 0xfc, 0x42, 0x90, 0xe6, 0xf1, 0x75, 0xdb, 0xb4, 0xa2, 0xde, - 0x95, 0x02, 0x6a, 0xae, 0x47, 0x13, 0x29, 0x85, 0x4a, 0xab, 0x8e, 0x19, 0xe4, 0x60, 0x8b, 0x56, - 0x0a, 0x38, 0xa1, 0x52, 0xf8, 0x8d, 0x5f, 0x41, 0x23, 0x81, 0xdb, 0xea, 0x3b, 0xa4, 0xe1, 0xf1, - 0x2b, 0x91, 0x33, 0x91, 0xc2, 0x0c, 0x39, 0xd7, 0xa3, 0xc9, 0xd4, 0xf8, 0xb2, 0x9f, 0xe0, 0x9f, - 0xdf, 0x75, 0x8c, 0x0a, 0xd3, 0x99, 0xf9, 0x2e, 0x95, 0x12, 0xc7, 0xd3, 0xde, 0x09, 0xdf, 0x0e, - 0xf8, 0x05, 0x06, 0x8e, 0xd4, 0x32, 0x63, 0x19, 0xb4, 0x77, 0x84, 0x87, 0xa3, 0xd7, 0xa2, 0x29, - 0x90, 0x79, 0x62, 0xed, 0xb3, 0x91, 0x92, 0x1c, 0x3b, 0xd7, 0xa3, 0x45, 0x53, 0x26, 0xbf, 0x20, - 0xa5, 0xdf, 0xe5, 0xf1, 0x53, 0xa2, 0x52, 0xa5, 0x28, 0x41, 0xaa, 0x90, 0xa8, 0xf7, 0xb5, 0x68, - 0x3e, 0x58, 0x1e, 0x2d, 0xe5, 0x6c, 0x72, 0xd6, 0x50, 0xa1, 0x6a, 0x3f, 0x7f, 0xec, 0x0b, 0x52, - 0xde, 0x4e, 0x48, 0x8d, 0x9d, 0x50, 0xb5, 0xee, 0xe9, 0x62, 0xd5, 0xec, 0x7c, 0x29, 0x65, 0x90, - 0x84, 0x04, 0x37, 0xf1, 0x0e, 0x05, 0x9c, 0xd0, 0xa1, 0x2c, 0xdb, 0xe4, 0x0b, 0x52, 0x12, 0x13, - 0x9e, 0xc1, 0x26, 0xa8, 0x54, 0x40, 0xd1, 0x4a, 0xc5, 0x74, 0x27, 0x37, 0xc5, 0xdc, 0x1e, 0xca, - 0xb8, 0xdc, 0x41, 0x21, 0x86, 0x76, 0x90, 0x90, 0x03, 0xa4, 0x08, 0x79, 0x03, 0x14, 0x0c, 0xe4, - 0x43, 0x41, 0x0b, 0xa7, 0xab, 0x73, 0x3d, 0x1a, 0x64, 0x14, 0x50, 0x59, 0x46, 0x0a, 0xe5, 0x14, - 0x50, 0x0c, 0x07, 0xf9, 0x51, 0x1f, 0x92, 0xc6, 0x5c, 0x8f, 0xc6, 0xb2, 0x55, 0x3c, 0x27, 0xc4, - 0x7e, 0x56, 0x4e, 0xcb, 0x53, 0x44, 0x80, 0xa0, 0x53, 0x44, 0x18, 0x21, 0x7a, 0x36, 0x1e, 0x1f, - 0x59, 0x39, 0x23, 0xaf, 0xa8, 0x51, 0xfc, 0x5c, 0x8f, 0x16, 0x8f, 0xa9, 0xfc, 0x82, 0x14, 0x32, - 0x58, 0x39, 0x1b, 0x71, 0x69, 0x0e, 0x51, 0x54, 0x5c, 0x62, 0x70, 0xe1, 0xa5, 0xc4, 0x24, 0x5f, - 0xca, 0x39, 0x79, 0x39, 0x4e, 0x20, 0x99, 0xeb, 0xd1, 0x12, 0xd3, 0x83, 0x4d, 0xc7, 0x02, 0xf7, - 0x2a, 0x8a, 0xfc, 0x6e, 0x19, 0x41, 0xcf, 0xf5, 0x68, 0xb1, 0x50, 0xbf, 0x37, 0xc5, 0x88, 0xb9, - 0xca, 0x63, 0x72, 0x27, 0x86, 0x18, 0xda, 0x89, 0x42, 0x64, 0xdd, 0x9b, 0x62, 0x58, 0x59, 0x65, - 0x22, 0x5e, 0x2a, 0x9c, 0x39, 0x85, 0xf0, 0xb3, 0x5a, 0x72, 0xec, 0x55, 0xe5, 0x71, 0x1e, 0x9a, - 0x9f, 0x97, 0x4f, 0xa2, 0x99, 0xeb, 0xd1, 0x92, 0xe3, 0xb6, 0x6a, 0xc9, 0x41, 0x4b, 0x95, 0xf3, - 0xdd, 0x78, 0x06, 0xad, 0x4b, 0x0e, 0x78, 0xaa, 0x77, 0x09, 0x21, 0xa9, 0x5c, 0x90, 0xe3, 0x29, - 0xa5, 0x12, 0xce, 0xf5, 0x68, 0x5d, 0x02, 0x51, 0xde, 0x49, 0x89, 0xe7, 0xa8, 0x5c, 0x94, 0x33, - 0x73, 0x24, 0x12, 0xcd, 0xf5, 0x68, 0x29, 0xd1, 0x20, 0xef, 0xa4, 0x84, 0x42, 0x54, 0x8a, 0x5d, - 0xd9, 0x06, 0xf2, 0x48, 0x09, 0xa4, 0xb8, 0x94, 0x18, 0x45, 0x50, 0x79, 0x42, 0x56, 0xdd, 0x04, - 0x12, 0xaa, 0xba, 0x49, 0xf1, 0x07, 0x97, 0x12, 0xc3, 0xf8, 0x29, 0x4f, 0x76, 0x61, 0x18, 0xb4, - 0x31, 0x31, 0x00, 0xe0, 0x52, 0x62, 0x1c, 0x3d, 0x45, 0x95, 0x19, 0x26, 0x90, 0x50, 0x86, 0x49, - 0x11, 0xf8, 0x96, 0x12, 0x03, 0xd9, 0x29, 0x4f, 0x75, 0x61, 0x18, 0xb6, 0x30, 0x29, 0x04, 0xde, - 0x0b, 0x52, 0x24, 0x39, 0xe5, 0x43, 0xf2, 0xbc, 0x21, 0xa0, 0xe8, 0xbc, 0x21, 0xc6, 0x9c, 0x9b, - 0x8e, 0xc5, 0xca, 0x51, 0x3e, 0x2c, 0x0f, 0xf3, 0x08, 0x9a, 0x0e, 0xf3, 0x68, 0x74, 0x9d, 0xe9, - 0x58, 0xcc, 0x10, 0xe5, 0x52, 0x1a, 0x13, 0x40, 0xcb, 0x4c, 0x58, 0x94, 0x91, 0x4a, 0x42, 0xd0, - 0x0a, 0xe5, 0x69, 0xd9, 0xe6, 0x2e, 0x46, 0x30, 0xd7, 0xa3, 0x25, 0x84, 0xba, 0xd0, 0x92, 0x3d, - 0x34, 0x95, 0xcb, 0xf2, 0xb0, 0x4d, 0xa2, 0xa1, 0xc3, 0x36, 0xd1, 0xbb, 0x73, 0x3e, 0xc9, 0xbe, - 0x56, 0xb9, 0x22, 0x6f, 0xcc, 0xe2, 0x14, 0x74, 0x63, 0x96, 0x60, 0x97, 0xab, 0x25, 0x7b, 0x0d, - 0x2a, 0xcf, 0x74, 0x6d, 0x21, 0xd0, 0x24, 0xb4, 0x90, 0x39, 0xd1, 0x85, 0x7b, 0xa7, 0x3b, 0xed, - 0xa6, 0xad, 0x1b, 0xca, 0x47, 0x12, 0xf7, 0x4e, 0x0c, 0x29, 0xec, 0x9d, 0x18, 0x80, 0xae, 0xf2, - 0xa2, 0xfd, 0xa9, 0x72, 0x55, 0x5e, 0xe5, 0x45, 0x1c, 0x5d, 0xe5, 0x25, 0x5b, 0xd5, 0xe9, 0x98, - 0xad, 0xa6, 0xf2, 0xac, 0xac, 0x00, 0x11, 0x34, 0x55, 0x80, 0xa8, 0x75, 0xe7, 0xdb, 0xe9, 0xd6, - 0x8d, 0xca, 0x24, 0x70, 0x7b, 0x22, 0xc8, 0x00, 0x9f, 0x42, 0x37, 0xd7, 0xa3, 0xa5, 0x5b, 0x48, - 0x56, 0x12, 0x8c, 0x15, 0x95, 0x6b, 0xb2, 0x82, 0xc5, 0x08, 0xa8, 0x82, 0xc5, 0x4d, 0x1c, 0x2b, - 0x09, 0xd6, 0x86, 0xca, 0x47, 0x53, 0x59, 0x05, 0xdf, 0x9c, 0x60, 0xa3, 0x78, 0x53, 0x34, 0x17, - 0x54, 0x9e, 0x93, 0x17, 0xbb, 0x10, 0x43, 0x17, 0x3b, 0xc1, 0xac, 0xf0, 0xa6, 0x68, 0x28, 0xa7, - 0x5c, 0x8f, 0x97, 0x0a, 0x97, 0x48, 0xc1, 0xa0, 0x4e, 0x4b, 0xb6, 0x2f, 0x53, 0x6e, 0xc8, 0x5a, - 0x97, 0x44, 0x43, 0xb5, 0x2e, 0xd1, 0x36, 0x6d, 0x36, 0x6e, 0x26, 0xa6, 0xdc, 0x8c, 0xde, 0x25, - 0xc8, 0x78, 0xba, 0xf3, 0x89, 0x99, 0x96, 0xbd, 0x16, 0x0d, 0x1f, 0xa0, 0x7c, 0x2c, 0xf2, 0x98, - 0x21, 0x61, 0xe9, 0xfe, 0x36, 0x12, 0x6e, 0xe0, 0xb5, 0xa8, 0xc7, 0xbd, 0xf2, 0x7c, 0x32, 0x87, - 0x40, 0x57, 0xa2, 0x1e, 0xfa, 0xaf, 0x45, 0x9d, 0xd4, 0x95, 0x17, 0x92, 0x39, 0x04, 0xd2, 0x8d, - 0x3a, 0xb5, 0x3f, 0x27, 0x84, 0xcd, 0x53, 0x3e, 0x2e, 0x6f, 0x1d, 0x03, 0x04, 0xdd, 0x3a, 0x86, - 0xc1, 0xf5, 0x9e, 0x13, 0xc2, 0xcd, 0x29, 0x2f, 0xc6, 0x8a, 0x04, 0x8d, 0x15, 0x82, 0xd2, 0x3d, - 0x27, 0x84, 0x69, 0x53, 0x5e, 0x8a, 0x15, 0x09, 0x5a, 0x27, 0x04, 0x73, 0x33, 0xba, 0xf9, 0xe1, - 0x28, 0x9f, 0x90, 0xaf, 0x38, 0xd2, 0x29, 0xe7, 0x7a, 0xb4, 0x6e, 0xfe, 0x3c, 0x6f, 0xa7, 0x1b, - 0xdd, 0x29, 0x2f, 0xcb, 0x43, 0x38, 0x8d, 0x8e, 0x0e, 0xe1, 0x54, 0xc3, 0xbd, 0x57, 0x22, 0x3e, - 0xb9, 0xca, 0x2b, 0xf2, 0x14, 0x27, 0x21, 0xe9, 0x14, 0x17, 0xf5, 0xe0, 0x95, 0x9c, 0x4d, 0x95, - 0x4f, 0xca, 0x53, 0x9c, 0x88, 0xa3, 0x53, 0x9c, 0xe4, 0x98, 0x3a, 0x1d, 0xf3, 0x81, 0x54, 0x5e, - 0x95, 0xa7, 0xb8, 0x08, 0x9a, 0x4e, 0x71, 0x51, 0xaf, 0xc9, 0x57, 0x22, 0xae, 0x80, 0xca, 0x6b, - 0xc9, 0xed, 0x07, 0xa4, 0xd8, 0x7e, 0xe6, 0x38, 0xa8, 0x25, 0xfb, 0xb4, 0x29, 0x25, 0x79, 0xfc, - 0x26, 0xd1, 0xd0, 0xf1, 0x9b, 0xe8, 0x0f, 0xb7, 0x94, 0x98, 0x17, 0x53, 0x99, 0xea, 0x72, 0x70, - 0x08, 0xb7, 0x22, 0x49, 0x19, 0x35, 0xc5, 0x33, 0x32, 0x3b, 0x08, 0x4d, 0xa7, 0x9c, 0x91, 0xfd, - 0x63, 0x50, 0x84, 0x9e, 0xce, 0xae, 0x31, 0x1b, 0x30, 0xa5, 0x2c, 0xcf, 0xae, 0x31, 0x02, 0x3a, - 0xbb, 0xc6, 0x2d, 0xc7, 0x66, 0x51, 0x81, 0x6b, 0x11, 0x33, 0x6d, 0x33, 0xad, 0x55, 0x65, 0x26, - 0xe2, 0x52, 0x12, 0xc1, 0xd3, 0xd9, 0x29, 0x0a, 0x83, 0xf5, 0x9a, 0xc1, 0xa6, 0x9b, 0x66, 0x7b, - 0xc5, 0xd6, 0x1d, 0xa3, 0x46, 0x2c, 0x43, 0x99, 0x8d, 0xac, 0xd7, 0x09, 0x34, 0xb0, 0x5e, 0x27, - 0xc0, 0xc1, 0xe9, 0x3d, 0x02, 0xd7, 0x48, 0x83, 0x98, 0x0f, 0x88, 0x72, 0x0b, 0xd8, 0x16, 0xd3, - 0xd8, 0x72, 0xb2, 0xb9, 0x1e, 0x2d, 0x8d, 0x03, 0xdd, 0xab, 0x2f, 0x6c, 0xd6, 0xde, 0x98, 0x0f, - 0xdc, 0x28, 0xab, 0x0e, 0x69, 0xeb, 0x0e, 0x51, 0xe6, 0xe4, 0xbd, 0x7a, 0x22, 0x11, 0xdd, 0xab, - 0x27, 0x22, 0xe2, 0x6c, 0xfd, 0xb1, 0x50, 0xe9, 0xc6, 0x36, 0x1c, 0x11, 0xc9, 0xa5, 0xe9, 0xec, - 0x24, 0x23, 0xa8, 0x80, 0xe6, 0x6d, 0x6b, 0x15, 0x6e, 0x2a, 0x5e, 0x97, 0x67, 0xa7, 0x74, 0x4a, - 0x3a, 0x3b, 0xa5, 0x63, 0xa9, 0xaa, 0xcb, 0x58, 0x36, 0x06, 0x6f, 0xcb, 0xaa, 0x9e, 0x40, 0x42, - 0x55, 0x3d, 0x01, 0x1c, 0x67, 0xa8, 0x11, 0x97, 0x78, 0xca, 0x7c, 0x37, 0x86, 0x40, 0x12, 0x67, - 0x08, 0xe0, 0x38, 0xc3, 0x59, 0xe2, 0x35, 0xd6, 0x94, 0x85, 0x6e, 0x0c, 0x81, 0x24, 0xce, 0x10, - 0xc0, 0xf4, 0xb0, 0x29, 0x83, 0xa7, 0x3a, 0xcd, 0x75, 0xbf, 0xcf, 0x16, 0xe5, 0xc3, 0x66, 0x2a, - 0x21, 0x3d, 0x6c, 0xa6, 0x22, 0xf1, 0x0f, 0xee, 0xda, 0x46, 0x51, 0x59, 0x82, 0x0a, 0x27, 0xc3, - 0x7d, 0xc1, 0x6e, 0x4a, 0xcd, 0xf5, 0x68, 0xbb, 0xb5, 0x81, 0xfc, 0x48, 0x60, 0x4a, 0xa4, 0x54, - 0xa1, 0xaa, 0xb1, 0xe0, 0xae, 0x82, 0x81, 0xe7, 0x7a, 0xb4, 0xc0, 0xd8, 0xe8, 0x05, 0x34, 0x04, - 0x1f, 0x55, 0xb1, 0x4c, 0xaf, 0x3c, 0xa5, 0xbc, 0x21, 0x1f, 0x99, 0x04, 0x14, 0x3d, 0x32, 0x09, - 0x3f, 0xe9, 0x24, 0x0e, 0x3f, 0xd9, 0x14, 0x53, 0x9e, 0x52, 0x34, 0x79, 0x12, 0x97, 0x90, 0x74, - 0x12, 0x97, 0x00, 0x41, 0xbd, 0x65, 0xc7, 0x6e, 0x97, 0xa7, 0x94, 0x5a, 0x42, 0xbd, 0x0c, 0x15, - 0xd4, 0xcb, 0x7e, 0x06, 0xf5, 0xd6, 0xd6, 0x3a, 0x5e, 0x99, 0x7e, 0xe3, 0x72, 0x42, 0xbd, 0x3e, - 0x32, 0xa8, 0xd7, 0x07, 0xd0, 0xa9, 0x10, 0x00, 0x55, 0xc7, 0xa6, 0x93, 0xf6, 0x6d, 0xb3, 0xd9, - 0x54, 0xee, 0xc8, 0x53, 0x61, 0x14, 0x4f, 0xa7, 0xc2, 0x28, 0x8c, 0x6e, 0x3d, 0x59, 0xab, 0xc8, - 0x4a, 0x67, 0x55, 0xb9, 0x2b, 0x6f, 0x3d, 0x43, 0x0c, 0xdd, 0x7a, 0x86, 0xbf, 0xe0, 0x74, 0x41, - 0x7f, 0x69, 0xe4, 0xbe, 0x43, 0xdc, 0x35, 0xe5, 0x5e, 0xe4, 0x74, 0x21, 0xe0, 0xe0, 0x74, 0x21, - 0xfc, 0xc6, 0xab, 0xe8, 0x71, 0x69, 0xa1, 0xf1, 0xdf, 0x9e, 0x6a, 0x44, 0x77, 0x1a, 0x6b, 0xca, - 0x9b, 0xc0, 0xea, 0xa9, 0xc4, 0xa5, 0x4a, 0x26, 0x9d, 0xeb, 0xd1, 0xba, 0x71, 0x82, 0x63, 0xf9, - 0x1b, 0xf3, 0x2c, 0xb6, 0x8d, 0x56, 0x9d, 0xf6, 0x0f, 0xa1, 0x6f, 0x45, 0x8e, 0xe5, 0x71, 0x12, - 0x38, 0x96, 0xc7, 0xc1, 0xb8, 0x8d, 0x2e, 0x46, 0x8e, 0x6a, 0x0b, 0x7a, 0x93, 0x9e, 0x4b, 0x88, - 0x51, 0xd5, 0x1b, 0xeb, 0xc4, 0x53, 0x3e, 0x05, 0xbc, 0x2f, 0xa5, 0x1c, 0xf8, 0x22, 0xd4, 0x73, - 0x3d, 0xda, 0x0e, 0xfc, 0xb0, 0xca, 0x32, 0x2f, 0x2a, 0x9f, 0x96, 0xef, 0x37, 0x29, 0x6c, 0xae, - 0x47, 0x63, 0x59, 0x19, 0xdf, 0x46, 0xca, 0x9d, 0xf6, 0xaa, 0xa3, 0x1b, 0x84, 0x6d, 0xb4, 0x60, - 0xef, 0xc6, 0x37, 0xa0, 0x9f, 0x91, 0x77, 0x69, 0x69, 0x74, 0x74, 0x97, 0x96, 0x86, 0xa3, 0x8a, - 0x2a, 0x85, 0x71, 0x55, 0x3e, 0x2b, 0x2b, 0xaa, 0x84, 0xa4, 0x8a, 0x2a, 0x07, 0x7d, 0x7d, 0x13, - 0x9d, 0x0d, 0xce, 0xf3, 0x7c, 0xfd, 0x65, 0x9d, 0xa6, 0xbc, 0x0d, 0x7c, 0x2e, 0xc6, 0x1e, 0x03, - 0x24, 0xaa, 0xb9, 0x1e, 0x2d, 0xa5, 0x3c, 0x5d, 0x71, 0x63, 0x11, 0xca, 0xf9, 0xf6, 0xe2, 0xbb, - 0xe4, 0x15, 0x37, 0x85, 0x8c, 0xae, 0xb8, 0x29, 0xa8, 0x44, 0xe6, 0x5c, 0xa8, 0xfa, 0x0e, 0xcc, - 0x03, 0x99, 0xa6, 0x71, 0x48, 0x64, 0xce, 0x77, 0x6a, 0x2b, 0x3b, 0x30, 0x0f, 0x76, 0x6b, 0x69, - 0x1c, 0xf0, 0x65, 0xd4, 0x57, 0xab, 0x2d, 0x68, 0x1d, 0x4b, 0x69, 0x44, 0x6c, 0xc0, 0x00, 0x3a, - 0xd7, 0xa3, 0x71, 0x3c, 0xdd, 0x06, 0xcd, 0x34, 0x75, 0xd7, 0x33, 0x1b, 0x2e, 0x8c, 0x18, 0x7f, - 0x84, 0x18, 0xf2, 0x36, 0x28, 0x89, 0x86, 0x6e, 0x83, 0x92, 0xe0, 0x74, 0xbf, 0x38, 0xad, 0xbb, - 0xae, 0x6e, 0x19, 0x8e, 0x3e, 0x05, 0xcb, 0x04, 0x89, 0x58, 0xca, 0x4b, 0x58, 0xba, 0x5f, 0x94, - 0x21, 0x70, 0xf9, 0xee, 0x43, 0xfc, 0x6d, 0xce, 0xfd, 0xc8, 0xe5, 0x7b, 0x04, 0x0f, 0x97, 0xef, - 0x11, 0x18, 0xec, 0x3b, 0x7d, 0x98, 0x46, 0x56, 0x4d, 0xc8, 0x93, 0xbc, 0x1a, 0xd9, 0x77, 0x46, - 0x09, 0x60, 0xdf, 0x19, 0x05, 0x4a, 0x4d, 0xf2, 0x97, 0xdb, 0xb5, 0x94, 0x26, 0x85, 0xab, 0x6c, - 0xac, 0x0c, 0x5d, 0xbf, 0xc3, 0xc1, 0x51, 0xde, 0xb4, 0xf4, 0x96, 0x5d, 0x9e, 0xf2, 0xa5, 0x6e, - 0xca, 0xeb, 0x77, 0x2a, 0x21, 0x5d, 0xbf, 0x53, 0x91, 0x74, 0x76, 0xf5, 0x0f, 0x5a, 0x6b, 0xba, - 0x43, 0x8c, 0x20, 0x7b, 0x28, 0x3b, 0x1a, 0xbe, 0x23, 0xcf, 0xae, 0x5d, 0x48, 0xe9, 0xec, 0xda, - 0x05, 0x4d, 0x37, 0x79, 0xc9, 0x68, 0x8d, 0xe8, 0x86, 0xb2, 0x2e, 0x6f, 0xf2, 0xd2, 0x29, 0xe9, - 0x26, 0x2f, 0x1d, 0x9b, 0xfe, 0x39, 0xf7, 0x1c, 0xd3, 0x23, 0x4a, 0x73, 0x37, 0x9f, 0x03, 0xa4, - 0xe9, 0x9f, 0x03, 0x68, 0x7a, 0x20, 0x8c, 0x76, 0x48, 0x4b, 0x3e, 0x10, 0xc6, 0xbb, 0x21, 0x5a, - 0x82, 0xee, 0x58, 0xb8, 0xc3, 0x84, 0x62, 0xc9, 0x3b, 0x16, 0x0e, 0xa6, 0x3b, 0x96, 0xd0, 0xa5, - 0x42, 0x32, 0xd0, 0x57, 0x6c, 0x79, 0x0d, 0x15, 0x71, 0x74, 0x0d, 0x95, 0x8c, 0xf9, 0x5f, 0x90, - 0xac, 0x67, 0x95, 0xb6, 0xbc, 0xeb, 0x10, 0x50, 0x74, 0xd7, 0x21, 0xda, 0xd9, 0x4e, 0xa3, 0x31, - 0x78, 0x05, 0xd7, 0x3a, 0xc1, 0x3b, 0xce, 0xe7, 0xe4, 0xcf, 0x8c, 0xa0, 0xe9, 0x67, 0x46, 0x40, - 0x12, 0x13, 0x3e, 0x6d, 0x39, 0x29, 0x4c, 0xc2, 0xfb, 0xc1, 0x08, 0x08, 0xcf, 0x23, 0x5c, 0x2b, - 0x2d, 0xcc, 0x57, 0x8c, 0xaa, 0xf8, 0x44, 0xe6, 0xca, 0x37, 0xb0, 0x71, 0x8a, 0xb9, 0x1e, 0x2d, - 0xa1, 0x1c, 0x7e, 0x07, 0x9d, 0xe7, 0x50, 0xee, 0x0d, 0x07, 0x29, 0xd8, 0x8c, 0x60, 0x41, 0xf0, - 0x64, 0xeb, 0x8c, 0x6e, 0xb4, 0x73, 0x3d, 0x5a, 0x57, 0x5e, 0xe9, 0x75, 0xf1, 0xf5, 0xa1, 0xb3, - 0x9b, 0xba, 0x82, 0x45, 0xa2, 0x2b, 0xaf, 0xf4, 0xba, 0xb8, 0xdc, 0x1f, 0xec, 0xa6, 0xae, 0xa0, - 0x13, 0xba, 0xf2, 0xc2, 0x2e, 0x2a, 0x76, 0xc3, 0x97, 0x9a, 0x4d, 0x65, 0x03, 0xaa, 0x7b, 0x7a, - 0x37, 0xd5, 0x95, 0x60, 0xc3, 0xb9, 0x13, 0x47, 0x3a, 0x4b, 0x2f, 0xb5, 0x89, 0x55, 0x93, 0x16, - 0xa0, 0x87, 0xf2, 0x2c, 0x1d, 0x23, 0xa0, 0xb3, 0x74, 0x0c, 0x48, 0x07, 0x94, 0x68, 0x84, 0xad, - 0x6c, 0xca, 0x03, 0x4a, 0xc4, 0xd1, 0x01, 0x25, 0x19, 0x6c, 0x2f, 0xa1, 0x53, 0x4b, 0xeb, 0x9e, - 0xee, 0xef, 0x20, 0x5d, 0xde, 0x95, 0xef, 0x46, 0x1e, 0x99, 0xe2, 0x24, 0xf0, 0xc8, 0x14, 0x07, - 0xd3, 0x31, 0x42, 0xc1, 0xb5, 0x4d, 0xab, 0x31, 0xab, 0x9b, 0xcd, 0x8e, 0x43, 0x94, 0xff, 0x4f, - 0x1e, 0x23, 0x11, 0x34, 0x1d, 0x23, 0x11, 0x10, 0x5d, 0xa0, 0x29, 0xa8, 0xe4, 0xba, 0xe6, 0xaa, - 0xc5, 0xcf, 0x95, 0x9d, 0xa6, 0xa7, 0xfc, 0xff, 0xf2, 0x02, 0x9d, 0x44, 0x43, 0x17, 0xe8, 0x24, - 0x38, 0xdc, 0x3a, 0x25, 0xa4, 0x27, 0x54, 0xfe, 0x52, 0xe4, 0xd6, 0x29, 0x81, 0x06, 0x6e, 0x9d, - 0x92, 0x52, 0x1b, 0xce, 0xa2, 0x02, 0xdb, 0x93, 0xcd, 0x9b, 0xc1, 0x5b, 0xf5, 0x5f, 0x96, 0xd7, - 0xc7, 0x28, 0x9e, 0xae, 0x8f, 0x51, 0x98, 0xcc, 0x87, 0x77, 0xc1, 0x5f, 0x49, 0xe3, 0x13, 0xc8, - 0x3f, 0x56, 0x06, 0xdf, 0x12, 0xf9, 0xf0, 0x91, 0xf2, 0xdd, 0x99, 0x34, 0x46, 0xc1, 0xf0, 0x88, - 0x15, 0x92, 0x19, 0x69, 0xe4, 0x81, 0x49, 0x36, 0x94, 0xcf, 0xa7, 0x32, 0x62, 0x04, 0x32, 0x23, - 0x06, 0xc3, 0x6f, 0xa1, 0xb3, 0x21, 0x6c, 0x81, 0xb4, 0x56, 0x82, 0x99, 0xe9, 0x7b, 0x32, 0xf2, - 0x36, 0x38, 0x99, 0x8c, 0x6e, 0x83, 0x93, 0x31, 0x49, 0xac, 0xb9, 0xe8, 0xfe, 0xea, 0x0e, 0xac, - 0x03, 0x09, 0xa6, 0x30, 0x48, 0x62, 0xcd, 0xa5, 0xf9, 0xbd, 0x3b, 0xb0, 0x0e, 0x64, 0x9a, 0xc2, - 0x00, 0xff, 0x50, 0x06, 0x5d, 0x4a, 0x46, 0x95, 0x9a, 0xcd, 0x59, 0xdb, 0x09, 0x71, 0xca, 0xf7, - 0x65, 0xe4, 0x8b, 0x86, 0xdd, 0x15, 0x9b, 0xeb, 0xd1, 0x76, 0x59, 0x01, 0xfe, 0x24, 0x1a, 0x29, - 0x75, 0x0c, 0xd3, 0x83, 0x87, 0x37, 0xba, 0x71, 0xfe, 0xfe, 0x4c, 0xe4, 0x88, 0x23, 0x62, 0xe1, - 0x88, 0x23, 0x02, 0xf0, 0xeb, 0x68, 0xbc, 0x46, 0x1a, 0x1d, 0xc7, 0xf4, 0x36, 0x35, 0x48, 0x3d, - 0x49, 0x79, 0xfc, 0x40, 0x46, 0x9e, 0xc4, 0x62, 0x14, 0x74, 0x12, 0x8b, 0x01, 0x31, 0x41, 0x13, - 0x33, 0x0f, 0x3d, 0xe2, 0x58, 0x7a, 0x13, 0x2a, 0xa9, 0x79, 0xb6, 0xa3, 0xaf, 0x92, 0x19, 0x4b, - 0x5f, 0x69, 0x12, 0xe5, 0x8b, 0x19, 0x79, 0x5f, 0x95, 0x4e, 0x4a, 0xf7, 0x55, 0xe9, 0x58, 0xbc, - 0x86, 0x1e, 0x4f, 0xc2, 0x96, 0x4d, 0x17, 0xea, 0xf9, 0x52, 0x46, 0xde, 0x58, 0x75, 0xa1, 0xa5, - 0x1b, 0xab, 0x2e, 0x68, 0x08, 0xcf, 0x9d, 0xe4, 0x17, 0xa2, 0xfc, 0x58, 0x46, 0xbe, 0x64, 0x4c, - 0xa4, 0x9a, 0xeb, 0xd1, 0x52, 0xdc, 0x4a, 0xee, 0xa6, 0xf8, 0x54, 0x28, 0x3f, 0xde, 0x9d, 0x6f, - 0xa0, 0xf4, 0x29, 0x2e, 0x19, 0x77, 0x53, 0xfc, 0x11, 0x94, 0x9f, 0xe8, 0xce, 0x37, 0xb4, 0x8b, - 0x48, 0x76, 0x67, 0xa8, 0xa7, 0xdb, 0xf2, 0x2b, 0x3f, 0x99, 0x91, 0xcf, 0xe9, 0x69, 0x84, 0xf4, - 0x9c, 0x9e, 0xea, 0x10, 0xf0, 0x7a, 0x82, 0x45, 0xbd, 0xf2, 0x53, 0x11, 0x2d, 0x8c, 0x51, 0x50, - 0x2d, 0x8c, 0x1b, 0xe2, 0xbf, 0x9e, 0x60, 0x38, 0xae, 0xfc, 0xbd, 0x74, 0x5e, 0x81, 0x50, 0x13, - 0xec, 0xcd, 0x5f, 0x4f, 0xb0, 0x8f, 0x56, 0xfe, 0x7e, 0x3a, 0xaf, 0xf0, 0x79, 0x35, 0x6e, 0x56, - 0x4d, 0x27, 0xa4, 0x8e, 0x67, 0x33, 0xce, 0x92, 0x36, 0xfd, 0x5c, 0x74, 0x42, 0x4a, 0x24, 0x83, - 0x09, 0x29, 0x11, 0x93, 0xc4, 0x9a, 0x7f, 0xf7, 0xcf, 0xef, 0xc0, 0x5a, 0x98, 0x46, 0x13, 0x31, - 0x49, 0xac, 0xb9, 0x18, 0xbe, 0xb2, 0x03, 0x6b, 0x61, 0x1a, 0x4d, 0xc4, 0xe0, 0xcf, 0xa0, 0x73, - 0x21, 0xe6, 0x2e, 0x71, 0xdc, 0xb0, 0xeb, 0x7f, 0x21, 0x23, 0x5f, 0x25, 0xa4, 0xd0, 0xcd, 0xf5, - 0x68, 0x69, 0x2c, 0x12, 0xb9, 0x73, 0xa1, 0xfc, 0xe2, 0x4e, 0xdc, 0xc3, 0x5b, 0x90, 0x14, 0x54, - 0x22, 0x77, 0x2e, 0x97, 0x5f, 0xda, 0x89, 0x7b, 0x78, 0x0d, 0x92, 0x82, 0x9a, 0xea, 0x47, 0xbd, - 0xb0, 0xb7, 0x53, 0x7f, 0x2c, 0x83, 0x86, 0x6b, 0x9e, 0x43, 0xf4, 0x16, 0xf7, 0x4b, 0x9e, 0x40, - 0x03, 0xcc, 0x48, 0xc2, 0xb7, 0x53, 0xd6, 0x82, 0xdf, 0xf8, 0x12, 0x1a, 0x9d, 0xd7, 0x5d, 0x0f, - 0x4a, 0x56, 0x2c, 0x83, 0x3c, 0x04, 0x03, 0xe1, 0x9c, 0x16, 0x81, 0xe2, 0x79, 0x46, 0xc7, 0xca, - 0x41, 0x40, 0x88, 0xdc, 0x8e, 0xee, 0xb8, 0x03, 0xef, 0x6d, 0x15, 0x7b, 0xc0, 0xfb, 0x36, 0x52, - 0x56, 0xfd, 0x7a, 0x06, 0xc5, 0xcc, 0x37, 0xf6, 0xef, 0x3f, 0xb0, 0x84, 0xc6, 0x22, 0x41, 0x48, - 0xb8, 0x95, 0xf3, 0x2e, 0x63, 0x94, 0x44, 0x4b, 0xe3, 0xa7, 0x03, 0xeb, 0xda, 0x3b, 0xda, 0x3c, - 0x77, 0xb5, 0xee, 0xdf, 0xde, 0x2a, 0xe6, 0x3a, 0x4e, 0x53, 0x13, 0x50, 0xdc, 0x15, 0xf0, 0x1f, - 0x16, 0xc2, 0x08, 0x0b, 0xf8, 0x12, 0x77, 0x66, 0xc8, 0x84, 0x0e, 0xda, 0x91, 0x74, 0x1a, 0xcc, - 0x79, 0xe1, 0x93, 0x68, 0xb8, 0xd2, 0x6a, 0x13, 0xc7, 0xb5, 0x2d, 0xdd, 0xb3, 0xfd, 0xb4, 0x7d, - 0xe0, 0xbc, 0x6b, 0x0a, 0x70, 0xd1, 0xa1, 0x54, 0xa4, 0xc7, 0x57, 0xfc, 0x2c, 0xd5, 0x39, 0x88, - 0x6d, 0x71, 0x2a, 0x21, 0x4b, 0xb5, 0x9f, 0x6b, 0xfa, 0x0a, 0xea, 0xbd, 0xe3, 0xea, 0x60, 0x87, - 0x1d, 0x90, 0x76, 0x28, 0x40, 0x24, 0x05, 0x0a, 0x7c, 0x15, 0xf5, 0xc1, 0xb9, 0xd5, 0x55, 0x7a, - 0x81, 0x16, 0xdc, 0xc6, 0x9b, 0x00, 0x11, 0x9d, 0x74, 0x19, 0x0d, 0xbe, 0x8d, 0x0a, 0xe1, 0xa5, - 0x1c, 0x24, 0x9a, 0xf4, 0x63, 0x6c, 0x42, 0x6a, 0x8b, 0xf5, 0x00, 0xc7, 0x32, 0x54, 0x8a, 0x2c, - 0x62, 0x05, 0xf1, 0x1c, 0x1a, 0x0b, 0x61, 0x54, 0x44, 0x7e, 0x6c, 0x5f, 0x48, 0xed, 0x22, 0xf0, - 0xa2, 0xe2, 0x14, 0x59, 0x45, 0x8b, 0xe1, 0x0a, 0xea, 0xf7, 0x7d, 0xc6, 0x07, 0x76, 0x54, 0xd2, - 0x53, 0xdc, 0x67, 0xbc, 0x5f, 0xf4, 0x16, 0xf7, 0xcb, 0xe3, 0x59, 0x34, 0xaa, 0xd9, 0x1d, 0x8f, - 0x2c, 0xdb, 0xfc, 0xce, 0x91, 0x07, 0x7f, 0x84, 0x36, 0x39, 0x14, 0x53, 0xf7, 0x6c, 0x3f, 0x33, - 0x88, 0x98, 0xa1, 0x42, 0x2e, 0x85, 0x17, 0xd1, 0x78, 0xec, 0xfa, 0x52, 0xcc, 0xd7, 0x21, 0x7c, - 0x5e, 0x9c, 0x59, 0xbc, 0x28, 0xfe, 0xfe, 0x0c, 0xea, 0x5b, 0x76, 0x74, 0xd3, 0x73, 0xb9, 0x09, - 0xf7, 0x99, 0xc9, 0x0d, 0x47, 0x6f, 0x53, 0xfd, 0x98, 0x84, 0xe0, 0x25, 0x77, 0xf5, 0x66, 0x87, - 0xb8, 0x53, 0xf7, 0xe8, 0xd7, 0xfd, 0xfb, 0xad, 0xe2, 0x27, 0xf6, 0x90, 0x3d, 0xfc, 0x5a, 0xc0, - 0x89, 0xd5, 0x40, 0x55, 0xc0, 0x83, 0xbf, 0x44, 0x15, 0x60, 0x38, 0xbc, 0x88, 0x10, 0xff, 0xd4, - 0x52, 0xbb, 0xcd, 0xed, 0xc1, 0x05, 0x63, 0x57, 0x1f, 0xc3, 0x14, 0x3b, 0x10, 0x98, 0xde, 0x16, - 0xd3, 0x95, 0x0a, 0x1c, 0xa8, 0x16, 0x2c, 0xf3, 0x16, 0xf9, 0x62, 0x1a, 0x09, 0x25, 0xee, 0x37, - 0x36, 0x41, 0x48, 0xd1, 0x62, 0x78, 0x05, 0x8d, 0x71, 0xbe, 0x41, 0x34, 0xc6, 0x51, 0x79, 0x56, - 0x88, 0xa0, 0x99, 0xd2, 0x06, 0x6d, 0x34, 0x38, 0x58, 0xac, 0x23, 0x52, 0x02, 0x4f, 0x85, 0xc1, - 0xe2, 0x21, 0x37, 0xaa, 0x32, 0x06, 0x1a, 0x7b, 0x7e, 0x7b, 0xab, 0xa8, 0xf8, 0xe5, 0x59, 0x4a, - 0xd5, 0xa4, 0xc4, 0x29, 0x50, 0x44, 0xe4, 0xc1, 0xb4, 0xbe, 0x90, 0xc0, 0x23, 0xaa, 0xf3, 0x72, - 0x11, 0x3c, 0x8d, 0x46, 0x02, 0x73, 0xb4, 0x3b, 0x77, 0x2a, 0x65, 0x30, 0x38, 0xe7, 0x49, 0x40, - 0x23, 0x81, 0x1e, 0x45, 0x26, 0x52, 0x19, 0xc1, 0x33, 0x85, 0x59, 0xa0, 0x47, 0x3c, 0x53, 0xda, - 0x09, 0x9e, 0x29, 0x55, 0xfc, 0x0a, 0x1a, 0x2a, 0xdd, 0xab, 0x71, 0x8f, 0x1b, 0x57, 0x39, 0x15, - 0x46, 0xd8, 0x85, 0xdc, 0x39, 0xdc, 0x3b, 0x47, 0x6c, 0xba, 0x48, 0x8f, 0x67, 0xd0, 0xa8, 0xf4, - 0xa2, 0xe5, 0x2a, 0xa7, 0x81, 0x03, 0x4b, 0x5f, 0x0a, 0x98, 0x3a, 0xcf, 0xa0, 0x2b, 0x25, 0x08, - 0x92, 0x0b, 0x51, 0xad, 0xa1, 0xdb, 0xef, 0x66, 0xd3, 0xde, 0xd0, 0x08, 0x38, 0xf7, 0x80, 0xf9, - 0xfa, 0x00, 0xd3, 0x1a, 0x83, 0xa3, 0xea, 0x0e, 0xc3, 0x49, 0xe9, 0x9b, 0xe4, 0x62, 0xf8, 0x1d, - 0x84, 0x21, 0xbe, 0x29, 0x31, 0xfc, 0x0b, 0x8e, 0x4a, 0xd9, 0x55, 0xce, 0x42, 0x10, 0x27, 0x1c, - 0xf5, 0x2e, 0xab, 0x94, 0xa7, 0x2e, 0xf1, 0xe9, 0xe3, 0xa2, 0xce, 0x4a, 0xd5, 0x83, 0xec, 0xb5, - 0xa6, 0x21, 0xb6, 0x38, 0x81, 0x2b, 0xde, 0x40, 0xe7, 0xaa, 0x0e, 0x79, 0x60, 0xda, 0x1d, 0xd7, - 0x5f, 0x3e, 0xfc, 0x79, 0xeb, 0xdc, 0x8e, 0xf3, 0xd6, 0x93, 0xbc, 0xe2, 0x33, 0x6d, 0x87, 0x3c, - 0xa8, 0xfb, 0xa1, 0x7b, 0xa4, 0x98, 0x17, 0x69, 0xdc, 0xa9, 0xb8, 0xc0, 0xb1, 0x89, 0xc3, 0x4d, - 0xe2, 0x2a, 0x4a, 0x38, 0xd5, 0x32, 0x3f, 0x2d, 0x33, 0xc0, 0x89, 0xe2, 0x8a, 0x14, 0xc3, 0x1a, - 0xc2, 0xb7, 0xa6, 0xfd, 0xcb, 0xae, 0x52, 0xa3, 0x61, 0x77, 0x2c, 0xcf, 0x55, 0x1e, 0x03, 0x66, - 0x2a, 0x15, 0xcb, 0x6a, 0x23, 0x08, 0xe3, 0x55, 0xd7, 0x39, 0x5e, 0x14, 0x4b, 0xbc, 0x34, 0x9e, - 0x47, 0x85, 0xaa, 0x63, 0x3e, 0xd0, 0x3d, 0x72, 0x9b, 0x6c, 0x56, 0xed, 0xa6, 0xd9, 0xd8, 0x04, - 0x2b, 0x7a, 0x3e, 0x55, 0xb6, 0x19, 0xae, 0xbe, 0x4e, 0x36, 0xeb, 0x6d, 0xc0, 0x8a, 0xcb, 0x4a, - 0xb4, 0xa4, 0x18, 0x56, 0xe7, 0xf1, 0xdd, 0x85, 0xd5, 0x21, 0xa8, 0xc0, 0xaf, 0xca, 0x1e, 0x7a, - 0xc4, 0xa2, 0x4b, 0xbd, 0xcb, 0x2d, 0xe6, 0x95, 0xc8, 0xd5, 0x5a, 0x80, 0xe7, 0xa9, 0x9c, 0xd8, - 0x28, 0x23, 0x01, 0x58, 0x6c, 0x58, 0xb4, 0x88, 0xfa, 0xa5, 0x9c, 0x38, 0x75, 0xe2, 0xf3, 0x28, - 0x2f, 0x44, 0x75, 0x85, 0x68, 0x1c, 0x10, 0x01, 0x2b, 0xcf, 0x43, 0xfd, 0x0c, 0xf2, 0x6d, 0x47, - 0xe0, 0x36, 0x06, 0x21, 0xef, 0xfd, 0x50, 0x5b, 0xa6, 0xa1, 0x85, 0x04, 0x10, 0x6e, 0x3c, 0xcc, - 0x78, 0x99, 0x13, 0xc2, 0x8d, 0x87, 0x19, 0x2f, 0xa5, 0x7c, 0x97, 0xd7, 0xd1, 0x10, 0x9f, 0x36, - 0x85, 0x68, 0x34, 0x10, 0x2e, 0xcb, 0x4f, 0x7a, 0xc5, 0xa2, 0x71, 0x09, 0x44, 0xf8, 0x25, 0x48, - 0xfb, 0xe6, 0xbb, 0xe4, 0xf5, 0x86, 0xdb, 0x17, 0x71, 0xe0, 0x47, 0xf2, 0xbe, 0xf9, 0x9e, 0x79, - 0x53, 0x68, 0x44, 0xd4, 0x24, 0x3f, 0xc1, 0x02, 0xcc, 0x79, 0x92, 0xfa, 0x6d, 0x4a, 0x39, 0x8b, - 0xc5, 0x22, 0x78, 0x09, 0x8d, 0xc7, 0x94, 0x87, 0xc7, 0xae, 0x81, 0x74, 0x1b, 0x09, 0x9a, 0x27, - 0xae, 0xa9, 0xb1, 0xb2, 0xea, 0xf7, 0x64, 0x63, 0x2b, 0x06, 0x15, 0x0c, 0xa7, 0x12, 0x3a, 0x07, - 0x04, 0xe3, 0xb3, 0x66, 0x82, 0x11, 0x88, 0xf0, 0x65, 0x34, 0x10, 0xc9, 0xfc, 0x06, 0x4e, 0x9a, - 0x41, 0xda, 0xb7, 0x00, 0x8b, 0xaf, 0xa3, 0x01, 0x3a, 0x7f, 0x5b, 0x91, 0x98, 0x4f, 0x1d, 0x0e, - 0x13, 0x27, 0x5c, 0x9f, 0x8e, 0x96, 0x91, 0xa2, 0x0b, 0xfb, 0x09, 0xba, 0xe2, 0xab, 0x55, 0x18, - 0x3b, 0x3d, 0xd8, 0x2b, 0xf6, 0xee, 0xb4, 0x57, 0x54, 0x7f, 0x33, 0x13, 0xd7, 0x7e, 0x7c, 0x33, - 0x1e, 0xf8, 0x85, 0xe5, 0xe6, 0xf2, 0x81, 0x62, 0xad, 0x41, 0x08, 0x18, 0x29, 0x84, 0x4b, 0x76, - 0xdf, 0x21, 0x5c, 0x72, 0x7b, 0x0c, 0xe1, 0xa2, 0xfe, 0xef, 0x7c, 0x57, 0x83, 0x8b, 0x23, 0x71, - 0x55, 0x7e, 0x91, 0x9e, 0x77, 0x68, 0xed, 0x25, 0x37, 0xb6, 0x6b, 0x67, 0xef, 0xc9, 0x75, 0x9d, - 0x8d, 0x1a, 0x57, 0x93, 0x29, 0xc5, 0x4c, 0xe9, 0x10, 0x1a, 0x28, 0x9f, 0x90, 0x29, 0x3d, 0x9a, - 0x5e, 0x4d, 0x2c, 0x80, 0x3f, 0x86, 0x06, 0xc3, 0x9c, 0xef, 0xbd, 0x42, 0xa0, 0xa9, 0x84, 0x54, - 0xef, 0x21, 0x25, 0xfe, 0x2c, 0xea, 0x93, 0xf2, 0xfb, 0x5d, 0xdb, 0x85, 0x85, 0xca, 0xa4, 0x18, - 0xbe, 0x90, 0x9d, 0x1d, 0xa2, 0xb9, 0xfd, 0x38, 0x53, 0xbc, 0x8c, 0x4e, 0x55, 0x1d, 0x62, 0x80, - 0x2d, 0xd4, 0xcc, 0xc3, 0xb6, 0xc3, 0x83, 0x4b, 0xb2, 0x01, 0x0c, 0x4b, 0x47, 0xdb, 0x47, 0xd3, - 0x45, 0x8d, 0xe3, 0x05, 0x46, 0x49, 0xc5, 0xe9, 0x7e, 0x82, 0xb5, 0xe4, 0x36, 0xd9, 0xdc, 0xb0, - 0x1d, 0x83, 0xc5, 0x5f, 0xe4, 0xfb, 0x09, 0x2e, 0xe8, 0x75, 0x8e, 0x12, 0xf7, 0x13, 0x72, 0xa1, - 0x89, 0x17, 0xd1, 0xd0, 0x7e, 0x43, 0x00, 0xfe, 0x62, 0x36, 0xc5, 0x74, 0xf1, 0xd1, 0x4d, 0xdd, - 0x10, 0xa4, 0xd1, 0xe9, 0x4d, 0x49, 0xa3, 0xf3, 0xad, 0x6c, 0x8a, 0x5d, 0xe6, 0x23, 0x9d, 0xee, - 0x22, 0x10, 0x86, 0x9c, 0xee, 0x22, 0xcc, 0x34, 0x62, 0x1a, 0x9a, 0x48, 0x14, 0x49, 0x8c, 0xd3, - 0xb7, 0x63, 0x62, 0x9c, 0x9f, 0xce, 0x75, 0xb3, 0x5b, 0x3d, 0x91, 0xfd, 0x5e, 0x64, 0x7f, 0x1d, - 0x0d, 0x05, 0x92, 0xe5, 0x49, 0x92, 0x47, 0x82, 0x80, 0xa3, 0x0c, 0x0c, 0x65, 0x04, 0x22, 0x7c, - 0x85, 0xb5, 0xb5, 0x66, 0xbe, 0xcb, 0x82, 0xee, 0x8d, 0xf0, 0x70, 0x6a, 0xba, 0xa7, 0xd7, 0x5d, - 0xf3, 0x5d, 0xa2, 0x05, 0x68, 0xf5, 0x9f, 0x66, 0x13, 0x8d, 0x7f, 0x4f, 0xfa, 0x68, 0x0f, 0x7d, - 0x94, 0x20, 0x44, 0x66, 0xb6, 0x7c, 0x22, 0xc4, 0x3d, 0x08, 0xf1, 0x4f, 0xb2, 0x89, 0x46, 0xde, - 0x27, 0x42, 0xdc, 0xcb, 0x6c, 0x71, 0x15, 0x0d, 0x6a, 0xf6, 0x86, 0x3b, 0x0d, 0x67, 0x16, 0x36, - 0x57, 0xc0, 0x44, 0xed, 0xd8, 0x1b, 0x6e, 0x1d, 0x4e, 0x23, 0x5a, 0x48, 0xa0, 0x7e, 0x3b, 0xdb, - 0xc5, 0x0c, 0xfe, 0x44, 0xf0, 0xef, 0xe7, 0x12, 0xf9, 0x2b, 0x59, 0xc9, 0xcc, 0xfe, 0x91, 0xce, - 0x1b, 0x57, 0x6b, 0xac, 0x91, 0x96, 0x1e, 0xcd, 0x1b, 0xe7, 0x02, 0x94, 0xa7, 0x9d, 0x09, 0x49, - 0xd4, 0xaf, 0x66, 0x23, 0x7e, 0x06, 0x27, 0xb2, 0xdb, 0xb5, 0xec, 0x02, 0xad, 0xe3, 0xae, 0x13, - 0x27, 0x92, 0xdb, 0xad, 0xe4, 0x7e, 0x30, 0x1b, 0xf1, 0x32, 0x79, 0x74, 0x53, 0x48, 0x7d, 0x35, - 0x1b, 0xf7, 0x98, 0x79, 0x74, 0x35, 0xe9, 0x2a, 0x1a, 0xe4, 0x72, 0x08, 0x96, 0x0a, 0x36, 0xef, - 0x33, 0x20, 0x5c, 0xa0, 0x06, 0x04, 0xea, 0xf7, 0x65, 0x91, 0xec, 0xfd, 0xf3, 0x88, 0xea, 0xd0, - 0xaf, 0x64, 0x65, 0xbf, 0xa7, 0x47, 0x57, 0x7f, 0x26, 0x11, 0xaa, 0x75, 0x56, 0x1a, 0x3c, 0x6c, - 0x56, 0xaf, 0x70, 0x03, 0x1f, 0x40, 0x35, 0x81, 0x42, 0xfd, 0x3f, 0xd9, 0x44, 0x67, 0xac, 0x47, - 0x57, 0x80, 0x37, 0xe0, 0x56, 0xbc, 0x61, 0x85, 0x13, 0x39, 0x5c, 0x42, 0xd2, 0xf1, 0x17, 0x8b, - 0x76, 0xef, 0x13, 0xe2, 0x8f, 0x27, 0x6c, 0xd7, 0x20, 0x96, 0x60, 0x62, 0x0a, 0x6d, 0x71, 0xe3, - 0xf6, 0x2f, 0xb2, 0x3b, 0xf9, 0xae, 0x3d, 0xca, 0xab, 0x6a, 0x7f, 0x55, 0xdf, 0x84, 0x18, 0x2b, - 0xb4, 0x27, 0x86, 0x59, 0x2c, 0xf6, 0x36, 0x03, 0x89, 0x2f, 0x62, 0x9c, 0x4a, 0xfd, 0xe3, 0xde, - 0x64, 0xc7, 0xa9, 0x47, 0x57, 0x84, 0xe7, 0x51, 0xbe, 0xaa, 0x7b, 0x6b, 0x5c, 0x93, 0xe1, 0xb5, - 0xae, 0xad, 0x7b, 0x6b, 0x1a, 0x40, 0xf1, 0x15, 0x34, 0xa0, 0xe9, 0x1b, 0x62, 0xea, 0x70, 0xb8, - 0xd8, 0x71, 0xf4, 0x0d, 0x9e, 0x3f, 0x3e, 0x40, 0x63, 0x35, 0xc8, 0xd3, 0xc0, 0x6e, 0xbe, 0x21, - 0xc8, 0x39, 0xcb, 0xd3, 0x10, 0x64, 0x67, 0x38, 0x8f, 0xf2, 0x53, 0xb6, 0xb1, 0x09, 0xc6, 0x2c, - 0xc3, 0xac, 0xb2, 0x15, 0xdb, 0xd8, 0xd4, 0x00, 0x8a, 0x7f, 0x28, 0x83, 0xfa, 0xe7, 0x88, 0x6e, - 0xd0, 0x11, 0x32, 0xd8, 0xcd, 0x16, 0xe4, 0xcd, 0xc3, 0xb1, 0x05, 0x19, 0x5f, 0x63, 0x95, 0x89, - 0x8a, 0xc2, 0xeb, 0xc7, 0xb7, 0xd0, 0xc0, 0xb4, 0xee, 0x91, 0x55, 0xdb, 0xd9, 0x04, 0xeb, 0x96, - 0xd1, 0xd0, 0x7a, 0x54, 0xd2, 0x1f, 0x9f, 0x88, 0xbd, 0x8c, 0x35, 0xf8, 0x2f, 0x2d, 0x28, 0x4c, - 0xc5, 0xc2, 0xf3, 0xb7, 0x0d, 0x85, 0x62, 0x61, 0x89, 0xda, 0x82, 0x34, 0x6d, 0xc1, 0xb5, 0xf2, - 0x70, 0xf2, 0xb5, 0x32, 0xec, 0x1e, 0xc1, 0x02, 0x0e, 0xb2, 0x23, 0x8c, 0xc0, 0xa2, 0xcf, 0x76, - 0x8f, 0x00, 0x85, 0xe4, 0x08, 0x9a, 0x40, 0xa2, 0x7e, 0xa3, 0x17, 0x25, 0xba, 0x59, 0x9c, 0x28, - 0xf9, 0x89, 0x92, 0x87, 0x4a, 0x5e, 0x8e, 0x29, 0xf9, 0x44, 0xdc, 0x71, 0xe7, 0x03, 0xaa, 0xe1, - 0x3f, 0x92, 0x8f, 0xb9, 0xfd, 0x3d, 0xda, 0xa7, 0xcb, 0x50, 0x7a, 0xbd, 0x3b, 0x4a, 0x2f, 0x18, - 0x10, 0x7d, 0x3b, 0x0e, 0x88, 0xfe, 0xdd, 0x0e, 0x88, 0x81, 0xd4, 0x01, 0x11, 0x2a, 0xc8, 0x60, - 0xaa, 0x82, 0x54, 0xf8, 0xa0, 0x41, 0xdd, 0x33, 0xef, 0x9c, 0xdf, 0xde, 0x2a, 0x8e, 0xd2, 0xd1, - 0x94, 0x98, 0x73, 0x07, 0x58, 0xa8, 0x5f, 0xcf, 0x77, 0xf1, 0xd5, 0x3d, 0x12, 0x1d, 0xb9, 0x81, - 0x72, 0xa5, 0x76, 0x9b, 0xeb, 0xc7, 0x29, 0xc1, 0x4d, 0x38, 0xa5, 0x14, 0xa5, 0xc6, 0x2f, 0xa1, - 0x5c, 0xe9, 0x5e, 0x2d, 0x1a, 0x71, 0xb8, 0x74, 0xaf, 0xc6, 0xbf, 0x24, 0xb5, 0xec, 0xbd, 0x1a, - 0x7e, 0x39, 0x0c, 0xfd, 0xb3, 0xd6, 0xb1, 0xd6, 0xf9, 0x41, 0x91, 0x1b, 0xc1, 0xfa, 0x96, 0x36, - 0x0d, 0x8a, 0xa2, 0xc7, 0xc5, 0x08, 0x6d, 0x44, 0x9b, 0xfa, 0x76, 0xaf, 0x4d, 0xfd, 0x3b, 0x6a, - 0xd3, 0xc0, 0x6e, 0xb5, 0x69, 0x70, 0x17, 0xda, 0x84, 0x76, 0xd4, 0xa6, 0xa1, 0x83, 0x6b, 0x53, - 0x1b, 0x4d, 0xc4, 0xe3, 0x2b, 0x04, 0x1a, 0xa1, 0x21, 0x1c, 0xc7, 0x72, 0xc3, 0x12, 0x78, 0xfa, - 0xef, 0x30, 0x6c, 0x9d, 0xe5, 0x59, 0x8c, 0x66, 0x29, 0xd4, 0x12, 0x4a, 0xab, 0xbf, 0x98, 0x4d, - 0x0f, 0x0b, 0x71, 0x3c, 0xa7, 0xb8, 0xef, 0x4a, 0x94, 0x52, 0x5e, 0x76, 0x88, 0x4a, 0x97, 0x72, - 0x84, 0x6d, 0x92, 0xcc, 0xbe, 0x96, 0x49, 0x8b, 0x55, 0x71, 0x20, 0x89, 0x7d, 0x38, 0x6e, 0xac, - 0x06, 0xd6, 0xf3, 0xae, 0x6c, 0xa5, 0x16, 0x4d, 0xdb, 0x97, 0xdb, 0x67, 0xda, 0xbe, 0xdf, 0xcc, - 0xa0, 0x53, 0xb7, 0x3b, 0x2b, 0x84, 0x1b, 0xa7, 0x05, 0xcd, 0x78, 0x07, 0x21, 0x0a, 0xe6, 0x46, - 0x2c, 0x19, 0x30, 0x62, 0xf9, 0x88, 0x18, 0x67, 0x22, 0x52, 0x60, 0x32, 0xa4, 0x66, 0x06, 0x2c, - 0x17, 0x7c, 0x13, 0xcb, 0xf5, 0xce, 0x0a, 0xa9, 0xc7, 0x2c, 0x59, 0x04, 0xee, 0x13, 0xaf, 0x30, - 0xe3, 0xf5, 0xfd, 0x1a, 0x8d, 0xfc, 0x7c, 0x36, 0x35, 0xb4, 0xc7, 0xb1, 0xcd, 0xac, 0xf0, 0xe9, - 0xc4, 0x5e, 0x89, 0x66, 0x58, 0x48, 0x20, 0x89, 0x70, 0x4c, 0xe2, 0x92, 0x2c, 0xb0, 0x63, 0x9e, - 0xef, 0xe3, 0x7d, 0x15, 0xd8, 0xef, 0x65, 0x52, 0x43, 0xb0, 0x1c, 0x57, 0x81, 0xa9, 0xbf, 0x9d, - 0xf5, 0x23, 0xbf, 0x1c, 0xe8, 0x13, 0xae, 0xa2, 0x41, 0x1e, 0xde, 0x5e, 0xb6, 0xad, 0xe5, 0x57, - 0x79, 0x70, 0x35, 0x1c, 0x10, 0xd0, 0x65, 0xde, 0x8f, 0x4c, 0x11, 0x24, 0x7a, 0x84, 0x65, 0xde, - 0xe4, 0x50, 0x4a, 0x2f, 0x90, 0xd0, 0x85, 0x7c, 0xe6, 0xa1, 0xe9, 0xc1, 0xae, 0x80, 0xf6, 0x65, - 0x8e, 0x2d, 0xe4, 0xe4, 0xa1, 0xe9, 0xb1, 0x3d, 0x41, 0x80, 0xa6, 0x8b, 0x74, 0x2d, 0xcc, 0x66, - 0xc6, 0x17, 0x69, 0x97, 0x27, 0x75, 0xe3, 0xce, 0x5c, 0x57, 0xd1, 0x20, 0x37, 0x58, 0xe5, 0x66, - 0x26, 0xbc, 0xb5, 0xdc, 0xc4, 0x15, 0x5a, 0x1b, 0x10, 0x50, 0x8e, 0x1a, 0x59, 0x0d, 0x0d, 0xeb, - 0x80, 0xa3, 0x03, 0x10, 0x8d, 0x63, 0xd4, 0xed, 0x6c, 0x3c, 0x00, 0xcd, 0xa3, 0x7b, 0x28, 0xb8, - 0x22, 0x1b, 0xab, 0x81, 0x85, 0x26, 0x6c, 0xb8, 0x44, 0x5b, 0x59, 0xb6, 0xef, 0xba, 0x8e, 0x06, - 0x6e, 0x93, 0x4d, 0x66, 0x57, 0xd9, 0x17, 0x9a, 0xe2, 0xae, 0x73, 0x98, 0x78, 0xa3, 0xe9, 0xd3, - 0xa9, 0xbf, 0x91, 0x8d, 0x87, 0xd6, 0x79, 0x74, 0x85, 0xfd, 0x51, 0xd4, 0x0f, 0xa2, 0xac, 0xf8, - 0x57, 0xea, 0x20, 0x40, 0x10, 0xb7, 0x6c, 0xe1, 0xeb, 0x93, 0xa9, 0x3f, 0xde, 0x17, 0x8d, 0xb7, - 0xf4, 0xe8, 0x4a, 0xef, 0x13, 0x68, 0x68, 0xda, 0xb6, 0x5c, 0xd3, 0xf5, 0x88, 0xd5, 0xf0, 0x15, - 0xf6, 0x31, 0xba, 0x61, 0x69, 0x84, 0x60, 0xd1, 0xf3, 0x46, 0xa0, 0xde, 0x8f, 0xf2, 0xe2, 0xe7, - 0xd1, 0x20, 0x88, 0x1c, 0xec, 0x90, 0x85, 0x34, 0xb1, 0x2b, 0x14, 0x18, 0x35, 0x42, 0x0e, 0x49, - 0xf1, 0x1d, 0x34, 0x30, 0xbd, 0x66, 0x36, 0x0d, 0x87, 0x58, 0x3c, 0x1f, 0xfa, 0x93, 0xc9, 0xd1, - 0xb1, 0x26, 0xe1, 0x5f, 0xa0, 0x65, 0xcd, 0x69, 0xf0, 0x62, 0x92, 0xef, 0x11, 0x87, 0x4d, 0xfc, - 0xcd, 0x2c, 0x42, 0x61, 0x01, 0xfc, 0x04, 0xca, 0x06, 0x89, 0x78, 0xc0, 0x0c, 0x44, 0xd2, 0xa0, - 0x2c, 0x4c, 0xc5, 0x7c, 0x6c, 0x67, 0x77, 0x1c, 0xdb, 0x77, 0x50, 0x1f, 0xbb, 0x51, 0x02, 0x4b, - 0x6d, 0x21, 0x04, 0x4c, 0x6a, 0x83, 0x27, 0x81, 0x9e, 0x1d, 0x16, 0x61, 0x67, 0x27, 0x59, 0x3d, - 0x33, 0x66, 0x13, 0x0d, 0xd4, 0x0b, 0x7f, 0xe1, 0x4b, 0x28, 0x0f, 0x52, 0xcc, 0xc0, 0x39, 0x11, - 0xdc, 0x44, 0x23, 0xf2, 0x03, 0x3c, 0xed, 0xa6, 0x69, 0xdb, 0xf2, 0x68, 0xd5, 0xd0, 0xea, 0x61, - 0x2e, 0x17, 0x0e, 0x93, 0xe4, 0xc2, 0x61, 0xea, 0x3f, 0xcf, 0x26, 0x44, 0x02, 0x7b, 0x74, 0x87, - 0xc9, 0x8b, 0x08, 0x81, 0x23, 0x33, 0x95, 0xa7, 0xef, 0x02, 0x01, 0xa3, 0x04, 0x18, 0x81, 0xda, - 0x4a, 0xdb, 0xfa, 0x90, 0x58, 0xfd, 0x9d, 0x4c, 0x2c, 0x7c, 0xd4, 0x81, 0xe4, 0x28, 0xee, 0x7a, - 0xb2, 0xfb, 0xdc, 0x26, 0xfa, 0x7d, 0x91, 0xdb, 0x5b, 0x5f, 0xc8, 0xdf, 0x72, 0x08, 0x3b, 0xbf, - 0xa3, 0xfc, 0x96, 0x6f, 0x64, 0x93, 0x82, 0x69, 0x1d, 0x4f, 0x15, 0x0f, 0x73, 0x8d, 0xe7, 0xf7, - 0x90, 0x6b, 0xfc, 0x6d, 0x34, 0x16, 0x09, 0x31, 0xc5, 0xb3, 0x63, 0x5d, 0xea, 0x1e, 0xab, 0x2a, - 0xdd, 0x05, 0x5e, 0x22, 0x53, 0xff, 0x6f, 0xa6, 0x7b, 0x80, 0xb1, 0x23, 0x57, 0x9d, 0x04, 0x01, - 0xe4, 0xfe, 0x62, 0x04, 0x70, 0x08, 0xc7, 0xcc, 0xe3, 0x2d, 0x80, 0x0f, 0xc8, 0xe4, 0xf1, 0x7e, - 0x0b, 0xe0, 0xc7, 0x33, 0x3b, 0xc6, 0x87, 0x3b, 0x6a, 0x19, 0xa8, 0xff, 0x31, 0x93, 0x18, 0xc7, - 0xed, 0x40, 0xed, 0x7a, 0x19, 0xf5, 0x31, 0xb3, 0x15, 0xde, 0x2a, 0x21, 0xf2, 0x3d, 0x85, 0xa6, - 0x94, 0xe7, 0x65, 0xf0, 0x3c, 0xea, 0x67, 0x6d, 0x30, 0xa2, 0x19, 0x22, 0x13, 0xda, 0x69, 0xa4, - 0x4d, 0x8e, 0x1c, 0xad, 0xfe, 0x56, 0x26, 0x16, 0x56, 0xee, 0x08, 0xbf, 0x2d, 0x9c, 0xaa, 0x73, - 0xbb, 0x9f, 0xaa, 0xd5, 0x3f, 0xca, 0x26, 0x47, 0xb5, 0x3b, 0xc2, 0x0f, 0x39, 0x8c, 0xeb, 0xaa, - 0xfd, 0xad, 0x5b, 0xcb, 0x68, 0x54, 0x96, 0x05, 0x5f, 0xb6, 0x2e, 0x26, 0xc7, 0xf6, 0x4b, 0x69, - 0x45, 0x84, 0x87, 0xfa, 0x5e, 0x26, 0x1e, 0x90, 0xef, 0xc8, 0xe7, 0xa7, 0xfd, 0x69, 0x8b, 0xfc, - 0x29, 0x1f, 0x90, 0xb5, 0xe6, 0x30, 0x3e, 0xe5, 0x03, 0xb2, 0x6a, 0xec, 0xef, 0x53, 0x7e, 0x26, - 0x9b, 0x16, 0xcf, 0xf0, 0xc8, 0x3f, 0xe8, 0x53, 0xa2, 0x90, 0x59, 0xcb, 0xf8, 0xa7, 0x3d, 0x91, - 0x16, 0x40, 0x30, 0x85, 0x67, 0x8c, 0xcf, 0xfe, 0xc6, 0x78, 0xa2, 0xb0, 0x3e, 0x20, 0x8a, 0x7c, - 0x3c, 0x84, 0xf5, 0x01, 0x19, 0x2a, 0x1f, 0x3c, 0x61, 0xfd, 0x5a, 0x76, 0xb7, 0x41, 0x34, 0x4f, - 0x84, 0x17, 0x13, 0xde, 0x97, 0xb3, 0xf1, 0xe0, 0xae, 0x47, 0x2e, 0xa6, 0x59, 0xd4, 0xc7, 0xc3, - 0xcc, 0xa6, 0x0a, 0x87, 0xe1, 0xd3, 0x76, 0x34, 0xfc, 0x3b, 0x6e, 0x22, 0xfe, 0x50, 0xb2, 0x3b, - 0x91, 0x30, 0x5a, 0xf5, 0xdb, 0x99, 0x48, 0x24, 0xd4, 0x23, 0xb9, 0x42, 0xd8, 0xd7, 0x92, 0x84, - 0x5f, 0xf1, 0x2f, 0x33, 0xf3, 0x91, 0x24, 0x93, 0xc1, 0xf7, 0x94, 0x89, 0xa7, 0x9b, 0xcd, 0x68, - 0x79, 0xee, 0x73, 0xff, 0x1b, 0x59, 0x34, 0x1e, 0x23, 0xc5, 0x97, 0xa4, 0x28, 0x34, 0x70, 0x2d, - 0x19, 0x31, 0xce, 0x66, 0xf1, 0x68, 0xf6, 0x70, 0x93, 0x7a, 0x09, 0xe5, 0xcb, 0xfa, 0x26, 0xfb, - 0xb6, 0x5e, 0xc6, 0xd2, 0xd0, 0x37, 0xc5, 0x1b, 0x37, 0xc0, 0xe3, 0x15, 0x74, 0x86, 0xbd, 0x87, - 0x98, 0xb6, 0xb5, 0x6c, 0xb6, 0x48, 0xc5, 0x5a, 0x30, 0x9b, 0x4d, 0xd3, 0xe5, 0x8f, 0x66, 0x57, - 0xb7, 0xb7, 0x8a, 0x97, 0x3d, 0xdb, 0xd3, 0x9b, 0x75, 0xe2, 0x93, 0xd5, 0x3d, 0xb3, 0x45, 0xea, - 0xa6, 0x55, 0x6f, 0x01, 0xa5, 0xc0, 0x32, 0x99, 0x15, 0xae, 0xb0, 0x2c, 0x98, 0xb5, 0x86, 0x6e, - 0x59, 0xc4, 0xa8, 0x58, 0x53, 0x9b, 0x1e, 0x61, 0x8f, 0x6d, 0x39, 0x76, 0x25, 0xc8, 0x7c, 0xaf, - 0x19, 0x9a, 0x32, 0x5e, 0xa1, 0x04, 0x5a, 0x42, 0x21, 0xf5, 0xd7, 0xf3, 0x09, 0x41, 0x70, 0x8f, - 0x91, 0xfa, 0xf8, 0x3d, 0x9d, 0xdf, 0xa1, 0xa7, 0xaf, 0xa1, 0x7e, 0x1e, 0x67, 0x92, 0x3f, 0x30, - 0x80, 0xb1, 0xf8, 0x03, 0x06, 0x12, 0x5f, 0x68, 0x38, 0x15, 0x6e, 0xa2, 0x89, 0x65, 0xda, 0x4d, - 0xc9, 0x9d, 0xd9, 0xb7, 0x8f, 0xce, 0xec, 0xc2, 0x0f, 0xbf, 0x85, 0xce, 0x01, 0x36, 0xa1, 0x5b, - 0xfb, 0xa1, 0x2a, 0x88, 0xcc, 0xc4, 0xaa, 0x4a, 0xee, 0xdc, 0xb4, 0xf2, 0xf8, 0x53, 0x68, 0x38, - 0x18, 0x20, 0x26, 0x71, 0xf9, 0xcb, 0x45, 0x97, 0x71, 0xc6, 0xc2, 0x9e, 0x51, 0x30, 0x98, 0x68, - 0xc9, 0xa1, 0xb3, 0x24, 0x5e, 0xea, 0x7f, 0xc8, 0x74, 0x0b, 0x7b, 0x7c, 0xe4, 0xb3, 0xf2, 0x2b, - 0xa8, 0xdf, 0x60, 0x1f, 0xc5, 0x75, 0xaa, 0x7b, 0x60, 0x64, 0x46, 0xaa, 0xf9, 0x65, 0xd4, 0x3f, - 0xcc, 0x74, 0x8d, 0xb6, 0x7c, 0xdc, 0x3f, 0xef, 0xcb, 0xb9, 0x94, 0xcf, 0xe3, 0x93, 0xe8, 0x15, - 0x54, 0x30, 0xc3, 0x48, 0xbe, 0xf5, 0x30, 0xbc, 0x93, 0x36, 0x26, 0xc0, 0x61, 0x74, 0xdd, 0x44, - 0x67, 0x7d, 0xc3, 0x42, 0xc7, 0xb7, 0xc0, 0x72, 0xeb, 0x1d, 0xc7, 0x64, 0xe3, 0x52, 0x3b, 0xed, - 0x46, 0xcc, 0xb3, 0xdc, 0x3b, 0x8e, 0x49, 0x2b, 0xd0, 0xbd, 0x35, 0x62, 0xe9, 0xf5, 0x0d, 0xdb, - 0x59, 0x87, 0xd8, 0x9a, 0x6c, 0x70, 0x6a, 0x63, 0x0c, 0x7e, 0xcf, 0x07, 0xe3, 0xa7, 0xd0, 0xc8, - 0x6a, 0xb3, 0x43, 0x82, 0x68, 0x86, 0xec, 0xad, 0x4f, 0x1b, 0xa6, 0xc0, 0xe0, 0x85, 0xe4, 0x02, - 0x42, 0x40, 0xe4, 0x41, 0x2c, 0x6c, 0x78, 0xd8, 0xd3, 0x06, 0x29, 0x64, 0x99, 0x77, 0xd7, 0x04, - 0xd3, 0x6a, 0x26, 0xa4, 0x7a, 0xd3, 0xb6, 0x56, 0xeb, 0x1e, 0x71, 0x5a, 0xd0, 0x50, 0x30, 0x4e, - 0xd4, 0xce, 0x02, 0x05, 0x3c, 0x9d, 0xb8, 0xf3, 0xb6, 0xb5, 0xba, 0x4c, 0x9c, 0x16, 0x6d, 0xea, - 0x55, 0x84, 0x79, 0x53, 0x1d, 0xb8, 0xf4, 0x60, 0x1f, 0x07, 0x76, 0x8a, 0x1a, 0xff, 0x08, 0x76, - 0x1b, 0x02, 0x1f, 0x56, 0x44, 0x43, 0x2c, 0xa4, 0x1b, 0x13, 0x1a, 0x98, 0x2a, 0x6a, 0x88, 0x81, - 0x40, 0x5e, 0x67, 0x11, 0xb7, 0x5e, 0x60, 0x56, 0xd3, 0x1a, 0xff, 0xa5, 0xfe, 0x6e, 0x36, 0x2d, - 0x50, 0xf2, 0x71, 0x7d, 0xe3, 0xc0, 0x73, 0x08, 0xf1, 0x8c, 0x92, 0xf4, 0x73, 0x23, 0x06, 0xad, - 0x21, 0x26, 0x85, 0x87, 0x50, 0x56, 0x58, 0x20, 0x7a, 0xf7, 0xb0, 0x5d, 0x4c, 0x12, 0xe9, 0x21, - 0x9c, 0xe2, 0xc2, 0xc6, 0x64, 0xf7, 0xb0, 0x5a, 0x1d, 0xbd, 0x10, 0x45, 0x55, 0xe8, 0xdd, 0xe7, - 0x6d, 0x6d, 0x92, 0x48, 0x8f, 0xf7, 0x4b, 0xdc, 0x91, 0x6b, 0xe9, 0xef, 0x65, 0x53, 0x03, 0x82, - 0x9f, 0xc8, 0xf4, 0x30, 0x65, 0x7a, 0x32, 0xf4, 0x0f, 0x34, 0xf4, 0x13, 0x65, 0x7a, 0x32, 0xf6, - 0x0f, 0xa2, 0xa7, 0xcf, 0x2c, 0xb0, 0xf0, 0x99, 0xb7, 0x4d, 0xcb, 0xc0, 0x8f, 0xa1, 0x33, 0x77, - 0x6a, 0x33, 0x5a, 0xfd, 0x76, 0x65, 0xb1, 0x5c, 0xbf, 0xb3, 0x58, 0xab, 0xce, 0x4c, 0x57, 0x66, - 0x2b, 0x33, 0xe5, 0x42, 0x0f, 0x3e, 0x85, 0xc6, 0x42, 0xd4, 0xdc, 0x9d, 0x85, 0xd2, 0x62, 0x21, - 0x83, 0xc7, 0xd1, 0x48, 0x08, 0x9c, 0x5a, 0x5a, 0x2e, 0x64, 0x9f, 0x79, 0x1a, 0x0d, 0xc1, 0xfe, - 0xa5, 0xc4, 0xda, 0x34, 0x8c, 0x06, 0x96, 0xa6, 0x6a, 0x33, 0xda, 0x5d, 0x60, 0x82, 0x50, 0x5f, - 0x79, 0x66, 0x91, 0x32, 0xcc, 0x3c, 0xf3, 0xbf, 0x32, 0x08, 0xd5, 0x66, 0x97, 0xab, 0x9c, 0x70, - 0x08, 0xf5, 0x57, 0x16, 0xef, 0x96, 0xe6, 0x2b, 0x94, 0x6e, 0x00, 0xe5, 0x97, 0xaa, 0x33, 0xb4, - 0x86, 0x41, 0xd4, 0x3b, 0x3d, 0xbf, 0x54, 0x9b, 0x29, 0x64, 0x29, 0x50, 0x9b, 0x29, 0x95, 0x0b, - 0x39, 0x0a, 0xbc, 0xa7, 0x55, 0x96, 0x67, 0x0a, 0x79, 0xfa, 0xe7, 0x7c, 0x6d, 0xb9, 0xb4, 0x5c, - 0xe8, 0xa5, 0x7f, 0xce, 0xc2, 0x9f, 0x7d, 0x94, 0x59, 0x6d, 0x66, 0x19, 0x7e, 0xf4, 0xd3, 0x26, - 0xcc, 0xfa, 0xbf, 0x06, 0x28, 0x8a, 0xb2, 0x2e, 0x57, 0xb4, 0xc2, 0x20, 0xfd, 0x41, 0x59, 0xd2, - 0x1f, 0x88, 0x36, 0x4e, 0x9b, 0x59, 0x58, 0xba, 0x3b, 0x53, 0x18, 0xa2, 0xbc, 0x16, 0x6e, 0x53, - 0xf0, 0x30, 0xfd, 0x53, 0x5b, 0xa0, 0x7f, 0x8e, 0x50, 0x4e, 0xda, 0x4c, 0x69, 0xbe, 0x5a, 0x5a, - 0x9e, 0x2b, 0x8c, 0xd2, 0xf6, 0x00, 0xcf, 0x31, 0x56, 0x72, 0xb1, 0xb4, 0x30, 0x53, 0x28, 0x70, - 0x9a, 0xf2, 0x7c, 0x65, 0xf1, 0x76, 0x61, 0x1c, 0x1a, 0xf2, 0xd6, 0x02, 0xfc, 0xc0, 0xb4, 0x00, - 0xfc, 0x75, 0xea, 0x99, 0xcf, 0xa0, 0xbe, 0xa5, 0x1a, 0x58, 0x26, 0x9d, 0x43, 0xa7, 0x96, 0x6a, - 0xf5, 0xe5, 0xb7, 0xaa, 0x33, 0x11, 0x79, 0x8f, 0xa3, 0x11, 0x1f, 0x31, 0x5f, 0x59, 0xbc, 0xf3, - 0x26, 0x93, 0xb6, 0x0f, 0x5a, 0x28, 0x4d, 0x2f, 0xd5, 0x0a, 0x59, 0xda, 0x2b, 0x3e, 0xe8, 0x5e, - 0x65, 0xb1, 0xbc, 0x74, 0xaf, 0x56, 0xc8, 0x3d, 0xf3, 0xc0, 0xcf, 0x68, 0xb5, 0xe4, 0x98, 0xab, - 0xa6, 0x85, 0x2f, 0xa0, 0xc7, 0xca, 0x33, 0x77, 0x2b, 0xd3, 0x33, 0xf5, 0x25, 0xad, 0x72, 0xab, - 0xb2, 0x18, 0xa9, 0xe9, 0x0c, 0x1a, 0x97, 0xd1, 0xa5, 0x6a, 0xa5, 0x90, 0xc1, 0x67, 0x11, 0x96, - 0xc1, 0xaf, 0x97, 0x16, 0x66, 0x0b, 0x59, 0xac, 0xa0, 0xd3, 0x32, 0xbc, 0xb2, 0xb8, 0x7c, 0x67, - 0x71, 0xa6, 0x90, 0x7b, 0xe6, 0x27, 0x33, 0xe8, 0x4c, 0xa2, 0x83, 0x2a, 0x56, 0xd1, 0xc5, 0x99, - 0xf9, 0x52, 0x6d, 0xb9, 0x32, 0x5d, 0x9b, 0x29, 0x69, 0xd3, 0x73, 0xf5, 0xe9, 0xd2, 0xf2, 0xcc, - 0xad, 0x25, 0xed, 0xad, 0xfa, 0xad, 0x99, 0xc5, 0x19, 0xad, 0x34, 0x5f, 0xe8, 0xc1, 0x4f, 0xa1, - 0x62, 0x0a, 0x4d, 0x6d, 0x66, 0xfa, 0x8e, 0x56, 0x59, 0x7e, 0xab, 0x90, 0xc1, 0x4f, 0xa2, 0x0b, - 0xa9, 0x44, 0xf4, 0x77, 0x21, 0x8b, 0x2f, 0xa2, 0x89, 0x34, 0x92, 0x37, 0xe6, 0x0b, 0xb9, 0x67, - 0x7e, 0x34, 0x83, 0x70, 0xdc, 0xc3, 0x10, 0x3f, 0x81, 0xce, 0x53, 0xbd, 0xa8, 0xa7, 0x37, 0xf0, - 0x49, 0x74, 0x21, 0x91, 0x42, 0x68, 0x5e, 0x11, 0x3d, 0x9e, 0x42, 0xc2, 0x1b, 0x77, 0x1e, 0x29, - 0xc9, 0x04, 0xb4, 0x69, 0x53, 0xe5, 0xf7, 0xfe, 0xd3, 0xc5, 0x9e, 0xf7, 0xbe, 0x79, 0x31, 0xf3, - 0xbb, 0xdf, 0xbc, 0x98, 0xf9, 0xa3, 0x6f, 0x5e, 0xcc, 0x7c, 0xea, 0xfa, 0x5e, 0x1c, 0x30, 0xd9, - 0x68, 0x5f, 0xe9, 0x03, 0x57, 0xa3, 0x1b, 0xff, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xe3, 0xba, 0x6d, - 0x64, 0x80, 0x37, 0x01, 0x00, + 0xa4, 0x14, 0x00, 0xa5, 0xb0, 0xff, 0x28, 0xda, 0x62, 0x2c, 0x94, 0x03, 0x0b, 0x56, 0xad, 0x7e, + 0x3b, 0x8f, 0x86, 0x7d, 0x63, 0x96, 0x43, 0xea, 0xc1, 0xe7, 0x50, 0x61, 0xd6, 0x16, 0xa2, 0x22, + 0x82, 0xf1, 0xcb, 0xaa, 0xed, 0x46, 0xac, 0x7a, 0x38, 0x11, 0xbe, 0x81, 0x8a, 0x0b, 0xb6, 0x21, + 0x9a, 0x6e, 0xc1, 0x98, 0xb6, 0x6c, 0x23, 0xe6, 0xfa, 0x12, 0x10, 0xe2, 0x4b, 0x28, 0x0f, 0x56, + 0x6f, 0xc2, 0xd5, 0x73, 0xc4, 0xd2, 0x0d, 0xf0, 0x82, 0x6e, 0x14, 0x76, 0xab, 0x1b, 0x03, 0x7b, + 0xd5, 0x8d, 0xe2, 0xc1, 0xea, 0xc6, 0x9b, 0x68, 0x18, 0x6a, 0xf2, 0xa3, 0x7e, 0x6f, 0xbf, 0xbc, + 0x3d, 0xc1, 0x57, 0xa0, 0x11, 0xd6, 0x6e, 0x1e, 0xfb, 0x1b, 0x16, 0x1e, 0x89, 0x55, 0x44, 0xed, + 0xd0, 0x3e, 0xd4, 0xee, 0x0f, 0x32, 0x68, 0xe0, 0xae, 0xb5, 0x66, 0xd9, 0xeb, 0xfb, 0xd3, 0xb8, + 0x1b, 0x68, 0x88, 0xb3, 0x11, 0xe6, 0x78, 0xf0, 0x66, 0xea, 0x32, 0x70, 0x03, 0x38, 0x69, 0x22, + 0x15, 0x7e, 0x25, 0x28, 0x04, 0x86, 0xad, 0xb9, 0x30, 0xae, 0xa8, 0x5f, 0xa8, 0x29, 0x87, 0x42, + 0x14, 0xc9, 0xf1, 0x79, 0x9e, 0x03, 0x5f, 0x08, 0xac, 0x43, 0x9b, 0xc2, 0x52, 0xe0, 0xab, 0xff, + 0x32, 0x8b, 0x46, 0x23, 0xd7, 0x4f, 0xcf, 0xa2, 0x41, 0x7e, 0xfd, 0x63, 0xfa, 0xb1, 0x19, 0xc1, + 0xf0, 0x35, 0x00, 0x6a, 0x45, 0xf6, 0x67, 0xd5, 0xc0, 0x1f, 0x43, 0x03, 0xb6, 0x0b, 0x4b, 0x13, + 0x7c, 0xcb, 0x68, 0x38, 0x84, 0x16, 0xeb, 0xb4, 0xed, 0x6c, 0x70, 0x70, 0x12, 0x51, 0x23, 0x6d, + 0x17, 0x3e, 0xed, 0x26, 0x1a, 0xd4, 0x5d, 0x97, 0x78, 0x0d, 0x4f, 0x5f, 0x11, 0xc3, 0x35, 0x06, + 0x40, 0x71, 0x74, 0x00, 0x70, 0x49, 0x5f, 0xc1, 0xaf, 0xa3, 0x91, 0xa6, 0x43, 0x60, 0xf1, 0xd2, + 0x5b, 0xb4, 0x95, 0xc2, 0xe6, 0x52, 0x42, 0x88, 0x37, 0xfe, 0x21, 0xa2, 0x6a, 0xe0, 0x7b, 0x68, + 0x84, 0x7f, 0x0e, 0xb3, 0x3a, 0x83, 0x81, 0x36, 0x1a, 0x2e, 0x26, 0x4c, 0x24, 0xcc, 0xee, 0x8c, + 0x1b, 0x1f, 0x8a, 0xe4, 0x22, 0x5f, 0x43, 0x20, 0x55, 0xbf, 0x91, 0xa1, 0x1b, 0x1e, 0x0a, 0x08, + 0xd2, 0x69, 0xb6, 0x77, 0xa9, 0x2b, 0xed, 0x30, 0x62, 0x7e, 0xc1, 0xed, 0x31, 0x3b, 0x69, 0x1c, + 0x8b, 0xc7, 0x51, 0xc1, 0x10, 0xef, 0x7e, 0xce, 0xc8, 0x1f, 0xe1, 0xd7, 0xa3, 0x71, 0x2a, 0x7c, + 0x19, 0xe5, 0xe9, 0x86, 0x36, 0x7a, 0xf0, 0x13, 0xd7, 0x48, 0x0d, 0x28, 0xd4, 0xef, 0xce, 0xa2, + 0x61, 0xe1, 0x6b, 0xae, 0xef, 0xeb, 0x73, 0x5e, 0xde, 0x59, 0x33, 0xb9, 0x1d, 0x2c, 0xc0, 0x82, + 0x26, 0xdf, 0x0c, 0x44, 0xb1, 0xa3, 0x27, 0x08, 0x2e, 0x98, 0x17, 0xf8, 0x87, 0x16, 0x76, 0x7e, + 0x08, 0xa2, 0xf4, 0xb7, 0xf3, 0xc5, 0x6c, 0x29, 0x77, 0x3b, 0x5f, 0xcc, 0x97, 0xfa, 0xc1, 0xb3, + 0x1e, 0xa2, 0x49, 0xb1, 0x13, 0xa6, 0xf5, 0xc0, 0x5c, 0x39, 0xe2, 0x76, 0x83, 0x07, 0x1b, 0x75, + 0x20, 0x22, 0x9b, 0x23, 0x6e, 0x44, 0xf8, 0xae, 0xca, 0xe6, 0x38, 0x81, 0x01, 0x97, 0xcd, 0xbf, + 0xcb, 0x20, 0x25, 0x51, 0x36, 0x95, 0x43, 0x7a, 0xf9, 0x3e, 0xb8, 0x34, 0x06, 0xdf, 0xca, 0xa2, + 0xb1, 0xaa, 0xe5, 0x91, 0x15, 0x76, 0xee, 0x39, 0xe2, 0x53, 0xc5, 0x1d, 0x96, 0xc6, 0x94, 0x7f, + 0x0c, 0xef, 0xf3, 0x27, 0x83, 0x53, 0x65, 0x88, 0x4a, 0xe1, 0x24, 0x96, 0x3e, 0xc0, 0xf4, 0x46, + 0x11, 0x21, 0x1f, 0xf1, 0x39, 0xe7, 0x68, 0x08, 0xf9, 0x88, 0x4f, 0x5e, 0xef, 0x51, 0x21, 0xff, + 0xf7, 0x0c, 0x3a, 0x99, 0x50, 0x39, 0x24, 0x07, 0xec, 0x2e, 0x43, 0xc8, 0x85, 0x8c, 0x90, 0x1c, + 0xb0, 0xbb, 0x0c, 0xd1, 0x16, 0x34, 0x1f, 0x89, 0x97, 0xc0, 0xb1, 0x6a, 0xb1, 0x3a, 0x35, 0xc9, + 0xa5, 0xaa, 0x0a, 0x2e, 0x62, 0x14, 0x9c, 0xf4, 0x65, 0x81, 0xf3, 0x95, 0x6d, 0x1a, 0xcd, 0x88, + 0xf3, 0x15, 0x2d, 0x83, 0x3f, 0x85, 0x06, 0x2b, 0xef, 0x74, 0x1d, 0x02, 0x7c, 0x99, 0xc4, 0xdf, + 0x17, 0xf0, 0xf5, 0x11, 0x49, 0x9c, 0x99, 0x1f, 0x19, 0xa5, 0x88, 0xf2, 0x0e, 0x19, 0xaa, 0x9f, + 0xcf, 0xa0, 0x73, 0xe9, 0xad, 0xc3, 0x1f, 0x42, 0x03, 0xf4, 0x64, 0x5b, 0xd1, 0x16, 0xf8, 0xa7, + 0xb3, 0x94, 0x1f, 0x76, 0x8b, 0x34, 0x74, 0x47, 0xdc, 0x78, 0xfb, 0x64, 0xf8, 0x55, 0x34, 0x54, + 0x75, 0xdd, 0x2e, 0x71, 0xea, 0x37, 0xee, 0x6a, 0x55, 0x7e, 0xa6, 0x82, 0x3d, 0xbb, 0x09, 0xe0, + 0x86, 0x7b, 0x23, 0x12, 0x54, 0x41, 0xa4, 0x57, 0x7f, 0x30, 0x83, 0xce, 0xf7, 0xfa, 0x2a, 0x7a, + 0x80, 0x5f, 0x22, 0x96, 0x6e, 0x79, 0x3c, 0xb5, 0x2e, 0x3f, 0xa2, 0x78, 0x00, 0x93, 0x0f, 0x19, + 0x01, 0x21, 0x2d, 0xc4, 0x6e, 0xc7, 0x82, 0xe7, 0x78, 0x76, 0x93, 0x07, 0xb0, 0x48, 0x21, 0x9f, + 0x50, 0xfd, 0x91, 0x37, 0x51, 0xff, 0xa2, 0x45, 0x16, 0x1f, 0xe0, 0xe7, 0x85, 0x04, 0x6e, 0x7c, + 0xa0, 0x8d, 0x89, 0x03, 0x06, 0x10, 0xb3, 0x7d, 0x9a, 0x90, 0xe6, 0xed, 0xa6, 0x98, 0x85, 0x8a, + 0xab, 0x03, 0x16, 0xcb, 0x30, 0xcc, 0x6c, 0x9f, 0x26, 0x66, 0xab, 0xba, 0x29, 0x26, 0x57, 0xe2, + 0x9d, 0x2d, 0x95, 0x62, 0x18, 0xbf, 0x14, 0x9f, 0x06, 0xe6, 0x92, 0x32, 0x10, 0x45, 0xf7, 0x04, + 0x71, 0x8a, 0xd9, 0x3e, 0x2d, 0x39, 0x73, 0xd1, 0xb0, 0x68, 0x18, 0x13, 0x7d, 0x90, 0x13, 0x71, + 0xb3, 0x7d, 0x9a, 0x44, 0x8b, 0x5f, 0x0c, 0xd2, 0x3c, 0xde, 0xb6, 0x4d, 0x2b, 0xea, 0x5d, 0x29, + 0xa0, 0x66, 0xfb, 0x34, 0x91, 0x52, 0xa8, 0xb4, 0xe6, 0x98, 0x41, 0x0e, 0xb6, 0x68, 0xa5, 0x80, + 0x13, 0x2a, 0x85, 0xdf, 0xf8, 0x55, 0x34, 0x12, 0xb8, 0xad, 0xbe, 0x4d, 0x9a, 0x1e, 0xbf, 0x12, + 0x39, 0x1d, 0x29, 0xcc, 0x90, 0xb3, 0x7d, 0x9a, 0x4c, 0x8d, 0x2f, 0xfb, 0x09, 0xfe, 0xf9, 0x5d, + 0xc7, 0xa8, 0x30, 0x9d, 0x99, 0xef, 0x50, 0x29, 0x71, 0x3c, 0xed, 0x9d, 0xf0, 0xed, 0x80, 0x5f, + 0x60, 0xe0, 0x48, 0x2d, 0xd3, 0x96, 0x41, 0x7b, 0x47, 0x78, 0x38, 0x7a, 0x3d, 0x9a, 0x02, 0x99, + 0x27, 0xd6, 0x3e, 0x13, 0x29, 0xc9, 0xb1, 0xb3, 0x7d, 0x5a, 0x34, 0x65, 0xf2, 0x8b, 0x52, 0xfa, + 0x5d, 0x1e, 0x3f, 0x25, 0x2a, 0x55, 0x8a, 0x12, 0xa4, 0x0a, 0x89, 0x7a, 0x5f, 0x8f, 0xe6, 0x83, + 0xe5, 0xd1, 0x52, 0xce, 0x24, 0x67, 0x0d, 0x15, 0xaa, 0xf6, 0xf3, 0xc7, 0xbe, 0x28, 0xe5, 0xed, + 0x84, 0xd4, 0xd8, 0x09, 0x55, 0xeb, 0x9e, 0x2e, 0x56, 0xcd, 0xce, 0x97, 0x52, 0x06, 0x49, 0x48, + 0x70, 0x13, 0xef, 0x50, 0xc0, 0x09, 0x1d, 0xca, 0xb2, 0x4d, 0xbe, 0x28, 0x25, 0x31, 0xe1, 0x19, + 0x6c, 0x82, 0x4a, 0x05, 0x14, 0xad, 0x54, 0x4c, 0x77, 0x72, 0x53, 0xcc, 0xed, 0xa1, 0x8c, 0xc9, + 0x1d, 0x14, 0x62, 0x68, 0x07, 0x09, 0x39, 0x40, 0xca, 0x90, 0x37, 0x40, 0xc1, 0x40, 0x3e, 0x14, + 0xb4, 0x70, 0xb2, 0x36, 0xdb, 0xa7, 0x41, 0x46, 0x01, 0x95, 0x65, 0xa4, 0x50, 0x4e, 0x02, 0xc5, + 0x70, 0x90, 0x1f, 0xf5, 0x11, 0x69, 0xce, 0xf6, 0x69, 0x2c, 0x5b, 0xc5, 0xf3, 0x42, 0xec, 0x67, + 0xe5, 0x94, 0x3c, 0x45, 0x04, 0x08, 0x3a, 0x45, 0x84, 0x11, 0xa2, 0x67, 0xe2, 0xf1, 0x91, 0x95, + 0xd3, 0xf2, 0x8a, 0x1a, 0xc5, 0xcf, 0xf6, 0x69, 0xf1, 0x98, 0xca, 0x2f, 0x4a, 0x21, 0x83, 0x95, + 0x33, 0x11, 0x97, 0xe6, 0x10, 0x45, 0xc5, 0x25, 0x06, 0x17, 0x5e, 0x4c, 0x4c, 0xf2, 0xa5, 0x9c, + 0x95, 0x97, 0xe3, 0x04, 0x92, 0xd9, 0x3e, 0x2d, 0x31, 0x3d, 0xd8, 0x64, 0x2c, 0x70, 0xaf, 0xa2, + 0xc8, 0xef, 0x96, 0x11, 0xf4, 0x6c, 0x9f, 0x16, 0x0b, 0xf5, 0x7b, 0x53, 0x8c, 0x98, 0xab, 0x3c, + 0x21, 0x77, 0x62, 0x88, 0xa1, 0x9d, 0x28, 0x44, 0xd6, 0xbd, 0x29, 0x86, 0x95, 0x55, 0xce, 0xc5, + 0x4b, 0x85, 0x33, 0xa7, 0x10, 0x7e, 0x56, 0x4b, 0x8e, 0xbd, 0xaa, 0x3c, 0xc9, 0x43, 0xf3, 0xf3, + 0xf2, 0x49, 0x34, 0xb3, 0x7d, 0x5a, 0x72, 0xdc, 0x56, 0x2d, 0x39, 0x68, 0xa9, 0x72, 0xbe, 0x17, + 0xcf, 0xa0, 0x75, 0xc9, 0x01, 0x4f, 0xf5, 0x1e, 0x21, 0x24, 0x95, 0x0b, 0x72, 0x3c, 0xa5, 0x54, + 0xc2, 0xd9, 0x3e, 0xad, 0x47, 0x20, 0xca, 0xbb, 0x29, 0xf1, 0x1c, 0x95, 0x8b, 0x72, 0x66, 0x8e, + 0x44, 0xa2, 0xd9, 0x3e, 0x2d, 0x25, 0x1a, 0xe4, 0xdd, 0x94, 0x50, 0x88, 0x4a, 0xb9, 0x27, 0xdb, + 0x40, 0x1e, 0x29, 0x81, 0x14, 0x17, 0x13, 0xa3, 0x08, 0x2a, 0x4f, 0xc9, 0xaa, 0x9b, 0x40, 0x42, + 0x55, 0x37, 0x29, 0xfe, 0xe0, 0x62, 0x62, 0x18, 0x3f, 0xe5, 0xe9, 0x1e, 0x0c, 0x83, 0x36, 0x26, + 0x06, 0x00, 0x5c, 0x4c, 0x8c, 0xa3, 0xa7, 0xa8, 0x32, 0xc3, 0x04, 0x12, 0xca, 0x30, 0x29, 0x02, + 0xdf, 0x62, 0x62, 0x20, 0x3b, 0xe5, 0x99, 0x1e, 0x0c, 0xc3, 0x16, 0x26, 0x85, 0xc0, 0x7b, 0x51, + 0x8a, 0x24, 0xa7, 0xbc, 0x4f, 0x9e, 0x37, 0x04, 0x14, 0x9d, 0x37, 0xc4, 0x98, 0x73, 0x93, 0xb1, + 0x58, 0x39, 0xca, 0xfb, 0xe5, 0x61, 0x1e, 0x41, 0xd3, 0x61, 0x1e, 0x8d, 0xae, 0x33, 0x19, 0x8b, + 0x19, 0xa2, 0x5c, 0x4a, 0x63, 0x02, 0x68, 0x99, 0x09, 0x8b, 0x32, 0x52, 0x4d, 0x08, 0x5a, 0xa1, + 0x7c, 0x40, 0xb6, 0xb9, 0x8b, 0x11, 0xcc, 0xf6, 0x69, 0x09, 0xa1, 0x2e, 0xb4, 0x64, 0x0f, 0x4d, + 0xe5, 0xb2, 0x3c, 0x6c, 0x93, 0x68, 0xe8, 0xb0, 0x4d, 0xf4, 0xee, 0x9c, 0x4b, 0xb2, 0xaf, 0x55, + 0xae, 0xc8, 0x1b, 0xb3, 0x38, 0x05, 0xdd, 0x98, 0x25, 0xd8, 0xe5, 0x6a, 0xc9, 0x5e, 0x83, 0xca, + 0xb3, 0x3d, 0x5b, 0x08, 0x34, 0x09, 0x2d, 0x64, 0x4e, 0x74, 0xe1, 0xde, 0xe9, 0x6e, 0xa7, 0x65, + 0xeb, 0x86, 0xf2, 0xc1, 0xc4, 0xbd, 0x13, 0x43, 0x0a, 0x7b, 0x27, 0x06, 0xa0, 0xab, 0xbc, 0x68, + 0x7f, 0xaa, 0x5c, 0x95, 0x57, 0x79, 0x11, 0x47, 0x57, 0x79, 0xc9, 0x56, 0x75, 0x32, 0x66, 0xab, + 0xa9, 0x3c, 0x27, 0x2b, 0x40, 0x04, 0x4d, 0x15, 0x20, 0x6a, 0xdd, 0xf9, 0x56, 0xba, 0x75, 0xa3, + 0x32, 0x0e, 0xdc, 0x9e, 0x0a, 0x32, 0xc0, 0xa7, 0xd0, 0xcd, 0xf6, 0x69, 0xe9, 0x16, 0x92, 0xd5, + 0x04, 0x63, 0x45, 0xe5, 0x9a, 0xac, 0x60, 0x31, 0x02, 0xaa, 0x60, 0x71, 0x13, 0xc7, 0x6a, 0x82, + 0xb5, 0xa1, 0xf2, 0xa1, 0x54, 0x56, 0xc1, 0x37, 0x27, 0xd8, 0x28, 0xde, 0x14, 0xcd, 0x05, 0x95, + 0xe7, 0xe5, 0xc5, 0x2e, 0xc4, 0xd0, 0xc5, 0x4e, 0x30, 0x2b, 0xbc, 0x29, 0x1a, 0xca, 0x29, 0xd7, + 0xe3, 0xa5, 0xc2, 0x25, 0x52, 0x30, 0xa8, 0xd3, 0x92, 0xed, 0xcb, 0x94, 0x1b, 0xb2, 0xd6, 0x25, + 0xd1, 0x50, 0xad, 0x4b, 0xb4, 0x4d, 0x9b, 0x89, 0x9b, 0x89, 0x29, 0x37, 0xa3, 0x77, 0x09, 0x32, + 0x9e, 0xee, 0x7c, 0x62, 0xa6, 0x65, 0xaf, 0x47, 0xc3, 0x07, 0x28, 0x1f, 0x8e, 0x3c, 0x66, 0x48, + 0x58, 0xba, 0xbf, 0x8d, 0x84, 0x1b, 0x78, 0x3d, 0xea, 0x71, 0xaf, 0xbc, 0x90, 0xcc, 0x21, 0xd0, + 0x95, 0xa8, 0x87, 0xfe, 0xeb, 0x51, 0x27, 0x75, 0xe5, 0xc5, 0x64, 0x0e, 0x81, 0x74, 0xa3, 0x4e, + 0xed, 0xcf, 0x0b, 0x61, 0xf3, 0x94, 0x8f, 0xc8, 0x5b, 0xc7, 0x00, 0x41, 0xb7, 0x8e, 0x61, 0x70, + 0xbd, 0xe7, 0x85, 0x70, 0x73, 0xca, 0x4b, 0xb1, 0x22, 0x41, 0x63, 0x85, 0xa0, 0x74, 0xcf, 0x0b, + 0x61, 0xda, 0x94, 0x97, 0x63, 0x45, 0x82, 0xd6, 0x09, 0xc1, 0xdc, 0x8c, 0x5e, 0x7e, 0x38, 0xca, + 0x47, 0xe5, 0x2b, 0x8e, 0x74, 0xca, 0xd9, 0x3e, 0xad, 0x97, 0x3f, 0xcf, 0x5b, 0xe9, 0x46, 0x77, + 0xca, 0x2b, 0xf2, 0x10, 0x4e, 0xa3, 0xa3, 0x43, 0x38, 0xd5, 0x70, 0xef, 0xd5, 0x88, 0x4f, 0xae, + 0xf2, 0xaa, 0x3c, 0xc5, 0x49, 0x48, 0x3a, 0xc5, 0x45, 0x3d, 0x78, 0x25, 0x67, 0x53, 0xe5, 0x63, + 0xf2, 0x14, 0x27, 0xe2, 0xe8, 0x14, 0x27, 0x39, 0xa6, 0x4e, 0xc6, 0x7c, 0x20, 0x95, 0xd7, 0xe4, + 0x29, 0x2e, 0x82, 0xa6, 0x53, 0x5c, 0xd4, 0x6b, 0xf2, 0xd5, 0x88, 0x2b, 0xa0, 0xf2, 0x7a, 0x72, + 0xfb, 0x01, 0x29, 0xb6, 0x9f, 0x39, 0x0e, 0x6a, 0xc9, 0x3e, 0x6d, 0x4a, 0x45, 0x1e, 0xbf, 0x49, + 0x34, 0x74, 0xfc, 0x26, 0xfa, 0xc3, 0x2d, 0x26, 0xe6, 0xc5, 0x54, 0x26, 0x7a, 0x1c, 0x1c, 0xc2, + 0xad, 0x48, 0x52, 0x46, 0x4d, 0xf1, 0x8c, 0xcc, 0x0e, 0x42, 0x93, 0x29, 0x67, 0x64, 0xff, 0x18, + 0x14, 0xa1, 0xa7, 0xb3, 0x6b, 0xcc, 0x06, 0x4c, 0x99, 0x92, 0x67, 0xd7, 0x18, 0x01, 0x9d, 0x5d, + 0xe3, 0x96, 0x63, 0x33, 0xa8, 0xc4, 0xb5, 0x88, 0x99, 0xb6, 0x99, 0xd6, 0x8a, 0x32, 0x1d, 0x71, + 0x29, 0x89, 0xe0, 0xe9, 0xec, 0x14, 0x85, 0xc1, 0x7a, 0xcd, 0x60, 0x93, 0x2d, 0xb3, 0xb3, 0x6c, + 0xeb, 0x8e, 0x51, 0x27, 0x96, 0xa1, 0xcc, 0x44, 0xd6, 0xeb, 0x04, 0x1a, 0x58, 0xaf, 0x13, 0xe0, + 0xe0, 0xf4, 0x1e, 0x81, 0x6b, 0xa4, 0x49, 0xcc, 0x87, 0x44, 0xb9, 0x05, 0x6c, 0xcb, 0x69, 0x6c, + 0x39, 0xd9, 0x6c, 0x9f, 0x96, 0xc6, 0x81, 0xee, 0xd5, 0xe7, 0x37, 0xea, 0x6f, 0xcc, 0x05, 0x6e, + 0x94, 0x35, 0x87, 0x74, 0x74, 0x87, 0x28, 0xb3, 0xf2, 0x5e, 0x3d, 0x91, 0x88, 0xee, 0xd5, 0x13, + 0x11, 0x71, 0xb6, 0xfe, 0x58, 0xa8, 0xf6, 0x62, 0x1b, 0x8e, 0x88, 0xe4, 0xd2, 0x74, 0x76, 0x92, + 0x11, 0x54, 0x40, 0x73, 0xb6, 0xb5, 0x02, 0x37, 0x15, 0xb7, 0xe5, 0xd9, 0x29, 0x9d, 0x92, 0xce, + 0x4e, 0xe9, 0x58, 0xaa, 0xea, 0x32, 0x96, 0x8d, 0xc1, 0x3b, 0xb2, 0xaa, 0x27, 0x90, 0x50, 0x55, + 0x4f, 0x00, 0xc7, 0x19, 0x6a, 0xc4, 0x25, 0x9e, 0x32, 0xd7, 0x8b, 0x21, 0x90, 0xc4, 0x19, 0x02, + 0x38, 0xce, 0x70, 0x86, 0x78, 0xcd, 0x55, 0x65, 0xbe, 0x17, 0x43, 0x20, 0x89, 0x33, 0x04, 0x30, + 0x3d, 0x6c, 0xca, 0xe0, 0x89, 0x6e, 0x6b, 0xcd, 0xef, 0xb3, 0x05, 0xf9, 0xb0, 0x99, 0x4a, 0x48, + 0x0f, 0x9b, 0xa9, 0x48, 0xfc, 0x83, 0x3b, 0xb6, 0x51, 0x54, 0x16, 0xa1, 0xc2, 0xf1, 0x70, 0x5f, + 0xb0, 0x93, 0x52, 0xb3, 0x7d, 0xda, 0x4e, 0x6d, 0x20, 0x3f, 0x18, 0x98, 0x12, 0x29, 0x35, 0xa8, + 0xea, 0x44, 0x70, 0x57, 0xc1, 0xc0, 0xb3, 0x7d, 0x5a, 0x60, 0x6c, 0xf4, 0x22, 0x1a, 0x82, 0x8f, + 0xaa, 0x5a, 0xa6, 0x37, 0x35, 0xa1, 0xbc, 0x21, 0x1f, 0x99, 0x04, 0x14, 0x3d, 0x32, 0x09, 0x3f, + 0xe9, 0x24, 0x0e, 0x3f, 0xd9, 0x14, 0x33, 0x35, 0xa1, 0x68, 0xf2, 0x24, 0x2e, 0x21, 0xe9, 0x24, + 0x2e, 0x01, 0x82, 0x7a, 0xa7, 0x1c, 0xbb, 0x33, 0x35, 0xa1, 0xd4, 0x13, 0xea, 0x65, 0xa8, 0xa0, + 0x5e, 0xf6, 0x33, 0xa8, 0xb7, 0xbe, 0xda, 0xf5, 0xa6, 0xe8, 0x37, 0x2e, 0x25, 0xd4, 0xeb, 0x23, + 0x83, 0x7a, 0x7d, 0x00, 0x9d, 0x0a, 0x01, 0x50, 0x73, 0x6c, 0x3a, 0x69, 0xdf, 0x31, 0x5b, 0x2d, + 0xe5, 0xae, 0x3c, 0x15, 0x46, 0xf1, 0x74, 0x2a, 0x8c, 0xc2, 0xe8, 0xd6, 0x93, 0xb5, 0x8a, 0x2c, + 0x77, 0x57, 0x94, 0x7b, 0xf2, 0xd6, 0x33, 0xc4, 0xd0, 0xad, 0x67, 0xf8, 0x0b, 0x4e, 0x17, 0xf4, + 0x97, 0x46, 0x1e, 0x38, 0xc4, 0x5d, 0x55, 0xee, 0x47, 0x4e, 0x17, 0x02, 0x0e, 0x4e, 0x17, 0xc2, + 0x6f, 0xbc, 0x82, 0x9e, 0x94, 0x16, 0x1a, 0xff, 0xed, 0xa9, 0x4e, 0x74, 0xa7, 0xb9, 0xaa, 0x7c, + 0x1c, 0x58, 0x3d, 0x93, 0xb8, 0x54, 0xc9, 0xa4, 0xb3, 0x7d, 0x5a, 0x2f, 0x4e, 0x70, 0x2c, 0x7f, + 0x63, 0x8e, 0xc5, 0xb6, 0xd1, 0x6a, 0x93, 0xfe, 0x21, 0xf4, 0xcd, 0xc8, 0xb1, 0x3c, 0x4e, 0x02, + 0xc7, 0xf2, 0x38, 0x18, 0x77, 0xd0, 0xc5, 0xc8, 0x51, 0x6d, 0x5e, 0x6f, 0xd1, 0x73, 0x09, 0x31, + 0x6a, 0x7a, 0x73, 0x8d, 0x78, 0xca, 0x27, 0x80, 0xf7, 0xa5, 0x94, 0x03, 0x5f, 0x84, 0x7a, 0xb6, + 0x4f, 0xdb, 0x86, 0x1f, 0x56, 0x59, 0xe6, 0x45, 0xe5, 0x93, 0xf2, 0xfd, 0x26, 0x85, 0xcd, 0xf6, + 0x69, 0x2c, 0x2b, 0xe3, 0x5b, 0x48, 0xb9, 0xdb, 0x59, 0x71, 0x74, 0x83, 0xb0, 0x8d, 0x16, 0xec, + 0xdd, 0xf8, 0x06, 0xf4, 0x53, 0xf2, 0x2e, 0x2d, 0x8d, 0x8e, 0xee, 0xd2, 0xd2, 0x70, 0x54, 0x51, + 0xa5, 0x30, 0xae, 0xca, 0xa7, 0x65, 0x45, 0x95, 0x90, 0x54, 0x51, 0xe5, 0xa0, 0xaf, 0x1f, 0x47, + 0x67, 0x82, 0xf3, 0x3c, 0x5f, 0x7f, 0x59, 0xa7, 0x29, 0x6f, 0x01, 0x9f, 0x8b, 0xb1, 0xc7, 0x00, + 0x89, 0x6a, 0xb6, 0x4f, 0x4b, 0x29, 0x4f, 0x57, 0xdc, 0x58, 0x84, 0x72, 0xbe, 0xbd, 0xf8, 0x2e, + 0x79, 0xc5, 0x4d, 0x21, 0xa3, 0x2b, 0x6e, 0x0a, 0x2a, 0x91, 0x39, 0x17, 0xaa, 0xbe, 0x0d, 0xf3, + 0x40, 0xa6, 0x69, 0x1c, 0x12, 0x99, 0xf3, 0x9d, 0xda, 0xf2, 0x36, 0xcc, 0x83, 0xdd, 0x5a, 0x1a, + 0x07, 0x7c, 0x19, 0x15, 0xea, 0xf5, 0x79, 0xad, 0x6b, 0x29, 0xcd, 0x88, 0x0d, 0x18, 0x40, 0x67, + 0xfb, 0x34, 0x8e, 0xa7, 0xdb, 0xa0, 0xe9, 0x96, 0xee, 0x7a, 0x66, 0xd3, 0x85, 0x11, 0xe3, 0x8f, + 0x10, 0x43, 0xde, 0x06, 0x25, 0xd1, 0xd0, 0x6d, 0x50, 0x12, 0x9c, 0xee, 0x17, 0x27, 0x75, 0xd7, + 0xd5, 0x2d, 0xc3, 0xd1, 0x27, 0x60, 0x99, 0x20, 0x11, 0x4b, 0x79, 0x09, 0x4b, 0xf7, 0x8b, 0x32, + 0x04, 0x2e, 0xdf, 0x7d, 0x88, 0xbf, 0xcd, 0x79, 0x10, 0xb9, 0x7c, 0x8f, 0xe0, 0xe1, 0xf2, 0x3d, + 0x02, 0x83, 0x7d, 0xa7, 0x0f, 0xd3, 0xc8, 0x8a, 0x09, 0x79, 0x92, 0x57, 0x22, 0xfb, 0xce, 0x28, + 0x01, 0xec, 0x3b, 0xa3, 0x40, 0xa9, 0x49, 0xfe, 0x72, 0xbb, 0x9a, 0xd2, 0xa4, 0x70, 0x95, 0x8d, + 0x95, 0xa1, 0xeb, 0x77, 0x38, 0x38, 0xa6, 0x36, 0x2c, 0xbd, 0x6d, 0x4f, 0x4d, 0xf8, 0x52, 0x37, + 0xe5, 0xf5, 0x3b, 0x95, 0x90, 0xae, 0xdf, 0xa9, 0x48, 0x3a, 0xbb, 0xfa, 0x07, 0xad, 0x55, 0xdd, + 0x21, 0x46, 0x90, 0x3d, 0x94, 0x1d, 0x0d, 0xdf, 0x96, 0x67, 0xd7, 0x1e, 0xa4, 0x74, 0x76, 0xed, + 0x81, 0xa6, 0x9b, 0xbc, 0x64, 0xb4, 0x46, 0x74, 0x43, 0x59, 0x93, 0x37, 0x79, 0xe9, 0x94, 0x74, + 0x93, 0x97, 0x8e, 0x4d, 0xff, 0x9c, 0xfb, 0x8e, 0xe9, 0x11, 0xa5, 0xb5, 0x93, 0xcf, 0x01, 0xd2, + 0xf4, 0xcf, 0x01, 0x34, 0x3d, 0x10, 0x46, 0x3b, 0xa4, 0x2d, 0x1f, 0x08, 0xe3, 0xdd, 0x10, 0x2d, + 0x41, 0x77, 0x2c, 0xdc, 0x61, 0x42, 0xb1, 0xe4, 0x1d, 0x0b, 0x07, 0xd3, 0x1d, 0x4b, 0xe8, 0x52, + 0x21, 0x19, 0xe8, 0x2b, 0xb6, 0xbc, 0x86, 0x8a, 0x38, 0xba, 0x86, 0x4a, 0xc6, 0xfc, 0x2f, 0x4a, + 0xd6, 0xb3, 0x4a, 0x47, 0xde, 0x75, 0x08, 0x28, 0xba, 0xeb, 0x10, 0xed, 0x6c, 0x27, 0xd1, 0x09, + 0x78, 0x05, 0xd7, 0xba, 0xc1, 0x3b, 0xce, 0x67, 0xe4, 0xcf, 0x8c, 0xa0, 0xe9, 0x67, 0x46, 0x40, + 0x12, 0x13, 0x3e, 0x6d, 0x39, 0x29, 0x4c, 0xc2, 0xfb, 0xc1, 0x08, 0x08, 0xcf, 0x21, 0x5c, 0xaf, + 0xcc, 0xcf, 0x55, 0x8d, 0x9a, 0xf8, 0x44, 0xe6, 0xca, 0x37, 0xb0, 0x71, 0x8a, 0xd9, 0x3e, 0x2d, + 0xa1, 0x1c, 0x7e, 0x1b, 0x9d, 0xe7, 0x50, 0xee, 0x0d, 0x07, 0x29, 0xd8, 0x8c, 0x60, 0x41, 0xf0, + 0x64, 0xeb, 0x8c, 0x5e, 0xb4, 0xb3, 0x7d, 0x5a, 0x4f, 0x5e, 0xe9, 0x75, 0xf1, 0xf5, 0xa1, 0xbb, + 0x93, 0xba, 0x82, 0x45, 0xa2, 0x27, 0xaf, 0xf4, 0xba, 0xb8, 0xdc, 0x1f, 0xee, 0xa4, 0xae, 0xa0, + 0x13, 0x7a, 0xf2, 0xc2, 0x2e, 0x2a, 0xf7, 0xc2, 0x57, 0x5a, 0x2d, 0x65, 0x1d, 0xaa, 0xfb, 0xc0, + 0x4e, 0xaa, 0xab, 0xc0, 0x86, 0x73, 0x3b, 0x8e, 0x74, 0x96, 0x5e, 0xec, 0x10, 0xab, 0x2e, 0x2d, + 0x40, 0x8f, 0xe4, 0x59, 0x3a, 0x46, 0x40, 0x67, 0xe9, 0x18, 0x90, 0x0e, 0x28, 0xd1, 0x08, 0x5b, + 0xd9, 0x90, 0x07, 0x94, 0x88, 0xa3, 0x03, 0x4a, 0x32, 0xd8, 0x5e, 0x44, 0x27, 0x17, 0xd7, 0x3c, + 0xdd, 0xdf, 0x41, 0xba, 0xbc, 0x2b, 0xdf, 0x89, 0x3c, 0x32, 0xc5, 0x49, 0xe0, 0x91, 0x29, 0x0e, + 0xa6, 0x63, 0x84, 0x82, 0xeb, 0x1b, 0x56, 0x73, 0x46, 0x37, 0x5b, 0x5d, 0x87, 0x28, 0xff, 0x9f, + 0x3c, 0x46, 0x22, 0x68, 0x3a, 0x46, 0x22, 0x20, 0xba, 0x40, 0x53, 0x50, 0xc5, 0x75, 0xcd, 0x15, + 0x8b, 0x9f, 0x2b, 0xbb, 0x2d, 0x4f, 0xf9, 0xff, 0xe5, 0x05, 0x3a, 0x89, 0x86, 0x2e, 0xd0, 0x49, + 0x70, 0xb8, 0x75, 0x4a, 0x48, 0x4f, 0xa8, 0xfc, 0x95, 0xc8, 0xad, 0x53, 0x02, 0x0d, 0xdc, 0x3a, + 0x25, 0xa5, 0x36, 0x9c, 0x41, 0x25, 0xb6, 0x27, 0x9b, 0x33, 0x83, 0xb7, 0xea, 0xbf, 0x2a, 0xaf, + 0x8f, 0x51, 0x3c, 0x5d, 0x1f, 0xa3, 0x30, 0x99, 0x0f, 0xef, 0x82, 0xbf, 0x96, 0xc6, 0x27, 0x90, + 0x7f, 0xac, 0x0c, 0xbe, 0x25, 0xf2, 0xe1, 0x23, 0xe5, 0xbb, 0x33, 0x69, 0x8c, 0x82, 0xe1, 0x11, + 0x2b, 0x24, 0x33, 0xd2, 0xc8, 0x43, 0x93, 0xac, 0x2b, 0x9f, 0x4d, 0x65, 0xc4, 0x08, 0x64, 0x46, + 0x0c, 0x86, 0xdf, 0x44, 0x67, 0x42, 0xd8, 0x3c, 0x69, 0x2f, 0x07, 0x33, 0xd3, 0xf7, 0x64, 0xe4, + 0x6d, 0x70, 0x32, 0x19, 0xdd, 0x06, 0x27, 0x63, 0x92, 0x58, 0x73, 0xd1, 0xfd, 0xf5, 0x6d, 0x58, + 0x07, 0x12, 0x4c, 0x61, 0x90, 0xc4, 0x9a, 0x4b, 0xf3, 0x7b, 0xb7, 0x61, 0x1d, 0xc8, 0x34, 0x85, + 0x01, 0xfe, 0xa1, 0x0c, 0xba, 0x94, 0x8c, 0xaa, 0xb4, 0x5a, 0x33, 0xb6, 0x13, 0xe2, 0x94, 0xef, + 0xcb, 0xc8, 0x17, 0x0d, 0x3b, 0x2b, 0x36, 0xdb, 0xa7, 0xed, 0xb0, 0x02, 0xfc, 0x31, 0x34, 0x52, + 0xe9, 0x1a, 0xa6, 0x07, 0x0f, 0x6f, 0x74, 0xe3, 0xfc, 0xfd, 0x99, 0xc8, 0x11, 0x47, 0xc4, 0xc2, + 0x11, 0x47, 0x04, 0xe0, 0xdb, 0x68, 0xac, 0x4e, 0x9a, 0x5d, 0xc7, 0xf4, 0x36, 0x34, 0x48, 0x3d, + 0x49, 0x79, 0xfc, 0x40, 0x46, 0x9e, 0xc4, 0x62, 0x14, 0x74, 0x12, 0x8b, 0x01, 0x31, 0x41, 0xe7, + 0xa6, 0x1f, 0x79, 0xc4, 0xb1, 0xf4, 0x16, 0x54, 0x52, 0xf7, 0x6c, 0x47, 0x5f, 0x21, 0xd3, 0x96, + 0xbe, 0xdc, 0x22, 0xca, 0xe7, 0x33, 0xf2, 0xbe, 0x2a, 0x9d, 0x94, 0xee, 0xab, 0xd2, 0xb1, 0x78, + 0x15, 0x3d, 0x99, 0x84, 0x9d, 0x32, 0x5d, 0xa8, 0xe7, 0x0b, 0x19, 0x79, 0x63, 0xd5, 0x83, 0x96, + 0x6e, 0xac, 0x7a, 0xa0, 0x21, 0x3c, 0x77, 0x92, 0x5f, 0x88, 0xf2, 0xe3, 0x19, 0xf9, 0x92, 0x31, + 0x91, 0x6a, 0xb6, 0x4f, 0x4b, 0x71, 0x2b, 0xb9, 0x97, 0xe2, 0x53, 0xa1, 0xfc, 0x44, 0x6f, 0xbe, + 0x81, 0xd2, 0xa7, 0xb8, 0x64, 0xdc, 0x4b, 0xf1, 0x47, 0x50, 0x7e, 0xb2, 0x37, 0xdf, 0xd0, 0x2e, + 0x22, 0xd9, 0x9d, 0xa1, 0x91, 0x6e, 0xcb, 0xaf, 0xfc, 0x54, 0x46, 0x3e, 0xa7, 0xa7, 0x11, 0xd2, + 0x73, 0x7a, 0xaa, 0x43, 0xc0, 0xed, 0x04, 0x8b, 0x7a, 0xe5, 0xa7, 0x23, 0x5a, 0x18, 0xa3, 0xa0, + 0x5a, 0x18, 0x37, 0xc4, 0xbf, 0x9d, 0x60, 0x38, 0xae, 0xfc, 0xfd, 0x74, 0x5e, 0x81, 0x50, 0x13, + 0xec, 0xcd, 0x6f, 0x27, 0xd8, 0x47, 0x2b, 0xff, 0x20, 0x9d, 0x57, 0xf8, 0xbc, 0x1a, 0x37, 0xab, + 0xa6, 0x13, 0x52, 0xd7, 0xb3, 0x19, 0x67, 0x49, 0x9b, 0x7e, 0x3e, 0x3a, 0x21, 0x25, 0x92, 0xc1, + 0x84, 0x94, 0x88, 0x49, 0x62, 0xcd, 0xbf, 0xfb, 0x17, 0xb6, 0x61, 0x2d, 0x4c, 0xa3, 0x89, 0x98, + 0x24, 0xd6, 0x5c, 0x0c, 0x5f, 0xda, 0x86, 0xb5, 0x30, 0x8d, 0x26, 0x62, 0xf0, 0xa7, 0xd0, 0xd9, + 0x10, 0x73, 0x8f, 0x38, 0x6e, 0xd8, 0xf5, 0xbf, 0x98, 0x91, 0xaf, 0x12, 0x52, 0xe8, 0x66, 0xfb, + 0xb4, 0x34, 0x16, 0x89, 0xdc, 0xb9, 0x50, 0x7e, 0x69, 0x3b, 0xee, 0xe1, 0x2d, 0x48, 0x0a, 0x2a, + 0x91, 0x3b, 0x97, 0xcb, 0x2f, 0x6f, 0xc7, 0x3d, 0xbc, 0x06, 0x49, 0x41, 0x4d, 0x0c, 0xa0, 0x7e, + 0xd8, 0xdb, 0xdd, 0x2e, 0x14, 0xbf, 0x9c, 0x29, 0x7d, 0x25, 0x73, 0xbb, 0x50, 0xfc, 0x4a, 0xa6, + 0xf4, 0x55, 0xfa, 0xff, 0x57, 0x33, 0xa5, 0x5f, 0xcf, 0x68, 0x4f, 0x84, 0x25, 0x2b, 0x2b, 0xc4, + 0xf2, 0x6a, 0x2d, 0x9d, 0x7f, 0x77, 0x22, 0x8a, 0xfd, 0x4c, 0x44, 0xb1, 0x1a, 0xd5, 0x1f, 0xcf, + 0xa0, 0xe1, 0xba, 0xe7, 0x10, 0xbd, 0xcd, 0x1d, 0xa0, 0xcf, 0xa1, 0x22, 0xb3, 0xc6, 0xf0, 0x0d, + 0xa2, 0xb5, 0xe0, 0x37, 0xbe, 0x84, 0x46, 0xe7, 0x74, 0xd7, 0x83, 0x26, 0x56, 0x2d, 0x83, 0x3c, + 0x02, 0x4b, 0xe4, 0x9c, 0x16, 0x81, 0xe2, 0x39, 0x46, 0xc7, 0xca, 0x41, 0xe4, 0x89, 0xdc, 0xb6, + 0x7e, 0xbf, 0xc5, 0xaf, 0x6f, 0x96, 0xfb, 0xc0, 0xcd, 0x37, 0x52, 0x56, 0xfd, 0x46, 0x06, 0xc5, + 0xec, 0x44, 0xf6, 0xee, 0xa8, 0xb0, 0x88, 0x4e, 0x44, 0xa2, 0x9d, 0x70, 0x73, 0xea, 0x1d, 0x06, + 0x43, 0x89, 0x96, 0xc6, 0x1f, 0x08, 0xcc, 0x78, 0xef, 0x6a, 0x73, 0xdc, 0xa7, 0x7b, 0x60, 0x6b, + 0xb3, 0x9c, 0xeb, 0x3a, 0x2d, 0x4d, 0x40, 0x71, 0x9f, 0xc3, 0x7f, 0x54, 0x0a, 0x43, 0x39, 0xe0, + 0x4b, 0xdc, 0x6b, 0x22, 0x13, 0x7a, 0x82, 0x47, 0xf2, 0x76, 0x30, 0x2f, 0x89, 0x8f, 0xa1, 0xe1, + 0x6a, 0xbb, 0x43, 0x1c, 0xd7, 0xb6, 0x74, 0xcf, 0xf6, 0xf3, 0x03, 0x82, 0x97, 0xb0, 0x29, 0xc0, + 0x45, 0xcf, 0x55, 0x91, 0x1e, 0x5f, 0xf1, 0xd3, 0x61, 0xe7, 0x20, 0x88, 0xc6, 0xc9, 0x84, 0x74, + 0xd8, 0x7e, 0x52, 0xeb, 0x2b, 0xa8, 0xff, 0xae, 0xab, 0x83, 0xc1, 0x77, 0x40, 0xda, 0xa5, 0x00, + 0x91, 0x14, 0x28, 0xf0, 0x55, 0x54, 0x80, 0x03, 0xb2, 0xab, 0xf4, 0x03, 0x2d, 0xf8, 0xa7, 0xb7, + 0x00, 0x22, 0x7a, 0x03, 0x33, 0x1a, 0x7c, 0x07, 0x95, 0xc2, 0xdb, 0x3f, 0xc8, 0x68, 0xe9, 0x07, + 0xf3, 0x84, 0x1c, 0x1a, 0x6b, 0x01, 0x8e, 0xa5, 0xc2, 0x14, 0x59, 0xc4, 0x0a, 0xe2, 0x59, 0x74, + 0x22, 0x84, 0x51, 0x11, 0xf9, 0x41, 0x84, 0x21, 0x87, 0x8c, 0xc0, 0x8b, 0x8a, 0x53, 0x64, 0x15, + 0x2d, 0x86, 0xab, 0x68, 0xc0, 0x77, 0x4e, 0x2f, 0x6e, 0xab, 0xa4, 0x27, 0xb9, 0x73, 0xfa, 0x80, + 0xe8, 0x96, 0xee, 0x97, 0xc7, 0x33, 0x68, 0x54, 0xb3, 0xbb, 0x1e, 0x59, 0xb2, 0xf9, 0xe5, 0x26, + 0x8f, 0x32, 0x09, 0x6d, 0x72, 0x28, 0xa6, 0xe1, 0xd9, 0x7e, 0x0a, 0x12, 0x31, 0x15, 0x86, 0x5c, + 0x0a, 0x2f, 0xa0, 0xb1, 0xd8, 0x3d, 0xa9, 0x98, 0x18, 0x44, 0xf8, 0xbc, 0x38, 0xb3, 0x78, 0x51, + 0xfc, 0xfd, 0x19, 0x54, 0x58, 0x72, 0x74, 0xd3, 0x73, 0xb9, 0xad, 0xf8, 0xe9, 0xf1, 0x75, 0x47, + 0xef, 0x50, 0xfd, 0x18, 0x87, 0x28, 0x29, 0xf7, 0xf4, 0x56, 0x97, 0xb8, 0x13, 0xf7, 0xe9, 0xd7, + 0xfd, 0xfb, 0xcd, 0xf2, 0x47, 0x77, 0x91, 0xa6, 0xfc, 0x5a, 0xc0, 0x89, 0xd5, 0x40, 0x55, 0xc0, + 0x83, 0xbf, 0x44, 0x15, 0x60, 0x38, 0xbc, 0x80, 0x10, 0xff, 0xd4, 0x4a, 0xa7, 0xc3, 0x0d, 0xcf, + 0x05, 0xab, 0x5a, 0x1f, 0xc3, 0x14, 0x3b, 0x10, 0x98, 0xde, 0x11, 0xf3, 0xa2, 0x0a, 0x1c, 0xa8, + 0x16, 0x2c, 0xf1, 0x16, 0xf9, 0x62, 0x1a, 0x09, 0x25, 0xee, 0x37, 0x36, 0x41, 0x48, 0xd1, 0x62, + 0x78, 0x19, 0x9d, 0xe0, 0x7c, 0x83, 0xb0, 0x8f, 0xa3, 0xf2, 0xac, 0x10, 0x41, 0x33, 0xa5, 0x0d, + 0xda, 0x68, 0x70, 0xb0, 0x58, 0x47, 0xa4, 0x04, 0x9e, 0x08, 0xa3, 0xd2, 0x43, 0x12, 0x56, 0xe5, + 0x04, 0x68, 0xec, 0xf9, 0xad, 0xcd, 0xb2, 0xe2, 0x97, 0x67, 0xb9, 0x5b, 0x93, 0x32, 0xb4, 0x40, + 0x11, 0x91, 0x07, 0xd3, 0xfa, 0x52, 0x02, 0x8f, 0xa8, 0xce, 0xcb, 0x45, 0xf0, 0x24, 0x1a, 0x09, + 0xec, 0xde, 0xee, 0xde, 0xad, 0x4e, 0x81, 0x65, 0x3b, 0xcf, 0x36, 0x1a, 0x89, 0x28, 0x29, 0x32, + 0x91, 0xca, 0x08, 0x2e, 0x30, 0xcc, 0xd4, 0x3d, 0xe2, 0x02, 0xd3, 0x49, 0x70, 0x81, 0xa9, 0xe1, + 0x57, 0xd1, 0x50, 0xe5, 0x7e, 0x9d, 0xbb, 0xf6, 0xb8, 0xca, 0xc9, 0x30, 0x94, 0x2f, 0x24, 0xe9, + 0xe1, 0x6e, 0x40, 0x62, 0xd3, 0x45, 0x7a, 0x3c, 0x8d, 0x46, 0xa5, 0xa7, 0x33, 0x57, 0x39, 0x05, + 0x1c, 0x58, 0x9e, 0x54, 0xc0, 0x34, 0x78, 0xaa, 0x5e, 0x29, 0x13, 0x91, 0x5c, 0x88, 0x6a, 0x0d, + 0xdd, 0xe7, 0xb7, 0x5a, 0xf6, 0xba, 0x46, 0xc0, 0x8b, 0x08, 0xec, 0xe4, 0x8b, 0x4c, 0x6b, 0x0c, + 0x8e, 0x6a, 0x38, 0x0c, 0x27, 0xe5, 0x89, 0x92, 0x8b, 0xe1, 0xb7, 0x11, 0x86, 0x40, 0xaa, 0xc4, + 0xf0, 0x6f, 0x52, 0xaa, 0x53, 0xae, 0x72, 0x06, 0xa2, 0x45, 0xe1, 0xa8, 0x1b, 0x5b, 0x75, 0x6a, + 0xe2, 0x12, 0x9f, 0x3e, 0x2e, 0xea, 0xac, 0x54, 0x23, 0x48, 0x93, 0x6b, 0x1a, 0x62, 0x8b, 0x13, + 0xb8, 0xe2, 0x75, 0x74, 0xb6, 0xe6, 0x90, 0x87, 0xa6, 0xdd, 0x75, 0xfd, 0xe5, 0xc3, 0x9f, 0xb7, + 0xce, 0x6e, 0x3b, 0x6f, 0x3d, 0xcd, 0x2b, 0x3e, 0xdd, 0x71, 0xc8, 0xc3, 0x86, 0x1f, 0x23, 0x48, + 0x0a, 0xae, 0x91, 0xc6, 0x9d, 0x8a, 0x0b, 0x3c, 0xa8, 0x38, 0xdc, 0x24, 0xae, 0xa2, 0x84, 0x53, + 0x2d, 0x73, 0x08, 0x33, 0x03, 0x9c, 0x28, 0xae, 0x48, 0x31, 0xac, 0x21, 0x7c, 0x6b, 0xd2, 0xbf, + 0x55, 0xab, 0x34, 0x9b, 0x76, 0xd7, 0xf2, 0x5c, 0xe5, 0x09, 0x60, 0xa6, 0x52, 0xb1, 0xac, 0x34, + 0x83, 0x78, 0x61, 0x0d, 0x9d, 0xe3, 0x45, 0xb1, 0xc4, 0x4b, 0xe3, 0x39, 0x54, 0xaa, 0x39, 0xe6, + 0x43, 0xdd, 0x23, 0x77, 0xc8, 0x46, 0xcd, 0x6e, 0x99, 0xcd, 0x0d, 0x30, 0xd7, 0xe7, 0x53, 0x65, + 0x87, 0xe1, 0x1a, 0x6b, 0x64, 0xa3, 0xd1, 0x01, 0xac, 0xb8, 0xac, 0x44, 0x4b, 0x8a, 0xf1, 0x7b, + 0x9e, 0xdc, 0x59, 0xfc, 0x1e, 0x82, 0x4a, 0xfc, 0x4e, 0xee, 0x91, 0x47, 0x2c, 0xba, 0xd4, 0xbb, + 0xdc, 0x34, 0x5f, 0x89, 0xdc, 0xe1, 0x05, 0x78, 0x9e, 0x33, 0x8a, 0x8d, 0x32, 0x12, 0x80, 0xc5, + 0x86, 0x45, 0x8b, 0xa8, 0x5f, 0xc8, 0x89, 0x53, 0x27, 0x3e, 0x8f, 0xf2, 0x42, 0xf8, 0x58, 0x08, + 0xfb, 0x01, 0xa1, 0xb6, 0xf2, 0x3c, 0xa6, 0xd0, 0x20, 0xdf, 0x76, 0x04, 0xfe, 0x69, 0x10, 0x5b, + 0xdf, 0x8f, 0xe9, 0x65, 0x1a, 0x5a, 0x48, 0x00, 0x71, 0xcd, 0xc3, 0xd4, 0x9a, 0x39, 0x21, 0xae, + 0x79, 0x98, 0x5a, 0x53, 0x4a, 0xac, 0x79, 0x1d, 0x0d, 0xf1, 0x69, 0x53, 0x08, 0x7b, 0x03, 0x71, + 0xb9, 0xfc, 0xec, 0x5a, 0x2c, 0xec, 0x97, 0x40, 0x84, 0x5f, 0x86, 0xfc, 0x72, 0xbe, 0xef, 0x5f, + 0x7f, 0xb8, 0x7d, 0x11, 0x07, 0x7e, 0x24, 0xc1, 0x9c, 0xef, 0x02, 0x38, 0x81, 0x46, 0x44, 0x4d, + 0xf2, 0x33, 0x39, 0xc0, 0x9c, 0x27, 0xa9, 0xdf, 0x86, 0x94, 0x1c, 0x59, 0x2c, 0x82, 0x17, 0xd1, + 0x58, 0x4c, 0x79, 0x78, 0x90, 0x1c, 0xc8, 0xeb, 0x91, 0xa0, 0x79, 0xe2, 0x9a, 0x1a, 0x2b, 0xab, + 0x7e, 0x4f, 0x36, 0xb6, 0x62, 0x50, 0xc1, 0x70, 0x2a, 0xa1, 0x73, 0x40, 0x30, 0x3e, 0x6b, 0x26, + 0x18, 0x81, 0x08, 0x5f, 0x46, 0xc5, 0x48, 0x8a, 0x39, 0xf0, 0x06, 0x0d, 0xf2, 0xcb, 0x05, 0x58, + 0x7c, 0x1d, 0x15, 0xe9, 0xfc, 0x6d, 0x45, 0x82, 0x4b, 0x75, 0x39, 0x4c, 0x9c, 0x70, 0x7d, 0x3a, + 0x5a, 0x46, 0x0a, 0x63, 0xec, 0x67, 0x02, 0x8b, 0xaf, 0x56, 0x61, 0x90, 0xf6, 0x60, 0xaf, 0xd8, + 0xbf, 0xdd, 0x5e, 0x51, 0xfd, 0xcd, 0x4c, 0x5c, 0xfb, 0xf1, 0xcd, 0x78, 0x84, 0x19, 0x96, 0x04, + 0xcc, 0x07, 0x8a, 0xb5, 0x06, 0xb1, 0x66, 0xa4, 0x58, 0x31, 0xd9, 0x3d, 0xc7, 0x8a, 0xc9, 0xed, + 0x32, 0x56, 0x8c, 0xfa, 0xbf, 0xf3, 0x3d, 0x2d, 0x3b, 0x0e, 0xc5, 0x27, 0xfa, 0x25, 0x7a, 0xde, + 0xa1, 0xb5, 0x57, 0xdc, 0xd8, 0xae, 0x9d, 0x3d, 0x5c, 0x37, 0x74, 0x36, 0x6a, 0x5c, 0x4d, 0xa6, + 0x14, 0x53, 0xb2, 0x43, 0x0c, 0xa2, 0x7c, 0x42, 0x4a, 0xf6, 0x68, 0x1e, 0x37, 0xb1, 0x00, 0xfe, + 0x30, 0x1a, 0x0c, 0x93, 0xcb, 0xf7, 0x0b, 0x11, 0xad, 0x12, 0x72, 0xca, 0x87, 0x94, 0xf8, 0xd3, + 0xa8, 0x20, 0x25, 0x12, 0xbc, 0xb6, 0x03, 0x53, 0x98, 0x71, 0x31, 0x4e, 0x22, 0x3b, 0x3b, 0x44, + 0x93, 0x08, 0x72, 0xa6, 0x78, 0x09, 0x9d, 0xac, 0x39, 0xc4, 0x00, 0xa3, 0xab, 0xe9, 0x47, 0x1d, + 0x87, 0x47, 0xb1, 0x64, 0x03, 0x18, 0x96, 0x8e, 0x8e, 0x8f, 0xa6, 0x8b, 0x1a, 0xc7, 0x0b, 0x8c, + 0x92, 0x8a, 0xd3, 0xfd, 0x04, 0x6b, 0xc9, 0x1d, 0xb2, 0xb1, 0x6e, 0x3b, 0x06, 0x0b, 0xf4, 0xc8, + 0xf7, 0x13, 0x5c, 0xd0, 0x6b, 0x1c, 0x25, 0xee, 0x27, 0xe4, 0x42, 0xe7, 0x5e, 0x42, 0x43, 0x7b, + 0x8d, 0x35, 0xf8, 0x4b, 0xd9, 0x14, 0x1b, 0xc9, 0xc7, 0x37, 0x47, 0x44, 0x90, 0xaf, 0xa7, 0x3f, + 0x25, 0x5f, 0xcf, 0xb7, 0xb3, 0x29, 0x06, 0xa0, 0x8f, 0x75, 0x5e, 0x8d, 0x40, 0x18, 0x72, 0x5e, + 0x8d, 0x30, 0xa5, 0x89, 0x69, 0x68, 0x22, 0x51, 0x24, 0x03, 0x4f, 0x61, 0xdb, 0x0c, 0x3c, 0x3f, + 0x93, 0xeb, 0x65, 0x20, 0x7b, 0x2c, 0xfb, 0xdd, 0xc8, 0xfe, 0x3a, 0x1a, 0x0a, 0x24, 0xcb, 0xb3, + 0x31, 0x8f, 0x04, 0x91, 0x4d, 0x19, 0x18, 0xca, 0x08, 0x44, 0xf8, 0x0a, 0x6b, 0x6b, 0xdd, 0x7c, + 0x87, 0x45, 0xf7, 0x1b, 0xe1, 0x71, 0xdb, 0x74, 0x4f, 0x6f, 0xb8, 0xe6, 0x3b, 0x44, 0x0b, 0xd0, + 0xea, 0x3f, 0xcd, 0x26, 0x5a, 0x19, 0x1f, 0xf7, 0xd1, 0x2e, 0xfa, 0x28, 0x41, 0x88, 0xcc, 0x3e, + 0xfa, 0x58, 0x88, 0xbb, 0x10, 0xe2, 0x9f, 0x64, 0x13, 0xad, 0xc9, 0x8f, 0x85, 0xb8, 0x9b, 0xd9, + 0xe2, 0x2a, 0x1a, 0xd4, 0xec, 0x75, 0x77, 0x12, 0xce, 0x2c, 0x6c, 0xae, 0x80, 0x89, 0xda, 0xb1, + 0xd7, 0xdd, 0x06, 0x9c, 0x46, 0xb4, 0x90, 0x40, 0xfd, 0xb3, 0x6c, 0x0f, 0x7b, 0xfb, 0x63, 0xc1, + 0xbf, 0x9b, 0x4b, 0xe4, 0xaf, 0x66, 0x25, 0x7b, 0xfe, 0xc7, 0x3a, 0x41, 0x5d, 0xbd, 0xb9, 0x4a, + 0xda, 0x7a, 0x34, 0x41, 0x9d, 0x0b, 0x50, 0x9e, 0xdf, 0x26, 0x24, 0x51, 0xbf, 0x9c, 0x8d, 0x38, + 0x34, 0x1c, 0xcb, 0x6e, 0xc7, 0xb2, 0x0b, 0xb4, 0x8e, 0xfb, 0x68, 0x1c, 0x4b, 0x6e, 0xa7, 0x92, + 0xfb, 0xc1, 0x6c, 0xc4, 0x9d, 0xe5, 0xf1, 0xcd, 0x55, 0xf5, 0xe5, 0x6c, 0xdc, 0x35, 0xe7, 0xf1, + 0xd5, 0xa4, 0xab, 0x68, 0x90, 0xcb, 0x21, 0x58, 0x2a, 0xd8, 0xbc, 0xcf, 0x80, 0x70, 0x81, 0x1a, + 0x10, 0xa8, 0xdf, 0x97, 0x45, 0xb2, 0x9b, 0xd1, 0x63, 0xaa, 0x43, 0xbf, 0x9a, 0x95, 0x1d, 0xac, + 0x1e, 0x5f, 0xfd, 0x19, 0x47, 0xa8, 0xde, 0x5d, 0x6e, 0xf2, 0xf8, 0x5c, 0xfd, 0xc2, 0x0d, 0x7c, + 0x00, 0xd5, 0x04, 0x0a, 0xf5, 0xff, 0x64, 0x13, 0xbd, 0xbe, 0x1e, 0x5f, 0x01, 0xde, 0x80, 0x5b, + 0xf1, 0xa6, 0x15, 0x4e, 0xe4, 0x70, 0x09, 0x49, 0xc7, 0x5f, 0x2c, 0xac, 0xbe, 0x4f, 0x88, 0x3f, + 0x92, 0xb0, 0x5d, 0x83, 0xa0, 0x85, 0x89, 0xb9, 0xba, 0xc5, 0x8d, 0xdb, 0xbf, 0xc8, 0x6e, 0xe7, + 0x24, 0xf7, 0x38, 0xaf, 0xaa, 0x03, 0x35, 0x7d, 0x03, 0x82, 0xb9, 0xd0, 0x9e, 0x18, 0x66, 0x41, + 0xdf, 0x3b, 0x0c, 0x24, 0xbe, 0x88, 0x71, 0x2a, 0xf5, 0x8f, 0xfb, 0x93, 0x3d, 0xb4, 0x1e, 0x5f, + 0x11, 0x9e, 0x47, 0xf9, 0x9a, 0xee, 0xad, 0x72, 0x4d, 0x86, 0xd7, 0xba, 0x8e, 0xee, 0xad, 0x6a, + 0x00, 0xc5, 0x57, 0x50, 0x51, 0xd3, 0xd7, 0xc5, 0x1c, 0xe5, 0x70, 0xb1, 0xe3, 0xe8, 0xeb, 0x3c, + 0x51, 0x7d, 0x80, 0xc6, 0x6a, 0x90, 0x10, 0x82, 0xdd, 0x7c, 0x43, 0x34, 0x75, 0x96, 0x10, 0x22, + 0x48, 0x03, 0x71, 0x1e, 0xe5, 0x27, 0x6c, 0x63, 0x03, 0x8c, 0x59, 0x86, 0x59, 0x65, 0xcb, 0xb6, + 0xb1, 0xa1, 0x01, 0x14, 0xff, 0x50, 0x06, 0x0d, 0xcc, 0x12, 0xdd, 0xa0, 0x23, 0x64, 0xb0, 0x97, + 0x2d, 0xc8, 0xc7, 0x0f, 0xc6, 0x16, 0x64, 0x6c, 0x95, 0x55, 0x26, 0x2a, 0x0a, 0xaf, 0x1f, 0xdf, + 0x42, 0xc5, 0x49, 0xdd, 0x23, 0x2b, 0xb6, 0xb3, 0x01, 0xd6, 0x2d, 0xa3, 0xa1, 0x99, 0xaa, 0xa4, + 0x3f, 0x3e, 0x11, 0x7b, 0x19, 0x6b, 0xf2, 0x5f, 0x5a, 0x50, 0x98, 0x8a, 0x85, 0x27, 0x8a, 0x1b, + 0x0a, 0xc5, 0xc2, 0x32, 0xc2, 0x05, 0xf9, 0xe0, 0x82, 0x6b, 0xe5, 0xe1, 0xe4, 0x6b, 0x65, 0xd8, + 0x3d, 0x82, 0x05, 0x1c, 0xa4, 0x61, 0x18, 0x81, 0x45, 0x9f, 0xed, 0x1e, 0x01, 0x0a, 0x59, 0x18, + 0x34, 0x81, 0x44, 0xfd, 0x66, 0x3f, 0x4a, 0xf4, 0xe7, 0x38, 0x56, 0xf2, 0x63, 0x25, 0x0f, 0x95, + 0x7c, 0x2a, 0xa6, 0xe4, 0xe7, 0xe2, 0x1e, 0x42, 0xef, 0x51, 0x0d, 0xff, 0xd1, 0x7c, 0xcc, 0xbf, + 0xf0, 0xf1, 0x3e, 0x5d, 0x86, 0xd2, 0xeb, 0xdf, 0x56, 0x7a, 0xc1, 0x80, 0x28, 0x6c, 0x3b, 0x20, + 0x06, 0x76, 0x3a, 0x20, 0x8a, 0xa9, 0x03, 0x22, 0x54, 0x90, 0xc1, 0x54, 0x05, 0xa9, 0xf2, 0x41, + 0x83, 0x7a, 0xa7, 0xf8, 0x39, 0xbf, 0xb5, 0x59, 0x1e, 0xa5, 0xa3, 0x29, 0x31, 0xb9, 0x0f, 0xb0, + 0x50, 0xbf, 0x91, 0xef, 0xe1, 0x14, 0x7c, 0x28, 0x3a, 0x72, 0x03, 0xe5, 0x2a, 0x9d, 0x0e, 0xd7, + 0x8f, 0x93, 0x82, 0x3f, 0x72, 0x4a, 0x29, 0x4a, 0x8d, 0x5f, 0x46, 0xb9, 0xca, 0xfd, 0x7a, 0x34, + 0xb4, 0x71, 0xe5, 0x7e, 0x9d, 0x7f, 0x49, 0x6a, 0xd9, 0xfb, 0x75, 0xfc, 0x4a, 0x18, 0x63, 0x68, + 0xb5, 0x6b, 0xad, 0xf1, 0x83, 0x22, 0x37, 0x82, 0xf5, 0x2d, 0x6d, 0x9a, 0x14, 0x45, 0x8f, 0x8b, + 0x11, 0xda, 0x88, 0x36, 0x15, 0x76, 0xae, 0x4d, 0x03, 0xdb, 0x6a, 0x53, 0x71, 0xa7, 0xda, 0x34, + 0xb8, 0x03, 0x6d, 0x42, 0xdb, 0x6a, 0xd3, 0xd0, 0xfe, 0xb5, 0xa9, 0x83, 0xce, 0xc5, 0x03, 0x39, + 0x04, 0x1a, 0xa1, 0x21, 0x1c, 0xc7, 0x72, 0xc3, 0x12, 0x78, 0xfa, 0xef, 0x32, 0x6c, 0x83, 0x25, + 0x74, 0x8c, 0xa6, 0x43, 0xd4, 0x12, 0x4a, 0xab, 0xbf, 0x94, 0x4d, 0x8f, 0x3f, 0x71, 0x34, 0xa7, + 0xb8, 0xef, 0x4a, 0x94, 0x52, 0x5e, 0xf6, 0xbc, 0x4a, 0x97, 0x72, 0x84, 0x6d, 0x92, 0xcc, 0xbe, + 0x96, 0x49, 0x0b, 0x8a, 0xb1, 0x2f, 0x89, 0xbd, 0x3f, 0x6e, 0xac, 0x06, 0xd6, 0xf3, 0xae, 0x6c, + 0xa5, 0x16, 0xcd, 0x0f, 0x98, 0xdb, 0x63, 0x7e, 0xc0, 0xdf, 0xcc, 0xa0, 0x93, 0x77, 0xba, 0xcb, + 0x84, 0x1b, 0xa7, 0x05, 0xcd, 0x78, 0x1b, 0x21, 0x0a, 0xe6, 0x46, 0x2c, 0x19, 0x30, 0x62, 0xf9, + 0xa0, 0x18, 0xd0, 0x22, 0x52, 0x60, 0x3c, 0xa4, 0x66, 0x06, 0x2c, 0x17, 0x7c, 0x13, 0xcb, 0xb5, + 0xee, 0x32, 0x69, 0xc4, 0x2c, 0x59, 0x04, 0xee, 0xe7, 0x5e, 0x65, 0xc6, 0xeb, 0x7b, 0x35, 0x1a, + 0xf9, 0x85, 0x6c, 0x6a, 0x0c, 0x91, 0x23, 0x9b, 0xc2, 0xe1, 0x93, 0x89, 0xbd, 0x12, 0x4d, 0xe5, + 0x90, 0x40, 0x12, 0xe1, 0x98, 0xc4, 0x25, 0x59, 0x60, 0x47, 0x3c, 0xb1, 0xc8, 0xbb, 0x2a, 0xb0, + 0xdf, 0xcb, 0xa4, 0xc6, 0x7a, 0x39, 0xaa, 0x02, 0x53, 0x7f, 0x3b, 0xeb, 0x87, 0x98, 0xd9, 0xd7, + 0x27, 0x5c, 0x45, 0x83, 0x3c, 0x8e, 0xbe, 0x6c, 0x5b, 0xcb, 0xaf, 0xf2, 0xe0, 0x6a, 0x38, 0x20, + 0xa0, 0xcb, 0xbc, 0x1f, 0x02, 0x23, 0xc8, 0x28, 0x09, 0xcb, 0xbc, 0xc9, 0xa1, 0x94, 0x5e, 0x20, + 0xa1, 0x0b, 0xf9, 0xf4, 0x23, 0xd3, 0x83, 0x5d, 0x01, 0xed, 0xcb, 0x1c, 0x5b, 0xc8, 0xc9, 0x23, + 0xd3, 0x63, 0x7b, 0x82, 0x00, 0x4d, 0x17, 0xe9, 0x7a, 0x98, 0x36, 0x8d, 0x2f, 0xd2, 0x2e, 0xcf, + 0x1e, 0xc7, 0x9d, 0xb9, 0xae, 0xa2, 0x41, 0x6e, 0xb0, 0xca, 0xcd, 0x4c, 0x78, 0x6b, 0xb9, 0x89, + 0x2b, 0xb4, 0x36, 0x20, 0xa0, 0x1c, 0x35, 0xb2, 0x12, 0x1a, 0xd6, 0x01, 0x47, 0x07, 0x20, 0x1a, + 0xc7, 0xa8, 0x5b, 0xd9, 0x78, 0xa4, 0x9b, 0xc7, 0xf7, 0x50, 0x70, 0x45, 0x36, 0x56, 0x03, 0x0b, + 0x4d, 0xd8, 0x70, 0x89, 0xb6, 0xb2, 0x6c, 0xdf, 0x75, 0x1d, 0x15, 0xef, 0x90, 0x0d, 0x66, 0x57, + 0x59, 0x08, 0x4d, 0x71, 0xd7, 0x38, 0x4c, 0xbc, 0xd1, 0xf4, 0xe9, 0xd4, 0xdf, 0xc8, 0xc6, 0x63, + 0xf8, 0x3c, 0xbe, 0xc2, 0xfe, 0x10, 0x1a, 0x00, 0x51, 0x56, 0xfd, 0x2b, 0x75, 0x10, 0x20, 0x88, + 0x5b, 0xb6, 0xf0, 0xf5, 0xc9, 0xd4, 0x9f, 0x28, 0x44, 0x03, 0x3b, 0x3d, 0xbe, 0xd2, 0xfb, 0x28, + 0x1a, 0x9a, 0xb4, 0x2d, 0xd7, 0x74, 0x3d, 0x62, 0x35, 0x7d, 0x85, 0x7d, 0x82, 0x6e, 0x58, 0x9a, + 0x21, 0x58, 0xf4, 0xbc, 0x11, 0xa8, 0xf7, 0xa2, 0xbc, 0xf8, 0x05, 0x34, 0x08, 0x22, 0x07, 0x3b, + 0x64, 0x21, 0x1f, 0xed, 0x32, 0x05, 0x46, 0x8d, 0x90, 0x43, 0x52, 0x7c, 0x17, 0x15, 0x27, 0x57, + 0xcd, 0x96, 0xe1, 0x10, 0x8b, 0x27, 0x5e, 0x7f, 0x3a, 0x39, 0x0c, 0xd7, 0x38, 0xfc, 0x0b, 0xb4, + 0xac, 0x39, 0x4d, 0x5e, 0x4c, 0xf2, 0x3d, 0xe2, 0xb0, 0x73, 0x7f, 0x2b, 0x8b, 0x50, 0x58, 0x00, + 0x3f, 0x85, 0xb2, 0x41, 0xc6, 0x1f, 0x30, 0x03, 0x91, 0x34, 0x28, 0x0b, 0x53, 0x31, 0x1f, 0xdb, + 0xd9, 0x6d, 0xc7, 0xf6, 0x5d, 0x54, 0x60, 0x37, 0x4a, 0x60, 0xa9, 0x2d, 0xc4, 0x9a, 0x49, 0x6d, + 0xf0, 0x38, 0xd0, 0xb3, 0xc3, 0x22, 0xec, 0xec, 0x24, 0xab, 0x67, 0xc6, 0xec, 0x5c, 0x13, 0xf5, + 0xc3, 0x5f, 0xf8, 0x12, 0xca, 0x83, 0x14, 0x33, 0x70, 0x4e, 0x04, 0x37, 0xd1, 0x88, 0xfc, 0x00, + 0x4f, 0xbb, 0x69, 0xd2, 0xb6, 0x3c, 0x5a, 0x35, 0xb4, 0x7a, 0x98, 0xcb, 0x85, 0xc3, 0x24, 0xb9, + 0x70, 0x98, 0xfa, 0xcf, 0xb3, 0x09, 0x21, 0xc7, 0x1e, 0xdf, 0x61, 0xf2, 0x12, 0x42, 0xe0, 0xc8, + 0x4c, 0xe5, 0xe9, 0xbb, 0x40, 0xc0, 0x28, 0x01, 0x46, 0xa0, 0xb6, 0xd2, 0xb6, 0x3e, 0x24, 0x56, + 0x7f, 0x27, 0x13, 0x8b, 0x53, 0xb5, 0x2f, 0x39, 0x8a, 0xbb, 0x9e, 0xec, 0x1e, 0xb7, 0x89, 0x7e, + 0x5f, 0xe4, 0x76, 0xd7, 0x17, 0xf2, 0xb7, 0x1c, 0xc0, 0xce, 0xef, 0x30, 0xbf, 0xe5, 0x9b, 0xd9, + 0xa4, 0xa8, 0x5d, 0x47, 0x53, 0xc5, 0xc3, 0xa4, 0xe6, 0xf9, 0x5d, 0x24, 0x35, 0x7f, 0x0b, 0x9d, + 0x88, 0xc4, 0xb2, 0xe2, 0x69, 0xb8, 0x2e, 0xf5, 0x0e, 0x8a, 0x95, 0xee, 0x02, 0x2f, 0x91, 0xa9, + 0xff, 0x37, 0xd3, 0x3b, 0x92, 0xd9, 0xa1, 0xab, 0x4e, 0x82, 0x00, 0x72, 0x7f, 0x39, 0x02, 0x38, + 0x80, 0x63, 0xe6, 0xd1, 0x16, 0xc0, 0x7b, 0x64, 0xf2, 0x78, 0xb7, 0x05, 0xf0, 0x13, 0x99, 0x6d, + 0x03, 0xd1, 0x1d, 0xb6, 0x0c, 0xd4, 0xff, 0x98, 0x49, 0x0c, 0x18, 0xb7, 0xaf, 0x76, 0xbd, 0x82, + 0x0a, 0xcc, 0x6c, 0x85, 0xb7, 0x4a, 0x08, 0xb1, 0x4f, 0xa1, 0x29, 0xe5, 0x79, 0x19, 0x3c, 0x87, + 0x06, 0x58, 0x1b, 0x8c, 0x68, 0x2a, 0xca, 0x84, 0x76, 0x1a, 0x69, 0x93, 0x23, 0x47, 0xab, 0xbf, + 0x95, 0x89, 0xc5, 0xaf, 0x3b, 0xc4, 0x6f, 0x0b, 0xa7, 0xea, 0xdc, 0xce, 0xa7, 0x6a, 0xf5, 0x8f, + 0xb2, 0xc9, 0xe1, 0xf3, 0x0e, 0xf1, 0x43, 0x0e, 0xe2, 0xba, 0x6a, 0x6f, 0xeb, 0xd6, 0x12, 0x1a, + 0x95, 0x65, 0xc1, 0x97, 0xad, 0x8b, 0xc9, 0x41, 0x04, 0x53, 0x5a, 0x11, 0xe1, 0xa1, 0x7e, 0x3d, + 0x13, 0x8f, 0xfc, 0x77, 0xe8, 0xf3, 0xd3, 0xde, 0xb4, 0x45, 0xfe, 0x94, 0xf7, 0xc8, 0x5a, 0x73, + 0x10, 0x9f, 0xf2, 0x1e, 0x59, 0x35, 0xf6, 0xf6, 0x29, 0x3f, 0x9b, 0x4d, 0x0b, 0x9c, 0x78, 0xe8, + 0x1f, 0xf4, 0x09, 0x51, 0xc8, 0xac, 0x65, 0xfc, 0xd3, 0x9e, 0x4a, 0x8b, 0x54, 0x98, 0xc2, 0x33, + 0xc6, 0x67, 0x6f, 0x63, 0x3c, 0x51, 0x58, 0xef, 0x11, 0x45, 0x3e, 0x1a, 0xc2, 0x7a, 0x8f, 0x0c, + 0x95, 0xf7, 0x9e, 0xb0, 0xbe, 0x92, 0xdd, 0x69, 0xb4, 0xce, 0x63, 0xe1, 0xc5, 0x84, 0xf7, 0xc5, + 0x6c, 0x3c, 0x8a, 0xec, 0xa1, 0x8b, 0x69, 0x06, 0x15, 0x78, 0x3c, 0xdb, 0x54, 0xe1, 0x30, 0x7c, + 0xda, 0x8e, 0x86, 0x7f, 0xc7, 0x4d, 0xc4, 0x1f, 0x4a, 0x76, 0x26, 0x12, 0x46, 0xab, 0xfe, 0x59, + 0x26, 0x12, 0x72, 0xf5, 0x50, 0xae, 0x10, 0xf6, 0xb4, 0x24, 0xe1, 0x57, 0xfd, 0xcb, 0xcc, 0x7c, + 0x24, 0x9b, 0x65, 0xf0, 0x3d, 0x53, 0xc4, 0xd3, 0xcd, 0x56, 0xb4, 0x3c, 0xf7, 0xb9, 0xff, 0x8d, + 0x2c, 0x1a, 0x8b, 0x91, 0xe2, 0x4b, 0x52, 0x14, 0x1a, 0xb8, 0x96, 0x8c, 0x18, 0x67, 0xb3, 0x78, + 0x34, 0xbb, 0xb8, 0x49, 0xbd, 0x84, 0xf2, 0x53, 0xfa, 0x06, 0xfb, 0xb6, 0x7e, 0xc6, 0xd2, 0xd0, + 0x37, 0xc4, 0x1b, 0x37, 0xc0, 0xe3, 0x65, 0x74, 0x9a, 0xbd, 0x87, 0x98, 0xb6, 0xb5, 0x64, 0xb6, + 0x49, 0xd5, 0x9a, 0x37, 0x5b, 0x2d, 0xd3, 0xe5, 0x8f, 0x66, 0x57, 0xb7, 0x36, 0xcb, 0x97, 0x3d, + 0xdb, 0xd3, 0x5b, 0x0d, 0xe2, 0x93, 0x35, 0x3c, 0xb3, 0x4d, 0x1a, 0xa6, 0xd5, 0x68, 0x03, 0xa5, + 0xc0, 0x32, 0x99, 0x15, 0xae, 0xb2, 0x74, 0x9b, 0xf5, 0xa6, 0x6e, 0x59, 0xc4, 0xa8, 0x5a, 0x13, + 0x1b, 0x1e, 0x61, 0x8f, 0x6d, 0x39, 0x76, 0x25, 0xc8, 0x7c, 0xaf, 0x19, 0x9a, 0x32, 0x5e, 0xa6, + 0x04, 0x5a, 0x42, 0x21, 0xf5, 0xab, 0xf9, 0x84, 0x68, 0xbb, 0x47, 0x48, 0x7d, 0xfc, 0x9e, 0xce, + 0x6f, 0xd3, 0xd3, 0xd7, 0xd0, 0x00, 0x0f, 0x68, 0xc9, 0x1f, 0x18, 0xc0, 0x58, 0xfc, 0x21, 0x03, + 0x89, 0x2f, 0x34, 0x9c, 0x0a, 0xb7, 0xd0, 0xb9, 0x25, 0xda, 0x4d, 0xc9, 0x9d, 0x59, 0xd8, 0x43, + 0x67, 0xf6, 0xe0, 0x87, 0xdf, 0x44, 0x67, 0x01, 0x9b, 0xd0, 0xad, 0x03, 0x50, 0x15, 0x44, 0x66, + 0x62, 0x55, 0x25, 0x77, 0x6e, 0x5a, 0x79, 0xfc, 0x09, 0x34, 0x1c, 0x0c, 0x10, 0x93, 0xb8, 0xfc, + 0xe5, 0xa2, 0xc7, 0x38, 0x63, 0x61, 0xcf, 0x28, 0x18, 0x4c, 0xb4, 0xe4, 0xd0, 0x59, 0x12, 0x2f, + 0xf5, 0x3f, 0x64, 0x7a, 0xc5, 0x57, 0x3e, 0xf4, 0x59, 0xf9, 0x55, 0x34, 0x60, 0xb0, 0x8f, 0xe2, + 0x3a, 0xd5, 0x3b, 0x02, 0x33, 0x23, 0xd5, 0xfc, 0x32, 0xea, 0x1f, 0x66, 0x7a, 0x86, 0x75, 0x3e, + 0xea, 0x9f, 0xf7, 0xc5, 0x5c, 0xca, 0xe7, 0xf1, 0x49, 0xf4, 0x0a, 0x2a, 0x99, 0x61, 0xc8, 0xe0, + 0x46, 0x18, 0xde, 0x49, 0x3b, 0x21, 0xc0, 0x61, 0x74, 0xdd, 0x44, 0x67, 0x7c, 0xc3, 0x42, 0xc7, + 0xb7, 0xc0, 0x72, 0x1b, 0x5d, 0xc7, 0x64, 0xe3, 0x52, 0x3b, 0xe5, 0x46, 0xcc, 0xb3, 0xdc, 0xbb, + 0x8e, 0x49, 0x2b, 0xd0, 0xbd, 0x55, 0x62, 0xe9, 0x8d, 0x75, 0xdb, 0x59, 0x83, 0xd8, 0x9a, 0x6c, + 0x70, 0x6a, 0x27, 0x18, 0xfc, 0xbe, 0x0f, 0xc6, 0xcf, 0xa0, 0x91, 0x95, 0x56, 0x97, 0x04, 0xd1, + 0x0c, 0xd9, 0x5b, 0x9f, 0x36, 0x4c, 0x81, 0xc1, 0x0b, 0xc9, 0x05, 0x84, 0x80, 0xc8, 0x83, 0xa0, + 0xdb, 0xf0, 0xb0, 0xa7, 0x0d, 0x52, 0xc8, 0x12, 0xef, 0xae, 0x73, 0x4c, 0xab, 0x99, 0x90, 0x1a, + 0x2d, 0xdb, 0x5a, 0x69, 0x78, 0xc4, 0x69, 0x43, 0x43, 0xc1, 0x38, 0x51, 0x3b, 0x03, 0x14, 0xf0, + 0x74, 0xe2, 0xce, 0xd9, 0xd6, 0xca, 0x12, 0x71, 0xda, 0xb4, 0xa9, 0x57, 0x11, 0xe6, 0x4d, 0x75, + 0xe0, 0xd2, 0x83, 0x7d, 0x1c, 0xd8, 0x29, 0x6a, 0xfc, 0x23, 0xd8, 0x6d, 0x08, 0x7c, 0x58, 0x19, + 0x0d, 0xb1, 0x90, 0x6e, 0x4c, 0x68, 0x60, 0xaa, 0xa8, 0x21, 0x06, 0x02, 0x79, 0x9d, 0x41, 0xdc, + 0x7a, 0x81, 0x59, 0x4d, 0x6b, 0xfc, 0x97, 0xfa, 0xbb, 0xd9, 0xb4, 0x88, 0xcc, 0x47, 0xf5, 0x8d, + 0x03, 0xcf, 0x22, 0xc4, 0x53, 0x57, 0xd2, 0xcf, 0x8d, 0x18, 0xb4, 0x86, 0x98, 0x14, 0x1e, 0x42, + 0x59, 0x61, 0x81, 0xe8, 0xdf, 0xc5, 0x76, 0x31, 0x49, 0xa4, 0x07, 0x70, 0x8a, 0x0b, 0x1b, 0x93, + 0xdd, 0xc5, 0x6a, 0x75, 0xf8, 0x42, 0x14, 0x55, 0xa1, 0x7f, 0x8f, 0xb7, 0xb5, 0x49, 0x22, 0x3d, + 0xda, 0x2f, 0x71, 0x87, 0xae, 0xa5, 0xbf, 0x97, 0x4d, 0x8d, 0x3c, 0x7e, 0x2c, 0xd3, 0x83, 0x94, + 0xe9, 0xf1, 0xd0, 0xdf, 0xd7, 0xd0, 0x4f, 0x94, 0xe9, 0xf1, 0xd8, 0xdf, 0x8f, 0x9e, 0x3e, 0x3b, + 0xcf, 0xc2, 0x67, 0xde, 0x31, 0x2d, 0x03, 0x3f, 0x81, 0x4e, 0xdf, 0xad, 0x4f, 0x6b, 0x8d, 0x3b, + 0xd5, 0x85, 0xa9, 0xc6, 0xdd, 0x85, 0x7a, 0x6d, 0x7a, 0xb2, 0x3a, 0x53, 0x9d, 0x9e, 0x2a, 0xf5, + 0xe1, 0x93, 0xe8, 0x44, 0x88, 0x9a, 0xbd, 0x3b, 0x5f, 0x59, 0x28, 0x65, 0xf0, 0x18, 0x1a, 0x09, + 0x81, 0x13, 0x8b, 0x4b, 0xa5, 0xec, 0xb3, 0x1f, 0x40, 0x43, 0xb0, 0x7f, 0xa9, 0xb0, 0x36, 0x0d, + 0xa3, 0xe2, 0xe2, 0x44, 0x7d, 0x5a, 0xbb, 0x07, 0x4c, 0x10, 0x2a, 0x4c, 0x4d, 0x2f, 0x50, 0x86, + 0x99, 0x67, 0xff, 0x57, 0x06, 0xa1, 0xfa, 0xcc, 0x52, 0x8d, 0x13, 0x0e, 0xa1, 0x81, 0xea, 0xc2, + 0xbd, 0xca, 0x5c, 0x95, 0xd2, 0x15, 0x51, 0x7e, 0xb1, 0x36, 0x4d, 0x6b, 0x18, 0x44, 0xfd, 0x93, + 0x73, 0x8b, 0xf5, 0xe9, 0x52, 0x96, 0x02, 0xb5, 0xe9, 0xca, 0x54, 0x29, 0x47, 0x81, 0xf7, 0xb5, + 0xea, 0xd2, 0x74, 0x29, 0x4f, 0xff, 0x9c, 0xab, 0x2f, 0x55, 0x96, 0x4a, 0xfd, 0xf4, 0xcf, 0x19, + 0xf8, 0xb3, 0x40, 0x99, 0xd5, 0xa7, 0x97, 0xe0, 0xc7, 0x00, 0x6d, 0xc2, 0x8c, 0xff, 0xab, 0x48, + 0x51, 0x94, 0xf5, 0x54, 0x55, 0x2b, 0x0d, 0xd2, 0x1f, 0x94, 0x25, 0xfd, 0x81, 0x68, 0xe3, 0xb4, + 0xe9, 0xf9, 0xc5, 0x7b, 0xd3, 0xa5, 0x21, 0xca, 0x6b, 0xfe, 0x0e, 0x05, 0x0f, 0xd3, 0x3f, 0xb5, + 0x79, 0xfa, 0xe7, 0x08, 0xe5, 0xa4, 0x4d, 0x57, 0xe6, 0x6a, 0x95, 0xa5, 0xd9, 0xd2, 0x28, 0x6d, + 0x0f, 0xf0, 0x3c, 0xc1, 0x4a, 0x2e, 0x54, 0xe6, 0xa7, 0x4b, 0x25, 0x4e, 0x33, 0x35, 0x57, 0x5d, + 0xb8, 0x53, 0x1a, 0x83, 0x86, 0xbc, 0x39, 0x0f, 0x3f, 0x30, 0x2d, 0x00, 0x7f, 0x9d, 0x7c, 0xf6, + 0x53, 0xa8, 0xb0, 0x58, 0x07, 0xcb, 0xa4, 0xb3, 0xe8, 0xe4, 0x62, 0xbd, 0xb1, 0xf4, 0x66, 0x6d, + 0x3a, 0x22, 0xef, 0x31, 0x34, 0xe2, 0x23, 0xe6, 0xaa, 0x0b, 0x77, 0x3f, 0xce, 0xa4, 0xed, 0x83, + 0xe6, 0x2b, 0x93, 0x8b, 0xf5, 0x52, 0x96, 0xf6, 0x8a, 0x0f, 0xba, 0x5f, 0x5d, 0x98, 0x5a, 0xbc, + 0x5f, 0x2f, 0xe5, 0x9e, 0x7d, 0xe8, 0xa7, 0xce, 0x5a, 0x74, 0xcc, 0x15, 0xd3, 0xc2, 0x17, 0xd0, + 0x13, 0x53, 0xd3, 0xf7, 0xaa, 0x93, 0xd3, 0x8d, 0x45, 0xad, 0x7a, 0xab, 0xba, 0x10, 0xa9, 0xe9, + 0x34, 0x1a, 0x93, 0xd1, 0x95, 0x5a, 0xb5, 0x94, 0xc1, 0x67, 0x10, 0x96, 0xc1, 0xb7, 0x2b, 0xf3, + 0x33, 0xa5, 0x2c, 0x56, 0xd0, 0x29, 0x19, 0x5e, 0x5d, 0x58, 0xba, 0xbb, 0x30, 0x5d, 0xca, 0x3d, + 0xfb, 0x53, 0x19, 0x74, 0x3a, 0xd1, 0x41, 0x15, 0xab, 0xe8, 0xe2, 0xf4, 0x5c, 0xa5, 0xbe, 0x54, + 0x9d, 0xac, 0x4f, 0x57, 0xb4, 0xc9, 0xd9, 0xc6, 0x64, 0x65, 0x69, 0xfa, 0xd6, 0xa2, 0xf6, 0x66, + 0xe3, 0xd6, 0xf4, 0xc2, 0xb4, 0x56, 0x99, 0x2b, 0xf5, 0xe1, 0x67, 0x50, 0x39, 0x85, 0xa6, 0x3e, + 0x3d, 0x79, 0x57, 0xab, 0x2e, 0xbd, 0x59, 0xca, 0xe0, 0xa7, 0xd1, 0x85, 0x54, 0x22, 0xfa, 0xbb, + 0x94, 0xc5, 0x17, 0xd1, 0xb9, 0x34, 0x92, 0x37, 0xe6, 0x4a, 0xb9, 0x67, 0x7f, 0x2c, 0x83, 0x70, + 0xdc, 0xc3, 0x10, 0x3f, 0x85, 0xce, 0x53, 0xbd, 0x68, 0xa4, 0x37, 0xf0, 0x69, 0x74, 0x21, 0x91, + 0x42, 0x68, 0x5e, 0x19, 0x3d, 0x99, 0x42, 0xc2, 0x1b, 0x77, 0x1e, 0x29, 0xc9, 0x04, 0xb4, 0x69, + 0x13, 0x53, 0x5f, 0xff, 0x4f, 0x17, 0xfb, 0xbe, 0xfe, 0xad, 0x8b, 0x99, 0xdf, 0xfd, 0xd6, 0xc5, + 0xcc, 0x1f, 0x7d, 0xeb, 0x62, 0xe6, 0x13, 0xd7, 0x77, 0xe3, 0x80, 0xc9, 0x46, 0xfb, 0x72, 0x01, + 0x5c, 0x8d, 0x6e, 0xfc, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x3b, 0xad, 0x29, 0xa5, 0xe9, 0x37, + 0x01, 0x00, } func (m *Metadata) Marshal() (dAtA []byte, err error) { diff --git a/api/types/installers/installer.sh.tmpl b/api/types/installers/installer.sh.tmpl index 1209c6dc2abc0..c2dd09bee8903 100644 --- a/api/types/installers/installer.sh.tmpl +++ b/api/types/installers/installer.sh.tmpl @@ -3,32 +3,6 @@ set -eu -upgrade_endpoint="{{ .PublicProxyAddr }}/v1/webapi/automaticupgrades/channel/default" - -# upgrade_endpoint_fetch loads the specified value from the upgrade endpoint. the only -# currently supported values are 'version' and 'critical'. -upgrade_endpoint_fetch() { - host_path="${upgrade_endpoint}/${1}" - - if sf_output="$(curl --proto '=https' --tlsv1.2 -sSf "https://${host_path}")"; then - # emit output with empty lines and extra whitespace removed - echo "$sf_output" | grep -v -e '^[[:space:]]*$' | awk '{$1=$1};1' - return 0 - else - return 1 - fi -} - -# get_target_version loads the current value of the /version endpoint. -get_target_version() { - if tv_output="$(upgrade_endpoint_fetch version)"; then - # emit version string with leading 'v' removed if one is present - echo "${tv_output#v}" - return 0 - fi - return 1 -} - on_ec2() { IMDS_TOKEN=$(curl -m5 -sS -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 300") [ -z "$IMDS_TOKEN" ] && return 1 @@ -51,102 +25,17 @@ on_gcp() { if test -f /usr/local/bin/teleport; then exit 0 fi - # shellcheck disable=SC1091 - . /etc/os-release - - TELEPORT_PACKAGE="{{ .TeleportPackage }}" - TELEPORT_UPDATER_PACKAGE="{{ .TeleportPackage }}-updater" - - if [ "$ID" = "debian" ] || [ "$ID" = "ubuntu" ]; then - # old versions of ubuntu require that keys get added by `apt-key add`, without - # adding the key apt shows a key signing error when installing teleport. - if [ "$VERSION_CODENAME" = "xenial" ] || [ "$VERSION_CODENAME" = "trusty" ]; then - curl -o /tmp/teleport-pubkey.asc https://apt.releases.teleport.dev/gpg - sudo apt-key add /tmp/teleport-pubkey.asc - echo "deb https://apt.releases.teleport.dev/ubuntu ${VERSION_CODENAME?} {{ .RepoChannel }}" | sudo tee /etc/apt/sources.list.d/teleport.list - rm /tmp/teleport-pubkey.asc - else - sudo curl https://apt.releases.teleport.dev/gpg \ - -o /usr/share/keyrings/teleport-archive-keyring.asc - echo "deb [signed-by=/usr/share/keyrings/teleport-archive-keyring.asc] https://apt.releases.teleport.dev/${ID?} ${VERSION_CODENAME?} {{ .RepoChannel }}" | sudo tee /etc/apt/sources.list.d/teleport.list >/dev/null - fi - sudo apt-get update - - # shellcheck disable=SC2050 - if [ "{{ .AutomaticUpgrades }}" = "true" ]; then - # automatic upgrades - if ! target_version="$(get_target_version)"; then - # error getting the target version - sudo apt-get install -y "$TELEPORT_PACKAGE" jq "$TELEPORT_UPDATER_PACKAGE" - elif [ "$target_version" == "none" ]; then - # no target version advertised - sudo apt-get install -y "$TELEPORT_PACKAGE" jq "$TELEPORT_UPDATER_PACKAGE" - else - # successfully retrieved target version - sudo apt-get install -y "$TELEPORT_PACKAGE=$target_version" jq "$TELEPORT_UPDATER_PACKAGE=$target_version" - fi - else - # no automatic upgrades - sudo apt-get install -y "$TELEPORT_PACKAGE" jq - fi - - elif [ "$ID" = "amzn" ] || [ "$ID" = "rhel" ]; then - if [ "$ID" = "rhel" ]; then - VERSION_ID=${VERSION_ID//\.*/} # convert version numbers like '7.2' to only include the major version - fi - sudo yum install -y yum-utils - sudo yum-config-manager --add-repo \ - "$(rpm --eval "https://yum.releases.teleport.dev/$ID/$VERSION_ID/Teleport/%{_arch}/{{ .RepoChannel }}/teleport.repo")" - - # shellcheck disable=SC2050 - if [ "{{ .AutomaticUpgrades }}" = "true" ]; then - # automatic upgrades - if ! target_version="$(get_target_version)"; then - # error getting the target version - sudo yum install -y "$TELEPORT_PACKAGE" jq "$TELEPORT_UPDATER_PACKAGE" - elif [ "$target_version" == "none" ]; then - # no target version advertised - sudo yum install -y "$TELEPORT_PACKAGE" jq "$TELEPORT_UPDATER_PACKAGE" - else - # successfully retrieved target version - sudo yum install -y "$TELEPORT_PACKAGE-$target_version" jq "$TELEPORT_UPDATER_PACKAGE-$target_version" - fi - else - # no automatic upgrades - sudo yum install -y "$TELEPORT_PACKAGE" jq - fi - - elif [ "$ID" = "sles" ] || [ "$ID" = "opensuse-tumbleweed" ] || [ "$ID" = "opensuse-leap" ]; then - if [ "$ID" = "opensuse-tumbleweed" ]; then - VERSION_ID="15" # tumbleweed uses dated VERSION_IDs like 20230702 - else - VERSION_ID="${VERSION_ID//.*/}" # convert version numbers like '7.2' to only include the major version - fi - sudo rpm --import "https://zypper.releases.teleport.dev/gpg" - sudo zypper --non-interactive addrepo "$(rpm --eval "https://zypper.releases.teleport.dev/sles/$VERSION_ID/Teleport/%{_arch}/{{ .RepoChannel }}/teleport.repo")" - sudo zypper --gpg-auto-import-keys refresh - - # shellcheck disable=SC2050 - if [ "{{ .AutomaticUpgrades }}" = "true" ]; then - # automatic upgrades - if ! target_version="$(get_target_version)"; then - # error getting the target version - sudo zypper --non-interactive install -y "$TELEPORT_PACKAGE" jq "$TELEPORT_UPDATER_PACKAGE" - elif [ "$target_version" == "none" ]; then - # no target version advertised - sudo zypper --non-interactive install -y "$TELEPORT_PACKAGE" jq "$TELEPORT_UPDATER_PACKAGE" - else - # successfully retrieved target version - sudo zypper --non-interactive install -y "$TELEPORT_PACKAGE-$target_version" jq "$TELEPORT_UPDATER_PACKAGE-$target_version" - fi - else - # no automatic upgrades - sudo zypper --non-interactive install -y "$TELEPORT_PACKAGE" jq - fi - else - echo "Unsupported distro: $ID" - exit 1 - fi + + INSTALL_SCRIPT_URL="https://{{.PublicProxyAddr}}/scripts/install.sh" + echo "Offloading the installation part to the generic Teleport install script hosted at: $INSTALL_SCRIPT_URL" + + TEMP_INSTALLER_SCRIPT="$(mktemp)" + curl -sSf "$INSTALL_SCRIPT_URL" -o "$TEMP_INSTALLER_SCRIPT" + + chmod +x "$TEMP_INSTALLER_SCRIPT" + + sudo "$TEMP_INSTALLER_SCRIPT" || (echo "The install script ($TEMP_INSTALLER_SCRIPT) returned a non-zero exit code" && exit 1) + rm "$TEMP_INSTALLER_SCRIPT" if on_azure; then API_VERSION=$(curl -m5 -sS -H "Metadata: true" --noproxy "*" "http://169.254.169.254/metadata/versions" | jq -r ".apiVersions[-1]") diff --git a/api/types/installers/installers.go b/api/types/installers/installers.go index fcd6e024574dd..632a6788c9173 100644 --- a/api/types/installers/installers.go +++ b/api/types/installers/installers.go @@ -22,8 +22,11 @@ import ( "github.com/gravitational/teleport/api/types" ) +//go:embed legacy-installer.sh.tmpl +var legacyDefaultInstallScript string + //go:embed installer.sh.tmpl -var defaultInstallScript string +var newDefaultInstallScript string //go:embed agentless-installer.sh.tmpl var defaultAgentlessInstallScript string @@ -36,9 +39,11 @@ const InstallerScriptName = types.DefaultInstallerScriptName // installer script when agentless mode is enabled for a matcher const InstallerScriptNameAgentless = types.DefaultInstallerScriptNameAgentless -// DefaultInstaller represents a the default installer script provided -// by teleport -var DefaultInstaller = types.MustNewInstallerV1(InstallerScriptName, defaultInstallScript) +// LegacyDefaultInstaller represents the default installer script provided by teleport. +var LegacyDefaultInstaller = types.MustNewInstallerV1(InstallerScriptName, legacyDefaultInstallScript) + +// NewDefaultInstaller represents the default installer script provided by teleport. +var NewDefaultInstaller = types.MustNewInstallerV1(InstallerScriptName, newDefaultInstallScript) // DefaultAgentlessInstaller represents a the default agentless installer script provided // by teleport diff --git a/api/types/installers/legacy-installer.sh.tmpl b/api/types/installers/legacy-installer.sh.tmpl new file mode 100644 index 0000000000000..1209c6dc2abc0 --- /dev/null +++ b/api/types/installers/legacy-installer.sh.tmpl @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1083,SC2215,SC2288 # caused by Go templating, and shellcheck won't parse if the lines are excluded individually + +set -eu + +upgrade_endpoint="{{ .PublicProxyAddr }}/v1/webapi/automaticupgrades/channel/default" + +# upgrade_endpoint_fetch loads the specified value from the upgrade endpoint. the only +# currently supported values are 'version' and 'critical'. +upgrade_endpoint_fetch() { + host_path="${upgrade_endpoint}/${1}" + + if sf_output="$(curl --proto '=https' --tlsv1.2 -sSf "https://${host_path}")"; then + # emit output with empty lines and extra whitespace removed + echo "$sf_output" | grep -v -e '^[[:space:]]*$' | awk '{$1=$1};1' + return 0 + else + return 1 + fi +} + +# get_target_version loads the current value of the /version endpoint. +get_target_version() { + if tv_output="$(upgrade_endpoint_fetch version)"; then + # emit version string with leading 'v' removed if one is present + echo "${tv_output#v}" + return 0 + fi + return 1 +} + +on_ec2() { + IMDS_TOKEN=$(curl -m5 -sS -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 300") + [ -z "$IMDS_TOKEN" ] && return 1 + EC2_STATUS=$(curl -o /dev/null -w "%{http_code}" -m5 -sS -H "X-aws-ec2-metadata-token: ${IMDS_TOKEN}" "http://169.254.169.254/latest/meta-data") + [ "$EC2_STATUS" = "200" ] +} + +on_azure() { + AZURE_STATUS=$(curl -o /dev/null -w "%{http_code}" -m5 -sS -H "Metadata: true" --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=2021-02-01") + [ "$AZURE_STATUS" = "200" ] +} + +on_gcp() { + GCP_STATUS=$(curl -o /dev/null -w "%{http_code}" -m5 -sS -H "Metadata-Flavor: Google" "http://metadata.google.internal/") + [ "$GCP_STATUS" = "200" ] +} + +( + flock -n 9 || exit 1 + if test -f /usr/local/bin/teleport; then + exit 0 + fi + # shellcheck disable=SC1091 + . /etc/os-release + + TELEPORT_PACKAGE="{{ .TeleportPackage }}" + TELEPORT_UPDATER_PACKAGE="{{ .TeleportPackage }}-updater" + + if [ "$ID" = "debian" ] || [ "$ID" = "ubuntu" ]; then + # old versions of ubuntu require that keys get added by `apt-key add`, without + # adding the key apt shows a key signing error when installing teleport. + if [ "$VERSION_CODENAME" = "xenial" ] || [ "$VERSION_CODENAME" = "trusty" ]; then + curl -o /tmp/teleport-pubkey.asc https://apt.releases.teleport.dev/gpg + sudo apt-key add /tmp/teleport-pubkey.asc + echo "deb https://apt.releases.teleport.dev/ubuntu ${VERSION_CODENAME?} {{ .RepoChannel }}" | sudo tee /etc/apt/sources.list.d/teleport.list + rm /tmp/teleport-pubkey.asc + else + sudo curl https://apt.releases.teleport.dev/gpg \ + -o /usr/share/keyrings/teleport-archive-keyring.asc + echo "deb [signed-by=/usr/share/keyrings/teleport-archive-keyring.asc] https://apt.releases.teleport.dev/${ID?} ${VERSION_CODENAME?} {{ .RepoChannel }}" | sudo tee /etc/apt/sources.list.d/teleport.list >/dev/null + fi + sudo apt-get update + + # shellcheck disable=SC2050 + if [ "{{ .AutomaticUpgrades }}" = "true" ]; then + # automatic upgrades + if ! target_version="$(get_target_version)"; then + # error getting the target version + sudo apt-get install -y "$TELEPORT_PACKAGE" jq "$TELEPORT_UPDATER_PACKAGE" + elif [ "$target_version" == "none" ]; then + # no target version advertised + sudo apt-get install -y "$TELEPORT_PACKAGE" jq "$TELEPORT_UPDATER_PACKAGE" + else + # successfully retrieved target version + sudo apt-get install -y "$TELEPORT_PACKAGE=$target_version" jq "$TELEPORT_UPDATER_PACKAGE=$target_version" + fi + else + # no automatic upgrades + sudo apt-get install -y "$TELEPORT_PACKAGE" jq + fi + + elif [ "$ID" = "amzn" ] || [ "$ID" = "rhel" ]; then + if [ "$ID" = "rhel" ]; then + VERSION_ID=${VERSION_ID//\.*/} # convert version numbers like '7.2' to only include the major version + fi + sudo yum install -y yum-utils + sudo yum-config-manager --add-repo \ + "$(rpm --eval "https://yum.releases.teleport.dev/$ID/$VERSION_ID/Teleport/%{_arch}/{{ .RepoChannel }}/teleport.repo")" + + # shellcheck disable=SC2050 + if [ "{{ .AutomaticUpgrades }}" = "true" ]; then + # automatic upgrades + if ! target_version="$(get_target_version)"; then + # error getting the target version + sudo yum install -y "$TELEPORT_PACKAGE" jq "$TELEPORT_UPDATER_PACKAGE" + elif [ "$target_version" == "none" ]; then + # no target version advertised + sudo yum install -y "$TELEPORT_PACKAGE" jq "$TELEPORT_UPDATER_PACKAGE" + else + # successfully retrieved target version + sudo yum install -y "$TELEPORT_PACKAGE-$target_version" jq "$TELEPORT_UPDATER_PACKAGE-$target_version" + fi + else + # no automatic upgrades + sudo yum install -y "$TELEPORT_PACKAGE" jq + fi + + elif [ "$ID" = "sles" ] || [ "$ID" = "opensuse-tumbleweed" ] || [ "$ID" = "opensuse-leap" ]; then + if [ "$ID" = "opensuse-tumbleweed" ]; then + VERSION_ID="15" # tumbleweed uses dated VERSION_IDs like 20230702 + else + VERSION_ID="${VERSION_ID//.*/}" # convert version numbers like '7.2' to only include the major version + fi + sudo rpm --import "https://zypper.releases.teleport.dev/gpg" + sudo zypper --non-interactive addrepo "$(rpm --eval "https://zypper.releases.teleport.dev/sles/$VERSION_ID/Teleport/%{_arch}/{{ .RepoChannel }}/teleport.repo")" + sudo zypper --gpg-auto-import-keys refresh + + # shellcheck disable=SC2050 + if [ "{{ .AutomaticUpgrades }}" = "true" ]; then + # automatic upgrades + if ! target_version="$(get_target_version)"; then + # error getting the target version + sudo zypper --non-interactive install -y "$TELEPORT_PACKAGE" jq "$TELEPORT_UPDATER_PACKAGE" + elif [ "$target_version" == "none" ]; then + # no target version advertised + sudo zypper --non-interactive install -y "$TELEPORT_PACKAGE" jq "$TELEPORT_UPDATER_PACKAGE" + else + # successfully retrieved target version + sudo zypper --non-interactive install -y "$TELEPORT_PACKAGE-$target_version" jq "$TELEPORT_UPDATER_PACKAGE-$target_version" + fi + else + # no automatic upgrades + sudo zypper --non-interactive install -y "$TELEPORT_PACKAGE" jq + fi + else + echo "Unsupported distro: $ID" + exit 1 + fi + + if on_azure; then + API_VERSION=$(curl -m5 -sS -H "Metadata: true" --noproxy "*" "http://169.254.169.254/metadata/versions" | jq -r ".apiVersions[-1]") + INSTANCE_INFO=$(curl -m5 -sS -H "Metadata: true" --noproxy "*" "http://169.254.169.254/metadata/instance?api-version=$API_VERSION&format=json") + + REGION="$(echo "$INSTANCE_INFO" | jq -r .compute.location)" + RESOURCE_GROUP="$(echo "$INSTANCE_INFO" | jq -r .compute.resourceGroupName)" + SUBSCRIPTION_ID="$(echo "$INSTANCE_INFO" | jq -r .compute.subscriptionId)" + VM_ID="$(echo "$INSTANCE_INFO" | jq -r .compute.vmId)" + + JOIN_METHOD=azure + LABELS="teleport.internal/vm-id=${VM_ID},teleport.internal/subscription-id=${SUBSCRIPTION_ID},teleport.internal/region=${REGION},teleport.internal/resource-group=${RESOURCE_GROUP}" + elif on_ec2; then + IMDS_TOKEN=$(curl -m5 -sS -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 300") + INSTANCE_INFO=$(curl -m5 -sS -H "X-aws-ec2-metadata-token: ${IMDS_TOKEN}" "http://169.254.169.254/latest/dynamic/instance-identity/document") + + ACCOUNT_ID="$(echo "$INSTANCE_INFO" | jq -r .accountId)" + INSTANCE_ID="$(echo "$INSTANCE_INFO" | jq -r .instanceId)" + + JOIN_METHOD=iam + LABELS="teleport.dev/instance-id=${INSTANCE_ID},teleport.dev/account-id=${ACCOUNT_ID}" + elif on_gcp; then + NAME="$(curl -m5 -sS -H "Metadata-Flavor:Google" "http://metadata.google.internal/computeMetadata/v1/instance/name")" + # GCP metadata returns fully qualified zone ("projects//zones/"), so we need to parse the zone name. + FULL_ZONE="$(curl -m5 -sS -H "Metadata-Flavor:Google" "http://metadata.google.internal/computeMetadata/v1/instance/zone")" + ZONE="$(basename $FULL_ZONE)" + PROJECT_ID=$(curl -m5 -sS -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/project/project-id") + + JOIN_METHOD=gcp + LABELS="teleport.internal/name=${NAME},teleport.internal/zone=${ZONE},teleport.internal/project-id=${PROJECT_ID}" + else + echo "Could not determine cloud provider" + exit 1 + fi + + # generate teleport ssh config + # token is read as a parameter from the AWS ssm script run and + # passed as the first argument to the script + sudo /usr/local/bin/teleport node configure \ + --proxy="{{ .PublicProxyAddr }}" \ + --join-method=${JOIN_METHOD} \ + {{- if .AzureClientID }} + --azure-client-id="{{ .AzureClientID }}" \ + {{ end -}} + --token="$1" \ + --output=file \ + --labels="${LABELS}" + + # enable and start teleport service + sudo systemctl enable --now teleport + +) 9>/var/lock/teleport_install.lock diff --git a/api/types/maintenance.go b/api/types/maintenance.go index 9cab6a9ad4765..507cf71f6a53e 100644 --- a/api/types/maintenance.go +++ b/api/types/maintenance.go @@ -26,13 +26,17 @@ import ( ) const ( - // UpgraderKindKuberController is a short name used to identify the kube-controller-based + // UpgraderKindKubeController is a short name used to identify the kube-controller-based // external upgrader variant. UpgraderKindKubeController = "kube" // UpgraderKindSystemdUnit is a short name used to identify the systemd-unit-based // external upgrader variant. UpgraderKindSystemdUnit = "unit" + + // UpgraderKindTeleportUpdate is a short name used to identify the teleport-update + // external upgrader variant. + UpgraderKindTeleportUpdate = "binary" ) var validWeekdays = [7]time.Weekday{ @@ -45,10 +49,10 @@ var validWeekdays = [7]time.Weekday{ time.Saturday, } -// parseWeekday attempts to interpret a string as a time.Weekday. In the interest of flexibility, +// ParseWeekday attempts to interpret a string as a time.Weekday. In the interest of flexibility, // parsing is case-insensitive and supports the common three-letter shorthand accepted by many // common scheduling utilites (e.g. contab, systemd timers). -func parseWeekday(s string) (day time.Weekday, ok bool) { +func ParseWeekday(s string) (day time.Weekday, ok bool) { for _, w := range validWeekdays { if strings.EqualFold(w.String(), s) || strings.EqualFold(w.String()[:3], s) { return w, true @@ -58,6 +62,42 @@ func parseWeekday(s string) (day time.Weekday, ok bool) { return time.Sunday, false } +// ParseWeekdays attempts to parse a slice of strings representing week days. +// The slice must not be empty but can also contain a single value "*", representing the whole week. +// Day order doesn't matter but the same week day must not be present multiple times. +// In the interest of flexibility, parsing is case-insensitive and supports the common three-letter shorthand +// accepted by many common scheduling utilites (e.g. contab, systemd timers). +func ParseWeekdays(days []string) (map[time.Weekday]struct{}, error) { + if len(days) == 0 { + return nil, trace.BadParameter("empty weekdays list") + } + // Special case, we support wildcards. + if len(days) == 1 && days[0] == Wildcard { + return map[time.Weekday]struct{}{ + time.Monday: {}, + time.Tuesday: {}, + time.Wednesday: {}, + time.Thursday: {}, + time.Friday: {}, + time.Saturday: {}, + time.Sunday: {}, + }, nil + } + weekdays := make(map[time.Weekday]struct{}, 7) + for _, day := range days { + weekday, ok := ParseWeekday(day) + if !ok { + return nil, trace.BadParameter("failed to parse weekday: %v", day) + } + // Check if this is a duplicate + if _, ok := weekdays[weekday]; ok { + return nil, trace.BadParameter("duplicate weekday: %v", weekday.String()) + } + weekdays[weekday] = struct{}{} + } + return weekdays, nil +} + // generator builds a closure that iterates valid maintenance config from the current day onward. Used in // schedule export logic and tests. func (w *AgentUpgradeWindow) generator(from time.Time) func() (start time.Time, end time.Time) { @@ -75,7 +115,7 @@ func (w *AgentUpgradeWindow) generator(from time.Time) func() (start time.Time, var weekdays []time.Weekday for _, d := range w.Weekdays { - if p, ok := parseWeekday(d); ok { + if p, ok := ParseWeekday(d); ok { weekdays = append(weekdays, p) } } @@ -203,7 +243,7 @@ func (m *ClusterMaintenanceConfigV1) CheckAndSetDefaults() error { } for _, day := range m.Spec.AgentUpgrades.Weekdays { - if _, ok := parseWeekday(day); !ok { + if _, ok := ParseWeekday(day); !ok { return trace.BadParameter("invalid weekday in agent upgrade window: %q", day) } } @@ -248,13 +288,14 @@ func (m *ClusterMaintenanceConfigV1) WithinUpgradeWindow(t time.Time) bool { } } - weekday := t.Weekday().String() - for _, upgradeWeekday := range upgradeWindow.Weekdays { - if weekday == upgradeWeekday { - if int(upgradeWindow.UTCStartHour) == t.Hour() { - return true - } - } + upgradeWeekDays, err := ParseWeekdays(upgradeWindow.Weekdays) + if err != nil { + return false } - return false + + if _, ok := upgradeWeekDays[t.Weekday()]; !ok { + return false + } + + return int(upgradeWindow.UTCStartHour) == t.Hour() } diff --git a/api/types/maintenance_test.go b/api/types/maintenance_test.go index 203006a8dee37..02a14014e76ba 100644 --- a/api/types/maintenance_test.go +++ b/api/types/maintenance_test.go @@ -205,7 +205,7 @@ func TestWeekdayParser(t *testing.T) { } for _, tt := range tts { - day, ok := parseWeekday(tt.input) + day, ok := ParseWeekday(tt.input) if tt.fail { require.False(t, ok) continue @@ -244,7 +244,7 @@ func TestWithinUpgradeWindow(t *testing.T) { desc: "within upgrade window weekday", upgradeWindow: AgentUpgradeWindow{ UTCStartHour: 8, - Weekdays: []string{"Monday"}, + Weekdays: []string{"Mon"}, }, date: "Mon, 02 Jan 2006 08:04:05 UTC", withinWindow: true, @@ -253,7 +253,7 @@ func TestWithinUpgradeWindow(t *testing.T) { desc: "not within upgrade window weekday", upgradeWindow: AgentUpgradeWindow{ UTCStartHour: 8, - Weekdays: []string{"Tuesday"}, + Weekdays: []string{"Tue"}, }, date: "Mon, 02 Jan 2006 08:04:05 UTC", withinWindow: false, @@ -271,3 +271,83 @@ func TestWithinUpgradeWindow(t *testing.T) { }) } } + +func TestParseWeekdays(t *testing.T) { + t.Parallel() + tests := []struct { + name string + input []string + expect map[time.Weekday]struct{} + expectError require.ErrorAssertionFunc + }{ + { + name: "Nil slice", + input: nil, + expect: nil, + expectError: require.Error, + }, + { + name: "Empty slice", + input: []string{}, + expect: nil, + expectError: require.Error, + }, + { + name: "Few valid days", + input: []string{"Mon", "Tuesday", "WEDNESDAY"}, + expect: map[time.Weekday]struct{}{ + time.Monday: {}, + time.Tuesday: {}, + time.Wednesday: {}, + }, + expectError: require.NoError, + }, + { + name: "Every day", + input: []string{"Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"}, + expect: map[time.Weekday]struct{}{ + time.Monday: {}, + time.Tuesday: {}, + time.Wednesday: {}, + time.Thursday: {}, + time.Friday: {}, + time.Saturday: {}, + time.Sunday: {}, + }, + expectError: require.NoError, + }, + { + name: "Wildcard", + input: []string{"*"}, + expect: map[time.Weekday]struct{}{ + time.Monday: {}, + time.Tuesday: {}, + time.Wednesday: {}, + time.Thursday: {}, + time.Friday: {}, + time.Saturday: {}, + time.Sunday: {}, + }, + expectError: require.NoError, + }, + { + name: "Duplicated day", + input: []string{"Mon", "Monday"}, + expect: nil, + expectError: require.Error, + }, + { + name: "Invalid days", + input: []string{"Mon", "Tuesday", "frurfday"}, + expect: nil, + expectError: require.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := ParseWeekdays(tt.input) + tt.expectError(t, err) + require.Equal(t, tt.expect, result) + }) + } +} diff --git a/api/types/resource_153.go b/api/types/resource_153.go index 8c79fcf2b89c0..ea5ac04a88186 100644 --- a/api/types/resource_153.go +++ b/api/types/resource_153.go @@ -18,6 +18,8 @@ import ( "encoding/json" "time" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" headerv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/header/v1" @@ -119,6 +121,10 @@ func (r *legacyToResource153Adapter) GetVersion() string { // Resource153ToLegacy transforms an RFD 153 style resource into a legacy // [Resource] type. // +// Resources153 implemented by proto-generated structs should use ProtoResource153ToLegacy +// instead as it will ensure the protobuf message is properly marshaled to JSON +// with protojson. +// // Note that CheckAndSetDefaults is a noop for the returned resource and // SetSubKind is not implemented and panics on use. func Resource153ToLegacy(r Resource153) Resource { @@ -233,3 +239,36 @@ func (r *resource153ToLegacyAdapter) SetRevision(rev string) { func (r *resource153ToLegacyAdapter) SetSubKind(subKind string) { panic("interface Resource153 does not implement SetSubKind") } + +// ProtoResource153 is a Resource153 implemented by a protobuf-generated struct. +type ProtoResource153 interface { + Resource153 + proto.Message +} + +type protoResource153ToLegacyAdapter struct { + inner ProtoResource153 + resource153ToLegacyAdapter +} + +// MarshalJSON adds support for marshaling the wrapped resource (instead of +// marshaling the adapter itself). +func (r *protoResource153ToLegacyAdapter) MarshalJSON() ([]byte, error) { + return protojson.MarshalOptions{ + UseProtoNames: true, + }.Marshal(r.inner) +} + +// ProtoResource153ToLegacy transforms an RFD 153 style resource implemented by +// a proto-generated struct into a legacy [Resource] type. Implements +// [ResourceWithLabels] and CloneResource (where the wrapped resource supports +// cloning). +// +// Note that CheckAndSetDefaults is a noop for the returned resource and +// SetSubKind is not implemented and panics on use. +func ProtoResource153ToLegacy(r ProtoResource153) Resource { + return &protoResource153ToLegacyAdapter{ + r, + resource153ToLegacyAdapter{r}, + } +} diff --git a/assets/aws/files/install-hardened.sh b/assets/aws/files/install-hardened.sh index 85ba8791d4773..fae3fb0058942 100644 --- a/assets/aws/files/install-hardened.sh +++ b/assets/aws/files/install-hardened.sh @@ -3,12 +3,12 @@ # Update packages dnf -y update -# Install +# Install # - uuid used for random token generation, # - python for certbot dnf install -y uuid python3 -# Install certbot +# Install certbot python3 -m venv /opt/certbot /opt/certbot/bin/pip install --upgrade pip /opt/certbot/bin/pip install certbot certbot-dns-route53 @@ -23,11 +23,16 @@ usermod -a -G adm teleport # Setup teleport run dir for pid files install -d -m 0700 -o teleport -g adm /var/lib/teleport install -d -m 0755 -o teleport -g adm /run/teleport /etc/teleport.d +# Setup teleport/system directory +install -d -m 0755 -o teleport -g adm /opt/teleport/system/bin +install -d -m 0755 -o teleport -g adm /opt/teleport/system/lib/systemd/system # Extract tarball to /tmp/teleport to get the binaries out mkdir /tmp/teleport tar -C /tmp/teleport -x -z -f /tmp/teleport.tar.gz --strip-components=1 -install -m 755 /tmp/teleport/{tctl,tsh,teleport,tbot} /usr/local/bin +install -m 755 /tmp/teleport/{tctl,tsh,teleport,tbot,teleport-update} /opt/teleport/system/bin +install -m 755 /tmp/teleport/examples/systemd/teleport.service /opt/teleport/system/lib/systemd/system +/opt/teleport/system/bin/teleport-update link-package rm -rf /tmp/teleport /tmp/teleport.tar.gz if [[ "${TELEPORT_FIPS}" == 1 ]]; then diff --git a/assets/install-scripts/install.sh b/assets/install-scripts/install.sh deleted file mode 100755 index ecc3179f659cf..0000000000000 --- a/assets/install-scripts/install.sh +++ /dev/null @@ -1,422 +0,0 @@ -#!/bin/bash -# Copyright 2022 Gravitational, Inc - -# This script detects the current Linux distribution and installs Teleport -# through its package manager, if supported, or downloading a tarball otherwise. -# We'll download Teleport from the official website and checksum it to make sure it was properly -# downloaded before executing. - -# The script is wrapped inside a function to protect against the connection being interrupted -# in the middle of the stream. - -# For more download options, head to https://goteleport.com/download/ - -set -euo pipefail - -# download uses curl or wget to download a teleport binary -download() { - URL=$1 - TMP_PATH=$2 - - echo "Downloading $URL" - if type curl &>/dev/null; then - set -x - # shellcheck disable=SC2086 - $SUDO $CURL -o "$TMP_PATH" "$URL" - else - set -x - # shellcheck disable=SC2086 - $SUDO $CURL -O "$TMP_PATH" "$URL" - fi - set +x -} - -install_via_apt_get() { - echo "Installing Teleport v$TELEPORT_VERSION via apt-get" - add_apt_key - set -x - $SUDO apt-get install -y "teleport$TELEPORT_SUFFIX=$TELEPORT_VERSION" - set +x - if [ "$TELEPORT_EDITION" = "cloud" ]; then - set -x - $SUDO apt-get install -y teleport-ent-updater - set +x - fi -} - -add_apt_key() { - APT_REPO_ID=$ID - APT_REPO_VERSION_CODENAME=$VERSION_CODENAME - IS_LEGACY=0 - - # check if we must use legacy .asc key - case "$ID" in - ubuntu | pop | neon | zorin) - if ! expr "$VERSION_ID" : "2.*" >/dev/null; then - IS_LEGACY=1 - fi - ;; - debian | raspbian) - if [ "$VERSION_ID" -lt 11 ]; then - IS_LEGACY=1 - fi - ;; - linuxmint | parrot) - if [ "$VERSION_ID" -lt 5 ]; then - IS_LEGACY=1 - fi - ;; - elementary) - if [ "$VERSION_ID" -lt 6 ]; then - IS_LEGACY=1 - fi - ;; - kali) - YEAR="$(echo "$VERSION_ID" | cut -f1 -d.)" - if [ "$YEAR" -lt 2021 ]; then - IS_LEGACY=1 - fi - ;; - esac - - if [[ "$IS_LEGACY" == 0 ]]; then - # set APT_REPO_ID if necessary - case "$ID" in - linuxmint | kali | elementary | pop | raspbian | neon | zorin | parrot) - APT_REPO_ID=$ID_LIKE - ;; - esac - - # set APT_REPO_VERSION_CODENAME if necessary - case "$ID" in - linuxmint | elementary | pop | neon | zorin) - APT_REPO_VERSION_CODENAME=$UBUNTU_CODENAME - ;; - kali) - APT_REPO_VERSION_CODENAME="bullseye" - ;; - parrot) - APT_REPO_VERSION_CODENAME="buster" - ;; - esac - fi - - echo "Downloading Teleport's PGP public key..." - TEMP_DIR=$(mktemp -d -t teleport-XXXXXXXXXX) - MAJOR=$(echo "$TELEPORT_VERSION" | cut -f1 -d.) - TELEPORT_REPO="" - - CHANNEL="stable/v${MAJOR}" - if [ "$TELEPORT_EDITION" = "cloud" ]; then - CHANNEL="stable/cloud" - fi - - if [[ "$IS_LEGACY" == 1 ]]; then - if ! type gpg >/dev/null; then - echo "Installing gnupg" - set -x - $SUDO apt-get update - $SUDO apt-get install -y gnupg - set +x - fi - TMP_KEY="$TEMP_DIR/teleport-pubkey.asc" - download "https://deb.releases.teleport.dev/teleport-pubkey.asc" "$TMP_KEY" - set -x - $SUDO apt-key add "$TMP_KEY" - set +x - TELEPORT_REPO="deb https://apt.releases.teleport.dev/${APT_REPO_ID?} ${APT_REPO_VERSION_CODENAME?} ${CHANNEL}" - else - TMP_KEY="$TEMP_DIR/teleport-pubkey.gpg" - download "https://apt.releases.teleport.dev/gpg" "$TMP_KEY" - set -x - $SUDO cp "$TMP_KEY" /usr/share/keyrings/teleport-archive-keyring.asc - set +x - TELEPORT_REPO="deb [signed-by=/usr/share/keyrings/teleport-archive-keyring.asc] https://apt.releases.teleport.dev/${APT_REPO_ID?} ${APT_REPO_VERSION_CODENAME?} ${CHANNEL}" - fi - - set -x - echo "$TELEPORT_REPO" | $SUDO tee /etc/apt/sources.list.d/teleport.list >/dev/null - set +x - - set -x - $SUDO apt-get update - set +x -} - -# $1 is the value of the $ID path segment in the YUM repo URL. In -# /etc/os-release, this is either the value of $ID or $ID_LIKE. -install_via_yum() { - # shellcheck source=/dev/null - source /etc/os-release - - # Get the major version from the version ID. - VERSION_ID=$(echo "$VERSION_ID" | grep -Eo "^[0-9]+") - TELEPORT_MAJOR_VERSION="v$(echo "$TELEPORT_VERSION" | grep -Eo "^[0-9]+")" - - CHANNEL="stable/${TELEPORT_MAJOR_VERSION}" - if [ "$TELEPORT_EDITION" = "cloud" ]; then - CHANNEL="stable/cloud" - fi - - if type dnf &>/dev/null; then - echo "Installing Teleport v$TELEPORT_VERSION through dnf" - $SUDO dnf install -y 'dnf-command(config-manager)' - $SUDO dnf config-manager --add-repo "$(rpm --eval "https://yum.releases.teleport.dev/$1/$VERSION_ID/Teleport/%{_arch}/$CHANNEL/teleport-yum.repo")" - $SUDO dnf install -y "teleport$TELEPORT_SUFFIX-$TELEPORT_VERSION" - - if [ "$TELEPORT_EDITION" = "cloud" ]; then - $SUDO dnf install -y teleport-ent-updater - fi - - else - echo "Installing Teleport v$TELEPORT_VERSION through yum" - $SUDO yum install -y yum-utils - $SUDO yum-config-manager --add-repo "$(rpm --eval "https://yum.releases.teleport.dev/$1/$VERSION_ID/Teleport/%{_arch}/$CHANNEL/teleport-yum.repo")" - $SUDO yum install -y "teleport$TELEPORT_SUFFIX-$TELEPORT_VERSION" - - if [ "$TELEPORT_EDITION" = "cloud" ]; then - $SUDO yum install -y teleport-ent-updater - fi - fi - set +x -} - -install_via_zypper() { - # shellcheck source=/dev/null - source /etc/os-release - - # Get the major version from the version ID. - VERSION_ID=$(echo "$VERSION_ID" | grep -Eo "^[0-9]+") - TELEPORT_MAJOR_VERSION="v$(echo "$TELEPORT_VERSION" | grep -Eo "^[0-9]+")" - - CHANNEL="stable/${TELEPORT_MAJOR_VERSION}" - if [ "$TELEPORT_EDITION" = "cloud" ]; then - CHANNEL="stable/cloud" - fi - - $SUDO rpm --import https://zypper.releases.teleport.dev/gpg - $SUDO zypper addrepo --refresh --repo "$(rpm --eval "https://zypper.releases.teleport.dev/$ID/$VERSION_ID/Teleport/%{_arch}/$CHANNEL/teleport-zypper.repo")" - $SUDO zypper --gpg-auto-import-keys refresh teleport - $SUDO zypper install -y "teleport$TELEPORT_SUFFIX" - - if [ "$TELEPORT_EDITION" = "cloud" ]; then - $SUDO zypper install -y teleport-ent-updater - fi - - set +x -} - - -# download .tar.gz file via curl/wget, unzip it and run the install sript -install_via_curl() { - TEMP_DIR=$(mktemp -d -t teleport-XXXXXXXXXX) - - TELEPORT_FILENAME="teleport$TELEPORT_SUFFIX-v$TELEPORT_VERSION-linux-$ARCH-bin.tar.gz" - URL="https://cdn.teleport.dev/${TELEPORT_FILENAME}" - download "${URL}" "${TEMP_DIR}/${TELEPORT_FILENAME}" - - TMP_CHECKSUM="${TEMP_DIR}/${TELEPORT_FILENAME}.sha256" - download "${URL}.sha256" "$TMP_CHECKSUM" - - set -x - cd "$TEMP_DIR" - # shellcheck disable=SC2086 - $SUDO $SHA_COMMAND -c "$TMP_CHECKSUM" - cd - - - $SUDO tar -xzf "${TEMP_DIR}/${TELEPORT_FILENAME}" -C "$TEMP_DIR" - $SUDO "$TEMP_DIR/teleport/install" - set +x -} - -# wrap script in a function so a partially downloaded script -# doesn't execute -install_teleport() { - # exit if not on Linux - if [[ $(uname) != "Linux" ]]; then - echo "ERROR: This script works only for Linux. Please go to the downloads page to find the proper installation method for your operating system:" - echo "https://goteleport.com/download/" - exit 1 - fi - - KERNEL_VERSION=$(uname -r) - MIN_VERSION="2.6.23" - if [ $MIN_VERSION != "$(echo -e "$MIN_VERSION\n$KERNEL_VERSION" | sort -V | head -n1)" ]; then - echo "ERROR: Teleport requires Linux kernel version $MIN_VERSION+" - exit 1 - fi - - # check if can run as admin either by running as root or by - # having 'sudo' or 'doas' installed - IS_ROOT="" - SUDO="" - if [ "$(id -u)" = 0 ]; then - # running as root, no need for sudo/doas - IS_ROOT="YES" - SUDO="" - elif type sudo &>/dev/null; then - SUDO="sudo" - elif type doas &>/dev/null; then - SUDO="doas" - fi - - if [ -z "$SUDO" ] && [ -z "$IS_ROOT" ]; then - echo "ERROR: The installer requires a way to run commands as root." - echo "Either run this script as root or install sudo/doas." - exit 1 - fi - - # require curl/wget - CURL="" - if type curl &>/dev/null; then - CURL="curl -fL" - elif type wget &>/dev/null; then - CURL="wget" - fi - if [ -z "$CURL" ]; then - echo "ERROR: This script requires either curl or wget in order to download files. Please install one of them and try again." - exit 1 - fi - - # require shasum/sha256sum - SHA_COMMAND="" - if type shasum &>/dev/null; then - SHA_COMMAND="shasum -a 256" - elif type sha256sum &>/dev/null; then - SHA_COMMAND="sha256sum" - else - echo "ERROR: This script requires sha256sum or shasum to validate the download. Please install it and try again." - exit 1 - fi - - # detect distro - OS_RELEASE=/etc/os-release - ID="" - ID_LIKE="" - VERSION_CODENAME="" - UBUNTU_CODENAME="" - if [[ -f "$OS_RELEASE" ]]; then - # shellcheck source=/dev/null - . $OS_RELEASE - fi - - # detect architecture - ARCH="" - case $(uname -m) in - x86_64) - ARCH="amd64" - ;; - i386) - ARCH="386" - ;; - armv7l) - ARCH="arm" - ;; - aarch64) - ARCH="arm64" - ;; - **) - echo "ERROR: Your system's architecture isn't officially supported or couldn't be determined." - echo "Please refer to the installation guide for more information:" - echo "https://goteleport.com/docs/installation/" - exit 1 - ;; - esac - - # select install method based on distribution - # if ID is debian derivate, run apt-get - case "$ID" in - debian | ubuntu | kali | linuxmint | pop | raspbian | neon | zorin | parrot | elementary) - install_via_apt_get - ;; - # if ID is amazon Linux 2/RHEL/etc, run yum - centos | rhel | amzn) - install_via_yum "$ID" - ;; - sles) - install_via_zypper - ;; - *) - # before downloading manually, double check if we didn't miss any debian or - # rh/fedora derived distros using the ID_LIKE var. Some $ID_LIKE values - # include multiple distro names in an arbitrary order, so evaluate the first - # one. - case "$(echo "$ID_LIKE" | awk '{print $1}')" in - ubuntu | debian) - install_via_apt_get - ;; - centos | fedora | rhel) - # There is no repository for "fedora", and there is no difference - # between the repositories for "centos" and "rhel", so pick an arbitrary - # one. - install_via_yum rhel - ;; - *) - if [ "$TELEPORT_EDITION" = "cloud" ]; then - echo "The system does not support a package manager, which is required for Teleport Enterprise Cloud." - exit 1 - fi - - # if ID and ID_LIKE didn't return a supported distro, download through curl - echo "There is no officially supported package for your package manager. Downloading and installing Teleport via curl." - install_via_curl - ;; - esac - ;; - esac - - GREEN='\033[0;32m' - COLOR_OFF='\033[0m' - - echo "" - echo -e "${GREEN}$(teleport version) installed successfully!${COLOR_OFF}" - echo "" - echo "The following commands are now available:" - if type teleport &>/dev/null; then - echo " teleport - The daemon that runs the Auth Service, Proxy Service, and other Teleport services." - fi - if type tsh &>/dev/null; then - echo " tsh - A tool that lets end users interact with Teleport." - fi - if type tctl &>/dev/null; then - echo " tctl - An administrative tool that can configure the Teleport Auth Service." - fi - if type tbot &>/dev/null; then - echo " tbot - Teleport Machine ID client." - fi -} - -# The suffix is "-ent" if we are installing a commercial edition of Teleport and -# empty for Teleport Community Edition. -TELEPORT_SUFFIX="" -TELEPORT_VERSION="" -TELEPORT_EDITION="" -if [ $# -ge 1 ] && [ -n "$1" ]; then - TELEPORT_VERSION=$1 -else - echo "ERROR: Please provide the version you want to install (e.g., 10.1.9)." - exit 1 -fi - -if ! echo "$1" | grep -qE "[0-9]+\.[0-9]+\.[0-9]+"; then - echo "ERROR: The first parameter must be a version number, e.g., 10.1.9." - exit 1 -fi - -if [ $# -ge 2 ] && [ -n "$2" ]; then - TELEPORT_EDITION=$2 - - case $TELEPORT_EDITION in - enterprise | cloud) - TELEPORT_SUFFIX="-ent" - ;; - # An empty edition defaults to OSS. - oss | "" ) - ;; - *) - echo 'ERROR: The second parameter must be "oss", "cloud", or "enterprise".' - exit 1 - ;; - esac -fi -install_teleport diff --git a/assets/install-scripts/install.sh b/assets/install-scripts/install.sh new file mode 120000 index 0000000000000..c41183ab5d100 --- /dev/null +++ b/assets/install-scripts/install.sh @@ -0,0 +1 @@ +../../lib/web/scripts/install/install.sh \ No newline at end of file diff --git a/build.assets/build-package.sh b/build.assets/build-package.sh index 770b067c17dd3..c27ad71d5c5ab 100755 --- a/build.assets/build-package.sh +++ b/build.assets/build-package.sh @@ -63,8 +63,8 @@ TARBALL_DIRECTORY="$s" GNUPG_DIR=${GNUPG_DIR:-/tmp/gnupg} # linux package configuration -LINUX_BINARY_DIR=/usr/local/bin -LINUX_SYSTEMD_DIR=/lib/systemd/system +LINUX_BINARY_DIR=/opt/teleport/system/bin +LINUX_SYSTEMD_DIR=/opt/teleport/system/lib/systemd/system LINUX_CONFIG_DIR=/etc LINUX_DATA_DIR=/var/lib/teleport @@ -183,6 +183,11 @@ if [[ "${RUNTIME}" == "fips" ]]; then OPTIONAL_RUNTIME_SECTION+="-fips" fi +# After install is --after-install except for RPM, we use --rpm-posttrans. +# This is because RPM runs after install scrips before the old package removal, +# so old Teleport are still here and we cannot run our teleport-update symlink logic. +AFTER_INSTALL_TARGET="--after-install" + # set variables appropriately depending on type of package being built if [[ "${TELEPORT_TYPE}" == "ent" ]]; then TARBALL_FILENAME="teleport-ent-v${TELEPORT_VERSION}-${PLATFORM}-${TARBALL_ARCH}${OPTIONAL_TARBALL_SECTION}${OPTIONAL_RUNTIME_SECTION}-bin.tar.gz" @@ -226,8 +231,8 @@ if [[ "${PACKAGE_TYPE}" == "pkg" ]]; then PKG_FILENAME="teleport-${TELEPORT_VERSION}${ARCH_TAG}.${PACKAGE_TYPE}" fi else - FILE_LIST="${TAR_PATH}/tsh ${TAR_PATH}/tctl ${TAR_PATH}/teleport ${TAR_PATH}/tbot ${TAR_PATH}/examples/systemd/teleport.service ${TAR_PATH}/examples/systemd/post-upgrade" - LINUX_BINARY_FILE_LIST="${TAR_PATH}/tsh ${TAR_PATH}/tctl ${TAR_PATH}/tbot ${TAR_PATH}/teleport" + FILE_LIST="${TAR_PATH}/tsh ${TAR_PATH}/tctl ${TAR_PATH}/teleport ${TAR_PATH}/tbot ${TAR_PATH}/teleport-update ${TAR_PATH}/examples/systemd/teleport.service ${TAR_PATH}/examples/systemd/post-install ${TAR_PATH}/examples/systemd/before-remove" + LINUX_BINARY_FILE_LIST="${TAR_PATH}/tsh ${TAR_PATH}/tctl ${TAR_PATH}/tbot ${TAR_PATH}/teleport ${TAR_PATH}/teleport-update" LINUX_SYSTEMD_FILE_LIST="${TAR_PATH}/examples/systemd/teleport.service" EXTRA_DOCKER_OPTIONS="" RPM_SIGN_STANZA="" @@ -237,6 +242,8 @@ else FILE_PERMISSIONS_STANZA="--rpm-user root --rpm-group root --rpm-use-file-permissions " # the rpm/rpmmacros file suppresses the creation of .build-id files (see https://github.com/gravitational/teleport/issues/7040) EXTRA_DOCKER_OPTIONS="-v $(pwd)/rpm/rpmmacros:/root/.rpmmacros" + + AFTER_INSTALL_TARGET="--rpm-posttrans" # if we set this environment variable, don't sign RPMs (can be useful for building test RPMs # without having the signing keys) if [ "${UNSIGNED_RPM}" == "true" ]; then @@ -291,8 +298,12 @@ if [[ "${PACKAGE_TYPE}" != "pkg" ]]; then CONFIG_FILE_STANZA="--config-files /src/buildroot${LINUX_CONFIG_DIR}/${LINUX_CONFIG_FILE} " fi - # include post-upgrade script - mv -v ${TAR_PATH}/examples/systemd/post-upgrade ${PACKAGE_TEMPDIR} + # include post-install and before-remove script + mv -v ${TAR_PATH}/examples/systemd/post-install ${PACKAGE_TEMPDIR} + mv -v ${TAR_PATH}/examples/systemd/before-remove ${PACKAGE_TEMPDIR} + + # create versions folder + mkdir -p ${PACKAGE_TEMPDIR}/buildroot${LINUX_DATA_DIR}/versions # /var/lib/teleport # shellcheck disable=SC2174 @@ -369,7 +380,8 @@ else --provides teleport \ --prefix / \ --verbose \ - --after-upgrade /src/post-upgrade \ + "$AFTER_INSTALL_TARGET" /src/post-install \ + --before-remove /src/before-remove \ ${CONFIG_FILE_STANZA} \ ${FILE_PERMISSIONS_STANZA} \ ${RPM_SIGN_STANZA} . diff --git a/build.assets/charts/Dockerfile-distroless b/build.assets/charts/Dockerfile-distroless index 0c3f30886750b..026470fce80c5 100644 --- a/build.assets/charts/Dockerfile-distroless +++ b/build.assets/charts/Dockerfile-distroless @@ -23,7 +23,10 @@ COPY $TELEPORT_DEB_FILE_NAME ./$TELEPORT_DEB_FILE_NAME RUN dpkg-deb -R $TELEPORT_DEB_FILE_NAME /opt/staging && \ mkdir -p /opt/staging/etc/teleport && \ mkdir -p /opt/staging/var/lib/dpkg/status.d/ && \ + mkdir -p /opt/staging/usr/local/bin && \ mv /opt/staging/DEBIAN/control /opt/staging/var/lib/dpkg/status.d/teleport && \ + mv /opt/staging/opt/teleport/system/bin/* /opt/staging/usr/local/bin/ && \ + rm -f /opt/staging/usr/local/bin/teleport-update && \ rm -rf /opt/staging/DEBIAN FROM $BASE_IMAGE diff --git a/build.assets/charts/Dockerfile-distroless-fips b/build.assets/charts/Dockerfile-distroless-fips index b6594eb266c70..30582d60a0b51 100644 --- a/build.assets/charts/Dockerfile-distroless-fips +++ b/build.assets/charts/Dockerfile-distroless-fips @@ -23,7 +23,10 @@ COPY $TELEPORT_DEB_FILE_NAME ./$TELEPORT_DEB_FILE_NAME RUN dpkg-deb -R $TELEPORT_DEB_FILE_NAME /opt/staging && \ mkdir -p /opt/staging/etc/teleport && \ mkdir -p /opt/staging/var/lib/dpkg/status.d/ && \ + mkdir -p /opt/staging/usr/local/bin && \ mv /opt/staging/DEBIAN/control /opt/staging/var/lib/dpkg/status.d/teleport && \ + mv /opt/staging/opt/teleport/system/bin/* /opt/staging/usr/local/bin/ && \ + rm -f /opt/staging/usr/local/bin/teleport-update && \ rm -rf /opt/staging/DEBIAN FROM $BASE_IMAGE diff --git a/build.assets/charts/Dockerfile-tbot-distroless b/build.assets/charts/Dockerfile-tbot-distroless index 9e1e4d8897c07..842157a175bbc 100644 --- a/build.assets/charts/Dockerfile-tbot-distroless +++ b/build.assets/charts/Dockerfile-tbot-distroless @@ -17,5 +17,5 @@ ENV TELEPORT_DEB_FILE_NAME=teleport${TELEPORT_RELEASE_INFIX}_${TELEPORT_VERSION} RUN --mount=type=bind,target=/ctx dpkg-deb -R /ctx/$TELEPORT_DEB_FILE_NAME /opt/staging FROM $BASE_IMAGE -COPY --from=teleport /opt/staging/usr/local/bin/tbot /usr/local/bin/tbot +COPY --from=teleport /opt/staging/opt/teleport/system/bin/tbot /usr/local/bin/tbot ENTRYPOINT ["/usr/local/bin/tbot"] diff --git a/build.assets/charts/Dockerfile-tbot-distroless-fips b/build.assets/charts/Dockerfile-tbot-distroless-fips index 7592a8993ec69..b6fb33caab877 100644 --- a/build.assets/charts/Dockerfile-tbot-distroless-fips +++ b/build.assets/charts/Dockerfile-tbot-distroless-fips @@ -17,5 +17,5 @@ ENV TELEPORT_DEB_FILE_NAME=teleport${TELEPORT_RELEASE_INFIX}_${TELEPORT_VERSION} RUN --mount=type=bind,target=/ctx dpkg-deb -R /ctx/$TELEPORT_DEB_FILE_NAME /opt/staging FROM $BASE_IMAGE -COPY --from=teleport /opt/staging/usr/local/bin/tbot /usr/local/bin/tbot +COPY --from=teleport /opt/staging/opt/teleport/system/bin/tbot /usr/local/bin/tbot ENTRYPOINT ["/usr/local/bin/tbot", "--fips"] diff --git a/build.assets/install b/build.assets/install index 691c627d192ce..eb52ec7d99959 100755 --- a/build.assets/install +++ b/build.assets/install @@ -14,7 +14,7 @@ VARDIR=/var/lib/teleport echo "Starting Teleport installation..." cd $(dirname $0) mkdir -p $VARDIR $BINDIR -cp -f teleport tctl tsh tbot $BINDIR/ || exit 1 +cp -f teleport tctl tsh tbot teleport-update $BINDIR/ || exit 1 # # What operating system is the user running? diff --git a/constants.go b/constants.go index ffaf2c598e7eb..fc867a201393e 100644 --- a/constants.go +++ b/constants.go @@ -274,7 +274,13 @@ const ( // ComponentAssist represents Teleport Assist ComponentAssist = "assist" - // VerboseLogEnvVar forces all logs to be verbose (down to DEBUG level) + // ComponentUpdater represents the teleport-update binary. + ComponentUpdater = "updater" + + // ComponentRolloutController represents the autoupdate_agent_rollout controller. + ComponentRolloutController = "rollout-controller" + + // VerboseLogsEnvVar forces all logs to be verbose (down to DEBUG level) VerboseLogsEnvVar = "TELEPORT_DEBUG" // IterationsEnvVar sets tests iterations to run diff --git a/docs/cspell.json b/docs/cspell.json index 3ebe87ae58c54..2ae254df2fe9c 100644 --- a/docs/cspell.json +++ b/docs/cspell.json @@ -283,6 +283,7 @@ "automations", "automount", "autoscale", + "autoupdate", "awly", "awsapp", "awsathena", diff --git a/e b/e index eaae8b4c02a06..e70cb5564864e 160000 --- a/e +++ b/e @@ -1 +1 @@ -Subproject commit eaae8b4c02a066e1d4d19e9cb410f2c4ec7e6ef9 +Subproject commit e70cb5564864e7f1186e78cb86e7d0c8577e01bb diff --git a/examples/chart/teleport-kube-agent/templates/updater/deployment.yaml b/examples/chart/teleport-kube-agent/templates/updater/deployment.yaml index 1528fa2f5b014..5790f16cd8dfc 100644 --- a/examples/chart/teleport-kube-agent/templates/updater/deployment.yaml +++ b/examples/chart/teleport-kube-agent/templates/updater/deployment.yaml @@ -1,5 +1,6 @@ {{- if .Values.updater.enabled -}} {{- $updater := mustMergeOverwrite (mustDeepCopy .Values) .Values.updater -}} +{{- $versionServerOverride := and $updater.versionServer (ne $updater.versionServer "https://{{ .Values.proxyAddr }}/v1/webapi/automaticupgrades/channel") }} apiVersion: apps/v1 kind: Deployment metadata: @@ -62,11 +63,22 @@ spec: - "--agent-name={{ .Release.Name }}" - "--agent-namespace={{ .Release.Namespace }}" - "--base-image={{ include "teleport-kube-agent.baseImage" . }}" + {{- if $updater.versionServer}} - "--version-server={{ tpl $updater.versionServer . }}" - "--version-channel={{ $updater.releaseChannel }}" - {{- if .Values.updater.extraArgs }} - {{- toYaml .Values.updater.extraArgs | nindent 10 }} - {{- end }} + {{- end }} + {{- /* We don't want to enable the RFD-184 update protocol if the user has set a custom versionServer as this + would be a breaking change when the teleport proxy starts override the explicitly set RFD-109 version server */ -}} + {{- if and $updater.proxyAddr (not $versionServerOverride)}} + - "--proxy-address={{ $updater.proxyAddr }}" + - "--update-group={{ default $updater.releaseChannel $updater.group }}" + {{- end }} +{{- if $updater.pullCredentials }} + - "--pull-credentials={{ $updater.pullCredentials }}" +{{- end }} +{{- if .Values.updater.extraArgs }} + {{- toYaml .Values.updater.extraArgs | nindent 10 }} +{{- end }} {{- if $updater.securityContext }} securityContext: {{- toYaml $updater.securityContext | nindent 10 }} {{- end }} diff --git a/examples/chart/teleport-kube-agent/tests/updater_deployment_test.yaml b/examples/chart/teleport-kube-agent/tests/updater_deployment_test.yaml index d577c87397b4f..830d4dcd2fcc3 100644 --- a/examples/chart/teleport-kube-agent/tests/updater_deployment_test.yaml +++ b/examples/chart/teleport-kube-agent/tests/updater_deployment_test.yaml @@ -67,6 +67,69 @@ tests: - contains: path: spec.template.spec.containers[0].args content: "--version-server=https://proxy.teleport.example.com:443/v1/webapi/automaticupgrades/channel" + - it: defaults the updater proxy server to the proxy address + set: + proxyAddr: proxy.teleport.example.com:443 + roles: "custom" + updater: + enabled: true + versionServer: "" + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: "--proxy-address=proxy.teleport.example.com:443" + - it: doesn't enable the RFD-184 proxy protocol if the versionServer is custom + set: + proxyAddr: proxy.teleport.example.com:443 + roles: "custom" + updater: + enabled: true + versionServer: "version-server.example.com" + group: foobar + asserts: + - notContains: + path: spec.template.spec.containers[0].args + content: "--proxy-address=proxy.teleport.example.com:443" + - notContains: + path: spec.template.spec.containers[0].args + content: "--update-group=foobar" + - it: defaults the update group to the release channel when group is unset + set: + proxyAddr: proxy.teleport.example.com:443 + roles: "custom" + updater: + enabled: true + versionServer: "" + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: "--update-group=stable/cloud" + - it: uses the update group when set + set: + proxyAddr: proxy.teleport.example.com:443 + roles: "custom" + updater: + enabled: true + versionServer: "" + group: "foobar" + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: "--update-group=foobar" + - it: unsets the version server when empty + set: + proxyAddr: proxy.teleport.example.com:443 + roles: "custom" + updater: + enabled: true + versionServer: "" + asserts: + - notContains: + path: spec.template.spec.containers[0].args + content: "--proxy-server=" + - notContains: + path: spec.template.spec.containers[0].args + content: "--version-channel=stable/cloud" - it: sets the updater version server values: - ../.lint/updater.yaml diff --git a/examples/chart/teleport-kube-agent/values.yaml b/examples/chart/teleport-kube-agent/values.yaml index c607fc33ddbc5..534e55e59fe2d 100644 --- a/examples/chart/teleport-kube-agent/values.yaml +++ b/examples/chart/teleport-kube-agent/values.yaml @@ -162,15 +162,75 @@ tls: # The filename inside the secret is important - it _must_ be ca.pem existingCASecretName: "" +# updater -- controls whether the Kube Agent Updater should be deployed alongside +# the `teleport-kube-agent`. The updater fetches the target version, validates the +# image signature, and updates the teleport deployment. +# +# The updater can fetch the update information using two protocols: +# - the webapi update protocol (in this case the Teleport Proxy Service is the one driving the version rollout) +# - the version server protocol (this is an HTTP server serving static files specifying the version and if the update is critical). +# +# The webapi protocol takes precedence over the version server one if the Teleport Proxy Service supports it. +# The version server protocol failover can be disabled by unsetting `updater.versionServer`. +# The webapi protocol can be disabled by setting `updater.proxyAddr` to `""`. +# For backward compatibility reasons, the webapi protocol is not enabled if a custom `updater.versionServer` is set. +# +# All Kubernetes-specific fields such as `tolerations`, `affinity`, `nodeSelector`, +# ... default to the agent values. However, they can be overridden from the +# `updater` object. For example: +# +# ```yaml +# # the agent pod requests 1cpu and 2 GiB of memory. It also has a memory limit. +# resources: +# requests: +# cpu: "1" +# memory: "2Gi" +# limits: +# memory: "2Gi" +# +# # the updater pod requests 0.5 cpu and 512MiB of memory. The memory limit has also been unset. +# updater: +# resources: +# requests: +# cpu: "0.5" +# memory: "512Mi" +# limits: ~ +# ``` +# +# Other updater-specific values that can be defined in `updater` are described +# below. updater: enabled: false # `updater.versionServer` is the URL of the version server the agent fetches # the target version from. The complete version endpoint is built by # concatenating `versionServer` and `releaseChannel`. # This field supports gotemplate. + # + # Setting this field makes the updater fetch the version using the version server protocol. + # Setting this field to a custom value disables the webapi update protocol to ensure backward compatibility. versionServer: "https://{{ .Values.proxyAddr }}/v1/webapi/automaticupgrades/channel" - # Release channel the agent subscribes to. + + # updater.releaseChannel(string) -- is the release channel the updater + # subscribes to. + # + # The complete version endpoint is built by concatenating + # [`versionServer`](#updaterversionserver) and [`releaseChannel`](#updaterreleasechannel). + # You must not change the default value if you are a Teleport Cloud user unless + # instructed by Teleport support. + # + # This value is used when the updater is fetching the version using the version server protocol. + # It is also used as a failover when fetching the version using the webapi protocol if `updater.group` is unset. releaseChannel: "stable/cloud" + + # updater.group(string) -- is the update group used when fetching the version using the webapi protocol. + # When unset, the group defaults to `update.releaseChannel`. + group: "" + + # updater.image(string) -- sets the container image used for Teleport updater + # pods run when `updater.enabled` is true. + # + # You can override this to use your own Teleport Kube Agent Updater image rather + # than a Teleport-published image. image: public.ecr.aws/gravitational/teleport-kube-agent-updater serviceAccount: # service account name defaults to "-updater" diff --git a/examples/systemd/before-remove b/examples/systemd/before-remove new file mode 100755 index 0000000000000..9bc9c6542999c --- /dev/null +++ b/examples/systemd/before-remove @@ -0,0 +1,12 @@ +#!/bin/bash + +# This before remove script is run each time the teleport package is removed. + +set -eu + +if [ $# -ge 1 ] && [ "$1" = "1" ]; then + echo "Skipping symlink removal as this is a package upgrade." +else + echo "Removing symlinks from Teleport system paths..." + /opt/teleport/system/bin/teleport-update unlink-package || true +fi diff --git a/examples/systemd/post-install b/examples/systemd/post-install new file mode 100755 index 0000000000000..189069bd2784d --- /dev/null +++ b/examples/systemd/post-install @@ -0,0 +1,8 @@ +#!/bin/bash + +# This post install script is run each time the teleport package is installed/upgraded. + +set -eu + +echo "Teleport system symlinks creation..." +/opt/teleport/system/bin/teleport-update link-package diff --git a/examples/systemd/post-upgrade b/examples/systemd/post-upgrade old mode 100644 new mode 100755 index 0fe4388403517..499bee029c7ce --- a/examples/systemd/post-upgrade +++ b/examples/systemd/post-upgrade @@ -1,11 +1,5 @@ #!/bin/bash -# this post upgrade script is run each time the teleport package is upgraded +# This post upgrade script is run each time the teleport package is upgraded. set -eu - -# skip reload and restart when systemd is disabled. This is only relevant when -# testing in a container. -if [ -d /run/systemd/system ]; then - systemctl --system daemon-reload >/dev/null || true -fi diff --git a/go.mod b/go.mod index 5de838c5a3f01..9eba9c698f487 100644 --- a/go.mod +++ b/go.mod @@ -211,6 +211,7 @@ require github.com/mailgun/minheap v0.0.0-20170619185613-3dbe6c6bf55f // indirec require ( github.com/Masterminds/sprig/v3 v3.2.3 + github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20231024185945-8841054dbdb8 github.com/distribution/reference v0.5.0 github.com/google/go-containerregistry v0.19.1 github.com/google/renameio/v2 v2.0.0 @@ -244,6 +245,8 @@ require ( github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 // indirect github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.20.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ecr v1.20.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.18.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 // indirect github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.3 // indirect diff --git a/go.sum b/go.sum index 26335f9351b93..048fa8442a81b 100644 --- a/go.sum +++ b/go.sum @@ -244,6 +244,7 @@ github.com/aws/aws-sdk-go v1.51.6 h1:Ld36dn9r7P9IjU8WZSaswQ8Y/XUCRpewim5980DwYiU github.com/aws/aws-sdk-go v1.51.6/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 h1:gTK2uhtAPtFcdRRJilZPx8uJLL2J85xK11nKtWL0wfU= @@ -262,9 +263,11 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0/go.mod h1:nQ3how7DMnFMWiU1 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9 h1:vXY/Hq1XdxHBIYgBUmug/AbMyIe1AKulPYS2/VE1X70= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.9/go.mod h1:GyJJTZoHVuENM4TeJEl5Ffs4W9m19u+4wKJcDi/GZ4A= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= @@ -329,6 +332,7 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod h1:0ih0Z83YDH/QeQ6Ori2yG github.com/aws/aws-sigv4-auth-cassandra-gocql-driver-plugin v0.0.0-20220331165046-e4d000c0d6a6 h1:+AQtpMAj/wOpgdmXSGKSBVozGsYbvaf73gTz4aSK9vM= github.com/aws/aws-sigv4-auth-cassandra-gocql-driver-plugin v0.0.0-20220331165046-e4d000c0d6a6/go.mod h1:Y5LTHeZGpeKFaXYfPYNfVqdpAjejlvXLhGqFqSJRQYc= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20231024185945-8841054dbdb8 h1:SoFYaT9UyGkR0+nogNyD/Lj+bsixB+SNuAS4ABlEs6M= diff --git a/integrations/kube-agent-updater/cmd/teleport-kube-agent-updater/constants.go b/integrations/kube-agent-updater/cmd/teleport-kube-agent-updater/constants.go index 3015b7150ae0f..d8238407ea418 100644 --- a/integrations/kube-agent-updater/cmd/teleport-kube-agent-updater/constants.go +++ b/integrations/kube-agent-updater/cmd/teleport-kube-agent-updater/constants.go @@ -34,3 +34,21 @@ EuIXJJox2oAL7NzdSi9VIUYnEnx+2EtkU/spAFRR6i1BnT6aoIy3521B76wnmRr9 atCSKjt6MdRxgj4htCjBWWJAGM9Z/avF4CYFmK7qiVxgpdrSM8Esbt2Ta+Lu3QMJ T8LjqFu3u3dxVOo9RuLk+BkCAwEAAQ== -----END PUBLIC KEY-----`) + +// teleportStageOCIPubKey is the key used to sign Teleport distroless images dev builds. +// The key lives in the Teleport staging AWS KMS. +// This key is only trusted on dev builds/pre-release versions of the kube updater. +var teleportStageOCIPubKey = []byte(`-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA8MPaUO4fcN3gS1psn3U7 +Pm/iM7jLQVg5MgcG9jbAkFsVOvmk3eq7cv0r94voz63IXhs4wKLK/e2QMljW1kz1 +AX7NvdXecCxwcyntgYnDXtxYBhcPGSM6cVnWlZ3pLNb8uVK7oxm0HjGUblcLreaI +aoLGmpyK+eCCLJso0Y7Yw0qRTJHg+2JQenbWps23AO96a6nqab2Ix7zEa3HyNZLa +P6rYV9q6vqZ3MBsDz5Lrc76JYSliqGVMVONhdXcqS2PYNti4Wm8o2CTJ0gRf2zYx +z2how6+rWM8HVoRYqG8JvCDvY6SGr5AbqIz/UCGm7XDH1S7M7C4FZ3MNTazoHY7h +VGAYLNPOtnQeZTtJDyRPH7csq+2tyvDPin3ymgRvvBrMrpBSmnnr67TxSIAv4xgu +B2hAgTL501B+s2m06bBcbKc03JsxgJBu4sBxKqIh1yeF8AW861bh90oZGI8/d9xM +fyI0BiELvY08HioQaAoC2VJx44I+KVDA1SLnMEx9n44eZ5Bk8G6PiZe5bikVDizF +RBVos6fjDapmGqVGoj+eotrI755FTKA3egB8DYw/H5yD1CO0QBBWXDhqM0ruTt4i +LzfxsdKEiXFMFZmXYzqwut9RXguGa/7LYPT7ijtW57z/wLytIjyYRkZH1P0dffFs +tiben+kjeNwFJ7Kg/WIDjjUCAwEAAQ== +-----END PUBLIC KEY-----`) diff --git a/integrations/kube-agent-updater/cmd/teleport-kube-agent-updater/main.go b/integrations/kube-agent-updater/cmd/teleport-kube-agent-updater/main.go index 8e7850b563394..e4591d28b89ad 100644 --- a/integrations/kube-agent-updater/cmd/teleport-kube-agent-updater/main.go +++ b/integrations/kube-agent-updater/cmd/teleport-kube-agent-updater/main.go @@ -18,6 +18,7 @@ package main import ( "flag" + "fmt" "net/url" "os" "strings" @@ -25,6 +26,7 @@ import ( "github.com/distribution/reference" "github.com/gravitational/trace" + "golang.org/x/mod/semver" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" @@ -37,6 +39,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "github.com/gravitational/teleport/api/client/webclient" kubeversionupdater "github.com/gravitational/teleport/integrations/kube-agent-updater" "github.com/gravitational/teleport/integrations/kube-agent-updater/pkg/controller" "github.com/gravitational/teleport/integrations/kube-agent-updater/pkg/img" @@ -68,6 +71,9 @@ func main() { var insecureNoVerify bool var insecureNoResolve bool var disableLeaderElection bool + var credSource string + var proxyAddress string + var updateGroup string flag.StringVar(&agentName, "agent-name", "", "The name of the agent that should be updated. This is mandatory.") flag.StringVar(&agentNamespace, "agent-namespace", "", "The namespace of the agent that should be updated. This is mandatory.") @@ -77,9 +83,16 @@ func main() { flag.BoolVar(&insecureNoVerify, "insecure-no-verify-image", false, "Disable image signature verification. The image tag is still resolved and image must exist.") flag.BoolVar(&insecureNoResolve, "insecure-no-resolve-image", false, "Disable image signature verification AND resolution. The updater can update to non-existing images.") flag.BoolVar(&disableLeaderElection, "disable-leader-election", false, "Disable leader election, used when running the kube-agent-updater outside of Kubernetes.") + flag.StringVar(&proxyAddress, "proxy-address", "", "The proxy address of the teleport cluster. When set, the updater will try to get update via the /find proxy endpoint.") + flag.StringVar(&updateGroup, "update-group", "", "The agent update group, as defined in the `autoupdate_config` resource. When unset or set to an unknown value, agent will update with the default group.") flag.StringVar(&versionServer, "version-server", "https://updates.releases.teleport.dev/v1/", "URL of the HTTP server advertising target version and critical maintenances. Trailing slash is optional.") flag.StringVar(&versionChannel, "version-channel", "cloud/stable", "Version channel to get updates from.") flag.StringVar(&baseImageName, "base-image", "public.ecr.aws/gravitational/teleport", "Image reference containing registry and repository.") + flag.StringVar(&credSource, "pull-credentials", img.NoCredentialSource, + fmt.Sprintf("Where to get registry pull credentials, values are '%s', '%s', '%s', '%s'.", + img.DockerCredentialSource, img.GoogleCredentialSource, img.AmazonCredentialSource, img.NoCredentialSource, + ), + ) opts := zap.Options{ Development: false, @@ -89,6 +102,7 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + // Validate configuration. if agentName == "" { ctrl.Log.Error(trace.BadParameter("--agent-name empty"), "agent-name must be provided") os.Exit(1) @@ -97,7 +111,16 @@ func main() { ctrl.Log.Error(trace.BadParameter("--agent-namespace empty"), "agent-namespace must be provided") os.Exit(1) } + if versionServer == "" && proxyAddress == "" { + ctrl.Log.Error( + trace.BadParameter("at least one of --proxy-address or --version-server must be provided"), + "the updater has no upstream configured, it cannot retrieve the version and check when to update", + ) + os.Exit(1) + } + // Build a new controller manager. We need to do this early as some trigger + // need a Kubernetes client and the manager is the one providing it. mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{BindAddress: metricsAddr}, @@ -120,16 +143,81 @@ func main() { os.Exit(1) } - versionServerURL, err := url.Parse(strings.TrimRight(versionServer, "/") + "/" + versionChannel) - if err != nil { - ctrl.Log.Error(err, "failed to pasre version server URL, exiting") - os.Exit(1) + // Craft the version getter and update triggers based on the configuration (use RFD-109 APIs, RFD-184, or both). + var criticalUpdateTriggers []maintenance.Trigger + var plannedMaintenanceTriggers []maintenance.Trigger + var versionGetters []version.Getter + + // If the proxy server is specified, we enabled RFD-184 updates + // See https://github.com/gravitational/teleport/blob/master/rfd/0184-agent-auto-updates.md#updater-apis + if proxyAddress != "" { + ctrl.Log.Info("fetching versions from the proxy /find endpoint", "proxy_server_url", proxyAddress, "update_group", updateGroup) + + proxyClt, err := webclient.NewReusableClient(&webclient.Config{ + Context: ctx, + ProxyAddr: proxyAddress, + UpdateGroup: updateGroup, + }) + if err != nil { + ctrl.Log.Error(err, "failed to create proxy client, exiting") + os.Exit(1) + } + + // We do a preflight check before starting to know if the proxy is correctly configured and reachable. + ctrl.Log.Info("preflight check: ping the proxy server", "proxy_server_url", proxyAddress) + pong, err := proxyClt.Ping() + if err != nil { + ctrl.Log.Error(err, "failed to ping proxy, either the proxy address is wrong, or the network blocks connections to the proxy", + "proxy_address", proxyAddress, + ) + os.Exit(1) + } + ctrl.Log.Info("proxy server successfully pinged", + "proxy_server_url", proxyAddress, + "proxy_cluster_name", pong.ClusterName, + "proxy_version", pong.ServerVersion, + ) + + versionGetters = append(versionGetters, version.NewProxyVersionGetter(proxyClt)) + + // In RFD 184, the server is driving the update, so both regular maintenances and + // critical ones are fetched from the proxy. Using the same trigger ensures we hit the cache if both triggers + // are evaluated and don't actually make 2 calls. + proxyTrigger := maintenance.NewProxyMaintenanceTrigger("proxy update protocol", proxyClt) + criticalUpdateTriggers = append(criticalUpdateTriggers, proxyTrigger) + plannedMaintenanceTriggers = append(plannedMaintenanceTriggers, proxyTrigger) } - versionGetter := version.NewBasicHTTPVersionGetter(versionServerURL) + + // If the version server is specified, we enable RFD-109 updates + // See https://github.com/gravitational/teleport/blob/master/rfd/0109-cloud-agent-upgrades.md#kubernetes-model + if versionServer != "" { + rawUrl := strings.TrimRight(versionServer, "/") + "/" + versionChannel + versionServerURL, err := url.Parse(rawUrl) + if err != nil { + ctrl.Log.Error(err, "failed to parse version server URL, exiting", "url", rawUrl) + os.Exit(1) + } + ctrl.Log.Info("fetching versions from the version server", "version_server_url", versionServerURL.String()) + + versionGetters = append(versionGetters, version.NewBasicHTTPVersionGetter(versionServerURL)) + // critical updates are advertised by the version channel + criticalUpdateTriggers = append(criticalUpdateTriggers, maintenance.NewBasicHTTPMaintenanceTrigger("critical update", versionServerURL)) + // planned maintenance windows are exported by the pods + plannedMaintenanceTriggers = append(plannedMaintenanceTriggers, podmaintenance.NewWindowTrigger("maintenance window", mgr.GetClient())) + } + maintenanceTriggers := maintenance.Triggers{ - maintenance.NewBasicHTTPMaintenanceTrigger("critical update", versionServerURL), + // We check if the update is critical. + maintenance.FailoverTrigger(criticalUpdateTriggers), + // We check if the agent in unhealthy. podmaintenance.NewUnhealthyWorkloadTrigger("unhealthy pods", mgr.GetClient()), - podmaintenance.NewWindowTrigger("maintenance window", mgr.GetClient()), + // We check if we're in a maintenance window. + maintenance.FailoverTrigger(plannedMaintenanceTriggers), + } + + kc, err := img.GetKeychain(credSource) + if err != nil { + ctrl.Log.Error(err, "failed to get keychain for registry auth") } var imageValidators img.Validators @@ -139,9 +227,18 @@ func main() { imageValidators = append(imageValidators, img.NewNopValidator("insecure no resolution")) case insecureNoVerify: ctrl.Log.Info("INSECURE: Image validation disabled") - imageValidators = append(imageValidators, img.NewInsecureValidator("insecure always verified")) + imageValidators = append(imageValidators, img.NewInsecureValidator("insecure always verified", kc)) + case semver.Prerelease("v"+kubeversionupdater.Version) != "": + ctrl.Log.Info("This is a pre-release updater version, the key usied to sign dev and pre-release builds of Teleport will be trusted.") + validator, err := img.NewCosignSingleKeyValidator(teleportStageOCIPubKey, "staging cosign signature validator", kc) + if err != nil { + ctrl.Log.Error(err, "failed to build pre-release image validator, exiting") + os.Exit(1) + } + imageValidators = append(imageValidators, validator) + fallthrough default: - validator, err := img.NewCosignSingleKeyValidator(teleportProdOCIPubKey, "cosign signature validator") + validator, err := img.NewCosignSingleKeyValidator(teleportProdOCIPubKey, "cosign signature validator", kc) if err != nil { ctrl.Log.Error(err, "failed to build image validator, exiting") os.Exit(1) @@ -155,7 +252,12 @@ func main() { os.Exit(1) } - versionUpdater := controller.NewVersionUpdater(versionGetter, imageValidators, maintenanceTriggers, baseImage) + versionUpdater := controller.NewVersionUpdater( + version.FailoverGetter(versionGetters), + imageValidators, + maintenanceTriggers, + baseImage, + ) // Controller registration deploymentController := controller.DeploymentVersionUpdater{ @@ -189,7 +291,7 @@ func main() { os.Exit(1) } - ctrl.Log.Info("starting the updater", "version", kubeversionupdater.Version, "url", versionServerURL.String()) + ctrl.Log.Info("starting the updater", "version", kubeversionupdater.Version) if err := mgr.Start(ctx); err != nil { ctrl.Log.Error(err, "failed to start manager, exiting") diff --git a/integrations/kube-agent-updater/pkg/img/cosign.go b/integrations/kube-agent-updater/pkg/img/cosign.go index f2c514cdd8d5d..b5bac4bf70f6a 100644 --- a/integrations/kube-agent-updater/pkg/img/cosign.go +++ b/integrations/kube-agent-updater/pkg/img/cosign.go @@ -22,7 +22,9 @@ import ( "encoding/hex" "github.com/distribution/reference" + "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/gravitational/trace" "github.com/opencontainers/go-digest" "github.com/sigstore/cosign/v2/pkg/cosign" @@ -101,7 +103,7 @@ func (v *cosignKeyValidator) ValidateAndResolveDigest(ctx context.Context, image // NewCosignSingleKeyValidator takes a PEM-encoded public key and returns an // img.Validator that checks the image was signed with cosign by the // corresponding private key. -func NewCosignSingleKeyValidator(pem []byte, name string) (Validator, error) { +func NewCosignSingleKeyValidator(pem []byte, name string, keyChain authn.Keychain) (Validator, error) { pubKey, err := cryptoutils.UnmarshalPEMToPublicKey(pem) if err != nil { return nil, trace.Wrap(err) @@ -115,9 +117,10 @@ func NewCosignSingleKeyValidator(pem []byte, name string) (Validator, error) { return nil, trace.Wrap(err) } return &cosignKeyValidator{ - verifier: verifier, - skid: skid, - name: name, + registryOptions: []ociremote.Option{ociremote.WithRemoteOptions(remote.WithAuthFromKeychain(keyChain))}, + verifier: verifier, + skid: skid, + name: name, }, nil } diff --git a/integrations/kube-agent-updater/pkg/img/cosign_test.go b/integrations/kube-agent-updater/pkg/img/cosign_test.go index 0fd3a2ab0c829..76aaecd73e17e 100644 --- a/integrations/kube-agent-updater/pkg/img/cosign_test.go +++ b/integrations/kube-agent-updater/pkg/img/cosign_test.go @@ -43,7 +43,7 @@ import ( var distrolessKey = []byte("-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEWZzVzkb8A+DbgDpaJId/bOmV8n7Q\nOqxYbK0Iro6GzSmOzxkn+N2AKawLyXi84WSwJQBK//psATakCgAQKkNTAA==\n-----END PUBLIC KEY-----") func Test_NewCosignSingleKeyValidator(t *testing.T) { - a, err := NewCosignSingleKeyValidator(distrolessKey, "distroless") + a, err := NewCosignSingleKeyValidator(distrolessKey, "distroless", nil) require.NoError(t, err) require.Equal(t, "distroless-799a5c21a7f8c39707274cbd065ba2e1969d8d29", a.Name()) } diff --git a/integrations/kube-agent-updater/pkg/img/insecure.go b/integrations/kube-agent-updater/pkg/img/insecure.go index a02cc17e2473e..cdcf0a7408325 100644 --- a/integrations/kube-agent-updater/pkg/img/insecure.go +++ b/integrations/kube-agent-updater/pkg/img/insecure.go @@ -20,12 +20,16 @@ import ( "context" "github.com/distribution/reference" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/gravitational/trace" "github.com/opencontainers/go-digest" + ociremote "github.com/sigstore/cosign/v2/pkg/oci/remote" ) type insecureValidator struct { - name string + name string + registryOptions []ociremote.Option } // Name returns the validator name @@ -45,7 +49,7 @@ func (v *insecureValidator) Name() string { // image is valid. Using this validator makes you vulnerable in case of image // registry compromise. func (v *insecureValidator) ValidateAndResolveDigest(ctx context.Context, image reference.NamedTagged) (NamedTaggedDigested, error) { - ref, err := NamedTaggedToDigest(image) + ref, err := NamedTaggedToDigest(image, v.registryOptions...) if err != nil { return nil, trace.Wrap(err) } @@ -57,8 +61,9 @@ func (v *insecureValidator) ValidateAndResolveDigest(ctx context.Context, image // NewInsecureValidator returns an img.Validator that only resolves the image // but does not check its signature. This must not be confused with // NewNopValidator that returns a validator that always validate without resolving. -func NewInsecureValidator(name string) Validator { +func NewInsecureValidator(name string, keyChain authn.Keychain) Validator { return &insecureValidator{ - name: name, + name: name, + registryOptions: []ociremote.Option{ociremote.WithRemoteOptions(remote.WithAuthFromKeychain(keyChain))}, } } diff --git a/integrations/kube-agent-updater/pkg/img/keychains.go b/integrations/kube-agent-updater/pkg/img/keychains.go new file mode 100644 index 0000000000000..1cfe460bd563c --- /dev/null +++ b/integrations/kube-agent-updater/pkg/img/keychains.go @@ -0,0 +1,54 @@ +/* +Copyright 2023 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package img + +import ( + "github.com/awslabs/amazon-ecr-credential-helper/ecr-login" + "github.com/awslabs/amazon-ecr-credential-helper/ecr-login/api" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/v1/google" + "github.com/gravitational/trace" +) + +type CredentialSource string + +const ( + DockerCredentialSource = "docker" + AmazonCredentialSource = "aws" + GoogleCredentialSource = "google" + NoCredentialSource = "none" +) + +// GetKeychain builds a ggcr keychain for image pulling and returns it. +// We could attempt to autodetect or build a multi-keychain but ECR login +// attempts take a lot of time and log errors. As most users don't need registry +// auth, it's acceptable to have them do an extra step and specify which auth +// they need. +func GetKeychain(credSource string) (authn.Keychain, error) { + switch credSource { + case DockerCredentialSource: + return authn.DefaultKeychain, nil + case AmazonCredentialSource: + return authn.NewKeychainFromHelper(ecr.NewECRHelper(ecr.WithClientFactory(api.DefaultClientFactory{}))), nil + case GoogleCredentialSource: + return google.Keychain, nil + case NoCredentialSource: + return nil, nil + default: + return nil, trace.BadParameter("credential source '%s' not recognized", credSource) + } +} diff --git a/lib/auth/auth.go b/lib/auth/auth.go index c8a53890645e8..3d74e66958c36 100644 --- a/lib/auth/auth.go +++ b/lib/auth/auth.go @@ -90,8 +90,8 @@ import ( "github.com/gravitational/teleport/lib/gitlab" "github.com/gravitational/teleport/lib/internal/context121" "github.com/gravitational/teleport/lib/inventory" + kubetoken "github.com/gravitational/teleport/lib/kube/token" kubeutils "github.com/gravitational/teleport/lib/kube/utils" - "github.com/gravitational/teleport/lib/kubernetestoken" "github.com/gravitational/teleport/lib/limiter" "github.com/gravitational/teleport/lib/loginrule" "github.com/gravitational/teleport/lib/modules" @@ -478,10 +478,10 @@ func NewServer(cfg *InitConfig, opts ...ServerOption) (*Server, error) { as.tpmValidator = tpm.Validate } if as.k8sTokenReviewValidator == nil { - as.k8sTokenReviewValidator = &kubernetestoken.TokenReviewValidator{} + as.k8sTokenReviewValidator = &kubetoken.TokenReviewValidator{} } if as.k8sJWKSValidator == nil { - as.k8sJWKSValidator = kubernetestoken.ValidateTokenWithJWKS + as.k8sJWKSValidator = kubetoken.ValidateTokenWithJWKS } if as.gcpIDTokenValidator == nil { @@ -1034,6 +1034,7 @@ func (a *Server) syncUpgradeWindowStartHour(ctx context.Context) error { agentWindow, _ := cmc.GetAgentUpgradeWindow() agentWindow.UTCStartHour = uint32(startHour) + agentWindow.Weekdays = []string{"Mon", "Tue", "Wed", "Thu"} cmc.SetAgentUpgradeWindow(agentWindow) @@ -5582,7 +5583,7 @@ func (a *Server) ExportUpgradeWindows(ctx context.Context, req proto.ExportUpgra } switch req.UpgraderKind { - case "": + case "", types.UpgraderKindTeleportUpdate: rsp.CanonicalSchedule = cached.CanonicalSchedule.Clone() case types.UpgraderKindKubeController: rsp.KubeControllerSchedule = cached.KubeControllerSchedule diff --git a/lib/auth/authclient/api.go b/lib/auth/authclient/api.go index 3949bce0edc05..cecb7decba1a8 100644 --- a/lib/auth/authclient/api.go +++ b/lib/auth/authclient/api.go @@ -305,6 +305,9 @@ type ReadProxyAccessPoint interface { // GetAutoUpdateVersion gets the AutoUpdateVersion from the backend. GetAutoUpdateVersion(ctx context.Context) (*autoupdate.AutoUpdateVersion, error) + + // GetAutoUpdateAgentRollout gets the AutoUpdateAgentRollout from the backend. + GetAutoUpdateAgentRollout(ctx context.Context) (*autoupdate.AutoUpdateAgentRollout, error) } // SnowflakeSessionWatcher is watcher interface used by Snowflake web session watcher. @@ -1151,6 +1154,9 @@ type Cache interface { // GetAutoUpdateVersion gets the AutoUpdateVersion from the backend. GetAutoUpdateVersion(ctx context.Context) (*autoupdate.AutoUpdateVersion, error) + + // GetAutoUpdateAgentRollout gets the AutoUpdateAgentRollout from the backend. + GetAutoUpdateAgentRollout(ctx context.Context) (*autoupdate.AutoUpdateAgentRollout, error) } type NodeWrapper struct { diff --git a/lib/auth/autoupdate/autoupdatev1/service.go b/lib/auth/autoupdate/autoupdatev1/service.go index 5bc902b739663..bf34575a58e76 100644 --- a/lib/auth/autoupdate/autoupdatev1/service.go +++ b/lib/auth/autoupdate/autoupdatev1/service.go @@ -21,12 +21,14 @@ package autoupdatev1 import ( "context" "log/slog" + "maps" "github.com/gravitational/trace" "google.golang.org/protobuf/types/known/emptypb" "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" "github.com/gravitational/teleport/api/types" + update "github.com/gravitational/teleport/api/types/autoupdate" apievents "github.com/gravitational/teleport/api/types/events" "github.com/gravitational/teleport/lib/authz" "github.com/gravitational/teleport/lib/events" @@ -41,6 +43,9 @@ type Cache interface { // GetAutoUpdateVersion gets the AutoUpdateVersion from the backend. GetAutoUpdateVersion(ctx context.Context) (*autoupdate.AutoUpdateVersion, error) + + // GetAutoUpdateAgentRollout gets the AutoUpdateAgentRollout from the backend. + GetAutoUpdateAgentRollout(ctx context.Context) (*autoupdate.AutoUpdateAgentRollout, error) } // ServiceConfig holds configuration options for the auto update gRPC service. @@ -115,6 +120,10 @@ func (s *Service) CreateAutoUpdateConfig(ctx context.Context, req *autoupdate.Cr return nil, trace.Wrap(err) } + if err := validateServerSideAgentConfig(req.Config); err != nil { + return nil, trace.Wrap(err) + } + config, err := s.backend.CreateAutoUpdateConfig(ctx, req.Config) var errMsg string if err != nil { @@ -151,6 +160,10 @@ func (s *Service) UpdateAutoUpdateConfig(ctx context.Context, req *autoupdate.Up return nil, trace.Wrap(err) } + if err := validateServerSideAgentConfig(req.Config); err != nil { + return nil, trace.Wrap(err) + } + config, err := s.backend.UpdateAutoUpdateConfig(ctx, req.Config) var errMsg string if err != nil { @@ -187,6 +200,10 @@ func (s *Service) UpsertAutoUpdateConfig(ctx context.Context, req *autoupdate.Up return nil, trace.Wrap(err) } + if err := validateServerSideAgentConfig(req.Config); err != nil { + return nil, trace.Wrap(err) + } + config, err := s.backend.UpsertAutoUpdateConfig(ctx, req.Config) var errMsg string if err != nil { @@ -430,6 +447,127 @@ func (s *Service) DeleteAutoUpdateVersion(ctx context.Context, req *autoupdate.D return &emptypb.Empty{}, trace.Wrap(err) } +// GetAutoUpdateAgentRollout gets the current AutoUpdateAgentRollout singleton. +func (s *Service) GetAutoUpdateAgentRollout(ctx context.Context, req *autoupdate.GetAutoUpdateAgentRolloutRequest) (*autoupdate.AutoUpdateAgentRollout, error) { + authCtx, err := s.authorizer.Authorize(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + + if err := authCtx.CheckAccessToKind(types.KindAutoUpdateAgentRollout, types.VerbRead); err != nil { + return nil, trace.Wrap(err) + } + + plan, err := s.cache.GetAutoUpdateAgentRollout(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + + return plan, nil +} + +// CreateAutoUpdateAgentRollout creates AutoUpdateAgentRollout singleton. +func (s *Service) CreateAutoUpdateAgentRollout(ctx context.Context, req *autoupdate.CreateAutoUpdateAgentRolloutRequest) (*autoupdate.AutoUpdateAgentRollout, error) { + authCtx, err := s.authorizer.Authorize(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + + // Editing the AU agent plan is restricted to cluster administrators. As of today we don't have any way of having + // resources that can only be edited by Teleport Cloud (when running cloud-hosted). + // The workaround is to check if the caller has the auth/admin system role. + // This is not ideal as it forces local tctl usage and can be bypassed if the user is very creative. + // In the future, if we expand the permission system and make cloud + // a first class citizen, we'll want to update this permission check. + if !(authz.HasBuiltinRole(*authCtx, string(types.RoleAuth)) || authz.HasBuiltinRole(*authCtx, string(types.RoleAdmin))) { + return nil, trace.AccessDenied("this request can be only executed by an auth server") + } + + if err := authCtx.CheckAccessToKind(types.KindAutoUpdateAgentRollout, types.VerbCreate); err != nil { + return nil, trace.Wrap(err) + } + + autoUpdateAgentRollout, err := s.backend.CreateAutoUpdateAgentRollout(ctx, req.Rollout) + return autoUpdateAgentRollout, trace.Wrap(err) +} + +// UpdateAutoUpdateAgentRollout updates AutoUpdateAgentRollout singleton. +func (s *Service) UpdateAutoUpdateAgentRollout(ctx context.Context, req *autoupdate.UpdateAutoUpdateAgentRolloutRequest) (*autoupdate.AutoUpdateAgentRollout, error) { + authCtx, err := s.authorizer.Authorize(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + + // Editing the AU agent plan is restricted to cluster administrators. As of today we don't have any way of having + // resources that can only be edited by Teleport Cloud (when running cloud-hosted). + // The workaround is to check if the caller has the auth/admin system role. + // This is not ideal as it forces local tctl usage and can be bypassed if the user is very creative. + // In the future, if we expand the permission system and make cloud + // a first class citizen, we'll want to update this permission check. + if !(authz.HasBuiltinRole(*authCtx, string(types.RoleAuth)) || authz.HasBuiltinRole(*authCtx, string(types.RoleAdmin))) { + return nil, trace.AccessDenied("this request can be only executed by an auth server") + } + + if err := authCtx.CheckAccessToKind(types.KindAutoUpdateAgentRollout, types.VerbUpdate); err != nil { + return nil, trace.Wrap(err) + } + + autoUpdateAgentRollout, err := s.backend.UpdateAutoUpdateAgentRollout(ctx, req.Rollout) + return autoUpdateAgentRollout, trace.Wrap(err) +} + +// UpsertAutoUpdateAgentRollout updates or creates AutoUpdateAgentRollout singleton. +func (s *Service) UpsertAutoUpdateAgentRollout(ctx context.Context, req *autoupdate.UpsertAutoUpdateAgentRolloutRequest) (*autoupdate.AutoUpdateAgentRollout, error) { + authCtx, err := s.authorizer.Authorize(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + + // Editing the AU agent plan is restricted to cluster administrators. As of today we don't have any way of having + // resources that can only be edited by Teleport Cloud (when running cloud-hosted). + // The workaround is to check if the caller has the auth/admin system role. + // This is not ideal as it forces local tctl usage and can be bypassed if the user is very creative. + // In the future, if we expand the permission system and make cloud + // a first class citizen, we'll want to update this permission check. + if !(authz.HasBuiltinRole(*authCtx, string(types.RoleAuth)) || authz.HasBuiltinRole(*authCtx, string(types.RoleAdmin))) { + return nil, trace.AccessDenied("this request can be only executed by an auth server") + } + + if err := authCtx.CheckAccessToKind(types.KindAutoUpdateAgentRollout, types.VerbCreate, types.VerbUpdate); err != nil { + return nil, trace.Wrap(err) + } + + autoUpdateAgentRollout, err := s.backend.UpsertAutoUpdateAgentRollout(ctx, req.Rollout) + return autoUpdateAgentRollout, trace.Wrap(err) +} + +// DeleteAutoUpdateAgentRollout deletes AutoUpdateAgentRollout singleton. +func (s *Service) DeleteAutoUpdateAgentRollout(ctx context.Context, req *autoupdate.DeleteAutoUpdateAgentRolloutRequest) (*emptypb.Empty, error) { + authCtx, err := s.authorizer.Authorize(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + + // Editing the AU agent plan is restricted to cluster administrators. As of today we don't have any way of having + // resources that can only be edited by Teleport Cloud (when running cloud-hosted). + // The workaround is to check if the caller has the auth/admin system role. + // This is not ideal as it forces local tctl usage and can be bypassed if the user is very creative. + // In the future, if we expand the permission system and make cloud + // a first class citizen, we'll want to update this permission check. + if !(authz.HasBuiltinRole(*authCtx, string(types.RoleAuth)) || authz.HasBuiltinRole(*authCtx, string(types.RoleAdmin))) { + return nil, trace.AccessDenied("this request can be only executed by an auth server") + } + + if err := authCtx.CheckAccessToKind(types.KindAutoUpdateAgentRollout, types.VerbDelete); err != nil { + return nil, trace.Wrap(err) + } + + if err := s.backend.DeleteAutoUpdateAgentRollout(ctx); err != nil { + return nil, trace.Wrap(err) + } + return &emptypb.Empty{}, nil +} + func (s *Service) emitEvent(ctx context.Context, e apievents.AuditEvent) { if err := s.emitter.EmitAuditEvent(ctx, e); err != nil { slog.WarnContext(ctx, "Failed to emit audit event", @@ -449,3 +587,131 @@ func checkAdminCloudAccess(authCtx *authz.Context) error { } return nil } + +// Those values are arbitrary, we will want to increase them as we test. We will also want to modulate them based on the +// cluster context. We don't want people to craft schedules that can't realistically finish within a week on Cloud as +// we usually do weekly updates. However, self-hosted users can craft more complex schedules, slower rollouts, and shoot +// themselves in the foot if they want. +const ( + maxGroupsTimeBasedStrategy = 20 + maxGroupsHaltOnErrorStrategy = 10 + maxGroupsHaltOnErrorStrategyCloud = 4 + maxRolloutDurationCloudHours = 72 +) + +var ( + cloudGroupUpdateDays = []string{"Mon", "Tue", "Wed", "Thu"} +) + +// validateServerSideAgentConfig validates that the autoupdate_config.agent spec meets the cluster rules. +// Rules may vary based on the cluster, and over time. +// +// This function should not be confused with api/types/autoupdate.ValidateAutoUpdateConfig which validates the integrity +// of the resource and does not enforce potentially changing rules. +func validateServerSideAgentConfig(config *autoupdate.AutoUpdateConfig) error { + agentsSpec := config.GetSpec().GetAgents() + if agentsSpec == nil { + return nil + } + // We must check resource integrity before, because it makes no sense to try to enforce rules on an invalid resource. + // The generic backend service will likely check integrity again, but it's not a large performance problem. + err := update.ValidateAutoUpdateConfig(config) + if err != nil { + return trace.Wrap(err, "validating autoupdate config") + } + + var maxGroups int + isCloud := modules.GetModules().Features().Cloud + + switch { + case isCloud && agentsSpec.GetStrategy() == update.AgentsStrategyHaltOnError: + maxGroups = maxGroupsHaltOnErrorStrategyCloud + case agentsSpec.GetStrategy() == update.AgentsStrategyHaltOnError: + maxGroups = maxGroupsHaltOnErrorStrategy + case agentsSpec.GetStrategy() == update.AgentsStrategyTimeBased: + maxGroups = maxGroupsTimeBasedStrategy + default: + return trace.BadParameter("unknown max group for strategy %v", agentsSpec.GetStrategy()) + } + + if len(agentsSpec.GetSchedules().GetRegular()) > maxGroups { + return trace.BadParameter("max groups (%d) exceeded for strategy %s, %s schedule contains %d groups", maxGroups, agentsSpec.GetStrategy(), update.AgentsScheduleRegular, len(agentsSpec.GetSchedules().GetRegular())) + } + + if !isCloud { + return nil + } + + cloudWeekdays, err := types.ParseWeekdays(cloudGroupUpdateDays) + if err != nil { + return trace.Wrap(err, "parsing cloud weekdays") + } + + for i, group := range agentsSpec.GetSchedules().GetRegular() { + weekdays, err := types.ParseWeekdays(group.Days) + if err != nil { + return trace.Wrap(err, "parsing weekdays from group %d", i) + } + + if !maps.Equal(cloudWeekdays, weekdays) { + return trace.BadParameter("weekdays must be set to %v in cloud", cloudGroupUpdateDays) + } + + } + + if duration := computeMinRolloutTime(agentsSpec.GetSchedules().GetRegular()); duration > maxRolloutDurationCloudHours { + return trace.BadParameter("rollout takes more than %d hours to complete: estimated completion time is %d hours", maxRolloutDurationCloudHours, duration) + } + + return nil +} + +func computeMinRolloutTime(groups []*autoupdate.AgentAutoUpdateGroup) int { + if len(groups) == 0 { + return 0 + } + + // We start the rollout at the first group hour, and we wait for the group to update (1 hour). + hours := groups[0].StartHour + 1 + + for _, group := range groups[1:] { + previousStartHour := (hours - 1) % 24 + previousEndHour := hours % 24 + + // compute the difference between the current hour and the group start hour + // we then check if it's less than the WaitHours, in this case we wait a day + diff := hourDifference(previousStartHour, group.StartHour) + if diff < group.WaitHours%24 { + hours += 24 + hourDifference(previousEndHour, group.StartHour) + } else { + hours += hourDifference(previousEndHour, group.StartHour) + } + + // Handle the case where WaitHours is > 24 + // This is an integer division + waitDays := group.WaitHours / 24 + // There's a special case where the difference modulo 24 is zero, the + // wait hours are non-null, but we already waited 23 hours. + // To avoid double counting we reduce the number of wait days by 1 if + // it's not zero already. + if diff == 0 { + waitDays = max(waitDays-1, 0) + } + hours += waitDays * 24 + + // We assume the group took an hour to update + hours += 1 + } + + // We remove the group start hour we added initially + return int(hours - groups[0].StartHour) +} + +// hourDifference computed the difference between two hours. +func hourDifference(a, b int32) int32 { + diff := b - a + if diff < 0 { + diff = diff + 24 + } + return diff +} diff --git a/lib/auth/autoupdate/autoupdatev1/service_test.go b/lib/auth/autoupdate/autoupdatev1/service_test.go index b9c6f6cccbef2..69737a1344c33 100644 --- a/lib/auth/autoupdate/autoupdatev1/service_test.go +++ b/lib/auth/autoupdate/autoupdatev1/service_test.go @@ -18,11 +18,13 @@ package autoupdatev1 import ( "context" + "strconv" "testing" + "time" - "github.com/google/uuid" "github.com/gravitational/trace" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/durationpb" autoupdatev1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" "github.com/gravitational/teleport/api/types" @@ -32,261 +34,16 @@ import ( "github.com/gravitational/teleport/lib/backend/memory" libevents "github.com/gravitational/teleport/lib/events" "github.com/gravitational/teleport/lib/events/eventstest" + "github.com/gravitational/teleport/lib/modules" "github.com/gravitational/teleport/lib/services" "github.com/gravitational/teleport/lib/services/local" - "github.com/gravitational/teleport/lib/tlsca" ) -func TestAutoUpdateCRUD(t *testing.T) { - t.Parallel() - - requireTraceErrorFn := func(traceFn func(error) bool) require.ErrorAssertionFunc { - return func(tt require.TestingT, err error, i ...interface{}) { - require.True(t, traceFn(err), "received an un-expected error: %v", err) - } - } - - ctx, localClient, resourceSvc := initSvc(t, "test-cluster", &eventstest.MockRecorderEmitter{}) - initialConfig, err := autoupdate.NewAutoUpdateConfig(&autoupdatev1pb.AutoUpdateConfigSpec{ - Tools: &autoupdatev1pb.AutoUpdateConfigSpecTools{ - Mode: "enabled", - }, - }) - require.NoError(t, err) - initialVersion, err := autoupdate.NewAutoUpdateVersion(&autoupdatev1pb.AutoUpdateVersionSpec{ - Tools: &autoupdatev1pb.AutoUpdateVersionSpecTools{ - TargetVersion: "1.2.3", - }, - }) - require.NoError(t, err) - - tt := []struct { - Name string - Role types.RoleSpecV6 - Setup func(t *testing.T, dcName string) - Test func(ctx context.Context, resourceSvc *Service, dcName string) error - ErrAssertion require.ErrorAssertionFunc - }{ - // Read - { - Name: "allowed read access to auto update resources", - Role: types.RoleSpecV6{ - Allow: types.RoleConditions{Rules: []types.Rule{{ - Resources: []string{types.KindAutoUpdateConfig, types.KindAutoUpdateVersion}, - Verbs: []string{types.VerbRead}, - }}}, - }, - Setup: func(t *testing.T, dcName string) { - _, err := localClient.CreateAutoUpdateConfig(ctx, initialConfig) - require.NoError(t, err) - _, err = localClient.CreateAutoUpdateVersion(ctx, initialVersion) - require.NoError(t, err) - }, - Test: func(ctx context.Context, resourceSvc *Service, dcName string) error { - _, errConfig := resourceSvc.GetAutoUpdateConfig(ctx, &autoupdatev1pb.GetAutoUpdateConfigRequest{}) - _, errVersion := resourceSvc.GetAutoUpdateVersion(ctx, &autoupdatev1pb.GetAutoUpdateVersionRequest{}) - return trace.NewAggregate(errConfig, errVersion) - }, - ErrAssertion: require.NoError, - }, - { - Name: "no access to auto update resources", - Role: types.RoleSpecV6{}, - Setup: func(t *testing.T, dcName string) { - _, err := localClient.CreateAutoUpdateConfig(ctx, initialConfig) - require.NoError(t, err) - _, err = localClient.CreateAutoUpdateVersion(ctx, initialVersion) - require.NoError(t, err) - }, - Test: func(ctx context.Context, resourceSvc *Service, dcName string) error { - _, errConfig := resourceSvc.GetAutoUpdateConfig(ctx, &autoupdatev1pb.GetAutoUpdateConfigRequest{}) - _, errVersion := resourceSvc.GetAutoUpdateVersion(ctx, &autoupdatev1pb.GetAutoUpdateVersionRequest{}) - return trace.NewAggregate(errConfig, errVersion) - }, - ErrAssertion: requireTraceErrorFn(trace.IsAccessDenied), - }, - // Create - { - Name: "no access to create auto update resources", - Role: types.RoleSpecV6{}, - Test: func(ctx context.Context, resourceSvc *Service, dcName string) error { - _, errConfig := resourceSvc.CreateAutoUpdateConfig(ctx, &autoupdatev1pb.CreateAutoUpdateConfigRequest{ - Config: initialConfig, - }) - _, errVersion := resourceSvc.CreateAutoUpdateVersion(ctx, &autoupdatev1pb.CreateAutoUpdateVersionRequest{ - Version: initialVersion, - }) - return trace.NewAggregate(errConfig, errVersion) - }, - ErrAssertion: requireTraceErrorFn(trace.IsAccessDenied), - }, - { - Name: "access to create auto update resources", - Role: types.RoleSpecV6{ - Allow: types.RoleConditions{Rules: []types.Rule{{ - Resources: []string{types.KindAutoUpdateConfig, types.KindAutoUpdateVersion}, - Verbs: []string{types.VerbCreate}, - }}}, - }, - Test: func(ctx context.Context, resourceSvc *Service, dcName string) error { - _, errConfig := resourceSvc.CreateAutoUpdateConfig(ctx, &autoupdatev1pb.CreateAutoUpdateConfigRequest{ - Config: initialConfig, - }) - _, errVersion := resourceSvc.CreateAutoUpdateVersion(ctx, &autoupdatev1pb.CreateAutoUpdateVersionRequest{ - Version: initialVersion, - }) - return trace.NewAggregate(errConfig, errVersion) - }, - ErrAssertion: require.NoError, - }, - // Update - { - Name: "no access to update auto update resources", - Role: types.RoleSpecV6{}, - Test: func(ctx context.Context, resourceSvc *Service, dcName string) error { - _, errConfig := resourceSvc.UpdateAutoUpdateConfig(ctx, &autoupdatev1pb.UpdateAutoUpdateConfigRequest{ - Config: initialConfig, - }) - _, errVersion := resourceSvc.UpdateAutoUpdateVersion(ctx, &autoupdatev1pb.UpdateAutoUpdateVersionRequest{ - Version: initialVersion, - }) - return trace.NewAggregate(errConfig, errVersion) - }, - ErrAssertion: requireTraceErrorFn(trace.IsAccessDenied), - }, - { - Name: "access to update auto update resources", - Role: types.RoleSpecV6{ - Allow: types.RoleConditions{Rules: []types.Rule{{ - Resources: []string{types.KindAutoUpdateConfig, types.KindAutoUpdateVersion}, - Verbs: []string{types.VerbUpdate}, - }}}, - }, - Setup: func(t *testing.T, dcName string) { - _, err := localClient.CreateAutoUpdateConfig(ctx, initialConfig) - require.NoError(t, err) - _, err = localClient.CreateAutoUpdateVersion(ctx, initialVersion) - require.NoError(t, err) - }, - Test: func(ctx context.Context, resourceSvc *Service, dcName string) error { - _, errConfig := resourceSvc.UpdateAutoUpdateConfig(ctx, &autoupdatev1pb.UpdateAutoUpdateConfigRequest{ - Config: initialConfig, - }) - _, errVersion := resourceSvc.UpdateAutoUpdateVersion(ctx, &autoupdatev1pb.UpdateAutoUpdateVersionRequest{ - Version: initialVersion, - }) - return trace.NewAggregate(errConfig, errVersion) - }, - ErrAssertion: require.NoError, - }, - // Upsert - { - Name: "no access to upsert auto update resources", - Role: types.RoleSpecV6{ - Allow: types.RoleConditions{Rules: []types.Rule{{ - Resources: []string{types.KindAutoUpdateConfig, types.KindAutoUpdateVersion}, - Verbs: []string{types.VerbUpdate}, // missing VerbCreate - }}}, - }, - Test: func(ctx context.Context, resourceSvc *Service, dcName string) error { - _, errConfig := resourceSvc.UpsertAutoUpdateConfig(ctx, &autoupdatev1pb.UpsertAutoUpdateConfigRequest{ - Config: initialConfig, - }) - _, errVersion := resourceSvc.UpsertAutoUpdateVersion(ctx, &autoupdatev1pb.UpsertAutoUpdateVersionRequest{ - Version: initialVersion, - }) - return trace.NewAggregate(errConfig, errVersion) - }, - ErrAssertion: requireTraceErrorFn(trace.IsAccessDenied), - }, - { - Name: "access to upsert auto update resources", - Role: types.RoleSpecV6{ - Allow: types.RoleConditions{Rules: []types.Rule{{ - Resources: []string{types.KindAutoUpdateConfig, types.KindAutoUpdateVersion}, - Verbs: []string{types.VerbUpdate, types.VerbCreate}, - }}}, - }, - Setup: func(t *testing.T, dcName string) {}, - Test: func(ctx context.Context, resourceSvc *Service, dcName string) error { - _, errConfig := resourceSvc.UpsertAutoUpdateConfig(ctx, &autoupdatev1pb.UpsertAutoUpdateConfigRequest{ - Config: initialConfig, - }) - _, errVersion := resourceSvc.UpsertAutoUpdateVersion(ctx, &autoupdatev1pb.UpsertAutoUpdateVersionRequest{ - Version: initialVersion, - }) - return trace.NewAggregate(errConfig, errVersion) - }, - ErrAssertion: require.NoError, - }, - // Delete - { - Name: "no access to delete auto update resources", - Role: types.RoleSpecV6{}, - Test: func(ctx context.Context, resourceSvc *Service, dcName string) error { - _, errConfig := resourceSvc.DeleteAutoUpdateConfig(ctx, &autoupdatev1pb.DeleteAutoUpdateConfigRequest{}) - _, errVersion := resourceSvc.DeleteAutoUpdateVersion(ctx, &autoupdatev1pb.DeleteAutoUpdateVersionRequest{}) - return trace.NewAggregate(errConfig, errVersion) - }, - ErrAssertion: requireTraceErrorFn(trace.IsAccessDenied), - }, - { - Name: "access to delete auto update resources", - Role: types.RoleSpecV6{ - Allow: types.RoleConditions{Rules: []types.Rule{{ - Resources: []string{types.KindAutoUpdateConfig, types.KindAutoUpdateVersion}, - Verbs: []string{types.VerbDelete}, - }}}, - }, - Setup: func(t *testing.T, dcName string) { - _, err := localClient.CreateAutoUpdateConfig(ctx, initialConfig) - require.NoError(t, err) - _, err = localClient.CreateAutoUpdateVersion(ctx, initialVersion) - require.NoError(t, err) - }, - Test: func(ctx context.Context, resourceSvc *Service, dcName string) error { - _, errConfig := resourceSvc.DeleteAutoUpdateConfig(ctx, &autoupdatev1pb.DeleteAutoUpdateConfigRequest{}) - _, errVersion := resourceSvc.DeleteAutoUpdateVersion(ctx, &autoupdatev1pb.DeleteAutoUpdateVersionRequest{}) - return trace.NewAggregate(errConfig, errVersion) - }, - ErrAssertion: require.NoError, - }, - } - - for _, tc := range tt { - tc := tc - t.Run(tc.Name, func(t *testing.T) { - localCtx := authorizerForDummyUser(t, ctx, tc.Role, localClient) - - dcName := uuid.NewString() - if tc.Setup != nil { - tc.Setup(t, dcName) - } - - err := tc.Test(localCtx, resourceSvc, dcName) - tc.ErrAssertion(t, err) - err = localClient.DeleteAutoUpdateConfig(ctx) - if !trace.IsNotFound(err) { - require.NoError(t, err) - } - err = localClient.DeleteAutoUpdateVersion(ctx) - if !trace.IsNotFound(err) { - require.NoError(t, err) - } - }) - } -} - func TestAutoUpdateConfigEvents(t *testing.T) { - role := types.RoleSpecV6{ - Allow: types.RoleConditions{Rules: []types.Rule{{ - Resources: []string{types.KindAutoUpdateConfig}, - Verbs: []string{types.VerbList, types.VerbCreate, types.VerbRead, types.VerbUpdate, types.VerbDelete}, - }}}, - } + rwVerbs := []string{types.VerbList, types.VerbCreate, types.VerbRead, types.VerbUpdate, types.VerbDelete} mockEmitter := &eventstest.MockRecorderEmitter{} - ctx, localClient, service := initSvc(t, "test-cluster", mockEmitter) - localCtx := authorizerForDummyUser(t, ctx, role, localClient) + service := newService(t, fakeChecker{allowedVerbs: rwVerbs}, mockEmitter) + ctx := context.Background() config, err := autoupdate.NewAutoUpdateConfig(&autoupdatev1pb.AutoUpdateConfigSpec{ Tools: &autoupdatev1pb.AutoUpdateConfigSpecTools{ @@ -295,7 +52,7 @@ func TestAutoUpdateConfigEvents(t *testing.T) { }) require.NoError(t, err) - _, err = service.CreateAutoUpdateConfig(localCtx, &autoupdatev1pb.CreateAutoUpdateConfigRequest{Config: config}) + _, err = service.CreateAutoUpdateConfig(ctx, &autoupdatev1pb.CreateAutoUpdateConfigRequest{Config: config}) require.NoError(t, err) require.Len(t, mockEmitter.Events(), 1) require.Equal(t, libevents.AutoUpdateConfigCreateEvent, mockEmitter.LastEvent().GetType()) @@ -303,7 +60,7 @@ func TestAutoUpdateConfigEvents(t *testing.T) { require.Equal(t, types.MetaNameAutoUpdateConfig, mockEmitter.LastEvent().(*apievents.AutoUpdateConfigCreate).Name) mockEmitter.Reset() - _, err = service.UpdateAutoUpdateConfig(localCtx, &autoupdatev1pb.UpdateAutoUpdateConfigRequest{Config: config}) + _, err = service.UpdateAutoUpdateConfig(ctx, &autoupdatev1pb.UpdateAutoUpdateConfigRequest{Config: config}) require.NoError(t, err) require.Len(t, mockEmitter.Events(), 1) require.Equal(t, libevents.AutoUpdateConfigUpdateEvent, mockEmitter.LastEvent().GetType()) @@ -311,7 +68,7 @@ func TestAutoUpdateConfigEvents(t *testing.T) { require.Equal(t, types.MetaNameAutoUpdateConfig, mockEmitter.LastEvent().(*apievents.AutoUpdateConfigUpdate).Name) mockEmitter.Reset() - _, err = service.UpsertAutoUpdateConfig(localCtx, &autoupdatev1pb.UpsertAutoUpdateConfigRequest{Config: config}) + _, err = service.UpsertAutoUpdateConfig(ctx, &autoupdatev1pb.UpsertAutoUpdateConfigRequest{Config: config}) require.NoError(t, err) require.Len(t, mockEmitter.Events(), 1) require.Equal(t, libevents.AutoUpdateConfigUpdateEvent, mockEmitter.LastEvent().GetType()) @@ -319,7 +76,7 @@ func TestAutoUpdateConfigEvents(t *testing.T) { require.Equal(t, types.MetaNameAutoUpdateConfig, mockEmitter.LastEvent().(*apievents.AutoUpdateConfigUpdate).Name) mockEmitter.Reset() - _, err = service.DeleteAutoUpdateConfig(localCtx, &autoupdatev1pb.DeleteAutoUpdateConfigRequest{}) + _, err = service.DeleteAutoUpdateConfig(ctx, &autoupdatev1pb.DeleteAutoUpdateConfigRequest{}) require.NoError(t, err) require.Len(t, mockEmitter.Events(), 1) require.Equal(t, libevents.AutoUpdateConfigDeleteEvent, mockEmitter.LastEvent().GetType()) @@ -329,15 +86,10 @@ func TestAutoUpdateConfigEvents(t *testing.T) { } func TestAutoUpdateVersionEvents(t *testing.T) { - role := types.RoleSpecV6{ - Allow: types.RoleConditions{Rules: []types.Rule{{ - Resources: []string{types.KindAutoUpdateVersion}, - Verbs: []string{types.VerbList, types.VerbCreate, types.VerbRead, types.VerbUpdate, types.VerbDelete}, - }}}, - } + rwVerbs := []string{types.VerbList, types.VerbCreate, types.VerbRead, types.VerbUpdate, types.VerbDelete} mockEmitter := &eventstest.MockRecorderEmitter{} - ctx, localClient, service := initSvc(t, "test-cluster", mockEmitter) - localCtx := authorizerForDummyUser(t, ctx, role, localClient) + service := newService(t, fakeChecker{allowedVerbs: rwVerbs}, mockEmitter) + ctx := context.Background() config, err := autoupdate.NewAutoUpdateVersion(&autoupdatev1pb.AutoUpdateVersionSpec{ Tools: &autoupdatev1pb.AutoUpdateVersionSpecTools{ @@ -346,7 +98,7 @@ func TestAutoUpdateVersionEvents(t *testing.T) { }) require.NoError(t, err) - _, err = service.CreateAutoUpdateVersion(localCtx, &autoupdatev1pb.CreateAutoUpdateVersionRequest{Version: config}) + _, err = service.CreateAutoUpdateVersion(ctx, &autoupdatev1pb.CreateAutoUpdateVersionRequest{Version: config}) require.NoError(t, err) require.Len(t, mockEmitter.Events(), 1) require.Equal(t, libevents.AutoUpdateVersionCreateEvent, mockEmitter.LastEvent().GetType()) @@ -354,7 +106,7 @@ func TestAutoUpdateVersionEvents(t *testing.T) { require.Equal(t, types.MetaNameAutoUpdateVersion, mockEmitter.LastEvent().(*apievents.AutoUpdateVersionCreate).Name) mockEmitter.Reset() - _, err = service.UpdateAutoUpdateVersion(localCtx, &autoupdatev1pb.UpdateAutoUpdateVersionRequest{Version: config}) + _, err = service.UpdateAutoUpdateVersion(ctx, &autoupdatev1pb.UpdateAutoUpdateVersionRequest{Version: config}) require.NoError(t, err) require.Len(t, mockEmitter.Events(), 1) require.Equal(t, libevents.AutoUpdateVersionUpdateEvent, mockEmitter.LastEvent().GetType()) @@ -362,7 +114,7 @@ func TestAutoUpdateVersionEvents(t *testing.T) { require.Equal(t, types.MetaNameAutoUpdateVersion, mockEmitter.LastEvent().(*apievents.AutoUpdateVersionUpdate).Name) mockEmitter.Reset() - _, err = service.UpsertAutoUpdateVersion(localCtx, &autoupdatev1pb.UpsertAutoUpdateVersionRequest{Version: config}) + _, err = service.UpsertAutoUpdateVersion(ctx, &autoupdatev1pb.UpsertAutoUpdateVersionRequest{Version: config}) require.NoError(t, err) require.Len(t, mockEmitter.Events(), 1) require.Equal(t, libevents.AutoUpdateVersionUpdateEvent, mockEmitter.LastEvent().GetType()) @@ -370,7 +122,7 @@ func TestAutoUpdateVersionEvents(t *testing.T) { require.Equal(t, types.MetaNameAutoUpdateVersion, mockEmitter.LastEvent().(*apievents.AutoUpdateVersionUpdate).Name) mockEmitter.Reset() - _, err = service.DeleteAutoUpdateVersion(localCtx, &autoupdatev1pb.DeleteAutoUpdateVersionRequest{}) + _, err = service.DeleteAutoUpdateVersion(ctx, &autoupdatev1pb.DeleteAutoUpdateVersionRequest{}) require.NoError(t, err) require.Len(t, mockEmitter.Events(), 1) require.Equal(t, libevents.AutoUpdateVersionDeleteEvent, mockEmitter.LastEvent().GetType()) @@ -379,101 +131,398 @@ func TestAutoUpdateVersionEvents(t *testing.T) { mockEmitter.Reset() } -func authorizerForDummyUser(t *testing.T, ctx context.Context, roleSpec types.RoleSpecV6, localClient localClient) context.Context { - // Create role - roleName := "role-" + uuid.NewString() - role, err := types.NewRole(roleName, roleSpec) - require.NoError(t, err) - - err = localClient.CreateRole(ctx, role) - require.NoError(t, err) +type fakeChecker struct { + allowedVerbs []string + builtinRole *authz.BuiltinRole + services.AccessChecker +} - // Create user - user, err := types.NewUser("user-" + uuid.NewString()) - require.NoError(t, err) - user.AddRole(roleName) - err = localClient.CreateUser(user) - require.NoError(t, err) +func (f fakeChecker) CheckAccessToRule(_ services.RuleContext, _ string, resource string, verb string) error { + if resource == types.KindAutoUpdateConfig || resource == types.KindAutoUpdateVersion || resource == types.KindAutoUpdateAgentRollout { + for _, allowedVerb := range f.allowedVerbs { + if allowedVerb == verb { + return nil + } + } + } - return authz.ContextWithUser(ctx, authz.LocalUser{ - Username: user.GetName(), - Identity: tlsca.Identity{ - Username: user.GetName(), - Groups: []string{role.GetName()}, - }, - }) + return trace.AccessDenied("access denied to rule=%v/verb=%v", resource, verb) } -type localClient interface { - CreateUser(user types.User) error - CreateRole(ctx context.Context, role types.Role) error - services.AutoUpdateService +func (f fakeChecker) HasRole(name string) bool { + if f.builtinRole == nil { + return false + } + return name == f.builtinRole.Role.String() } -func initSvc(t *testing.T, clusterName string, emitter apievents.Emitter) (context.Context, localClient, *Service) { - ctx := context.Background() - backend, err := memory.New(memory.Config{}) - require.NoError(t, err) +func (f fakeChecker) identityGetter() authz.IdentityGetter { + if f.builtinRole != nil { + return *f.builtinRole + } + return nil +} - trustSvc := local.NewCAService(backend) - roleSvc := local.NewAccessService(backend) - userSvc := local.NewTestIdentityService(backend) +func newService(t *testing.T, checker fakeChecker, emitter apievents.Emitter) *Service { + t.Helper() - clusterConfigSvc, err := local.NewClusterConfigurationService(backend) + bk, err := memory.New(memory.Config{}) require.NoError(t, err) - require.NoError(t, clusterConfigSvc.SetAuthPreference(ctx, types.DefaultAuthPreference())) - require.NoError(t, clusterConfigSvc.SetClusterAuditConfig(ctx, types.DefaultClusterAuditConfig())) - require.NoError(t, clusterConfigSvc.SetClusterNetworkingConfig(ctx, types.DefaultClusterNetworkingConfig())) - require.NoError(t, clusterConfigSvc.SetSessionRecordingConfig(ctx, types.DefaultSessionRecordingConfig())) - - accessPoint := struct { - services.ClusterConfiguration - services.Trust - services.RoleGetter - services.UserGetter - }{ - ClusterConfiguration: clusterConfigSvc, - Trust: trustSvc, - RoleGetter: roleSvc, - UserGetter: userSvc, - } - accessService := local.NewAccessService(backend) - eventService := local.NewEventsService(backend) - lockWatcher, err := services.NewLockWatcher(ctx, services.LockWatcherConfig{ - ResourceWatcherConfig: services.ResourceWatcherConfig{ - Client: eventService, - Component: "test", - }, - LockGetter: accessService, - }) + storage, err := local.NewAutoUpdateService(bk) require.NoError(t, err) - authorizer, err := authz.NewAuthorizer(authz.AuthorizerOpts{ - ClusterName: clusterName, - AccessPoint: accessPoint, - LockWatcher: lockWatcher, - }) - require.NoError(t, err) + return newServiceWithStorage(t, checker, storage, emitter) +} - localResourceService, err := local.NewAutoUpdateService(backend) - require.NoError(t, err) +func newServiceWithStorage(t *testing.T, checker fakeChecker, storage services.AutoUpdateService, emitter apievents.Emitter) *Service { + t.Helper() - resourceSvc, err := NewService(ServiceConfig{ - Backend: localResourceService, - Cache: localResourceService, + authorizer := authz.AuthorizerFunc(func(ctx context.Context) (*authz.Context, error) { + user, err := types.NewUser("alice") + if err != nil { + return nil, err + } + + return &authz.Context{ + User: user, + Checker: checker, + Identity: checker.identityGetter(), + }, nil + }) + + service, err := NewService(ServiceConfig{ Authorizer: authorizer, + Backend: storage, + Cache: storage, Emitter: emitter, }) require.NoError(t, err) + return service +} + +func TestComputeMinRolloutTime(t *testing.T) { + t.Parallel() + tests := []struct { + name string + groups []*autoupdatev1pb.AgentAutoUpdateGroup + expectedHours int + }{ + { + name: "nil groups", + groups: nil, + expectedHours: 0, + }, + { + name: "empty groups", + groups: []*autoupdatev1pb.AgentAutoUpdateGroup{}, + expectedHours: 0, + }, + { + name: "single group", + groups: []*autoupdatev1pb.AgentAutoUpdateGroup{ + { + Name: "g1", + }, + }, + expectedHours: 1, + }, + { + name: "two groups, same day, different start hour, no wait time", + groups: []*autoupdatev1pb.AgentAutoUpdateGroup{ + { + Name: "g1", + StartHour: 2, + }, + { + Name: "g2", + StartHour: 4, + }, + }, + // g1 updates from 2:00 to 3:00, g2 updates from 4:00 to 5:00, rollout updates from 2:00 to 5:00. + expectedHours: 3, + }, + { + name: "two groups, same day, same start hour, no wait time", + groups: []*autoupdatev1pb.AgentAutoUpdateGroup{ + { + Name: "g1", + StartHour: 2, + }, + { + Name: "g2", + StartHour: 2, + }, + }, + // g1 and g2 can't update at the same time, the g1 updates from 2:00 to 3:00 days one, + // and g2 updates from 2:00 to 3:00 the next day. Total update spans from 2:00 day 1, to 3:00 day 2 + expectedHours: 25, + }, + { + name: "two groups, cannot happen on the same day because of wait_hours", + groups: []*autoupdatev1pb.AgentAutoUpdateGroup{ + { + Name: "g1", + StartHour: 2, + }, + { + Name: "g2", + StartHour: 4, + WaitHours: 6, + }, + }, + // g1 updates from 2:00 to 3:00. At 4:00 g2 can't update yet, so we wait the next day. + // On day 2, g2 updates from 4:00 to 5:00. Rollout spans from 2:00 day on to 7:00 day 2. + expectedHours: 27, + }, + { + name: "two groups, wait hours is several days", + groups: []*autoupdatev1pb.AgentAutoUpdateGroup{ + { + Name: "g1", + StartHour: 2, + }, + { + Name: "g2", + StartHour: 4, + WaitHours: 48, + }, + }, + // g1 updates from 2:00 to 3:00. At 4:00 g2 can't update yet, so we wait 2 days. + // On day 3, g2 updates from 4:00 to 5:00. Rollout spans from 2:00 day on to 7:00 day 3. + expectedHours: 51, + }, + { + name: "two groups, one wait hour", + groups: []*autoupdatev1pb.AgentAutoUpdateGroup{ + { + Name: "g1", + StartHour: 2, + }, + { + Name: "g2", + StartHour: 3, + WaitHours: 1, + }, + }, + expectedHours: 2, + }, + { + name: "two groups different days", + groups: []*autoupdatev1pb.AgentAutoUpdateGroup{ + { + Name: "g1", + StartHour: 23, + }, + { + Name: "g2", + StartHour: 1, + }, + }, + expectedHours: 3, + }, + { + name: "two groups different days, hour diff == wait hours == 1 day", + groups: []*autoupdatev1pb.AgentAutoUpdateGroup{ + { + Name: "g1", + StartHour: 12, + }, + { + Name: "g2", + StartHour: 12, + WaitHours: 24, + }, + }, + expectedHours: 25, + }, + { + name: "two groups different days, hour diff == wait hours", + groups: []*autoupdatev1pb.AgentAutoUpdateGroup{ + { + Name: "g1", + StartHour: 12, + }, + { + Name: "g2", + StartHour: 11, + WaitHours: 23, + }, + }, + expectedHours: 24, + }, + { + name: "everything at once", + groups: []*autoupdatev1pb.AgentAutoUpdateGroup{ + { + Name: "g1", + StartHour: 23, + }, + { + Name: "g2", + StartHour: 1, + WaitHours: 4, + }, + { + Name: "g3", + StartHour: 1, + }, + { + Name: "g4", + StartHour: 10, + WaitHours: 6, + }, + }, + expectedHours: 60, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expectedHours, computeMinRolloutTime(tt.groups)) + }) + } +} + +func generateGroups(n int, days []string) []*autoupdatev1pb.AgentAutoUpdateGroup { + groups := make([]*autoupdatev1pb.AgentAutoUpdateGroup, n) + for i := range groups { + groups[i] = &autoupdatev1pb.AgentAutoUpdateGroup{ + Name: strconv.Itoa(i), + Days: days, + StartHour: int32(i % 24), + } + } + return groups +} - return ctx, struct { - *local.AccessService - *local.IdentityService - *local.AutoUpdateService +func TestValidateServerSideAgentConfig(t *testing.T) { + cloudModules := &modules.TestModules{ + TestFeatures: modules.Features{ + Cloud: true, + }, + } + selfHostedModules := &modules.TestModules{ + TestFeatures: modules.Features{ + Cloud: false, + }, + } + tests := []struct { + name string + config *autoupdatev1pb.AutoUpdateConfigSpecAgents + modules modules.Modules + expectErr require.ErrorAssertionFunc }{ - AccessService: roleSvc, - IdentityService: userSvc, - AutoUpdateService: localResourceService, - }, resourceSvc + { + name: "empty agent config", + modules: selfHostedModules, + config: nil, + expectErr: require.NoError, + }, + { + name: "over max groups time-based", + modules: selfHostedModules, + config: &autoupdatev1pb.AutoUpdateConfigSpecAgents{ + Mode: autoupdate.AgentsUpdateModeEnabled, + Strategy: autoupdate.AgentsStrategyTimeBased, + MaintenanceWindowDuration: durationpb.New(time.Hour), + Schedules: &autoupdatev1pb.AgentAutoUpdateSchedules{ + Regular: generateGroups(maxGroupsTimeBasedStrategy+1, cloudGroupUpdateDays), + }, + }, + expectErr: require.Error, + }, + { + name: "over max groups halt-on-error", + modules: selfHostedModules, + config: &autoupdatev1pb.AutoUpdateConfigSpecAgents{ + Mode: autoupdate.AgentsUpdateModeEnabled, + Strategy: autoupdate.AgentsStrategyHaltOnError, + Schedules: &autoupdatev1pb.AgentAutoUpdateSchedules{ + Regular: generateGroups(maxGroupsHaltOnErrorStrategy+1, cloudGroupUpdateDays), + }, + }, + expectErr: require.Error, + }, + { + name: "over max groups halt-on-error cloud", + modules: cloudModules, + config: &autoupdatev1pb.AutoUpdateConfigSpecAgents{ + Mode: autoupdate.AgentsUpdateModeEnabled, + Strategy: autoupdate.AgentsStrategyHaltOnError, + Schedules: &autoupdatev1pb.AgentAutoUpdateSchedules{ + Regular: generateGroups(maxGroupsHaltOnErrorStrategyCloud+1, cloudGroupUpdateDays), + }, + }, + expectErr: require.Error, + }, + { + name: "cloud should reject custom weekdays", + modules: cloudModules, + config: &autoupdatev1pb.AutoUpdateConfigSpecAgents{ + Mode: autoupdate.AgentsUpdateModeEnabled, + Strategy: autoupdate.AgentsStrategyHaltOnError, + Schedules: &autoupdatev1pb.AgentAutoUpdateSchedules{ + Regular: generateGroups(maxGroupsHaltOnErrorStrategyCloud, []string{"Mon"}), + }, + }, + expectErr: require.Error, + }, + { + name: "self-hosted should allow custom weekdays", + modules: selfHostedModules, + config: &autoupdatev1pb.AutoUpdateConfigSpecAgents{ + Mode: autoupdate.AgentsUpdateModeEnabled, + Strategy: autoupdate.AgentsStrategyHaltOnError, + Schedules: &autoupdatev1pb.AgentAutoUpdateSchedules{ + Regular: generateGroups(maxGroupsHaltOnErrorStrategyCloud, []string{"Mon"}), + }, + }, + expectErr: require.NoError, + }, + { + name: "cloud should reject long rollouts", + modules: cloudModules, + config: &autoupdatev1pb.AutoUpdateConfigSpecAgents{ + Mode: autoupdate.AgentsUpdateModeEnabled, + Strategy: autoupdate.AgentsStrategyHaltOnError, + Schedules: &autoupdatev1pb.AgentAutoUpdateSchedules{ + Regular: []*autoupdatev1pb.AgentAutoUpdateGroup{ + {Name: "g1", Days: cloudGroupUpdateDays}, + {Name: "g2", Days: cloudGroupUpdateDays, WaitHours: maxRolloutDurationCloudHours}, + }, + }, + }, + expectErr: require.Error, + }, + { + name: "self-hosted should allow long rollouts", + modules: selfHostedModules, + config: &autoupdatev1pb.AutoUpdateConfigSpecAgents{ + Mode: autoupdate.AgentsUpdateModeEnabled, + Strategy: autoupdate.AgentsStrategyHaltOnError, + Schedules: &autoupdatev1pb.AgentAutoUpdateSchedules{ + Regular: []*autoupdatev1pb.AgentAutoUpdateGroup{ + {Name: "g1", Days: cloudGroupUpdateDays}, + {Name: "g2", Days: cloudGroupUpdateDays, WaitHours: maxRolloutDurationCloudHours}, + }, + }, + }, + expectErr: require.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test setup: crafing a config and setting modules + config, err := autoupdate.NewAutoUpdateConfig( + &autoupdatev1pb.AutoUpdateConfigSpec{ + Tools: nil, + Agents: tt.config, + }) + require.NoError(t, err) + modules.SetTestModules(t, tt.modules) + + // Test execution. + tt.expectErr(t, validateServerSideAgentConfig(config)) + }) + } } diff --git a/lib/auth/grpcserver.go b/lib/auth/grpcserver.go index 3482ffef13f83..e61cd580534c6 100644 --- a/lib/auth/grpcserver.go +++ b/lib/auth/grpcserver.go @@ -4919,6 +4919,18 @@ func (g *GRPCServer) DeleteUIConfig(ctx context.Context, _ *emptypb.Empty) (*emp return &emptypb.Empty{}, nil } +func (g *GRPCServer) defaultInstaller(ctx context.Context) (*types.InstallerV1, error) { + _, err := g.AuthServer.GetAutoUpdateAgentRollout(ctx) + switch { + case trace.IsNotFound(err): + return installers.LegacyDefaultInstaller, nil + case err != nil: + return nil, trace.Wrap(err, "failed to get query autoupdate state to build installer") + default: + return installers.NewDefaultInstaller, nil + } +} + // GetInstaller retrieves the installer script resource func (g *GRPCServer) GetInstaller(ctx context.Context, req *types.ResourceRequest) (*types.InstallerV1, error) { auth, err := g.authenticate(ctx) @@ -4930,7 +4942,7 @@ func (g *GRPCServer) GetInstaller(ctx context.Context, req *types.ResourceReques if trace.IsNotFound(err) { switch req.Name { case installers.InstallerScriptName: - return installers.DefaultInstaller, nil + return g.defaultInstaller(ctx) case installers.InstallerScriptNameAgentless: return installers.DefaultAgentlessInstaller, nil } @@ -4955,8 +4967,14 @@ func (g *GRPCServer) GetInstallers(ctx context.Context, _ *emptypb.Empty) (*type return nil, trace.Wrap(err) } var installersV1 []*types.InstallerV1 + + defaultInstaller, err := g.defaultInstaller(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + defaultInstallers := map[string]*types.InstallerV1{ - installers.InstallerScriptName: installers.DefaultInstaller, + types.DefaultInstallerScriptName: defaultInstaller, installers.InstallerScriptNameAgentless: installers.DefaultAgentlessInstaller, } diff --git a/lib/auth/grpcserver_test.go b/lib/auth/grpcserver_test.go index 06ba85359f390..c9ab8f60485bf 100644 --- a/lib/auth/grpcserver_test.go +++ b/lib/auth/grpcserver_test.go @@ -48,6 +48,7 @@ import ( otlpresourcev1 "go.opentelemetry.io/proto/otlp/resource/v1" otlptracev1 "go.opentelemetry.io/proto/otlp/trace/v1" "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/emptypb" "github.com/gravitational/teleport" @@ -55,11 +56,13 @@ import ( "github.com/gravitational/teleport/api/client/proto" "github.com/gravitational/teleport/api/constants" apidefaults "github.com/gravitational/teleport/api/defaults" + autoupdatev1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" clusterconfigpb "github.com/gravitational/teleport/api/gen/proto/go/teleport/clusterconfig/v1" "github.com/gravitational/teleport/api/internalutils/stream" "github.com/gravitational/teleport/api/metadata" "github.com/gravitational/teleport/api/observability/tracing" "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/api/types/autoupdate" "github.com/gravitational/teleport/api/types/installers" "github.com/gravitational/teleport/api/utils" "github.com/gravitational/teleport/api/utils/keys" @@ -4121,12 +4124,21 @@ func TestGRPCServer_GetInstallers(t *testing.T) { tests := []struct { name string inputInstallers map[string]string + hasAgentRollout bool expectedInstallers map[string]string }{ { name: "default installers only", expectedInstallers: map[string]string{ - installers.InstallerScriptName: installers.DefaultInstaller.GetScript(), + types.DefaultInstallerScriptName: installers.LegacyDefaultInstaller.GetScript(), + installers.InstallerScriptNameAgentless: installers.DefaultAgentlessInstaller.GetScript(), + }, + }, + { + name: "new default installers", + hasAgentRollout: true, + expectedInstallers: map[string]string{ + types.DefaultInstallerScriptName: installers.NewDefaultInstaller.GetScript(), installers.InstallerScriptNameAgentless: installers.DefaultAgentlessInstaller.GetScript(), }, }, @@ -4137,7 +4149,7 @@ func TestGRPCServer_GetInstallers(t *testing.T) { }, expectedInstallers: map[string]string{ "my-custom-installer": "echo test", - installers.InstallerScriptName: installers.DefaultInstaller.GetScript(), + types.DefaultInstallerScriptName: installers.LegacyDefaultInstaller.GetScript(), installers.InstallerScriptNameAgentless: installers.DefaultAgentlessInstaller.GetScript(), }, }, @@ -4159,6 +4171,25 @@ func TestGRPCServer_GetInstallers(t *testing.T) { require.NoError(t, err) }) + if tc.hasAgentRollout { + rollout, err := autoupdate.NewAutoUpdateAgentRollout( + &autoupdatev1pb.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: autoupdate.AgentsScheduleImmediate, + AutoupdateMode: autoupdate.AgentsUpdateModeEnabled, + Strategy: autoupdate.AgentsStrategyTimeBased, + MaintenanceWindowDuration: durationpb.New(1 * time.Hour), + }) + require.NoError(t, err) + _, err = grpc.AuthServer.CreateAutoUpdateAgentRollout(ctx, rollout) + require.NoError(t, err) + + t.Cleanup(func() { + assert.NoError(t, grpc.AuthServer.DeleteAutoUpdateAgentRollout(ctx)) + }) + } + for name, script := range tc.inputInstallers { installer, err := types.NewInstallerV1(name, script) require.NoError(t, err) diff --git a/lib/auth/init_test.go b/lib/auth/init_test.go index bcc2a1c3c8f78..80bdd5c6ccf00 100644 --- a/lib/auth/init_test.go +++ b/lib/auth/init_test.go @@ -1452,6 +1452,7 @@ func TestSyncUpgradeWindowStartHour(t *testing.T) { require.True(t, ok) require.Equal(t, uint32(0), agentWindow.UTCStartHour) + require.Equal(t, []string{"Mon", "Tue", "Wed", "Thu"}, agentWindow.Weekdays) // change the served hour mu.Lock() diff --git a/lib/auth/join/join.go b/lib/auth/join/join.go index d6da0da214b39..dd432dda57135 100644 --- a/lib/auth/join/join.go +++ b/lib/auth/join/join.go @@ -52,7 +52,7 @@ import ( "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/githubactions" "github.com/gravitational/teleport/lib/gitlab" - "github.com/gravitational/teleport/lib/kubernetestoken" + kubetoken "github.com/gravitational/teleport/lib/kube/token" "github.com/gravitational/teleport/lib/spacelift" "github.com/gravitational/teleport/lib/srv/alpnproxy/common" "github.com/gravitational/teleport/lib/tlsca" @@ -201,7 +201,7 @@ func Register(ctx context.Context, params RegisterParams) (certs *proto.Certs, e return nil, trace.Wrap(err) } } else if params.JoinMethod == types.JoinMethodKubernetes { - params.IDToken, err = kubernetestoken.GetIDToken(os.Getenv, os.ReadFile) + params.IDToken, err = kubetoken.GetIDToken(os.Getenv, os.ReadFile) if err != nil { return nil, trace.Wrap(err) } diff --git a/lib/auth/join_kubernetes.go b/lib/auth/join_kubernetes.go index afbd0e3cc759e..819b992b8067a 100644 --- a/lib/auth/join_kubernetes.go +++ b/lib/auth/join_kubernetes.go @@ -25,16 +25,16 @@ import ( "github.com/sirupsen/logrus" "github.com/gravitational/teleport/api/types" - "github.com/gravitational/teleport/lib/kubernetestoken" + kubetoken "github.com/gravitational/teleport/lib/kube/token" ) type k8sTokenReviewValidator interface { - Validate(context.Context, string) (*kubernetestoken.ValidationResult, error) + Validate(context.Context, string) (*kubetoken.ValidationResult, error) } -type k8sJWKSValidator func(now time.Time, jwksData []byte, clusterName string, token string) (*kubernetestoken.ValidationResult, error) +type k8sJWKSValidator func(now time.Time, jwksData []byte, clusterName string, token string) (*kubetoken.ValidationResult, error) -func (a *Server) checkKubernetesJoinRequest(ctx context.Context, req *types.RegisterUsingTokenRequest) (*kubernetestoken.ValidationResult, error) { +func (a *Server) checkKubernetesJoinRequest(ctx context.Context, req *types.RegisterUsingTokenRequest) (*kubetoken.ValidationResult, error) { if req.IDToken == "" { return nil, trace.BadParameter("IDToken not provided for Kubernetes join request") } @@ -51,7 +51,7 @@ func (a *Server) checkKubernetesJoinRequest(ctx context.Context, req *types.Regi } // Switch to join method subtype token validation. - var result *kubernetestoken.ValidationResult + var result *kubetoken.ValidationResult switch token.Spec.Kubernetes.Type { case types.KubernetesJoinTypeStaticJWKS: clusterName, err := a.GetDomainName() @@ -87,10 +87,10 @@ func (a *Server) checkKubernetesJoinRequest(ctx context.Context, req *types.Regi return result, trace.Wrap(checkKubernetesAllowRules(token, result)) } -func checkKubernetesAllowRules(pt *types.ProvisionTokenV2, got *kubernetestoken.ValidationResult) error { +func checkKubernetesAllowRules(pt *types.ProvisionTokenV2, got *kubetoken.ValidationResult) error { // If a single rule passes, accept the token for _, rule := range pt.Spec.Kubernetes.Allow { - wantUsername := fmt.Sprintf("%s:%s", kubernetestoken.ServiceAccountNamePrefix, rule.ServiceAccount) + wantUsername := fmt.Sprintf("%s:%s", kubetoken.ServiceAccountNamePrefix, rule.ServiceAccount) if wantUsername != got.Username { continue } diff --git a/lib/auth/join_kubernetes_test.go b/lib/auth/join_kubernetes_test.go index ea9a02b4f45ca..36572e486e974 100644 --- a/lib/auth/join_kubernetes_test.go +++ b/lib/auth/join_kubernetes_test.go @@ -26,14 +26,14 @@ import ( "github.com/gravitational/teleport/api/types" "github.com/gravitational/teleport/lib/auth/testauthority" - "github.com/gravitational/teleport/lib/kubernetestoken" + kubetoken "github.com/gravitational/teleport/lib/kube/token" ) type mockK8STokenReviewValidator struct { - tokens map[string]*kubernetestoken.ValidationResult + tokens map[string]*kubetoken.ValidationResult } -func (m *mockK8STokenReviewValidator) Validate(_ context.Context, token string) (*kubernetestoken.ValidationResult, error) { +func (m *mockK8STokenReviewValidator) Validate(_ context.Context, token string) (*kubetoken.ValidationResult, error) { result, ok := m.tokens[token] if !ok { return nil, errMockInvalidToken @@ -46,14 +46,14 @@ func TestAuth_RegisterUsingToken_Kubernetes(t *testing.T) { // Test setup // Creating an auth server with mock Kubernetes token validator - tokenReviewTokens := map[string]*kubernetestoken.ValidationResult{ + tokenReviewTokens := map[string]*kubetoken.ValidationResult{ "matching-implicit-in-cluster": {Username: "system:serviceaccount:namespace1:service-account1"}, // "matching-explicit-in-cluster" intentionally matches the second allow // rule of explicitInCluster to ensure all rules are processed. "matching-explicit-in-cluster": {Username: "system:serviceaccount:namespace2:service-account2"}, "user-token": {Username: "namespace1:service-account1"}, } - jwksTokens := map[string]*kubernetestoken.ValidationResult{ + jwksTokens := map[string]*kubetoken.ValidationResult{ "jwks-matching-service-account": {Username: "system:serviceaccount:static-jwks:matching"}, "jwks-mismatched-service-account": {Username: "system:serviceaccount:static-jwks:mismatched"}, } @@ -61,7 +61,7 @@ func TestAuth_RegisterUsingToken_Kubernetes(t *testing.T) { ctx := context.Background() p, err := newTestPack(ctx, t.TempDir(), func(server *Server) error { server.k8sTokenReviewValidator = &mockK8STokenReviewValidator{tokens: tokenReviewTokens} - server.k8sJWKSValidator = func(_ time.Time, _ []byte, _ string, token string) (*kubernetestoken.ValidationResult, error) { + server.k8sJWKSValidator = func(_ time.Time, _ []byte, _ string, token string) (*kubetoken.ValidationResult, error) { result, ok := jwksTokens[token] if !ok { return nil, errMockInvalidToken diff --git a/lib/authz/permissions.go b/lib/authz/permissions.go index 717d3bc27bd84..1b6d37a1043dc 100644 --- a/lib/authz/permissions.go +++ b/lib/authz/permissions.go @@ -690,6 +690,7 @@ func roleSpecForProxy(clusterName string) types.RoleSpecV6 { types.NewRule(types.KindClusterMaintenanceConfig, services.RO()), types.NewRule(types.KindAutoUpdateConfig, services.RO()), types.NewRule(types.KindAutoUpdateVersion, services.RO()), + types.NewRule(types.KindAutoUpdateAgentRollout, services.RO()), types.NewRule(types.KindIntegration, append(services.RO(), types.VerbUse)), types.NewRule(types.KindAuditQuery, services.RO()), types.NewRule(types.KindSecurityReport, services.RO()), diff --git a/lib/automaticupgrades/channel.go b/lib/automaticupgrades/channel.go index 275f8d93b69d1..347a268ec5e2f 100644 --- a/lib/automaticupgrades/channel.go +++ b/lib/automaticupgrades/channel.go @@ -99,6 +99,15 @@ func (c Channels) DefaultVersion(ctx context.Context) (string, error) { return targetVersion, trace.Wrap(err) } +// DefaultChannel returns the default upgrade channel. +func (c Channels) DefaultChannel() (*Channel, error) { + defaultChannel, ok := c[DefaultChannelName] + if ok && defaultChannel != nil { + return defaultChannel, nil + } + return NewDefaultChannel() +} + // Channel describes an automatic update channel configuration. // It can be configured to serve a static version, or forward version requests // to an upstream version server. Forwarded results are cached for 1 minute. diff --git a/lib/automaticupgrades/maintenance/mock.go b/lib/automaticupgrades/maintenance/mock.go index a777bfca0764c..76f538f412357 100644 --- a/lib/automaticupgrades/maintenance/mock.go +++ b/lib/automaticupgrades/maintenance/mock.go @@ -27,6 +27,7 @@ import ( type StaticTrigger struct { name string canStart bool + err error } // Name returns the StaticTrigger name. @@ -36,7 +37,7 @@ func (m StaticTrigger) Name() string { // CanStart returns the statically defined maintenance approval result. func (m StaticTrigger) CanStart(_ context.Context, _ client.Object) (bool, error) { - return m.canStart, nil + return m.canStart, m.err } // Default returns the default behavior if the trigger fails. This cannot diff --git a/lib/automaticupgrades/maintenance/proxy.go b/lib/automaticupgrades/maintenance/proxy.go new file mode 100644 index 0000000000000..ceb2495e5c17a --- /dev/null +++ b/lib/automaticupgrades/maintenance/proxy.go @@ -0,0 +1,85 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package maintenance + +import ( + "context" + + "github.com/gravitational/trace" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/gravitational/teleport/api/client/webclient" + "github.com/gravitational/teleport/lib/automaticupgrades/cache" + "github.com/gravitational/teleport/lib/automaticupgrades/constants" +) + +type proxyMaintenanceClient struct { + client *webclient.ReusableClient +} + +// Get does the HTTPS call to the Teleport Proxy sevrice to check if the update should happen now. +// If the proxy response does not contain the auto_update.agent_version field, +// this means the proxy does not support autoupdates. In this case we return trace.NotImplementedErr. +func (b *proxyMaintenanceClient) Get(ctx context.Context) (bool, error) { + resp, err := b.client.Find() + if err != nil { + return false, trace.Wrap(err) + } + // We check if a version is advertised to know if the proxy implements RFD-184 or not. + if resp.AutoUpdate.AgentVersion == "" { + return false, trace.NotImplemented("proxy does not seem to implement RFD-184") + } + return resp.AutoUpdate.AgentAutoUpdate, nil +} + +// ProxyMaintenanceTrigger checks if the maintenance should be triggered from the Teleport Proxy service /find endpoint, +// as specified in the RFD-184: https://github.com/gravitational/teleport/blob/master/rfd/0184-agent-auto-updates.md +// The Trigger returns trace.NotImplementedErr when running against a proxy that does not seem to +// expose automatic update instructions over the /find endpoint (proxy too old). +type ProxyMaintenanceTrigger struct { + name string + cachedGetter func(context.Context) (bool, error) +} + +// Name implements maintenance.Trigger returns the trigger name for logging +// and debugging purposes. +func (g ProxyMaintenanceTrigger) Name() string { + return g.name +} + +// Default implements maintenance.Trigger and returns what to do if the trigger can't be evaluated. +// ProxyMaintenanceTrigger should fail open, so the function returns true. +func (g ProxyMaintenanceTrigger) Default() bool { + return false +} + +// CanStart implements maintenance.Trigger. +func (g ProxyMaintenanceTrigger) CanStart(ctx context.Context, _ client.Object) (bool, error) { + result, err := g.cachedGetter(ctx) + return result, trace.Wrap(err) +} + +// NewProxyMaintenanceTrigger builds and return a Trigger checking a public HTTP endpoint. +func NewProxyMaintenanceTrigger(name string, clt *webclient.ReusableClient) Trigger { + maintenanceClient := &proxyMaintenanceClient{ + client: clt, + } + + return ProxyMaintenanceTrigger{name, cache.NewTimedMemoize[bool](maintenanceClient.Get, constants.CacheDuration).Get} +} diff --git a/lib/automaticupgrades/maintenance/trigger.go b/lib/automaticupgrades/maintenance/trigger.go index 457078f34fa49..d32e8a713e3ea 100644 --- a/lib/automaticupgrades/maintenance/trigger.go +++ b/lib/automaticupgrades/maintenance/trigger.go @@ -18,7 +18,9 @@ package maintenance import ( "context" + "strings" + "github.com/gravitational/trace" "sigs.k8s.io/controller-runtime/pkg/client" ctrllog "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -49,7 +51,10 @@ func (t Triggers) CanStart(ctx context.Context, object client.Object) bool { start, err := trigger.CanStart(ctx, object) if err != nil { start = trigger.Default() - log.Error(err, "trigger failed to evaluate, using its default value", "trigger", trigger.Name(), "defaultValue", start) + log.Error( + err, "trigger failed to evaluate, using its default value", "trigger", trigger.Name(), "defaultValue", + start, + ) } else { log.Info("trigger evaluated", "trigger", trigger.Name(), "result", start) } @@ -60,3 +65,48 @@ func (t Triggers) CanStart(ctx context.Context, object client.Object) bool { } return false } + +// FailoverTrigger wraps multiple Triggers and tries them sequentially. +// Any error is considered fatal, except for the trace.NotImplementedErr +// which indicates the trigger is not supported yet and we should +// failover to the next trigger. +type FailoverTrigger []Trigger + +// Name implements Trigger +func (f FailoverTrigger) Name() string { + names := make([]string, len(f)) + for i, t := range f { + names[i] = t.Name() + } + + return strings.Join(names, ", failover ") +} + +// CanStart implements Trigger +// Triggers are evaluated sequentially, the result of the first trigger not returning +// trace.NotImplementedErr is used. +func (f FailoverTrigger) CanStart(ctx context.Context, object client.Object) (bool, error) { + for _, trigger := range f { + canStart, err := trigger.CanStart(ctx, object) + switch { + case err == nil: + return canStart, nil + case trace.IsNotImplemented(err): + continue + default: + return false, trace.Wrap(err) + } + } + return false, trace.NotFound("every trigger returned NotImplemented") +} + +// Default implements Trigger. +// The default is the logical OR of every Trigger.Default. +func (f FailoverTrigger) Default() bool { + for _, trigger := range f { + if trigger.Default() { + return true + } + } + return false +} diff --git a/lib/automaticupgrades/maintenance/trigger_test.go b/lib/automaticupgrades/maintenance/trigger_test.go new file mode 100644 index 0000000000000..435b73f0f9bc4 --- /dev/null +++ b/lib/automaticupgrades/maintenance/trigger_test.go @@ -0,0 +1,169 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package maintenance + +import ( + "context" + "testing" + + "github.com/gravitational/trace" + "github.com/stretchr/testify/require" +) + +// checkTraceError is a test helper that converts trace.IsXXXError into a require.ErrorAssertionFunc +func checkTraceError(check func(error) bool) require.ErrorAssertionFunc { + return func(t require.TestingT, err error, i ...interface{}) { + require.True(t, check(err), i...) + } +} + +func TestFailoverTrigger_CanStart(t *testing.T) { + t.Parallel() + + // Test setup + ctx := context.Background() + tests := []struct { + name string + triggers []Trigger + expectResult bool + expectErr require.ErrorAssertionFunc + }{ + { + name: "nil", + triggers: nil, + expectResult: false, + expectErr: checkTraceError(trace.IsNotFound), + }, + { + name: "empty", + triggers: []Trigger{}, + expectResult: false, + expectErr: checkTraceError(trace.IsNotFound), + }, + { + name: "first trigger success firing", + triggers: []Trigger{ + StaticTrigger{canStart: true}, + StaticTrigger{canStart: false}, + }, + expectResult: true, + expectErr: require.NoError, + }, + { + name: "first trigger success not firing", + triggers: []Trigger{ + StaticTrigger{canStart: false}, + StaticTrigger{canStart: true}, + }, + expectResult: false, + expectErr: require.NoError, + }, + { + name: "first trigger failure", + triggers: []Trigger{ + StaticTrigger{err: trace.LimitExceeded("got rate-limited")}, + StaticTrigger{canStart: true}, + }, + expectResult: false, + expectErr: checkTraceError(trace.IsLimitExceeded), + }, + { + name: "first trigger skipped, second getter success", + triggers: []Trigger{ + StaticTrigger{err: trace.NotImplemented("proxy does not seem to implement RFD-184")}, + StaticTrigger{canStart: true}, + }, + expectResult: true, + expectErr: require.NoError, + }, + { + name: "first trigger skipped, second getter failure", + triggers: []Trigger{ + StaticTrigger{err: trace.NotImplemented("proxy does not seem to implement RFD-184")}, + StaticTrigger{err: trace.LimitExceeded("got rate-limited")}, + }, + expectResult: false, + expectErr: checkTraceError(trace.IsLimitExceeded), + }, + { + name: "first trigger skipped, second getter skipped", + triggers: []Trigger{ + StaticTrigger{err: trace.NotImplemented("proxy does not seem to implement RFD-184")}, + StaticTrigger{err: trace.NotImplemented("proxy does not seem to implement RFD-184")}, + }, + expectResult: false, + expectErr: checkTraceError(trace.IsNotFound), + }, + } + for _, tt := range tests { + t.Run( + tt.name, func(t *testing.T) { + // Test execution + trigger := FailoverTrigger(tt.triggers) + result, err := trigger.CanStart(ctx, nil) + require.Equal(t, tt.expectResult, result) + tt.expectErr(t, err) + }, + ) + } +} + +func TestFailoverTrigger_Name(t *testing.T) { + tests := []struct { + name string + triggers []Trigger + expectResult string + }{ + { + name: "nil", + triggers: nil, + expectResult: "", + }, + { + name: "empty", + triggers: []Trigger{}, + expectResult: "", + }, + { + name: "one trigger", + triggers: []Trigger{ + StaticTrigger{name: "proxy"}, + }, + expectResult: "proxy", + }, + { + name: "two triggers", + triggers: []Trigger{ + StaticTrigger{name: "proxy"}, + StaticTrigger{name: "version-server"}, + }, + expectResult: "proxy, failover version-server", + }, + } + for _, tt := range tests { + t.Run( + tt.name, func(t *testing.T) { + // Test execution + trigger := FailoverTrigger(tt.triggers) + result := trigger.Name() + require.Equal(t, tt.expectResult, result) + }, + ) + } +} diff --git a/lib/automaticupgrades/version/proxy.go b/lib/automaticupgrades/version/proxy.go new file mode 100644 index 0000000000000..90ec4859586e2 --- /dev/null +++ b/lib/automaticupgrades/version/proxy.go @@ -0,0 +1,72 @@ +/* + * Teleport + * Copyright (C) 2023 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package version + +import ( + "context" + + "github.com/gravitational/trace" + + "github.com/gravitational/teleport/api/client/webclient" + "github.com/gravitational/teleport/lib/automaticupgrades/cache" + "github.com/gravitational/teleport/lib/automaticupgrades/constants" +) + +type Finder interface { + Find() (*webclient.PingResponse, error) +} + +type proxyVersionClient struct { + client Finder +} + +func (b *proxyVersionClient) Get(_ context.Context) (string, error) { + resp, err := b.client.Find() + if err != nil { + return "", trace.Wrap(err) + } + // We check if a version is advertised to know if the proxy implements RFD-184 or not. + if resp.AutoUpdate.AgentVersion == "" { + return "", trace.NotImplemented("proxy does not seem to implement RFD-184") + } + return EnsureSemver(resp.AutoUpdate.AgentVersion) +} + +// ProxyVersionGetter gets the target version from the Teleport Proxy Service /find endpoint, as +// specified in the RFD-184: https://github.com/gravitational/teleport/blob/master/rfd/0184-agent-auto-updates.md +// The Getter returns trace.NotImplementedErr when running against a proxy that does not seem to +// expose automatic update instructions over the /find endpoint (proxy too old). +type ProxyVersionGetter struct { + cachedGetter func(context.Context) (string, error) +} + +// GetVersion implements Getter +func (g ProxyVersionGetter) GetVersion(ctx context.Context) (string, error) { + return g.cachedGetter(ctx) +} + +// NewProxyVersionGetter creates a ProxyVersionGetter from a webclient. +// The answer is cached for a minute. +func NewProxyVersionGetter(clt *webclient.ReusableClient) Getter { + versionClient := &proxyVersionClient{ + client: clt, + } + + return ProxyVersionGetter{cache.NewTimedMemoize[string](versionClient.Get, constants.CacheDuration).Get} +} diff --git a/lib/automaticupgrades/version/proxy_test.go b/lib/automaticupgrades/version/proxy_test.go new file mode 100644 index 0000000000000..2360f271c25a1 --- /dev/null +++ b/lib/automaticupgrades/version/proxy_test.go @@ -0,0 +1,116 @@ +/* + * Teleport + * Copyright (C) 2025 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package version + +import ( + "context" + "testing" + + "github.com/gravitational/trace" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/gravitational/teleport/api/client/webclient" +) + +type mockWebClient struct { + mock.Mock +} + +func (m *mockWebClient) Find() (*webclient.PingResponse, error) { + args := m.Called() + return args.Get(0).(*webclient.PingResponse), args.Error(1) +} + +func TestProxyVersionClient(t *testing.T) { + ctx := context.Background() + tests := []struct { + name string + pong *webclient.PingResponse + pongErr error + expectedVersion string + expectErr require.ErrorAssertionFunc + }{ + { + name: "semver without leading v", + pong: &webclient.PingResponse{ + AutoUpdate: webclient.AutoUpdateSettings{ + AgentVersion: "1.2.3", + }, + }, + expectedVersion: "v1.2.3", + expectErr: require.NoError, + }, + { + name: "semver with leading v", + pong: &webclient.PingResponse{ + AutoUpdate: webclient.AutoUpdateSettings{ + AgentVersion: "v1.2.3", + }, + }, + expectedVersion: "v1.2.3", + expectErr: require.NoError, + }, + { + name: "semver with prerelease and no leading v", + pong: &webclient.PingResponse{ + AutoUpdate: webclient.AutoUpdateSettings{ + AgentVersion: "1.2.3-dev.bartmoss.1", + }, + }, + expectedVersion: "v1.2.3-dev.bartmoss.1", + expectErr: require.NoError, + }, + { + name: "invalid semver", + pong: &webclient.PingResponse{ + AutoUpdate: webclient.AutoUpdateSettings{ + AgentVersion: "v", + }, + }, + expectedVersion: "", + expectErr: require.Error, + }, + { + name: "empty response", + pong: &webclient.PingResponse{}, + expectedVersion: "", + expectErr: func(t require.TestingT, err error, i ...interface{}) { + require.ErrorIs(t, err, trace.NotImplemented("proxy does not seem to implement RFD-184")) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test setup: create mock and load fixtures. + webClient := &mockWebClient{} + webClient.On("Find").Once().Return(tt.pong, tt.pongErr) + + // Test execution. + clt := proxyVersionClient{client: webClient} + v, err := clt.Get(ctx) + + // Test validation. + tt.expectErr(t, err) + require.Equal(t, tt.expectedVersion, v) + webClient.AssertExpectations(t) + }) + } +} diff --git a/lib/automaticupgrades/version/versionget.go b/lib/automaticupgrades/version/versionget.go index 00ffdc31fda3f..774921358db18 100644 --- a/lib/automaticupgrades/version/versionget.go +++ b/lib/automaticupgrades/version/versionget.go @@ -34,13 +34,42 @@ type Getter interface { GetVersion(context.Context) (string, error) } +// FailoverGetter wraps multiple Getters and tries them sequentially. +// Any error is considered fatal, except for the trace.NotImplementedErr +// which indicates the version getter is not supported yet and we should +// failover to the next version getter. +type FailoverGetter []Getter + +// GetVersion implements Getter +// Getters are evaluated sequentially, the result of the first getter not returning +// trace.NotImplementedErr is used. +func (f FailoverGetter) GetVersion(ctx context.Context) (string, error) { + for _, getter := range f { + version, err := getter.GetVersion(ctx) + switch { + case err == nil: + return version, nil + case trace.IsNotImplemented(err): + continue + default: + return "", trace.Wrap(err) + } + } + return "", trace.NotFound("every versionGetter returned NotImplemented") +} + // ValidVersionChange receives the current version and the candidate next version // and evaluates if the version transition is valid. func ValidVersionChange(ctx context.Context, current, next string) bool { log := ctrllog.FromContext(ctx).V(1) // Cannot upgrade to a non-valid version if !semver.IsValid(next) { - log.Error(trace.BadParameter("next version is not following semver"), "version change is invalid", "nextVersion", next) + log.Error( + trace.BadParameter("next version is not following semver"), + "version change is invalid", + "current_version", current, + "next_version", next, + ) return false } switch semver.Compare(next, current) { diff --git a/lib/automaticupgrades/version/versionget_test.go b/lib/automaticupgrades/version/versionget_test.go index f1d1b46fe56bf..3456c2e70e95d 100644 --- a/lib/automaticupgrades/version/versionget_test.go +++ b/lib/automaticupgrades/version/versionget_test.go @@ -20,6 +20,7 @@ import ( "context" "testing" + "github.com/gravitational/trace" "github.com/stretchr/testify/require" ) @@ -64,8 +65,99 @@ func TestValidVersionChange(t *testing.T) { }, } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - require.Equal(t, tt.want, ValidVersionChange(ctx, tt.current, tt.next)) - }) + t.Run( + tt.name, func(t *testing.T) { + require.Equal(t, tt.want, ValidVersionChange(ctx, tt.current, tt.next)) + }, + ) + } +} + +// checkTraceError is a test helper that converts trace.IsXXXError into a require.ErrorAssertionFunc +func checkTraceError(check func(error) bool) require.ErrorAssertionFunc { + return func(t require.TestingT, err error, i ...interface{}) { + require.True(t, check(err), i...) + } +} + +func TestFailoverGetter_GetVersion(t *testing.T) { + t.Parallel() + + // Test setup + ctx := context.Background() + tests := []struct { + name string + getters []Getter + expectResult string + expectErr require.ErrorAssertionFunc + }{ + { + name: "nil", + getters: nil, + expectResult: "", + expectErr: checkTraceError(trace.IsNotFound), + }, + { + name: "empty", + getters: []Getter{}, + expectResult: "", + expectErr: checkTraceError(trace.IsNotFound), + }, + { + name: "first getter success", + getters: []Getter{ + StaticGetter{version: semverMid}, + StaticGetter{version: semverHigh}, + }, + expectResult: semverMid, + expectErr: require.NoError, + }, + { + name: "first getter failure", + getters: []Getter{ + StaticGetter{err: trace.LimitExceeded("got rate-limited")}, + StaticGetter{version: semverHigh}, + }, + expectResult: "", + expectErr: checkTraceError(trace.IsLimitExceeded), + }, + { + name: "first getter skipped, second getter success", + getters: []Getter{ + StaticGetter{err: trace.NotImplemented("proxy does not seem to implement RFD-184")}, + StaticGetter{version: semverHigh}, + }, + expectResult: semverHigh, + expectErr: require.NoError, + }, + { + name: "first getter skipped, second getter failure", + getters: []Getter{ + StaticGetter{err: trace.NotImplemented("proxy does not seem to implement RFD-184")}, + StaticGetter{err: trace.LimitExceeded("got rate-limited")}, + }, + expectResult: "", + expectErr: checkTraceError(trace.IsLimitExceeded), + }, + { + name: "first getter skipped, second getter skipped", + getters: []Getter{ + StaticGetter{err: trace.NotImplemented("proxy does not seem to implement RFD-184")}, + StaticGetter{err: trace.NotImplemented("proxy does not seem to implement RFD-184")}, + }, + expectResult: "", + expectErr: checkTraceError(trace.IsNotFound), + }, + } + for _, tt := range tests { + t.Run( + tt.name, func(t *testing.T) { + // Test execution + getter := FailoverGetter(tt.getters) + result, err := getter.GetVersion(ctx) + require.Equal(t, tt.expectResult, result) + tt.expectErr(t, err) + }, + ) } } diff --git a/lib/autoupdate/agent/config.go b/lib/autoupdate/agent/config.go new file mode 100644 index 0000000000000..c2f51f3240cb4 --- /dev/null +++ b/lib/autoupdate/agent/config.go @@ -0,0 +1,270 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "time" + + "github.com/google/renameio/v2" + "github.com/gravitational/trace" + "gopkg.in/yaml.v3" + + "github.com/gravitational/teleport/lib/autoupdate" +) + +const ( + // updateConfigName specifies the name of the file inside versionsDirName containing configuration for the teleport update. + updateConfigName = "update.yaml" + + // UpdateConfig metadata + updateConfigVersion = "v1" + updateConfigKind = "update_config" +) + +// UpdateConfig describes the update.yaml file schema. +type UpdateConfig struct { + // Version of the configuration file + Version string `yaml:"version"` + // Kind of configuration file (always "update_config") + Kind string `yaml:"kind"` + // Spec contains user-specified configuration. + Spec UpdateSpec `yaml:"spec"` + // Status contains state configuration. + Status UpdateStatus `yaml:"status"` +} + +// UpdateSpec describes the spec field in update.yaml. +type UpdateSpec struct { + // Proxy address + Proxy string `yaml:"proxy"` + // Path is the location the Teleport binaries are linked into. + Path string `yaml:"path"` + // Group specifies the update group identifier for the agent. + Group string `yaml:"group,omitempty"` + // BaseURL is CDN base URL used for the Teleport tgz download URL. + BaseURL string `yaml:"base_url,omitempty"` + // Enabled controls whether auto-updates are enabled. + Enabled bool `yaml:"enabled"` + // Pinned controls whether the active_version is pinned. + Pinned bool `yaml:"pinned"` +} + +// UpdateStatus describes the status field in update.yaml. +type UpdateStatus struct { + // IDFile is the path to a temporary file containing the updater ID. + IDFile string `yaml:"id_file,omitempty"` + // LastUpdate status, if attempted + LastUpdate *LastUpdate `yaml:"last_update,omitempty"` + // Active is the currently active revision of Teleport. + Active Revision `yaml:"active"` + // Backup is the last working revision of Teleport. + Backup *Revision `yaml:"backup,omitempty"` + // Skip is the skipped revision of Teleport. + // Skipped revisions are not applied because they + // are known to crash. + Skip *Revision `yaml:"skip,omitempty"` +} + +// LastUpdate describes the last attempted updated. +type LastUpdate struct { + // Success or failure of the attempted update + Success bool `yaml:"success"` + // Time the update occurred + Time time.Time `yaml:"time"` + // Target revision for the update + Target Revision `yaml:"target"` +} + +// Revision is a version and edition of Teleport. +type Revision struct { + // Version is the version of Teleport. + Version string `yaml:"version" json:"version"` + // Flags describe the edition of Teleport. + Flags autoupdate.InstallFlags `yaml:"flags,flow,omitempty" json:"flags,omitempty"` +} + +// NewRevision create a Revision. +// If version is not set, no flags are returned. +// This ensures that all Revisions without versions are zero-valued. +func NewRevision(version string, flags autoupdate.InstallFlags) Revision { + if version != "" { + return Revision{ + Version: version, + Flags: flags, + } + } + return Revision{} +} + +// NewRevisionFromDir translates a directory path containing Teleport into a Revision. +func NewRevisionFromDir(dir string) (Revision, error) { + parts := strings.Split(dir, "_") + var out Revision + if len(parts) == 0 { + return out, trace.Errorf("dir name empty") + } + out.Version = parts[0] + if out.Version == "" { + return out, trace.Errorf("version missing in dir %s", dir) + } + switch flags := parts[1:]; len(flags) { + case 2: + if flags[1] != autoupdate.FlagFIPS.DirFlag() { + break + } + out.Flags |= autoupdate.FlagFIPS + fallthrough + case 1: + if flags[0] != autoupdate.FlagEnterprise.DirFlag() { + break + } + out.Flags |= autoupdate.FlagEnterprise + fallthrough + case 0: + return out, nil + } + return out, trace.Errorf("invalid flag in %s", dir) +} + +// Dir returns the directory path name of a Revision. +func (r Revision) Dir() string { + // Do not change the order of these statements. + // Otherwise, installed versions will no longer match update.yaml. + var suffix string + if r.Flags&(autoupdate.FlagEnterprise|autoupdate.FlagFIPS) != 0 { + suffix += "_" + autoupdate.FlagEnterprise.DirFlag() + } + if r.Flags&autoupdate.FlagFIPS != 0 { + suffix += "_" + autoupdate.FlagFIPS.DirFlag() + } + return r.Version + suffix +} + +// String returns a human-readable description of a Teleport revision. +func (r Revision) String() string { + if flags := r.Flags.Strings(); len(flags) > 0 { + return fmt.Sprintf("%s+%s", r.Version, strings.Join(flags, "+")) + } + return r.Version +} + +// readConfig reads UpdateConfig from a file. +func readConfig(path string) (*UpdateConfig, error) { + f, err := os.Open(path) + if errors.Is(err, fs.ErrNotExist) { + return &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + }, nil + } + if err != nil { + return nil, trace.Wrap(err, "failed to open") + } + defer f.Close() + var cfg UpdateConfig + if err := yaml.NewDecoder(f).Decode(&cfg); err != nil { + return nil, trace.Wrap(err, "failed to parse") + } + if k := cfg.Kind; k != updateConfigKind { + return nil, trace.Errorf("invalid kind %s", k) + } + if v := cfg.Version; v != updateConfigVersion { + return nil, trace.Errorf("invalid version %s", v) + } + return &cfg, nil +} + +// writeConfig writes UpdateConfig to a file atomically, ensuring the file cannot be corrupted. +func writeConfig(filename string, cfg *UpdateConfig) error { + opts := []renameio.Option{ + renameio.WithPermissions(configFileMode), + renameio.WithExistingPermissions(), + renameio.WithTempDir(filepath.Dir(filename)), + } + t, err := renameio.NewPendingFile(filename, opts...) + if err != nil { + return trace.Wrap(err) + } + defer t.Cleanup() + err = yaml.NewEncoder(t).Encode(cfg) + if err != nil { + return trace.Wrap(err) + } + return trace.Wrap(t.CloseAtomicallyReplace()) +} + +func validateConfigSpec(spec *UpdateSpec, override OverrideConfig) error { + if override.Proxy != "" { + spec.Proxy = override.Proxy + } + if override.Path != "" { + spec.Path = override.Path + } + spec.Group = overrideOptional(spec.Group, override.Group) + spec.BaseURL = overrideOptional(spec.BaseURL, override.BaseURL) + if spec.BaseURL != "" && + !strings.HasPrefix(strings.ToLower(spec.BaseURL), "https://") { + return trace.Errorf("Teleport download base URL %s must use TLS (https://)", spec.BaseURL) + } + if override.Enabled { + spec.Enabled = true + } + if override.Pinned { + spec.Pinned = true + } + return nil +} + +func overrideOptional(orig, override string) string { + switch override { + case "": + return orig + case "default": + return "" + default: + return override + } +} + +// Status of the agent auto-updates system. +type Status struct { + UpdateSpec `yaml:",inline"` + UpdateStatus `yaml:",inline"` + FindResp `yaml:",inline"` + // ID is the updater ID. + ID string `yaml:"id,omitempty"` +} + +// FindResp summarizes the auto-update status response from cluster. +type FindResp struct { + // Target revision of Teleport to install + Target Revision `yaml:"target"` + // InWindow is true when the install should happen now. + InWindow bool `yaml:"in_window"` + // Jitter duration before an automated install + Jitter time.Duration `yaml:"jitter"` + // AGPL installations cannot use the official CDN. + AGPL bool `yaml:"agpl,omitempty"` +} diff --git a/lib/autoupdate/agent/config_test.go b/lib/autoupdate/agent/config_test.go new file mode 100644 index 0000000000000..136a9efe49ee6 --- /dev/null +++ b/lib/autoupdate/agent/config_test.go @@ -0,0 +1,237 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/gravitational/teleport/lib/autoupdate" +) + +func TestNewRevisionFromDir(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + dir string + rev Revision + errMatch string + }{ + { + name: "version", + dir: "1.2.3", + rev: Revision{ + Version: "1.2.3", + }, + }, + { + name: "full", + dir: "1.2.3_ent_fips", + rev: Revision{ + Version: "1.2.3", + Flags: autoupdate.FlagEnterprise | autoupdate.FlagFIPS, + }, + }, + { + name: "ent", + dir: "1.2.3_ent", + rev: Revision{ + Version: "1.2.3", + Flags: autoupdate.FlagEnterprise, + }, + }, + { + name: "empty", + errMatch: "missing", + }, + { + name: "trailing", + dir: "1.2.3_", + errMatch: "invalid", + }, + { + name: "more trailing", + dir: "1.2.3___", + errMatch: "invalid", + }, + { + name: "no version", + dir: "_fips", + errMatch: "missing", + }, + { + name: "fips no ent", + dir: "1.2.3_fips", + errMatch: "invalid", + }, + { + name: "unknown start fips", + dir: "1.2.3_test_fips", + errMatch: "invalid", + }, + { + name: "unknown start ent", + dir: "1.2.3_test_ent", + errMatch: "invalid", + }, + { + name: "unknown end fips", + dir: "1.2.3_fips_test", + errMatch: "invalid", + }, + { + name: "unknown end ent", + dir: "1.2.3_ent_test", + errMatch: "invalid", + }, + { + name: "bad order", + dir: "1.2.3_fips_ent", + errMatch: "invalid", + }, + { + name: "underscore", + dir: "_", + errMatch: "missing", + }, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + rev, err := NewRevisionFromDir(tt.dir) + if tt.errMatch != "" { + require.ErrorContains(t, err, tt.errMatch) + return + } + require.NoError(t, err) + require.Equal(t, tt.rev, rev) + require.Equal(t, tt.dir, rev.Dir()) + }) + } +} + +func TestValidateConfigSpec(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + config UpdateSpec + override UpdateSpec + result UpdateSpec + errMatch string + }{ + { + name: "overrides", + config: UpdateSpec{ + Proxy: "proxy", + Path: "/path", + Group: "group", + BaseURL: "https://example.com", + }, + override: UpdateSpec{ + Enabled: true, + Pinned: true, + Proxy: "overrideProxy", + Path: "/overridePath", + Group: "group2", + BaseURL: "https://example.com", + }, + result: UpdateSpec{ + Enabled: true, + Pinned: true, + Proxy: "overrideProxy", + Path: "/overridePath", + Group: "group2", + BaseURL: "https://example.com", + }, + }, + { + name: "default overrides", + config: UpdateSpec{ + Proxy: "proxy", + Path: "/path", + Group: "group", + BaseURL: "https://example.com", + }, + override: UpdateSpec{ + Proxy: "default", + Path: "default", + Group: "default", + BaseURL: "default", + }, + result: UpdateSpec{ + Proxy: "default", + Path: "default", + }, + }, + { + name: "only overrides", + override: UpdateSpec{ + Enabled: true, + Pinned: true, + Proxy: "overrideProxy", + Path: "/overridePath", + Group: "group2", + BaseURL: "https://example.com", + }, + result: UpdateSpec{ + Enabled: true, + Pinned: true, + Proxy: "overrideProxy", + Path: "/overridePath", + Group: "group2", + BaseURL: "https://example.com", + }, + }, + { + name: "no overrides", + config: UpdateSpec{ + Proxy: "proxy", + Path: "/path", + Group: "group", + BaseURL: "https://example.com", + }, + result: UpdateSpec{ + Proxy: "proxy", + Path: "/path", + Group: "group", + BaseURL: "https://example.com", + }, + }, + { + name: "BaseURL validation fails", + override: UpdateSpec{ + BaseURL: "http://example.com", + }, + errMatch: "must use TLS", + }, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + err := validateConfigSpec(&tt.config, OverrideConfig{UpdateSpec: tt.override}) + if tt.errMatch != "" { + require.ErrorContains(t, err, tt.errMatch) + return + } + require.NoError(t, err) + require.Equal(t, tt.result, tt.config) + }) + } +} diff --git a/lib/autoupdate/agent/installer.go b/lib/autoupdate/agent/installer.go new file mode 100644 index 0000000000000..901b59a9c82a1 --- /dev/null +++ b/lib/autoupdate/agent/installer.go @@ -0,0 +1,889 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "log/slog" + "net/http" + "os" + "path" + "path/filepath" + "syscall" + "time" + + "github.com/google/renameio/v2" + "github.com/gravitational/trace" + + "github.com/gravitational/teleport" + "github.com/gravitational/teleport/lib/autoupdate" + "github.com/gravitational/teleport/lib/utils" +) + +const ( + // checksumType for Teleport tgzs + checksumType = "sha256" + // checksumHexLen is the length of the Teleport checksum. + checksumHexLen = sha256.Size * 2 // bytes to hex + // maxServiceFileSize is the maximum size allowed for a systemd service file. + maxServiceFileSize = 1_000_000 // 1 MB + // configFileMode is the mode used for new configuration files. + configFileMode = 0644 + // systemDirMode is the mode used for new directories. + systemDirMode = 0755 +) + +const ( + // serviceDir contains the relative path to the Teleport SystemD service dir. + serviceDir = "lib/systemd/system" + // serviceName contains the upstream name of the Teleport SystemD service file. + serviceName = "teleport.service" +) + +// LocalInstaller manages the creation and removal of installations +// of Teleport. +// SetRequiredUmask must be called before any methods are executed. +type LocalInstaller struct { + // InstallDir contains each installation, named by version. + InstallDir string + // TargetServiceFile contains a copy of the linked installation's systemd service. + TargetServiceFile string + // SystemBinDir contains binaries for the system (packaged) install of Teleport. + SystemBinDir string + // SystemServiceFile contains the systemd service file for the system (packaged) install of Teleport. + SystemServiceFile string + // HTTP is an HTTP client for downloading Teleport. + HTTP *http.Client + // Log contains a logger. + Log *slog.Logger + // ReservedFreeTmpDisk is the amount of disk that must remain free in /tmp + ReservedFreeTmpDisk uint64 + // ReservedFreeInstallDisk is the amount of disk that must remain free in the install directory. + ReservedFreeInstallDisk uint64 + // TransformService transforms the systemd service during copying. + TransformService func(cfg []byte, pathDir string, flags autoupdate.InstallFlags) []byte + // ValidateBinary returns true if a file is a linkable binary, or + // false if a file should not be linked. + ValidateBinary func(ctx context.Context, path string) (bool, error) + // Template is download URI Template of Teleport packages. + Template string +} + +// Remove a Teleport version directory from InstallDir. +// This function is idempotent. +// See Installer interface for additional specs. +func (li *LocalInstaller) Remove(ctx context.Context, rev Revision) error { + // os.RemoveAll is dangerous because it can remove an entire directory tree. + // We must validate the version to ensure that we remove only a single path + // element under the InstallDir, and not InstallDir or its parents. + // revisionDir performs these validations. + versionDir, err := li.revisionDir(rev) + if err != nil { + return trace.Wrap(err) + } + + // invalidate checksum first, to protect against partially-removed + // directory with valid checksum. + err = os.Remove(filepath.Join(versionDir, checksumType)) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return trace.Wrap(err) + } + if err := os.RemoveAll(versionDir); err != nil { + return trace.Wrap(err) + } + return nil +} + +// Install a Teleport version directory in InstallDir. +// This function is idempotent. +// See Installer interface for additional specs. +func (li *LocalInstaller) Install(ctx context.Context, rev Revision, baseURL string, force bool) (err error) { + versionDir, err := li.revisionDir(rev) + if err != nil { + return trace.Wrap(err) + } + sumPath := filepath.Join(versionDir, checksumType) + + // generate download URI from Template + uri, err := autoupdate.MakeURL(li.Template, baseURL, autoupdate.DefaultPackage, rev.Version, rev.Flags) + if err != nil { + return trace.Wrap(err) + } + + // Get new and old checksums. If they match, skip download. + // Otherwise, clear the old version directory and re-download. + checksumURI := uri + "." + checksumType + newSum, err := li.getChecksum(ctx, checksumURI) + if err != nil { + return trace.Wrap(err, "failed to download checksum from %s", checksumURI) + } + oldSum, err := readChecksum(sumPath) + versionPresent := err == nil + if versionPresent && bytes.Equal(oldSum, newSum) { + li.Log.InfoContext(ctx, "Version already present.", "version", rev) + return nil + } + if versionPresent { + li.Log.WarnContext(ctx, "Removing version that does not match checksum.", "version", rev) + } else if !errors.Is(err, os.ErrNotExist) { + li.Log.WarnContext(ctx, "Removing version with unreadable checksum.", "version", rev, "error", err) + } + if versionPresent || !errors.Is(err, os.ErrNotExist) { + if force { + if err := li.Remove(ctx, rev); err != nil { + return trace.Wrap(err) + } + } else { + return trace.Errorf("refusing to remove linked installation of Teleport") + } + } + + // Verify that we have enough free temp space, then download tgz + freeTmp, err := utils.FreeDiskWithReserve(os.TempDir(), li.ReservedFreeTmpDisk) + if err != nil { + return trace.Wrap(err, "failed to calculate free disk") + } + f, err := os.CreateTemp("", "teleport-update-") + if err != nil { + return trace.Wrap(err, "failed to create temporary file") + } + defer func() { + _ = f.Close() // data never read after close + if err := os.Remove(f.Name()); err != nil { + li.Log.WarnContext(ctx, "Failed to cleanup temporary download.", "error", err) + } + }() + pathSum, err := li.download(ctx, f, int64(freeTmp), uri) + if err != nil { + return trace.Wrap(err, "failed to download teleport") + } + // Seek to the start of the tgz file after writing + if _, err := f.Seek(0, io.SeekStart); err != nil { + return trace.Wrap(err, "failed seek to start of download") + } + + // If interrupted, close the file immediately to stop extracting. + ctx, cancel := context.WithCancel(ctx) + defer cancel() + context.AfterFunc(ctx, func() { + _ = f.Close() // safe to close file multiple times + }) + // Check integrity before decompression + if !bytes.Equal(newSum, pathSum) { + return trace.Errorf("mismatched checksum, download possibly corrupt") + } + // Get uncompressed size of the tgz + n, err := uncompressedSize(f) + if err != nil { + return trace.Wrap(err, "failed to determine uncompressed size") + } + // Seek to start of tgz after reading size + if _, err := f.Seek(0, io.SeekStart); err != nil { + return trace.Wrap(err, "failed seek to start") + } + + // If there's an error after we start extracting, delete the version dir. + defer func() { + if err != nil { + if err := os.RemoveAll(versionDir); err != nil { + li.Log.WarnContext(ctx, "Failed to cleanup broken version extraction.", "error", err, "dir", versionDir) + } + } + }() + + // Extract tgz into version directory. + if err := li.extract(ctx, versionDir, f, n, rev.Flags); err != nil { + return trace.Wrap(err, "failed to extract teleport") + } + // Write the checksum last. This marks the version directory as valid. + if err := os.WriteFile(sumPath, []byte(hex.EncodeToString(newSum)), configFileMode); err != nil { + return trace.Wrap(err, "failed to write checksum") + } + return nil +} + +// readChecksum from the version directory. +func readChecksum(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, trace.Wrap(err) + } + defer f.Close() + var buf bytes.Buffer + _, err = io.CopyN(&buf, f, checksumHexLen) + if err != nil { + return nil, trace.Wrap(err) + } + raw := buf.String() + sum, err := hex.DecodeString(raw) + if err != nil { + return nil, trace.Wrap(err) + } + return sum, nil +} + +func (li *LocalInstaller) getChecksum(ctx context.Context, url string) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, trace.Wrap(err) + } + resp, err := li.HTTP.Do(req) + if err != nil { + return nil, trace.Wrap(err) + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil, trace.Errorf("checksum not found: %s", url) + } + if resp.StatusCode != http.StatusOK { + return nil, trace.Errorf("unexpected HTTP status code: %d", resp.StatusCode) + } + + var buf bytes.Buffer + _, err = io.CopyN(&buf, resp.Body, checksumHexLen) + if err != nil { + return nil, trace.Wrap(err) + } + sum, err := hex.DecodeString(buf.String()) + if err != nil { + return nil, trace.Wrap(err) + } + return sum, nil +} + +func (li *LocalInstaller) download(ctx context.Context, w io.Writer, max int64, url string) (sum []byte, err error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, trace.Wrap(err) + } + startTime := time.Now() + resp, err := li.HTTP.Do(req) + if err != nil { + return nil, trace.Wrap(err) + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil, trace.Errorf("Teleport download not found: %s", url) + } + if resp.StatusCode != http.StatusOK { + return nil, trace.Errorf("unexpected HTTP status code: %d", resp.StatusCode) + } + li.Log.InfoContext(ctx, "Downloading Teleport tarball.", "url", url, "size", resp.ContentLength) + + // Ensure there's enough space in /tmp for the download. + size := resp.ContentLength + if size < 0 { + li.Log.WarnContext(ctx, "Content length missing from response, unable to verify Teleport download size.") + size = max + } else if size > max { + return nil, trace.Errorf("size of download (%d bytes) exceeds available disk space (%d bytes)", resp.ContentLength, max) + } + // Calculate checksum concurrently with download. + shaReader := sha256.New() + tee := io.TeeReader(resp.Body, shaReader) + tee = io.TeeReader(tee, &progressLogger{ + ctx: ctx, + log: li.Log, + level: slog.LevelInfo, + name: path.Base(resp.Request.URL.Path), + max: int(resp.ContentLength), + lines: 5, + }) + n, err := io.CopyN(w, tee, size) + if err != nil { + return nil, trace.Wrap(err) + } + if resp.ContentLength >= 0 && n != resp.ContentLength { + return nil, trace.Errorf("mismatch in Teleport download size") + } + li.Log.InfoContext(ctx, "Download complete.", "duration", time.Since(startTime), "size", n) + return shaReader.Sum(nil), nil +} + +func (li *LocalInstaller) extract(ctx context.Context, dstDir string, src io.Reader, max int64, flags autoupdate.InstallFlags) error { + if err := os.MkdirAll(dstDir, systemDirMode); err != nil { + return trace.Wrap(err) + } + free, err := utils.FreeDiskWithReserve(dstDir, li.ReservedFreeInstallDisk) + if err != nil { + return trace.Wrap(err, "failed to calculate free disk in %s", dstDir) + } + // Bail if there's not enough free disk space at the target + if d := int64(free) - max; d < 0 { + return trace.Errorf("%s needs %d additional bytes of disk space for decompression", dstDir, -d) + } + zr, err := gzip.NewReader(src) + if err != nil { + return trace.Wrap(err, "requires gzip-compressed body") + } + li.Log.InfoContext(ctx, "Extracting Teleport tarball.", "path", dstDir, "size", max) + + err = utils.Extract(zr, dstDir, tgzExtractPaths(flags&(autoupdate.FlagEnterprise|autoupdate.FlagFIPS) != 0)...) + if err != nil { + return trace.Wrap(err) + } + return nil +} + +// tgzExtractPaths describes how to extract the Teleport tgz. +// See utils.Extract for more details on how this list is parsed. +// Paths must use tarball-style / separators (not filepath). +func tgzExtractPaths(ent bool) []utils.ExtractPath { + prefix := "teleport" + if ent { + prefix += "-ent" + } + return []utils.ExtractPath{ + {Src: path.Join(prefix, "examples/systemd/teleport.service"), Dst: filepath.Join(serviceDir, serviceName), DirMode: systemDirMode}, + {Src: path.Join(prefix, "examples"), Skip: true, DirMode: systemDirMode}, + {Src: path.Join(prefix, "install"), Skip: true, DirMode: systemDirMode}, + {Src: path.Join(prefix, "README.md"), Dst: "share/README.md", DirMode: systemDirMode}, + {Src: path.Join(prefix, "CHANGELOG.md"), Dst: "share/CHANGELOG.md", DirMode: systemDirMode}, + {Src: path.Join(prefix, "VERSION"), Dst: "share/VERSION", DirMode: systemDirMode}, + {Src: path.Join(prefix, "LICENSE-community"), Dst: "share/LICENSE-community", DirMode: systemDirMode}, + {Src: prefix, Dst: "bin", DirMode: systemDirMode}, + } +} + +func uncompressedSize(f io.Reader) (int64, error) { + // NOTE: The gzip length trailer is very unreliable, + // but we could optimize this in the future if + // we are willing to verify that all published + // Teleport tarballs have valid trailers. + r, err := gzip.NewReader(f) + if err != nil { + return 0, trace.Wrap(err) + } + n, err := io.Copy(io.Discard, r) + if err != nil { + return 0, trace.Wrap(err) + } + return n, nil +} + +// List installed versions of Teleport. +func (li *LocalInstaller) List(ctx context.Context) (revs []Revision, err error) { + entries, err := os.ReadDir(li.InstallDir) + if err != nil { + return nil, trace.Wrap(err) + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + rev, err := NewRevisionFromDir(entry.Name()) + if err != nil { + return nil, trace.Wrap(err) + } + revs = append(revs, rev) + } + return revs, nil +} + +// Link the specified version into pathDir and TargetServiceFile. +// The revert function restores the previous linking. +// If force is true, Link will overwrite files that are not symlinks. +// See Installer interface for additional specs. +func (li *LocalInstaller) Link(ctx context.Context, rev Revision, pathDir string, force bool) (revert func(context.Context) bool, err error) { + revert = func(context.Context) bool { return true } + versionDir, err := li.revisionDir(rev) + if err != nil { + return revert, trace.Wrap(err) + } + revert, err = li.forceLinks(ctx, + filepath.Join(versionDir, "bin"), + filepath.Join(versionDir, serviceDir, serviceName), + pathDir, force, rev.Flags, + ) + if err != nil { + return revert, trace.Wrap(err) + } + return revert, nil +} + +// LinkSystem links the system (package) version into defaultPathDir and TargetServiceFile. +// This prevents namespaced installations in /opt/teleport from linking to the system package. +// The revert function restores the previous linking. +// See Installer interface for additional specs. +func (li *LocalInstaller) LinkSystem(ctx context.Context) (revert func(context.Context) bool, err error) { + // The system package service file is always removed without flags, so pass + // no flags here to match the behavior. + revert, err = li.forceLinks(ctx, li.SystemBinDir, li.SystemServiceFile, defaultPathDir, false, 0) + return revert, trace.Wrap(err) +} + +// TryLink links the specified version into pathDir, but only in the case that +// no installation of Teleport is already linked or partially linked. +// See Installer interface for additional specs. +func (li *LocalInstaller) TryLink(ctx context.Context, revision Revision, pathDir string) error { + versionDir, err := li.revisionDir(revision) + if err != nil { + return trace.Wrap(err) + } + return trace.Wrap(li.tryLinks(ctx, + filepath.Join(versionDir, "bin"), + filepath.Join(versionDir, serviceDir, serviceName), + pathDir, revision.Flags, + )) +} + +// TryLinkSystem links the system installation to defaultPathDir, but only in the case that +// no installation of Teleport is already linked or partially linked. +// See Installer interface for additional specs. +func (li *LocalInstaller) TryLinkSystem(ctx context.Context) error { + // The system package service file is always removed without flags, so pass + // no flags here to match the behavior. + return trace.Wrap(li.tryLinks(ctx, li.SystemBinDir, li.SystemServiceFile, defaultPathDir, 0)) +} + +// Unlink unlinks a version from pathDir and TargetServiceFile. +// See Installer interface for additional specs. +func (li *LocalInstaller) Unlink(ctx context.Context, rev Revision, pathDir string) error { + versionDir, err := li.revisionDir(rev) + if err != nil { + return trace.Wrap(err) + } + return trace.Wrap(li.removeLinks(ctx, + filepath.Join(versionDir, "bin"), + filepath.Join(versionDir, serviceDir, serviceName), + pathDir, rev.Flags, + )) +} + +// UnlinkSystem unlinks the system (package) version from defaultPathDir and TargetServiceFile. +// See Installer interface for additional specs. +func (li *LocalInstaller) UnlinkSystem(ctx context.Context) error { + // The system package service file is always linked without flags, so pass + // no flags here to match the behavior. + return trace.Wrap(li.removeLinks(ctx, li.SystemBinDir, li.SystemServiceFile, defaultPathDir, 0)) +} + +// symlink from oldname to newname +type symlink struct { + oldname, newname string +} + +// smallFile is a file small enough to be stored in memory. +type smallFile struct { + name string + data []byte + mode os.FileMode +} + +// forceLinks replaces binary links and service files using files in binDir and svcDir. +// Existing links and files are replaced, but mismatched links and files will result in error. +// forceLinks will revert any overridden links or files if it hits an error. +// If successful, forceLinks may also be reverted after it returns by calling revert. +// The revert function returns true if reverting succeeds. +// If force is true, non-link files will be overwritten. +func (li *LocalInstaller) forceLinks(ctx context.Context, srcBinDir, srcSvcFile, dstBinDir string, force bool, flags autoupdate.InstallFlags) (revert func(context.Context) bool, err error) { + // setup revert function + var ( + revertLinks []symlink + revertFiles []smallFile + ) + revert = func(ctx context.Context) bool { + // This function is safe to call repeatedly. + // Returns true only when all changes are successfully reverted. + var ( + keepLinks []symlink + keepFiles []smallFile + ) + for _, l := range revertLinks { + err := renameio.Symlink(l.oldname, l.newname) + if err != nil { + keepLinks = append(keepLinks, l) + li.Log.ErrorContext(ctx, "Failed to revert symlink", "oldname", l.oldname, "newname", l.newname, errorKey, err) + } + } + for _, f := range revertFiles { + err := writeFileAtomicWithinDir(f.name, f.data, f.mode) + if err != nil { + keepFiles = append(keepFiles, f) + li.Log.ErrorContext(ctx, "Failed to revert files", "name", f.name, errorKey, err) + } + } + revertLinks = keepLinks + revertFiles = keepFiles + return len(revertLinks) == 0 && len(revertFiles) == 0 + } + // revert immediately on error, so caller can ignore revert arg + defer func() { + if err != nil { + revert(ctx) + } + }() + + // ensure source directory exists + entries, err := os.ReadDir(srcBinDir) + if errors.Is(err, os.ErrNotExist) { + return revert, trace.Wrap(ErrNoBinaries) + } + if err != nil { + return revert, trace.Wrap(err, "failed to read Teleport binary directory") + } + + // ensure target directories exist before trying to create links + err = os.MkdirAll(dstBinDir, systemDirMode) + if err != nil { + return revert, trace.Wrap(err) + } + err = os.MkdirAll(filepath.Dir(li.TargetServiceFile), systemDirMode) + if err != nil { + return revert, trace.Wrap(err) + } + + // create binary links + var linked int + for _, entry := range entries { + if entry.IsDir() { + continue + } + oldname := filepath.Join(srcBinDir, entry.Name()) + newname := filepath.Join(dstBinDir, entry.Name()) + exec, err := li.ValidateBinary(ctx, oldname) + if err != nil { + return revert, trace.Wrap(err) + } + if !exec { + continue + } + orig, err := forceLink(oldname, newname, force) + if err != nil && !errors.Is(err, os.ErrExist) { + return revert, trace.Wrap(err, "failed to create symlink for %s", entry.Name()) + } + if orig != "" { + revertLinks = append(revertLinks, symlink{ + oldname: orig, + newname: newname, + }) + } + linked++ + } + if linked == 0 { + return revert, trace.Wrap(ErrNoBinaries) + } + + // create systemd service file + + orig, err := li.forceCopyService(li.TargetServiceFile, srcSvcFile, maxServiceFileSize, dstBinDir, flags) + if err != nil && !errors.Is(err, os.ErrExist) { + return revert, trace.Wrap(err, "failed to copy service") + } + if orig != nil { + revertFiles = append(revertFiles, *orig) + } + return revert, nil +} + +// forceCopyService uses forceCopy to copy a systemd service file from src to dst. +// The contents of both src and dst must be smaller than n. +// See forceCopy for more details. +func (li *LocalInstaller) forceCopyService(dst, src string, n int64, dstBinDir string, flags autoupdate.InstallFlags) (orig *smallFile, err error) { + srcData, err := readFileAtMost(src, n) + if err != nil { + return nil, trace.Wrap(err) + } + return forceCopy(dst, li.TransformService(srcData, dstBinDir, flags), n) +} + +// forceLink attempts to create a symlink, atomically replacing an existing link if already present. +// If a non-symlink file or directory exists in newname already and force is false, forceLink errors with ErrFilePresent. +// If the link is already present with the desired oldname, forceLink returns os.ErrExist. +func forceLink(oldname, newname string, force bool) (orig string, err error) { + orig, err = os.Readlink(newname) + if errors.Is(err, os.ErrInvalid) || + errors.Is(err, syscall.EINVAL) { // workaround missing ErrInvalid wrapper + if force { + return "", trace.Wrap(renameio.Symlink(oldname, newname)) + } + // important: do not attempt to replace a non-linked install of Teleport without force + return "", trace.Wrap(ErrFilePresent, "refusing to replace file at %s", newname) + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return "", trace.Wrap(err) + } + if orig == oldname { + return "", trace.Wrap(os.ErrExist) + } + err = renameio.Symlink(oldname, newname) + if err != nil { + return "", trace.Wrap(err) + } + return orig, nil +} + +// forceCopy atomically copies a file from srcData to dst, replacing an existing file at dst if needed. +// The contents of dst must be smaller than n. +// forceCopy returns the original file path, mode, and contents as orig. +// If an irregular file, too large file, or directory exists in dst already, forceCopy errors. +// If the file is already present with the desired contents, forceCopy returns os.ErrExist. +func forceCopy(dst string, srcData []byte, n int64) (orig *smallFile, err error) { + fi, err := os.Lstat(dst) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, trace.Wrap(err) + } + if err == nil { + orig = &smallFile{ + name: dst, + mode: fi.Mode(), + } + if !orig.mode.IsRegular() { + return nil, trace.Errorf("refusing to replace irregular file at %s", dst) + } + orig.data, err = readFileAtMost(dst, n) + if err != nil { + return nil, trace.Wrap(err) + } + if bytes.Equal(srcData, orig.data) { + return nil, trace.Wrap(os.ErrExist) + } + } + err = writeFileAtomicWithinDir(dst, srcData, configFileMode) + if err != nil { + return nil, trace.Wrap(err) + } + return orig, nil +} + +// writeFileAtomicWithinDir atomically creates a new file with renameio, while ensuring that temporary +// files use the same directory as the target file (with format: .[base][randints]). +// This ensures that SELinux contexts for important files are set correctly. +func writeFileAtomicWithinDir(filename string, data []byte, perm os.FileMode) error { + dir := filepath.Dir(filename) + err := renameio.WriteFile(filename, data, perm, renameio.WithTempDir(dir)) + return trace.Wrap(err) +} + +// readFileAtMost reads a file up to n, or errors if it is too large. +func readFileAtMost(name string, n int64) ([]byte, error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + defer f.Close() + data, err := utils.ReadAtMost(f, n) + return data, trace.Wrap(err) +} + +func (li *LocalInstaller) removeLinks(ctx context.Context, srcBinDir, srcSvcFile, dstBinDir string, flags autoupdate.InstallFlags) error { + removeService := false + entries, err := os.ReadDir(srcBinDir) + if err != nil { + return trace.Wrap(err, "failed to find Teleport binary directory") + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + oldname := filepath.Join(srcBinDir, entry.Name()) + newname := filepath.Join(dstBinDir, entry.Name()) + v, err := os.Readlink(newname) + if errors.Is(err, os.ErrNotExist) || + errors.Is(err, os.ErrInvalid) || + errors.Is(err, syscall.EINVAL) { + li.Log.DebugContext(ctx, "Link not present.", "oldname", oldname, "newname", newname) + continue + } + if err != nil { + return trace.Wrap(err, "error reading link for %s", filepath.Base(newname)) + } + if v != oldname { + li.Log.DebugContext(ctx, "Skipping link to different binary.", "oldname", oldname, "newname", newname) + continue + } + if err := os.Remove(newname); err != nil { + li.Log.ErrorContext(ctx, "Unable to remove link.", "oldname", oldname, "newname", newname, errorKey, err) + continue + } + if filepath.Base(newname) == teleport.ComponentTeleport { + removeService = true + } + } + // only remove service if teleport was removed + if !removeService { + li.Log.DebugContext(ctx, "Teleport binary not unlinked. Skipping removal of teleport.service.") + return nil + } + srcBytes, err := readFileAtMost(srcSvcFile, maxServiceFileSize) + if err != nil { + return trace.Wrap(err) + } + dstBytes, err := readFileAtMost(li.TargetServiceFile, maxServiceFileSize) + if errors.Is(err, os.ErrNotExist) { + li.Log.DebugContext(ctx, "Service not present.", "path", li.TargetServiceFile) + return nil + } + if err != nil { + return trace.Wrap(err) + } + if !bytes.Equal(li.TransformService(srcBytes, dstBinDir, flags), dstBytes) { + li.Log.WarnContext(ctx, "Removed teleport binary link, but skipping removal of custom teleport.service: the service file does not match the reference file for this version. The file might have been manually edited.") + return nil + } + if err := os.Remove(li.TargetServiceFile); err != nil { + return trace.Wrap(err, "error removing copy of %s", filepath.Base(li.TargetServiceFile)) + } + return nil +} + +// tryLinks create binary and service links for files in binDir and svcDir if links are not already present. +// Existing links that point to files outside binDir or svcDir, as well as existing non-link files, will error. +// tryLinks will not attempt to create any links if linking could result in an error. +// However, concurrent changes to links may result in an error with partially-complete linking. +func (li *LocalInstaller) tryLinks(ctx context.Context, srcBinDir, srcSvcFile, dstBinDir string, flags autoupdate.InstallFlags) error { + // ensure source directory exists + entries, err := os.ReadDir(srcBinDir) + if errors.Is(err, os.ErrNotExist) { + return trace.Wrap(ErrNoBinaries) + } + if err != nil { + return trace.Wrap(err, "failed to read Teleport binary directory") + } + + // ensure target directories exist before trying to create links + err = os.MkdirAll(dstBinDir, systemDirMode) + if err != nil { + return trace.Wrap(err) + } + err = os.MkdirAll(filepath.Dir(li.TargetServiceFile), systemDirMode) + if err != nil { + return trace.Wrap(err) + } + + // validate that we can link all system binaries before attempting linking + var links []symlink + var linked int + for _, entry := range entries { + if entry.IsDir() { + continue + } + oldname := filepath.Join(srcBinDir, entry.Name()) + newname := filepath.Join(dstBinDir, entry.Name()) + exec, err := li.ValidateBinary(ctx, oldname) + if err != nil { + return trace.Wrap(err) + } + if !exec { + continue + } + ok, err := needsLink(oldname, newname) + if err != nil { + return trace.Wrap(err, "error evaluating link for %s", filepath.Base(oldname)) + } + if ok { + links = append(links, symlink{oldname, newname}) + } + linked++ + } + // bail if no binaries can be linked + if linked == 0 { + return trace.Wrap(ErrNoBinaries) + } + + // link binaries that are missing links + for _, link := range links { + if err := os.Symlink(link.oldname, link.newname); err != nil { + return trace.Wrap(err, "failed to create symlink for %s", filepath.Base(link.oldname)) + } + } + + // if any binaries are linked from srcBinDir, always link the service from svcDir + _, err = li.forceCopyService(li.TargetServiceFile, srcSvcFile, maxServiceFileSize, dstBinDir, flags) + if err != nil && !errors.Is(err, os.ErrExist) { + return trace.Wrap(err, "failed to copy service") + } + + return nil +} + +// needsLink returns true when a symlink from oldname to newname needs to be created, or false if it exists. +// If a non-symlink file or directory exists at newname, needsLink errors with ErrFilePresent. +// If a symlink to a different location exists, needsLink errors with ErrLinked. +func needsLink(oldname, newname string) (ok bool, err error) { + orig, err := os.Readlink(newname) + if errors.Is(err, os.ErrInvalid) || + errors.Is(err, syscall.EINVAL) { // workaround missing ErrInvalid wrapper + // important: do not attempt to replace a non-linked install of Teleport + return false, trace.Wrap(ErrFilePresent, "refusing to replace file at %s", newname) + } + if errors.Is(err, os.ErrNotExist) { + return true, nil + } + if err != nil { + return false, trace.Wrap(err) + } + if orig != oldname { + return false, trace.Wrap(ErrLinked, "refusing to replace link at %s", newname) + } + return false, nil +} + +// revisionDir returns the storage directory for a Teleport revision. +// revisionDir will fail if the revision cannot be used to construct the directory name. +// For example, it ensures that ".." cannot be provided to return a system directory. +func (li *LocalInstaller) revisionDir(rev Revision) (string, error) { + installDir, err := filepath.Abs(li.InstallDir) + if err != nil { + return "", trace.Wrap(err) + } + versionDir := filepath.Join(installDir, rev.Dir()) + if filepath.Dir(versionDir) != filepath.Clean(installDir) { + return "", trace.Errorf("refusing to link directory outside of version directory") + } + return versionDir, nil +} + +// IsLinked returns true if any binaries for Revision rev are linked to pathDir. +// Returns os.ErrNotExist error if the revision does not exist. +func (li *LocalInstaller) IsLinked(ctx context.Context, rev Revision, pathDir string) (bool, error) { + versionDir, err := li.revisionDir(rev) + if err != nil { + return false, trace.Wrap(err) + } + binDir := filepath.Join(versionDir, "bin") + entries, err := os.ReadDir(binDir) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, trace.Wrap(err) + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + v, err := os.Readlink(filepath.Join(pathDir, entry.Name())) + if err != nil { + continue + } + if filepath.Clean(v) == filepath.Join(binDir, entry.Name()) { + return true, nil + } + } + return false, nil +} diff --git a/lib/autoupdate/agent/installer_test.go b/lib/autoupdate/agent/installer_test.go new file mode 100644 index 0000000000000..ce629657d8841 --- /dev/null +++ b/lib/autoupdate/agent/installer_test.go @@ -0,0 +1,1170 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "slices" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/gravitational/teleport/lib/autoupdate" +) + +func TestLocalInstaller_Install(t *testing.T) { + t.Parallel() + const version = "new-version" + + _, testSum := testTGZ(t, version) + + tests := []struct { + name string + reservedTmp uint64 + reservedInstall uint64 + existingSum string + flags autoupdate.InstallFlags + force bool + + errMatch string + }{ + { + name: "not present", + }, + { + name: "present", + existingSum: testSum, + }, + { + name: "mismatched checksum", + existingSum: hex.EncodeToString(sha256.New().Sum(nil)), + force: true, + }, + { + name: "unreadable checksum", + existingSum: "bad", + force: true, + }, + { + name: "unreadable checksum, force false", + existingSum: "bad", + force: false, + errMatch: "refusing", + }, + { + name: "out of space in /tmp", + reservedTmp: reservedFreeDisk * 1_000_000_000, + errMatch: "no free space left", + }, + { + name: "out of space in install dir", + reservedInstall: reservedFreeDisk * 1_000_000_000, + errMatch: "no free space left", + }, + // TODO(sclevine): test flags + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + err := os.MkdirAll(filepath.Join(dir, version), os.ModePerm) + require.NoError(t, err) + + if tt.existingSum != "" { + err := os.WriteFile(filepath.Join(dir, version, checksumType), []byte(tt.existingSum), os.ModePerm) + require.NoError(t, err) + } + + // test parameters + var dlPath, shaPath, shasum string + + // test server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tgz, sum := testTGZ(t, version) + shasum = sum + var out *bytes.Buffer + if strings.HasSuffix(r.URL.Path, "."+checksumType) { // checksum request + shaPath = r.URL.Path + out = bytes.NewBufferString(sum) + } else { // tgz request + dlPath = r.URL.Path + out = tgz + } + w.Header().Set("Content-Length", strconv.Itoa(out.Len())) + _, err := io.Copy(w, out) + if err != nil { + t.Fatal(err) + } + })) + t.Cleanup(server.Close) + + installer := &LocalInstaller{ + InstallDir: dir, + HTTP: http.DefaultClient, + Log: slog.Default(), + ReservedFreeTmpDisk: tt.reservedTmp, + ReservedFreeInstallDisk: tt.reservedInstall, + Template: "{{.BaseURL}}/{{.Package}}-{{.OS}}/{{.Arch}}/{{.Version}}", + } + ctx := context.Background() + err = installer.Install(ctx, NewRevision(version, tt.flags), server.URL, tt.force) + if tt.errMatch != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMatch) + return + } + require.NoError(t, err) + + const expectedPath = "/teleport-" + runtime.GOOS + "/" + runtime.GOARCH + "/" + version + require.Equal(t, expectedPath+"."+checksumType, shaPath) + + if tt.existingSum == testSum { + return + } + + require.Equal(t, expectedPath, dlPath) + + for _, p := range []string{ + filepath.Join(dir, version, "lib", "systemd", "system", "teleport.service"), + filepath.Join(dir, version, "bin", "teleport"), + filepath.Join(dir, version, "bin", "tsh"), + } { + v, err := os.ReadFile(p) + require.NoError(t, err) + require.Equal(t, version, string(v)) + } + + sum, err := os.ReadFile(filepath.Join(dir, version, checksumType)) + require.NoError(t, err) + require.Equal(t, string(sum), shasum) + }) + } +} + +func testTGZ(t *testing.T, version string) (tgz *bytes.Buffer, shasum string) { + t.Helper() + + var buf bytes.Buffer + + sha := sha256.New() + gz := gzip.NewWriter(io.MultiWriter(&buf, sha)) + tw := tar.NewWriter(gz) + + var files = []struct { + Name, Body string + }{ + {"teleport/examples/systemd/teleport.service", version}, + {"teleport/teleport", version}, + {"teleport/tsh", version}, + } + for _, file := range files { + hdr := &tar.Header{ + Name: file.Name, + Mode: 0600, + Size: int64(len(file.Body)), + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(file.Body)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return &buf, hex.EncodeToString(sha.Sum(nil)) +} + +func TestLocalInstaller_Link(t *testing.T) { + t.Parallel() + const version = "new-version" + servicePath := filepath.Join(serviceDir, serviceName) + + tests := []struct { + name string + installDirs []string + installFiles []string + installFileMode os.FileMode + existingLinks []string + existingFiles []string + force bool + + resultLinks []string + resultServices []string + errMatch string + }{ + { + name: "present with new links", + installDirs: []string{ + "bin", + "bin/somedir", + "lib", + "lib/systemd", + "lib/systemd/system", + "somedir", + }, + installFiles: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + servicePath, + "README", + }, + installFileMode: os.ModePerm, + + resultLinks: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + }, + resultServices: []string{ + "lib/systemd/system/teleport.service", + }, + }, + { + name: "present with non-executable files", + installDirs: []string{ + "bin", + "bin/somedir", + "lib", + "lib/systemd", + "lib/systemd/system", + "somedir", + }, + installFiles: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + servicePath, + "README", + }, + installFileMode: 0644, + + errMatch: ErrNoBinaries.Error(), + }, + { + name: "present with existing links", + installDirs: []string{ + "bin", + "bin/somedir", + "lib", + "lib/systemd", + "lib/systemd/system", + "somedir", + }, + installFiles: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + servicePath, + "README", + }, + installFileMode: os.ModePerm, + existingLinks: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + }, + existingFiles: []string{ + "lib/systemd/system/teleport.service", + }, + + resultLinks: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + }, + resultServices: []string{ + "lib/systemd/system/teleport.service", + }, + }, + { + name: "conflicting systemd files", + installDirs: []string{ + "bin", + "bin/somedir", + "lib", + "lib/systemd", + "lib/systemd/system", + "somedir", + }, + installFiles: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + servicePath, + "README", + }, + installFileMode: os.ModePerm, + existingLinks: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + "lib/systemd/system/teleport.service", + }, + + errMatch: "refusing", + }, + { + name: "conflicting bin files", + installDirs: []string{ + "bin", + "bin/somedir", + "lib", + "lib/systemd", + "lib/systemd/system", + "somedir", + }, + installFiles: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + servicePath, + "README", + }, + installFileMode: os.ModePerm, + existingLinks: []string{ + "bin/teleport", + "bin/tbot", + }, + existingFiles: []string{ + "lib/systemd/system/teleport.service", + "bin/tsh", + }, + + errMatch: ErrFilePresent.Error(), + }, + { + name: "overwriting bin files", + installDirs: []string{ + "bin", + "bin/somedir", + "lib", + "lib/systemd", + "lib/systemd/system", + "somedir", + }, + installFiles: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + servicePath, + "README", + }, + installFileMode: os.ModePerm, + existingLinks: []string{ + "bin/teleport", + "bin/tbot", + }, + existingFiles: []string{ + "lib/systemd/system/teleport.service", + "bin/tsh", + }, + force: true, + + resultLinks: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + }, + resultServices: []string{ + "lib/systemd/system/teleport.service", + }, + }, + { + name: "no links", + installFiles: []string{"README"}, + installDirs: []string{"bin"}, + + errMatch: ErrNoBinaries.Error(), + }, + { + name: "no bin directory", + installFiles: []string{"README"}, + + errMatch: ErrNoBinaries.Error(), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + versionsDir := t.TempDir() + versionDir := filepath.Join(versionsDir, version+"_ent") + err := os.MkdirAll(versionDir, 0o755) + require.NoError(t, err) + + // setup files in version directory + for _, d := range tt.installDirs { + err := os.Mkdir(filepath.Join(versionDir, d), os.ModePerm) + require.NoError(t, err) + } + for _, n := range tt.installFiles { + err := os.WriteFile(filepath.Join(versionDir, n), []byte(filepath.Base(n)), tt.installFileMode) + require.NoError(t, err) + } + + // setup files in system links directory + linkDir := t.TempDir() + for _, n := range tt.existingLinks { + err := os.MkdirAll(filepath.Dir(filepath.Join(linkDir, n)), os.ModePerm) + require.NoError(t, err) + err = os.Symlink(filepath.Base(n)+".old", filepath.Join(linkDir, n)) + require.NoError(t, err) + } + for _, n := range tt.existingFiles { + err := os.MkdirAll(filepath.Dir(filepath.Join(linkDir, n)), os.ModePerm) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(linkDir, n), []byte(filepath.Base(n)), os.ModePerm) + require.NoError(t, err) + } + + validator := Validator{Log: slog.Default()} + installer := &LocalInstaller{ + InstallDir: versionsDir, + TargetServiceFile: filepath.Join(linkDir, serviceDir, serviceName), + Log: slog.Default(), + TransformService: func(b []byte, pathDir string, flags autoupdate.InstallFlags) []byte { + return []byte(fmt.Sprintf("[service=%s][path=%s][flags=%s]", string(b), pathDir, flags.Strings())) + }, + ValidateBinary: validator.IsExecutable, + Template: autoupdate.DefaultCDNURITemplate, + } + ctx := context.Background() + revert, err := installer.Link(ctx, NewRevision(version, autoupdate.FlagEnterprise), filepath.Join(linkDir, "bin"), tt.force) + if tt.errMatch != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMatch) + + // verify automatic revert + for _, link := range tt.existingLinks { + v, err := os.Readlink(filepath.Join(linkDir, link)) + require.NoError(t, err) + require.Equal(t, filepath.Base(link)+".old", v) + } + for _, n := range tt.existingFiles { + v, err := os.ReadFile(filepath.Join(linkDir, n)) + require.NoError(t, err) + require.Equal(t, filepath.Base(n), string(v)) + } + + // ensure revert still succeeds + ok := revert(ctx) + require.True(t, ok) + return + } + require.NoError(t, err) + + // verify links + for _, link := range tt.resultLinks { + v, err := os.ReadFile(filepath.Join(linkDir, link)) + require.NoError(t, err) + require.Equal(t, filepath.Base(link), string(v)) + } + for _, svc := range tt.resultServices { + v, err := os.ReadFile(filepath.Join(linkDir, svc)) + require.NoError(t, err) + require.Equal(t, fmt.Sprintf("[service=%s][path=%s][flags=[Enterprise]]", filepath.Base(svc), filepath.Join(linkDir, "bin")), string(v)) + } + + // verify manual revert + ok := revert(ctx) + require.True(t, ok) + for _, link := range tt.existingLinks { + v, err := os.Readlink(filepath.Join(linkDir, link)) + require.NoError(t, err) + require.Equal(t, filepath.Base(link)+".old", v) + } + for _, n := range tt.existingFiles { + v, err := os.ReadFile(filepath.Join(linkDir, n)) + require.NoError(t, err) + require.Equal(t, filepath.Base(n), string(v)) + } + }) + } +} + +func TestLocalInstaller_TryLink(t *testing.T) { + t.Parallel() + const version = "new-version" + servicePath := filepath.Join(serviceDir, serviceName) + + tests := []struct { + name string + installDirs []string + installFiles []string + installFileMode os.FileMode + existingLinks []string + existingFiles []string + + resultLinks []string + resultServices []string + errMatch string + }{ + { + name: "present with new links", + installDirs: []string{ + "bin", + "bin/somedir", + "lib", + "lib/systemd", + "lib/systemd/system", + "somedir", + }, + installFiles: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + servicePath, + "README", + }, + installFileMode: os.ModePerm, + + resultLinks: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + }, + resultServices: []string{ + "lib/systemd/system/teleport.service", + }, + }, + { + name: "present with non-executable files", + installDirs: []string{ + "bin", + "bin/somedir", + "lib", + "lib/systemd", + "lib/systemd/system", + "somedir", + }, + installFiles: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + servicePath, + "README", + }, + installFileMode: 0644, + + errMatch: ErrNoBinaries.Error(), + }, + { + name: "present with existing links", + installDirs: []string{ + "bin", + "bin/somedir", + "lib", + "lib/systemd", + "lib/systemd/system", + "somedir", + }, + installFiles: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + servicePath, + "README", + }, + installFileMode: os.ModePerm, + existingLinks: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + }, + existingFiles: []string{ + "lib/systemd/system/teleport.service", + }, + + errMatch: "refusing", + }, + { + name: "conflicting systemd files", + installDirs: []string{ + "bin", + "bin/somedir", + "lib", + "lib/systemd", + "lib/systemd/system", + "somedir", + }, + installFiles: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + servicePath, + "README", + }, + installFileMode: os.ModePerm, + existingLinks: []string{ + "lib/systemd/system/teleport.service", + }, + + errMatch: "replace irregular file", + }, + { + name: "conflicting bin files", + installDirs: []string{ + "bin", + "bin/somedir", + "lib", + "lib/systemd", + "lib/systemd/system", + "somedir", + }, + installFiles: []string{ + "bin/teleport", + "bin/tsh", + "bin/tbot", + servicePath, + "README", + }, + installFileMode: os.ModePerm, + existingFiles: []string{ + "bin/tsh", + }, + + errMatch: ErrFilePresent.Error(), + }, + { + name: "no links", + installFiles: []string{"README"}, + installDirs: []string{"bin"}, + + errMatch: ErrNoBinaries.Error(), + }, + { + name: "no bin directory", + installFiles: []string{"README"}, + + errMatch: ErrNoBinaries.Error(), + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + versionsDir := t.TempDir() + versionDir := filepath.Join(versionsDir, version+"_ent") + err := os.MkdirAll(versionDir, 0o755) + require.NoError(t, err) + + // setup files in version directory + for _, d := range tt.installDirs { + err := os.Mkdir(filepath.Join(versionDir, d), os.ModePerm) + require.NoError(t, err) + } + for _, n := range tt.installFiles { + err := os.WriteFile(filepath.Join(versionDir, n), []byte(filepath.Base(n)), tt.installFileMode) + require.NoError(t, err) + } + + // setup files in system links directory + linkDir := t.TempDir() + for _, n := range tt.existingLinks { + err := os.MkdirAll(filepath.Dir(filepath.Join(linkDir, n)), os.ModePerm) + require.NoError(t, err) + err = os.Symlink(filepath.Base(n)+".old", filepath.Join(linkDir, n)) + require.NoError(t, err) + } + for _, n := range tt.existingFiles { + err := os.MkdirAll(filepath.Dir(filepath.Join(linkDir, n)), os.ModePerm) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(linkDir, n), []byte(filepath.Base(n)), os.ModePerm) + require.NoError(t, err) + } + + validator := Validator{Log: slog.Default()} + installer := &LocalInstaller{ + InstallDir: versionsDir, + TargetServiceFile: filepath.Join(linkDir, serviceDir, serviceName), + Log: slog.Default(), + TransformService: func(b []byte, pathDir string, flags autoupdate.InstallFlags) []byte { + return []byte(fmt.Sprintf("[service=%s][path=%s][flags=%s]", string(b), pathDir, flags.Strings())) + }, + ValidateBinary: validator.IsExecutable, + } + ctx := context.Background() + err = installer.TryLink(ctx, NewRevision(version, autoupdate.FlagEnterprise), filepath.Join(linkDir, "bin")) + if tt.errMatch != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMatch) + + // verify no changes + for _, link := range tt.existingLinks { + v, err := os.Readlink(filepath.Join(linkDir, link)) + require.NoError(t, err) + require.Equal(t, filepath.Base(link)+".old", v) + } + for _, n := range tt.existingFiles { + v, err := os.ReadFile(filepath.Join(linkDir, n)) + require.NoError(t, err) + require.Equal(t, filepath.Base(n), string(v)) + } + return + } + require.NoError(t, err) + + // verify links + for _, link := range tt.resultLinks { + v, err := os.ReadFile(filepath.Join(linkDir, link)) + require.NoError(t, err) + require.Equal(t, filepath.Base(link), string(v)) + } + for _, svc := range tt.resultServices { + v, err := os.ReadFile(filepath.Join(linkDir, svc)) + require.NoError(t, err) + require.Equal(t, fmt.Sprintf("[service=%s][path=%s][flags=[Enterprise]]", filepath.Base(svc), filepath.Join(linkDir, "bin")), string(v)) + } + + }) + } +} + +func TestLocalInstaller_Remove(t *testing.T) { + t.Parallel() + const version = "existing-version" + + tests := []struct { + name string + dirs []string + files []string + createVersion string + removeVersion string + + errMatch string + }{ + { + name: "present", + dirs: []string{"bin/somedir", "somedir"}, + files: []string{checksumType, "bin/teleport", "bin/tsh", "bin/tbot", "README"}, + createVersion: version, + removeVersion: version, + }, + { + name: "present missing checksum", + dirs: []string{"bin/somedir", "somedir"}, + files: []string{"bin/teleport", "bin/tsh", "bin/tbot", "README"}, + createVersion: version, + removeVersion: version, + }, + { + name: "not present", + dirs: []string{"bin/somedir", "somedir"}, + files: []string{checksumType, "bin/teleport", "bin/tsh", "bin/tbot", "README"}, + createVersion: version, + removeVersion: "missing-version", + }, + { + name: "version empty", + dirs: []string{"bin/somedir", "somedir"}, + files: []string{checksumType, "bin/teleport", "bin/tsh", "bin/tbot", "README"}, + createVersion: version, + removeVersion: "", + + errMatch: "outside", + }, + { + name: "version has path", + dirs: []string{"bin/somedir", "somedir"}, + files: []string{checksumType, "bin/teleport", "bin/tsh", "bin/tbot", "README"}, + createVersion: version, + removeVersion: "one/two", + + errMatch: "outside", + }, + { + name: "version is ..", + dirs: []string{"bin/somedir", "somedir"}, + files: []string{checksumType, "bin/teleport", "bin/tsh", "bin/tbot", "README"}, + createVersion: version, + removeVersion: "..", + + errMatch: "outside", + }, + { + name: "version is .", + dirs: []string{"bin/somedir", "somedir"}, + files: []string{checksumType, "bin/teleport", "bin/tsh", "bin/tbot", "README"}, + createVersion: version, + removeVersion: ".", + + errMatch: "outside", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + versionsDir := t.TempDir() + versionDir := filepath.Join(versionsDir, tt.createVersion) + err := os.MkdirAll(versionDir, 0o755) + require.NoError(t, err) + + for _, d := range tt.dirs { + err := os.MkdirAll(filepath.Join(versionDir, d), os.ModePerm) + require.NoError(t, err) + } + for _, n := range tt.files { + err := os.WriteFile(filepath.Join(versionDir, n), []byte(filepath.Base(n)), os.ModePerm) + require.NoError(t, err) + } + + linkDir := t.TempDir() + + validator := Validator{Log: slog.Default()} + installer := &LocalInstaller{ + InstallDir: versionsDir, + TargetServiceFile: filepath.Join(linkDir, serviceDir, serviceName), + Log: slog.Default(), + TransformService: func(b []byte, pathDir string, flags autoupdate.InstallFlags) []byte { + return []byte(fmt.Sprintf("[service=%s][path=%s][flags=%s]", string(b), pathDir, flags.Strings())) + }, + ValidateBinary: validator.IsExecutable, + } + ctx := context.Background() + err = installer.Remove(ctx, NewRevision(tt.removeVersion, 0)) + if tt.errMatch != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMatch) + return + } + require.NoError(t, err) + _, err = os.Stat(filepath.Join(versionsDir, tt.removeVersion)) + require.ErrorIs(t, err, os.ErrNotExist) + }) + } +} + +func TestLocalInstaller_IsLinked(t *testing.T) { + t.Parallel() + const version = "existing-version" + servicePath := filepath.Join(serviceDir, serviceName) + + tests := []struct { + name string + dirs []string + files []string + createVersion string + linkVersion string + checkVersion string + + result bool + errMatch string + }{ + { + name: "linked", + dirs: []string{"bin/somedir", "somedir", serviceDir}, + files: []string{checksumType, "bin/teleport", "bin/tsh", "bin/tbot", "README", servicePath}, + createVersion: version, + linkVersion: version, + checkVersion: version, + result: true, + }, + { + name: "other linked", + dirs: []string{"bin/somedir", "somedir", serviceDir}, + files: []string{checksumType, "bin/teleport", "bin/tsh", "bin/tbot", "README", servicePath}, + createVersion: version, + linkVersion: version, + checkVersion: "other", + result: false, + }, + { + name: "not linked", + checkVersion: version, + result: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + versionsDir := t.TempDir() + linkDir := t.TempDir() + + validator := Validator{Log: slog.Default()} + installer := &LocalInstaller{ + InstallDir: versionsDir, + TargetServiceFile: filepath.Join(linkDir, serviceDir, serviceName), + Log: slog.Default(), + TransformService: func(b []byte, pathDir string, flags autoupdate.InstallFlags) []byte { + return []byte(fmt.Sprintf("[service=%s][path=%s][flags=%s]", string(b), pathDir, flags.Strings())) + }, + ValidateBinary: validator.IsExecutable, + } + ctx := context.Background() + if tt.createVersion != "" { + versionDir := filepath.Join(versionsDir, tt.createVersion) + err := os.MkdirAll(versionDir, 0o755) + require.NoError(t, err) + + for _, d := range tt.dirs { + err := os.MkdirAll(filepath.Join(versionDir, d), os.ModePerm) + require.NoError(t, err) + } + for _, n := range tt.files { + err := os.WriteFile(filepath.Join(versionDir, n), []byte(filepath.Base(n)), os.ModePerm) + require.NoError(t, err) + } + } + if tt.linkVersion != "" { + _, err := installer.Link(ctx, NewRevision(tt.linkVersion, 0), linkDir, false) + require.NoError(t, err) + } + result, err := installer.IsLinked(ctx, NewRevision(tt.checkVersion, 0), linkDir) + if tt.errMatch != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMatch) + return + } + require.NoError(t, err) + require.Equal(t, tt.result, result) + }) + } +} + +func TestLocalInstaller_Unlink(t *testing.T) { + t.Parallel() + const version = "existing-version" + servicePath := filepath.Join(serviceDir, serviceName) + + tests := []struct { + name string + bins []string + svcOrig []byte + + links []symlink + svcCopy []byte + + remaining []string + errMatch string + }{ + { + name: "normal", + bins: []string{"teleport", "tsh"}, + svcOrig: []byte("orig"), + links: []symlink{ + {oldname: "bin/teleport", newname: "bin/teleport"}, + {oldname: "bin/tsh", newname: "bin/tsh"}, + }, + svcCopy: []byte("[service=orig][path=bin][flags=[]]"), + }, + { + name: "different services", + bins: []string{"teleport", "tsh"}, + svcOrig: []byte("orig"), + links: []symlink{ + {oldname: "bin/teleport", newname: "bin/teleport"}, + {oldname: "bin/tsh", newname: "bin/tsh"}, + }, + svcCopy: []byte("custom"), + remaining: []string{servicePath}, + }, + { + name: "missing target service", + bins: []string{"teleport", "tsh"}, + svcOrig: []byte("orig"), + links: []symlink{ + {oldname: "bin/teleport", newname: "bin/teleport"}, + {oldname: "bin/tsh", newname: "bin/tsh"}, + }, + }, + { + name: "missing source service", + bins: []string{"teleport", "tsh"}, + links: []symlink{ + {oldname: "bin/teleport", newname: "bin/teleport"}, + {oldname: "bin/tsh", newname: "bin/tsh"}, + }, + svcCopy: []byte("custom"), + remaining: []string{servicePath}, + errMatch: "no such", + }, + { + name: "missing teleport link", + bins: []string{"teleport", "tsh"}, + svcOrig: []byte("orig"), + links: []symlink{ + {oldname: "bin/tsh", newname: "bin/tsh"}, + }, + svcCopy: []byte("[service=orig][path=bin][flags=[]]"), + remaining: []string{servicePath}, + }, + { + name: "missing other link", + bins: []string{"teleport", "tsh"}, + svcOrig: []byte("orig"), + links: []symlink{ + {oldname: "bin/teleport", newname: "bin/teleport"}, + }, + svcCopy: []byte("[service=orig][path=bin][flags=[]]"), + }, + { + name: "wrong teleport link", + bins: []string{"teleport", "tsh"}, + svcOrig: []byte("orig"), + links: []symlink{ + {oldname: "other", newname: "bin/teleport"}, + {oldname: "bin/tsh", newname: "bin/tsh"}, + }, + svcCopy: []byte("[service=orig][path=bin][flags=[]]"), + remaining: []string{servicePath, "bin/teleport"}, + }, + { + name: "wrong other link", + bins: []string{"teleport", "tsh"}, + svcOrig: []byte("orig"), + links: []symlink{ + {oldname: "bin/teleport", newname: "bin/teleport"}, + {oldname: "wrong", newname: "bin/tsh"}, + }, + svcCopy: []byte("[service=orig][path=bin][flags=[]]"), + remaining: []string{"bin/tsh"}, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + versionsDir := t.TempDir() + versionDir := filepath.Join(versionsDir, version) + err := os.MkdirAll(versionDir, 0o755) + require.NoError(t, err) + linkDir := t.TempDir() + + var files []smallFile + for _, n := range tt.bins { + files = append(files, smallFile{ + name: filepath.Join(versionDir, "bin", n), + data: []byte("binary"), + mode: os.ModePerm, + }) + } + if tt.svcOrig != nil { + files = append(files, smallFile{ + name: filepath.Join(versionDir, servicePath), + data: tt.svcOrig, + mode: os.ModePerm, + }) + } + if tt.svcCopy != nil { + files = append(files, smallFile{ + name: filepath.Join(linkDir, servicePath), + data: tt.svcCopy, + mode: os.ModePerm, + }) + } + + for _, n := range files { + err = os.MkdirAll(filepath.Dir(n.name), os.ModePerm) + require.NoError(t, err) + err = os.WriteFile(n.name, n.data, n.mode) + require.NoError(t, err) + } + for _, n := range tt.links { + newname := filepath.Join(linkDir, n.newname) + oldname := filepath.Join(versionDir, n.oldname) + err = os.MkdirAll(filepath.Dir(newname), os.ModePerm) + require.NoError(t, err) + err = os.Symlink(oldname, newname) + require.NoError(t, err) + } + + installer := &LocalInstaller{ + InstallDir: versionsDir, + TargetServiceFile: filepath.Join(linkDir, serviceDir, serviceName), + Log: slog.Default(), + TransformService: func(b []byte, pathDir string, flags autoupdate.InstallFlags) []byte { + return []byte(fmt.Sprintf("[service=%s][path=%s][flags=%s]", string(b), filepath.Base(pathDir), flags.Strings())) + }, + } + ctx := context.Background() + err = installer.Unlink(ctx, NewRevision(version, 0), filepath.Join(linkDir, "bin")) + if tt.errMatch != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMatch) + } else { + require.NoError(t, err) + } + for _, n := range tt.remaining { + _, err = os.Lstat(filepath.Join(linkDir, n)) + require.NoError(t, err) + } + for _, n := range tt.links { + if slices.Contains(tt.remaining, n.newname) { + continue + } + _, err = os.Lstat(filepath.Join(linkDir, n.newname)) + require.ErrorIs(t, err, os.ErrNotExist) + } + if !slices.Contains(tt.remaining, servicePath) { + _, err = os.Lstat(filepath.Join(linkDir, servicePath)) + require.ErrorIs(t, err, os.ErrNotExist) + } + }) + } +} + +func TestLocalInstaller_List(t *testing.T) { + installDir := t.TempDir() + versions := []string{"v1", "v2"} + + for _, d := range versions { + err := os.Mkdir(filepath.Join(installDir, d), os.ModePerm) + require.NoError(t, err) + } + for _, n := range []string{"file1", "file2"} { + err := os.WriteFile(filepath.Join(installDir, n), []byte(filepath.Base(n)), os.ModePerm) + require.NoError(t, err) + } + installer := &LocalInstaller{ + InstallDir: installDir, + Log: slog.Default(), + } + ctx := context.Background() + revisions, err := installer.List(ctx) + require.NoError(t, err) + require.Equal(t, []Revision{ + NewRevision("v1", 0), + NewRevision("v2", 0), + }, revisions) +} diff --git a/lib/autoupdate/agent/logger.go b/lib/autoupdate/agent/logger.go new file mode 100644 index 0000000000000..bd50ee50859a4 --- /dev/null +++ b/lib/autoupdate/agent/logger.go @@ -0,0 +1,108 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "bytes" + "context" + "fmt" + "log/slog" + + "github.com/gravitational/trace" +) + +// progressLogger logs progress of any data written as it approaches max. +// max(lines, call_count(Write)) lines are written for each multiple of max. +// progressLogger uses the variability of chunk size as a proxy for speed, and avoids +// logging extraneous lines that do not improve UX for waiting humans. +type progressLogger struct { + ctx context.Context + log *slog.Logger + level slog.Level + name string + max int + lines int + + l int + n int +} + +func (w *progressLogger) Write(p []byte) (n int, err error) { + w.n += len(p) + if w.n >= w.max*(w.l+1)/w.lines { + w.log.Log(w.ctx, w.level, "Downloading", + "file", w.name, + "progress", fmt.Sprintf("%d%%", w.n*100/w.max), + ) + w.l++ + } + return len(p), nil +} + +// lineLogger logs each line written to it. +type lineLogger struct { + ctx context.Context + log *slog.Logger + level slog.Level + prefix string + + last bytes.Buffer +} + +func (w *lineLogger) out(s string) { + w.log.Log(w.ctx, w.level, w.prefix+s) //nolint:sloglint // msg cannot be constant +} + +func (w *lineLogger) Write(p []byte) (n int, err error) { + lines := bytes.Split(p, []byte("\n")) + // Finish writing line + if len(lines) > 0 { + n, err = w.last.Write(lines[0]) + lines = lines[1:] + } + // Quit if no newline + if len(lines) == 0 || err != nil { + return n, trace.Wrap(err) + } + + // Newline found, log line + w.out(w.last.String()) + n += 1 + w.last.Reset() + + // Log lines that are already newline-terminated + for _, line := range lines[:len(lines)-1] { + w.out(string(line)) + n += len(line) + 1 + } + + // Store remaining line non-newline-terminated line. + n2, err := w.last.Write(lines[len(lines)-1]) + n += n2 + return n, trace.Wrap(err) +} + +// Flush logs any trailing bytes that were never terminated with a newline. +func (w *lineLogger) Flush() { + if w.last.Len() == 0 { + return + } + w.out(w.last.String()) + w.last.Reset() +} diff --git a/lib/autoupdate/agent/logger_test.go b/lib/autoupdate/agent/logger_test.go new file mode 100644 index 0000000000000..becc4a70a1ac7 --- /dev/null +++ b/lib/autoupdate/agent/logger_test.go @@ -0,0 +1,187 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "bytes" + "context" + "fmt" + "io" + "log/slog" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLineLogger(t *testing.T) { + t.Parallel() + + out := &bytes.Buffer{} + ll := lineLogger{ + ctx: context.Background(), + log: slog.New(slog.NewTextHandler(out, + &slog.HandlerOptions{ReplaceAttr: msgOnly}, + )), + } + + for _, e := range []struct { + v string + n int + }{ + {v: "", n: 0}, + {v: "a", n: 1}, + {v: "b\n", n: 2}, + {v: "c\nd", n: 3}, + {v: "e\nf\ng", n: 5}, + {v: "h", n: 1}, + {v: "", n: 0}, + {v: "\n", n: 1}, + {v: "i\n", n: 2}, + {v: "j", n: 1}, + } { + n, err := ll.Write([]byte(e.v)) + require.NoError(t, err) + require.Equal(t, e.n, n) + } + require.Equal(t, "msg=ab\nmsg=c\nmsg=de\nmsg=f\nmsg=gh\nmsg=i\n", out.String()) + ll.Flush() + require.Equal(t, "msg=ab\nmsg=c\nmsg=de\nmsg=f\nmsg=gh\nmsg=i\nmsg=j\n", out.String()) +} + +func msgOnly(_ []string, a slog.Attr) slog.Attr { + switch a.Key { + case "time", "level": + return slog.Attr{} + } + return slog.Attr{Key: a.Key, Value: a.Value} +} + +func TestProgressLogger(t *testing.T) { + t.Parallel() + + type write struct { + n int + out string + } + for _, tt := range []struct { + name string + max, lines int + writes []write + }{ + { + name: "even", + max: 100, + lines: 5, + writes: []write{ + {n: 10}, + {n: 10, out: "20%"}, + {n: 10}, + {n: 10, out: "40%"}, + {n: 10}, + {n: 10, out: "60%"}, + {n: 10}, + {n: 10, out: "80%"}, + {n: 10}, + {n: 10, out: "100%"}, + {n: 10}, + {n: 10, out: "120%"}, + }, + }, + { + name: "fast", + max: 100, + lines: 5, + writes: []write{ + {n: 100, out: "100%"}, + {n: 100, out: "200%"}, + }, + }, + { + name: "over fast", + max: 100, + lines: 5, + writes: []write{ + {n: 200, out: "200%"}, + }, + }, + { + name: "slow down when uneven", + max: 100, + lines: 5, + writes: []write{ + {n: 50, out: "50%"}, + {n: 10, out: "60%"}, + {n: 10, out: "70%"}, + {n: 10, out: "80%"}, + {n: 10}, + {n: 10, out: "100%"}, + {n: 10}, + {n: 10, out: "120%"}, + }, + }, + { + name: "slow down when very uneven", + max: 100, + lines: 5, + writes: []write{ + {n: 50, out: "50%"}, + {n: 1, out: "51%"}, + {n: 1}, + {n: 20, out: "72%"}, + {n: 10, out: "82%"}, + {n: 10}, + {n: 10, out: "102%"}, + }, + }, + { + name: "close", + max: 1000, + lines: 5, + writes: []write{ + {n: 999, out: "99%"}, + {n: 1, out: "100%"}, + }, + }, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + out := &bytes.Buffer{} + ll := progressLogger{ + ctx: context.Background(), + log: slog.New(slog.NewTextHandler(out, + &slog.HandlerOptions{ReplaceAttr: msgOnly}, + )), + name: "test", + max: tt.max, + lines: tt.lines, + } + for _, e := range tt.writes { + n, err := ll.Write(make([]byte, e.n)) + require.NoError(t, err) + require.Equal(t, e.n, n) + v, err := io.ReadAll(out) + require.NoError(t, err) + if len(v) > 0 { + e.out = fmt.Sprintf(`msg=Downloading file=test progress=%s`+"\n", e.out) + } + require.Equal(t, e.out, string(v)) + } + }) + } +} diff --git a/lib/autoupdate/agent/process.go b/lib/autoupdate/agent/process.go new file mode 100644 index 0000000000000..0aee3a10a924d --- /dev/null +++ b/lib/autoupdate/agent/process.go @@ -0,0 +1,482 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "bytes" + "context" + "errors" + "log/slog" + "os" + "os/exec" + "strconv" + "strings" + "syscall" + "time" + + "github.com/gravitational/trace" + "golang.org/x/sync/errgroup" +) + +// process monitoring consts +const ( + // monitorTimeout is the timeout for determining whether the process has started. + monitorTimeout = 1 * time.Minute + // monitorInterval is the polling interval for determining whether the process has started. + monitorInterval = 2 * time.Second + // minRunningIntervalsBeforeStable is the number of consecutive intervals with the same running PID detected + // before the service is determined stable. + minRunningIntervalsBeforeStable = 6 + // maxCrashesBeforeFailure is the number of total crashes detected before the service is marked as crash-looping. + maxCrashesBeforeFailure = 2 +) + +// log keys +const ( + unitKey = "unit" +) + +// SystemdService manages a systemd service (e.g., teleport or teleport-update). +type SystemdService struct { + // ServiceName specifies the systemd service name. + ServiceName string + // PIDFile is a path to a file containing the service's PID. + PIDFile string + // Log contains a logger. + Log *slog.Logger +} + +// Reload the systemd service. +// Attempts a graceful reload before a hard restart. +// See Process interface for more details. +func (s SystemdService) Reload(ctx context.Context) error { + // TODO(sclevine): allow server to force restart instead of reload + + if err := s.checkSystem(ctx); err != nil { + return trace.Wrap(err) + } + + // Command error codes < 0 indicate that we are unable to run the command. + // Errors from s.systemctl are logged along with stderr and stdout (debug only). + + // If the service is not running, return ErrNotNeeded. + // Note systemctl reload returns an error if the unit is not active, and + // try-reload-or-restart is too recent of an addition for centos7. + code := s.systemctl(ctx, slog.LevelDebug, "is-active", "--quiet", s.ServiceName) + switch { + case code < 0: + return trace.Errorf("unable to determine if systemd service is active") + case code > 0: + s.Log.WarnContext(ctx, "Systemd service not running.", unitKey, s.ServiceName) + return trace.Wrap(ErrNotNeeded) + } + + // Get initial PID for crash monitoring. + + initPID, err := readInt(s.PIDFile) + if errors.Is(err, os.ErrNotExist) { + s.Log.InfoContext(ctx, "No existing process detected. Skipping crash monitoring.", unitKey, s.ServiceName) + } else if err != nil { + s.Log.ErrorContext(ctx, "Error reading initial PID value. Skipping crash monitoring.", unitKey, s.ServiceName, errorKey, err) + } + + // Attempt graceful reload of running service. + code = s.systemctl(ctx, slog.LevelError, "reload", s.ServiceName) + switch { + case code < 0: + return trace.Errorf("unable to reload systemd service") + case code > 0: + // Graceful reload fails, try hard restart. + code = s.systemctl(ctx, slog.LevelError, "try-restart", s.ServiceName) + if code != 0 { + return trace.Errorf("hard restart of systemd service failed") + } + s.Log.WarnContext(ctx, "Service ungracefully restarted. Connections potentially dropped.", unitKey, s.ServiceName) + default: + s.Log.InfoContext(ctx, "Gracefully reloaded.", unitKey, s.ServiceName) + } + // monitor logs all relevant errors, so we filter for a few outcomes + err = s.monitor(ctx, initPID) + if errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, context.Canceled) { + return trace.Wrap(err) + } + if err != nil { + return trace.Errorf("failed to monitor process") + } + return nil +} + +// monitor for a started, healthy process. +// monitor logs all errors that should be displayed to the user. +func (s SystemdService) monitor(ctx context.Context, initPID int) error { + ctx, cancel := context.WithTimeout(ctx, monitorTimeout) + defer cancel() + ticker := time.NewTicker(monitorInterval) + defer ticker.Stop() + + if initPID != 0 { + s.Log.InfoContext(ctx, "Monitoring PID file to detect crashes.", unitKey, s.ServiceName) + var err error + _, err = s.monitorPID(ctx, initPID, ticker.C) + if errors.Is(err, context.DeadlineExceeded) { + s.Log.ErrorContext(ctx, "Timed out monitoring for crashing PID.", unitKey, s.ServiceName) + return trace.Wrap(err) + } + if err != nil { + s.Log.ErrorContext(ctx, "Error monitoring for crashing PID.", errorKey, err, unitKey, s.ServiceName) + return trace.Wrap(err) + } + } + + s.Log.InfoContext(ctx, "Monitoring diagnostic socket to detect readiness.", unitKey, s.ServiceName) + ticker = time.NewTicker(monitorInterval) + defer ticker.Stop() + // There is no debug service in v14, ignoring the readiness check + return nil +} + +// monitorPID for the started process to ensure it's running by polling PIDFile. +// This function detects several types of crashes while minimizing its own runtime during updates. +// For example, the process may crash by failing to fork (non-running PID), or looping (repeatedly changing PID), +// or getting stuck on quit (no change in PID). +// initPID is the PID before the restart operation has been issued. +// The final PID is returned. +func (s SystemdService) monitorPID(ctx context.Context, initPID int, tickC <-chan time.Time) (int, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + pidC := make(chan int) + var g errgroup.Group + g.Go(func() error { + return tickFile(ctx, s.PIDFile, pidC, tickC) + }) + stablePID, err := s.waitForStablePID(ctx, minRunningIntervalsBeforeStable, maxCrashesBeforeFailure, + initPID, pidC, func(pid int) error { + p, err := os.FindProcess(pid) + if err != nil { + return trace.Wrap(err) + } + return trace.Wrap(p.Signal(syscall.Signal(0))) + }) + cancel() + if err := g.Wait(); err != nil { + s.Log.ErrorContext(ctx, "Error monitoring for crashing process.", errorKey, err, unitKey, s.ServiceName) + } + return stablePID, trace.Wrap(err) +} + +// waitForStablePID monitors a service's PID via pidC and determines whether the service is crashing. +// verifyPID must be passed so that waitForStablePID can determine whether the process is running. +// verifyPID must return os.ErrProcessDone in the case that the PID cannot be found, or nil otherwise. +// baselinePID is the initial PID before any operation that might cause the process to start crashing. +// minStable is the number of times pidC must return the same running PID before waitForStablePID returns nil. +// minCrashes is the number of times pidC conveys a process crash or bad state before waitForStablePID returns an error. +// The last reported PID is returned. +func (s SystemdService) waitForStablePID(ctx context.Context, minStable, maxCrashes, baselinePID int, pidC <-chan int, verifyPID func(pid int) error) (int, error) { + pid := baselinePID + var last, stale int + var crashes int + for stable := 0; stable < minStable; stable++ { + select { + case <-ctx.Done(): + return pid, ctx.Err() + case p := <-pidC: + last = pid + pid = p + } + // A "crash" is defined as a transition away from a new (non-baseline) PID, or + // an interval where the current PID remains non-running (stale) since the last check. + if (last != 0 && pid != last && last != baselinePID) || + (stale != 0 && pid == stale && last == stale) { + crashes++ + } + if crashes > maxCrashes { + return pid, trace.Errorf("detected crashing process") + } + + // PID can only be stable if it is a real PID that is not new, + // has changed at least once, and hasn't been observed as missing. + if pid == 0 || + pid == baselinePID || + pid == stale || + pid != last { + stable = -1 + continue + } + err := verifyPID(pid) + // A stale PID most likely indicates that the process forked and crashed without systemd noticing. + // There is a small chance that we read the PID file before systemd removed it. + // Note: we only perform this check on PIDs that survive one iteration. + if errors.Is(err, os.ErrProcessDone) || + errors.Is(err, syscall.ESRCH) { + if pid != stale && + pid != baselinePID { + stale = pid + s.Log.WarnContext(ctx, "Detected stale PID.", unitKey, s.ServiceName, "pid", stale) + } + stable = -1 + continue + } + if err != nil { + return pid, trace.Wrap(err) + } + } + return pid, nil +} + +// readInt reads an integer from a file. +func readInt(path string) (int, error) { + p, err := readFileAtMost(path, 32) + if err != nil { + return 0, trace.Wrap(err) + } + i, err := strconv.ParseInt(string(bytes.TrimSpace(p)), 10, 64) + if err != nil { + return 0, trace.Wrap(err) + } + return int(i), nil +} + +// tickFile reads the current time on tickC, and outputs the last read int from path on ch for each received tick. +// If the path cannot be read, tickFile sends 0 on ch. +// Any error from the last attempt to read path is returned when ctx is canceled, unless the error is os.ErrNotExist. +func tickFile(ctx context.Context, path string, ch chan<- int, tickC <-chan time.Time) error { + var err error + for { + // two select statements -> never skip reads + select { + case <-tickC: + case <-ctx.Done(): + return err + } + var t int + t, err = readInt(path) + if errors.Is(err, os.ErrNotExist) { + err = nil + } + select { + case ch <- t: + case <-ctx.Done(): + return err + } + } +} + +// Sync systemd service configuration by running systemctl daemon-reload. +// See Process interface for more details. +func (s SystemdService) Sync(ctx context.Context) error { + if err := s.checkSystem(ctx); err != nil { + return trace.Wrap(err) + } + code := s.systemctl(ctx, slog.LevelError, "daemon-reload") + if code != 0 { + return trace.Errorf("unable to reload systemd configuration") + } + s.Log.InfoContext(ctx, "Systemd configuration synced.", unitKey, s.ServiceName) + return nil +} + +// Enable the systemd service. +func (s SystemdService) Enable(ctx context.Context, now bool) error { + if err := s.checkSystem(ctx); err != nil { + return trace.Wrap(err) + } + // The --now flag is not supported in systemd versions older than 220, + // so perform enable + start commands instead. + code := s.systemctl(ctx, slog.LevelInfo, "enable", s.ServiceName) + if code != 0 { + return trace.Errorf("unable to enable systemd service") + } + if now { + code := s.systemctl(ctx, slog.LevelInfo, "start", s.ServiceName) + if code != 0 { + return trace.Errorf("unable to start systemd service") + } + } + s.Log.InfoContext(ctx, "Systemd service enabled.", unitKey, s.ServiceName, "now", now) + return nil +} + +// Disable the systemd service. +func (s SystemdService) Disable(ctx context.Context, now bool) error { + if err := s.checkSystem(ctx); err != nil { + return trace.Wrap(err) + } + // The --now flag is not supported in systemd versions older than 220, + // so perform disable + stop commands instead. + code := s.systemctl(ctx, slog.LevelInfo, "disable", s.ServiceName) + if code != 0 { + return trace.Errorf("unable to disable systemd service") + } + if now { + code := s.systemctl(ctx, slog.LevelInfo, "stop", s.ServiceName) + if code != 0 { + return trace.Errorf("unable to stop systemd service") + } + } + s.Log.InfoContext(ctx, "Systemd service disabled.", unitKey, s.ServiceName, "now", now) + return nil +} + +// IsEnabled returns true if the service is enabled. +func (s SystemdService) IsEnabled(ctx context.Context) (bool, error) { + if err := s.checkSystem(ctx); err != nil { + return false, trace.Wrap(err) + } + if hasSystemDBelow(ctx, 238) { + return false, trace.Wrap(ErrNotAvailable) + } + code := s.systemctl(ctx, slog.LevelDebug, "is-enabled", "--quiet", s.ServiceName) + switch { + case code < 0: + return false, trace.Errorf("unable to determine if systemd service %s is enabled", s.ServiceName) + case code == 0: + return true, nil + } + return false, nil +} + +// IsActive returns true if the service is active. +func (s SystemdService) IsActive(ctx context.Context) (bool, error) { + if err := s.checkSystem(ctx); err != nil { + return false, trace.Wrap(err) + } + code := s.systemctl(ctx, slog.LevelDebug, "is-active", "--quiet", s.ServiceName) + switch { + case code < 0: + return false, trace.Errorf("unable to determine if systemd service %s is active", s.ServiceName) + case code == 0: + return true, nil + } + return false, nil +} + +// IsPresent returns true if the service exists. +func (s SystemdService) IsPresent(ctx context.Context) (bool, error) { + if err := s.checkSystem(ctx); err != nil { + return false, trace.Wrap(err) + } + if hasSystemDBelow(ctx, 246) { + return false, trace.Wrap(ErrNotAvailable) + } + code := s.systemctl(ctx, slog.LevelDebug, "list-unit-files", "--quiet", s.ServiceName) + if code < 0 { + return false, trace.Errorf("unable to determine if systemd service %s is present", s.ServiceName) + } + return code == 0, nil +} + +// checkSystem returns an error if the system is not compatible with this process manager. +func (s SystemdService) checkSystem(ctx context.Context) error { + present, err := hasSystemD() + if err != nil { + return trace.Wrap(err) + } + if !present { + return trace.Wrap(ErrNotSupported) + } + return nil +} + +// hasSystemD returns true if the system uses the SystemD process manager. +func hasSystemD() (bool, error) { + _, err := os.Stat("/run/systemd/system") + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, trace.Wrap(err) + } + return true, nil +} + +// hasSystemDBelow returns true the version of systemd can be determined, and it +// is below the provided version. +func hasSystemDBelow(ctx context.Context, i int) bool { + cmd := exec.CommandContext(ctx, "systemctl", "--version") + out, err := cmd.Output() + if err != nil { + return false + } + v, ok := parseSystemDVersion(out) + return ok && v < i +} + +// parseSystemDVersion parses the SystemD version from systemctl command output. +func parseSystemDVersion(out []byte) (int, bool) { + first, _, _ := strings.Cut(string(out), "\n") + parts := strings.SplitN(first, " ", 3) + if len(parts) < 2 || parts[0] != "systemd" { + return 0, false + } + version, err := strconv.Atoi(parts[1]) + if err != nil { + return 0, false + } + return version, true +} + +// systemctl returns a systemctl subcommand, converting the output to logs. +// Output sent to stdout is logged at debug level. +// Output sent to stderr is logged at the level specified by errLevel. +func (s SystemdService) systemctl(ctx context.Context, errLevel slog.Level, args ...string) int { + cmd := &localExec{ + Log: s.Log, + ErrLevel: errLevel, + OutLevel: slog.LevelDebug, + } + code, err := cmd.Run(ctx, "systemctl", args...) + if err == nil { + return code + } + if code >= 0 { + s.Log.Log(ctx, errLevel, "Non-zero exit code or error running systemctl.", + "args", args, "code", code) + return code + } + s.Log.Log(ctx, errLevel, "Unable to run systemctl.", + "args", args, "code", code, errorKey, err) + return code +} + +// localExec runs a command locally, logging any output. +type localExec struct { + // Log contains a slog logger. + // Defaults to slog.Default() if nil. + Log *slog.Logger + // ErrLevel is the log level for stderr. + ErrLevel slog.Level + // OutLevel is the log level for stdout. + OutLevel slog.Level +} + +// Run the command. Same arguments as exec.CommandContext. +// Outputs the status code, or -1 if out-of-range or unstarted. +func (c *localExec) Run(ctx context.Context, name string, args ...string) (int, error) { + cmd := exec.CommandContext(ctx, name, args...) + stderr := &lineLogger{ctx: ctx, log: c.Log, level: c.ErrLevel, prefix: "[stderr] "} + stdout := &lineLogger{ctx: ctx, log: c.Log, level: c.OutLevel, prefix: "[stdout] "} + cmd.Stderr = stderr + cmd.Stdout = stdout + err := cmd.Run() + stderr.Flush() + stdout.Flush() + code := cmd.ProcessState.ExitCode() + return code, trace.Wrap(err) +} diff --git a/lib/autoupdate/agent/process_test.go b/lib/autoupdate/agent/process_test.go new file mode 100644 index 0000000000000..5446b9b4109db --- /dev/null +++ b/lib/autoupdate/agent/process_test.go @@ -0,0 +1,360 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestWaitForStablePID(t *testing.T) { + t.Parallel() + + svc := &SystemdService{ + Log: slog.Default(), + } + + for _, tt := range []struct { + name string + ticks []int + baseline int + minStable int + maxCrashes int + findErrs map[int]error + + finalPID int + errored bool + canceled bool + }{ + { + name: "immediate restart", + ticks: []int{2, 2}, + baseline: 1, + minStable: 1, + maxCrashes: 1, + finalPID: 2, + }, + { + name: "zero stable", + }, + { + name: "immediate crash", + ticks: []int{2, 3}, + baseline: 1, + minStable: 1, + maxCrashes: 0, + errored: true, + finalPID: 3, + }, + { + name: "no changes times out", + ticks: []int{1, 1, 1, 1}, + baseline: 1, + minStable: 3, + maxCrashes: 2, + canceled: true, + finalPID: 1, + }, + { + name: "baseline restart", + ticks: []int{2, 2, 2, 2}, + baseline: 1, + minStable: 3, + maxCrashes: 2, + finalPID: 2, + }, + { + name: "one restart then stable", + ticks: []int{1, 2, 2, 2, 2}, + baseline: 1, + minStable: 3, + maxCrashes: 2, + finalPID: 2, + }, + { + name: "two restarts then stable", + ticks: []int{1, 2, 3, 3, 3, 3}, + baseline: 1, + minStable: 3, + maxCrashes: 2, + finalPID: 3, + }, + { + name: "three restarts then stable", + ticks: []int{1, 2, 3, 4, 4, 4, 4}, + baseline: 1, + minStable: 3, + maxCrashes: 2, + finalPID: 4, + }, + { + name: "too many restarts excluding baseline", + ticks: []int{1, 2, 3, 4, 5}, + baseline: 1, + minStable: 3, + maxCrashes: 2, + errored: true, + finalPID: 5, + }, + { + name: "too many restarts including baseline", + ticks: []int{1, 2, 3, 4}, + baseline: 0, + minStable: 3, + maxCrashes: 2, + errored: true, + finalPID: 4, + }, + { + name: "too many restarts slow", + ticks: []int{1, 1, 1, 2, 2, 2, 3, 3, 3, 4}, + baseline: 0, + minStable: 3, + maxCrashes: 2, + errored: true, + finalPID: 4, + }, + { + name: "too many restarts after stable", + ticks: []int{1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4}, + baseline: 0, + minStable: 3, + maxCrashes: 2, + finalPID: 3, + }, + { + name: "stable after too many restarts", + ticks: []int{1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4}, + baseline: 0, + minStable: 3, + maxCrashes: 2, + errored: true, + finalPID: 4, + }, + { + name: "cancel", + ticks: []int{1, 1, 1}, + baseline: 0, + minStable: 3, + maxCrashes: 2, + canceled: true, + finalPID: 1, + }, + { + name: "stale PID crash", + ticks: []int{2, 2, 2, 2, 2}, + baseline: 1, + minStable: 3, + maxCrashes: 2, + findErrs: map[int]error{ + 2: os.ErrProcessDone, + }, + errored: true, + finalPID: 2, + }, + { + name: "stale PID but fixed", + ticks: []int{2, 2, 3, 3, 3, 3}, + baseline: 1, + minStable: 3, + maxCrashes: 2, + findErrs: map[int]error{ + 2: os.ErrProcessDone, + }, + finalPID: 3, + }, + { + name: "error PID", + ticks: []int{2, 2, 3, 3, 3, 3}, + baseline: 1, + minStable: 3, + maxCrashes: 2, + findErrs: map[int]error{ + 2: errors.New("bad"), + }, + errored: true, + finalPID: 2, + }, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + ch := make(chan int) + go func() { + defer cancel() // always quit after last tick + for _, tick := range tt.ticks { + ch <- tick + } + }() + pid, err := svc.waitForStablePID(ctx, tt.minStable, tt.maxCrashes, + tt.baseline, ch, func(pid int) error { + return tt.findErrs[pid] + }) + require.Equal(t, tt.finalPID, pid) + require.Equal(t, tt.canceled, errors.Is(err, context.Canceled)) + if !tt.canceled { + require.Equal(t, tt.errored, err != nil) + } + }) + } +} + +func TestTickFile(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + ticks []int + errored bool + }{ + { + name: "consistent", + ticks: []int{1, 1, 1}, + errored: false, + }, + { + name: "divergent", + ticks: []int{1, 2, 3}, + errored: false, + }, + { + name: "start error", + ticks: []int{-1, 1, 1}, + errored: false, + }, + { + name: "ephemeral error", + ticks: []int{1, -1, 1}, + errored: false, + }, + { + name: "end error", + ticks: []int{1, 1, -1}, + errored: true, + }, + { + name: "start missing", + ticks: []int{0, 1, 1}, + errored: false, + }, + { + name: "ephemeral missing", + ticks: []int{1, 0, 1}, + errored: false, + }, + { + name: "end missing", + ticks: []int{1, 1, 0}, + errored: false, + }, + { + name: "cancel-only", + errored: false, + }, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "file") + + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + tickC := make(chan time.Time) + ch := make(chan int) + + go func() { + defer cancel() // always quit after last tick or fail + for _, tick := range tt.ticks { + _ = os.RemoveAll(filePath) + switch { + case tick > 0: + err := os.WriteFile(filePath, []byte(fmt.Sprintln(tick)), os.ModePerm) + require.NoError(t, err) + case tick < 0: + err := os.Mkdir(filePath, os.ModePerm) + require.NoError(t, err) + } + tickC <- time.Now() + res := <-ch + if tick < 0 { + tick = 0 + } + require.Equal(t, tick, res) + } + }() + err := tickFile(ctx, filePath, ch, tickC) + require.Equal(t, tt.errored, err != nil) + }) + } +} + +func TestParseSystemdVersion(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name string + output string + version int + }{ + { + name: "valid", + output: "systemd 249 (249.4-1ubuntu1.1)\n+PAM +AUDIT\n", + version: 249, + }, + { + name: "short", + output: "systemd 249\n", + version: 249, + }, + { + name: "stripped", + output: "systemd 249", + version: 249, + }, + { + name: "missing", + output: "systemd", + }, + { + name: "bad", + output: "not found", + }, + { + name: "empty", + }, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + v, ok := parseSystemDVersion([]byte(tt.output)) + if tt.version == 0 { + require.False(t, ok) + } + require.Equal(t, tt.version, v) + }) + } +} diff --git a/lib/autoupdate/agent/setup.go b/lib/autoupdate/agent/setup.go new file mode 100644 index 0000000000000..95c0d4a7f88c5 --- /dev/null +++ b/lib/autoupdate/agent/setup.go @@ -0,0 +1,553 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "bytes" + "context" + "errors" + "io/fs" + "log/slog" + "os" + "path/filepath" + "regexp" + "strings" + "text/template" + + "github.com/google/renameio/v2" + "github.com/gravitational/trace" + "gopkg.in/yaml.v3" + + "github.com/gravitational/teleport/lib/autoupdate" + "github.com/gravitational/teleport/lib/defaults" + libdefaults "github.com/gravitational/teleport/lib/defaults" + libutils "github.com/gravitational/teleport/lib/utils" +) + +// Base paths for constructing namespaced directories. +const ( + defaultInstallDir = "/opt/teleport" + defaultPathDir = "/usr/local/bin" + systemdAdminDir = "/etc/systemd/system" + systemdPIDDir = "/run" + needrestartConfDir = "/etc/needrestart/conf.d" + versionsDirName = "versions" + lockFileName = "update.lock" + defaultNamespace = "default" + systemNamespace = "system" +) + +const ( + // deprecatedTimerName is the timer for the deprecated upgrader should be disabled on setup. + deprecatedTimerName = "teleport-upgrade.timer" + // deprecatedServiceName is the service for the deprecated upgrader should be disabled on setup. + deprecatedServiceName = "teleport-upgrade.service" +) + +const ( + updateServiceTemplate = `# teleport-update +# DO NOT EDIT THIS FILE +[Unit] +Description=Teleport auto-update service + +[Service] +Type=oneshot +ExecStart={{.UpdaterBinary}} --install-suffix={{.InstallSuffix}} "--install-dir={{escape .InstallDir}}" update +` + updateTimerTemplate = `# teleport-update +# DO NOT EDIT THIS FILE +[Unit] +Description=Teleport auto-update timer unit + +[Timer] +OnActiveSec=1m +OnUnitActiveSec=5m +RandomizedDelaySec=1m + +[Install] +WantedBy={{.TeleportService}} +` + teleportDropInTemplate = `# teleport-update +# DO NOT EDIT THIS FILE +[Service] +Environment="TELEPORT_UPDATE_CONFIG_FILE={{escape .UpdaterConfigFile}}" +Environment="TELEPORT_UPDATE_INSTALL_DIR={{escape .InstallDir}}" +` + + deprecatedDropInTemplate = `# teleport-update +# DO NOT EDIT THIS FILE +[Service] +ExecStart= +ExecStart=-/bin/echo "The teleport-upgrade script has been disabled by teleport-update. Please remove the teleport-ent-updater package." +` + // This configuration sets the default value for needrestart-trigger automatic restarts for teleport.service to disabled. + // Users may still choose to enable needrestart for teleport.service when installing packaging interactively (or via dpkg config), + // but doing so will result in a hard restart that disconnects the agent whenever any dependent libraries are updated. + // Other network services, like openvpn, follow this pattern. + // It is possible to configure needrestart to trigger a soft restart (via restart.d script), but given that Teleport subprocesses + // can use a wide variety of installed binaries (when executed by the user), this could trigger many unexpected reloads. + needrestartConfTemplate = `$nrconf{override_rc}{qr(^{{replace .TeleportService "." "\\."}})} = 0; +` +) + +type confParams struct { + TeleportService string + UpdaterBinary string + InstallSuffix string + InstallDir string + Path string + UpdaterConfigFile string +} + +// Namespace represents a namespace within various system paths for a isolated installation of Teleport. +type Namespace struct { + log *slog.Logger + // name of namespace + name string + // installDir for Teleport namespaces (/opt/teleport) + installDir string + // defaultPathDir for Teleport binaries (ns: /opt/teleport/myns/bin) + defaultPathDir string + // dataDir parsed from teleport.yaml, if present + dataDir string + // defaultProxyAddr parsed from teleport.yaml, if present + defaultProxyAddr string + // serviceFile for the Teleport systemd service (ns: /etc/systemd/system/teleport_myns.service) + serviceFile string + // configFile for Teleport config (ns: /etc/teleport_myns.yaml) + configFile string + // pidFile for Teleport (ns: /run/teleport_myns.pid) + pidFile string + // updaterIDFile contains the updater's temporary ID file + updaterIDFile string + // updaterServiceFile is the systemd service path for the updater + updaterServiceFile string + // updaterTimerFile is the systemd timer path for the updater + updaterTimerFile string + // teleportDropInFile is the Teleport systemd drop-in path extending Teleport + teleportDropInFile string + // deprecatedDropInFile is the deprecated upgrader's systemd drop-in path + deprecatedDropInFile string + // needrestartConfFile is the path to needrestart configuration for Teleport + needrestartConfFile string +} + +var alphanum = regexp.MustCompile("^[a-zA-Z0-9-]*$") + +// NewNamespace validates and returns a Namespace. +// Namespaces must be alphanumeric + `-`. +// defaultPathDir overrides the destination directory for namespace setup (i.e., /usr/local) +func NewNamespace(ctx context.Context, log *slog.Logger, name, installDir string) (ns *Namespace, err error) { + defer func() { ns.overrideFromConfig(ctx) }() + + if name == defaultNamespace || + name == systemNamespace { + return nil, trace.Errorf("namespace %s is reserved", name) + } + if !alphanum.MatchString(name) { + return nil, trace.Errorf("invalid namespace name %s, must be alphanumeric", name) + } + if installDir == "" { + installDir = defaultInstallDir + } + if name == "" { + linkDir := defaultPathDir + return &Namespace{ + log: log, + name: name, + installDir: installDir, + defaultPathDir: linkDir, + dataDir: defaults.DataDir, + serviceFile: filepath.Join("/", serviceDir, serviceName), + configFile: defaults.ConfigFilePath, + pidFile: filepath.Join(systemdPIDDir, "teleport.pid"), + updaterIDFile: filepath.Join(os.TempDir(), BinaryName+".id"), + updaterServiceFile: filepath.Join(systemdAdminDir, BinaryName+".service"), + updaterTimerFile: filepath.Join(systemdAdminDir, BinaryName+".timer"), + teleportDropInFile: filepath.Join(systemdAdminDir, "teleport.service.d", BinaryName+".conf"), + deprecatedDropInFile: filepath.Join(systemdAdminDir, deprecatedServiceName+".d", BinaryName+".conf"), + needrestartConfFile: filepath.Join(needrestartConfDir, BinaryName+".conf"), + }, nil + } + + prefix := "teleport_" + name + linkDir := filepath.Join(installDir, name, "bin") + return &Namespace{ + log: log, + name: name, + installDir: installDir, + defaultPathDir: linkDir, + dataDir: filepath.Join(filepath.Dir(defaults.DataDir), prefix), + serviceFile: filepath.Join(systemdAdminDir, prefix+".service"), + configFile: filepath.Join(filepath.Dir(defaults.ConfigFilePath), prefix+".yaml"), + pidFile: filepath.Join(systemdPIDDir, prefix+".pid"), + updaterIDFile: filepath.Join(os.TempDir(), BinaryName+"_"+name+".id"), + updaterServiceFile: filepath.Join(systemdAdminDir, BinaryName+"_"+name+".service"), + updaterTimerFile: filepath.Join(systemdAdminDir, BinaryName+"_"+name+".timer"), + teleportDropInFile: filepath.Join(systemdAdminDir, prefix+".service.d", BinaryName+"_"+name+".conf"), + needrestartConfFile: filepath.Join(needrestartConfDir, BinaryName+"_"+name+".conf"), + // no deprecatedDropInFile, as teleport-upgrade does not conflict with namespaced installs + }, nil +} + +func (ns *Namespace) Dir() string { + name := ns.name + if name == "" { + name = defaultNamespace + } + return filepath.Join(ns.installDir, name) +} + +// Init creates the initial directory structure and returns the lockfile for a Namespace. +// Init should be called before the namespace is locked. +func (ns *Namespace) Init() (lockFile string, err error) { + if err := os.MkdirAll(filepath.Join(ns.Dir(), versionsDirName), systemDirMode); err != nil { + return "", trace.Wrap(err) + } + return filepath.Join(ns.Dir(), lockFileName), nil +} + +// Setup installs service and timer files for the teleport-update binary. +// Afterwords, Setup reloads systemd and enables the timer with --now. +func (ns *Namespace) Setup(ctx context.Context, path string) error { + if ok, err := hasSystemD(); err == nil && !ok { + ns.log.WarnContext(ctx, "Systemd is not running, skipping updater installation.") + return nil + } + + err := ns.writeConfigFiles(ctx, path) + if err != nil { + return trace.Wrap(err, "failed to write teleport-update systemd config files") + } + timer := &SystemdService{ + ServiceName: filepath.Base(ns.updaterTimerFile), + Log: ns.log, + } + if err := timer.Sync(ctx); err != nil { + return trace.Wrap(err, "failed to sync systemd config") + } + if err := timer.Enable(ctx, true); err != nil { + return trace.Wrap(err, "failed to enable teleport-update systemd timer") + } + if ns.name == "" { + oldTimer := &SystemdService{ + ServiceName: deprecatedTimerName, + Log: ns.log, + } + // If the old teleport-upgrade script is detected, disable it to ensure they do not interfere. + // Note that the schedule is also set to nop by the Teleport agent -- this just prevents restarts. + present, err := oldTimer.IsPresent(ctx) + if errors.Is(err, ErrNotAvailable) { // systemd too old + if err := oldTimer.Disable(ctx, true); err != nil { + ns.log.DebugContext(ctx, "The deprecated teleport-ent-updater package is either missing, or could not be disabled.", errorKey, err) + } + return nil + } + if err != nil { + return trace.Wrap(err, "failed to determine if deprecated teleport-upgrade systemd timer is present") + } + if present { + if err := oldTimer.Disable(ctx, true); err != nil { + ns.log.ErrorContext(ctx, "The deprecated teleport-ent-updater package is installed on this server and cannot be disabled due to an error.", errorKey, err) + ns.log.ErrorContext(ctx, "You must remove the teleport-ent-updater package after verifying that teleport-update is working.", errorKey, err) + } else { + ns.log.WarnContext(ctx, "The deprecated teleport-ent-updater package is installed on this server.") + ns.log.WarnContext(ctx, "The systemd timer included in this package has been disabled to prevent conflicts.", "timer", deprecatedTimerName) + ns.log.WarnContext(ctx, "The systemd service included in this package will no longer perform updates.", "service", deprecatedServiceName) + ns.log.WarnContext(ctx, "Please remove the teleport-ent-updater package after verifying that teleport-update is working.") + } + } + } + return nil +} + +// Teardown removes all traces of the auto-updater, including its configuration. +func (ns *Namespace) Teardown(ctx context.Context) error { + if ok, err := hasSystemD(); err == nil && !ok { + ns.log.WarnContext(ctx, "Systemd is not running, skipping updater removal.") + if err := os.RemoveAll(ns.Dir()); err != nil { + return trace.Wrap(err, "failed to remove versions directory") + } + return nil + } + + svc := &SystemdService{ + ServiceName: filepath.Base(ns.updaterTimerFile), + Log: ns.log, + } + if err := svc.Disable(ctx, true); err != nil { + ns.log.WarnContext(ctx, "Unable to disable teleport-update systemd timer before removing.") + ns.log.DebugContext(ctx, "Error disabling teleport-update systemd timer.", errorKey, err) + } + for _, p := range []string{ + ns.updaterServiceFile, + ns.updaterTimerFile, + ns.teleportDropInFile, + ns.deprecatedDropInFile, + ns.needrestartConfFile, + } { + if p == "" { + continue + } + if err := os.Remove(p); err != nil && !errors.Is(err, fs.ErrNotExist) { + return trace.Wrap(err, "failed to remove %s", filepath.Base(p)) + } + } + if err := svc.Sync(ctx); err != nil { + return trace.Wrap(err, "failed to sync systemd config") + } + if err := os.RemoveAll(ns.Dir()); err != nil { + return trace.Wrap(err, "failed to remove versions directory") + } + if ns.name == "" { + oldTimer := &SystemdService{ + ServiceName: deprecatedTimerName, + Log: ns.log, + } + // If the old upgrader exists, attempt to re-enable it automatically + present, err := oldTimer.IsPresent(ctx) + if errors.Is(err, ErrNotAvailable) { // systemd too old + if err := oldTimer.Enable(ctx, true); err != nil { + ns.log.DebugContext(ctx, "The deprecated teleport-ent-updater package is either missing, or could not be enabled.", errorKey, err) + } + return nil + } + if err != nil { + return trace.Wrap(err, "failed to determine if deprecated teleport-upgrade systemd timer is present") + } + if present { + if err := oldTimer.Enable(ctx, true); err != nil { + ns.log.ErrorContext(ctx, "The deprecated teleport-ent-updater package is installed on this server, and it cannot be re-enabled due to an error.", errorKey, err) + ns.log.ErrorContext(ctx, "Please fix the systemd timer included in the teleport-ent-updater package if you intend to use the deprecated updater.") + } else { + ns.log.WarnContext(ctx, "The deprecated teleport-ent-updater package is installed on this server.") + ns.log.WarnContext(ctx, "The systemd timer included in this package has been re-enabled to ensure continued updates.", "timer", deprecatedTimerName) + ns.log.WarnContext(ctx, "To disable updates entirely, please remove the teleport-ent-updater package.") + } + } + } + return nil +} + +func (ns *Namespace) writeConfigFiles(ctx context.Context, path string) error { + teleportService := filepath.Base(ns.serviceFile) + params := confParams{ + TeleportService: teleportService, + UpdaterBinary: filepath.Join(path, BinaryName), + InstallSuffix: ns.name, + InstallDir: ns.installDir, + Path: path, + UpdaterConfigFile: filepath.Join(ns.Dir(), updateConfigName), + } + + for _, v := range []struct { + path, tmpl string + }{ + {ns.updaterServiceFile, updateServiceTemplate}, + {ns.updaterTimerFile, updateTimerTemplate}, + {ns.teleportDropInFile, teleportDropInTemplate}, + {ns.deprecatedDropInFile, deprecatedDropInTemplate}, + } { + if v.path == "" { + continue + } + err := writeSystemTemplate(v.path, v.tmpl, params) + if err != nil { + return trace.Wrap(err) + } + } + // Needrestart config is non-critical for updater functionality. + _, err := os.Stat(filepath.Dir(ns.needrestartConfFile)) + if os.IsNotExist(err) { + return nil // needrestart is not present + } + if err != nil { + ns.log.ErrorContext(ctx, "Unable to disable needrestart.", errorKey, err) + return nil + } + ns.log.InfoContext(ctx, "Disabling needrestart.", unitKey, teleportService) + err = writeSystemTemplate(ns.needrestartConfFile, needrestartConfTemplate, params) + if err != nil { + ns.log.ErrorContext(ctx, "Unable to disable needrestart.", errorKey, err) + return nil + } + return nil +} + +// writeSystemTemplate atomically writes a template to a system file, creating any needed directories. +// Temporarily files are stored in the target path to ensure the file has needed SELinux contexts. +func writeSystemTemplate(path, t string, values any) error { + dir, file := filepath.Split(path) + if err := os.MkdirAll(dir, systemDirMode); err != nil { + return trace.Wrap(err) + } + opts := []renameio.Option{ + renameio.WithPermissions(configFileMode), + renameio.WithExistingPermissions(), + renameio.WithTempDir(dir), + } + f, err := renameio.NewPendingFile(path, opts...) + if err != nil { + return trace.Wrap(err) + } + defer f.Cleanup() + + tmpl, err := template.New(file).Funcs(template.FuncMap{ + "replace": func(s, old, new string) string { + return strings.ReplaceAll(s, old, new) + }, + // escape is a best-effort function for escaping quotes in systemd service templates. + // Paths that are escaped with this method should not be advertised to the user as + // configurable until a more robust escaping mechanism is shipped. + // See: https://www.freedesktop.org/software/systemd/man/latest/systemd.syntax.html + "escape": func(s string) string { + replacer := strings.NewReplacer( + `"`, `\"`, + `\`, `\\`, + ) + return replacer.Replace(s) + }, + }).Parse(t) + if err != nil { + return trace.Wrap(err) + } + err = tmpl.Execute(f, values) + if err != nil { + return trace.Wrap(err) + } + return trace.Wrap(f.CloseAtomicallyReplace()) +} + +// ReplaceTeleportService replaces the default paths in the Teleport service config with namespaced paths. +func (ns *Namespace) ReplaceTeleportService(cfg []byte, pathDir string, flags autoupdate.InstallFlags) []byte { + if pathDir == "" { + pathDir = ns.defaultPathDir + } + var startFlags []string + if flags&autoupdate.FlagFIPS != 0 { + startFlags = append(startFlags, "--fips") + } + for _, rep := range []struct { + old, new string + }{ + { + old: "/usr/local/bin/", + new: pathDir + "/", + }, + { + old: "/etc/teleport.yaml", + new: ns.configFile, + }, + { + old: "/run/teleport.pid", + new: ns.pidFile, + }, + { + old: "/teleport start ", + new: "/teleport start " + joinTerminal(startFlags, " "), + }, + } { + cfg = bytes.ReplaceAll(cfg, []byte(rep.old), []byte(rep.new)) + } + return cfg +} + +func joinTerminal(s []string, sep string) string { + v := strings.Join(s, sep) + if len(v) > 0 { + return v + sep + } + return v +} + +func (ns *Namespace) LogWarnings(ctx context.Context, pathDir string) { + if ns.name == "" { + return + } + if pathDir == "" { + pathDir = ns.defaultPathDir + } + ns.log.WarnContext(ctx, "Custom install suffix specified. Teleport data_dir must be configured in the config file.", + "data_dir", ns.dataDir, + "path_dir", pathDir, + "config_file", ns.configFile, + "service", filepath.Base(ns.serviceFile), + "pid_file", ns.pidFile, + ) +} + +// unversionedConfig is used to read all versions of teleport.yaml, including +// versions that may now be unsupported. +type unversionedConfig struct { + Teleport unversionedTeleport `yaml:"teleport"` +} + +type unversionedTeleport struct { + AuthServers []string `yaml:"auth_servers"` + AuthServer string `yaml:"auth_server"` + ProxyServer string `yaml:"proxy_server"` + DataDir string `yaml:"data_dir"` +} + +// overrideFromConfig loads fields from teleport.yaml into the namespace, overriding any defaults. +func (ns *Namespace) overrideFromConfig(ctx context.Context) { + if ns == nil || ns.configFile == "" { + return + } + path := ns.configFile + f, err := libutils.OpenFileAllowingUnsafeLinks(path) + if err != nil { + ns.log.DebugContext(ctx, "Unable to open Teleport config to read proxy or data dir", "config", path, errorKey, err) + return + } + defer f.Close() + var cfg unversionedConfig + if err := yaml.NewDecoder(f).Decode(&cfg); err != nil { + ns.log.DebugContext(ctx, "Unable to parse Teleport config to read proxy or data dir", "config", path, errorKey, err) + return + } + if cfg.Teleport.DataDir != "" { + ns.dataDir = cfg.Teleport.DataDir + } + + // Any implicitly defaulted port in teleport.yaml is explicitly defaulted (to 3080). + + var addr string + var port int + switch t := cfg.Teleport; { + case t.ProxyServer != "": + addr = t.ProxyServer + port = libdefaults.HTTPListenPort + case t.AuthServer != "": + addr = t.AuthServer + port = libdefaults.AuthListenPort + case len(t.AuthServers) > 0: + addr = t.AuthServers[0] + port = libdefaults.AuthListenPort + default: + ns.log.DebugContext(ctx, "Unable to find proxy in Teleport config", "config", path, errorKey, err) + return + } + netaddr, err := libutils.ParseHostPortAddr(addr, port) + if err != nil { + ns.log.DebugContext(ctx, "Unable to parse proxy in Teleport config", "config", path, "proxy_addr", addr, "proxy_port", port, errorKey, err) + return + } + ns.defaultProxyAddr = netaddr.String() +} diff --git a/lib/autoupdate/agent/setup_test.go b/lib/autoupdate/agent/setup_test.go new file mode 100644 index 0000000000000..641d8a3623b15 --- /dev/null +++ b/lib/autoupdate/agent/setup_test.go @@ -0,0 +1,463 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "bytes" + "context" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/gravitational/teleport/lib/autoupdate" + "github.com/gravitational/teleport/lib/config" + "github.com/gravitational/teleport/lib/utils/testutils/golden" +) + +func TestNewNamespace(t *testing.T) { + for _, p := range []struct { + name string + namespace string + installDir string + errMatch string + ns *Namespace + }{ + { + name: "no namespace", + ns: &Namespace{ + dataDir: "/var/lib/teleport", + installDir: "/opt/teleport", + defaultPathDir: "/usr/local/bin", + serviceFile: "/lib/systemd/system/teleport.service", + configFile: "/etc/teleport.yaml", + pidFile: "/run/teleport.pid", + updaterIDFile: "/TMP/teleport-update.id", + updaterServiceFile: "/etc/systemd/system/teleport-update.service", + updaterTimerFile: "/etc/systemd/system/teleport-update.timer", + teleportDropInFile: "/etc/systemd/system/teleport.service.d/teleport-update.conf", + deprecatedDropInFile: "/etc/systemd/system/teleport-upgrade.service.d/teleport-update.conf", + needrestartConfFile: "/etc/needrestart/conf.d/teleport-update.conf", + }, + }, + { + name: "no namespace with dirs", + installDir: "/install", + ns: &Namespace{ + dataDir: "/var/lib/teleport", + installDir: "/install", + defaultPathDir: "/usr/local/bin", + serviceFile: "/lib/systemd/system/teleport.service", + configFile: "/etc/teleport.yaml", + pidFile: "/run/teleport.pid", + updaterIDFile: "/TMP/teleport-update.id", + updaterServiceFile: "/etc/systemd/system/teleport-update.service", + updaterTimerFile: "/etc/systemd/system/teleport-update.timer", + teleportDropInFile: "/etc/systemd/system/teleport.service.d/teleport-update.conf", + deprecatedDropInFile: "/etc/systemd/system/teleport-upgrade.service.d/teleport-update.conf", + needrestartConfFile: "/etc/needrestart/conf.d/teleport-update.conf", + }, + }, + { + name: "test namespace", + namespace: "test", + ns: &Namespace{ + name: "test", + dataDir: "/var/lib/teleport_test", + installDir: "/opt/teleport", + defaultPathDir: "/opt/teleport/test/bin", + serviceFile: "/etc/systemd/system/teleport_test.service", + configFile: "/etc/teleport_test.yaml", + pidFile: "/run/teleport_test.pid", + updaterIDFile: "/TMP/teleport-update_test.id", + updaterServiceFile: "/etc/systemd/system/teleport-update_test.service", + updaterTimerFile: "/etc/systemd/system/teleport-update_test.timer", + teleportDropInFile: "/etc/systemd/system/teleport_test.service.d/teleport-update_test.conf", + needrestartConfFile: "/etc/needrestart/conf.d/teleport-update_test.conf", + }, + }, + { + name: "test namespace with dirs", + namespace: "test", + installDir: "/install", + ns: &Namespace{ + name: "test", + dataDir: "/var/lib/teleport_test", + installDir: "/install", + defaultPathDir: "/install/test/bin", + configFile: "/etc/teleport_test.yaml", + pidFile: "/run/teleport_test.pid", + serviceFile: "/etc/systemd/system/teleport_test.service", + updaterIDFile: "/TMP/teleport-update_test.id", + updaterServiceFile: "/etc/systemd/system/teleport-update_test.service", + updaterTimerFile: "/etc/systemd/system/teleport-update_test.timer", + teleportDropInFile: "/etc/systemd/system/teleport_test.service.d/teleport-update_test.conf", + needrestartConfFile: "/etc/needrestart/conf.d/teleport-update_test.conf", + }, + }, + { + name: "reserved default", + namespace: defaultNamespace, + errMatch: "reserved", + }, + { + name: "reserved system", + namespace: systemNamespace, + errMatch: "reserved", + }, + } { + p := p + t.Run(p.name, func(t *testing.T) { + log := slog.Default() + ctx := context.Background() + ns, err := NewNamespace(ctx, log, p.namespace, p.installDir) + if p.errMatch != "" { + require.Error(t, err) + require.Contains(t, err.Error(), p.errMatch) + return + } + require.NoError(t, err) + ns.log = nil + ns.updaterIDFile = strings.Replace(ns.updaterIDFile, + strings.TrimSuffix(os.TempDir(), "/"), "/TMP", 1, + ) + require.Equal(t, p.ns, ns) + }) + } +} + +func TestWriteConfigFiles(t *testing.T) { + for _, p := range []struct { + name string + namespace string + }{ + { + name: "no namespace", + }, + { + name: "test namespace", + namespace: "test", + }, + } { + p := p + t.Run(p.name, func(t *testing.T) { + log := slog.Default() + linkDir := t.TempDir() + ctx := context.Background() + ns, err := NewNamespace(ctx, log, p.namespace, "") + require.NoError(t, err) + ns.updaterServiceFile = rebasePath(filepath.Join(linkDir, serviceDir), filepath.Base(ns.updaterServiceFile)) + ns.updaterServiceFile = rebasePath(filepath.Join(linkDir, serviceDir), ns.updaterServiceFile) + ns.updaterTimerFile = rebasePath(filepath.Join(linkDir, serviceDir), ns.updaterTimerFile) + ns.teleportDropInFile = rebasePath(filepath.Join(linkDir, serviceDir, filepath.Base(filepath.Dir(ns.teleportDropInFile))), ns.teleportDropInFile) + ns.deprecatedDropInFile = rebasePath(filepath.Join(linkDir, serviceDir, filepath.Base(filepath.Dir(ns.deprecatedDropInFile))), ns.deprecatedDropInFile) + ns.needrestartConfFile = rebasePath(linkDir, filepath.Base(ns.needrestartConfFile)) + err = ns.writeConfigFiles(ctx, linkDir) + require.NoError(t, err) + + for _, tt := range []struct { + name string + path string + }{ + {name: "service", path: ns.updaterServiceFile}, + {name: "timer", path: ns.updaterTimerFile}, + {name: "dropin", path: ns.teleportDropInFile}, + {name: "deprecated", path: ns.deprecatedDropInFile}, + {name: "needrestart", path: ns.needrestartConfFile}, + } { + if tt.path == "" { + continue + } + t.Run(tt.name, func(t *testing.T) { + data, err := os.ReadFile(tt.path) + require.NoError(t, err) + data = replaceValues(data, map[string]string{ + defaultPathDir: linkDir, + }) + if golden.ShouldSet() { + golden.Set(t, data) + } + require.Equal(t, string(golden.Get(t)), string(data)) + }) + } + }) + } +} + +func rebasePath(newBase, oldPath string) string { + if oldPath == "" { + return "" + } + return filepath.Join(newBase, filepath.Base(oldPath)) +} + +func replaceValues(data []byte, m map[string]string) []byte { + for k, v := range m { + data = bytes.ReplaceAll(data, []byte(v), []byte(k)) + } + return data +} + +func TestNamespace_overrideFromConfig(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + cfg *unversionedTeleport + want Namespace + }{ + { + name: "default", + cfg: &unversionedTeleport{ + ProxyServer: "example.com", + DataDir: "/data", + }, + want: Namespace{ + defaultProxyAddr: "example.com:3080", + dataDir: "/data", + }, + }, + { + name: "empty", + cfg: &unversionedTeleport{}, + want: Namespace{ + defaultProxyAddr: "default.example.com", + dataDir: "/var/lib/teleport", + }, + }, + { + name: "full proxy", + cfg: &unversionedTeleport{ + ProxyServer: "https://example.com:8080", + }, + want: Namespace{ + defaultProxyAddr: "example.com:8080", + dataDir: "/var/lib/teleport", + }, + }, + { + name: "protocol and host", + cfg: &unversionedTeleport{ + ProxyServer: "https://example.com", + }, + want: Namespace{ + defaultProxyAddr: "example.com:3080", + dataDir: "/var/lib/teleport", + }, + }, + { + name: "host and port", + cfg: &unversionedTeleport{ + ProxyServer: "example.com:443", + }, + want: Namespace{ + defaultProxyAddr: "example.com:443", + dataDir: "/var/lib/teleport", + }, + }, + { + name: "host", + cfg: &unversionedTeleport{ + ProxyServer: "example.com", + }, + want: Namespace{ + defaultProxyAddr: "example.com:3080", + dataDir: "/var/lib/teleport", + }, + }, + { + name: "auth server (v3)", + cfg: &unversionedTeleport{ + AuthServer: "example.com", + }, + want: Namespace{ + defaultProxyAddr: "example.com:3025", + dataDir: "/var/lib/teleport", + }, + }, + { + name: "auth server (v1/2)", + cfg: &unversionedTeleport{ + AuthServers: []string{ + "one.example.com", + "two.example.com", + }, + }, + want: Namespace{ + defaultProxyAddr: "one.example.com:3025", + dataDir: "/var/lib/teleport", + }, + }, + { + name: "proxy priority", + cfg: &unversionedTeleport{ + ProxyServer: "one.example.com", + AuthServer: "two.example.com", + AuthServers: []string{"three.example.com"}, + }, + want: Namespace{ + defaultProxyAddr: "one.example.com:3080", + dataDir: "/var/lib/teleport", + }, + }, + { + name: "auth priority", + cfg: &unversionedTeleport{ + AuthServer: "two.example.com", + AuthServers: []string{"three.example.com"}, + }, + want: Namespace{ + defaultProxyAddr: "two.example.com:3025", + dataDir: "/var/lib/teleport", + }, + }, + { + name: "missing", + want: Namespace{ + defaultProxyAddr: "default.example.com", + dataDir: "/var/lib/teleport", + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + ns := &Namespace{ + log: slog.Default(), + configFile: filepath.Join(t.TempDir(), "teleport.yaml"), + defaultProxyAddr: "default.example.com", + dataDir: "/var/lib/teleport", + } + if tt.cfg != nil { + out, err := yaml.Marshal(unversionedConfig{Teleport: *tt.cfg}) + require.NoError(t, err) + err = os.WriteFile(ns.configFile, out, os.ModePerm) + require.NoError(t, err) + } + ctx := context.Background() + ns.overrideFromConfig(ctx) + ns.configFile = "" + ns.log = nil + require.Equal(t, &tt.want, ns) + }) + } +} + +// In the future, the latest version of the updater may need to read a version of teleport.yaml that has +// an unsupported version which is supported by the updater-managed version of Teleport. +// This test will break if Teleport removes a field that the updater reads. +func TestUnversionedTeleportConfig(t *testing.T) { + in := unversionedConfig{ + Teleport: unversionedTeleport{ + ProxyServer: "proxy.example.com", + AuthServer: "auth.example.com", + AuthServers: []string{"auth1.example.com", "auth2.example.com"}, + DataDir: "example_dir", + }, + } + var inB bytes.Buffer + err := yaml.NewEncoder(&inB).Encode(in) + require.NoError(t, err) + fc, err := config.ReadConfig(&inB) + require.NoError(t, err) + + var outB bytes.Buffer + err = yaml.NewEncoder(&outB).Encode(fc) + require.NoError(t, err) + + var out unversionedConfig + err = yaml.NewDecoder(&outB).Decode(&out) + require.NoError(t, err) + require.Equal(t, in, out) +} + +func TestReplaceTeleportService(t *testing.T) { + t.Parallel() + + const defaultService = ` +[Unit] +Description=Teleport Service +After=network.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +EnvironmentFile=-/etc/default/teleport +ExecStart=/usr/local/bin/teleport start --config /etc/teleport.yaml --pid-file=/run/teleport.pid +# systemd before 239 needs an absolute path +ExecReload=/bin/sh -c "exec pkill -HUP -L -F /run/teleport.pid" +PIDFile=/run/teleport.pid +LimitNOFILE=524288 + +[Install] +WantedBy=multi-user.target +` + + tests := []struct { + name string + in string + + pidFile string + configFile string + pathDir string + flags autoupdate.InstallFlags + }{ + { + name: "default", + in: defaultService, + pidFile: "/var/run/teleport.pid", + configFile: "/etc/teleport.yaml", + pathDir: "/usr/local/bin", + }, + { + name: "custom", + in: defaultService, + pidFile: "/some/path/teleport.pid", + configFile: "/some/path/teleport.yaml", + pathDir: "/some/path/bin", + }, + { + name: "FIPS", + in: defaultService, + pidFile: "/var/run/teleport.pid", + configFile: "/etc/teleport.yaml", + pathDir: "/usr/local/bin", + flags: autoupdate.FlagFIPS, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ns := &Namespace{ + log: slog.Default(), + configFile: tt.configFile, + pidFile: tt.pidFile, + } + data := ns.ReplaceTeleportService([]byte(tt.in), tt.pathDir, tt.flags) + if golden.ShouldSet() { + golden.Set(t, data) + } + require.Equal(t, string(golden.Get(t)), string(data)) + }) + } +} diff --git a/lib/autoupdate/agent/telemetry.go b/lib/autoupdate/agent/telemetry.go new file mode 100644 index 0000000000000..bd6bb887cad40 --- /dev/null +++ b/lib/autoupdate/agent/telemetry.go @@ -0,0 +1,105 @@ +/* + * Teleport + * Copyright (C) 2025 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "os" + "path/filepath" + "strings" + + "github.com/gravitational/trace" +) + +const installDirEnvVar = "TELEPORT_UPDATE_INSTALL_DIR" + +// IsManagedByUpdater returns true if the local Teleport binary is managed by teleport-update. +// Note that true may be returned even if auto-updates is disabled or the version is pinned. +// The binary is considered managed if it lives under /opt/teleport, but not within the package +// path at /opt/teleport/system. +func IsManagedByUpdater() (bool, error) { + systemd, err := hasSystemD() + if err != nil { + return false, trace.Wrap(err) + } + if !systemd { + return false, nil + } + teleportPath, err := os.Readlink("/proc/self/exe") + if err != nil { + return false, trace.Wrap(err, "cannot find Teleport binary") + } + installDir := os.Getenv(installDirEnvVar) + if installDir == "" { + installDir = defaultInstallDir + } + // Check if current binary is under the updater-managed path. + managed, err := hasParentDir(teleportPath, installDir) + if err != nil { + return false, trace.Wrap(err) + } + if !managed { + return false, nil + } + // Return false if the binary is under the updater-managed path, but in the system prefix reserved for the package. + system, err := hasParentDir(teleportPath, packageSystemDir) + return !system, trace.Wrap(err) +} + +// IsManagedAndDefault returns true if the local Teleport binary is both managed by teleport-update +// and the default installation (with teleport.service as the unit file name). +// The binary is considered managed and default if it lives within /opt/teleport/default. +func IsManagedAndDefault() (bool, error) { + systemd, err := hasSystemD() + if err != nil { + return false, trace.Wrap(err) + } + if !systemd { + return false, nil + } + teleportPath, err := os.Readlink("/proc/self/exe") + if err != nil { + return false, trace.Wrap(err, "cannot find Teleport binary") + } + installDir := os.Getenv(installDirEnvVar) + if installDir == "" { + installDir = defaultInstallDir + } + isDefault, err := hasParentDir(teleportPath, filepath.Join(installDir, defaultNamespace)) + return isDefault, trace.Wrap(err) +} + +// hasParentDir returns true if dir is any parent directory of parent. +// hasParentDir does not resolve symlinks, and requires that files be represented the same way in dir and parent. +func hasParentDir(dir, parent string) (bool, error) { + // Note that os.Stat + os.SameFile would be more reliable, + // but does not work well for arbitrarily nested subdirectories. + absDir, err := filepath.Abs(dir) + if err != nil { + return false, trace.Wrap(err, "cannot get absolute path for directory %s", dir) + } + absParent, err := filepath.Abs(parent) + if err != nil { + return false, trace.Wrap(err, "cannot get absolute path for parent directory %s", dir) + } + sep := string(filepath.Separator) + if !strings.HasSuffix(absParent, sep) { + absParent += sep + } + return strings.HasPrefix(absDir, absParent), nil +} diff --git a/lib/autoupdate/agent/telemetry_test.go b/lib/autoupdate/agent/telemetry_test.go new file mode 100644 index 0000000000000..5c5f329c95d5f --- /dev/null +++ b/lib/autoupdate/agent/telemetry_test.go @@ -0,0 +1,110 @@ +/* + * Teleport + * Copyright (C) 2025 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestHasParentDir(t *testing.T) { + tests := []struct { + name string + path string + parent string + wantResult bool + }{ + { + name: "Has valid parent directory", + path: "/opt/teleport/dir/test", + parent: "/opt/teleport", + wantResult: true, + }, + { + name: "Has valid parent directory with slash", + path: "/opt/teleport/dir/test", + parent: "/opt/teleport/", + wantResult: true, + }, + { + name: "Parent directory is root", + path: "/opt/teleport/dir", + parent: "/", + wantResult: true, + }, + { + name: "Parent is the same as the path", + path: "/opt/teleport/dir", + parent: "/opt/teleport/dir", + wantResult: false, + }, + { + name: "Parent the same as the path but without slash", + path: "/opt/teleport/dir/", + parent: "/opt/teleport/dir", + wantResult: false, + }, + { + name: "Parent the same as the path but with slash", + path: "/opt/teleport/dir", + parent: "/opt/teleport/dir/", + wantResult: false, + }, + { + name: "Parent is substring of the path", + path: "/opt/teleport/dir-place", + parent: "/opt/teleport/dir", + wantResult: false, + }, + { + name: "Parent is in path", + path: "/opt/teleport", + parent: "/opt/teleport/dir", + wantResult: false, + }, + { + name: "Empty parent", + path: "/opt/teleport/dir", + parent: "", + wantResult: false, + }, + { + name: "Empty path", + path: "", + parent: "/opt/teleport", + wantResult: false, + }, + { + name: "Both empty", + path: "", + parent: "", + wantResult: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + result, err := hasParentDir(tt.path, tt.parent) + require.NoError(t, err) + require.Equal(t, tt.wantResult, result) + }) + } +} diff --git a/lib/autoupdate/agent/testdata/TestReplaceTeleportService/FIPS.golden b/lib/autoupdate/agent/testdata/TestReplaceTeleportService/FIPS.golden new file mode 100644 index 0000000000000..787db53a9549d --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestReplaceTeleportService/FIPS.golden @@ -0,0 +1,18 @@ + +[Unit] +Description=Teleport Service +After=network.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +EnvironmentFile=-/etc/default/teleport +ExecStart=/usr/local/bin/teleport start --fips --config /etc/teleport.yaml --pid-file=/var/run/teleport.pid +# systemd before 239 needs an absolute path +ExecReload=/bin/sh -c "exec pkill -HUP -L -F /var/run/teleport.pid" +PIDFile=/var/run/teleport.pid +LimitNOFILE=524288 + +[Install] +WantedBy=multi-user.target diff --git a/lib/autoupdate/agent/testdata/TestReplaceTeleportService/custom.golden b/lib/autoupdate/agent/testdata/TestReplaceTeleportService/custom.golden new file mode 100644 index 0000000000000..c0c056488e286 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestReplaceTeleportService/custom.golden @@ -0,0 +1,18 @@ + +[Unit] +Description=Teleport Service +After=network.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +EnvironmentFile=-/etc/default/teleport +ExecStart=/some/path/bin/teleport start --config /some/path/teleport.yaml --pid-file=/some/path/teleport.pid +# systemd before 239 needs an absolute path +ExecReload=/bin/sh -c "exec pkill -HUP -L -F /some/path/teleport.pid" +PIDFile=/some/path/teleport.pid +LimitNOFILE=524288 + +[Install] +WantedBy=multi-user.target diff --git a/lib/autoupdate/agent/testdata/TestReplaceTeleportService/default.golden b/lib/autoupdate/agent/testdata/TestReplaceTeleportService/default.golden new file mode 100644 index 0000000000000..282d75dc28a43 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestReplaceTeleportService/default.golden @@ -0,0 +1,18 @@ + +[Unit] +Description=Teleport Service +After=network.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +EnvironmentFile=-/etc/default/teleport +ExecStart=/usr/local/bin/teleport start --config /etc/teleport.yaml --pid-file=/var/run/teleport.pid +# systemd before 239 needs an absolute path +ExecReload=/bin/sh -c "exec pkill -HUP -L -F /var/run/teleport.pid" +PIDFile=/var/run/teleport.pid +LimitNOFILE=524288 + +[Install] +WantedBy=multi-user.target diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Disable/already_disabled.golden b/lib/autoupdate/agent/testdata/TestUpdater_Disable/already_disabled.golden new file mode 100644 index 0000000000000..6e104086250e3 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Disable/already_disabled.golden @@ -0,0 +1,10 @@ +version: v1 +kind: update_config +spec: + proxy: "" + path: "" + enabled: false + pinned: false +status: + active: + version: "" diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Disable/enabled.golden b/lib/autoupdate/agent/testdata/TestUpdater_Disable/enabled.golden new file mode 100644 index 0000000000000..6e104086250e3 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Disable/enabled.golden @@ -0,0 +1,10 @@ +version: v1 +kind: update_config +spec: + proxy: "" + path: "" + enabled: false + pinned: false +status: + active: + version: "" diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/FIPS_and_Enterprise_flags.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/FIPS_and_Enterprise_flags.golden new file mode 100644 index 0000000000000..0a42425a845c0 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/FIPS_and_Enterprise_flags.golden @@ -0,0 +1,12 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + enabled: false + pinned: false +status: + id_file: updater-id-file + active: + version: 16.3.0 + flags: [Enterprise, FIPS] diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/agpl_requires_base_URL.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/agpl_requires_base_URL.golden new file mode 100644 index 0000000000000..6e104086250e3 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/agpl_requires_base_URL.golden @@ -0,0 +1,10 @@ +version: v1 +kind: update_config +spec: + proxy: "" + path: "" + enabled: false + pinned: false +status: + active: + version: "" diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/backup_version_kept_for_validation.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/backup_version_kept_for_validation.golden new file mode 100644 index 0000000000000..d7028f84581d7 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/backup_version_kept_for_validation.golden @@ -0,0 +1,13 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + enabled: false + pinned: false +status: + id_file: updater-id-file + active: + version: 16.3.0 + backup: + version: backup-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/backup_version_removed_on_install.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/backup_version_removed_on_install.golden new file mode 100644 index 0000000000000..c2342d86c90e9 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/backup_version_removed_on_install.golden @@ -0,0 +1,13 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + enabled: false + pinned: false +status: + id_file: updater-id-file + active: + version: 16.3.0 + backup: + version: old-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/config_does_not_exist.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/config_does_not_exist.golden new file mode 100644 index 0000000000000..e16d92d41752e --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/config_does_not_exist.golden @@ -0,0 +1,11 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + enabled: false + pinned: false +status: + id_file: updater-id-file + active: + version: 16.3.0 diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/config_from_file.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/config_from_file.golden new file mode 100644 index 0000000000000..fb2c1f4c3156e --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/config_from_file.golden @@ -0,0 +1,15 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /path + group: group + base_url: https://example.com + enabled: true + pinned: false +status: + id_file: updater-id-file + active: + version: 16.3.0 + backup: + version: old-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/config_from_user.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/config_from_user.golden new file mode 100644 index 0000000000000..42a4ebb9cedb4 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/config_from_user.golden @@ -0,0 +1,15 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /path + group: new-group + base_url: https://example.com/new + enabled: true + pinned: false +status: + id_file: updater-id-file + active: + version: new-version + backup: + version: old-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/defaults.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/defaults.golden new file mode 100644 index 0000000000000..c2342d86c90e9 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/defaults.golden @@ -0,0 +1,13 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + enabled: false + pinned: false +status: + id_file: updater-id-file + active: + version: 16.3.0 + backup: + version: old-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/insecure_URL.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/insecure_URL.golden new file mode 100644 index 0000000000000..69b08b9c83c9d --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/insecure_URL.golden @@ -0,0 +1,11 @@ +version: v1 +kind: update_config +spec: + proxy: "" + path: "" + base_url: http://example.com + enabled: false + pinned: false +status: + active: + version: "" diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/install_error.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/install_error.golden new file mode 100644 index 0000000000000..6e104086250e3 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/install_error.golden @@ -0,0 +1,10 @@ +version: v1 +kind: update_config +spec: + proxy: "" + path: "" + enabled: false + pinned: false +status: + active: + version: "" diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/invalid_metadata.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/invalid_metadata.golden new file mode 100644 index 0000000000000..0c3dcaac8edbd --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/invalid_metadata.golden @@ -0,0 +1,10 @@ +version: "" +kind: "" +spec: + proxy: "" + path: "" + enabled: false + pinned: false +status: + active: + version: "" diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/no_need_to_reload.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/no_need_to_reload.golden new file mode 100644 index 0000000000000..e16d92d41752e --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/no_need_to_reload.golden @@ -0,0 +1,11 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + enabled: false + pinned: false +status: + id_file: updater-id-file + active: + version: 16.3.0 diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/not_started_or_enabled.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/not_started_or_enabled.golden new file mode 100644 index 0000000000000..e16d92d41752e --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/not_started_or_enabled.golden @@ -0,0 +1,11 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + enabled: false + pinned: false +status: + id_file: updater-id-file + active: + version: 16.3.0 diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/override_skip.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/override_skip.golden new file mode 100644 index 0000000000000..c2342d86c90e9 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/override_skip.golden @@ -0,0 +1,13 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + enabled: false + pinned: false +status: + id_file: updater-id-file + active: + version: 16.3.0 + backup: + version: old-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/setup_fails_already_installed.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/setup_fails_already_installed.golden new file mode 100644 index 0000000000000..067fcf60bd527 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/setup_fails_already_installed.golden @@ -0,0 +1,10 @@ +version: v1 +kind: update_config +spec: + proxy: "" + path: "" + enabled: false + pinned: false +status: + active: + version: 16.3.0 diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Install/version_already_installed.golden b/lib/autoupdate/agent/testdata/TestUpdater_Install/version_already_installed.golden new file mode 100644 index 0000000000000..e16d92d41752e --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Install/version_already_installed.golden @@ -0,0 +1,11 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + enabled: false + pinned: false +status: + id_file: updater-id-file + active: + version: 16.3.0 diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Unpin/not_pinned.golden b/lib/autoupdate/agent/testdata/TestUpdater_Unpin/not_pinned.golden new file mode 100644 index 0000000000000..6e104086250e3 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Unpin/not_pinned.golden @@ -0,0 +1,10 @@ +version: v1 +kind: update_config +spec: + proxy: "" + path: "" + enabled: false + pinned: false +status: + active: + version: "" diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Unpin/pinned.golden b/lib/autoupdate/agent/testdata/TestUpdater_Unpin/pinned.golden new file mode 100644 index 0000000000000..6e104086250e3 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Unpin/pinned.golden @@ -0,0 +1,10 @@ +version: v1 +kind: update_config +spec: + proxy: "" + path: "" + enabled: false + pinned: false +status: + active: + version: "" diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/FIPS_and_Enterprise_flags.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/FIPS_and_Enterprise_flags.golden new file mode 100644 index 0000000000000..14a0b83b8539e --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/FIPS_and_Enterprise_flags.golden @@ -0,0 +1,22 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + base_url: https://example.com + enabled: true + pinned: false +status: + id_file: updater-id-file + last_update: + success: true + time: 2025-01-01T00:00:00Z + target: + version: 16.3.0 + flags: [Enterprise, FIPS] + active: + version: 16.3.0 + flags: [Enterprise, FIPS] + backup: + version: old-version + flags: [Enterprise, FIPS] diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/agpl_requires_base_URL.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/agpl_requires_base_URL.golden new file mode 100644 index 0000000000000..667353a7c24ae --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/agpl_requires_base_URL.golden @@ -0,0 +1,18 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + enabled: true + pinned: false +status: + id_file: updater-id-file + last_update: + success: false + time: 2025-01-01T00:00:00Z + target: + version: 16.3.0 + active: + version: old-version + backup: + version: backup-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/backup_version_is_linked.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/backup_version_is_linked.golden new file mode 100644 index 0000000000000..7697a84f3326c --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/backup_version_is_linked.golden @@ -0,0 +1,19 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + base_url: https://example.com + enabled: true + pinned: false +status: + id_file: updater-id-file + last_update: + success: true + time: 2025-01-01T00:00:00Z + target: + version: 16.3.0 + active: + version: 16.3.0 + backup: + version: old-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/backup_version_kept_when_no_change.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/backup_version_kept_when_no_change.golden new file mode 100644 index 0000000000000..d257cd6c30282 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/backup_version_kept_when_no_change.golden @@ -0,0 +1,13 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + base_url: https://example.com + enabled: true + pinned: false +status: + active: + version: 16.3.0 + backup: + version: backup-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/backup_version_removed_on_install.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/backup_version_removed_on_install.golden new file mode 100644 index 0000000000000..7697a84f3326c --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/backup_version_removed_on_install.golden @@ -0,0 +1,19 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + base_url: https://example.com + enabled: true + pinned: false +status: + id_file: updater-id-file + last_update: + success: true + time: 2025-01-01T00:00:00Z + target: + version: 16.3.0 + active: + version: 16.3.0 + backup: + version: old-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/insecure_URL.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/insecure_URL.golden new file mode 100644 index 0000000000000..297b00ce4ecf8 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/insecure_URL.golden @@ -0,0 +1,11 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + base_url: http://example.com + enabled: true + pinned: false +status: + active: + version: "" diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/install_error.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/install_error.golden new file mode 100644 index 0000000000000..575463dc75100 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/install_error.golden @@ -0,0 +1,16 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + enabled: true + pinned: false +status: + id_file: updater-id-file + last_update: + success: false + time: 2025-01-01T00:00:00Z + target: + version: 16.3.0 + active: + version: "" diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/invalid_metadata.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/invalid_metadata.golden new file mode 100644 index 0000000000000..a771da1fc9e25 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/invalid_metadata.golden @@ -0,0 +1,10 @@ +version: "" +kind: "" +spec: + proxy: localhost + path: "" + enabled: false + pinned: false +status: + active: + version: "" diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/missing_path_during_window.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/missing_path_during_window.golden new file mode 100644 index 0000000000000..f29735ca0230b --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/missing_path_during_window.golden @@ -0,0 +1,12 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: "" + group: group + base_url: https://example.com + enabled: true + pinned: false +status: + active: + version: old-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/pinned_version.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/pinned_version.golden new file mode 100644 index 0000000000000..501506cf96c78 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/pinned_version.golden @@ -0,0 +1,13 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + base_url: https://example.com + enabled: true + pinned: true +status: + active: + version: old-version + backup: + version: backup-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/setup_fails.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/setup_fails.golden new file mode 100644 index 0000000000000..170e821a409b2 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/setup_fails.golden @@ -0,0 +1,21 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + base_url: https://example.com + enabled: true + pinned: false +status: + id_file: updater-id-file + last_update: + success: false + time: 2025-01-01T00:00:00Z + target: + version: 16.3.0 + active: + version: old-version + backup: + version: backup-version + skip: + version: 16.3.0 diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/skip_version.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/skip_version.golden new file mode 100644 index 0000000000000..10b36430ffed1 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/skip_version.golden @@ -0,0 +1,15 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + base_url: https://example.com + enabled: true + pinned: false +status: + active: + version: old-version + backup: + version: backup-version + skip: + version: 16.3.0 diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_disabled_during_window.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_disabled_during_window.golden new file mode 100644 index 0000000000000..b6b43595a5903 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_disabled_during_window.golden @@ -0,0 +1,12 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + group: group + base_url: https://example.com + enabled: false + pinned: false +status: + active: + version: old-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_disabled_outside_of_window.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_disabled_outside_of_window.golden new file mode 100644 index 0000000000000..b6b43595a5903 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_disabled_outside_of_window.golden @@ -0,0 +1,12 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + group: group + base_url: https://example.com + enabled: false + pinned: false +status: + active: + version: old-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_enabled_during_window.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_enabled_during_window.golden new file mode 100644 index 0000000000000..efdd858023547 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_enabled_during_window.golden @@ -0,0 +1,20 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + group: group + base_url: https://example.com + enabled: true + pinned: false +status: + id_file: updater-id-file + last_update: + success: true + time: 2025-01-01T00:00:00Z + target: + version: 16.3.0 + active: + version: 16.3.0 + backup: + version: old-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_enabled_now,_not_started_or_enabled.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_enabled_now,_not_started_or_enabled.golden new file mode 100644 index 0000000000000..efdd858023547 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_enabled_now,_not_started_or_enabled.golden @@ -0,0 +1,20 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + group: group + base_url: https://example.com + enabled: true + pinned: false +status: + id_file: updater-id-file + last_update: + success: true + time: 2025-01-01T00:00:00Z + target: + version: 16.3.0 + active: + version: 16.3.0 + backup: + version: old-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_enabled_now.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_enabled_now.golden new file mode 100644 index 0000000000000..efdd858023547 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_enabled_now.golden @@ -0,0 +1,20 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + group: group + base_url: https://example.com + enabled: true + pinned: false +status: + id_file: updater-id-file + last_update: + success: true + time: 2025-01-01T00:00:00Z + target: + version: 16.3.0 + active: + version: 16.3.0 + backup: + version: old-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_enabled_outside_of_window.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_enabled_outside_of_window.golden new file mode 100644 index 0000000000000..a4cac37b8733c --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/updates_enabled_outside_of_window.golden @@ -0,0 +1,12 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + group: group + base_url: https://example.com + enabled: true + pinned: false +status: + active: + version: old-version diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/version_already_installed_in_window.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/version_already_installed_in_window.golden new file mode 100644 index 0000000000000..926667a2e7fc0 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/version_already_installed_in_window.golden @@ -0,0 +1,11 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + base_url: https://example.com + enabled: true + pinned: false +status: + active: + version: 16.3.0 diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/version_already_installed_outside_of_window.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/version_already_installed_outside_of_window.golden new file mode 100644 index 0000000000000..926667a2e7fc0 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/version_already_installed_outside_of_window.golden @@ -0,0 +1,11 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + base_url: https://example.com + enabled: true + pinned: false +status: + active: + version: 16.3.0 diff --git a/lib/autoupdate/agent/testdata/TestUpdater_Update/version_detects_as_linked.golden b/lib/autoupdate/agent/testdata/TestUpdater_Update/version_detects_as_linked.golden new file mode 100644 index 0000000000000..7697a84f3326c --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestUpdater_Update/version_detects_as_linked.golden @@ -0,0 +1,19 @@ +version: v1 +kind: update_config +spec: + proxy: localhost + path: /usr/local/bin + base_url: https://example.com + enabled: true + pinned: false +status: + id_file: updater-id-file + last_update: + success: true + time: 2025-01-01T00:00:00Z + target: + version: 16.3.0 + active: + version: 16.3.0 + backup: + version: old-version diff --git a/lib/autoupdate/agent/testdata/TestWriteConfigFiles/no_namespace/deprecated.golden b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/no_namespace/deprecated.golden new file mode 100644 index 0000000000000..3f18b9cdf3065 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/no_namespace/deprecated.golden @@ -0,0 +1,5 @@ +# teleport-update +# DO NOT EDIT THIS FILE +[Service] +ExecStart= +ExecStart=-/bin/echo "The teleport-upgrade script has been disabled by teleport-update. Please remove the teleport-ent-updater package." diff --git a/lib/autoupdate/agent/testdata/TestWriteConfigFiles/no_namespace/dropin.golden b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/no_namespace/dropin.golden new file mode 100644 index 0000000000000..cb09143fe9fdf --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/no_namespace/dropin.golden @@ -0,0 +1,5 @@ +# teleport-update +# DO NOT EDIT THIS FILE +[Service] +Environment="TELEPORT_UPDATE_CONFIG_FILE=/opt/teleport/default/update.yaml" +Environment="TELEPORT_UPDATE_INSTALL_DIR=/opt/teleport" diff --git a/lib/autoupdate/agent/testdata/TestWriteConfigFiles/no_namespace/needrestart.golden b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/no_namespace/needrestart.golden new file mode 100644 index 0000000000000..b5d6a74435cb2 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/no_namespace/needrestart.golden @@ -0,0 +1 @@ +$nrconf{override_rc}{qr(^teleport\.service)} = 0; diff --git a/lib/autoupdate/agent/testdata/TestWriteConfigFiles/no_namespace/service.golden b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/no_namespace/service.golden new file mode 100644 index 0000000000000..45d778ec09f25 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/no_namespace/service.golden @@ -0,0 +1,8 @@ +# teleport-update +# DO NOT EDIT THIS FILE +[Unit] +Description=Teleport auto-update service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/teleport-update --install-suffix= "--install-dir=/opt/teleport" update diff --git a/lib/autoupdate/agent/testdata/TestWriteConfigFiles/no_namespace/timer.golden b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/no_namespace/timer.golden new file mode 100644 index 0000000000000..d14a43d679e53 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/no_namespace/timer.golden @@ -0,0 +1,12 @@ +# teleport-update +# DO NOT EDIT THIS FILE +[Unit] +Description=Teleport auto-update timer unit + +[Timer] +OnActiveSec=1m +OnUnitActiveSec=5m +RandomizedDelaySec=1m + +[Install] +WantedBy=teleport.service diff --git a/lib/autoupdate/agent/testdata/TestWriteConfigFiles/test_namespace/dropin.golden b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/test_namespace/dropin.golden new file mode 100644 index 0000000000000..dc6445dc6e7f9 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/test_namespace/dropin.golden @@ -0,0 +1,5 @@ +# teleport-update +# DO NOT EDIT THIS FILE +[Service] +Environment="TELEPORT_UPDATE_CONFIG_FILE=/opt/teleport/test/update.yaml" +Environment="TELEPORT_UPDATE_INSTALL_DIR=/opt/teleport" diff --git a/lib/autoupdate/agent/testdata/TestWriteConfigFiles/test_namespace/needrestart.golden b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/test_namespace/needrestart.golden new file mode 100644 index 0000000000000..ad6bd606a74cb --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/test_namespace/needrestart.golden @@ -0,0 +1 @@ +$nrconf{override_rc}{qr(^teleport_test\.service)} = 0; diff --git a/lib/autoupdate/agent/testdata/TestWriteConfigFiles/test_namespace/service.golden b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/test_namespace/service.golden new file mode 100644 index 0000000000000..f698deec24bb9 --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/test_namespace/service.golden @@ -0,0 +1,8 @@ +# teleport-update +# DO NOT EDIT THIS FILE +[Unit] +Description=Teleport auto-update service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/teleport-update --install-suffix=test "--install-dir=/opt/teleport" update diff --git a/lib/autoupdate/agent/testdata/TestWriteConfigFiles/test_namespace/timer.golden b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/test_namespace/timer.golden new file mode 100644 index 0000000000000..f57a3c08055bc --- /dev/null +++ b/lib/autoupdate/agent/testdata/TestWriteConfigFiles/test_namespace/timer.golden @@ -0,0 +1,12 @@ +# teleport-update +# DO NOT EDIT THIS FILE +[Unit] +Description=Teleport auto-update timer unit + +[Timer] +OnActiveSec=1m +OnUnitActiveSec=5m +RandomizedDelaySec=1m + +[Install] +WantedBy=teleport_test.service diff --git a/lib/autoupdate/agent/updater.go b/lib/autoupdate/agent/updater.go new file mode 100644 index 0000000000000..e4cf6b26ef145 --- /dev/null +++ b/lib/autoupdate/agent/updater.go @@ -0,0 +1,1217 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "bytes" + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/asn1" + "errors" + "io/fs" + "log/slog" + "math/rand/v2" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "syscall" + "time" + + "github.com/google/uuid" + "github.com/gravitational/trace" + + "github.com/gravitational/teleport/api/client/webclient" + "github.com/gravitational/teleport/api/constants" + "github.com/gravitational/teleport/lib/autoupdate" + libdefaults "github.com/gravitational/teleport/lib/defaults" + "github.com/gravitational/teleport/lib/modules" + libutils "github.com/gravitational/teleport/lib/utils" +) + +const ( + // BinaryName specifies the name of the updater binary. + BinaryName = "teleport-update" +) + +const ( + // packageSystemDir is the location where packaged Teleport binaries and services are installed. + packageSystemDir = "/opt/teleport/system" + // reservedFreeDisk is the minimum required free space left on disk during downloads. + // TODO(sclevine): This value is arbitrary and could be replaced by, e.g., min(1%, 200mb) in the future + // to account for a range of disk sizes. + reservedFreeDisk = 10_000_000 + // requiredUmask must be set before this package can be used. + // Use syscall.Umask to set when no other goroutines are running. + requiredUmask = 0o022 + // teleportHostIDFileName is the name of Teleport's host ID file. + teleportHostIDFileName = "host_uuid" + // systemdMachineIDFile is a path containing a machine-unique identifier created by systemd. + systemdMachineIDFile = "/etc/machine-id" +) + +// Log keys +const ( + targetKey = "target_version" + activeKey = "active_version" + backupKey = "backup_version" + errorKey = "error" +) + +var ( + initTime = time.Now().UTC() +) + +// SetRequiredUmask sets the umask to match the systemd umask that the teleport-update service will execute with. +// This ensures consistent file permissions. +// NOTE: This must be run in main.go before any goroutines that create files are started. +func SetRequiredUmask(ctx context.Context, log *slog.Logger) { + warnUmask(ctx, log, syscall.Umask(requiredUmask)) +} + +func warnUmask(ctx context.Context, log *slog.Logger, old int) { + if old&^requiredUmask != 0 { + log.WarnContext(ctx, "Restrictive umask detected. Umask has been changed to 0022 for teleport-update and all child processes.") + log.WarnContext(ctx, "All files created by teleport-update will have permissions set according to this umask.") + } +} + +// NewLocalUpdater returns a new Updater that auto-updates local +// installations of the Teleport agent. +// The AutoUpdater uses an HTTP client with sane defaults for downloads, and +// will not fill disk to within 10 MB of available capacity. +func NewLocalUpdater(cfg LocalUpdaterConfig, ns *Namespace) (*Updater, error) { + certPool, err := x509.SystemCertPool() + if err != nil { + return nil, trace.Wrap(err) + } + tr, err := libdefaults.Transport() + if err != nil { + return nil, trace.Wrap(err) + } + tr.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: cfg.InsecureSkipVerify, + RootCAs: certPool, + } + client := &http.Client{ + Transport: tr, + Timeout: cfg.DownloadTimeout, + } + if cfg.Log == nil { + cfg.Log = slog.Default() + } + if cfg.SystemDir == "" { + cfg.SystemDir = packageSystemDir + } + validator := Validator{Log: cfg.Log} + return &Updater{ + Log: cfg.Log, + Pool: certPool, + InsecureSkipVerify: cfg.InsecureSkipVerify, + UpdateConfigFile: filepath.Join(ns.Dir(), updateConfigName), + UpdateIDFile: ns.updaterIDFile, + MachineIDFile: systemdMachineIDFile, + TeleportIDFile: filepath.Join(ns.dataDir, teleportHostIDFileName), + TeleportConfigFile: ns.configFile, + TeleportServiceName: filepath.Base(ns.serviceFile), + DefaultProxyAddr: ns.defaultProxyAddr, + DefaultPathDir: ns.defaultPathDir, + Installer: &LocalInstaller{ + InstallDir: filepath.Join(ns.Dir(), versionsDirName), + TargetServiceFile: ns.serviceFile, + SystemBinDir: filepath.Join(cfg.SystemDir, "bin"), + SystemServiceFile: filepath.Join(cfg.SystemDir, serviceDir, serviceName), + HTTP: client, + Log: cfg.Log, + ReservedFreeTmpDisk: reservedFreeDisk, + ReservedFreeInstallDisk: reservedFreeDisk, + TransformService: ns.ReplaceTeleportService, + ValidateBinary: validator.IsBinary, + Template: autoupdate.DefaultCDNURITemplate, + }, + Process: &SystemdService{ + ServiceName: filepath.Base(ns.serviceFile), + PIDFile: ns.pidFile, + Log: cfg.Log, + }, + ReexecSetup: func(ctx context.Context, pathDir string, reload bool) error { + name := filepath.Join(pathDir, BinaryName) + if cfg.SelfSetup && runtime.GOOS == constants.LinuxOS { + name = "/proc/self/exe" + } + args := []string{ + "--install-dir", ns.installDir, + "--install-suffix", ns.name, + } + if cfg.Debug { + args = append(args, "--debug") + } + args = append(args, "setup", "--path", pathDir) + if reload { + args = append(args, "--reload") + } + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + cfg.Log.InfoContext(ctx, "Executing new teleport-update binary to update configuration.") + defer cfg.Log.InfoContext(ctx, "Finished executing new teleport-update binary.") + return trace.Wrap(cmd.Run()) + }, + SetupNamespace: ns.Setup, + TeardownNamespace: ns.Teardown, + LogConfigWarnings: ns.LogWarnings, + }, nil +} + +// LocalUpdaterConfig specifies configuration for managing local agent auto-updates. +type LocalUpdaterConfig struct { + // Log contains a slog logger. + // Defaults to slog.Default() if nil. + Log *slog.Logger + // InsecureSkipVerify turns off TLS certificate verification. + InsecureSkipVerify bool + // DownloadTimeout is a timeout for file download requests. + // Defaults to no timeout. + DownloadTimeout time.Duration + // SystemDir for package-installed Teleport installations (usually /opt/teleport/system). + SystemDir string + // SelfSetup mode for using the current version of the teleport-update to setup the update service. + SelfSetup bool + // Debug logs enabled. + Debug bool +} + +// Updater implements the agent-local logic for Teleport agent auto-updates. +// SetRequiredUmask must be called before any methods are executed, except for Status. +type Updater struct { + // Log contains a logger. + Log *slog.Logger + // Pool used for requests to the Teleport web API. + Pool *x509.CertPool + // InsecureSkipVerify skips TLS verification. + InsecureSkipVerify bool + // UpdateConfigFile contains the path to the agent auto-updates configuration. + UpdateConfigFile string + // UpdateIDFile contains the path to the ID used to track the Teleport agent during updates. + // This ID is written and read by the Teleport agent and used to schedule progressive updates. + UpdateIDFile string + // MachineIDFile contains the path to a system-unique ID file. + MachineIDFile string + // TeleportIDFile contains the path to Teleport's host ID. + TeleportIDFile string + // TeleportConfigFile contains the path to Teleport's configuration. + TeleportConfigFile string + // TeleportServiceName contains the full name of the systemd service for Teleport + TeleportServiceName string + // DefaultProxyAddr contains Teleport's proxy address. This may differ from the updater's. + DefaultProxyAddr string + // DefaultPathDir contains the default path that Teleport binaries should be installed into. + DefaultPathDir string + // Installer manages installations of the Teleport agent. + Installer Installer + // Process manages a running instance of Teleport. + Process Process + // ReexecSetup re-execs teleport-update with the setup command. + // This configures the updater service, verifies the installation, and optionally reloads Teleport. + ReexecSetup func(ctx context.Context, path string, reload bool) error + // SetupNamespace configures the Teleport updater service for the current Namespace. + SetupNamespace func(ctx context.Context, path string) error + // TeardownNamespace removes all traces of the updater service in the current Namespace, including Teleport. + TeardownNamespace func(ctx context.Context) error + // LogConfigWarnings logs warnings related to the configuration Namespace. + LogConfigWarnings func(ctx context.Context, pathDir string) +} + +// Installer provides an API for installing Teleport agents. +type Installer interface { + // Install the Teleport agent at revision from the download Template. + // If force is true, Install will remove broken revisions. + // Install must be idempotent. + Install(ctx context.Context, rev Revision, baseURL string, force bool) error + // Link the Teleport agent at the specified revision of Teleport into path. + // The revert function must restore the previous linking, returning false on any failure. + // If force is true, Link will overwrite non-symlinks. + // Link must be idempotent. Link's revert function must be idempotent. + Link(ctx context.Context, rev Revision, pathDir string, force bool) (revert func(context.Context) bool, err error) + // LinkSystem links the system installation of Teleport into the system linking location. + // The revert function must restore the previous linking, returning false on any failure. + // LinkSystem must be idempotent. LinkSystem's revert function must be idempotent. + LinkSystem(ctx context.Context) (revert func(context.Context) bool, err error) + // TryLink links the specified revision of Teleport into path. + // Unlike Link, TryLink will fail if existing links to other locations are present. + // TryLink must be idempotent. + TryLink(ctx context.Context, rev Revision, pathDir string) error + // TryLinkSystem links the system (package) installation of Teleport into the system linking location. + // Unlike LinkSystem, TryLinkSystem will fail if existing links to other locations are present. + // TryLinkSystem must be idempotent. + TryLinkSystem(ctx context.Context) error + // Unlink unlinks the specified revision of Teleport from path. + // Unlink must be idempotent. + Unlink(ctx context.Context, rev Revision, pathDir string) error + // UnlinkSystem unlinks the system (package) installation of Teleport from the system linking location. + // UnlinkSystem must be idempotent. + UnlinkSystem(ctx context.Context) error + // List the installed revisions of Teleport. + List(ctx context.Context) (revisions []Revision, err error) + // Remove the Teleport agent at revision. + // Remove must be idempotent. + Remove(ctx context.Context, rev Revision) error + // IsLinked returns true if the revision is linked to path. + IsLinked(ctx context.Context, rev Revision, pathDir string) (bool, error) +} + +var ( + // ErrLinked is returned when a linked version cannot be operated on. + ErrLinked = errors.New("version is linked") + // ErrNotNeeded is returned when the operation is not needed. + ErrNotNeeded = errors.New("not needed") + // ErrNotSupported is returned when the operation is not supported on the platform. + ErrNotSupported = errors.New("not supported on this platform") + // ErrNotAvailable is returned when the operation is not available at the current version of the platform. + ErrNotAvailable = errors.New("not available at this version") + // ErrNoBinaries is returned when no binaries are available to be linked. + ErrNoBinaries = errors.New("no binaries available to link") + // ErrFilePresent is returned when a file is present. + ErrFilePresent = errors.New("file present") + // ErrNotInstalled is returned when Teleport is not installed. + ErrNotInstalled = errors.New("not installed") +) + +// Process provides an API for interacting with a running Teleport process. +type Process interface { + // Reload must reload the Teleport process as gracefully as possible. + // If the process is not healthy after reloading, Reload must return an error. + // If the process did not require reloading, Reload must return ErrNotNeeded. + // E.g., if the process is not enabled, or it was already reloaded after the last Sync. + // If the type implementing Process does not support the system process manager, + // Reload must return ErrNotSupported. + Reload(ctx context.Context) error + // Sync must validate and synchronize process configuration. + // After the linked Teleport installation is changed, failure to call Sync without + // error before Reload may result in undefined behavior. + // If the type implementing Process does not support the system process manager, + // Sync must return ErrNotSupported. + Sync(ctx context.Context) error + // IsEnabled must return true if the Process is configured to run on system boot. + // If the type implementing Process does not support the system process manager, + // IsEnabled must return ErrNotSupported. + IsEnabled(ctx context.Context) (bool, error) + // IsActive must return true if the Process is currently running. + // If the type implementing Process does not support the system process manager, + // IsActive must return ErrNotSupported. + IsActive(ctx context.Context) (bool, error) + // IsPresent must return true if the Process is installed on the system. + // If the type implementing Process does not support the system process manager, + // IsPresent must return ErrNotSupported. + IsPresent(ctx context.Context) (bool, error) +} + +// OverrideConfig contains overrides for individual update operations. +// If validated, these overrides may be persisted to disk. +type OverrideConfig struct { + UpdateSpec + + // The fields below override the behavior of + // Updater.Install for a single run. + + // ForceVersion to the specified version. + ForceVersion string + // ForceFlags in installed Teleport. + ForceFlags []string + // AllowOverwrite of installed binaries. + AllowOverwrite bool + // AllowProxyConflict when proxies in teleport.yaml and update.yaml are mismatched. + AllowProxyConflict bool +} + +func deref[T any](ptr *T) T { + if ptr != nil { + return *ptr + } + var t T + return t +} + +func toPtr[T any](t T) *T { + return &t +} + +// Install attempts an initial installation of Teleport. +// If the initial installation succeeds, the override configuration is persisted. +// Otherwise, the configuration is not changed. +// This function is idempotent. +func (u *Updater) Install(ctx context.Context, override OverrideConfig) error { + // Read configuration from update.yaml and override any new values passed as flags. + cfg, err := readConfig(u.UpdateConfigFile) + if err != nil { + return trace.Wrap(err, "failed to read %s", updateConfigName) + } + if err := validateConfigSpec(&cfg.Spec, override); err != nil { + return trace.Wrap(err) + } + + if cfg.Spec.Proxy == "" { + cfg.Spec.Proxy = u.DefaultProxyAddr + } else if u.DefaultProxyAddr != "" && + !sameProxies(cfg.Spec.Proxy, u.DefaultProxyAddr) && + !override.AllowProxyConflict { + u.Log.ErrorContext(ctx, "Proxy specified in update.yaml does not match teleport.yaml.", "update_proxy", cfg.Spec.Proxy, "teleport_proxy", u.DefaultProxyAddr) + return trace.Errorf("refusing to install with conflicting proxy addresses, pass --allow-proxy-conflict to override") + } + if cfg.Spec.Path == "" { + cfg.Spec.Path = u.DefaultPathDir + } + cfg.Status.IDFile = u.UpdateIDFile + + active := cfg.Status.Active + skip := deref(cfg.Status.Skip) + + // Lookup target version from the proxy. + + resp, err := u.find(ctx, cfg, u.readID(ctx)) + if err != nil { + return trace.Wrap(err) + } + targetVersion := resp.Target.Version + targetFlags := resp.Target.Flags + targetFlags |= autoupdate.NewInstallFlagsFromStrings(override.ForceFlags) + if override.ForceVersion != "" { + targetVersion = override.ForceVersion + } + target := NewRevision(targetVersion, targetFlags) + + switch target.Version { + case "": + return trace.Errorf("agent version not available from Teleport cluster") + case skip.Version: + u.Log.WarnContext(ctx, "Target version was previously marked as broken. Retrying update.", targetKey, target, activeKey, active) + default: + u.Log.InfoContext(ctx, "Initiating installation.", targetKey, target, activeKey, active) + } + + if err := u.update(ctx, cfg, target, override.AllowOverwrite, resp.AGPL); err != nil { + if errors.Is(err, ErrFilePresent) && !override.AllowOverwrite { + u.Log.ErrorContext(ctx, "A non-packaged or outdated installation of Teleport was detected on this system.") + u.Log.ErrorContext(ctx, "Use --overwrite to force immediate removal of any existing binaries installed manually or via script.") + u.Log.ErrorContext(ctx, "Alternatively, if a Teleport RPM or DEB package is installed, upgrade it to the latest version and retry without --overwrite.") + } + return trace.Wrap(err) + } + if target.Version == skip.Version { + cfg.Status.Skip = nil + } + + // Only write the configuration file if the initial update succeeds. + // Note: skip_version is never set on failed enable, only failed update. + + if err := writeConfig(u.UpdateConfigFile, cfg); err != nil { + return trace.Wrap(err, "failed to write %s", updateConfigName) + } + u.Log.InfoContext(ctx, "Configuration updated.") + u.LogConfigWarnings(ctx, cfg.Spec.Path) + u.notices(ctx) + return nil +} + +// sameProxies returns true if both proxies addresses are the same. +// Note that the port is defaulted to 443, which is different from teleport.yaml's default. +func sameProxies(a, b string) bool { + const defaultPort = 443 + if a == b { + return true + } + addrA, err := libutils.ParseAddr(a) + if err != nil { + return false + } + addrB, err := libutils.ParseAddr(b) + if err != nil { + return false + } + return addrA.Host() == addrB.Host() && + addrA.Port(defaultPort) == addrB.Port(defaultPort) +} + +// Remove removes everything created by the updater for the given namespace. +// Before attempting this, Remove attempts to gracefully recover the system-packaged version of Teleport (if present). +// This function is idempotent. +func (u *Updater) Remove(ctx context.Context, force bool) error { + cfg, err := readConfig(u.UpdateConfigFile) + if err != nil { + return trace.Wrap(err, "failed to read %s", updateConfigName) + } + if err := validateConfigSpec(&cfg.Spec, OverrideConfig{}); err != nil { + return trace.Wrap(err) + } + active := cfg.Status.Active + if active.Version == "" { + u.Log.InfoContext(ctx, "No installation of Teleport managed by the updater. Removing updater configuration.") + if err := u.TeardownNamespace(ctx); err != nil { + return trace.Wrap(err) + } + u.Log.InfoContext(ctx, "Automatic update configuration for Teleport successfully uninstalled.") + return nil + } + + // Do not link system package installation if the installation we are removing + // is not installed into /usr/local/bin. In this case, we also need to make sure + // it is clear we are not going to recover the package's systemd service if it + // was overwritten. + if filepath.Clean(cfg.Spec.Path) != filepath.Clean(defaultPathDir) { + if u.TeleportServiceName == serviceName { + if !force { + u.Log.ErrorContext(ctx, "Default Teleport systemd service would be removed, and --force was not passed.") + u.Log.ErrorContext(ctx, "Refusing to remove Teleport from this system.") + return trace.Errorf("unable to remove Teleport completely without --force") + } else { + u.Log.WarnContext(ctx, "Default Teleport systemd service will be removed since --force was passed.") + u.Log.WarnContext(ctx, "Teleport will be removed from this system.") + } + } + return u.removeWithoutSystem(ctx, cfg) + } + revert, err := u.Installer.LinkSystem(ctx) + if errors.Is(err, ErrNoBinaries) { + if !force { + u.Log.ErrorContext(ctx, "No packaged installation of Teleport was found, and --force was not passed.") + u.Log.ErrorContext(ctx, "Refusing to remove Teleport from this system entirely without --force.") + return trace.Errorf("unable to remove Teleport completely without --force") + } else { + u.Log.WarnContext(ctx, "No packaged installation of Teleport was found, but --force was passed.") + u.Log.WarnContext(ctx, "Teleport will be removed from this system entirely.") + } + return u.removeWithoutSystem(ctx, cfg) + } + if err != nil { + return trace.Wrap(err, "failed to link") + } + + u.Log.InfoContext(ctx, "Updater-managed installation of Teleport detected.") + u.Log.InfoContext(ctx, "Restoring packaged version of Teleport before removing.") + + revertConfig := func(ctx context.Context) bool { + if ok := revert(ctx); !ok { + u.Log.ErrorContext(ctx, "Failed to revert Teleport symlinks. Installation likely broken.") + return false + } + if err := u.Process.Sync(ctx); err != nil { + u.Log.ErrorContext(ctx, "Failed to revert systemd configuration after failed restart.", errorKey, err) + return false + } + return true + } + + // Sync systemd. + + err = u.Process.Sync(ctx) + if errors.Is(err, context.Canceled) { + return trace.Errorf("sync canceled") + } + if errors.Is(err, ErrNotSupported) { + u.Log.WarnContext(ctx, "Not syncing systemd configuration because systemd is not running.") + } else if err != nil { + // If sync fails, we may have left the host in a bad state, so we revert linking and re-Sync. + u.Log.ErrorContext(ctx, "Reverting symlinks due to invalid configuration.") + if ok := revertConfig(ctx); ok { + u.Log.WarnContext(ctx, "Teleport updater encountered a configuration error and successfully reverted the installation.") + } + return trace.Wrap(err, "failed to validate configuration for system package version of Teleport") + } + + // Restart Teleport. + + u.Log.InfoContext(ctx, "Teleport package successfully restored.") + err = u.Process.Reload(ctx) + if errors.Is(err, context.Canceled) { + return trace.Errorf("reload canceled") + } + if err != nil && + !errors.Is(err, ErrNotNeeded) && // no output if restart not needed + !errors.Is(err, ErrNotSupported) { // already logged above for Sync + + // If reloading Teleport at the new version fails, revert and reload. + u.Log.ErrorContext(ctx, "Reverting symlinks due to failed restart.") + if ok := revertConfig(ctx); ok { + if err := u.Process.Reload(ctx); err != nil && !errors.Is(err, ErrNotNeeded) { + u.Log.ErrorContext(ctx, "Failed to reload Teleport after reverting.", errorKey, err) + u.Log.ErrorContext(ctx, "Installation likely broken.") + } else { + u.Log.WarnContext(ctx, "Teleport updater detected an error with the new installation and successfully reverted it.") + } + } + return trace.Wrap(err, "failed to start system package version of Teleport") + } + u.Log.InfoContext(ctx, "Auto-updating Teleport removed and replaced by Teleport package.", "version", active) + if err := u.TeardownNamespace(ctx); err != nil { + return trace.Wrap(err) + } + u.Log.InfoContext(ctx, "Auto-update configuration for Teleport successfully uninstalled.") + return nil +} + +func (u *Updater) removeWithoutSystem(ctx context.Context, cfg *UpdateConfig) error { + u.Log.InfoContext(ctx, "Updater-managed installation of Teleport detected.") + u.Log.InfoContext(ctx, "Attempting to unlink and remove.") + ok, err := u.Process.IsActive(ctx) + if err != nil && !errors.Is(err, ErrNotSupported) { + return trace.Wrap(err) + } + if ok { + return trace.Errorf("refusing to remove active installation of Teleport, please stop and disable Teleport first") + } + if err := u.Installer.Unlink(ctx, cfg.Status.Active, cfg.Spec.Path); err != nil { + return trace.Wrap(err) + } + u.Log.InfoContext(ctx, "Teleport uninstalled.", "version", cfg.Status.Active) + if err := u.TeardownNamespace(ctx); err != nil { + return trace.Wrap(err) + } + u.Log.InfoContext(ctx, "Automatic update configuration for Teleport successfully uninstalled.") + return nil +} + +// readID generates a DBPID based on both the systemd machine ID and Teleport host ID. +// This reduces the chance that multiple hosts will have the same updater ID, while +// allowing both the teleport and teleport-update binaries to deterministically derive +// the same value. This also avoids issues caused by non-UUID values in host_uuid, +// and ensures that updaters without running agents have a unique value. +// The ID must be persisted to ensure it does not change between Teleport updates. +// Only the Teleport Agent may persist the ID, as it may start first at system boot and must +// know the value immediately when it starts. +// Errors will be logged. +func (u *Updater) readID(ctx context.Context) string { + tid, err := os.ReadFile(u.TeleportIDFile) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + u.Log.WarnContext(ctx, "Failed to read Teleport host ID.", "path", u.TeleportIDFile, errorKey, err) + } + mid, err := os.ReadFile(u.MachineIDFile) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + u.Log.WarnContext(ctx, "Failed to read systemd machine ID.", "path", u.MachineIDFile, errorKey, err) + } + out, err := FindDBPID(u.UpdateIDFile, bytes.TrimSpace(mid), bytes.TrimSpace(tid), false) + if err != nil { + u.Log.ErrorContext(ctx, "Unable to generate unique ID for this host.", errorKey, err) + u.Log.ErrorContext(ctx, "The Teleport agent may not be tracked, and may fail if used as a canary.") + return "" + } + return out +} + +func readIfPresent(path string) string { + idBytes, err := os.ReadFile(path) + if err != nil { + return "" + } + return string(bytes.TrimSpace(idBytes)) +} + +// FindDBPID returns a deterministic boot-persistent identifier (DBPID). +// This is a sha256-based UUIDv5 that is regenerated at each boot using a +// machine-derived identifier and a namespace identifier. +// DBPIDs are stable across reboots as long as their identifiers remain the +// same and the logic implementing the ID generation remains the same. +// This reduces the risk that the ID changes unexpectedly when a new version +// of the process in launched, because both the system must be rebooted +// and the IDs (or logic) must change for the DBPID to change. +// ASN.1 is used to support binary identifiers and to ensure stability across +// versions of the Go standard library. +// +// It is safe for other code to read path directly to determine the cached DBPID. +// FindDBPID may be called concurrently on the same path with persist set to false. +// Calls with persist set to true are not reentrant. +func FindDBPID(path string, systemID, namespaceID []byte, persist bool) (string, error) { + idBytes, err := os.ReadFile(path) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return "", trace.Wrap(err) + } + if s := bytes.TrimSpace(idBytes); err == nil && len(s) > 0 { + return string(s), nil + } + id, err := generateDBPID(systemID, namespaceID) + if err != nil { + return id, trace.Wrap(err) + } + if persist { + err = writeFileAtomicWithinDir(path, []byte(id), configFileMode) + } + return id, trace.Wrap(err) +} + +func generateDBPID(systemID, namespaceID []byte) (string, error) { + /* + --- ASN.1 Schema + + DBPID DEFINITIONS ::= BEGIN + DBPID ::= SEQUENCE { + systemID OCTET STRING, + namespaceID OCTET STRING + } + END + */ + + // changing this struct will change all hashes + obj := struct { + SID []byte + NSID []byte + }{ + SID: systemID, + NSID: namespaceID, + } + if len(obj.SID) == 0 && len(obj.NSID) == 0 { + return "", trace.BadParameter("all provided IDs are empty") + } + der, err := asn1.Marshal(obj) + if err != nil { + return "", trace.Wrap(err) + } + // this only uses the first 16 bytes of the sha256 hash, which is acceptable + // for uniquely identify agents connected to a cluster + return uuid.NewHash(sha256.New(), uuid.Nil, der, 5).String(), nil +} + +// Status returns all available local and remote fields related to agent auto-updates. +// Status is safe to run concurrently with other Updater commands. +// Status does not write files, and therefore does not require SetRequiredUmask. +func (u *Updater) Status(ctx context.Context) (Status, error) { + var out Status + // Read configuration from update.yaml. + cfg, err := readConfig(u.UpdateConfigFile) + if err != nil { + return out, trace.Wrap(err, "failed to read %s", updateConfigName) + } + if err := validateConfigSpec(&cfg.Spec, OverrideConfig{}); err != nil { + return out, trace.Wrap(err) + } + if cfg.Spec.Proxy == "" { + return out, trace.Wrap(ErrNotInstalled) + } + out.UpdateSpec = cfg.Spec + out.UpdateStatus = cfg.Status + + // Lookup target version from the proxy. + out.ID = readIfPresent(cfg.Status.IDFile) + resp, err := u.find(ctx, cfg, out.ID) + if err != nil { + return out, trace.Wrap(err) + } + out.FindResp = resp + out.IDFile = "" + return out, nil +} + +// Disable disables agent auto-updates. +// This function is idempotent. +func (u *Updater) Disable(ctx context.Context) error { + cfg, err := readConfig(u.UpdateConfigFile) + if err != nil { + return trace.Wrap(err, "failed to read %s", updateConfigName) + } + if !cfg.Spec.Enabled { + u.Log.InfoContext(ctx, "Automatic updates already disabled.") + return nil + } + cfg.Spec.Enabled = false + if err := writeConfig(u.UpdateConfigFile, cfg); err != nil { + return trace.Wrap(err, "failed to write %s", updateConfigName) + } + return nil +} + +// Unpin allows the current version to be changed by Update. +// This function is idempotent. +func (u *Updater) Unpin(ctx context.Context) error { + cfg, err := readConfig(u.UpdateConfigFile) + if err != nil { + return trace.Wrap(err, "failed to read %s", updateConfigName) + } + if !cfg.Spec.Pinned { + u.Log.InfoContext(ctx, "Current version not pinned.", activeKey, cfg.Status.Active) + return nil + } + cfg.Spec.Pinned = false + if err := writeConfig(u.UpdateConfigFile, cfg); err != nil { + return trace.Wrap(err, "failed to write %s", updateConfigName) + } + return nil +} + +// Update initiates an agent update. +// If the update succeeds, the new installed version is marked as active. +// Otherwise, the auto-updates configuration is not changed. +// Unlike Enable, Update will not validate or repair the current version. +// This function is idempotent. +func (u *Updater) Update(ctx context.Context, now bool) error { + // Read configuration from update.yaml and override any new values passed as flags. + cfg, err := readConfig(u.UpdateConfigFile) + if err != nil { + return trace.Wrap(err, "failed to read %s", updateConfigName) + } + if err := validateConfigSpec(&cfg.Spec, OverrideConfig{}); err != nil { + return trace.Wrap(err) + } + + active := cfg.Status.Active + skip := deref(cfg.Status.Skip) + if !cfg.Spec.Enabled { + u.Log.InfoContext(ctx, "Automatic updates disabled.", activeKey, active) + return nil + } + + if u.DefaultProxyAddr != "" && + !sameProxies(cfg.Spec.Proxy, u.DefaultProxyAddr) { + u.Log.WarnContext(ctx, "Proxy specified in update.yaml does not match teleport.yaml.", "update_proxy", cfg.Spec.Proxy, "teleport_proxy", u.DefaultProxyAddr) + u.Log.WarnContext(ctx, "Unexpected updates may occur.") + } + + if cfg.Spec.Path == "" { + return trace.Errorf("failed to read destination path for binary links from %s", updateConfigName) + } + cfg.Status.IDFile = u.UpdateIDFile + + resp, err := u.find(ctx, cfg, u.readID(ctx)) + if err != nil { + return trace.Wrap(err) + } + target := resp.Target + + if cfg.Spec.Pinned { + switch target { + case active: + u.Log.InfoContext(ctx, "Teleport is up-to-date. Installation is pinned to prevent future updates.", activeKey, active) + default: + u.Log.InfoContext(ctx, "Teleport version is pinned. Skipping update.", targetKey, target, activeKey, active) + } + return nil + } + + // If a version fails and is marked skip, we ignore any edition changes as well. + // If a cluster is broadcasting a version that failed to start, changing ent/fips is unlikely to fix the issue. + + if !resp.InWindow && !now { + switch { + case target.Version == "": + u.Log.WarnContext(ctx, "Cannot determine target agent version. Waiting for both version and update window.") + case target == active: + u.Log.InfoContext(ctx, "Teleport is up-to-date. Update window is not active.", activeKey, active) + case target.Version == skip.Version: + u.Log.InfoContext(ctx, "Update available, but the new version is marked as broken. Update window is not active.", targetKey, target, activeKey, active) + default: + u.Log.InfoContext(ctx, "Update available, but update window is not active.", targetKey, target, activeKey, active) + } + return nil + } + + switch { + case target.Version == "": + if resp.InWindow { + u.Log.ErrorContext(ctx, "Update window is active, but target version is not available.", activeKey, active) + } + return trace.Errorf("target version missing") + case target == active: + if resp.InWindow { + u.Log.InfoContext(ctx, "Teleport is up-to-date. Update window is active, but no action is needed.", activeKey, active) + } else { + u.Log.InfoContext(ctx, "Teleport is up-to-date. No action is needed.", activeKey, active) + } + return nil + case target.Version == skip.Version: + u.Log.InfoContext(ctx, "Update available, but the new version is marked as broken. Skipping update.", targetKey, target, activeKey, active) + return nil + default: + u.Log.InfoContext(ctx, "Update available. Initiating update.", targetKey, target, activeKey, active) + } + if !now && resp.Jitter > 0 { + select { + case <-time.After(time.Duration(rand.Int64N(int64(resp.Jitter)))): + case <-ctx.Done(): + return trace.Wrap(ctx.Err()) + } + } + cfg.Status.LastUpdate = &LastUpdate{ + Time: initTime.Truncate(time.Millisecond), + Target: target, + } + updateErr := u.update(ctx, cfg, target, false, resp.AGPL) + if updateErr == nil { + cfg.Status.LastUpdate.Success = true + } + writeErr := writeConfig(u.UpdateConfigFile, cfg) + if writeErr != nil { + writeErr = trace.Wrap(writeErr, "failed to write %s", updateConfigName) + } else { + u.Log.InfoContext(ctx, "Configuration updated.") + } + // Show notices last + if updateErr == nil && now { + u.notices(ctx) + } + return trace.NewAggregate(updateErr, writeErr) +} + +func (u *Updater) find(ctx context.Context, cfg *UpdateConfig, id string) (FindResp, error) { + if cfg.Spec.Proxy == "" { + return FindResp{}, trace.Errorf("Teleport proxy URL must be specified with --proxy or present in %s", updateConfigName) + } + addr, err := libutils.ParseAddr(cfg.Spec.Proxy) + if err != nil { + return FindResp{}, trace.Wrap(err, "failed to parse proxy server address") + } + group := cfg.Spec.Group + if group == "" { + group = "default" + } + resp, err := webclient.Find(&webclient.Config{ + Context: ctx, + ProxyAddr: addr.Addr, + Insecure: u.InsecureSkipVerify, + Timeout: 30 * time.Second, + UpdateGroup: group, + UpdateID: id, + Pool: u.Pool, + }) + if err != nil { + return FindResp{}, trace.Wrap(err, "failed to request version from proxy") + } + var flags autoupdate.InstallFlags + var agpl bool + switch resp.Edition { + case modules.BuildEnterprise: + flags |= autoupdate.FlagEnterprise + case modules.BuildCommunity: + case modules.BuildOSS: + agpl = true + default: + agpl = true + u.Log.WarnContext(ctx, "Unknown edition detected, defaulting to OSS.", "edition", resp.Edition) + } + if resp.FIPS { + flags |= autoupdate.FlagFIPS + } + jitterSec := resp.AutoUpdate.AgentUpdateJitterSeconds + return FindResp{ + Target: NewRevision(resp.AutoUpdate.AgentVersion, flags), + InWindow: resp.AutoUpdate.AgentAutoUpdate, + Jitter: time.Duration(jitterSec) * time.Second, + AGPL: agpl, + }, nil +} + +func (u *Updater) removeRevision(ctx context.Context, cfg *UpdateConfig, rev Revision) error { + linked, err := u.Installer.IsLinked(ctx, rev, cfg.Spec.Path) + if err != nil { + return trace.Wrap(err, "failed to determine if linked") + } + if linked { + return trace.Wrap(ErrLinked, "refusing to remove") + } + return trace.Wrap(u.Installer.Remove(ctx, rev)) +} + +func (u *Updater) update(ctx context.Context, cfg *UpdateConfig, target Revision, force, agpl bool) error { + baseURL := cfg.Spec.BaseURL + if baseURL == "" { + baseURL = autoupdate.DefaultBaseURL + } + + active := cfg.Status.Active + backup := deref(cfg.Status.Backup) + switch backup { + case Revision{}, target, active: + default: + if target == active { + // Keep backup version if we are only verifying active version + break + } + err := u.removeRevision(ctx, cfg, backup) + if err != nil { + // this could happen if it was already removed due to a failed installation + u.Log.WarnContext(ctx, "Failed to remove backup version of Teleport before new install.", errorKey, err, backupKey, backup) + } + } + + // Install and link the desired version (or validate existing installation) + + linked, err := u.Installer.IsLinked(ctx, target, cfg.Spec.Path) + if err != nil { + return trace.Wrap(err, "failed to determine if linked") + } + err = u.Installer.Install(ctx, target, baseURL, !linked) + if err != nil { + return trace.Wrap(err, "failed to install") + } + + // If the target version has fewer binaries, this will leave old binaries linked. + // This may prevent the installation from being removed. + // Cleanup logic at the end of this function will ensure that they are removed + // eventually. + + revert, err := u.Installer.Link(ctx, target, cfg.Spec.Path, force) + if err != nil { + return trace.Wrap(err, "failed to link") + } + + // If we fail to revert after this point, the next update/enable will + // fix the link to restore the active version. + + revertConfig := func(ctx context.Context) bool { + if target.Version != "" { + cfg.Status.Skip = toPtr(target) + } + if force { + u.Log.ErrorContext(ctx, "Unable to revert Teleport symlinks in overwrite mode. Installation likely broken.") + return false + } + if ok := revert(ctx); !ok { + u.Log.ErrorContext(ctx, "Failed to revert Teleport symlinks. Installation likely broken.") + return false + } + if err := u.SetupNamespace(ctx, cfg.Spec.Path); err != nil { + u.Log.ErrorContext(ctx, "Failed to revert configuration after failed restart.", errorKey, err) + return false + } + return true + } + + if cfg.Status.Active != target { + err := u.ReexecSetup(ctx, cfg.Spec.Path, true) + if errors.Is(err, context.Canceled) { + return trace.Errorf("check canceled") + } + if err != nil { + // If reloading Teleport at the new version fails, revert and reload. + u.Log.ErrorContext(ctx, "Reverting symlinks due to failed restart.") + if ok := revertConfig(ctx); ok { + if err := u.Process.Reload(ctx); err != nil && !errors.Is(err, ErrNotNeeded) { + u.Log.ErrorContext(ctx, "Failed to reload Teleport after reverting. Installation likely broken.", errorKey, err) + } else { + u.Log.WarnContext(ctx, "Teleport updater detected an error with the new installation and successfully reverted it.") + } + } + return trace.Wrap(err, "failed to start new version %s of Teleport", target) + } + u.Log.InfoContext(ctx, "Target version successfully installed.", targetKey, target) + + if r := cfg.Status.Active; r.Version != "" { + cfg.Status.Backup = toPtr(r) + } + cfg.Status.Active = target + } else { + err := u.ReexecSetup(ctx, cfg.Spec.Path, false) + if errors.Is(err, context.Canceled) { + return trace.Errorf("check canceled") + } + if err != nil { + // If sync fails, we may have left the host in a bad state, so we revert linking and re-Sync. + u.Log.ErrorContext(ctx, "Reverting symlinks due to invalid configuration.") + if ok := revertConfig(ctx); ok { + u.Log.WarnContext(ctx, "Teleport updater encountered a configuration error and successfully reverted the installation.") + } + return trace.Wrap(err, "failed to validate new version %s of Teleport", target) + } + u.Log.InfoContext(ctx, "Target version successfully validated.", targetKey, target) + } + if r := deref(cfg.Status.Backup); r.Version != "" { + u.Log.InfoContext(ctx, "Backup version set.", backupKey, r) + } + u.cleanup(ctx, cfg, []Revision{ + target, active, backup, + }) + return nil +} + +// Setup writes updater configuration and verifies the Teleport installation. +// If restart is true, Setup also restarts Teleport. +// Setup is safe to run concurrently with other Updater commands. +func (u *Updater) Setup(ctx context.Context, path string, restart bool) error { + // Setup teleport-updater configuration and sync systemd. + + err := u.SetupNamespace(ctx, path) + if errors.Is(err, context.Canceled) { + return trace.Errorf("sync canceled") + } + if err != nil { + return trace.Wrap(err, "failed to setup updater") + } + + present, err := u.Process.IsPresent(ctx) + if errors.Is(err, context.Canceled) { + return trace.Errorf("config check canceled") + } + if errors.Is(err, ErrNotSupported) { + u.Log.WarnContext(ctx, "Skipping all systemd setup because systemd is not running.") + return nil + } + if errors.Is(err, ErrNotAvailable) { + u.Log.DebugContext(ctx, "Systemd version is outdated. Skipping SELinux verification.") + } else if err != nil { + return trace.Wrap(err, "failed to determine if new version of Teleport has an installed systemd service") + } else if !present { + return trace.Errorf("cannot find systemd service for new version of Teleport, check SELinux settings") + } + + // Restart Teleport if necessary. + + if restart { + err = u.Process.Reload(ctx) + if errors.Is(err, context.Canceled) { + return trace.Errorf("reload canceled") + } + if err != nil && + !errors.Is(err, ErrNotNeeded) { // skip if not needed + return trace.Wrap(err, "failed to reload Teleport") + } + } + return nil +} + +// notices displays final notices after install or update. +func (u *Updater) notices(ctx context.Context) { + enabled, err := u.Process.IsEnabled(ctx) + if errors.Is(err, ErrNotSupported) { + u.Log.WarnContext(ctx, "Teleport is installed, but systemd is not present to start it.") + u.Log.WarnContext(ctx, "After configuring teleport.yaml, your system must also be configured to start Teleport.") + return + } + if errors.Is(err, ErrNotAvailable) { + u.Log.WarnContext(ctx, "Remember to use systemctl to enable and start Teleport.") + return + } + if err != nil { + u.Log.ErrorContext(ctx, "Failed to determine if Teleport is enabled.", errorKey, err) + return + } + active, err := u.Process.IsActive(ctx) + if err != nil { + u.Log.ErrorContext(ctx, "Failed to determine if Teleport is active.", errorKey, err) + return + } + if !enabled && active { + u.Log.WarnContext(ctx, "Teleport is installed and started, but not configured to start on boot.") + u.Log.WarnContext(ctx, "After configuring teleport.yaml, you must enable it.", + "command", "systemctl enable "+u.TeleportServiceName) + } + if !active && enabled { + u.Log.WarnContext(ctx, "Teleport is installed and enabled at boot, but not running.") + u.Log.WarnContext(ctx, "After configuring teleport.yaml, you must start it.", + "command", "systemctl start "+u.TeleportServiceName) + } + if !active && !enabled { + u.Log.WarnContext(ctx, "Teleport is installed, but not running or enabled at boot.") + u.Log.WarnContext(ctx, "After configuring teleport.yaml, you must enable and start.", + "command", "systemctl enable --now "+u.TeleportServiceName) + } +} + +// cleanup orphan installations +func (u *Updater) cleanup(ctx context.Context, cfg *UpdateConfig, keep []Revision) { + revs, err := u.Installer.List(ctx) + if err != nil { + u.Log.ErrorContext(ctx, "Failed to read installed versions.", errorKey, err) + return + } + if len(revs) < 3 { + return + } + u.Log.WarnContext(ctx, "More than two versions of Teleport are installed. Removing unused versions.", "count", len(revs)) + for _, v := range revs { + if v.Version == "" || slices.Contains(keep, v) { + continue + } + err := u.removeRevision(ctx, cfg, v) + if errors.Is(err, ErrLinked) { + u.Log.WarnContext(ctx, "Refusing to remove version with orphan links.", "version", v) + continue + } + if err != nil { + u.Log.WarnContext(ctx, "Failed to remove unused version of Teleport.", errorKey, err, "version", v) + continue + } + u.Log.WarnContext(ctx, "Deleted unused version of Teleport.", "version", v) + } +} + +// LinkPackage creates links from the system (package) installation of Teleport, if they are needed. +// LinkPackage returns nil and warns if an auto-updates version is already linked, but auto-updates is disabled. +// LinkPackage returns an error only if an unknown version of Teleport is present (e.g., manually copied files). +// This function is idempotent. +func (u *Updater) LinkPackage(ctx context.Context) error { + cfg, err := readConfig(u.UpdateConfigFile) + if err != nil { + return trace.Wrap(err, "failed to read %s", updateConfigName) + } + if err := validateConfigSpec(&cfg.Spec, OverrideConfig{}); err != nil { + return trace.Wrap(err) + } + active := cfg.Status.Active + if cfg.Spec.Enabled { + u.Log.InfoContext(ctx, "Automatic updates is enabled. Skipping system package link.", activeKey, active) + return nil + } + if cfg.Spec.Pinned { + u.Log.InfoContext(ctx, "Automatic update version is pinned. Skipping system package link.", activeKey, active) + return nil + } + // If an active version is set, but auto-updates is disabled, try to link the system installation in case the config is stale. + // If any links are present, this will return ErrLinked and not create any system links. + // This state is important to log as a warning, + if err := u.Installer.TryLinkSystem(ctx); errors.Is(err, ErrLinked) { + u.Log.WarnContext(ctx, "Automatic updates is disabled, but a non-package version of Teleport is linked.", activeKey, active) + return nil + } else if err != nil { + return trace.Wrap(err, "failed to link system package installation") + } + + // If syncing succeeds, ensure the installed systemd service can be found via systemctl. + // SELinux contexts can interfere with systemctl's ability to read service files. + if err := u.Process.Sync(ctx); errors.Is(err, ErrNotSupported) { + u.Log.WarnContext(ctx, "Systemd is not installed. Skipping sync.") + } else if err != nil { + return trace.Wrap(err, "failed to sync systemd configuration") + } else { + present, err := u.Process.IsPresent(ctx) + if errors.Is(err, ErrNotAvailable) { + u.Log.DebugContext(ctx, "Systemd version is outdated. Skipping SELinux verification.") + } else if err != nil { + return trace.Wrap(err, "failed to determine if Teleport has an installed systemd service") + } else if !present { + return trace.Errorf("cannot find systemd service for Teleport, check SELinux settings") + } + } + u.Log.InfoContext(ctx, "Successfully linked system package installation.") + return nil +} + +// UnlinkPackage removes links from the system (package) installation of Teleport, if they are present. +// This function is idempotent. +func (u *Updater) UnlinkPackage(ctx context.Context) error { + if err := u.Installer.UnlinkSystem(ctx); err != nil { + return trace.Wrap(err, "failed to unlink system package installation") + } + if err := u.Process.Sync(ctx); errors.Is(err, ErrNotSupported) { + u.Log.WarnContext(ctx, "Systemd is not installed. Skipping sync.") + } else if err != nil { + return trace.Wrap(err, "failed to sync systemd configuration") + } + u.Log.InfoContext(ctx, "Successfully unlinked system package installation.") + return nil +} diff --git a/lib/autoupdate/agent/updater_test.go b/lib/autoupdate/agent/updater_test.go new file mode 100644 index 0000000000000..57961cb7f26c6 --- /dev/null +++ b/lib/autoupdate/agent/updater_test.go @@ -0,0 +1,2167 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/gravitational/teleport/api/client/webclient" + "github.com/gravitational/teleport/lib/autoupdate" + "github.com/gravitational/teleport/lib/utils/testutils/golden" +) + +func TestMain(m *testing.M) { + initTime = time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + os.Exit(m.Run()) +} + +func TestWarnUmask(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + old int + warn bool + }{ + {old: 0o000, warn: false}, + {old: 0o001, warn: true}, + {old: 0o011, warn: true}, + {old: 0o111, warn: true}, + {old: 0o002, warn: false}, + {old: 0o020, warn: false}, + {old: 0o022, warn: false}, + {old: 0o220, warn: true}, + {old: 0o200, warn: true}, + {old: 0o222, warn: true}, + {old: 0o333, warn: true}, + {old: 0o444, warn: true}, + {old: 0o555, warn: true}, + {old: 0o666, warn: true}, + {old: 0o777, warn: true}, + } { + tt := tt + t.Run(fmt.Sprintf("old umask %o", tt.old), func(t *testing.T) { + ctx := context.Background() + out := &bytes.Buffer{} + warnUmask(ctx, slog.New(slog.NewTextHandler(out, + &slog.HandlerOptions{ReplaceAttr: msgOnly}, + )), tt.old) + assert.Equal(t, tt.warn, strings.Contains(out.String(), "detected")) + }) + } +} + +func TestUpdater_Disable(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + cfg *UpdateConfig // nil -> file not present + errMatch string + }{ + { + name: "enabled", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Enabled: true, + }, + }, + }, + { + name: "already disabled", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Enabled: false, + }, + }, + }, + { + name: "config does not exist", + }, + { + name: "invalid metadata", + cfg: &UpdateConfig{ + Spec: UpdateSpec{ + Enabled: true, + }, + }, + errMatch: "invalid", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + ns := &Namespace{installDir: dir} + _, err := ns.Init() + require.NoError(t, err) + cfgPath := filepath.Join(ns.Dir(), updateConfigName) + updater, err := NewLocalUpdater(LocalUpdaterConfig{ + InsecureSkipVerify: true, + }, ns) + require.NoError(t, err) + + // Create config file only if provided in test case + if tt.cfg != nil { + b, err := yaml.Marshal(tt.cfg) + require.NoError(t, err) + err = os.WriteFile(cfgPath, b, 0600) + require.NoError(t, err) + } + + err = updater.Disable(context.Background()) + if tt.errMatch != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMatch) + return + } + require.NoError(t, err) + + data, err := os.ReadFile(cfgPath) + + // If no config is present, disable should not create it + if tt.cfg == nil { + require.ErrorIs(t, err, os.ErrNotExist) + return + } + require.NoError(t, err) + + if golden.ShouldSet() { + golden.Set(t, data) + } + require.Equal(t, string(golden.Get(t)), string(data)) + }) + } +} + +func TestUpdater_Unpin(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + cfg *UpdateConfig // nil -> file not present + errMatch string + }{ + { + name: "pinned", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Pinned: true, + }, + }, + }, + { + name: "not pinned", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Pinned: false, + }, + }, + }, + { + name: "config does not exist", + }, + { + name: "invalid metadata", + cfg: &UpdateConfig{ + Spec: UpdateSpec{ + Enabled: true, + }, + }, + errMatch: "invalid", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + ns := &Namespace{installDir: dir} + _, err := ns.Init() + require.NoError(t, err) + cfgPath := filepath.Join(ns.Dir(), updateConfigName) + + updater, err := NewLocalUpdater(LocalUpdaterConfig{ + InsecureSkipVerify: true, + }, ns) + require.NoError(t, err) + + // Create config file only if provided in test case + if tt.cfg != nil { + b, err := yaml.Marshal(tt.cfg) + require.NoError(t, err) + err = os.WriteFile(cfgPath, b, 0600) + require.NoError(t, err) + } + + err = updater.Unpin(context.Background()) + if tt.errMatch != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMatch) + return + } + require.NoError(t, err) + + data, err := os.ReadFile(cfgPath) + + // If no config is present, disable should not create it + if tt.cfg == nil { + require.ErrorIs(t, err, os.ErrNotExist) + return + } + require.NoError(t, err) + + if golden.ShouldSet() { + golden.Set(t, data) + } + require.Equal(t, string(golden.Get(t)), string(data)) + }) + } +} + +func TestUpdater_Update(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + cfg *UpdateConfig // nil -> file not present + flags autoupdate.InstallFlags + inWindow bool + now bool + agpl bool + installErr error + setupErr error + reloadErr error + notActive bool + notEnabled bool + linkedRevisions []Revision + + removedRevisions []Revision + installedRevision Revision + installedBaseURL string + linkedRevision Revision + requestGroup string + reloadCalls int + revertCalls int + setupCalls int + restarted bool + errMatch string + }{ + { + name: "updates enabled during window", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + Group: "group", + BaseURL: "https://example.com", + Enabled: true, + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + }, + }, + inWindow: true, + + removedRevisions: []Revision{NewRevision("unknown-version", 0)}, + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: "https://example.com", + linkedRevision: NewRevision("16.3.0", 0), + requestGroup: "group", + setupCalls: 1, + restarted: true, + }, + { + name: "updates enabled now", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + Group: "group", + BaseURL: "https://example.com", + Enabled: true, + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + }, + }, + now: true, + + removedRevisions: []Revision{NewRevision("unknown-version", 0)}, + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: "https://example.com", + linkedRevision: NewRevision("16.3.0", 0), + requestGroup: "group", + setupCalls: 1, + restarted: true, + }, + { + name: "updates enabled now, not started or enabled", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + Group: "group", + BaseURL: "https://example.com", + Enabled: true, + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + }, + }, + now: true, + notEnabled: true, + notActive: true, + + removedRevisions: []Revision{NewRevision("unknown-version", 0)}, + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: "https://example.com", + linkedRevision: NewRevision("16.3.0", 0), + requestGroup: "group", + setupCalls: 1, + restarted: true, + }, + { + name: "updates disabled during window", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + Group: "group", + BaseURL: "https://example.com", + Enabled: false, + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + }, + }, + inWindow: true, + }, + { + name: "missing path during window", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Group: "group", + BaseURL: "https://example.com", + Enabled: true, + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + }, + }, + inWindow: true, + errMatch: "destination path", + }, + { + name: "updates enabled outside of window", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + Group: "group", + BaseURL: "https://example.com", + Enabled: true, + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + }, + }, + requestGroup: "group", + }, + { + name: "updates disabled outside of window", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + Group: "group", + BaseURL: "https://example.com", + Enabled: false, + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + }, + }, + }, + { + name: "insecure URL", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + BaseURL: "http://example.com", + Enabled: true, + }, + }, + inWindow: true, + + errMatch: "must use TLS", + }, + { + name: "install error", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + Enabled: true, + }, + }, + inWindow: true, + installErr: errors.New("install error"), + + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: autoupdate.DefaultBaseURL, + requestGroup: "default", + errMatch: "install error", + }, + { + name: "version already installed in window", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + BaseURL: "https://example.com", + Enabled: true, + }, + Status: UpdateStatus{ + Active: NewRevision("16.3.0", 0), + }, + }, + inWindow: true, + requestGroup: "default", + }, + { + name: "version already installed outside of window", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + BaseURL: "https://example.com", + Enabled: true, + }, + Status: UpdateStatus{ + Active: NewRevision("16.3.0", 0), + }, + }, + requestGroup: "default", + }, + { + name: "version detects as linked", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + BaseURL: "https://example.com", + Enabled: true, + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + }, + }, + linkedRevisions: []Revision{NewRevision("16.3.0", 0)}, + inWindow: true, + + requestGroup: "default", + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: "https://example.com", + linkedRevision: NewRevision("16.3.0", 0), + removedRevisions: []Revision{ + NewRevision("unknown-version", 0), + }, + setupCalls: 1, + restarted: true, + }, + { + name: "backup version removed on install", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + BaseURL: "https://example.com", + Enabled: true, + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + Backup: toPtr(NewRevision("backup-version", 0)), + }, + }, + inWindow: true, + + requestGroup: "default", + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: "https://example.com", + linkedRevision: NewRevision("16.3.0", 0), + removedRevisions: []Revision{ + NewRevision("backup-version", 0), + NewRevision("unknown-version", 0), + }, + setupCalls: 1, + restarted: true, + }, + { + name: "backup version is linked", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + BaseURL: "https://example.com", + Enabled: true, + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + Backup: toPtr(NewRevision("backup-version", 0)), + }, + }, + inWindow: true, + linkedRevisions: []Revision{NewRevision("backup-version", 0)}, + + requestGroup: "default", + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: "https://example.com", + linkedRevision: NewRevision("16.3.0", 0), + removedRevisions: []Revision{ + NewRevision("unknown-version", 0), + }, + setupCalls: 1, + restarted: true, + }, + { + name: "backup version kept when no change", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + BaseURL: "https://example.com", + Enabled: true, + }, + Status: UpdateStatus{ + Active: NewRevision("16.3.0", 0), + Backup: toPtr(NewRevision("backup-version", 0)), + }, + }, + inWindow: true, + requestGroup: "default", + }, + { + name: "config does not exist", + }, + { + name: "FIPS and Enterprise flags", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + BaseURL: "https://example.com", + Enabled: true, + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", autoupdate.FlagEnterprise|autoupdate.FlagFIPS), + Backup: toPtr(NewRevision("backup-version", autoupdate.FlagEnterprise|autoupdate.FlagFIPS)), + }, + }, + inWindow: true, + flags: autoupdate.FlagEnterprise | autoupdate.FlagFIPS, + + requestGroup: "default", + installedRevision: NewRevision("16.3.0", autoupdate.FlagEnterprise|autoupdate.FlagFIPS), + installedBaseURL: "https://example.com", + linkedRevision: NewRevision("16.3.0", autoupdate.FlagEnterprise|autoupdate.FlagFIPS), + removedRevisions: []Revision{ + NewRevision("backup-version", autoupdate.FlagEnterprise|autoupdate.FlagFIPS), + NewRevision("unknown-version", 0), + }, + setupCalls: 1, + restarted: true, + }, + { + name: "invalid metadata", + cfg: &UpdateConfig{}, + errMatch: "invalid", + }, + { + name: "setup fails", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + BaseURL: "https://example.com", + Enabled: true, + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + Backup: toPtr(NewRevision("backup-version", 0)), + }, + }, + inWindow: true, + setupErr: errors.New("setup error"), + + requestGroup: "default", + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: "https://example.com", + linkedRevision: NewRevision("16.3.0", 0), + removedRevisions: []Revision{ + NewRevision("backup-version", 0), + }, + reloadCalls: 1, + revertCalls: 1, + setupCalls: 1, + restarted: true, + errMatch: "setup error", + }, + { + name: "skip version", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + BaseURL: "https://example.com", + Enabled: true, + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + Backup: toPtr(NewRevision("backup-version", 0)), + Skip: toPtr(NewRevision("16.3.0", 0)), + }, + }, + inWindow: true, + requestGroup: "default", + }, + { + name: "pinned version", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + BaseURL: "https://example.com", + Enabled: true, + Pinned: true, + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + Backup: toPtr(NewRevision("backup-version", 0)), + }, + }, + inWindow: true, + requestGroup: "default", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + var requestedGroup string + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestedGroup = r.URL.Query().Get("group") + config := webclient.PingResponse{ + AutoUpdate: webclient.AutoUpdateSettings{ + AgentVersion: "16.3.0", + AgentAutoUpdate: tt.inWindow, + }, + } + config.Edition = "community" + if tt.flags&autoupdate.FlagEnterprise != 0 { + config.Edition = "ent" + } + if tt.agpl { + config.Edition = "oss" + } + config.FIPS = tt.flags&autoupdate.FlagFIPS != 0 + err := json.NewEncoder(w).Encode(config) + require.NoError(t, err) + })) + t.Cleanup(server.Close) + + dir := t.TempDir() + ns := &Namespace{ + installDir: dir, + defaultPathDir: "ignored", + updaterIDFile: "updater-id-file", + } + _, err := ns.Init() + require.NoError(t, err) + cfgPath := filepath.Join(ns.Dir(), updateConfigName) + + updater, err := NewLocalUpdater(LocalUpdaterConfig{ + InsecureSkipVerify: true, + }, ns) + require.NoError(t, err) + + // Create config file only if provided in test case + if tt.cfg != nil { + tt.cfg.Spec.Proxy = strings.TrimPrefix(server.URL, "https://") + b, err := yaml.Marshal(tt.cfg) + require.NoError(t, err) + err = os.WriteFile(cfgPath, b, 0600) + require.NoError(t, err) + } + + var ( + installedRevision Revision + installedBaseURL string + linkedRevision Revision + removedRevisions []Revision + revertFuncCalls int + setupCalls int + revertSetupCalls int + reloadCalls int + ) + updater.Installer = &testInstaller{ + FuncInstall: func(_ context.Context, rev Revision, baseURL string, force bool) error { + for _, r := range tt.linkedRevisions { + if r == rev { + require.False(t, force) + } + } + installedRevision = rev + installedBaseURL = baseURL + return tt.installErr + }, + FuncLink: func(_ context.Context, rev Revision, path string, force bool) (revert func(context.Context) bool, err error) { + linkedRevision = rev + return func(_ context.Context) bool { + revertFuncCalls++ + return true + }, nil + }, + FuncList: func(_ context.Context) (revs []Revision, err error) { + return slices.Compact([]Revision{ + installedRevision, + tt.cfg.Status.Active, + NewRevision("unknown-version", 0), + }), nil + }, + FuncRemove: func(_ context.Context, rev Revision) error { + removedRevisions = append(removedRevisions, rev) + return nil + }, + FuncIsLinked: func(ctx context.Context, rev Revision, path string) (bool, error) { + for _, r := range tt.linkedRevisions { + if r == rev { + return true, nil + } + } + return false, nil + }, + } + updater.Process = &testProcess{ + FuncReload: func(_ context.Context) error { + reloadCalls++ + return tt.reloadErr + }, + FuncIsPresent: func(ctx context.Context) (bool, error) { + return true, nil + }, + FuncIsEnabled: func(ctx context.Context) (bool, error) { + return !tt.notEnabled, nil + }, + FuncIsActive: func(ctx context.Context) (bool, error) { + return !tt.notActive, nil + }, + } + var restarted bool + updater.ReexecSetup = func(_ context.Context, path string, reload bool) error { + restarted = reload + setupCalls++ + return tt.setupErr + } + updater.SetupNamespace = func(_ context.Context, path string) error { + revertSetupCalls++ + return nil + } + + ctx := context.Background() + err = updater.Update(ctx, tt.now) + if tt.errMatch != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMatch) + } else { + require.NoError(t, err) + } + require.Equal(t, tt.installedRevision, installedRevision) + require.Equal(t, tt.installedBaseURL, installedBaseURL) + require.Equal(t, tt.linkedRevision, linkedRevision) + require.Equal(t, tt.removedRevisions, removedRevisions) + require.Equal(t, tt.flags, installedRevision.Flags) + require.Equal(t, tt.requestGroup, requestedGroup) + require.Equal(t, tt.reloadCalls, reloadCalls) + require.Equal(t, tt.revertCalls, revertSetupCalls) + require.Equal(t, tt.revertCalls, revertFuncCalls) + require.Equal(t, tt.setupCalls, setupCalls) + require.Equal(t, tt.restarted, restarted) + + if tt.cfg == nil { + _, err := os.Stat(cfgPath) + require.Error(t, err) + return + } + + data, err := os.ReadFile(cfgPath) + require.NoError(t, err) + data = blankTestAddr(data) + + if golden.ShouldSet() { + golden.Set(t, data) + } + require.Equal(t, string(golden.Get(t)), string(data)) + }) + } +} + +func TestUpdater_LinkPackage(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + cfg *UpdateConfig // nil -> file not present + tryLinkSystemErr error + syncErr error + notPresent bool + + syncCalls int + tryLinkSystemCalls int + errMatch string + }{ + { + name: "updates enabled", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Enabled: true, + }, + }, + + tryLinkSystemCalls: 0, + syncCalls: 0, + }, + { + name: "pinned", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Pinned: true, + }, + }, + + tryLinkSystemCalls: 0, + syncCalls: 0, + }, + { + name: "updates disabled", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Enabled: false, + }, + }, + + tryLinkSystemCalls: 1, + syncCalls: 1, + }, + { + name: "already linked", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Enabled: false, + }, + }, + tryLinkSystemErr: ErrLinked, + + tryLinkSystemCalls: 1, + syncCalls: 0, + }, + { + name: "link error", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Enabled: false, + }, + }, + tryLinkSystemErr: errors.New("bad"), + + tryLinkSystemCalls: 1, + syncCalls: 0, + errMatch: "bad", + }, + { + name: "no config", + tryLinkSystemCalls: 1, + syncCalls: 1, + }, + { + name: "systemd is not installed", + tryLinkSystemCalls: 1, + syncCalls: 1, + syncErr: ErrNotSupported, + }, + { + name: "systemd is not installed, already linked", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Enabled: false, + }, + }, + tryLinkSystemCalls: 1, + syncCalls: 1, + syncErr: ErrNotSupported, + }, + { + name: "SELinux blocks service from being read", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Enabled: false, + }, + }, + tryLinkSystemCalls: 1, + syncCalls: 1, + notPresent: true, + errMatch: "cannot find systemd service", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + ns := &Namespace{installDir: dir} + _, err := ns.Init() + require.NoError(t, err) + cfgPath := filepath.Join(ns.Dir(), updateConfigName) + + updater, err := NewLocalUpdater(LocalUpdaterConfig{ + InsecureSkipVerify: true, + }, ns) + require.NoError(t, err) + + // Create config file only if provided in test case + if tt.cfg != nil { + b, err := yaml.Marshal(tt.cfg) + require.NoError(t, err) + err = os.WriteFile(cfgPath, b, 0600) + require.NoError(t, err) + } + + var tryLinkSystemCalls int + updater.Installer = &testInstaller{ + FuncTryLinkSystem: func(_ context.Context) error { + tryLinkSystemCalls++ + return tt.tryLinkSystemErr + }, + } + var syncCalls int + updater.Process = &testProcess{ + FuncSync: func(_ context.Context) error { + syncCalls++ + return tt.syncErr + }, + FuncIsPresent: func(ctx context.Context) (bool, error) { + return !tt.notPresent, nil + }, + } + + ctx := context.Background() + err = updater.LinkPackage(ctx) + if tt.errMatch != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMatch) + } else { + require.NoError(t, err) + } + require.Equal(t, tt.tryLinkSystemCalls, tryLinkSystemCalls) + require.Equal(t, tt.syncCalls, syncCalls) + }) + } +} + +func TestUpdater_Remove(t *testing.T) { + t.Parallel() + + const version = "active-version" + + tests := []struct { + name string + cfg *UpdateConfig // nil -> file not present + linkSystemErr error + isActiveErr error + syncErr error + reloadErr error + processActive bool + force bool + serviceName string + + unlinkedVersion string + teardownCalls int + syncCalls int + revertFuncCalls int + linkSystemCalls int + reloadCalls int + errMatch string + }{ + { + name: "no config", + teardownCalls: 1, + }, + { + name: "no active version", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + }, + teardownCalls: 1, + }, + { + name: "no conflicting system links, process disabled, force", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Status: UpdateStatus{ + Active: NewRevision(version, 0), + }, + }, + unlinkedVersion: version, + teardownCalls: 1, + force: true, + }, + { + name: "no system links, process active, force", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + }, + Status: UpdateStatus{ + Active: NewRevision(version, 0), + }, + }, + linkSystemErr: ErrNoBinaries, + linkSystemCalls: 1, + processActive: true, + force: true, + errMatch: "refusing to remove", + }, + { + name: "no system links, process disabled, force", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + }, + Status: UpdateStatus{ + Active: NewRevision(version, 0), + }, + }, + linkSystemErr: ErrNoBinaries, + linkSystemCalls: 1, + unlinkedVersion: version, + teardownCalls: 1, + force: true, + }, + { + name: "no system links, process disabled, no force", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + }, + Status: UpdateStatus{ + Active: NewRevision(version, 0), + }, + }, + linkSystemErr: ErrNoBinaries, + linkSystemCalls: 1, + errMatch: "unable to remove", + }, + { + name: "no system links, process disabled, no systemd, force", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + }, + Status: UpdateStatus{ + Active: NewRevision(version, 0), + }, + }, + linkSystemErr: ErrNoBinaries, + linkSystemCalls: 1, + isActiveErr: ErrNotSupported, + unlinkedVersion: version, + teardownCalls: 1, + force: true, + }, + { + name: "no system links, process disabled, custom path, force", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: "custom", + }, + Status: UpdateStatus{ + Active: NewRevision(version, 0), + }, + }, + unlinkedVersion: version, + teardownCalls: 1, + force: true, + }, + { + name: "no system links, process disabled, custom path, no force", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: "custom", + }, + Status: UpdateStatus{ + Active: NewRevision(version, 0), + }, + }, + errMatch: "unable to remove", + }, + { + name: "no system links, process disabled, custom path, no force, custom service", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: "custom", + }, + Status: UpdateStatus{ + Active: NewRevision(version, 0), + }, + }, + serviceName: "custom", + unlinkedVersion: version, + teardownCalls: 1, + force: true, + }, + { + name: "active version", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + }, + Status: UpdateStatus{ + Active: NewRevision(version, 0), + }, + }, + linkSystemCalls: 1, + syncCalls: 1, + reloadCalls: 1, + teardownCalls: 1, + }, + { + name: "active version, no systemd", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + }, + Status: UpdateStatus{ + Active: NewRevision(version, 0), + }, + }, + linkSystemCalls: 1, + syncCalls: 1, + reloadCalls: 1, + teardownCalls: 1, + syncErr: ErrNotSupported, + reloadErr: ErrNotSupported, + }, + { + name: "active version, no reload", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + }, + Status: UpdateStatus{ + Active: NewRevision(version, 0), + }, + }, + linkSystemCalls: 1, + syncCalls: 1, + reloadCalls: 1, + teardownCalls: 1, + reloadErr: ErrNotNeeded, + }, + { + name: "active version, sync error", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + }, + Status: UpdateStatus{ + Active: NewRevision(version, 0), + }, + }, + linkSystemCalls: 1, + syncCalls: 2, + revertFuncCalls: 1, + syncErr: errors.New("sync error"), + errMatch: "configuration", + }, + { + name: "active version, reload error", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Path: defaultPathDir, + }, + Status: UpdateStatus{ + Active: NewRevision(version, 0), + }, + }, + linkSystemCalls: 1, + syncCalls: 2, + reloadCalls: 2, + revertFuncCalls: 1, + reloadErr: errors.New("reload error"), + errMatch: "start", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + ns := &Namespace{installDir: dir} + _, err := ns.Init() + require.NoError(t, err) + cfgPath := filepath.Join(ns.Dir(), updateConfigName) + + updater, err := NewLocalUpdater(LocalUpdaterConfig{ + InsecureSkipVerify: true, + }, ns) + require.NoError(t, err) + updater.TeleportServiceName = serviceName + if tt.serviceName != "" { + updater.TeleportServiceName = tt.serviceName + } + + // Create config file only if provided in test case + if tt.cfg != nil { + b, err := yaml.Marshal(tt.cfg) + require.NoError(t, err) + err = os.WriteFile(cfgPath, b, 0600) + require.NoError(t, err) + } + + var ( + linkSystemCalls int + revertFuncCalls int + syncCalls int + reloadCalls int + teardownCalls int + unlinkedVersion string + ) + updater.Installer = &testInstaller{ + FuncLinkSystem: func(_ context.Context) (revert func(context.Context) bool, err error) { + linkSystemCalls++ + return func(_ context.Context) bool { + revertFuncCalls++ + return true + }, tt.linkSystemErr + }, + FuncUnlink: func(_ context.Context, rev Revision, path string) error { + unlinkedVersion = rev.Version + return nil + }, + } + updater.Process = &testProcess{ + FuncSync: func(_ context.Context) error { + syncCalls++ + return tt.syncErr + }, + FuncReload: func(_ context.Context) error { + reloadCalls++ + return tt.reloadErr + }, + FuncIsActive: func(_ context.Context) (bool, error) { + return tt.processActive, tt.isActiveErr + }, + } + updater.TeardownNamespace = func(_ context.Context) error { + teardownCalls++ + return nil + } + + ctx := context.Background() + err = updater.Remove(ctx, tt.force) + if tt.errMatch != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMatch) + } else { + require.NoError(t, err) + } + require.Equal(t, tt.syncCalls, syncCalls) + require.Equal(t, tt.reloadCalls, reloadCalls) + require.Equal(t, tt.linkSystemCalls, linkSystemCalls) + require.Equal(t, tt.revertFuncCalls, revertFuncCalls) + require.Equal(t, tt.unlinkedVersion, unlinkedVersion) + require.Equal(t, tt.teardownCalls, teardownCalls) + }) + } +} + +func TestUpdater_Install(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + cfg *UpdateConfig // nil -> file not present + userCfg OverrideConfig + flags autoupdate.InstallFlags + agpl bool + installErr error + setupErr error + reloadErr error + notPresent bool + notEnabled bool + notActive bool + + removedRevision Revision + installedRevision Revision + installedBaseURL string + linkedRevision Revision + requestGroup string + reloadCalls int + revertCalls int + setupCalls int + restarted bool + errMatch string + }{ + { + name: "config from file", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Enabled: true, + Group: "group", + Path: "/path", + BaseURL: "https://example.com", + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + }, + }, + + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: "https://example.com", + linkedRevision: NewRevision("16.3.0", 0), + requestGroup: "group", + setupCalls: 1, + restarted: true, + }, + { + name: "config from user", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + Group: "old-group", + BaseURL: "https://example.com/old", + }, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + }, + }, + userCfg: OverrideConfig{ + UpdateSpec: UpdateSpec{ + Enabled: true, + Path: "/path", + Group: "new-group", + BaseURL: "https://example.com/new", + }, + ForceVersion: "new-version", + }, + + installedRevision: NewRevision("new-version", 0), + installedBaseURL: "https://example.com/new", + linkedRevision: NewRevision("new-version", 0), + requestGroup: "new-group", + setupCalls: 1, + restarted: true, + }, + { + name: "defaults", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + }, + }, + + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: autoupdate.DefaultBaseURL, + linkedRevision: NewRevision("16.3.0", 0), + requestGroup: "default", + setupCalls: 1, + restarted: true, + }, + { + name: "override skip", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + Skip: toPtr(NewRevision("16.3.0", 0)), + }, + }, + + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: autoupdate.DefaultBaseURL, + linkedRevision: NewRevision("16.3.0", 0), + requestGroup: "default", + setupCalls: 1, + restarted: true, + }, + { + name: "insecure URL", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Spec: UpdateSpec{ + BaseURL: "http://example.com", + }, + }, + + errMatch: "must use TLS", + }, + { + name: "install error", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + }, + installErr: errors.New("install error"), + + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: autoupdate.DefaultBaseURL, + requestGroup: "default", + errMatch: "install error", + }, + { + name: "version already installed", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Status: UpdateStatus{ + Active: NewRevision("16.3.0", 0), + }, + }, + + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: autoupdate.DefaultBaseURL, + linkedRevision: NewRevision("16.3.0", 0), + requestGroup: "default", + setupCalls: 1, + restarted: false, + }, + { + name: "backup version removed on install", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Status: UpdateStatus{ + Active: NewRevision("old-version", 0), + Backup: toPtr(NewRevision("backup-version", 0)), + }, + }, + + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: autoupdate.DefaultBaseURL, + linkedRevision: NewRevision("16.3.0", 0), + removedRevision: NewRevision("backup-version", 0), + requestGroup: "default", + setupCalls: 1, + restarted: true, + }, + { + name: "backup version kept for validation", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Status: UpdateStatus{ + Active: NewRevision("16.3.0", 0), + Backup: toPtr(NewRevision("backup-version", 0)), + }, + }, + + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: autoupdate.DefaultBaseURL, + linkedRevision: NewRevision("16.3.0", 0), + requestGroup: "default", + setupCalls: 1, + }, + { + name: "config does not exist", + + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: autoupdate.DefaultBaseURL, + linkedRevision: NewRevision("16.3.0", 0), + requestGroup: "default", + setupCalls: 1, + restarted: true, + }, + { + name: "FIPS and Enterprise flags", + flags: autoupdate.FlagEnterprise | autoupdate.FlagFIPS, + installedRevision: NewRevision("16.3.0", autoupdate.FlagEnterprise|autoupdate.FlagFIPS), + installedBaseURL: autoupdate.DefaultBaseURL, + linkedRevision: NewRevision("16.3.0", autoupdate.FlagEnterprise|autoupdate.FlagFIPS), + requestGroup: "default", + setupCalls: 1, + restarted: true, + }, + { + name: "invalid metadata", + cfg: &UpdateConfig{}, + errMatch: "invalid", + }, + { + name: "setup fails", + setupErr: errors.New("setup error"), + + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: autoupdate.DefaultBaseURL, + linkedRevision: NewRevision("16.3.0", 0), + requestGroup: "default", + revertCalls: 1, + setupCalls: 1, + reloadCalls: 1, + restarted: true, + errMatch: "setup error", + }, + { + name: "setup fails already installed", + cfg: &UpdateConfig{ + Version: updateConfigVersion, + Kind: updateConfigKind, + Status: UpdateStatus{ + Active: NewRevision("16.3.0", 0), + }, + }, + setupErr: errors.New("setup error"), + + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: autoupdate.DefaultBaseURL, + linkedRevision: NewRevision("16.3.0", 0), + requestGroup: "default", + revertCalls: 1, + setupCalls: 1, + errMatch: "setup error", + }, + { + name: "no need to reload", + reloadErr: ErrNotNeeded, + + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: autoupdate.DefaultBaseURL, + linkedRevision: NewRevision("16.3.0", 0), + requestGroup: "default", + setupCalls: 1, + restarted: true, + }, + { + name: "not started or enabled", + notEnabled: true, + notActive: true, + + installedRevision: NewRevision("16.3.0", 0), + installedBaseURL: autoupdate.DefaultBaseURL, + linkedRevision: NewRevision("16.3.0", 0), + requestGroup: "default", + setupCalls: 1, + restarted: true, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + ns := &Namespace{ + installDir: dir, + defaultPathDir: defaultPathDir, + defaultProxyAddr: "default-proxy", + updaterIDFile: "updater-id-file", + } + _, err := ns.Init() + require.NoError(t, err) + cfgPath := filepath.Join(ns.Dir(), updateConfigName) + + updater, err := NewLocalUpdater(LocalUpdaterConfig{ + InsecureSkipVerify: true, + }, ns) + require.NoError(t, err) + + // Create config file only if provided in test case + if tt.cfg != nil { + b, err := yaml.Marshal(tt.cfg) + require.NoError(t, err) + err = os.WriteFile(cfgPath, b, 0600) + require.NoError(t, err) + } + + var requestedGroup string + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestedGroup = r.URL.Query().Get("group") + config := webclient.PingResponse{ + AutoUpdate: webclient.AutoUpdateSettings{ + AgentVersion: "16.3.0", + }, + } + config.Edition = "community" + if tt.flags&autoupdate.FlagEnterprise != 0 { + config.Edition = "ent" + } + if tt.agpl { + config.Edition = "oss" + } + config.FIPS = tt.flags&autoupdate.FlagFIPS != 0 + err := json.NewEncoder(w).Encode(config) + require.NoError(t, err) + })) + t.Cleanup(server.Close) + + if tt.userCfg.Proxy == "" { + tt.userCfg.Proxy = strings.TrimPrefix(server.URL, "https://") + } + updater.DefaultProxyAddr = tt.userCfg.Proxy + + var ( + installedRevision Revision + installedBaseURL string + linkedRevision Revision + removedRevision Revision + revertFuncCalls int + reloadCalls int + setupCalls int + revertSetupCalls int + ) + updater.Installer = &testInstaller{ + FuncInstall: func(_ context.Context, rev Revision, baseURL string, force bool) error { + installedRevision = rev + installedBaseURL = baseURL + return tt.installErr + }, + FuncLink: func(_ context.Context, rev Revision, path string, force bool) (revert func(context.Context) bool, err error) { + linkedRevision = rev + return func(_ context.Context) bool { + revertFuncCalls++ + return true + }, nil + }, + FuncList: func(_ context.Context) (revs []Revision, err error) { + return []Revision{}, nil + }, + FuncRemove: func(_ context.Context, rev Revision) error { + removedRevision = rev + return nil + }, + FuncIsLinked: func(ctx context.Context, rev Revision, path string) (bool, error) { + return false, nil + }, + } + updater.Process = &testProcess{ + FuncReload: func(_ context.Context) error { + reloadCalls++ + return tt.reloadErr + }, + FuncIsPresent: func(ctx context.Context) (bool, error) { + return !tt.notPresent, nil + }, + FuncIsEnabled: func(ctx context.Context) (bool, error) { + return !tt.notEnabled, nil + }, + FuncIsActive: func(ctx context.Context) (bool, error) { + return !tt.notActive, nil + }, + } + var restarted bool + updater.ReexecSetup = func(_ context.Context, path string, reload bool) error { + setupCalls++ + restarted = reload + return tt.setupErr + } + updater.SetupNamespace = func(_ context.Context, path string) error { + revertSetupCalls++ + return nil + } + + ctx := context.Background() + err = updater.Install(ctx, tt.userCfg) + if tt.errMatch != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMatch) + } else { + require.NoError(t, err) + } + require.Equal(t, tt.installedRevision, installedRevision) + require.Equal(t, tt.installedBaseURL, installedBaseURL) + require.Equal(t, tt.linkedRevision, linkedRevision) + require.Equal(t, tt.removedRevision, removedRevision) + require.Equal(t, tt.flags, installedRevision.Flags) + require.Equal(t, tt.requestGroup, requestedGroup) + require.Equal(t, tt.reloadCalls, reloadCalls) + require.Equal(t, tt.revertCalls, revertSetupCalls) + require.Equal(t, tt.revertCalls, revertFuncCalls) + require.Equal(t, tt.setupCalls, setupCalls) + require.Equal(t, tt.restarted, restarted) + + if tt.cfg == nil && err != nil { + _, err := os.Stat(cfgPath) + require.Error(t, err) + return + } + + data, err := os.ReadFile(cfgPath) + require.NoError(t, err) + data = blankTestAddr(data) + + if golden.ShouldSet() { + golden.Set(t, data) + } + require.Equal(t, string(golden.Get(t)), string(data)) + }) + } +} + +func TestSameProxies(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + a, b string + match bool + }{ + { + name: "protocol missing with port", + a: "https://example.com:8080", + b: "example.com:8080", + match: true, + }, + { + name: "protocol missing without port", + a: "https://example.com", + b: "example.com", + match: true, + }, + { + name: "same with port", + a: "example.com:443", + b: "example.com:443", + match: true, + }, + { + name: "does not set default teleport port", + a: "example.com", + b: "example.com:3080", + match: false, + }, + { + name: "does set default standard port", + a: "example.com", + b: "example.com:443", + match: true, + }, + { + name: "other formats if equal", + a: "@123", + b: "@123", + match: true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + s := sameProxies(tt.a, tt.b) + require.Equal(t, tt.match, s) + }) + } +} + +func TestUpdater_Setup(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + restart bool + present bool + setupErr error + presentErr error + reloadErr error + + errMatch string + }{ + { + name: "no restart", + restart: false, + present: true, + }, + { + name: "restart", + restart: true, + present: true, + }, + { + name: "reload not needed", + restart: true, + present: true, + reloadErr: ErrNotNeeded, + }, + { + name: "not present", + restart: true, + present: false, + errMatch: "cannot find systemd", + }, + { + name: "setup error", + restart: false, + setupErr: errors.New("some error"), + errMatch: "some error", + }, + { + name: "setup error canceled", + restart: false, + setupErr: context.Canceled, + errMatch: "canceled", + }, + { + name: "present error", + restart: false, + presentErr: errors.New("some error"), + errMatch: "some error", + }, + { + name: "present error canceled", + restart: false, + presentErr: context.Canceled, + errMatch: "canceled", + }, + { + name: "preset error not supported", + restart: false, + presentErr: ErrNotSupported, + }, + { + name: "reload error canceled", + restart: true, + present: true, + reloadErr: context.Canceled, + errMatch: "canceled", + }, + { + name: "reload error", + restart: true, + present: true, + reloadErr: errors.New("some error"), + errMatch: "some error", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + ns := &Namespace{} + updater, err := NewLocalUpdater(LocalUpdaterConfig{}, ns) + require.NoError(t, err) + + updater.Process = &testProcess{ + FuncReload: func(_ context.Context) error { + return tt.reloadErr + }, + FuncIsPresent: func(_ context.Context) (bool, error) { + return tt.present, tt.presentErr + }, + } + updater.SetupNamespace = func(_ context.Context, path string) error { + require.Equal(t, "test", path) + return tt.setupErr + } + + ctx := context.Background() + err = updater.Setup(ctx, "test", tt.restart) + if tt.errMatch != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errMatch) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestFindDBPID(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + existingID string + systemID string + namespaceID string + persist bool + + result string + resFile string + errMatch string + }{ + { + name: "no ids", + errMatch: "empty", + }, + { + name: "both ids", + systemID: "test1", + namespaceID: "test2", + result: "2c652f3c-ae11-5e5a-ba40-73cba91149cd", + }, + { + name: "systemID", + systemID: "test1", + result: "6abf58ef-f6cc-55ae-91e2-376af6b0ba90", + }, + { + name: "systemID matching namespaceID", + namespaceID: "test1", + result: "35321002-ed1c-51e3-a3b3-cbf0f0bc2d88", + }, + { + name: "namespaceID", + namespaceID: "test2", + result: "906a648e-038b-576a-893e-d6b32a9d8aee", + }, + { + name: "namespaceID matching systemID", + systemID: "test2", + result: "42929c01-bcc8-5a49-af36-e5f21344d5f5", + }, + { + name: "existing file not replaced", + existingID: "existing", + systemID: "test1", + namespaceID: "test2", + result: "existing", + resFile: "existing", + }, + { + name: "existing file not replaced on persist", + existingID: "existing", + systemID: "test1", + namespaceID: "test2", + persist: true, + result: "existing", + resFile: "existing", + }, + { + name: "persisted when missing", + systemID: "test1", + namespaceID: "test2", + persist: true, + result: "2c652f3c-ae11-5e5a-ba40-73cba91149cd", + resFile: "2c652f3c-ae11-5e5a-ba40-73cba91149cd", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + existing := filepath.Join(t.TempDir(), "existing") + if tt.existingID != "" { + err := os.WriteFile(existing, []byte(tt.existingID), os.ModePerm) + require.NoError(t, err) + } + s, err := FindDBPID(existing, []byte(tt.systemID), []byte(tt.namespaceID), tt.persist) + if tt.errMatch != "" { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, tt.result, s) + + if tt.resFile != "" { + b, err := os.ReadFile(existing) + require.NoError(t, err) + require.Equal(t, tt.resFile, string(b)) + } else { + require.NoFileExists(t, existing) + } + }) + } +} + +var serverRegexp = regexp.MustCompile("127.0.0.1:[0-9]+") + +func blankTestAddr(s []byte) []byte { + return serverRegexp.ReplaceAll(s, []byte("localhost")) +} + +type testInstaller struct { + FuncInstall func(ctx context.Context, rev Revision, baseURL string, force bool) error + FuncRemove func(ctx context.Context, rev Revision) error + FuncLink func(ctx context.Context, rev Revision, path string, force bool) (revert func(context.Context) bool, err error) + FuncLinkSystem func(ctx context.Context) (revert func(context.Context) bool, err error) + FuncTryLink func(ctx context.Context, rev Revision, path string) error + FuncTryLinkSystem func(ctx context.Context) error + FuncUnlink func(ctx context.Context, rev Revision, path string) error + FuncUnlinkSystem func(ctx context.Context) error + FuncList func(ctx context.Context) (revs []Revision, err error) + FuncIsLinked func(ctx context.Context, rev Revision, path string) (bool, error) +} + +func (ti *testInstaller) Install(ctx context.Context, rev Revision, baseURL string, force bool) error { + return ti.FuncInstall(ctx, rev, baseURL, force) +} + +func (ti *testInstaller) Remove(ctx context.Context, rev Revision) error { + return ti.FuncRemove(ctx, rev) +} + +func (ti *testInstaller) Link(ctx context.Context, rev Revision, path string, force bool) (revert func(context.Context) bool, err error) { + return ti.FuncLink(ctx, rev, path, force) +} + +func (ti *testInstaller) LinkSystem(ctx context.Context) (revert func(context.Context) bool, err error) { + return ti.FuncLinkSystem(ctx) +} + +func (ti *testInstaller) TryLink(ctx context.Context, rev Revision, path string) error { + return ti.FuncTryLink(ctx, rev, path) +} + +func (ti *testInstaller) TryLinkSystem(ctx context.Context) error { + return ti.FuncTryLinkSystem(ctx) +} + +func (ti *testInstaller) Unlink(ctx context.Context, rev Revision, path string) error { + return ti.FuncUnlink(ctx, rev, path) +} + +func (ti *testInstaller) UnlinkSystem(ctx context.Context) error { + return ti.FuncUnlinkSystem(ctx) +} + +func (ti *testInstaller) List(ctx context.Context) (revs []Revision, err error) { + return ti.FuncList(ctx) +} + +func (ti *testInstaller) IsLinked(ctx context.Context, rev Revision, path string) (bool, error) { + return ti.FuncIsLinked(ctx, rev, path) +} + +type testProcess struct { + FuncReload func(ctx context.Context) error + FuncSync func(ctx context.Context) error + FuncIsEnabled func(ctx context.Context) (bool, error) + FuncIsActive func(ctx context.Context) (bool, error) + FuncIsPresent func(ctx context.Context) (bool, error) +} + +func (tp *testProcess) Reload(ctx context.Context) error { + return tp.FuncReload(ctx) +} + +func (tp *testProcess) Sync(ctx context.Context) error { + return tp.FuncSync(ctx) +} + +func (tp *testProcess) IsEnabled(ctx context.Context) (bool, error) { + return tp.FuncIsEnabled(ctx) +} + +func (tp *testProcess) IsActive(ctx context.Context) (bool, error) { + return tp.FuncIsActive(ctx) +} + +func (tp *testProcess) IsPresent(ctx context.Context) (bool, error) { + return tp.FuncIsPresent(ctx) +} diff --git a/lib/autoupdate/agent/validate.go b/lib/autoupdate/agent/validate.go new file mode 100644 index 0000000000000..8ce1732d3d5d1 --- /dev/null +++ b/lib/autoupdate/agent/validate.go @@ -0,0 +1,130 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "bytes" + "context" + "io" + "log/slog" + "os" + "path/filepath" + "unicode" + + "github.com/gravitational/trace" +) + +const ( + // fileHeaderSniffBytes is the max size to read to determine a file's MIME type + fileHeaderSniffBytes = 512 // MIME standard size + // execModeMask is the minimum required set of bits to consider a file executable. + execModeMask = 0111 +) + +// Validator validates filesystem paths. +type Validator struct { + Log *slog.Logger +} + +// IsBinary returns true for working binaries that are executable by all users. +// If the file is irregular, non-executable, or a shell script, IsBinary returns false and logs a warning. +// IsBinary errors if lstat fails, a regular file is unreadable, or an executable file does not execute. +func (v *Validator) IsBinary(ctx context.Context, path string) (bool, error) { + // The behavior of this method is intended to protect against executable files + // being adding to the Teleport tgz that should not be made available on PATH, + // and additionally, should not cause installation to fail. + // While known copies of these files (e.g., "install") are excluded during extraction, + // it is safer to assume others could be present in past or future tgzs. + + if exec, err := v.IsExecutable(ctx, path); err != nil || !exec { + return exec, trace.Wrap(err) + } + name := filepath.Base(path) + d, err := readFileLimit(path, fileHeaderSniffBytes) + if err != nil { + return false, trace.Wrap(err) + } + // Refuse to test or link shell scripts + if isTextScript(d) { + v.Log.WarnContext(ctx, "Found unexpected shell script", "name", name) + return false, nil + } + v.Log.InfoContext(ctx, "Validating binary", "name", name) + r := localExec{ + Log: v.Log, + ErrLevel: slog.LevelDebug, + OutLevel: slog.LevelInfo, // always show version + } + code, err := r.Run(ctx, path, "version") + if code < 0 { + return false, trace.Wrap(err, "error validating binary %s", name) + } + if code > 0 { + v.Log.InfoContext(ctx, "Binary does not support version command", "name", name) + } + return true, nil +} + +// IsExecutable returns true for regular, executable files. +func (v *Validator) IsExecutable(ctx context.Context, path string) (bool, error) { + name := filepath.Base(path) + fi, err := os.Lstat(path) + if err != nil { + return false, trace.Wrap(err) + } + if !fi.Mode().IsRegular() { + v.Log.WarnContext(ctx, "Found unexpected irregular file", "name", name) + return false, nil + } + if fi.Mode()&execModeMask != execModeMask { + v.Log.WarnContext(ctx, "Found unexpected non-executable file", "name", name) + return false, nil + } + return true, nil +} + +func isTextScript(data []byte) bool { + data = bytes.TrimLeftFunc(data, unicode.IsSpace) + if !bytes.HasPrefix(data, []byte("#!")) { + return false + } + // Assume presence of MIME binary data bytes means binary: + // https://mimesniff.spec.whatwg.org/#terminology + for _, b := range data { + switch { + case b <= 0x08, b == 0x0B, + 0x0E <= b && b <= 0x1A, + 0x1C <= b && b <= 0x1F: + return false + } + } + return true +} + +// readFileLimit the first n bytes of a file. +func readFileLimit(name string, n int64) ([]byte, error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + defer f.Close() + var buf bytes.Buffer + _, err = io.Copy(&buf, io.LimitReader(f, n)) + return buf.Bytes(), trace.Wrap(err) +} diff --git a/lib/autoupdate/agent/validate_test.go b/lib/autoupdate/agent/validate_test.go new file mode 100644 index 0000000000000..9949792ef490e --- /dev/null +++ b/lib/autoupdate/agent/validate_test.go @@ -0,0 +1,103 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package agent + +import ( + "bytes" + "context" + "log/slog" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValidator_IsBinary(t *testing.T) { + for _, tt := range []struct { + name string + mode os.FileMode + contents string + + valid bool + errMatch string + logMatch string + }{ + { + name: "missing", + errMatch: "no such", + }, + { + name: "non-executable", + contents: "test", + mode: 0666, + logMatch: "non-executable", + }, + { + name: "shell script", + contents: " #!bash ", + mode: 0777, + logMatch: "unexpected shell", + }, + { + name: "unqualified shell script", + contents: " #!bash" + string([]byte{0x0B}), + mode: 0777, + errMatch: "validating binary", + }, + { + name: "exit 0", + contents: "#!/bin/sh\nexit 0\n" + string([]byte{0x0B}), + mode: 0777, + valid: true, + }, + { + name: "exit 1", + contents: "#!/bin/sh\nexit 1\n" + string([]byte{0x0B}), + mode: 0777, + valid: true, + logMatch: "version command", + }, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + opts := &slog.HandlerOptions{AddSource: true} + log := slog.New(slog.NewTextHandler(&buf, opts)) + v := Validator{Log: log} + ctx := context.Background() + path := filepath.Join(t.TempDir(), "file") + if tt.contents != "" { + os.WriteFile(path, []byte(tt.contents), tt.mode) + } + val, err := v.IsBinary(ctx, path) + if tt.logMatch != "" { + require.Contains(t, buf.String(), tt.logMatch) + } + if tt.errMatch != "" { + require.Error(t, err) + require.False(t, val) + require.Contains(t, err.Error(), tt.errMatch) + return + } + require.Equal(t, tt.valid, val) + require.NoError(t, err) + }) + } +} diff --git a/lib/autoupdate/package_url.go b/lib/autoupdate/package_url.go index 9b283c3da59c2..b00eb59fea5c9 100644 --- a/lib/autoupdate/package_url.go +++ b/lib/autoupdate/package_url.go @@ -20,10 +20,12 @@ package autoupdate import ( "bytes" + "encoding/json" "runtime" "text/template" "github.com/gravitational/trace" + "gopkg.in/yaml.v3" ) // InstallFlags sets flags for the Teleport installation. @@ -54,6 +56,82 @@ const ( BaseURLEnvVar = "TELEPORT_CDN_BASE_URL" ) +// NewInstallFlagsFromStrings returns InstallFlags given a slice of human-readable strings. +func NewInstallFlagsFromStrings(s []string) InstallFlags { + var out InstallFlags + for _, f := range s { + for _, flag := range []InstallFlags{ + FlagEnterprise, + FlagFIPS, + } { + if f == flag.String() { + out |= flag + } + } + } + return out +} + +// Strings converts InstallFlags to a slice of human-readable strings. +func (i InstallFlags) Strings() []string { + var out []string + for _, flag := range []InstallFlags{ + FlagEnterprise, + FlagFIPS, + } { + if i&flag != 0 { + out = append(out, flag.String()) + } + } + return out +} + +// String returns the string representation of a single InstallFlag flag, or "Unknown". +func (i InstallFlags) String() string { + switch i { + case 0: + return "" + case FlagEnterprise: + return "Enterprise" + case FlagFIPS: + return "FIPS" + } + return "Unknown" +} + +// DirFlag returns the directory path representation of a single InstallFlag flag, or "unknown". +func (i InstallFlags) DirFlag() string { + switch i { + case 0: + return "" + case FlagEnterprise: + return "ent" + case FlagFIPS: + return "fips" + } + return "unknown" +} + +func (i InstallFlags) MarshalYAML() (any, error) { + return i.Strings(), nil +} + +func (i InstallFlags) MarshalJSON() ([]byte, error) { + return json.Marshal(i.Strings()) +} + +func (i *InstallFlags) UnmarshalYAML(n *yaml.Node) error { + var s []string + if err := n.Decode(&s); err != nil { + return trace.Wrap(err) + } + if i == nil { + return trace.BadParameter("nil install flags while parsing YAML") + } + *i = NewInstallFlagsFromStrings(s) + return nil +} + // MakeURL constructs the package download URL from template, base URL and revision. func MakeURL(uriTmpl string, baseURL string, pkg string, version string, flags InstallFlags) (string, error) { tmpl, err := template.New("uri").Parse(uriTmpl) diff --git a/lib/autoupdate/package_url_test.go b/lib/autoupdate/package_url_test.go new file mode 100644 index 0000000000000..b3eca4be38d7e --- /dev/null +++ b/lib/autoupdate/package_url_test.go @@ -0,0 +1,95 @@ +/* + * Teleport + * Copyright (C) 2025 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package autoupdate + +import ( + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestInstallFlagsYAML(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + yaml string + flags InstallFlags + skipYAML bool + }{ + { + name: "both", + yaml: `["Enterprise", "FIPS"]`, + flags: FlagEnterprise | FlagFIPS, + }, + { + name: "order", + yaml: `["FIPS", "Enterprise"]`, + flags: FlagEnterprise | FlagFIPS, + skipYAML: true, + }, + { + name: "extra", + yaml: `["FIPS", "Enterprise", "bad"]`, + flags: FlagEnterprise | FlagFIPS, + skipYAML: true, + }, + { + name: "enterprise", + yaml: `["Enterprise"]`, + flags: FlagEnterprise, + }, + { + name: "fips", + yaml: `["FIPS"]`, + flags: FlagFIPS, + }, + { + name: "empty", + yaml: `[]`, + }, + { + name: "nil", + skipYAML: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + var flags InstallFlags + err := yaml.Unmarshal([]byte(tt.yaml), &flags) + require.NoError(t, err) + require.Equal(t, tt.flags, flags) + + // verify test YAML + var v any + err = yaml.Unmarshal([]byte(tt.yaml), &v) + require.NoError(t, err) + res, err := yaml.Marshal(v) + require.NoError(t, err) + + // compare verified YAML to flag output + out, err := yaml.Marshal(flags) + require.NoError(t, err) + + if !tt.skipYAML { + require.Equal(t, string(res), string(out)) + } + }) + } +} diff --git a/lib/autoupdate/rollout/client.go b/lib/autoupdate/rollout/client.go new file mode 100644 index 0000000000000..bde2267d095de --- /dev/null +++ b/lib/autoupdate/rollout/client.go @@ -0,0 +1,50 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package rollout + +import ( + "context" + + autoupdatepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + "github.com/gravitational/teleport/api/types" +) + +// Client is the subset of the Teleport client RPCs the controller needs. +type Client interface { + // GetAutoUpdateConfig gets the AutoUpdateConfig singleton resource. + GetAutoUpdateConfig(ctx context.Context) (*autoupdatepb.AutoUpdateConfig, error) + + // GetAutoUpdateVersion gets the AutoUpdateVersion singleton resource. + GetAutoUpdateVersion(ctx context.Context) (*autoupdatepb.AutoUpdateVersion, error) + + // GetAutoUpdateAgentRollout gets the AutoUpdateAgentRollout singleton resource. + GetAutoUpdateAgentRollout(ctx context.Context) (*autoupdatepb.AutoUpdateAgentRollout, error) + + // CreateAutoUpdateAgentRollout creates the AutoUpdateAgentRollout singleton resource. + CreateAutoUpdateAgentRollout(ctx context.Context, rollout *autoupdatepb.AutoUpdateAgentRollout) (*autoupdatepb.AutoUpdateAgentRollout, error) + + // UpdateAutoUpdateAgentRollout updates the AutoUpdateAgentRollout singleton resource. + UpdateAutoUpdateAgentRollout(ctx context.Context, rollout *autoupdatepb.AutoUpdateAgentRollout) (*autoupdatepb.AutoUpdateAgentRollout, error) + + // DeleteAutoUpdateAgentRollout deletes the AutoUpdateAgentRollout singleton resource. + DeleteAutoUpdateAgentRollout(ctx context.Context) error + + // GetClusterMaintenanceConfig loads the current maintenance config singleton. + GetClusterMaintenanceConfig(ctx context.Context) (types.ClusterMaintenanceConfig, error) +} diff --git a/lib/autoupdate/rollout/client_test.go b/lib/autoupdate/rollout/client_test.go new file mode 100644 index 0000000000000..782251a562025 --- /dev/null +++ b/lib/autoupdate/rollout/client_test.go @@ -0,0 +1,229 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package rollout + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/protoadapt" + + "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + "github.com/gravitational/teleport/api/types" + apiutils "github.com/gravitational/teleport/api/utils" +) + +// mockClient is a mock implementation if the Client interface for testing purposes. +// This is used to precisely check which calls are made by the reconciler during tests. +// Use newMockClient to create one from stubs. Once the test is over, you must call +// mockClient.checkIfEmpty to validate all expected calls were made. +type mockClient struct { + getAutoUpdateConfig *getHandler[*autoupdate.AutoUpdateConfig] + getAutoUpdateVersion *getHandler[*autoupdate.AutoUpdateVersion] + getAutoUpdateAgentRollout *getHandler[*autoupdate.AutoUpdateAgentRollout] + createAutoUpdateAgentRollout *createUpdateHandler[*autoupdate.AutoUpdateAgentRollout] + updateAutoUpdateAgentRollout *createUpdateHandler[*autoupdate.AutoUpdateAgentRollout] + deleteAutoUpdateAgentRollout *deleteHandler + getClusterMaintenanceConfig *legacyGetHandler[*types.ClusterMaintenanceConfigV1] +} + +func (m mockClient) GetAutoUpdateConfig(ctx context.Context) (*autoupdate.AutoUpdateConfig, error) { + return m.getAutoUpdateConfig.handle(ctx) +} + +func (m mockClient) GetAutoUpdateVersion(ctx context.Context) (*autoupdate.AutoUpdateVersion, error) { + return m.getAutoUpdateVersion.handle(ctx) +} + +func (m mockClient) GetAutoUpdateAgentRollout(ctx context.Context) (*autoupdate.AutoUpdateAgentRollout, error) { + return m.getAutoUpdateAgentRollout.handle(ctx) +} + +func (m mockClient) CreateAutoUpdateAgentRollout(ctx context.Context, rollout *autoupdate.AutoUpdateAgentRollout) (*autoupdate.AutoUpdateAgentRollout, error) { + return m.createAutoUpdateAgentRollout.handle(ctx, rollout) +} + +func (m mockClient) UpdateAutoUpdateAgentRollout(ctx context.Context, rollout *autoupdate.AutoUpdateAgentRollout) (*autoupdate.AutoUpdateAgentRollout, error) { + return m.updateAutoUpdateAgentRollout.handle(ctx, rollout) +} + +func (m mockClient) DeleteAutoUpdateAgentRollout(ctx context.Context) error { + return m.deleteAutoUpdateAgentRollout.handle(ctx) +} + +func (m mockClient) GetClusterMaintenanceConfig(ctx context.Context) (types.ClusterMaintenanceConfig, error) { + return m.getClusterMaintenanceConfig.handle(ctx) +} + +func (m mockClient) checkIfEmpty(t *testing.T) { + require.True(t, m.getAutoUpdateConfig.isEmpty(), "Get autoupdate_config mock not empty") + require.True(t, m.getAutoUpdateVersion.isEmpty(), "Get autoupdate_version mock not empty") + require.True(t, m.getAutoUpdateAgentRollout.isEmpty(), "Get autoupdate_agent_rollout mock not empty") + require.True(t, m.createAutoUpdateAgentRollout.isEmpty(), "Create autoupdate_agent_rollout mock not empty") + require.True(t, m.updateAutoUpdateAgentRollout.isEmpty(), "Update autoupdate_agent_rollout mock not empty") + require.True(t, m.deleteAutoUpdateAgentRollout.isEmpty(), "Delete autoupdate_agent_rollout mock not empty") + require.True(t, m.getClusterMaintenanceConfig.isEmpty(), "Get cluster_maintenance config mock not empty") +} + +func newMockClient(t *testing.T, stubs mockClientStubs) *mockClient { + // Fail early if there's a mismatch + require.Equal(t, len(stubs.createRolloutAnswers), len(stubs.createRolloutExpects), "invalid stubs, create validations and answers slices are not the same length") + require.Equal(t, len(stubs.updateRolloutAnswers), len(stubs.updateRolloutExpects), "invalid stubs, update validations and answers slices are not the same length") + + return &mockClient{ + getAutoUpdateConfig: &getHandler[*autoupdate.AutoUpdateConfig]{t, stubs.configAnswers}, + getAutoUpdateVersion: &getHandler[*autoupdate.AutoUpdateVersion]{t, stubs.versionAnswers}, + getAutoUpdateAgentRollout: &getHandler[*autoupdate.AutoUpdateAgentRollout]{t, stubs.rolloutAnswers}, + createAutoUpdateAgentRollout: &createUpdateHandler[*autoupdate.AutoUpdateAgentRollout]{t, stubs.createRolloutExpects, stubs.createRolloutAnswers}, + updateAutoUpdateAgentRollout: &createUpdateHandler[*autoupdate.AutoUpdateAgentRollout]{t, stubs.updateRolloutExpects, stubs.updateRolloutAnswers}, + deleteAutoUpdateAgentRollout: &deleteHandler{t, stubs.deleteRolloutAnswers}, + getClusterMaintenanceConfig: &legacyGetHandler[*types.ClusterMaintenanceConfigV1]{t, stubs.cmcAnswers}, + } +} + +type mockClientStubs struct { + configAnswers []callAnswer[*autoupdate.AutoUpdateConfig] + versionAnswers []callAnswer[*autoupdate.AutoUpdateVersion] + rolloutAnswers []callAnswer[*autoupdate.AutoUpdateAgentRollout] + createRolloutAnswers []callAnswer[*autoupdate.AutoUpdateAgentRollout] + createRolloutExpects []require.ValueAssertionFunc + updateRolloutAnswers []callAnswer[*autoupdate.AutoUpdateAgentRollout] + updateRolloutExpects []require.ValueAssertionFunc + deleteRolloutAnswers []error + cmcAnswers []callAnswer[*types.ClusterMaintenanceConfigV1] +} + +type callAnswer[T any] struct { + result T + err error +} + +// getHandler is used in a mock client to answer get resource requests during tests. +// It takes a list of answers and errors and will return them when invoked. +// If there are no stubs left it fails the test. +type getHandler[T proto.Message] struct { + t *testing.T + answers []callAnswer[T] +} + +func (h *getHandler[T]) handle(_ context.Context) (T, error) { + if len(h.answers) == 0 { + require.Fail(h.t, "no answers left") + } + + entry := h.answers[0] + h.answers = h.answers[1:] + + // We need to deep copy because the reconciler might do updates in place. + // We don't want the original resource to be edited as this would mess with other tests. + return proto.Clone(entry.result).(T), entry.err +} + +// isEmpty returns true only if all stubs were consumed +func (h *getHandler[T]) isEmpty() bool { + return len(h.answers) == 0 +} + +// legacyGetHandler is a getHandler for legacy teleport types (gogo proto-based) +// A first iteration was trying to be smart and reuse the getHandler logic +// by converting fixtures before to protoadapt.MessageV2, and converting back to +// protoadapt.MessageV1 before returning. The resulting code was hard to read and +// duplicating the logic seems more maintainable. +type legacyGetHandler[T protoadapt.MessageV1] struct { + t *testing.T + answers []callAnswer[T] +} + +func (h *legacyGetHandler[T]) handle(_ context.Context) (T, error) { + if len(h.answers) == 0 { + require.Fail(h.t, "no answers left") + } + + entry := h.answers[0] + h.answers = h.answers[1:] + + // We need to deep copy because the reconciler might do updates in place. + // We don't want the original resource to be edited as this would mess with other tests. + result := apiutils.CloneProtoMsg(entry.result) + return result, entry.err +} + +// isEmpty returns true only if all stubs were consumed +func (h *legacyGetHandler[T]) isEmpty() bool { + return len(h.answers) == 0 +} + +// createUpdateHandler is used in a mock client to answer create or update resource requests during tests (any request whose arity is 2). +// It first validates the input using the provided validation function, then it returns the predefined answer and error. +// If there are no stubs left it fails the test. +type createUpdateHandler[T proto.Message] struct { + t *testing.T + expect []require.ValueAssertionFunc + answers []callAnswer[T] +} + +func (h *createUpdateHandler[T]) handle(_ context.Context, object T) (T, error) { + if len(h.expect) == 0 { + require.Fail(h.t, "not expecting more calls") + } + h.expect[0](h.t, object) + h.expect = h.expect[1:] + + if len(h.answers) == 0 { + require.Fail(h.t, "no answers left") + } + + entry := h.answers[0] + h.answers = h.answers[1:] + + // We need to deep copy because the reconciler might do updates in place. + // We don't want the original resource to be edited as this would mess with other tests. + return proto.Clone(entry.result).(T), entry.err +} + +// isEmpty returns true only if all stubs were consumed +func (h *createUpdateHandler[T]) isEmpty() bool { + return len(h.answers) == 0 && len(h.expect) == 0 +} + +// deleteHandler is used in a mock client to answer delete resource requests during tests. +// It takes a list of errors and returns them when invoked. +// If there are no stubs left it fails the test. +type deleteHandler struct { + t *testing.T + answers []error +} + +func (h *deleteHandler) handle(_ context.Context) error { + if len(h.answers) == 0 { + require.Fail(h.t, "no answers left") + } + + entry := h.answers[0] + h.answers = h.answers[1:] + + return entry +} + +// isEmpty returns true only if all stubs were consumed +func (h *deleteHandler) isEmpty() bool { + return len(h.answers) == 0 +} diff --git a/lib/autoupdate/rollout/controller.go b/lib/autoupdate/rollout/controller.go new file mode 100644 index 0000000000000..2b483072736b5 --- /dev/null +++ b/lib/autoupdate/rollout/controller.go @@ -0,0 +1,155 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package rollout + +import ( + "context" + "time" + + "github.com/gravitational/trace" + "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + + "github.com/gravitational/teleport" + "github.com/gravitational/teleport/api/utils/retryutils" + "github.com/gravitational/teleport/lib/utils/interval" +) + +const ( + defaultReconcilerPeriod = time.Minute +) + +// Controller wakes up every minute to reconcile the autoupdate_agent_rollout resource. +// See the reconciler godoc for more details about the reconciliation process. +// We currently wake up every minute, in the future we might decide to also watch for events +// (from autoupdate_config and autoupdate_version changefeed) to react faster. +type Controller struct { + // TODO(hugoShaka) add prometheus metrics describing the reconciliation status + reconciler reconciler + clock clockwork.Clock + log *logrus.Entry + period time.Duration + metrics *metrics +} + +// NewController creates a new Controller for the autoupdate_agent_rollout kind. +// The period can be specified to control the sync frequency. This is mainly +// used to speed up tests or for demo purposes. When empty, the controller picks +// a sane default value. +func NewController(client Client, log *logrus.Entry, clock clockwork.Clock, period time.Duration, reg prometheus.Registerer) (*Controller, error) { + if client == nil { + return nil, trace.BadParameter("missing client") + } + if log == nil { + return nil, trace.BadParameter("missing log") + } + if clock == nil { + return nil, trace.BadParameter("missing clock") + } + if reg == nil { + return nil, trace.BadParameter("missing prometheus.Registerer") + } + + if period <= 0 { + period = defaultReconcilerPeriod + } + + log = log.WithField(teleport.ComponentLabel, teleport.ComponentRolloutController) + + haltOnError, err := newHaltOnErrorStrategy(log) + if err != nil { + return nil, trace.Wrap(err, "failed to initialize halt-on-error strategy") + } + timeBased, err := newTimeBasedStrategy(log) + if err != nil { + return nil, trace.Wrap(err, "failed to initialize time-based strategy") + } + + m, err := newMetrics(reg) + if err != nil { + return nil, trace.Wrap(err, "failed to initialize metrics") + } + + return &Controller{ + metrics: m, + clock: clock, + log: log, + reconciler: reconciler{ + clt: client, + log: log, + clock: clock, + metrics: m, + rolloutStrategies: []rolloutStrategy{ + timeBased, + haltOnError, + }, + }, + period: period, + }, nil +} + +// Run the autoupdate_agent_rollout controller. This function returns only when its context is canceled. +func (c *Controller) Run(ctx context.Context) error { + config := interval.Config{ + Duration: c.period, + FirstDuration: c.period, + Jitter: retryutils.NewSeventhJitter(), + Clock: c.clock, + } + ticker := interval.New(config) + defer ticker.Stop() + + c.log.WithField("period", c.period).Info("Starting autoupdate_agent_rollout controller") + for { + select { + case <-ctx.Done(): + c.log.WithField("reason", ctx.Err()).Info("Stopping autoupdate_agent_rollout controller") + return ctx.Err() + case <-ticker.Next(): + c.log.Debug("Reconciling autoupdate_agent_rollout") + if err := c.tryAndCatch(ctx); err != nil { + c.log.WithError(err).Error("Failed to reconcile autoudpate_agent_controller") + } + } + } +} + +// tryAndCatch tries to run the controller reconciliation logic and recovers from potential panic by converting them +// into errors. This ensures that a critical bug in the reconciler cannot bring down the whole Teleport cluster. +func (c *Controller) tryAndCatch(ctx context.Context) (err error) { + startTime := c.clock.Now() + // If something terribly bad happens during the reconciliation, we recover and return an error + defer func() { + if r := recover(); r != nil { + c.log.Errorf("Recovered from panic in the autoupdate_agent_rollout controller: %#v", r) + err = trace.NewAggregate(err, trace.Errorf("Panic recovered during reconciliation: %v", r)) + c.metrics.observeReconciliation(metricsReconciliationResultLabelValuePanic, c.clock.Now().Sub(startTime)) + } + }() + + err = trace.Wrap(c.reconciler.reconcile(ctx)) + endTime := c.clock.Now() + result := metricsReconciliationResultLabelValueSuccess + if err != nil { + result = metricsReconciliationResultLabelValueFail + } + c.metrics.observeReconciliation(result, endTime.Sub(startTime)) + return +} diff --git a/lib/autoupdate/rollout/metrics.go b/lib/autoupdate/rollout/metrics.go new file mode 100644 index 0000000000000..f3f6740299236 --- /dev/null +++ b/lib/autoupdate/rollout/metrics.go @@ -0,0 +1,361 @@ +/* + * Teleport + * Copyright (C) 2025 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package rollout + +import ( + "fmt" + "strconv" + "strings" + "sync" + "time" + + "github.com/gravitational/trace" + "github.com/prometheus/client_golang/prometheus" + "golang.org/x/exp/constraints" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" + + "github.com/gravitational/teleport" + autoupdatepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + "github.com/gravitational/teleport/api/types/autoupdate" +) + +const ( + metricsSubsystem = "agent_autoupdates" + metricVersionLabelRetention = 24 * time.Hour +) + +type metrics struct { + // lock protects previousVersions and groupCount. + // This should only be acquired by setVersionMetric. + lock sync.Mutex + + // previousVersions is a list of the version we exported metrics for. + // We track those to zero every old version if metrics labels contain the version. + previousVersions map[string]time.Time + groupCount int + + // controller metrics + reconciliations *prometheus.CounterVec + reconciliationDuration *prometheus.HistogramVec + reconciliationTries *prometheus.CounterVec + reconciliationTryDuration *prometheus.HistogramVec + + // resource spec metrics + versionPresent prometheus.Gauge + versionStart *prometheus.GaugeVec + versionTarget *prometheus.GaugeVec + versionMode prometheus.Gauge + + configPresent prometheus.Gauge + configMode prometheus.Gauge + + rolloutPresent prometheus.Gauge + rolloutStart *prometheus.GaugeVec + rolloutTarget *prometheus.GaugeVec + rolloutMode prometheus.Gauge + rolloutStrategy *prometheus.GaugeVec + + // rollout status metrics + rolloutTimeOverride prometheus.Gauge + rolloutState prometheus.Gauge + rolloutGroupState *prometheus.GaugeVec +} + +const ( + metricsReconciliationResultLabelName = "result" + metricsReconciliationResultLabelValueFail = "fail" + metricsReconciliationResultLabelValuePanic = "panic" + metricsReconciliationResultLabelValueRetry = "retry" + metricsReconciliationResultLabelValueSuccess = "success" + + metricsGroupNumberLabelName = "group_number" + metricsVersionLabelName = "version" + + metricsStrategyLabelName = "strategy" +) + +func newMetrics(reg prometheus.Registerer) (*metrics, error) { + m := metrics{ + previousVersions: make(map[string]time.Time), + reconciliations: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "reconciliations_total", + Help: "Count the rollout reconciliations triggered by the controller, and their result (success, failure, panic). One reconciliation might imply several tries in case of conflict.", + }, []string{metricsReconciliationResultLabelName}), + reconciliationDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "reconciliation_duration_seconds", + Help: "Time spent reconciling the autoupdate_agent_rollout resource. One reconciliation might imply several tries in case of conflict.", + }, []string{metricsReconciliationResultLabelName}), + reconciliationTries: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "reconciliation_tries_total", + Help: "Count the rollout reconciliations tried by the controller, and their result (success, failure, conflict).", + }, []string{metricsReconciliationResultLabelName}), + reconciliationTryDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "reconciliation_try_duration_seconds", + Help: "Time spent trying to reconcile the autoupdate_agent_rollout resource.", + }, []string{metricsReconciliationResultLabelName}), + + versionPresent: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "version_present", + Help: "Boolean describing if an autoupdate_version resource exists in Teleport and its 'spec.agents' field is not nil.", + }), + versionTarget: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "version_target", + Help: "Metric describing the agent target version from the autoupdate_version resource.", + }, []string{metricsVersionLabelName}), + versionStart: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "version_start", + Help: "Metric describing the agent start version from the autoupdate_version resource.", + }, []string{"version"}), + versionMode: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "version_mode", + Help: fmt.Sprintf("Metric describing the agent update mode from the autoupdate_version resource. %s", valuesHelpString(codeToAgentMode)), + }), + + configPresent: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "config_present", + Help: "Boolean describing if an autoupdate_config resource exists in Teleport and its 'spec.agents' field is not nil.", + }), + configMode: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "config_mode", + Help: fmt.Sprintf("Metric describing the agent update mode from the autoupdate_agent_config resource. %s", valuesHelpString(codeToAgentMode)), + }), + + rolloutPresent: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "rollout_present", + Help: "Boolean describing if an autoupdate_agent_rollout resource exists in Teleport.", + }), + rolloutTarget: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "rollout_target", + Help: "Metric describing the agent target version from the autoupdate_gent_rollout resource.", + }, []string{metricsVersionLabelName}), + rolloutStart: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "rollout_start", + Help: "Metric describing the agent start version from the autoupdate_agent_rollout resource.", + }, []string{metricsVersionLabelName}), + rolloutMode: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "rollout_mode", + Help: fmt.Sprintf("Metric describing the agent update mode from the autoupdate_agent_rollout resource. %s", valuesHelpString(codeToAgentMode)), + }), + rolloutStrategy: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "rollout_strategy", + Help: "Metric describing the strategy of the autoupdate_agent_rollout resource.", + }, []string{metricsStrategyLabelName}), + rolloutTimeOverride: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "rollout_time_override_timestamp_seconds", + Help: "Describes the autoupdate_agent_rollout time override if set in (seconds since epoch). Zero means no time override.", + }), + rolloutState: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "rollout_state", + Help: fmt.Sprintf("Describes the autoupdate_agent_rollout state. %s", valuesHelpString(autoupdatepb.AutoUpdateAgentRolloutState_name)), + }), + rolloutGroupState: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: teleport.MetricNamespace, + Subsystem: metricsSubsystem, + Name: "rollout_group_state", + Help: fmt.Sprintf("Describes the autoupdate_agent_rollout state for each group. Groups are identified by their position in the schedule. %s", valuesHelpString(autoupdatepb.AutoUpdateAgentGroupState_name)), + }, []string{metricsGroupNumberLabelName}), + } + + errs := trace.NewAggregate( + reg.Register(m.reconciliations), + reg.Register(m.reconciliationDuration), + reg.Register(m.reconciliationTries), + reg.Register(m.reconciliationTryDuration), + + reg.Register(m.versionPresent), + reg.Register(m.versionTarget), + reg.Register(m.versionStart), + reg.Register(m.versionMode), + reg.Register(m.configPresent), + reg.Register(m.configMode), + reg.Register(m.rolloutPresent), + reg.Register(m.rolloutTarget), + reg.Register(m.rolloutStart), + reg.Register(m.rolloutMode), + reg.Register(m.rolloutStrategy), + + reg.Register(m.rolloutTimeOverride), + reg.Register(m.rolloutState), + reg.Register(m.rolloutGroupState), + ) + + return &m, errs +} + +func valuesHelpString[K constraints.Integer](possibleValues map[K]string) string { + sb := strings.Builder{} + sb.WriteString("Possible values are") + + // maps are nor ordered, so we must sort keys to consistently generate the help message. + keys := maps.Keys(possibleValues) + slices.Sort(keys) + for _, k := range keys { + sb.WriteString(fmt.Sprintf(" %d:%s", k, possibleValues[k])) + } + + sb.WriteRune('.') + return sb.String() +} + +func (m *metrics) setVersionMetric(version string, metric *prometheus.GaugeVec, now time.Time) { + m.lock.Lock() + defer m.lock.Unlock() + + // for every version we've seen + for v, ts := range m.previousVersions { + labels := prometheus.Labels{metricsVersionLabelName: v} + // if the version is too old, we forget about it to limit cardinality + if now.After(ts.Add(metricVersionLabelRetention)) { + metric.Delete(labels) + delete(m.previousVersions, v) + } else { + // Else we just mark the version as not set anymore + metric.With(labels).Set(0) + } + } + // We set the new version + metric.With(prometheus.Labels{metricsVersionLabelName: version}).Set(1) + m.previousVersions[version] = now +} + +func (m *metrics) observeReconciliation(result string, duration time.Duration) { + m.reconciliations.With(prometheus.Labels{metricsReconciliationResultLabelName: result}).Inc() + m.reconciliationDuration.With(prometheus.Labels{metricsReconciliationResultLabelName: result}).Observe(duration.Seconds()) +} + +func (m *metrics) observeReconciliationTry(result string, duration time.Duration) { + m.reconciliationTries.With(prometheus.Labels{metricsReconciliationResultLabelName: result}).Inc() + m.reconciliationTryDuration.With(prometheus.Labels{metricsReconciliationResultLabelName: result}).Observe(duration.Seconds()) +} + +func (m *metrics) observeConfig(config *autoupdatepb.AutoUpdateConfig) { + if config.GetSpec().GetAgents() == nil { + m.configPresent.Set(0) + m.configMode.Set(float64(agentModeCode[defaultConfigMode])) + return + } + m.configPresent.Set(1) + m.configMode.Set(float64(agentModeCode[config.GetSpec().GetAgents().GetMode()])) +} + +func (m *metrics) observeVersion(version *autoupdatepb.AutoUpdateVersion, now time.Time) { + if version.GetSpec().GetAgents() == nil { + m.versionPresent.Set(0) + m.versionMode.Set(float64(agentModeCode[defaultConfigMode])) + return + } + m.versionPresent.Set(1) + m.versionMode.Set(float64(agentModeCode[version.GetSpec().GetAgents().GetMode()])) + m.setVersionMetric(version.GetSpec().GetAgents().GetStartVersion(), m.versionStart, now) + m.setVersionMetric(version.GetSpec().GetAgents().GetTargetVersion(), m.versionTarget, now) +} + +func (m *metrics) setGroupStates(groups []*autoupdatepb.AutoUpdateAgentRolloutStatusGroup) { + m.lock.Lock() + defer m.lock.Unlock() + + // Set the state for the groups specified in the rollout. + for i, group := range groups { + labels := prometheus.Labels{metricsGroupNumberLabelName: strconv.Itoa(i)} + m.rolloutGroupState.With(labels).Set(float64(group.State)) + } + + // If we have as many or more groups than before, no cleanup to do. + if len(groups) >= m.groupCount { + m.groupCount = len(groups) + return + } + + // If we have less groups than before, we must unset the metrics for higher group numbers. + for i := len(groups); i < m.groupCount; i++ { + labels := prometheus.Labels{metricsGroupNumberLabelName: strconv.Itoa(i)} + m.rolloutGroupState.With(labels).Set(float64(0)) + } + m.groupCount = len(groups) +} + +func (m *metrics) observeRollout(rollout *autoupdatepb.AutoUpdateAgentRollout, now time.Time) { + if rollout.GetSpec() == nil { + m.rolloutPresent.Set(0) + m.rolloutMode.Set(0) + } else { + m.rolloutPresent.Set(1) + m.rolloutMode.Set(float64(agentModeCode[rollout.GetSpec().GetAutoupdateMode()])) + m.setVersionMetric(rollout.GetSpec().GetStartVersion(), m.rolloutStart, now) + m.setVersionMetric(rollout.GetSpec().GetTargetVersion(), m.rolloutTarget, now) + } + + m.setStrategyMetric(rollout.GetSpec().GetStrategy(), m.rolloutStrategy) + + if to := rollout.GetStatus().GetTimeOverride().AsTime(); !(to.IsZero() || to.Unix() == 0) { + m.rolloutTimeOverride.Set(float64(to.Second())) + } else { + m.rolloutTimeOverride.Set(0) + } + + m.rolloutState.Set(float64(rollout.GetStatus().GetState())) + m.setGroupStates(rollout.GetStatus().GetGroups()) +} + +var strategies = []string{autoupdate.AgentsStrategyHaltOnError, autoupdate.AgentsStrategyTimeBased} + +func (m *metrics) setStrategyMetric(strategy string, metric *prometheus.GaugeVec) { + for _, s := range strategies { + if s == strategy { + metric.With(prometheus.Labels{metricsStrategyLabelName: s}).Set(1) + } else { + metric.With(prometheus.Labels{metricsStrategyLabelName: s}).Set(0) + } + } +} diff --git a/lib/autoupdate/rollout/metrics_test.go b/lib/autoupdate/rollout/metrics_test.go new file mode 100644 index 0000000000000..b0315d8455109 --- /dev/null +++ b/lib/autoupdate/rollout/metrics_test.go @@ -0,0 +1,293 @@ +/* + * Teleport + * Copyright (C) 2025 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package rollout + +import ( + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + + autoupdatepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" +) + +func newMetricsForTest(t *testing.T) *metrics { + reg := prometheus.NewRegistry() + m, err := newMetrics(reg) + require.NoError(t, err) + return m +} + +func Test_setVersionMetric(t *testing.T) { + now := clockwork.NewFakeClock().Now() + aMinuteAgo := now.Add(-time.Minute) + aWeekAgo := now.Add(-time.Hour * 24 * 7) + testVersion := "1.2.3-alpha.1" + previousVersion := "1.2.1" + testMetricLabels := []string{metricsVersionLabelName} + tests := []struct { + name string + previousVersions map[string]time.Time + previousMetrics map[string]float64 + expectedVersions map[string]time.Time + expectedMetrics map[string]float64 + }{ + { + name: "no versions", + previousVersions: map[string]time.Time{}, + previousMetrics: map[string]float64{}, + expectedVersions: map[string]time.Time{ + testVersion: now, + }, + expectedMetrics: map[string]float64{ + testVersion: 1, + }, + }, + { + name: "same version, not expired", + previousVersions: map[string]time.Time{ + testVersion: aMinuteAgo, + }, + previousMetrics: map[string]float64{ + testVersion: 1, + }, + expectedVersions: map[string]time.Time{ + testVersion: now, + }, + expectedMetrics: map[string]float64{ + testVersion: 1, + }, + }, + { + name: "same version, expired", + previousVersions: map[string]time.Time{ + testVersion: aWeekAgo, + }, + previousMetrics: map[string]float64{ + testVersion: 1, + }, + expectedVersions: map[string]time.Time{ + testVersion: now, + }, + expectedMetrics: map[string]float64{ + testVersion: 1, + }, + }, + { + name: "old non-expired versions", + previousVersions: map[string]time.Time{ + previousVersion: aMinuteAgo, + }, + previousMetrics: map[string]float64{ + previousVersion: 1, + }, + expectedVersions: map[string]time.Time{ + previousVersion: aMinuteAgo, + testVersion: now, + }, + expectedMetrics: map[string]float64{ + previousVersion: 0, + testVersion: 1, + }, + }, + { + name: "old expired versions", + previousVersions: map[string]time.Time{ + previousVersion: aWeekAgo, + }, + previousMetrics: map[string]float64{ + previousVersion: 1, + }, + expectedVersions: map[string]time.Time{ + testVersion: now, + }, + expectedMetrics: map[string]float64{ + testVersion: 1, + }, + }, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + // Test setup: create metrics and load previous metrics. + m := metrics{ + previousVersions: test.previousVersions, + } + + testGauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{}, testMetricLabels) + for k, v := range test.previousMetrics { + testGauge.With(prometheus.Labels{testMetricLabels[0]: k}).Set(v) + } + + // Test execution: set the version metric. + m.setVersionMetric(testVersion, testGauge, now) + + // Test validation: collect the metrics and check that the state match what we expect. + require.Equal(t, test.expectedVersions, m.previousVersions) + metricsChan := make(chan prometheus.Metric, 100) + testGauge.Collect(metricsChan) + close(metricsChan) + metricsResult := collectMetricsByLabel(t, metricsChan, testMetricLabels[0]) + require.Equal(t, test.expectedMetrics, metricsResult) + }) + } +} + +func Test_setGroupStates(t *testing.T) { + testMetricLabels := []string{metricsGroupNumberLabelName} + testGroups := []*autoupdatepb.AutoUpdateAgentRolloutStatusGroup{ + {State: autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE}, + {State: autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE}, + {State: autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED}, + } + tests := []struct { + name string + previousGroupCount int + previousMetrics map[string]float64 + expectedGroupCount int + expectedMetrics map[string]float64 + }{ + { + name: "no groups", + previousGroupCount: 0, + previousMetrics: map[string]float64{}, + expectedGroupCount: len(testGroups), + expectedMetrics: map[string]float64{ + "0": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE), + "1": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE), + "2": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED), + }, + }, + { + name: "same groups, same states", + previousGroupCount: len(testGroups), + previousMetrics: map[string]float64{ + "0": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE), + "1": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE), + "2": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED), + }, + expectedGroupCount: len(testGroups), + expectedMetrics: map[string]float64{ + "0": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE), + "1": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE), + "2": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED), + }, + }, + { + name: "same groups, different states", + previousGroupCount: len(testGroups), + previousMetrics: map[string]float64{ + "0": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE), + "1": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED), + "2": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED), + }, + expectedGroupCount: len(testGroups), + expectedMetrics: map[string]float64{ + "0": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE), + "1": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE), + "2": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED), + }, + }, + { + name: "less groups", + previousGroupCount: 1, + previousMetrics: map[string]float64{ + "0": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE), + }, + expectedGroupCount: len(testGroups), + expectedMetrics: map[string]float64{ + "0": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE), + "1": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE), + "2": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED), + }, + }, + { + name: "more groups", + previousGroupCount: 5, + previousMetrics: map[string]float64{ + "0": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE), + "1": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED), + "2": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED), + "3": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED), + "4": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED), + }, + expectedGroupCount: len(testGroups), + expectedMetrics: map[string]float64{ + "0": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE), + "1": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE), + "2": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED), + "3": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSPECIFIED), + "4": float64(autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSPECIFIED), + }, + }, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + testGauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{}, testMetricLabels) + for k, v := range test.previousMetrics { + testGauge.With(prometheus.Labels{testMetricLabels[0]: k}).Set(v) + } + + // Test setup: create metrics and load previous metrics. + m := metrics{ + groupCount: test.previousGroupCount, + rolloutGroupState: testGauge, + } + + // Test execution: set the version metric. + m.setGroupStates(testGroups) + + // Test validation: collect the metrics and check that the state match what we expect. + require.Equal(t, test.expectedGroupCount, m.groupCount) + metricsChan := make(chan prometheus.Metric, 100) + m.rolloutGroupState.Collect(metricsChan) + close(metricsChan) + metricsResult := collectMetricsByLabel(t, metricsChan, testMetricLabels[0]) + require.Equal(t, test.expectedMetrics, metricsResult) + + }) + } +} + +func collectMetricsByLabel(t *testing.T, ch <-chan prometheus.Metric, labelName string) map[string]float64 { + t.Helper() + result := make(map[string]float64) + + var protoMetric dto.Metric + for { + m, ok := <-ch + if !ok { + return result + } + require.NoError(t, m.Write(&protoMetric)) + ll := protoMetric.GetLabel() + require.Len(t, ll, 1) + require.Equal(t, labelName, ll[0].GetName()) + gg := protoMetric.GetGauge() + require.NotNil(t, gg) + result[ll[0].GetValue()] = gg.GetValue() + } +} diff --git a/lib/autoupdate/rollout/reconciler.go b/lib/autoupdate/rollout/reconciler.go new file mode 100644 index 0000000000000..5fa1cb818b7d0 --- /dev/null +++ b/lib/autoupdate/rollout/reconciler.go @@ -0,0 +1,439 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package rollout + +import ( + "context" + "sync" + "time" + + "github.com/gravitational/trace" + "github.com/jonboulle/clockwork" + "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + "github.com/gravitational/teleport/api/types" + update "github.com/gravitational/teleport/api/types/autoupdate" + "github.com/gravitational/teleport/api/utils" +) + +const ( + reconciliationTimeout = 30 * time.Second + defaultConfigMode = update.AgentsUpdateModeEnabled + defaultStrategy = update.AgentsStrategyHaltOnError + maxConflictRetry = 3 + + defaultGroupName = "default" + defaultCMCGroupName = defaultGroupName + "-cmc" + defaultStartHour = 12 +) + +var ( + // defaultUpdateDays is the default list of days when groups can be updated. + defaultUpdateDays = []string{"Mon", "Tue", "Wed", "Thu"} +) + +// reconciler reconciles the AutoUpdateAgentRollout singleton based on the content of the AutoUpdateVersion and +// AutoUpdateConfig singletons. This reconciler is not based on the services.GenericReconciler because: +// - we reconcile 2 resources with one +// - both input and output are singletons, we don't need the multi resource logic nor stream/paginated APIs +type reconciler struct { + clt Client + log *logrus.Entry + clock clockwork.Clock + metrics *metrics + + rolloutStrategies []rolloutStrategy + + // mutex ensures we only run one reconciliation at a time + mutex sync.Mutex +} + +// reconcile the AutoUpdateAgentRollout singleton. The reconciliation can fail because of a conflict (multiple auths +// are racing), in this case we retry the reconciliation immediately. +func (r *reconciler) reconcile(ctx context.Context) error { + r.mutex.Lock() + defer r.mutex.Unlock() + + ctx, cancel := context.WithTimeout(ctx, reconciliationTimeout) + defer cancel() + + var startTime time.Time + tries := 0 + var err error + for tries < maxConflictRetry { + tries++ + select { + case <-ctx.Done(): + return ctx.Err() + default: + startTime = r.clock.Now() + err = r.tryReconcile(ctx) + duration := r.clock.Since(startTime) + switch { + case err == nil: + r.metrics.observeReconciliationTry(metricsReconciliationResultLabelValueSuccess, duration) + return nil + case trace.IsCompareFailed(err), trace.IsNotFound(err): + // The resource changed since we last saw it + // We must have raced against another auth + // Let's retry the reconciliation + r.log.WithError(err).Debug("retrying reconciliation") + r.metrics.observeReconciliationTry(metricsReconciliationResultLabelValueRetry, duration) + default: + // error is non-nil and non-retryable + r.metrics.observeReconciliationTry(metricsReconciliationResultLabelValueFail, duration) + return trace.Wrap(err, "failed to reconcile rollout") + } + } + } + return trace.CompareFailed("compare failed, tried %d times, last error: %s", tries, err) +} + +// tryReconcile tries to reconcile the AutoUpdateAgentRollout singleton. +// This function should be nilpotent if the AutoUpdateAgentRollout is already up-to-date. +// The creation/update/deletion can fail with a trace.CompareFailedError or trace.NotFoundError +// if the resource change while we were computing it. +// The caller must handle those error and retry the reconciliation. +func (r *reconciler) tryReconcile(ctx context.Context) (err error) { + // get autoupdate_config + var config *autoupdate.AutoUpdateConfig + if c, err := r.clt.GetAutoUpdateConfig(ctx); err == nil { + config = c + } else if !trace.IsNotFound(err) { + return trace.Wrap(err, "getting autoupdate_config") + } + r.metrics.observeConfig(config) + + // get autoupdate_version + var version *autoupdate.AutoUpdateVersion + if v, err := r.clt.GetAutoUpdateVersion(ctx); err == nil { + version = v + } else if !trace.IsNotFound(err) { + return trace.Wrap(err, "getting autoupdate version") + } + r.metrics.observeVersion(version, r.clock.Now()) + + // get autoupdate_agent_rollout + rolloutExists := true + rollout, err := r.clt.GetAutoUpdateAgentRollout(ctx) + if err != nil && !trace.IsNotFound(err) { + return trace.Wrap(err, "getting autoupdate_agent_rollout") + } + if trace.IsNotFound(err) { + // rollout doesn't exist yet, we'll need to call Create instead of Update. + rolloutExists = false + } + + // We observe the current rollout. + r.metrics.observeRollout(rollout, r.clock.Now()) + // If the reconciliation succeeded, we observe the rollout again to reflect its new values. + defer func() { + if err != nil { + return + } + r.metrics.observeRollout(rollout, r.clock.Now()) + }() + + // if autoupdate_version does not exist or does not contain spec.agents, we should not configure a rollout + if version.GetSpec().GetAgents() == nil { + if !rolloutExists { + // the rollout doesn't exist, nothing to do + return nil + } + // the rollout exists, we must delete it. We also clear the rollout object for metrics purposes. + rollout = nil + return r.clt.DeleteAutoUpdateAgentRollout(ctx) + } + + // compute what the spec should look like + newSpec, err := r.buildRolloutSpec(config.GetSpec().GetAgents(), version.GetSpec().GetAgents()) + if err != nil { + return trace.Wrap(err, "mutating rollout") + } + newStatus, err := r.computeStatus(ctx, rollout, newSpec, config.GetSpec().GetAgents().GetSchedules()) + if err != nil { + return trace.Wrap(err, "computing rollout status") + } + + // We compute if something changed. + specChanged := !proto.Equal(rollout.GetSpec(), newSpec) + statusChanged := !proto.Equal(rollout.GetStatus(), newStatus) + rolloutChanged := specChanged || statusChanged + + // if nothing changed, no need to update the resource + if !rolloutChanged { + r.log.Debug("rollout unchanged") + return nil + } + + // if there are no existing rollout, we create a new one and set the status + if !rolloutExists { + r.log.Debug("creating rollout") + rollout, err = update.NewAutoUpdateAgentRollout(newSpec) + rollout.Status = newStatus + if err != nil { + return trace.Wrap(err, "validating new rollout") + } + rollout, err = r.clt.CreateAutoUpdateAgentRollout(ctx, rollout) + return trace.Wrap(err, "creating rollout") + } + + r.log.Debug("updating rollout") + // If there was a previous rollout, we update its spec and status and do an update. + // We don't create a new resource to keep the metadata containing the revision ID. + rollout.Spec = newSpec + rollout.Status = newStatus + err = update.ValidateAutoUpdateAgentRollout(rollout) + if err != nil { + return trace.Wrap(err, "validating mutated rollout") + } + rollout, err = r.clt.UpdateAutoUpdateAgentRollout(ctx, rollout) + return trace.Wrap(err, "updating rollout") +} + +func (r *reconciler) buildRolloutSpec(config *autoupdate.AutoUpdateConfigSpecAgents, version *autoupdate.AutoUpdateVersionSpecAgents) (*autoupdate.AutoUpdateAgentRolloutSpec, error) { + // reconcile mode + mode, err := getMode(config.GetMode(), version.GetMode()) + if err != nil { + return nil, trace.Wrap(err, "computing agent update mode") + } + + strategy := config.GetStrategy() + if strategy == "" { + strategy = defaultStrategy + } + + return &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: version.GetStartVersion(), + TargetVersion: version.GetTargetVersion(), + Schedule: version.GetSchedule(), + AutoupdateMode: mode, + Strategy: strategy, + MaintenanceWindowDuration: config.GetMaintenanceWindowDuration(), + }, nil + +} + +// agentModeCode maps agents mode to integers. +// When config and version modes don't match, the lowest integer takes precedence. +var ( + agentModeCode = map[string]int{ + update.AgentsUpdateModeDisabled: 1, + update.AgentsUpdateModeSuspended: 2, + update.AgentsUpdateModeEnabled: 3, + } + codeToAgentMode = map[int]string{ + 1: update.AgentsUpdateModeDisabled, + 2: update.AgentsUpdateModeSuspended, + 3: update.AgentsUpdateModeEnabled, + } +) + +// getMode merges the agent modes coming from the version and config resources into a single mode. +// "disabled" takes precedence over "suspended", which takes precedence over "enabled". +func getMode(configMode, versionMode string) (string, error) { + if configMode == "" { + configMode = defaultConfigMode + } + if versionMode == "" { + return "", trace.BadParameter("version mode empty") + } + + configCode, ok := agentModeCode[configMode] + if !ok { + return "", trace.BadParameter("unsupported agent config mode: %v", configMode) + } + versionCode, ok := agentModeCode[versionMode] + if !ok { + return "", trace.BadParameter("unsupported agent version mode: %v", versionMode) + } + + // The lowest code takes precedence + if configCode <= versionCode { + return codeToAgentMode[configCode], nil + } + return codeToAgentMode[versionCode], nil +} + +// computeStatus computes the new rollout status based on the existing rollout, +// new rollout spec, and autoupdate_config. existingRollout might be nil if this +// is a new rollout. +// Even if the returned new status might be derived from the existing rollout +// status, it is a new deep-cloned structure. +func (r *reconciler) computeStatus( + ctx context.Context, + existingRollout *autoupdate.AutoUpdateAgentRollout, + newSpec *autoupdate.AutoUpdateAgentRolloutSpec, + configSchedules *autoupdate.AgentAutoUpdateSchedules, +) (*autoupdate.AutoUpdateAgentRolloutStatus, error) { + + var status *autoupdate.AutoUpdateAgentRolloutStatus + + // First, we check if a major spec change happened and we should reset the rollout status + shouldResetRollout := existingRollout.GetSpec().GetStartVersion() != newSpec.GetStartVersion() || + existingRollout.GetSpec().GetTargetVersion() != newSpec.GetTargetVersion() || + existingRollout.GetSpec().GetSchedule() != newSpec.GetSchedule() || + existingRollout.GetSpec().GetStrategy() != newSpec.GetStrategy() + + // We create a new status if the rollout should be reset or the previous status was nil + if shouldResetRollout || existingRollout.GetStatus() == nil { + status = new(autoupdate.AutoUpdateAgentRolloutStatus) + // We set the start time if this is a new rollout + status.StartTime = timestamppb.New(r.clock.Now()) + } else { + status = utils.CloneProtoMsg(existingRollout.GetStatus()) + } + + // Then, we check if the selected schedule uses groups + switch newSpec.GetSchedule() { + case update.AgentsScheduleImmediate: + // There are no groups with the immediate schedule, we must clean them + status.Groups = nil + return status, nil + case update.AgentsScheduleRegular: + // Regular schedule has groups, we will compute them after + default: + return nil, trace.BadParameter("unsupported agent schedule type %q", newSpec.GetSchedule()) + } + + // capture the current time to put it in the status update timestamps and to + // compute the group state changes + now := r.clock.Now() + + // If timeOverride is set to a non-zero value (we have two potential zeros, go time's zero and timestamppb's zero) + // we use this instead of the clock's time. + if timeOverride := status.GetTimeOverride().AsTime(); !(timeOverride.IsZero() || timeOverride.Unix() == 0) { + r.log.WithFields(logrus.Fields{ + "time_override": timeOverride, + "real_time": now, + }).Debug("reconciling with synthetic time instead of real time") + now = timeOverride + } + + // If this is a new rollout or the rollout has been reset, we create groups from the config + groups := status.GetGroups() + var err error + if len(groups) == 0 { + groups, err = r.makeGroupsStatus(ctx, configSchedules, now) + if err != nil { + return nil, trace.Wrap(err, "creating groups status") + } + } + status.Groups = groups + + err = r.progressRollout(ctx, newSpec, status, now) + // Failing to progress the update is not a hard failure. + // We want to update the status even if something went wrong to surface the failed reconciliation and potential errors to the user. + if err != nil { + r.log.WithError(err).Error("Errors encountered during rollout progress. Some groups might not get updated properly.") + } + + status.State = computeRolloutState(groups) + return status, nil +} + +// progressRollout picks the right rollout strategy and updates groups to progress the rollout. +// groups are updated in place. +// If an error is returned, the groups should still be upserted, depending on the strategy, +// failing to update a group might not be fatal (other groups can still progress independently). +func (r *reconciler) progressRollout(ctx context.Context, spec *autoupdate.AutoUpdateAgentRolloutSpec, status *autoupdate.AutoUpdateAgentRolloutStatus, now time.Time) error { + for _, strategy := range r.rolloutStrategies { + if strategy.name() == spec.GetStrategy() { + return strategy.progressRollout(ctx, spec, status, now) + } + } + return trace.NotImplemented("rollout strategy %q not implemented", spec.GetStrategy()) +} + +// makeGroupStatus creates the autoupdate_agent_rollout.status.groups based on the autoupdate_config. +// This should be called if the status groups have not been initialized or must be reset. +func (r *reconciler) makeGroupsStatus(ctx context.Context, schedules *autoupdate.AgentAutoUpdateSchedules, now time.Time) ([]*autoupdate.AutoUpdateAgentRolloutStatusGroup, error) { + configGroups := schedules.GetRegular() + if len(configGroups) == 0 { + defaultGroup, err := r.defaultConfigGroup(ctx) + if err != nil { + return nil, trace.Wrap(err, "retrieving default group") + } + configGroups = []*autoupdate.AgentAutoUpdateGroup{defaultGroup} + } + + groups := make([]*autoupdate.AutoUpdateAgentRolloutStatusGroup, len(configGroups)) + for i, group := range configGroups { + groups[i] = &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: group.Name, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(now), + LastUpdateReason: updateReasonCreated, + ConfigDays: group.Days, + ConfigStartHour: group.StartHour, + ConfigWaitHours: group.WaitHours, + } + } + return groups, nil +} + +// defaultConfigGroup returns the default group in case of missing autoupdate_config resource. +// This is a function and not a variable because we will need to add more logic there in the future +// lookup maintenance information from RFD 109's cluster_maintenance_config. +func (r *reconciler) defaultConfigGroup(ctx context.Context) (*autoupdate.AgentAutoUpdateGroup, error) { + cmc, err := r.clt.GetClusterMaintenanceConfig(ctx) + if err != nil { + if trace.IsNotFound(err) { + // There's no CMC, we return the default group. + return defaultGroup(), nil + } + + // If we had an error, and it's not trace.ErrNotFound, we stop. + return nil, trace.Wrap(err, "retrieving the cluster maintenance config") + } + // We got a CMC, we generate the default from it. + upgradeWindow, ok := cmc.GetAgentUpgradeWindow() + + if !ok { + // The CMC is here but does not contain upgrade window. + return defaultGroup(), nil + } + + weekdays := upgradeWindow.Weekdays + // A CMC upgrade window not specifying weekdays should update every day. + if len(weekdays) == 0 { + weekdays = []string{types.Wildcard} + } + + return &autoupdate.AgentAutoUpdateGroup{ + Name: defaultCMCGroupName, + Days: weekdays, + StartHour: int32(upgradeWindow.UTCStartHour), + WaitHours: 0, + }, nil + +} + +func defaultGroup() *autoupdate.AgentAutoUpdateGroup { + return &autoupdate.AgentAutoUpdateGroup{ + Name: defaultGroupName, + Days: defaultUpdateDays, + StartHour: defaultStartHour, + WaitHours: 0, + } +} diff --git a/lib/autoupdate/rollout/reconciler_test.go b/lib/autoupdate/rollout/reconciler_test.go new file mode 100644 index 0000000000000..0f6f76e593e1b --- /dev/null +++ b/lib/autoupdate/rollout/reconciler_test.go @@ -0,0 +1,986 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package rollout + +import ( + "context" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "github.com/gravitational/trace" + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + "github.com/gravitational/teleport/api/types" + update "github.com/gravitational/teleport/api/types/autoupdate" + apiutils "github.com/gravitational/teleport/api/utils" + "github.com/gravitational/teleport/lib/utils" +) + +// rolloutEquals returns a require.ValueAssertionFunc that checks the rollout is identical. +// The comparison does not take into account the proto internal state. +func rolloutEquals(expected *autoupdate.AutoUpdateAgentRollout) require.ValueAssertionFunc { + return func(t require.TestingT, i interface{}, _ ...interface{}) { + require.IsType(t, &autoupdate.AutoUpdateAgentRollout{}, i, "resource should be an autoupdate_agent_rollout") + actual := i.(*autoupdate.AutoUpdateAgentRollout) + require.Empty(t, cmp.Diff(expected, actual, protocmp.Transform())) + } +} + +// cancelContext wraps a require.ValueAssertionFunc so that the given context is canceled before checking the assertion. +// This is used to test how the reconciler behaves when its context is canceled. +func cancelContext(assertionFunc require.ValueAssertionFunc, cancel func()) require.ValueAssertionFunc { + return func(t require.TestingT, i interface{}, i2 ...interface{}) { + cancel() + assertionFunc(t, i, i2...) + } +} + +// withRevisionID creates a deep copy of an agent rollout and sets the revisionID in its metadata. +// This is used to test the conditional update retry logic. +func withRevisionID(original *autoupdate.AutoUpdateAgentRollout, revision string) *autoupdate.AutoUpdateAgentRollout { + revisioned := apiutils.CloneProtoMsg(original) + revisioned.Metadata.Revision = revision + return revisioned +} + +func TestGetMode(t *testing.T) { + t.Parallel() + tests := []struct { + name string + configMode string + versionMode string + expected string + checkErr require.ErrorAssertionFunc + }{ + { + name: "config and version equal", + configMode: update.AgentsUpdateModeEnabled, + versionMode: update.AgentsUpdateModeEnabled, + expected: update.AgentsUpdateModeEnabled, + checkErr: require.NoError, + }, + { + name: "config suspends, version enables", + configMode: update.AgentsUpdateModeSuspended, + versionMode: update.AgentsUpdateModeEnabled, + expected: update.AgentsUpdateModeSuspended, + checkErr: require.NoError, + }, + { + name: "config enables, version suspends", + configMode: update.AgentsUpdateModeEnabled, + versionMode: update.AgentsUpdateModeSuspended, + expected: update.AgentsUpdateModeSuspended, + checkErr: require.NoError, + }, + { + name: "config suspends, version disables", + configMode: update.AgentsUpdateModeSuspended, + versionMode: update.AgentsUpdateModeDisabled, + expected: update.AgentsUpdateModeDisabled, + checkErr: require.NoError, + }, + { + name: "version enables, no config", + configMode: "", + versionMode: update.AgentsUpdateModeEnabled, + expected: update.AgentsUpdateModeEnabled, + checkErr: require.NoError, + }, + { + name: "config enables, no version", + configMode: update.AgentsUpdateModeEnabled, + versionMode: "", + expected: "", + checkErr: require.Error, + }, + { + name: "unknown mode", + configMode: "this in not a mode", + versionMode: update.AgentsUpdateModeEnabled, + expected: "", + checkErr: require.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := getMode(tt.configMode, tt.versionMode) + tt.checkErr(t, err) + require.Equal(t, tt.expected, result) + }) + } +} + +func TestTryReconcile(t *testing.T) { + t.Parallel() + log := utils.NewLoggerForTests().WithField("component", "reconciler") + ctx := context.Background() + clock := clockwork.NewFakeClock() + + // Test setup: creating fixtures + configOK, err := update.NewAutoUpdateConfig(&autoupdate.AutoUpdateConfigSpec{ + Tools: &autoupdate.AutoUpdateConfigSpecTools{ + Mode: update.ToolsUpdateModeEnabled, + }, + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: update.AgentsUpdateModeEnabled, + Strategy: update.AgentsStrategyHaltOnError, + }, + }) + require.NoError(t, err) + + configNoAgent, err := update.NewAutoUpdateConfig(&autoupdate.AutoUpdateConfigSpec{ + Tools: &autoupdate.AutoUpdateConfigSpecTools{ + Mode: update.ToolsUpdateModeEnabled, + }, + }) + require.NoError(t, err) + + versionOK, err := update.NewAutoUpdateVersion(&autoupdate.AutoUpdateVersionSpec{ + Tools: &autoupdate.AutoUpdateVersionSpecTools{ + TargetVersion: "1.2.3", + }, + Agents: &autoupdate.AutoUpdateVersionSpecAgents{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: update.AgentsScheduleImmediate, + Mode: update.AgentsUpdateModeEnabled, + }, + }) + require.NoError(t, err) + + versionNoAgent, err := update.NewAutoUpdateVersion(&autoupdate.AutoUpdateVersionSpec{ + Tools: &autoupdate.AutoUpdateVersionSpecTools{ + TargetVersion: "1.2.3", + }, + }) + require.NoError(t, err) + + upToDateRollout, err := update.NewAutoUpdateAgentRollout(&autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: update.AgentsScheduleImmediate, + AutoupdateMode: update.AgentsUpdateModeEnabled, + Strategy: update.AgentsStrategyHaltOnError, + }) + require.NoError(t, err) + upToDateRollout.Status = &autoupdate.AutoUpdateAgentRolloutStatus{StartTime: timestamppb.New(clock.Now())} + + outOfDateRollout, err := update.NewAutoUpdateAgentRollout(&autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.2", + TargetVersion: "1.2.3", + Schedule: update.AgentsScheduleImmediate, + AutoupdateMode: update.AgentsUpdateModeEnabled, + Strategy: update.AgentsStrategyHaltOnError, + }) + require.NoError(t, err) + outOfDateRollout.Status = &autoupdate.AutoUpdateAgentRolloutStatus{} + + tests := []struct { + name string + config *autoupdate.AutoUpdateConfig + version *autoupdate.AutoUpdateVersion + existingRollout *autoupdate.AutoUpdateAgentRollout + createExpect *autoupdate.AutoUpdateAgentRollout + updateExpect *autoupdate.AutoUpdateAgentRollout + deleteExpect bool + }{ + { + name: "config and version exist, no existing rollout", + // rollout should be created + config: configOK, + version: versionOK, + createExpect: upToDateRollout, + }, + { + name: "version exist, no existing rollout nor config", + // rollout should be created + version: versionOK, + createExpect: upToDateRollout, + }, + { + name: "version exist, no existing rollout, config exist but doesn't contain agent section", + // rollout should be created + config: configNoAgent, + version: versionOK, + createExpect: upToDateRollout, + }, + { + name: "config exist, no existing rollout nor version", + // rollout should not be created as there is no version + config: configOK, + }, + { + name: "config exist, no existing rollout, version exist but doesn't contain agent section", + // rollout should not be created as there is no version + config: configOK, + version: versionNoAgent, + }, + { + name: "no existing rollout, config, nor version", + // rollout should not be created as there is no version + }, + { + name: "existing out-of-date rollout, config and version exist", + // rollout should be updated + config: configOK, + version: versionOK, + existingRollout: outOfDateRollout, + updateExpect: upToDateRollout, + }, + { + name: "existing up-to-date rollout, config and version exist", + // rollout should not be updated as its spec is already good + config: configOK, + version: versionOK, + existingRollout: upToDateRollout, + }, + { + name: "existing rollout and config but no version", + // rollout should be deleted as there is no version + config: configOK, + existingRollout: upToDateRollout, + deleteExpect: true, + }, + { + name: "existing rollout but no config nor version", + // rollout should be deleted as there is no version + existingRollout: upToDateRollout, + deleteExpect: true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + // Test setup: creating a fake client answering fixtures + var stubs mockClientStubs + + if tt.config != nil { + stubs.configAnswers = []callAnswer[*autoupdate.AutoUpdateConfig]{{tt.config, nil}} + } else { + stubs.configAnswers = []callAnswer[*autoupdate.AutoUpdateConfig]{{nil, trace.NotFound("no config")}} + } + + if tt.version != nil { + stubs.versionAnswers = []callAnswer[*autoupdate.AutoUpdateVersion]{{tt.version, nil}} + } else { + stubs.versionAnswers = []callAnswer[*autoupdate.AutoUpdateVersion]{{nil, trace.NotFound("no version")}} + } + + if tt.existingRollout != nil { + stubs.rolloutAnswers = []callAnswer[*autoupdate.AutoUpdateAgentRollout]{{tt.existingRollout, nil}} + } else { + stubs.rolloutAnswers = []callAnswer[*autoupdate.AutoUpdateAgentRollout]{{nil, trace.NotFound("no rollout")}} + } + + if tt.createExpect != nil { + stubs.createRolloutAnswers = []callAnswer[*autoupdate.AutoUpdateAgentRollout]{{tt.createExpect, nil}} + stubs.createRolloutExpects = []require.ValueAssertionFunc{rolloutEquals(tt.createExpect)} + } + + if tt.updateExpect != nil { + stubs.updateRolloutAnswers = []callAnswer[*autoupdate.AutoUpdateAgentRollout]{{tt.updateExpect, nil}} + stubs.updateRolloutExpects = []require.ValueAssertionFunc{rolloutEquals(tt.updateExpect)} + } + + if tt.deleteExpect { + stubs.deleteRolloutAnswers = []error{nil} + } + + client := newMockClient(t, stubs) + + // Test execution: Running the reconciliation + + reconciler := &reconciler{ + clt: client, + log: log, + clock: clock, + metrics: newMetricsForTest(t), + } + + require.NoError(t, reconciler.tryReconcile(ctx)) + // Test validation: Checking that the mock client is now empty + + client.checkIfEmpty(t) + }) + } +} + +func TestReconciler_Reconcile(t *testing.T) { + log := utils.NewLoggerForTests().WithField("component", "reconciler") + ctx := context.Background() + clock := clockwork.NewFakeClock() + // Test setup: creating fixtures + config, err := update.NewAutoUpdateConfig(&autoupdate.AutoUpdateConfigSpec{ + Tools: &autoupdate.AutoUpdateConfigSpecTools{ + Mode: update.ToolsUpdateModeEnabled, + }, + Agents: &autoupdate.AutoUpdateConfigSpecAgents{ + Mode: update.AgentsUpdateModeEnabled, + Strategy: update.AgentsStrategyHaltOnError, + }, + }) + require.NoError(t, err) + version, err := update.NewAutoUpdateVersion(&autoupdate.AutoUpdateVersionSpec{ + Tools: &autoupdate.AutoUpdateVersionSpecTools{ + TargetVersion: "1.2.3", + }, + Agents: &autoupdate.AutoUpdateVersionSpecAgents{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: update.AgentsScheduleImmediate, + Mode: update.AgentsUpdateModeEnabled, + }, + }) + require.NoError(t, err) + upToDateRollout, err := update.NewAutoUpdateAgentRollout(&autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: update.AgentsScheduleImmediate, + AutoupdateMode: update.AgentsUpdateModeEnabled, + Strategy: update.AgentsStrategyHaltOnError, + }) + require.NoError(t, err) + upToDateRollout.Status = &autoupdate.AutoUpdateAgentRolloutStatus{StartTime: timestamppb.New(clock.Now())} + + outOfDateRollout, err := update.NewAutoUpdateAgentRollout(&autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.2", + TargetVersion: "1.2.3", + Schedule: update.AgentsScheduleImmediate, + AutoupdateMode: update.AgentsUpdateModeEnabled, + Strategy: update.AgentsStrategyHaltOnError, + }) + require.NoError(t, err) + outOfDateRollout.Status = &autoupdate.AutoUpdateAgentRolloutStatus{} + + // Those tests are not written in table format because the fixture setup it too complex and this would harm + // readability. + t.Run("reconciliation has nothing to do, should exit", func(t *testing.T) { + // Test setup: build mock client + stubs := mockClientStubs{ + configAnswers: []callAnswer[*autoupdate.AutoUpdateConfig]{{config, nil}}, + versionAnswers: []callAnswer[*autoupdate.AutoUpdateVersion]{{version, nil}}, + rolloutAnswers: []callAnswer[*autoupdate.AutoUpdateAgentRollout]{{upToDateRollout, nil}}, + } + + client := newMockClient(t, stubs) + reconciler := &reconciler{ + clt: client, + log: log, + clock: clock, + metrics: newMetricsForTest(t), + } + + // Test execution: run the reconciliation loop + require.NoError(t, reconciler.reconcile(ctx)) + + // Test validation: check that all the expected calls were received + client.checkIfEmpty(t) + }) + + t.Run("reconciliation succeeds on first try, should exit", func(t *testing.T) { + stubs := mockClientStubs{ + configAnswers: []callAnswer[*autoupdate.AutoUpdateConfig]{{config, nil}}, + versionAnswers: []callAnswer[*autoupdate.AutoUpdateVersion]{{version, nil}}, + rolloutAnswers: []callAnswer[*autoupdate.AutoUpdateAgentRollout]{{outOfDateRollout, nil}}, + updateRolloutExpects: []require.ValueAssertionFunc{rolloutEquals(upToDateRollout)}, + updateRolloutAnswers: []callAnswer[*autoupdate.AutoUpdateAgentRollout]{{upToDateRollout, nil}}, + } + + client := newMockClient(t, stubs) + reconciler := &reconciler{ + clt: client, + log: log, + clock: clock, + metrics: newMetricsForTest(t), + } + + // Test execution: run the reconciliation loop + require.NoError(t, reconciler.reconcile(ctx)) + + // Test validation: check that all the expected calls were received + client.checkIfEmpty(t) + }) + + t.Run("reconciliation faces conflict on first try, should retry and see that there's nothing left to do", func(t *testing.T) { + stubs := mockClientStubs{ + // because of the retry, we expect 2 GETs on every resource + configAnswers: []callAnswer[*autoupdate.AutoUpdateConfig]{{config, nil}, {config, nil}}, + versionAnswers: []callAnswer[*autoupdate.AutoUpdateVersion]{{version, nil}, {version, nil}}, + rolloutAnswers: []callAnswer[*autoupdate.AutoUpdateAgentRollout]{{outOfDateRollout, nil}, {upToDateRollout, nil}}, + // Single update expected, because there's nothing to do after the retry + updateRolloutExpects: []require.ValueAssertionFunc{rolloutEquals(upToDateRollout)}, + updateRolloutAnswers: []callAnswer[*autoupdate.AutoUpdateAgentRollout]{{nil, trace.CompareFailed("conflict")}}, + } + + client := newMockClient(t, stubs) + reconciler := &reconciler{ + clt: client, + log: log, + clock: clock, + metrics: newMetricsForTest(t), + } + + // Test execution: run the reconciliation loop + require.NoError(t, reconciler.reconcile(ctx)) + + // Test validation: check that all the expected calls were received + client.checkIfEmpty(t) + }) + + t.Run("reconciliation faces conflict on first try, should retry and update a second time", func(t *testing.T) { + rev1, err := uuid.NewUUID() + require.NoError(t, err) + rev2, err := uuid.NewUUID() + require.NoError(t, err) + rev3, err := uuid.NewUUID() + require.NoError(t, err) + + stubs := mockClientStubs{ + // because of the retry, we expect 2 GETs on every resource + configAnswers: []callAnswer[*autoupdate.AutoUpdateConfig]{{config, nil}, {config, nil}}, + versionAnswers: []callAnswer[*autoupdate.AutoUpdateVersion]{{version, nil}, {version, nil}}, + rolloutAnswers: []callAnswer[*autoupdate.AutoUpdateAgentRollout]{ + {withRevisionID(outOfDateRollout, rev1.String()), nil}, + {withRevisionID(outOfDateRollout, rev2.String()), nil}}, + // Two updates expected, one with the old revision, then a second one with the new + updateRolloutExpects: []require.ValueAssertionFunc{ + rolloutEquals(withRevisionID(upToDateRollout, rev1.String())), + rolloutEquals(withRevisionID(upToDateRollout, rev2.String())), + }, + // We mimic a race and reject the first update because of the outdated revision + updateRolloutAnswers: []callAnswer[*autoupdate.AutoUpdateAgentRollout]{ + {nil, trace.CompareFailed("conflict")}, + {withRevisionID(upToDateRollout, rev3.String()), nil}, + }, + } + + client := newMockClient(t, stubs) + reconciler := &reconciler{ + clt: client, + log: log, + clock: clock, + metrics: newMetricsForTest(t), + } + + // Test execution: run the reconciliation loop + require.NoError(t, reconciler.reconcile(ctx)) + + // Test validation: check that all the expected calls were received + client.checkIfEmpty(t) + }) + + t.Run("reconciliation faces missing rollout on first try, should retry and create the rollout", func(t *testing.T) { + stubs := mockClientStubs{ + // because of the retry, we expect 2 GETs on every resource + configAnswers: []callAnswer[*autoupdate.AutoUpdateConfig]{{config, nil}, {config, nil}}, + versionAnswers: []callAnswer[*autoupdate.AutoUpdateVersion]{{version, nil}, {version, nil}}, + rolloutAnswers: []callAnswer[*autoupdate.AutoUpdateAgentRollout]{ + {outOfDateRollout, nil}, + {nil, trace.NotFound("no rollout")}}, + // One update expected on the first try, the second try should create + updateRolloutExpects: []require.ValueAssertionFunc{ + rolloutEquals(upToDateRollout), + }, + // We mimic the fact the rollout got deleted in the meantime + updateRolloutAnswers: []callAnswer[*autoupdate.AutoUpdateAgentRollout]{ + {nil, trace.NotFound("no rollout")}, + }, + // One create expected on the second try + createRolloutExpects: []require.ValueAssertionFunc{ + rolloutEquals(upToDateRollout), + }, + createRolloutAnswers: []callAnswer[*autoupdate.AutoUpdateAgentRollout]{ + {upToDateRollout, nil}, + }, + } + + client := newMockClient(t, stubs) + reconciler := &reconciler{ + clt: client, + log: log, + clock: clock, + metrics: newMetricsForTest(t), + } + + // Test execution: run the reconciliation loop + require.NoError(t, reconciler.reconcile(ctx)) + + // Test validation: check that all the expected calls were received + client.checkIfEmpty(t) + }) + + t.Run("reconciliation meets a hard unexpected failure on first try, should exit in error", func(t *testing.T) { + stubs := mockClientStubs{ + configAnswers: []callAnswer[*autoupdate.AutoUpdateConfig]{{config, nil}}, + versionAnswers: []callAnswer[*autoupdate.AutoUpdateVersion]{{version, nil}}, + rolloutAnswers: []callAnswer[*autoupdate.AutoUpdateAgentRollout]{{outOfDateRollout, nil}}, + updateRolloutExpects: []require.ValueAssertionFunc{rolloutEquals(upToDateRollout)}, + updateRolloutAnswers: []callAnswer[*autoupdate.AutoUpdateAgentRollout]{ + {nil, trace.ConnectionProblem(trace.Errorf("io/timeout"), "the DB fell on the floor")}, + }, + } + + client := newMockClient(t, stubs) + reconciler := &reconciler{ + clt: client, + log: log, + clock: clock, + metrics: newMetricsForTest(t), + } + + // Test execution: run the reconciliation loop + require.ErrorContains(t, reconciler.reconcile(ctx), "the DB fell on the floor") + + // Test validation: check that all the expected calls were received + client.checkIfEmpty(t) + }) + + t.Run("reconciliation faces conflict on first try, should retry but context is expired so it bails out", func(t *testing.T) { + cancelableCtx, cancel := context.WithCancel(ctx) + // just in case + t.Cleanup(cancel) + + stubs := mockClientStubs{ + // we expect a single GET because the context expires before the second retry + configAnswers: []callAnswer[*autoupdate.AutoUpdateConfig]{{config, nil}}, + versionAnswers: []callAnswer[*autoupdate.AutoUpdateVersion]{{version, nil}}, + rolloutAnswers: []callAnswer[*autoupdate.AutoUpdateAgentRollout]{{outOfDateRollout, nil}}, + // Single update expected, because there's nothing to do after the retry. + // We wrap the update validation function into a context canceler, so the context is done after the first update + updateRolloutExpects: []require.ValueAssertionFunc{cancelContext(rolloutEquals(upToDateRollout), cancel)}, + // return a retryable error + updateRolloutAnswers: []callAnswer[*autoupdate.AutoUpdateAgentRollout]{{nil, trace.CompareFailed("conflict")}}, + } + + client := newMockClient(t, stubs) + reconciler := &reconciler{ + clt: client, + log: log, + clock: clock, + metrics: newMetricsForTest(t), + } + + // Test execution: run the reconciliation loop + require.ErrorIs(t, reconciler.reconcile(cancelableCtx), context.Canceled) + + // Test validation: check that all the expected calls were received + client.checkIfEmpty(t) + }) +} + +func Test_makeGroupsStatus(t *testing.T) { + now := time.Now() + ctx := context.Background() + + tests := []struct { + name string + schedules *autoupdate.AgentAutoUpdateSchedules + expected []*autoupdate.AutoUpdateAgentRolloutStatusGroup + }{ + { + name: "nil schedules", + schedules: nil, + expected: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: defaultGroupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(now), + LastUpdateReason: updateReasonCreated, + ConfigDays: defaultUpdateDays, + ConfigStartHour: defaultStartHour, + }, + }, + }, + { + name: "no groups in schedule", + schedules: &autoupdate.AgentAutoUpdateSchedules{Regular: make([]*autoupdate.AgentAutoUpdateGroup, 0)}, + expected: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: defaultGroupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(now), + LastUpdateReason: updateReasonCreated, + ConfigDays: defaultUpdateDays, + ConfigStartHour: defaultStartHour, + }, + }, + }, + { + name: "one group in schedule", + schedules: &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + { + Name: "group1", + Days: everyWeekday, + StartHour: matchingStartHour, + }, + }, + }, + expected: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: "group1", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(now), + LastUpdateReason: updateReasonCreated, + ConfigDays: everyWeekday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "multiple groups in schedule", + schedules: &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + { + Name: "group1", + Days: everyWeekday, + StartHour: matchingStartHour, + }, + { + Name: "group2", + Days: everyWeekdayButSunday, + StartHour: nonMatchingStartHour, + WaitHours: 1, + }, + }, + }, + expected: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: "group1", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(now), + LastUpdateReason: updateReasonCreated, + ConfigDays: everyWeekday, + ConfigStartHour: matchingStartHour, + }, + { + Name: "group2", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(now), + LastUpdateReason: updateReasonCreated, + ConfigDays: everyWeekdayButSunday, + ConfigStartHour: nonMatchingStartHour, + ConfigWaitHours: 1, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // We craft a mock client always answering there's no cmc. + // It's not the point of this test to check the cmc client usage so we don't count the number of calls here. + // CMC-specific tests happen in TestDefaultConfigGroup(). + clt := newMockClient(t, mockClientStubs{cmcAnswers: []callAnswer[*types.ClusterMaintenanceConfigV1]{{ + result: nil, + err: trace.NotFound("no cmc"), + }}}) + r := reconciler{clt: clt} + result, err := r.makeGroupsStatus(ctx, tt.schedules, now) + require.NoError(t, err) + require.Equal(t, tt.expected, result) + }) + } +} + +const fakeRolloutStrategyName = "fake" + +type fakeRolloutStrategy struct { + strategyName string + // calls counts how many times the fake rollout strategy was called. + // This is not thread safe. + calls int +} + +func (f *fakeRolloutStrategy) name() string { + return f.strategyName +} + +func (f *fakeRolloutStrategy) progressRollout(ctx context.Context, spec *autoupdate.AutoUpdateAgentRolloutSpec, status *autoupdate.AutoUpdateAgentRolloutStatus, now time.Time) error { + f.calls++ + return nil +} + +func Test_reconciler_computeStatus(t *testing.T) { + log := utils.NewLoggerForTests().WithField("component", "reconciler") + clock := clockwork.NewFakeClock() + ctx := context.Background() + + oldStatus := &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: "old group", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + }, + }, + State: autoupdate.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSTARTED, + } + oldSpec := &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: update.AgentsScheduleRegular, + AutoupdateMode: update.AgentsUpdateModeEnabled, + Strategy: fakeRolloutStrategyName, + } + schedules := &autoupdate.AgentAutoUpdateSchedules{ + Regular: []*autoupdate.AgentAutoUpdateGroup{ + { + Name: "new group", + Days: everyWeekday, + }, + }, + } + r := reconciler{} + newGroups, err := r.makeGroupsStatus(ctx, schedules, clock.Now()) + require.NoError(t, err) + newStatus := &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: newGroups, + State: autoupdate.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSTARTED, + StartTime: timestamppb.New(clock.Now()), + } + + tests := []struct { + name string + existingRollout *autoupdate.AutoUpdateAgentRollout + newSpec *autoupdate.AutoUpdateAgentRolloutSpec + expectedStatus *autoupdate.AutoUpdateAgentRolloutStatus + expectedStrategyCalls int + }{ + { + name: "status is reset if start version changes", + existingRollout: &autoupdate.AutoUpdateAgentRollout{ + Spec: oldSpec, + Status: oldStatus, + }, + newSpec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.2", + TargetVersion: "1.2.4", + Schedule: update.AgentsScheduleRegular, + AutoupdateMode: update.AgentsUpdateModeEnabled, + Strategy: fakeRolloutStrategyName, + }, + // status should have been reset and is now the new status + expectedStatus: newStatus, + expectedStrategyCalls: 1, + }, + { + name: "status is reset if target version changes", + existingRollout: &autoupdate.AutoUpdateAgentRollout{ + Spec: oldSpec, + Status: oldStatus, + }, + newSpec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.5", + Schedule: update.AgentsScheduleRegular, + AutoupdateMode: update.AgentsUpdateModeEnabled, + Strategy: fakeRolloutStrategyName, + }, + // status should have been reset and is now the new status + expectedStatus: newStatus, + expectedStrategyCalls: 1, + }, + { + name: "status is reset if strategy changes", + existingRollout: &autoupdate.AutoUpdateAgentRollout{ + Spec: oldSpec, + Status: oldStatus, + }, + newSpec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: update.AgentsScheduleRegular, + AutoupdateMode: update.AgentsUpdateModeEnabled, + Strategy: fakeRolloutStrategyName + "2", + }, + // status should have been reset and is now the new status + expectedStatus: newStatus, + expectedStrategyCalls: 1, + }, + { + name: "status is not reset if mode changes", + existingRollout: &autoupdate.AutoUpdateAgentRollout{ + Spec: oldSpec, + Status: oldStatus, + }, + newSpec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: update.AgentsScheduleRegular, + AutoupdateMode: update.AgentsUpdateModeSuspended, + Strategy: fakeRolloutStrategyName, + }, + // status should NOT have been reset and still contain the old groups + expectedStatus: oldStatus, + expectedStrategyCalls: 1, + }, + { + name: "groups are unset if schedule is immediate", + existingRollout: &autoupdate.AutoUpdateAgentRollout{ + Spec: oldSpec, + Status: oldStatus, + }, + newSpec: &autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: update.AgentsScheduleImmediate, + AutoupdateMode: update.AgentsUpdateModeEnabled, + Strategy: fakeRolloutStrategyName, + }, + // groups should be unset + expectedStatus: &autoupdate.AutoUpdateAgentRolloutStatus{ + StartTime: timestamppb.New(clock.Now()), + }, + expectedStrategyCalls: 0, + }, + { + name: "new groups are populated if previous ones were empty", + existingRollout: &autoupdate.AutoUpdateAgentRollout{ + Spec: oldSpec, + // old groups were empty + Status: &autoupdate.AutoUpdateAgentRolloutStatus{ + StartTime: timestamppb.New(clock.Now()), + }, + }, + // no spec change + newSpec: oldSpec, + // still, we have the new groups set + expectedStatus: newStatus, + expectedStrategyCalls: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + strategy := &fakeRolloutStrategy{strategyName: tt.newSpec.Strategy} + r := &reconciler{ + log: log, + clock: clock, + rolloutStrategies: []rolloutStrategy{strategy}, + metrics: newMetricsForTest(t), + } + result, err := r.computeStatus(ctx, tt.existingRollout, tt.newSpec, schedules) + require.NoError(t, err) + require.Empty(t, cmp.Diff(tt.expectedStatus, result, protocmp.Transform())) + require.Equal(t, tt.expectedStrategyCalls, strategy.calls) + }) + } +} + +func TestDefaultConfigGroup(t *testing.T) { + ctx := context.Background() + testStartHour := 16 + + tests := []struct { + name string + cmcAnswer callAnswer[*types.ClusterMaintenanceConfigV1] + expectedResult *autoupdate.AgentAutoUpdateGroup + expectError require.ErrorAssertionFunc + }{ + { + name: "no CMC", + cmcAnswer: callAnswer[*types.ClusterMaintenanceConfigV1]{ + nil, trace.NotFound("no cmc"), + }, + expectedResult: defaultGroup(), + expectError: require.NoError, + }, + { + name: "CMC with no upgrade window", + cmcAnswer: callAnswer[*types.ClusterMaintenanceConfigV1]{ + &types.ClusterMaintenanceConfigV1{ + Spec: types.ClusterMaintenanceConfigSpecV1{ + AgentUpgrades: nil, + }, + }, nil, + }, + expectedResult: defaultGroup(), + expectError: require.NoError, + }, + { + name: "CMC with no weekdays", + cmcAnswer: callAnswer[*types.ClusterMaintenanceConfigV1]{ + &types.ClusterMaintenanceConfigV1{ + Spec: types.ClusterMaintenanceConfigSpecV1{ + AgentUpgrades: &types.AgentUpgradeWindow{ + UTCStartHour: uint32(testStartHour), + Weekdays: nil, + }, + }, + }, nil, + }, + expectedResult: &autoupdate.AgentAutoUpdateGroup{ + Name: defaultCMCGroupName, + Days: []string{"*"}, + StartHour: int32(testStartHour), + WaitHours: 0, + }, + expectError: require.NoError, + }, + { + name: "CMC with weekdays", + cmcAnswer: callAnswer[*types.ClusterMaintenanceConfigV1]{ + &types.ClusterMaintenanceConfigV1{ + Spec: types.ClusterMaintenanceConfigSpecV1{ + AgentUpgrades: &types.AgentUpgradeWindow{ + UTCStartHour: uint32(testStartHour), + Weekdays: everyWeekdayButSunday, + }, + }, + }, nil, + }, + expectedResult: &autoupdate.AgentAutoUpdateGroup{ + Name: defaultCMCGroupName, + Days: everyWeekdayButSunday, + StartHour: int32(testStartHour), + WaitHours: 0, + }, + expectError: require.NoError, + }, + { + name: "unexpected error getting CMC", + cmcAnswer: callAnswer[*types.ClusterMaintenanceConfigV1]{ + nil, trace.ConnectionProblem(trace.Errorf("oh no"), "connection failed"), + }, + expectedResult: nil, + expectError: require.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test setup: loading fixtures. + clt := newMockClient(t, mockClientStubs{cmcAnswers: []callAnswer[*types.ClusterMaintenanceConfigV1]{tt.cmcAnswer}}) + r := &reconciler{clt: clt} + // Test execution. + result, err := r.defaultConfigGroup(ctx) + tt.expectError(t, err) + require.Equal(t, tt.expectedResult, result) + // Test validation: the mock client should be empty. + clt.checkIfEmpty(t) + }) + } +} diff --git a/lib/autoupdate/rollout/strategy.go b/lib/autoupdate/rollout/strategy.go new file mode 100644 index 0000000000000..d5b8236ce8f90 --- /dev/null +++ b/lib/autoupdate/rollout/strategy.go @@ -0,0 +1,153 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package rollout + +import ( + "context" + "time" + + "github.com/gravitational/trace" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + "github.com/gravitational/teleport/api/types" +) + +const ( + // Common update reasons + updateReasonCreated = "created" + updateReasonReconcilerError = "reconciler_error" + updateReasonRolloutChanged = "rollout_changed_during_window" +) + +// rolloutStrategy is responsible for rolling out the update across groups. +// This interface allows us to inject dummy strategies for simpler testing. +type rolloutStrategy interface { + name() string + // progressRollout takes the new rollout spec, existing rollout status and current time. + // It updates the status resource in-place to progress the rollout to the next step if possible/needed. + progressRollout(context.Context, *autoupdate.AutoUpdateAgentRolloutSpec, *autoupdate.AutoUpdateAgentRolloutStatus, time.Time) error +} + +// inWindow checks if the time is in the group's maintenance window. +// The maintenance window is the semi-open interval: [windowStart, windowEnd). +func inWindow(group *autoupdate.AutoUpdateAgentRolloutStatusGroup, now time.Time, duration time.Duration) (bool, error) { + dayOK, err := canUpdateToday(group.ConfigDays, now) + if err != nil { + return false, trace.Wrap(err, "checking the day of the week") + } + if !dayOK { + return false, nil + } + + // We compute the theoretical window start and end + windowStart := now.Truncate(24 * time.Hour).Add(time.Duration(group.ConfigStartHour) * time.Hour) + windowEnd := windowStart.Add(duration) + + return !now.Before(windowStart) && now.Before(windowEnd), nil +} + +// rolloutChangedInWindow checks if the rollout got created after the theoretical group start time +func rolloutChangedInWindow(group *autoupdate.AutoUpdateAgentRolloutStatusGroup, now, rolloutStart time.Time, duration time.Duration) (bool, error) { + // If the rollout is older than 24h, we know it did not change during the window + if now.Sub(rolloutStart) > 24*time.Hour { + return false, nil + } + // Else we check if the rollout happened in the group window. + return inWindow(group, rolloutStart, duration) +} + +func canUpdateToday(allowedDays []string, now time.Time) (bool, error) { + for _, allowedDay := range allowedDays { + if allowedDay == types.Wildcard { + return true, nil + } + weekday, ok := types.ParseWeekday(allowedDay) + if !ok { + return false, trace.BadParameter("failed to parse weekday %q", allowedDay) + } + if weekday == now.Weekday() { + return true, nil + } + } + return false, nil +} + +func setGroupState(group *autoupdate.AutoUpdateAgentRolloutStatusGroup, newState autoupdate.AutoUpdateAgentGroupState, reason string, now time.Time) { + changed := false + previousState := group.State + + // Check if there is a state transition + if previousState != newState { + group.State = newState + changed = true + // If we just started the group, also update the start time + if newState == autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE { + group.StartTime = timestamppb.New(now) + } + } + + // Check if there is a reason change. Even if the state did not change, we + // might want to explain why. + if group.LastUpdateReason != reason { + group.LastUpdateReason = reason + changed = true + } + + if changed { + group.LastUpdateTime = timestamppb.New(now) + } +} + +func computeRolloutState(groups []*autoupdate.AutoUpdateAgentRolloutStatusGroup) autoupdate.AutoUpdateAgentRolloutState { + groupCount := len(groups) + + if groupCount == 0 { + return autoupdate.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSPECIFIED + } + + var doneGroups, unstartedGroups int + + for _, group := range groups { + switch group.State { + // If one or more groups have been rolled back, we consider the rollout rolledback + case autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK: + return autoupdate.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_ROLLEDBACK + + case autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED: + unstartedGroups++ + + case autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE: + doneGroups++ + } + } + + // If every group is done, the rollout is done. + if doneGroups == groupCount { + return autoupdate.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_DONE + } + + // If every group is unstarted, the rollout is unstarted. + if unstartedGroups == groupCount { + return autoupdate.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSTARTED + } + + // Else at least one group is active or done, but not everything is finished. We consider the rollout active. + return autoupdate.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_ACTIVE +} diff --git a/lib/autoupdate/rollout/strategy_haltonerror.go b/lib/autoupdate/rollout/strategy_haltonerror.go new file mode 100644 index 0000000000000..cc5d91cc9718b --- /dev/null +++ b/lib/autoupdate/rollout/strategy_haltonerror.go @@ -0,0 +1,164 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package rollout + +import ( + "context" + "time" + + "github.com/gravitational/trace" + "github.com/sirupsen/logrus" + + "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + update "github.com/gravitational/teleport/api/types/autoupdate" +) + +const ( + updateReasonCanStart = "can_start" + updateReasonCannotStart = "cannot_start" + updateReasonPreviousGroupsNotDone = "previous_groups_not_done" + updateReasonUpdateComplete = "update_complete" + updateReasonUpdateInProgress = "update_in_progress" + haltOnErrorWindowDuration = time.Hour +) + +type haltOnErrorStrategy struct { + log *logrus.Entry +} + +func (h *haltOnErrorStrategy) name() string { + return update.AgentsStrategyHaltOnError +} + +func newHaltOnErrorStrategy(log *logrus.Entry) (rolloutStrategy, error) { + if log == nil { + return nil, trace.BadParameter("missing log") + } + return &haltOnErrorStrategy{ + log: log.WithField("strategy", update.AgentsStrategyHaltOnError), + }, nil +} + +func (h *haltOnErrorStrategy) progressRollout(ctx context.Context, _ *autoupdate.AutoUpdateAgentRolloutSpec, status *autoupdate.AutoUpdateAgentRolloutStatus, now time.Time) error { + // We process every group in order, all the previous groups must be in the DONE state + // for the next group to become active. Even if some early groups are not DONE, + // later groups might be ACTIVE and need to transition to DONE, so we cannot + // return early and must process every group. + // + // For example, in a dev/staging/prod setup, the "dev" group might get rolled + // back while "staging" is still ACTIVE. We must not start PROD but still need + // to transition "staging" to DONE. + previousGroupsAreDone := true + + for i, group := range status.Groups { + switch group.State { + case autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED: + var previousGroup *autoupdate.AutoUpdateAgentRolloutStatusGroup + if i != 0 { + previousGroup = status.Groups[i-1] + } + canStart, err := canStartHaltOnError(group, previousGroup, now) + if err != nil { + // In halt-on-error rollouts, groups are dependent. + // Failing to transition a group should prevent other groups from transitioning. + setGroupState(group, group.State, updateReasonReconcilerError, now) + return err + } + + // Check if the rollout got created after the theoretical group start time + rolloutChangedDuringWindow, err := rolloutChangedInWindow(group, now, status.StartTime.AsTime(), haltOnErrorWindowDuration) + if err != nil { + setGroupState(group, group.State, updateReasonReconcilerError, now) + return err + } + + switch { + case !previousGroupsAreDone: + // All previous groups are not DONE + setGroupState(group, group.State, updateReasonPreviousGroupsNotDone, now) + case !canStart: + // All previous groups are DONE, but time-related criteria are not met + // This can be because we are outside an update window, or because the + // specified wait_hours doesn't let us update yet. + setGroupState(group, group.State, updateReasonCannotStart, now) + case rolloutChangedDuringWindow: + // All previous groups are DONE and time-related criteria are met. + // However, the rollout changed during the maintenance window. + setGroupState(group, group.State, updateReasonRolloutChanged, now) + default: + // All previous groups are DONE and time-related criteria are met. + // We can start. + setGroupState(group, autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, updateReasonCanStart, now) + } + previousGroupsAreDone = false + case autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK: + // The group has been manually rolled back. We don't touch anything and + // don't process the next groups. + previousGroupsAreDone = false + case autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE: + // The group has already been updated, we can look at the next group + case autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE: + // The group is currently being updated. We check if we can transition it to the done state + done, reason := isDoneHaltOnError(group, now) + + if done { + // We transition to the done state. We continue processing the groups as we might be able to start the next one. + setGroupState(group, autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, reason, now) + } else { + setGroupState(group, autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, reason, now) + } + previousGroupsAreDone = false + + default: + return trace.BadParameter("unknown autoupdate group state: %v", group.State) + } + } + return nil +} + +func canStartHaltOnError(group, previousGroup *autoupdate.AutoUpdateAgentRolloutStatusGroup, now time.Time) (bool, error) { + // check wait hours + if group.ConfigWaitHours != 0 { + if previousGroup == nil { + return false, trace.BadParameter("the first group cannot have non-zero wait hours") + } + + previousStart := previousGroup.StartTime.AsTime() + if previousStart.IsZero() || previousStart.Unix() == 0 { + return false, trace.BadParameter("the previous group doesn't have a start time, cannot check the 'wait_hours' criterion") + } + + // Check if the wait_hours criterion is OK, if we are at least after 'wait_hours' hours since the previous start. + if now.Before(previousGroup.StartTime.AsTime().Add(time.Duration(group.ConfigWaitHours) * time.Hour)) { + return false, nil + } + } + + return inWindow(group, now, haltOnErrorWindowDuration) +} + +func isDoneHaltOnError(group *autoupdate.AutoUpdateAgentRolloutStatusGroup, now time.Time) (bool, string) { + // Currently we don't implement status reporting from groups/agents. + // So we just wait 60 minutes and consider the maintenance done. + // This will change as we introduce agent status report and aggregated agent counts. + if group.StartTime.AsTime().Add(haltOnErrorWindowDuration).Before(now) { + return true, updateReasonUpdateComplete + } + return false, updateReasonUpdateInProgress +} diff --git a/lib/autoupdate/rollout/strategy_haltonerror_test.go b/lib/autoupdate/rollout/strategy_haltonerror_test.go new file mode 100644 index 0000000000000..01cb4eef416ee --- /dev/null +++ b/lib/autoupdate/rollout/strategy_haltonerror_test.go @@ -0,0 +1,512 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package rollout + +import ( + "context" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + "github.com/gravitational/teleport/lib/utils" +) + +func Test_canStartHaltOnError(t *testing.T) { + now := testSunday + yesterday := testSaturday + + tests := []struct { + name string + group *autoupdate.AutoUpdateAgentRolloutStatusGroup + previousGroup *autoupdate.AutoUpdateAgentRolloutStatusGroup + want bool + wantErr require.ErrorAssertionFunc + }{ + { + name: "first group, no wait_hours", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: "test-group", + ConfigDays: everyWeekday, + ConfigStartHour: int32(now.Hour()), + ConfigWaitHours: 0, + }, + want: true, + wantErr: require.NoError, + }, + { + name: "first group, wait_days (invalid)", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: "test-group", + ConfigDays: everyWeekday, + ConfigStartHour: int32(now.Hour()), + ConfigWaitHours: 1, + }, + want: false, + wantErr: require.Error, + }, + { + name: "second group, no wait_days", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: "test-group", + ConfigDays: everyWeekday, + ConfigStartHour: int32(now.Hour()), + ConfigWaitHours: 0, + }, + previousGroup: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: "previous-group", + StartTime: timestamppb.New(now), + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + ConfigDays: everyWeekday, + ConfigStartHour: int32(now.Hour()), + ConfigWaitHours: 0, + }, + want: true, + wantErr: require.NoError, + }, + { + name: "second group, wait_days not over", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: "test-group", + ConfigDays: everyWeekday, + ConfigStartHour: int32(now.Hour()), + ConfigWaitHours: 48, + }, + previousGroup: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: "previous-group", + StartTime: timestamppb.New(yesterday), + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + ConfigDays: everyWeekday, + ConfigStartHour: int32(now.Hour()), + ConfigWaitHours: 0, + }, + want: false, + wantErr: require.NoError, + }, + { + name: "second group, wait_days over", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: "test-group", + ConfigDays: everyWeekday, + ConfigStartHour: int32(now.Hour()), + ConfigWaitHours: 24, + }, + previousGroup: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: "previous-group", + StartTime: timestamppb.New(yesterday), + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + ConfigDays: everyWeekday, + ConfigStartHour: int32(now.Hour()), + ConfigWaitHours: 0, + }, + want: true, + wantErr: require.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := canStartHaltOnError(tt.group, tt.previousGroup, now) + tt.wantErr(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func Test_progressGroupsHaltOnError(t *testing.T) { + clock := clockwork.NewFakeClockAt(testSunday) + log := utils.NewLoggerForTests().WithField("component", "reconciler") + strategy, err := newHaltOnErrorStrategy(log) + require.NoError(t, err) + + fewMinutesAgo := clock.Now().Add(-5 * time.Minute) + yesterday := testSaturday + canStartToday := everyWeekday + cannotStartToday := everyWeekdayButSunday + ctx := context.Background() + + group1Name := "group1" + group2Name := "group2" + group3Name := "group3" + + tests := []struct { + name string + initialState []*autoupdate.AutoUpdateAgentRolloutStatusGroup + rolloutStartTime *timestamppb.Timestamp + expectedState []*autoupdate.AutoUpdateAgentRolloutStatusGroup + }{ + { + name: "single group unstarted -> unstarted", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: updateReasonCreated, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonCannotStart, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "single group unstarted -> unstarted because rollout changed in window", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: updateReasonCreated, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + rolloutStartTime: timestamppb.New(clock.Now()), + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonRolloutChanged, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "single group unstarted -> active", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: updateReasonCreated, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: timestamppb.New(clock.Now()), + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonCanStart, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "single group active -> active", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: timestamppb.New(fewMinutesAgo), + LastUpdateTime: timestamppb.New(fewMinutesAgo), + LastUpdateReason: updateReasonCanStart, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: timestamppb.New(fewMinutesAgo), + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonUpdateInProgress, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "single group active -> done", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: timestamppb.New(yesterday), + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: updateReasonUpdateInProgress, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + StartTime: timestamppb.New(yesterday), + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonUpdateComplete, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "single group done -> done", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + StartTime: timestamppb.New(yesterday), + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: updateReasonUpdateComplete, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + StartTime: timestamppb.New(yesterday), + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: updateReasonUpdateComplete, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "single group rolledback -> rolledback", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + StartTime: timestamppb.New(yesterday), + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: "manual_rollback", + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + StartTime: timestamppb.New(yesterday), + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: "manual_rollback", + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "first group done, second should activate, third should not progress", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + StartTime: timestamppb.New(yesterday), + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: updateReasonUpdateComplete, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: group2Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: updateReasonCreated, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + ConfigWaitHours: 24, + }, + { + Name: group3Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: updateReasonCreated, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + ConfigWaitHours: 0, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + StartTime: timestamppb.New(yesterday), + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: updateReasonUpdateComplete, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: group2Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: timestamppb.New(clock.Now()), + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonCanStart, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + ConfigWaitHours: 24, + }, + { + Name: group3Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonPreviousGroupsNotDone, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + ConfigWaitHours: 0, + }, + }, + }, + { + name: "first group rolledback, second should not start", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + StartTime: timestamppb.New(yesterday), + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: "manual_rollback", + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: group2Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: updateReasonCreated, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + ConfigWaitHours: 24, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + StartTime: timestamppb.New(yesterday), + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: "manual_rollback", + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: group2Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonPreviousGroupsNotDone, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + ConfigWaitHours: 24, + }, + }, + }, + { + name: "first group rolledback, second is active and should become done, third should not progress", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + StartTime: timestamppb.New(yesterday), + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: "manual_rollback", + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: group2Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: timestamppb.New(yesterday), + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: updateReasonCanStart, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + ConfigWaitHours: 0, + }, + { + Name: group3Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: updateReasonCreated, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + ConfigWaitHours: 0, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: group1Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + StartTime: timestamppb.New(yesterday), + LastUpdateTime: timestamppb.New(yesterday), + LastUpdateReason: "manual_rollback", + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: group2Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + StartTime: timestamppb.New(yesterday), + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonUpdateComplete, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + ConfigWaitHours: 0, + }, + { + Name: group3Name, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonPreviousGroupsNotDone, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + ConfigWaitHours: 0, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + status := &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: tt.initialState, + State: 0, + StartTime: tt.rolloutStartTime, + } + err := strategy.progressRollout(ctx, nil, status, clock.Now()) + require.NoError(t, err) + // We use require.Equal instead of Elements match because group order matters. + // It's not super important for time-based, but is crucial for halt-on-error. + // So it's better to be more conservative and validate order never changes for + // both strategies. + require.Equal(t, tt.expectedState, tt.initialState) + }) + } +} diff --git a/lib/autoupdate/rollout/strategy_test.go b/lib/autoupdate/rollout/strategy_test.go new file mode 100644 index 0000000000000..0711d4043ae9c --- /dev/null +++ b/lib/autoupdate/rollout/strategy_test.go @@ -0,0 +1,468 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package rollout + +import ( + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" +) + +var ( + // 2024-11-30 is a Saturday + testSaturday = time.Date(2024, 11, 30, 12, 30, 0, 0, time.UTC) + // 2024-12-01 is a Sunday + testSunday = time.Date(2024, 12, 1, 12, 30, 0, 0, time.UTC) + matchingStartHour = int32(12) + nonMatchingStartHour = int32(15) + everyWeekday = []string{"Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"} + everyWeekdayButSunday = []string{"Mon", "Tue", "Wed", "Thu", "Fri", "Sat"} +) + +func Test_canUpdateToday(t *testing.T) { + tests := []struct { + name string + allowedDays []string + now time.Time + want bool + wantErr require.ErrorAssertionFunc + }{ + { + name: "Empty list", + allowedDays: []string{}, + now: time.Now(), + want: false, + wantErr: require.NoError, + }, + { + name: "Wildcard", + allowedDays: []string{"*"}, + now: time.Now(), + want: true, + wantErr: require.NoError, + }, + { + name: "Matching day", + allowedDays: everyWeekday, + now: testSunday, + want: true, + wantErr: require.NoError, + }, + { + name: "No matching day", + allowedDays: everyWeekdayButSunday, + now: testSunday, + want: false, + wantErr: require.NoError, + }, + { + name: "Malformed day", + allowedDays: []string{"Mon", "Tue", "HelloThereGeneralKenobi"}, + now: testSunday, + want: false, + wantErr: require.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := canUpdateToday(tt.allowedDays, tt.now) + tt.wantErr(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func Test_inWindow(t *testing.T) { + tests := []struct { + name string + group *autoupdate.AutoUpdateAgentRolloutStatusGroup + now time.Time + duration time.Duration + want bool + wantErr require.ErrorAssertionFunc + }{ + { + name: "out of window", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + ConfigDays: everyWeekdayButSunday, + ConfigStartHour: matchingStartHour, + }, + now: testSunday, + duration: time.Hour, + want: false, + wantErr: require.NoError, + }, + { + name: "inside window, wrong hour", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + ConfigDays: everyWeekday, + ConfigStartHour: nonMatchingStartHour, + }, + now: testSunday, + duration: time.Hour, + want: false, + wantErr: require.NoError, + }, + { + name: "inside window, correct hour", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + ConfigDays: everyWeekday, + ConfigStartHour: matchingStartHour, + }, + now: testSunday, + duration: time.Hour, + want: true, + wantErr: require.NoError, + }, + { + name: "invalid weekdays", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + ConfigDays: []string{"HelloThereGeneralKenobi"}, + ConfigStartHour: matchingStartHour, + }, + now: testSunday, + duration: time.Hour, + want: false, + wantErr: require.Error, + }, + { + name: "short window", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + ConfigDays: everyWeekday, + ConfigStartHour: matchingStartHour, + }, + now: testSunday, + duration: time.Second, + want: false, + wantErr: require.NoError, + }, + { + name: "window start time is included", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + ConfigDays: everyWeekday, + ConfigStartHour: matchingStartHour, + }, + now: testSunday.Truncate(24 * time.Hour).Add(time.Duration(matchingStartHour) * time.Hour), + duration: time.Hour, + want: true, + wantErr: require.NoError, + }, + { + name: "window end time is not included", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + ConfigDays: everyWeekday, + ConfigStartHour: matchingStartHour, + }, + now: testSunday.Truncate(24 * time.Hour).Add(time.Duration(matchingStartHour+1) * time.Hour), + duration: time.Hour, + want: false, + wantErr: require.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := inWindow(tt.group, tt.now, tt.duration) + tt.wantErr(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func Test_rolloutChangedInWindow(t *testing.T) { + // Test setup: creating fixtures. + group := &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: "test-group", + ConfigDays: everyWeekdayButSunday, + ConfigStartHour: 12, + } + tests := []struct { + name string + now time.Time + rolloutStart time.Time + want bool + }{ + { + name: "zero rollout start time", + now: testSaturday, + rolloutStart: time.Time{}, + want: false, + }, + { + name: "epoch rollout start time", + now: testSaturday, + // tspb counts since epoch, wile go's zero is 0000-00-00 00:00:00 UTC + rolloutStart: (×tamppb.Timestamp{}).AsTime(), + want: false, + }, + { + name: "rollout changed a week ago", + now: testSaturday, + rolloutStart: testSaturday.Add(-7 * 24 * time.Hour), + want: false, + }, + { + name: "rollout changed the same day, before the window", + now: testSaturday, + rolloutStart: testSaturday.Add(-2 * time.Hour), + want: false, + }, + { + name: "rollout changed the same day, during the window", + now: testSaturday, + rolloutStart: testSaturday.Add(-2 * time.Minute), + want: true, + }, + { + name: "rollout just changed but we are not in a window", + now: testSunday, + rolloutStart: testSunday.Add(-2 * time.Minute), + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test execution. + result, err := rolloutChangedInWindow(group, tt.now, tt.rolloutStart, time.Hour) + require.NoError(t, err) + require.Equal(t, tt.want, result) + }) + } +} + +func Test_setGroupState(t *testing.T) { + groupName := "test-group" + + clock := clockwork.NewFakeClock() + // oldUpdateTime is 5 minutes in the past + oldUpdateTime := clock.Now() + clock.Advance(5 * time.Minute) + + tests := []struct { + name string + group *autoupdate.AutoUpdateAgentRolloutStatusGroup + newState autoupdate.AutoUpdateAgentGroupState + reason string + now time.Time + expected *autoupdate.AutoUpdateAgentRolloutStatusGroup + }{ + { + name: "same state, no change", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(oldUpdateTime), + LastUpdateReason: updateReasonCannotStart, + }, + newState: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + reason: updateReasonCannotStart, + now: clock.Now(), + expected: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + // update time has not been bumped as nothing changed + LastUpdateTime: timestamppb.New(oldUpdateTime), + LastUpdateReason: updateReasonCannotStart, + }, + }, + { + name: "same state, reason change", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(oldUpdateTime), + LastUpdateReason: updateReasonCannotStart, + }, + newState: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + reason: updateReasonReconcilerError, + now: clock.Now(), + expected: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + // update time has been bumped because reason changed + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonReconcilerError, + }, + }, + { + name: "new state, no reason change", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(oldUpdateTime), + LastUpdateReason: updateReasonCannotStart, + }, + newState: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + reason: updateReasonCannotStart, + now: clock.Now(), + expected: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + // update time has been bumped because state changed + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonCannotStart, + }, + }, + { + name: "new state, reason change", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(oldUpdateTime), + LastUpdateReason: updateReasonCannotStart, + }, + newState: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + reason: updateReasonReconcilerError, + now: clock.Now(), + expected: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + // update time has been bumped because state and reason changed + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonReconcilerError, + }, + }, + { + name: "new state, transition to active", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: nil, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(oldUpdateTime), + LastUpdateReason: updateReasonCannotStart, + }, + newState: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + reason: updateReasonCanStart, + now: clock.Now(), + expected: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + // We set start time during the transition + StartTime: timestamppb.New(clock.Now()), + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + // update time has been bumped because state and reason changed + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonCanStart, + }, + }, + { + name: "same state, transition from active to active", + group: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + StartTime: timestamppb.New(oldUpdateTime), + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + LastUpdateTime: timestamppb.New(oldUpdateTime), + LastUpdateReason: updateReasonCanStart, + }, + newState: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + reason: updateReasonReconcilerError, + now: clock.Now(), + expected: &autoupdate.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + // As the state was already active, the start time should not be refreshed + StartTime: timestamppb.New(oldUpdateTime), + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + // update time has been bumped because reason changed + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonReconcilerError, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + setGroupState(tt.group, tt.newState, tt.reason, tt.now) + require.Equal(t, tt.expected, tt.group) + }) + } +} + +func Test_computeRolloutState(t *testing.T) { + tests := []struct { + name string + groups []*autoupdate.AutoUpdateAgentRolloutStatusGroup + expectedState autoupdate.AutoUpdateAgentRolloutState + }{ + { + name: "empty groups", + groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{}, + expectedState: autoupdate.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSPECIFIED, + }, + { + name: "all groups unstarted", + groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + {State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED}, + {State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED}, + {State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED}, + }, + expectedState: autoupdate.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSTARTED, + }, + { + name: "one group active", + groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + {State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE}, + {State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED}, + {State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED}, + }, + expectedState: autoupdate.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_ACTIVE, + }, + { + name: "one group done", + groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + {State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE}, + {State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED}, + {State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED}, + }, + expectedState: autoupdate.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_ACTIVE, + }, + { + name: "every group done", + groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + {State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE}, + {State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE}, + {State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE}, + }, + expectedState: autoupdate.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_DONE, + }, + { + name: "one group rolledback", + groups: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + {State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE}, + {State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK}, + {State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE}, + }, + expectedState: autoupdate.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_ROLLEDBACK, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expectedState, computeRolloutState(tt.groups)) + }) + } +} diff --git a/lib/autoupdate/rollout/strategy_timebased.go b/lib/autoupdate/rollout/strategy_timebased.go new file mode 100644 index 0000000000000..e485144224e3a --- /dev/null +++ b/lib/autoupdate/rollout/strategy_timebased.go @@ -0,0 +1,122 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package rollout + +import ( + "context" + "time" + + "github.com/gravitational/trace" + "github.com/sirupsen/logrus" + + "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + update "github.com/gravitational/teleport/api/types/autoupdate" +) + +const ( + updateReasonInWindow = "in_window" + updateReasonOutsideWindow = "outside_window" +) + +type timeBasedStrategy struct { + log *logrus.Entry +} + +func (h *timeBasedStrategy) name() string { + return update.AgentsStrategyTimeBased +} + +func newTimeBasedStrategy(log *logrus.Entry) (rolloutStrategy, error) { + if log == nil { + return nil, trace.BadParameter("missing log") + } + return &timeBasedStrategy{ + log: log.WithField("strategy", update.AgentsStrategyTimeBased), + }, nil +} + +func (h *timeBasedStrategy) progressRollout(ctx context.Context, spec *autoupdate.AutoUpdateAgentRolloutSpec, status *autoupdate.AutoUpdateAgentRolloutStatus, now time.Time) error { + windowDuration := spec.GetMaintenanceWindowDuration().AsDuration() + // Backward compatibility for resources previously created without duration. + if windowDuration == 0 { + windowDuration = haltOnErrorWindowDuration + } + + // We always process every group regardless of the order. + var errs []error + for _, group := range status.Groups { + switch group.State { + case autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE: + // We start any group unstarted group in window. + // Done groups can transition back to active if they enter their maintenance window again. + // Some agents might have missed the previous windows and might expected to try again. + shouldBeActive, err := inWindow(group, now, windowDuration) + if err != nil { + // In time-based rollouts, groups are not dependent. + // Failing to transition a group should affect other groups. + // We reflect that something went wrong in the status and go to the next group. + setGroupState(group, group.State, updateReasonReconcilerError, now) + errs = append(errs, err) + continue + } + + // Check if the rollout got created after the theoretical group start time + rolloutChangedDuringWindow, err := rolloutChangedInWindow(group, now, status.StartTime.AsTime(), windowDuration) + if err != nil { + setGroupState(group, group.State, updateReasonReconcilerError, now) + errs = append(errs, err) + continue + } + + switch { + case !shouldBeActive: + setGroupState(group, group.State, updateReasonOutsideWindow, now) + case rolloutChangedDuringWindow: + setGroupState(group, group.State, updateReasonRolloutChanged, now) + default: + setGroupState(group, autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, updateReasonInWindow, now) + } + case autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK: + // We don't touch any group that was manually rolled back. + // Something happened and we should not try to update again. + case autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE: + // The group is currently being updated. We check if the maintenance + // is over and if we should transition it to the done state + shouldBeActive, err := inWindow(group, now, windowDuration) + if err != nil { + // In time-based rollouts, groups are not dependent. + // Failing to transition a group should affect other groups. + // We reflect that something went wrong in the status and go to the next group. + setGroupState(group, group.State, updateReasonReconcilerError, now) + errs = append(errs, err) + continue + } + + if shouldBeActive { + setGroupState(group, autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, updateReasonInWindow, now) + } else { + setGroupState(group, autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, updateReasonOutsideWindow, now) + } + default: + return trace.BadParameter("unknown autoupdate group state: %v", group.State) + } + } + return trace.NewAggregate(errs...) +} diff --git a/lib/autoupdate/rollout/strategy_timebased_test.go b/lib/autoupdate/rollout/strategy_timebased_test.go new file mode 100644 index 0000000000000..77615663b4867 --- /dev/null +++ b/lib/autoupdate/rollout/strategy_timebased_test.go @@ -0,0 +1,350 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package rollout + +import ( + "context" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + "github.com/gravitational/teleport/lib/utils" +) + +func Test_progressGroupsTimeBased(t *testing.T) { + clock := clockwork.NewFakeClockAt(testSunday) + log := utils.NewLoggerForTests().WithField("component", "reconciler") + strategy, err := newTimeBasedStrategy(log) + require.NoError(t, err) + + groupName := "test-group" + canStartToday := everyWeekday + cannotStartToday := everyWeekdayButSunday + lastUpdate := timestamppb.New(clock.Now().Add(-5 * time.Minute)) + ctx := context.Background() + + tests := []struct { + name string + initialState []*autoupdate.AutoUpdateAgentRolloutStatusGroup + rolloutStartTime *timestamppb.Timestamp + expectedState []*autoupdate.AutoUpdateAgentRolloutStatusGroup + }{ + { + name: "unstarted -> unstarted", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: lastUpdate, + LastUpdateReason: updateReasonCreated, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonOutsideWindow, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "unstarted -> unstarted because rollout just changed", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: lastUpdate, + LastUpdateReason: updateReasonCreated, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + rolloutStartTime: timestamppb.New(clock.Now()), + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonRolloutChanged, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "unstarted -> active", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: lastUpdate, + LastUpdateReason: updateReasonCreated, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: timestamppb.New(clock.Now()), + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonInWindow, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "done -> done", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + LastUpdateTime: lastUpdate, + LastUpdateReason: updateReasonOutsideWindow, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + LastUpdateTime: lastUpdate, + LastUpdateReason: updateReasonOutsideWindow, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "done -> active", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + LastUpdateTime: lastUpdate, + StartTime: lastUpdate, + LastUpdateReason: updateReasonOutsideWindow, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: timestamppb.New(clock.Now()), + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonInWindow, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "active -> active", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: lastUpdate, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonInWindow, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: lastUpdate, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonInWindow, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "active -> done", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: lastUpdate, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonInWindow, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + StartTime: lastUpdate, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonOutsideWindow, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "rolledback is a dead end", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName + "-in-maintenance", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + LastUpdateTime: lastUpdate, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: groupName + "-out-of-maintenance", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + LastUpdateTime: lastUpdate, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName + "-in-maintenance", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + LastUpdateTime: lastUpdate, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: groupName + "-out-of-maintenance", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + LastUpdateTime: lastUpdate, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + { + name: "mix of everything", + initialState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: "new group should start", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateTime: lastUpdate, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: "done group should start", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + LastUpdateTime: lastUpdate, + StartTime: lastUpdate, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: "rolledback group should do nothing", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + LastUpdateTime: lastUpdate, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: "old group should stop", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + LastUpdateTime: lastUpdate, + StartTime: lastUpdate, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + expectedState: []*autoupdate.AutoUpdateAgentRolloutStatusGroup{ + { + Name: "new group should start", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: timestamppb.New(clock.Now()), + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonInWindow, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: "done group should start", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + StartTime: timestamppb.New(clock.Now()), + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonInWindow, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: "rolledback group should do nothing", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK, + LastUpdateTime: lastUpdate, + ConfigDays: canStartToday, + ConfigStartHour: matchingStartHour, + }, + { + Name: "old group should stop", + State: autoupdate.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + StartTime: lastUpdate, + LastUpdateTime: timestamppb.New(clock.Now()), + LastUpdateReason: updateReasonOutsideWindow, + ConfigDays: cannotStartToday, + ConfigStartHour: matchingStartHour, + }, + }, + }, + } + + spec := &autoupdate.AutoUpdateAgentRolloutSpec{ + MaintenanceWindowDuration: durationpb.New(time.Hour), + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + status := &autoupdate.AutoUpdateAgentRolloutStatus{ + Groups: tt.initialState, + State: 0, + StartTime: tt.rolloutStartTime, + } + err := strategy.progressRollout(ctx, spec, status, clock.Now()) + require.NoError(t, err) + // We use require.Equal instead of Elements match because group order matters. + // It's not super important for time-based, but is crucial for halt-on-error. + // So it's better to be more conservative and validate order never changes for + // both strategies. + require.Equal(t, tt.expectedState, status.Groups) + }) + } +} diff --git a/lib/cache/cache.go b/lib/cache/cache.go index 63843d26f9abe..bf6e0f1d75f9f 100644 --- a/lib/cache/cache.go +++ b/lib/cache/cache.go @@ -170,6 +170,7 @@ func ForAuth(cfg Config) Config { {Kind: types.KindKubeWaitingContainer}, {Kind: types.KindAutoUpdateVersion}, {Kind: types.KindAutoUpdateConfig}, + {Kind: types.KindAutoUpdateAgentRollout}, } cfg.QueueSize = defaults.AuthQueueSize // We don't want to enable partial health for auth cache because auth uses an event stream @@ -224,6 +225,7 @@ func ForProxy(cfg Config) Config { {Kind: types.KindKubeWaitingContainer}, {Kind: types.KindAutoUpdateConfig}, {Kind: types.KindAutoUpdateVersion}, + {Kind: types.KindAutoUpdateAgentRollout}, } cfg.QueueSize = defaults.ProxyQueueSize return cfg @@ -1912,6 +1914,29 @@ func (c *Cache) GetAutoUpdateVersion(ctx context.Context) (*autoupdate.AutoUpdat return rg.reader.GetAutoUpdateVersion(ctx) } +// GetAutoUpdateAgentRollout gets the AutoUpdateAgentRollout from the backend. +func (c *Cache) GetAutoUpdateAgentRollout(ctx context.Context) (*autoupdate.AutoUpdateAgentRollout, error) { + ctx, span := c.Tracer.Start(ctx, "cache/GetAutoUpdateAgentRollout") + defer span.End() + + rg, err := readCollectionCache(c, c.collections.autoUpdateAgentRollouts) + if err != nil { + return nil, trace.Wrap(err) + } + defer rg.Release() + if !rg.IsCacheRead() { + cachedAgentRollout, err := utils.FnCacheGet(ctx, c.fnCache, autoUpdateCacheKey{"rollout"}, func(ctx context.Context) (*autoupdate.AutoUpdateAgentRollout, error) { + version, err := rg.reader.GetAutoUpdateAgentRollout(ctx) + return version, err + }) + if err != nil { + return nil, trace.Wrap(err) + } + return protobuf.Clone(cachedAgentRollout).(*autoupdate.AutoUpdateAgentRollout), nil + } + return rg.reader.GetAutoUpdateAgentRollout(ctx) +} + func (c *Cache) GetUIConfig(ctx context.Context) (types.UIConfig, error) { ctx, span := c.Tracer.Start(ctx, "cache/GetUIConfig") defer span.End() diff --git a/lib/cache/cache_test.go b/lib/cache/cache_test.go index 9f1384bb27d42..a31aa2d686fbd 100644 --- a/lib/cache/cache_test.go +++ b/lib/cache/cache_test.go @@ -2538,6 +2538,42 @@ func TestAutoUpdateVersion(t *testing.T) { }) } +// TestAutoUpdateAgentRollout tests that CRUD operations on AutoUpdateAgentRollout resource are +// replicated from the backend to the cache. +func TestAutoUpdateAgentRollout(t *testing.T) { + t.Parallel() + + p := newTestPack(t, ForAuth) + t.Cleanup(p.Close) + + testResources153(t, p, testFuncs153[*autoupdate.AutoUpdateAgentRollout]{ + newResource: func(name string) (*autoupdate.AutoUpdateAgentRollout, error) { + return newAutoUpdateAgentRollout(t), nil + }, + create: func(ctx context.Context, item *autoupdate.AutoUpdateAgentRollout) error { + _, err := p.autoUpdateService.UpsertAutoUpdateAgentRollout(ctx, item) + return trace.Wrap(err) + }, + list: func(ctx context.Context) ([]*autoupdate.AutoUpdateAgentRollout, error) { + item, err := p.autoUpdateService.GetAutoUpdateAgentRollout(ctx) + if trace.IsNotFound(err) { + return []*autoupdate.AutoUpdateAgentRollout{}, nil + } + return []*autoupdate.AutoUpdateAgentRollout{item}, trace.Wrap(err) + }, + cacheList: func(ctx context.Context) ([]*autoupdate.AutoUpdateAgentRollout, error) { + item, err := p.cache.GetAutoUpdateAgentRollout(ctx) + if trace.IsNotFound(err) { + return []*autoupdate.AutoUpdateAgentRollout{}, nil + } + return []*autoupdate.AutoUpdateAgentRollout{item}, trace.Wrap(err) + }, + deleteAll: func(ctx context.Context) error { + return trace.Wrap(p.autoUpdateService.DeleteAutoUpdateAgentRollout(ctx)) + }, + }) +} + // testResources153 is a generic tester for RFD153-style resources. func testResources153[T types.Resource153](t *testing.T, p *testPack, funcs testFuncs153[T]) { ctx := context.Background() @@ -3111,6 +3147,7 @@ func TestCacheWatchKindExistsInEvents(t *testing.T) { types.KindKubeWaitingContainer: newKubeWaitingContainer(t), types.KindAutoUpdateConfig: types.Resource153ToLegacy(newAutoUpdateConfig(t)), types.KindAutoUpdateVersion: types.Resource153ToLegacy(newAutoUpdateVersion(t)), + types.KindAutoUpdateAgentRollout: types.Resource153ToLegacy(newAutoUpdateAgentRollout(t)), } for name, cfg := range cases { @@ -3583,6 +3620,20 @@ func newAutoUpdateVersion(t *testing.T) *autoupdate.AutoUpdateVersion { return r } +func newAutoUpdateAgentRollout(t *testing.T) *autoupdate.AutoUpdateAgentRollout { + t.Helper() + + r, err := update.NewAutoUpdateAgentRollout(&autoupdate.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "2.3.4", + Schedule: update.AgentsScheduleImmediate, + AutoupdateMode: update.AgentsUpdateModeEnabled, + Strategy: update.AgentsStrategyTimeBased, + }) + require.NoError(t, err) + return r +} + func withKeepalive[T any](fn func(context.Context, T) (*types.KeepAlive, error)) func(context.Context, T) error { return func(ctx context.Context, resource T) error { _, err := fn(ctx, resource) diff --git a/lib/cache/collections.go b/lib/cache/collections.go index 64e99834ac638..58675b67f71f3 100644 --- a/lib/cache/collections.go +++ b/lib/cache/collections.go @@ -234,6 +234,7 @@ type cacheCollections struct { windowsDesktopServices collectionReader[windowsDesktopServiceGetter] autoUpdateConfigs collectionReader[autoUpdateConfigGetter] autoUpdateVersions collectionReader[autoUpdateVersionGetter] + autoUpdateAgentRollouts collectionReader[autoUpdateAgentRolloutGetter] } // setupCollections returns a registry of collections. @@ -693,6 +694,16 @@ func setupCollections(c *Cache, watches []types.WatchKind) (*cacheCollections, e watch: watch, } collections.byKind[resourceKind] = collections.autoUpdateVersions + case types.KindAutoUpdateAgentRollout: + if c.AutoUpdateService == nil { + return nil, trace.BadParameter("missing parameter AutoUpdateService") + } + collections.autoUpdateAgentRollouts = &genericCollection[*autoupdate.AutoUpdateAgentRollout, autoUpdateAgentRolloutGetter, autoUpdateAgentRolloutExecutor]{ + cache: c, + watch: watch, + } + collections.byKind[resourceKind] = collections.autoUpdateAgentRollouts + default: return nil, trace.BadParameter("resource %q is not supported", watch.Kind) } @@ -1268,6 +1279,41 @@ type autoUpdateVersionGetter interface { var _ executor[*autoupdate.AutoUpdateVersion, autoUpdateVersionGetter] = autoUpdateVersionExecutor{} +type autoUpdateAgentRolloutExecutor struct{} + +func (autoUpdateAgentRolloutExecutor) getAll(ctx context.Context, cache *Cache, loadSecrets bool) ([]*autoupdate.AutoUpdateAgentRollout, error) { + plan, err := cache.AutoUpdateService.GetAutoUpdateAgentRollout(ctx) + return []*autoupdate.AutoUpdateAgentRollout{plan}, trace.Wrap(err) +} + +func (autoUpdateAgentRolloutExecutor) upsert(ctx context.Context, cache *Cache, resource *autoupdate.AutoUpdateAgentRollout) error { + _, err := cache.autoUpdateCache.UpsertAutoUpdateAgentRollout(ctx, resource) + return trace.Wrap(err) +} + +func (autoUpdateAgentRolloutExecutor) deleteAll(ctx context.Context, cache *Cache) error { + return cache.autoUpdateCache.DeleteAutoUpdateAgentRollout(ctx) +} + +func (autoUpdateAgentRolloutExecutor) delete(ctx context.Context, cache *Cache, resource types.Resource) error { + return cache.autoUpdateCache.DeleteAutoUpdateAgentRollout(ctx) +} + +func (autoUpdateAgentRolloutExecutor) isSingleton() bool { return true } + +func (autoUpdateAgentRolloutExecutor) getReader(cache *Cache, cacheOK bool) autoUpdateAgentRolloutGetter { + if cacheOK { + return cache.autoUpdateCache + } + return cache.Config.AutoUpdateService +} + +type autoUpdateAgentRolloutGetter interface { + GetAutoUpdateAgentRollout(ctx context.Context) (*autoupdate.AutoUpdateAgentRollout, error) +} + +var _ executor[*autoupdate.AutoUpdateAgentRollout, autoUpdateAgentRolloutGetter] = autoUpdateAgentRolloutExecutor{} + type userExecutor struct{} func (userExecutor) getAll(ctx context.Context, cache *Cache, loadSecrets bool) ([]types.User, error) { diff --git a/lib/config/configuration.go b/lib/config/configuration.go index 042ca7d72a7e8..3a08262896553 100644 --- a/lib/config/configuration.go +++ b/lib/config/configuration.go @@ -2513,6 +2513,14 @@ func Configure(clf *CommandLineFlags, cfg *servicecfg.Config, legacyAppFlags boo } } + if rawPeriod := os.Getenv("TELEPORT_UNSTABLE_AGENT_ROLLOUT_SYNC_PERIOD"); rawPeriod != "" { + period, err := time.ParseDuration(rawPeriod) + if err != nil { + return trace.Wrap(err, "invalid agent rollout period %q", rawPeriod) + } + cfg.Auth.AgentRolloutControllerSyncPeriod = period + } + return nil } diff --git a/lib/kubernetestoken/token_source.go b/lib/kube/token/source.go similarity index 98% rename from lib/kubernetestoken/token_source.go rename to lib/kube/token/source.go index 321e07c8e4022..526c15adb6440 100644 --- a/lib/kubernetestoken/token_source.go +++ b/lib/kube/token/source.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubernetestoken +package token import ( "strings" diff --git a/lib/kubernetestoken/token_source_test.go b/lib/kube/token/source_test.go similarity index 99% rename from lib/kubernetestoken/token_source_test.go rename to lib/kube/token/source_test.go index 0d0a67a613776..a0980acd54e1f 100644 --- a/lib/kubernetestoken/token_source_test.go +++ b/lib/kube/token/source_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubernetestoken +package token import ( "io/fs" diff --git a/lib/kubernetestoken/token_validator.go b/lib/kube/token/validator.go similarity index 99% rename from lib/kubernetestoken/token_validator.go rename to lib/kube/token/validator.go index d3029af88a3f0..21d4936ec1132 100644 --- a/lib/kubernetestoken/token_validator.go +++ b/lib/kube/token/validator.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubernetestoken +package token import ( "context" diff --git a/lib/kubernetestoken/token_validator_test.go b/lib/kube/token/validator_test.go similarity index 99% rename from lib/kubernetestoken/token_validator_test.go rename to lib/kube/token/validator_test.go index 5d27f524b7f21..c1ceff7fdb946 100644 --- a/lib/kubernetestoken/token_validator_test.go +++ b/lib/kube/token/validator_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubernetestoken +package token import ( "context" diff --git a/lib/modules/modules.go b/lib/modules/modules.go index cfe87fdd429ca..7700b242795b9 100644 --- a/lib/modules/modules.go +++ b/lib/modules/modules.go @@ -289,6 +289,8 @@ type Modules interface { } const ( + // BuildCommunity is the Teleport Community Edition build type (only used in v16). + BuildCommunity = "community" // BuildOSS specifies open source build type BuildOSS = "oss" // BuildEnterprise specifies enterprise build type diff --git a/lib/service/service.go b/lib/service/service.go index 4b819577b250c..31e0a9ff4ce1a 100644 --- a/lib/service/service.go +++ b/lib/service/service.go @@ -47,6 +47,7 @@ import ( "github.com/gravitational/roundtrip" "github.com/gravitational/trace" "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -88,6 +89,8 @@ import ( "github.com/gravitational/teleport/lib/auth/storage" "github.com/gravitational/teleport/lib/authz" "github.com/gravitational/teleport/lib/automaticupgrades" + autoupdate "github.com/gravitational/teleport/lib/autoupdate/agent" + "github.com/gravitational/teleport/lib/autoupdate/rollout" "github.com/gravitational/teleport/lib/backend" "github.com/gravitational/teleport/lib/backend/dynamo" "github.com/gravitational/teleport/lib/backend/etcdbk" @@ -442,6 +445,15 @@ type TeleportProcess struct { // resolver is used to identify the reverse tunnel address when connecting via // the proxy. resolver reversetunnelclient.Resolver + + // metricRegistry is the prometheus metric registry for the process. + // Every teleport service that wants to register metrics should use this + // instead of the global prometheus.DefaultRegisterer to avoid registration + // conflicts. + // + // Both the metricsRegistry and the default global registry are gathered by + // Telepeort's metric service. + metricsRegistry *prometheus.Registry } type keyPairKey struct { @@ -856,6 +868,15 @@ func NewTeleport(cfg *servicecfg.Config) (*TeleportProcess, error) { "pid": fmt.Sprintf("%v.%v", os.Getpid(), processID), })) + // Use the custom metrics registry if specified, else create a new one. + // We must create the registry in NewTeleport, as opposed to ApplyConfig(), + // because some tests are running multiple Teleport instances from the same + // config. + metricsRegistry := cfg.MetricsRegistry + if metricsRegistry == nil { + metricsRegistry = prometheus.NewRegistry() + } + // If FIPS mode was requested make sure binary is build against BoringCrypto. if cfg.FIPS { if !modules.GetModules().IsBoringBinary() { @@ -994,6 +1015,7 @@ func NewTeleport(cfg *servicecfg.Config) (*TeleportProcess, error) { keyPairs: make(map[keyPairKey]KeyPair), cloudLabels: cloudLabels, TracingProvider: tracing.NoopProvider(), + metricsRegistry: metricsRegistry, } process.registerExpectedServices(cfg) @@ -1037,18 +1059,7 @@ func NewTeleport(cfg *servicecfg.Config) (*TeleportProcess, error) { return nil, trace.Wrap(err) } - upgraderKind := os.Getenv(automaticupgrades.EnvUpgrader) - upgraderVersion := automaticupgrades.GetUpgraderVersion(process.GracefulExitContext()) - if upgraderVersion == "" { - upgraderKind = "" - } - - // Instances deployed using the AWS OIDC integration are automatically updated - // by the proxy. The instance heartbeat should properly reflect that. - externalUpgrader := upgraderKind - if externalUpgrader == "" && os.Getenv(types.InstallMethodAWSOIDCDeployServiceEnvVar) == "true" { - externalUpgrader = types.OriginIntegrationAWSOIDC - } + upgraderKind, externalUpgrader, upgraderVersion := process.detectUpgrader() // note: we must create the inventory handle *after* registerExpectedServices because that function determines // the list of services (instance roles) to be included in the heartbeat. @@ -1077,7 +1088,10 @@ func NewTeleport(cfg *servicecfg.Config) (*TeleportProcess, error) { process.log.Warnf("Use of external upgraders on control-plane instances is not recommended.") } - if upgraderKind == "unit" { + switch upgraderKind { + case types.UpgraderKindTeleportUpdate: + // Exports are not required for teleport-update + case types.UpgraderKindSystemdUnit: process.RegisterFunc("autoupdates.endpoint.export", func() error { component := teleport.Component("autoupdates:endpoint:export", process.id) logger := process.log.WithFields(logrus.Fields{ @@ -1106,28 +1120,14 @@ func NewTeleport(cfg *servicecfg.Config) (*TeleportProcess, error) { logger.Infof("Exported autoupdates endpoint (addr=%s).", resolverAddr.String()) return nil }) + if err := process.configureUpgraderExporter(upgraderKind); err != nil { + return nil, trace.Wrap(err) + } + default: + if err := process.configureUpgraderExporter(upgraderKind); err != nil { + return nil, trace.Wrap(err) + } } - - driver, err := uw.NewDriver(upgraderKind) - if err != nil { - return nil, trace.Wrap(err) - } - - exporter, err := uw.NewExporter(uw.ExporterConfig[inventory.DownstreamSender]{ - Driver: driver, - ExportFunc: process.exportUpgradeWindows, - AuthConnectivitySentinel: process.inventoryHandle.Sender(), - }) - if err != nil { - return nil, trace.Wrap(err) - } - - process.RegisterCriticalFunc("upgradeewindow.export", exporter.Run) - process.OnExit("upgradewindow.export.stop", func(_ interface{}) { - exporter.Close() - }) - - process.log.Infof("Configured upgrade window exporter for external upgrader. kind=%s", upgraderKind) } if process.Config.Proxy.Enabled { @@ -1336,6 +1336,63 @@ func NewTeleport(cfg *servicecfg.Config) (*TeleportProcess, error) { return process, nil } +// detectUpgrader returns metadata about auto-upgraders that may be active. +// Note that kind and externalName are usually the same. +// However, some unregistered upgraders like the AWS ODIC upgrader are not valid kinds. +// For these upgraders, kind is empty and externalName is set to a non-kind value. +func (process *TeleportProcess) detectUpgrader() (kind, externalName, version string) { + // Check if the deprecated teleport-upgrader script is being used. + kind = os.Getenv(automaticupgrades.EnvUpgrader) + version = automaticupgrades.GetUpgraderVersion(process.GracefulExitContext()) + if version == "" { + kind = "" + } + + // If the installation is managed by teleport-update, it supersedes the teleport-upgrader script. + ok, err := autoupdate.IsManagedByUpdater() + if err != nil { + process.log.WithError(err).Warn("Failed to determine if auto-updates are enabled.") + } else if ok { + // If this is a teleport-update managed installation, the version + // managed by the timer will always match the installed version of teleport. + kind = types.UpgraderKindTeleportUpdate + version = "v" + teleport.Version + } + + // Instances deployed using the AWS OIDC integration are automatically updated + // by the proxy. The instance heartbeat should properly reflect that. + externalName = kind + if externalName == "" && os.Getenv(types.InstallMethodAWSOIDCDeployServiceEnvVar) == "true" { + externalName = types.OriginIntegrationAWSOIDC + } + return kind, externalName, version +} + +// configureUpgraderExporter configures the window exporter for upgraders that export windows. +func (process *TeleportProcess) configureUpgraderExporter(kind string) error { + driver, err := uw.NewDriver(kind) + if err != nil { + return trace.Wrap(err) + } + + exporter, err := uw.NewExporter(uw.ExporterConfig[inventory.DownstreamSender]{ + Driver: driver, + ExportFunc: process.exportUpgradeWindows, + AuthConnectivitySentinel: process.inventoryHandle.Sender(), + }) + if err != nil { + return trace.Wrap(err) + } + + process.RegisterCriticalFunc("upgradeewindow.export", exporter.Run) + process.OnExit("upgradewindow.export.stop", func(_ interface{}) { + exporter.Close() + }) + + process.log.WithField("kind", kind).Info("Configured upgrade window exporter for external upgrader.") + return nil +} + // enterpriseServicesEnabled will return true if any enterprise services are enabled. func (process *TeleportProcess) enterpriseServicesEnabled() bool { return modules.GetModules().BuildType() == modules.BuildEnterprise && @@ -2294,6 +2351,14 @@ func (process *TeleportProcess) initAuthService() error { } process.RegisterFunc("auth.heartbeat", heartbeat.Run) + agentRolloutController, err := rollout.NewController(authServer, log, process.Clock, cfg.Auth.AgentRolloutControllerSyncPeriod, process.metricsRegistry) + if err != nil { + return trace.Wrap(err, "creating the rollout controller") + } + process.RegisterFunc("auth.autoupdate_agent_rollout_controller", func() error { + return trace.Wrap(agentRolloutController.Run(process.GracefulExitContext()), "running autoupdate_agent_rollout controller") + }) + process.RegisterFunc("auth.server_info", func() error { return trace.Wrap(authServer.ReconcileServerInfos(process.GracefulExitContext())) }) @@ -3171,11 +3236,23 @@ func (process *TeleportProcess) initUploaderService() error { return nil } +// promHTTPLogAdapter adapts a slog.Logger into a promhttp.Logger. +type promHTTPLogAdapter struct { + ctx context.Context + *logrus.Entry +} + +// Println implements the promhttp.Logger interface. +func (l promHTTPLogAdapter) Println(v ...interface{}) { + //nolint:sloglint // msg cannot be constant + l.Error(v...) +} + // initMetricsService starts the metrics service currently serving metrics for // prometheus consumption func (process *TeleportProcess) initMetricsService() error { mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) + mux.Handle("/metrics", process.newMetricsHandler()) log := process.log.WithFields(logrus.Fields{ trace.Component: teleport.Component(teleport.ComponentMetrics, process.id), @@ -3263,6 +3340,35 @@ func (process *TeleportProcess) initMetricsService() error { return nil } +// newMetricsHandler creates a new metrics handler serving metrics both from the global prometheus registry and the +// in-process one. +func (process *TeleportProcess) newMetricsHandler() http.Handler { + // We gather metrics both from the in-process registry (preferred metrics registration method) + // and the global registry (used by some Teleport services and many dependencies). + gatherers := prometheus.Gatherers{ + process.metricsRegistry, + prometheus.DefaultGatherer, + } + + metricsHandler := promhttp.InstrumentMetricHandler( + process.metricsRegistry, promhttp.HandlerFor(gatherers, promhttp.HandlerOpts{ + // Errors can happen if metrics are registered with identical names in both the local and the global registry. + // In this case, we log the error but continue collecting metrics. The first collected metric will win + // (the one from the local metrics registry takes precedence). + // As we move more things to the local registry, especially in other tools like tbot, we will have less + // conflicts in tests. + ErrorHandling: promhttp.ContinueOnError, + ErrorLog: promHTTPLogAdapter{ + ctx: process.ExitContext(), + Entry: process.log.WithFields(logrus.Fields{ + trace.Component: teleport.ComponentMetrics, + }), + }, + }), + ) + return metricsHandler +} + // initDiagnosticService starts diagnostic service currently serving healthz // and prometheus endpoints func (process *TeleportProcess) initDiagnosticService() error { @@ -3272,7 +3378,7 @@ func (process *TeleportProcess) initDiagnosticService() error { // metrics will otherwise be served by the metrics service if it's enabled // in the config. if !process.Config.Metrics.Enabled { - mux.Handle("/metrics", promhttp.Handler()) + mux.Handle("/metrics", process.newMetricsHandler()) } if process.Config.Debug { diff --git a/lib/service/service_test.go b/lib/service/service_test.go index a0a01065f47a6..b1408b20a5436 100644 --- a/lib/service/service_test.go +++ b/lib/service/service_test.go @@ -20,8 +20,10 @@ import ( "crypto/tls" "errors" "fmt" + "io" "net" "net/http" + "net/url" "os" "path/filepath" "strings" @@ -35,7 +37,9 @@ import ( "github.com/google/uuid" "github.com/gravitational/trace" "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/crypto/ssh" "google.golang.org/grpc" @@ -44,7 +48,9 @@ import ( "github.com/gravitational/teleport" "github.com/gravitational/teleport/api/breaker" + autoupdatepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/api/types/autoupdate" "github.com/gravitational/teleport/lib" "github.com/gravitational/teleport/lib/auth" "github.com/gravitational/teleport/lib/auth/authclient" @@ -1591,3 +1597,229 @@ func TestSingleProcessModeResolver(t *testing.T) { }) } } + +// TestMetricsService tests that the optional metrics service exposes +// metrics from both the in-process and global metrics registry. When the +// service is disabled, metrics are served by the diagnostics service +// (tested in TestMetricsInDiagnosticsService). +func TestMetricsService(t *testing.T) { + t.Parallel() + // Test setup: create a listener for the metrics server, get its file descriptor. + + // Note: this code is copied from integrations/helpers/NewListenerOn() to avoid including helpers in a production + // build and avoid a cyclic dependency. + metricsListener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, metricsListener.Close()) + }) + require.IsType(t, &net.TCPListener{}, metricsListener) + metricsListenerFile, err := metricsListener.(*net.TCPListener).File() + require.NoError(t, err) + + // Test setup: create a new teleport process + dataDir := makeTempDir(t) + cfg := servicecfg.MakeDefaultConfig() + cfg.DataDir = dataDir + cfg.SetAuthServerAddress(utils.NetAddr{AddrNetwork: "tcp", Addr: "127.0.0.1:0"}) + cfg.Auth.Enabled = true + cfg.Proxy.Enabled = false + cfg.SSH.Enabled = false + cfg.Auth.StorageConfig.Params["path"] = dataDir + cfg.Auth.ListenAddr = utils.NetAddr{AddrNetwork: "tcp", Addr: "127.0.0.1:0"} + cfg.Metrics.Enabled = true + + // Configure the metrics server to use the listener we previously created. + cfg.Metrics.ListenAddr = &utils.NetAddr{AddrNetwork: "tcp", Addr: metricsListener.Addr().String()} + cfg.FileDescriptors = []*servicecfg.FileDescriptor{ + {Type: string(ListenerMetrics), Address: metricsListener.Addr().String(), File: metricsListenerFile}, + } + + // Create and start the Teleport service. + process, err := NewTeleport(cfg) + require.NoError(t, err) + require.NoError(t, process.Start()) + t.Cleanup(func() { + assert.NoError(t, process.Close()) + assert.NoError(t, process.Wait()) + }) + + // Test setup: create our test metrics. + nonce := strings.ReplaceAll(uuid.NewString(), "-", "") + localMetric := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "test", + Name: "local_metric_" + nonce, + }) + globalMetric := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "test", + Name: "global_metric_" + nonce, + }) + require.NoError(t, process.metricsRegistry.Register(localMetric)) + require.NoError(t, prometheus.Register(globalMetric)) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + t.Cleanup(cancel) + _, err = process.WaitForEvent(ctx, MetricsReady) + require.NoError(t, err) + + // Test execution: get metrics and check the tests metrics are here. + metricsURL, err := url.Parse("http://" + metricsListener.Addr().String()) + require.NoError(t, err) + metricsURL.Path = "/metrics" + resp, err := http.Get(metricsURL.String()) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + + // Test validation: check that the metrics server served both the local and global registry. + require.Contains(t, string(body), "local_metric_"+nonce) + require.Contains(t, string(body), "global_metric_"+nonce) +} + +// TestMetricsInDiagnosticsService tests that the diagnostics service exposes +// metrics from both the in-process and global metrics registry when the metrics +// service is disabled. +func TestMetricsInDiagnosticsService(t *testing.T) { + t.Parallel() + // Test setup: create a new teleport process + dataDir := makeTempDir(t) + cfg := servicecfg.MakeDefaultConfig() + cfg.DataDir = dataDir + cfg.SetAuthServerAddress(utils.NetAddr{AddrNetwork: "tcp", Addr: "127.0.0.1:0"}) + cfg.Auth.Enabled = true + cfg.Proxy.Enabled = false + cfg.SSH.Enabled = false + cfg.Auth.StorageConfig.Params["path"] = dataDir + cfg.Auth.ListenAddr = utils.NetAddr{AddrNetwork: "tcp", Addr: "127.0.0.1:0"} + cfg.DiagnosticAddr = utils.NetAddr{AddrNetwork: "tcp", Addr: "127.0.0.1:0"} + + // Test setup: Create and start the Teleport service. + process, err := NewTeleport(cfg) + require.NoError(t, err) + require.NoError(t, process.Start()) + t.Cleanup(func() { + assert.NoError(t, process.Close()) + assert.NoError(t, process.Wait()) + }) + + // Test setup: create our test metrics. + nonce := strings.ReplaceAll(uuid.NewString(), "-", "") + localMetric := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "test", + Name: "local_metric_" + nonce, + }) + globalMetric := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "test", + Name: "global_metric_" + nonce, + }) + require.NoError(t, process.metricsRegistry.Register(localMetric)) + require.NoError(t, prometheus.Register(globalMetric)) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + t.Cleanup(cancel) + _, err = process.WaitForEvent(ctx, TeleportReadyEvent) + require.NoError(t, err) + + // Test execution: query the metrics endpoint and check the tests metrics are here. + diagAddr, err := process.DiagnosticAddr() + require.NoError(t, err) + metricsURL, err := url.Parse("http://" + diagAddr.String()) + require.NoError(t, err) + metricsURL.Path = "/metrics" + resp, err := http.Get(metricsURL.String()) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + + // Test validation: check that the metrics server served both the local and global registry. + require.Contains(t, string(body), "local_metric_"+nonce) + require.Contains(t, string(body), "global_metric_"+nonce) +} + +// makeTempDir makes a temp dir with a shorter name than t.TempDir() in order to +// avoid https://github.com/golang/go/issues/62614. +func makeTempDir(t *testing.T) string { + t.Helper() + + tempDir, err := os.MkdirTemp("", "teleport-test-") + require.NoError(t, err, "os.MkdirTemp() failed") + t.Cleanup(func() { os.RemoveAll(tempDir) }) + return tempDir +} + +// TestAgentRolloutController validates that the agent rollout controller is started +// when we run the Auth Service. It does so by creating a dummy autoupdate_version resource +// and checking that the corresponding autoupdate_agent_rollout resource is created by the auth. +// If you want to test the reconciliation logic, add tests to the rolloutcontroller package instead. +func TestAgentRolloutController(t *testing.T) { + t.Parallel() + + dataDir := makeTempDir(t) + + cfg := servicecfg.MakeDefaultConfig() + // We use a real clock because too many sevrices are using the clock and it's not possible to accurately wait for + // each one of them to reach the point where they wait for the clock to advance. If we add a WaitUntil(X waiters) + // check, this will break the next time we add a new waiter. + cfg.Clock = clockwork.NewRealClock() + cfg.DataDir = dataDir + cfg.SetAuthServerAddress(utils.NetAddr{AddrNetwork: "tcp", Addr: "127.0.0.1:0"}) + cfg.Auth.Enabled = true + cfg.Proxy.Enabled = false + cfg.SSH.Enabled = false + cfg.Auth.StorageConfig.Params["path"] = dataDir + cfg.Auth.ListenAddr = utils.NetAddr{AddrNetwork: "tcp", Addr: "127.0.0.1:0"} + // Speed up the reconciliation period for testing purposes. + cfg.Auth.AgentRolloutControllerSyncPeriod = 200 * time.Millisecond + cfg.CircuitBreakerConfig = breaker.NoopBreakerConfig() + + process, err := NewTeleport(cfg) + require.NoError(t, err) + + // Test setup: start the Teleport auth and wait for it to beocme ready + require.NoError(t, process.Start()) + + // Test setup: wait for every service to start + ctx, cancel := context.WithTimeout(process.ExitContext(), 30*time.Second) + defer cancel() + for _, eventName := range []string{AuthTLSReady, InstanceReady} { + _, err := process.WaitForEvent(ctx, eventName) + require.NoError(t, err) + } + + // Test cleanup: close the Teleport process and wait for every service to exist before returning. + // This ensures that a service will not make the test fail by writing a file to the temporary directory while it's + // being removed. + t.Cleanup(func() { + require.NoError(t, process.Close()) + require.NoError(t, process.Wait()) + }) + + // Test execution: create the autoupdate_version resource + authServer := process.GetAuthServer() + version, err := autoupdate.NewAutoUpdateVersion(&autoupdatepb.AutoUpdateVersionSpec{ + Agents: &autoupdatepb.AutoUpdateVersionSpecAgents{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: autoupdate.AgentsScheduleImmediate, + Mode: autoupdate.AgentsUpdateModeEnabled, + }, + }) + require.NoError(t, err) + version, err = authServer.CreateAutoUpdateVersion(ctx, version) + require.NoError(t, err) + + // Test validation: check that a new autoupdate_agent_rollout config was created + require.Eventually(t, func() bool { + rollout, err := authServer.GetAutoUpdateAgentRollout(ctx) + if err != nil { + return false + } + return rollout.Spec.GetTargetVersion() == version.Spec.GetAgents().GetTargetVersion() + }, 5*time.Second, 10*time.Millisecond) +} diff --git a/lib/service/servicecfg/auth.go b/lib/service/servicecfg/auth.go index f76c0842f031b..d8e1219ec2368 100644 --- a/lib/service/servicecfg/auth.go +++ b/lib/service/servicecfg/auth.go @@ -15,6 +15,8 @@ package servicecfg import ( + "time" + "github.com/coreos/go-oidc/oauth2" "github.com/dustin/go-humanize" "github.com/gravitational/trace" @@ -115,6 +117,12 @@ type AuthConfig struct { // AccessMonitoring configures access monitoring. AccessMonitoring *AccessMonitoringOptions + + // AgentRolloutControllerSyncPeriod controls the period between two + // reconciliations of the agent rollout controller. This value is jittered. + // Empty value means the controller uses its default. + // Used in tests. + AgentRolloutControllerSyncPeriod time.Duration } // AccessMonitoringOptions configures access monitoring. diff --git a/lib/service/servicecfg/config.go b/lib/service/servicecfg/config.go index fb4c5ada42ba9..48efa308d3dab 100644 --- a/lib/service/servicecfg/config.go +++ b/lib/service/servicecfg/config.go @@ -27,6 +27,7 @@ import ( "github.com/ghodss/yaml" "github.com/gravitational/trace" "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" "github.com/sashabaranov/go-openai" "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" @@ -247,6 +248,12 @@ type Config struct { // AccessGraph represents AccessGraph server config AccessGraph AccessGraphConfig + // MetricsRegistry is the prometheus metrics registry used by the Teleport process to register its metrics. + // As of today, not every Teleport metric is registered against this registry. Some Teleport services + // and Teleport dependencies are using the global registry. + // Both the MetricsRegistry and the default global registry are gathered by Teleport's metric service. + MetricsRegistry *prometheus.Registry + // token is either the token needed to join the auth server, or a path pointing to a file // that contains the token // diff --git a/lib/services/autoupdates.go b/lib/services/autoupdates.go index 5fa7a4eed4677..f57d384df2dde 100644 --- a/lib/services/autoupdates.go +++ b/lib/services/autoupdates.go @@ -31,6 +31,9 @@ type AutoUpdateServiceGetter interface { // GetAutoUpdateVersion gets the AutoUpdateVersion singleton resource. GetAutoUpdateVersion(ctx context.Context) (*autoupdate.AutoUpdateVersion, error) + + // GetAutoUpdateAgentRollout gets the AutoUpdateAgentRollout singleton resource. + GetAutoUpdateAgentRollout(ctx context.Context) (*autoupdate.AutoUpdateAgentRollout, error) } // AutoUpdateService stores the autoupdate service. @@ -60,4 +63,16 @@ type AutoUpdateService interface { // DeleteAutoUpdateVersion deletes the AutoUpdateVersion singleton resource. DeleteAutoUpdateVersion(ctx context.Context) error + + // CreateAutoUpdateAgentRollout creates the AutoUpdateAgentRollout singleton resource. + CreateAutoUpdateAgentRollout(ctx context.Context, rollout *autoupdate.AutoUpdateAgentRollout) (*autoupdate.AutoUpdateAgentRollout, error) + + // UpdateAutoUpdateAgentRollout updates the AutoUpdateAgentRollout singleton resource. + UpdateAutoUpdateAgentRollout(ctx context.Context, rollout *autoupdate.AutoUpdateAgentRollout) (*autoupdate.AutoUpdateAgentRollout, error) + + // UpsertAutoUpdateAgentRollout sets the AutoUpdateAgentRollout singleton resource. + UpsertAutoUpdateAgentRollout(ctx context.Context, rollout *autoupdate.AutoUpdateAgentRollout) (*autoupdate.AutoUpdateAgentRollout, error) + + // DeleteAutoUpdateAgentRollout deletes the AutoUpdateAgentRollout singleton resource. + DeleteAutoUpdateAgentRollout(ctx context.Context) error } diff --git a/lib/services/github.go b/lib/services/github.go index e5dbeb6f73d7b..b147b055725bc 100644 --- a/lib/services/github.go +++ b/lib/services/github.go @@ -146,7 +146,15 @@ func unmarshalGithubConnector(bytes []byte) (types.GithubConnector, error) { // MarshalGithubConnector marshals the GithubConnector resource to JSON. func MarshalGithubConnector(connector types.GithubConnector, opts ...MarshalOption) ([]byte, error) { - return MarshalResource(connector, opts...) + marshal, ok := getResourceMarshaler(connector.GetKind()) + if !ok { + return nil, trace.NotImplemented("cannot dynamically marshal resources of kind %q", connector.GetKind()) + } + m, err := marshal(connector, opts...) + if err != nil { + return nil, trace.Wrap(err) + } + return m, nil } func marshalGithubConnector(githubConnector types.GithubConnector, opts ...MarshalOption) ([]byte, error) { diff --git a/lib/services/local/autoupdate.go b/lib/services/local/autoupdate.go index f6e6a23abd2b1..dfe479de0ff71 100644 --- a/lib/services/local/autoupdate.go +++ b/lib/services/local/autoupdate.go @@ -32,14 +32,16 @@ import ( ) const ( - autoUpdateConfigPrefix = "auto_update_config" - autoUpdateVersionPrefix = "auto_update_version" + autoUpdateConfigPrefix = "auto_update_config" + autoUpdateVersionPrefix = "auto_update_version" + autoUpdateAgentRolloutPrefix = "auto_update_agent_rollout" ) // AutoUpdateService is responsible for managing AutoUpdateConfig and AutoUpdateVersion singleton resources. type AutoUpdateService struct { config *generic.ServiceWrapper[*autoupdate.AutoUpdateConfig] version *generic.ServiceWrapper[*autoupdate.AutoUpdateVersion] + rollout *generic.ServiceWrapper[*autoupdate.AutoUpdateAgentRollout] } // NewAutoUpdateService returns a new AutoUpdateService. @@ -74,10 +76,26 @@ func NewAutoUpdateService(backend backend.Backend) (*AutoUpdateService, error) { if err != nil { return nil, trace.Wrap(err) } + rollout, err := generic.NewServiceWrapper( + generic.ServiceWrapperConfig[*autoupdate.AutoUpdateAgentRollout]{ + Backend: backend, + ResourceKind: types.KindAutoUpdateAgentRollout, + BackendPrefix: autoUpdateAgentRolloutPrefix, + MarshalFunc: services.MarshalProtoResource[*autoupdate.AutoUpdateAgentRollout], + UnmarshalFunc: services.UnmarshalProtoResource[*autoupdate.AutoUpdateAgentRollout], + ValidateFunc: update.ValidateAutoUpdateAgentRollout, + KeyFunc: func(_ *autoupdate.AutoUpdateAgentRollout) string { + return types.MetaNameAutoUpdateAgentRollout + }, + }) + if err != nil { + return nil, trace.Wrap(err) + } return &AutoUpdateService{ config: config, version: version, + rollout: rollout, }, nil } @@ -156,3 +174,41 @@ func (s *AutoUpdateService) GetAutoUpdateVersion(ctx context.Context) (*autoupda func (s *AutoUpdateService) DeleteAutoUpdateVersion(ctx context.Context) error { return trace.Wrap(s.version.DeleteResource(ctx, types.MetaNameAutoUpdateVersion)) } + +// CreateAutoUpdateAgentRollout creates the AutoUpdateAgentRollout singleton resource. +func (s *AutoUpdateService) CreateAutoUpdateAgentRollout( + ctx context.Context, + v *autoupdate.AutoUpdateAgentRollout, +) (*autoupdate.AutoUpdateAgentRollout, error) { + rollout, err := s.rollout.CreateResource(ctx, v) + return rollout, trace.Wrap(err) +} + +// UpdateAutoUpdateAgentRollout updates the AutoUpdateAgentRollout singleton resource. +func (s *AutoUpdateService) UpdateAutoUpdateAgentRollout( + ctx context.Context, + v *autoupdate.AutoUpdateAgentRollout, +) (*autoupdate.AutoUpdateAgentRollout, error) { + rollout, err := s.rollout.UpdateResource(ctx, v) + return rollout, trace.Wrap(err) +} + +// UpsertAutoUpdateAgentRollout sets the AutoUpdateAgentRollout singleton resource. +func (s *AutoUpdateService) UpsertAutoUpdateAgentRollout( + ctx context.Context, + v *autoupdate.AutoUpdateAgentRollout, +) (*autoupdate.AutoUpdateAgentRollout, error) { + rollout, err := s.rollout.UpsertResource(ctx, v) + return rollout, trace.Wrap(err) +} + +// GetAutoUpdateAgentRollout gets the AutoUpdateAgentRollout singleton resource. +func (s *AutoUpdateService) GetAutoUpdateAgentRollout(ctx context.Context) (*autoupdate.AutoUpdateAgentRollout, error) { + rollout, err := s.rollout.GetResource(ctx, types.MetaNameAutoUpdateAgentRollout) + return rollout, trace.Wrap(err) +} + +// DeleteAutoUpdateAgentRollout deletes the AutoUpdateAgentRollout singleton resource. +func (s *AutoUpdateService) DeleteAutoUpdateAgentRollout(ctx context.Context) error { + return trace.Wrap(s.rollout.DeleteResource(ctx, types.MetaNameAutoUpdateAgentRollout)) +} diff --git a/lib/services/local/autoupdate_test.go b/lib/services/local/autoupdate_test.go index 92ec037fbc5a6..ccd85531d0d34 100644 --- a/lib/services/local/autoupdate_test.go +++ b/lib/services/local/autoupdate_test.go @@ -92,8 +92,7 @@ func TestAutoUpdateServiceConfigCRUD(t *testing.T) { var notFoundError *trace.NotFoundError require.ErrorAs(t, err, ¬FoundError) - _, err = service.UpdateAutoUpdateConfig(ctx, config) - require.ErrorAs(t, err, ¬FoundError) + // In the v14 backport, we removed conditional updates (the backend primitives are not available). } // TestAutoUpdateServiceVersionCRUD verifies get/create/update/upsert/delete methods of the backend service @@ -153,8 +152,7 @@ func TestAutoUpdateServiceVersionCRUD(t *testing.T) { var notFoundError *trace.NotFoundError require.ErrorAs(t, err, ¬FoundError) - _, err = service.UpdateAutoUpdateVersion(ctx, version) - require.ErrorAs(t, err, ¬FoundError) + // In the v14 backport, we removed conditional updates (the backend primitives are not available). } // TestAutoUpdateServiceInvalidNameCreate verifies that configuration and version diff --git a/lib/services/local/events.go b/lib/services/local/events.go index ce90af3616e65..3add048277011 100644 --- a/lib/services/local/events.go +++ b/lib/services/local/events.go @@ -88,6 +88,8 @@ func (e *EventsService) NewWatcher(ctx context.Context, watch types.Watch) (type parser = newAutoUpdateConfigParser() case types.KindAutoUpdateVersion: parser = newAutoUpdateVersionParser() + case types.KindAutoUpdateAgentRollout: + parser = newAutoUpdateAgentRolloutParser() case types.KindNamespace: parser = newNamespaceParser(kind.Name) case types.KindRole: @@ -737,6 +739,41 @@ func (p *autoUpdateVersionParser) parse(event backend.Event) (types.Resource, er } } +func newAutoUpdateAgentRolloutParser() *autoUpdateAgentRolloutParser { + return &autoUpdateAgentRolloutParser{ + baseParser: newBaseParser(backend.NewKey(autoUpdateAgentRolloutPrefix)), + } +} + +type autoUpdateAgentRolloutParser struct { + baseParser +} + +func (p *autoUpdateAgentRolloutParser) parse(event backend.Event) (types.Resource, error) { + switch event.Type { + case types.OpDelete: + return &types.ResourceHeader{ + Kind: types.KindAutoUpdateAgentRollout, + Version: types.V1, + Metadata: types.Metadata{ + Name: types.MetaNameAutoUpdateAgentRollout, + Namespace: apidefaults.Namespace, + }, + }, nil + case types.OpPut: + autoUpdateAgentRollout, err := services.UnmarshalProtoResource[*autoupdate.AutoUpdateAgentRollout](event.Item.Value, + services.WithExpires(event.Item.Expires), + services.WithRevision(event.Item.Revision), + ) + if err != nil { + return nil, trace.Wrap(err) + } + return types.Resource153ToLegacy(autoUpdateAgentRollout), nil + default: + return nil, trace.BadParameter("event %v is not supported", event.Type) + } +} + func newNamespaceParser(name string) *namespaceParser { prefix := backend.NewKey(namespacesPrefix) if name != "" { diff --git a/lib/services/presets.go b/lib/services/presets.go index 56e4327c62673..8a57e38b69f50 100644 --- a/lib/services/presets.go +++ b/lib/services/presets.go @@ -171,6 +171,7 @@ func NewPresetEditorRole() types.Role { types.NewRule(types.KindServerInfo, RW()), types.NewRule(types.KindAutoUpdateVersion, RW()), types.NewRule(types.KindAutoUpdateConfig, RW()), + types.NewRule(types.KindAutoUpdateAgentRollout, RO()), }, }, }, diff --git a/lib/services/resource.go b/lib/services/resource.go index 95e61618270b4..fd6eee72cbabc 100644 --- a/lib/services/resource.go +++ b/lib/services/resource.go @@ -233,6 +233,8 @@ func ParseShortcut(in string) (string, error) { return types.KindAutoUpdateConfig, nil case types.KindAutoUpdateVersion: return types.KindAutoUpdateVersion, nil + case types.KindAutoUpdateAgentRollout: + return types.KindAutoUpdateAgentRollout, nil } return "", trace.BadParameter("unsupported resource: %q - resources should be expressed as 'type/name', for example 'connector/github'", in) } @@ -660,35 +662,6 @@ func CheckAndSetDefaults(r any) error { return nil } -// MarshalResource attempts to marshal a resource dynamically, returning NotImplementedError -// if no marshaler has been registered. -// -// NOTE: This function only supports the subset of resources which may be imported/exported -// by users (e.g. via `tctl get`). -func MarshalResource(resource types.Resource, opts ...MarshalOption) ([]byte, error) { - if err := resource.CheckAndSetDefaults(); err != nil { - return nil, trace.Wrap(err) - } - - marshal, ok := getResourceMarshaler(resource.GetKind()) - if !ok { - return nil, trace.NotImplemented("cannot dynamically marshal resources of kind %q", resource.GetKind()) - } - // Handle the case where `resource` was never fully unmarshaled. - if r, ok := resource.(*UnknownResource); ok { - u, err := UnmarshalResource(r.GetKind(), r.Raw, opts...) - if err != nil { - return nil, trace.Wrap(err) - } - resource = u - } - m, err := marshal(resource, opts...) - if err != nil { - return nil, trace.Wrap(err) - } - return m, nil -} - // UnmarshalResource attempts to unmarshal a resource dynamically, returning NotImplementedError // if no unmarshaler has been registered. // @@ -706,22 +679,26 @@ func UnmarshalResource(kind string, raw []byte, opts ...MarshalOption) (types.Re return u, nil } +type MetadataWithRawID struct { + ID json.RawMessage `json:"id,omitempty"` + types.Metadata +} + // UnknownResource is used to detect resources type UnknownResource struct { - types.ResourceHeader + Kind string `json:"kind,omitempty"` + MetadataWithRawID `json:"metadata,omitempty"` // Raw is raw representation of the resource - Raw []byte + Raw []byte `json:"-"` } // UnmarshalJSON unmarshals header and captures raw state func (u *UnknownResource) UnmarshalJSON(raw []byte) error { - var h types.ResourceHeader - if err := json.Unmarshal(raw, &h); err != nil { + type rawUnknownResource UnknownResource + if err := json.Unmarshal(raw, (*rawUnknownResource)(u)); err != nil { return trace.Wrap(err) } - u.Raw = make([]byte, len(raw)) - u.ResourceHeader = h - copy(u.Raw, raw) + u.Raw = append([]byte(nil), raw...) return nil } diff --git a/lib/utils/teleportassets/teleportassets.go b/lib/utils/teleportassets/teleportassets.go new file mode 100644 index 0000000000000..302977a8bd9ef --- /dev/null +++ b/lib/utils/teleportassets/teleportassets.go @@ -0,0 +1,99 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package teleportassets + +import ( + "fmt" + + "github.com/coreos/go-semver/semver" + + "github.com/gravitational/teleport" + "github.com/gravitational/teleport/lib/modules" +) + +const ( + // TeleportReleaseCDN is the Teleport CDN URL for release builds. + // This can be used to download the Teleport binary for release builds. + TeleportReleaseCDN = "https://cdn.teleport.dev" + // teleportPreReleaseCDN is the Teleport CDN URL for pre-release builds. + // This can be used to download the Teleport binary for pre-release builds. + teleportPreReleaseCDN = "https://cdn.cloud.gravitational.io" +) + +// CDNBaseURL returns the URL of the CDN that can be used to download Teleport +// binary assets. +func CDNBaseURL() string { + return cdnBaseURL(*teleport.SemVersion) +} + +// cdnBaseURL returns the base URL of the CDN that can be used to download +// Teleport binary assets. +func cdnBaseURL(version semver.Version) string { + if version.PreRelease != "" { + return teleportPreReleaseCDN + } + return TeleportReleaseCDN +} + +// CDNBaseURLForVersion returns the CDN base URL for a given artifact version. +// This function ensures that a Teleport production build cannot download from +// the pre-release CDN while Teleport pre-release builds can download both form +// the production and pre-release CDN. +func CDNBaseURLForVersion(artifactVersion *semver.Version) string { + return cdnBaseURLForVersion(artifactVersion, teleport.SemVersion) +} + +func cdnBaseURLForVersion(artifactVersion, teleportVersion *semver.Version) string { + if teleportVersion.PreRelease != "" && artifactVersion.PreRelease != "" { + return teleportPreReleaseCDN + } + return TeleportReleaseCDN +} + +const ( + // teleportReleaseECR is the official release repo for Teleport images. + teleportReleaseECR = "public.ecr.aws/gravitational" + // teleportReleaseECR is the pre-release repo for Teleport images. + teleportPreReleaseECR = "public.ecr.aws/gravitational-staging" + // distrolessTeleportOSSImage is the distroless image of the OSS version of Teleport + distrolessTeleportOSSImage = "teleport-distroless" + // distrolessTeleportEntImage is the distroless image of the Enterprise version of Teleport + distrolessTeleportEntImage = "teleport-ent-distroless" +) + +// DistrolessImage returns the distroless teleport image repo. +func DistrolessImage(version semver.Version) string { + repo := distrolessImageRepo(version) + name := distrolessImageName(modules.GetModules().BuildType()) + return fmt.Sprintf("%s/%s:%s", repo, name, version) +} + +func distrolessImageRepo(version semver.Version) string { + if version.PreRelease != "" { + return teleportPreReleaseECR + } + return teleportReleaseECR +} + +func distrolessImageName(buildType string) string { + if buildType == modules.BuildEnterprise { + return distrolessTeleportEntImage + } + return distrolessTeleportOSSImage +} diff --git a/lib/utils/teleportassets/teleportassets_test.go b/lib/utils/teleportassets/teleportassets_test.go new file mode 100644 index 0000000000000..3115fa470a69f --- /dev/null +++ b/lib/utils/teleportassets/teleportassets_test.go @@ -0,0 +1,117 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package teleportassets + +import ( + "testing" + + "github.com/coreos/go-semver/semver" + "github.com/stretchr/testify/require" + + "github.com/gravitational/teleport/lib/modules" +) + +func TestDistrolessTeleportImageRepo(t *testing.T) { + tests := []struct { + desc string + buildType string + version string + want string + }{ + { + desc: "ent release", + buildType: modules.BuildEnterprise, + version: "16.0.0", + want: "public.ecr.aws/gravitational/teleport-ent-distroless:16.0.0", + }, + { + desc: "oss release", + buildType: modules.BuildOSS, + version: "16.0.0", + want: "public.ecr.aws/gravitational/teleport-distroless:16.0.0", + }, + { + desc: "ent pre-release", + buildType: modules.BuildEnterprise, + version: "16.0.0-alpha.1", + want: "public.ecr.aws/gravitational-staging/teleport-ent-distroless:16.0.0-alpha.1", + }, + { + desc: "oss pre-release", + buildType: modules.BuildOSS, + version: "16.0.0-alpha.1", + want: "public.ecr.aws/gravitational-staging/teleport-distroless:16.0.0-alpha.1", + }, + } + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + semVer, err := semver.NewVersion(test.version) + require.NoError(t, err) + modules.SetTestModules(t, &modules.TestModules{TestBuildType: test.buildType}) + require.Equal(t, test.want, DistrolessImage(*semVer)) + }) + } +} + +func Test_cdnBaseURLForVersion(t *testing.T) { + t.Parallel() + tests := []struct { + name string + artifactVersion string + teleportVersion string + want string + }{ + { + name: "both official releases", + artifactVersion: "16.3.2", + teleportVersion: "16.1.0", + want: TeleportReleaseCDN, + }, + { + name: "both pre-releases", + artifactVersion: "16.3.2-dev.1", + teleportVersion: "16.1.0-foo.25", + want: teleportPreReleaseCDN, + }, + { + name: "official teleport should not be able to install pre-release artifacts", + artifactVersion: "16.3.2-dev.1", + teleportVersion: "16.1.0", + want: TeleportReleaseCDN, + }, + { + name: "pre-release teleport should be able to install official artifacts", + artifactVersion: "16.3.2", + teleportVersion: "16.1.0-dev.1", + want: TeleportReleaseCDN, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test setup: parse version. + av, err := semver.NewVersion(tt.artifactVersion) + require.NoError(t, err) + tv, err := semver.NewVersion(tt.teleportVersion) + require.NoError(t, err) + + // Test execution and validation. + require.Equal(t, tt.want, cdnBaseURLForVersion(av, tv)) + }) + } +} diff --git a/lib/utils/testutils/golden/golden.go b/lib/utils/testutils/golden/golden.go new file mode 100644 index 0000000000000..6f85064c2941b --- /dev/null +++ b/lib/utils/testutils/golden/golden.go @@ -0,0 +1,132 @@ +/* + * Teleport + * Copyright (C) 2023 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// Golden files are a convenient way of storing data that we want to assert in +// unit tests. They are stored under the `testdata/` directory in a directory +// based on the name of the test. They are especially useful for storing large +// pieces of data that can be unwieldy to embed directly into your test tables. +// +// The convenience factor comes from the update mode which causes the tests to +// write data, rather than assert against it. This allows expected outputs +// to be updated easily when the underlying implementation is adjusted. +// This mode can be enabled by setting `GOLDEN_UPDATE=1` when running the tests +// you wish to update. +// +// Usage: +// +// Golden is ideal for testing the results of marshaling, or units that output +// large amounts of data to stdout or a file: +// +// func TestMarshalFooStruct(t *testing.T) { +// got, err := json.Marshal(FooStruct{Some: "Data"}) +// require.NoError(t, err) +// +// if golden.Update() { +// golden.Set(t, got) +// } +// require.Equal(t, golden.Get(t), got) +// } +// +// It is possible to have multiple golden files per test using `GetNamed` and +// `SetNamed`. This is useful for cases where your unit under test produces +// multiple pieces of output e.g stdout and stderr: +// +// func TestFooCommand(t *testing.T) { +// stdoutBuf := new(bytes.Buffer) +// stderrBuf := new(bytes.Buffer) +// +// FooCommand(stdoutBuf, stderrBuf) +// +// stdout := stdoutBuf.Bytes() +// stderr := stderrBuf.Bytes() +// +// if golden.Update() { +// golden.SetNamed(t, "stdout", stdout) +// golden.SetNamed(t, "stderr", stderr) +// } +// require.Equal(t, golden.GetNamed(t, "stdout"), stdout) +// require.Equal(t, golden.GetNamed(t, "stderr"), stderr) +// } + +package golden + +import ( + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +func pathForFile(t *testing.T, name string) string { + pathComponents := []string{ + "testdata", + t.Name(), + } + + if name != "" { + pathComponents = append(pathComponents, name) + } + + return filepath.Join(pathComponents...) + ".golden" +} + +// ShouldSet provides a boolean value that indicates if your code should then +// call `Set` or `SetNamed` to update the stored golden file value with new +// data. +func ShouldSet() bool { + env := os.Getenv("GOLDEN_UPDATE") + should, _ := strconv.ParseBool(env) + return should +} + +// SetNamed writes the supplied data to a named golden file for the current +// test. +func SetNamed(t *testing.T, name string, data []byte) { + p := pathForFile(t, name) + dir := filepath.Dir(p) + + err := os.MkdirAll(dir, 0o755) + require.NoError(t, err) + + err = os.WriteFile(p, data, 0o644) + require.NoError(t, err) +} + +// Set writes the supplied data to the golden file for the current test. +func Set(t *testing.T, data []byte) { + SetNamed(t, "", data) +} + +// GetNamed returns the contents of a named golden file for the current test. If +// the specified golden file does not exist for the test, the test will be +// failed. +func GetNamed(t *testing.T, name string) []byte { + p := pathForFile(t, name) + data, err := os.ReadFile(p) + require.NoError(t, err) + + return data +} + +// Get returns the contents of the golden file for the current test. If there is +// no golden file for the test, the test will be failed. +func Get(t *testing.T) []byte { + return GetNamed(t, "") +} diff --git a/lib/utils/testutils/testutils.go b/lib/utils/testutils/testutils.go new file mode 100644 index 0000000000000..2bbe39dd58f53 --- /dev/null +++ b/lib/utils/testutils/testutils.go @@ -0,0 +1,239 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package testutils + +import ( + "fmt" + "reflect" + "strings" +) + +// ExhaustiveNonEmpty is a helper that uses reflection to check if a given value and its sub-elements are non-empty. Exhaustive +// non-emptiness is evaluated in the following ways: +// +// - Pointers/Interfaces are considered exhaustively non-empty if their underlying value is exhaustively non-empty. +// - Slices/Arrays are considered exhaustively non-empty if they have at least one exhaustively non-empty element. +// - Maps are considered exhaustively non-empty if they have at least one exhaustively non-empty value. +// - Structs are considered exhaustively non-empty if all their exported fields are non-empty. +// - All other types are considered exhaustively non-empty if reflect.Value.IsZero is false. +// +// The ignoreOpts parameter is a variadic list of strings that represent the fully qualified field names of struct fields that +// should be ignored when checking for non-emptiness. For example, to ignore the field Bar on type Foo pass in "Foo.Bar" as an +// ignore option. Note that embedded type fields have to be ignored by the parent type's name (i.e. `Outer.Field` rather than +// `Inner.Field`). +// +// The intended usecase of this helper is to ensure that new fields added to a struct are included in test cases that want to +// cover all fields. For example, a test of serialization/deserialization logic might assert that the sample struct is exhaustively +// non-empty in order to force new fields to be covered by the test. +func ExhaustiveNonEmpty(item any, ignoreOpts ...string) bool { + value := reflect.ValueOf(item) + + ignore := make(map[string]struct{}, len(ignoreOpts)) + for _, opt := range ignoreOpts { + ignore[opt] = struct{}{} + } + + return exhaustiveNonEmpty(value, ignore) +} + +func exhaustiveNonEmpty(value reflect.Value, ignore map[string]struct{}) bool { + if !value.IsValid() { + // indicates that reflect.ValueOf/Value.Elem was called on a nil pointer/interface + return false + } + + switch value.Kind() { + case reflect.Pointer, reflect.Interface: + // recursively check the underlying value + return exhaustiveNonEmpty(value.Elem(), ignore) + case reflect.Slice, reflect.Array: + if value.Len() == 0 { + return false + } + + for i := 0; i < value.Len(); i++ { + if exhaustiveNonEmpty(value.Index(i), ignore) { + return true + } + } + return false + case reflect.Map: + if value.Len() == 0 { + return false + } + + mr := value.MapRange() + + for mr.Next() { + if exhaustiveNonEmpty(mr.Value(), ignore) { + return true + } + } + + return false + case reflect.Struct: + var fieldsConsidered int + for _, vf := range reflect.VisibleFields(value.Type()) { + if vf.Anonymous { + // skip the embedded type itself since this loop will + // end up processing each of the embedded type's fields as + // a member of this type's fields. + continue + } + + if !vf.IsExported() { + // skip non-exported fields + continue + } + + fieldsConsidered++ + + // skip fields if `.` is in the ignore list + if _, ok := ignore[fmt.Sprintf("%s.%s", value.Type().Name(), vf.Name)]; ok { + continue + } + + if !exhaustiveNonEmpty(value.FieldByIndex(vf.Index), ignore) { + return false + } + } + + if fieldsConsidered == 0 { + // fallback to basic nonzeroness check for structs with no exported fields (necessary + // in order to achieve expected behavior for types like time.Time). + return !value.IsZero() + } + + return true + default: + // fallback to basic nonzeroness check for all other types + return !value.IsZero() + } +} + +// FindAllEmpty is a helper that uses reflection to find all empty sub-components of a given value. It functions similarly to the ExhaustiveNonEmpty +// check, but may return a non-empty list of paths in cases where ExhaustiveNonEmpty would return false since it records all empty members of +// collections even if the collection contains a non-empty member. +// +// The intended usecase for FindAllEmpty is to build helpful failure messages in tests that assert that a struct is non-empty. +// +// Note that this function panics if the top-level item passed in is nil. +func FindAllEmpty(item any, ignoreOpts ...string) []string { + value := reflect.ValueOf(item) + + if !value.IsValid() { + panic("FindAllEmpty called with nil top-level item") + } + + // dereference pointers and interfaces so that the root find logic starts from + // a concrete type (makes the returned paths more consistent/understandable). + switch value.Kind() { + case reflect.Ptr, reflect.Interface: + if value.IsNil() { + panic("FindAllEmpty called with nil top-level pointer/interface") + } + return FindAllEmpty(value.Elem().Interface(), ignoreOpts...) + } + + ignore := make(map[string]struct{}, len(ignoreOpts)) + for _, opt := range ignoreOpts { + ignore[opt] = struct{}{} + } + + path := []string{value.Type().Name()} + + return findAllEmpty(value, ignore, path) +} + +func findAllEmpty(value reflect.Value, ignore map[string]struct{}, path []string) []string { + if !value.IsValid() { + // indicates that reflect.ValueOf/Value.Elem was called on a nil pointer/interface + return []string{strings.Join(path, ".")} + } + + switch value.Kind() { + case reflect.Pointer, reflect.Interface: + // recursively check the underlying value + return findAllEmpty(value.Elem(), ignore, path) + case reflect.Slice, reflect.Array: + if value.Len() == 0 { + return []string{strings.Join(path, ".")} + } + + var emptyPaths []string + for i := 0; i < value.Len(); i++ { + emptyPaths = append(emptyPaths, findAllEmpty(value.Index(i), ignore, append(path, fmt.Sprintf("%d", i)))...) + } + return emptyPaths + case reflect.Map: + if value.Len() == 0 { + return []string{strings.Join(path, ".")} + } + + mr := value.MapRange() + + var emptyPaths []string + for mr.Next() { + emptyPaths = append(emptyPaths, findAllEmpty(mr.Value(), ignore, append(path, fmt.Sprintf("%v", mr.Key().Interface())))...) + } + + return emptyPaths + case reflect.Struct: + emptyPaths := make([]string, 0, value.NumField()) + var fieldsConsidered int + for _, vf := range reflect.VisibleFields(value.Type()) { + if vf.Anonymous { + // skip the embedded type itself since this loop will + // end up processing each of the embedded type's fields as + // a member of this type's fields. + continue + } + + if !vf.IsExported() { + // skip non-exported fields + continue + } + + fieldsConsidered++ + + // skip fields if `.` is in the ignore list + if _, ok := ignore[fmt.Sprintf("%s.%s", value.Type().Name(), vf.Name)]; ok { + continue + } + + emptyPaths = append(emptyPaths, findAllEmpty(value.FieldByIndex(vf.Index), ignore, append(path, vf.Name))...) + } + + if fieldsConsidered == 0 { + // fallback to basic nonzeroness check for structs with no exported fields (necessary + // in order to achieve expected behavior for types like time.Time). + if value.IsZero() { + return []string{strings.Join(path, ".")} + } + } + + return emptyPaths + default: + // fallback to basic nonzeroness check for all other types + if value.IsZero() { + return []string{strings.Join(path, ".")} + } + return nil + } +} diff --git a/lib/utils/testutils/testutils_test.go b/lib/utils/testutils/testutils_test.go new file mode 100644 index 0000000000000..35c73d15b7122 --- /dev/null +++ b/lib/utils/testutils/testutils_test.go @@ -0,0 +1,564 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package testutils + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestExhaustiveNonEmptyBasics tests the basic functionality of ExhaustiveNonEmpty using various +// combinations of simple types. +func TestExhaustiveNonEmptyBasics(t *testing.T) { + t.Parallel() + tts := []struct { + desc string + value any + expect bool + }{ + { + desc: "basic nil", + value: nil, + expect: false, + }, + { + desc: "nil slice", + value: []string(nil), + expect: false, + }, + { + desc: "empty slice", + value: []string{}, + expect: false, + }, + { + desc: "slice with empty element", + value: []string{""}, + expect: false, + }, + { + desc: "non-empty slice", + value: []string{"a"}, + expect: true, + }, + { + desc: "slice with mix of empty and non-empty elements", + value: []string{"", "a"}, + expect: true, + }, + { + desc: "nil pointer", + value: (*string)(nil), + expect: false, + }, + { + desc: "pointer to empty string", + value: new(string), + expect: false, + }, + { + desc: "pointer to non-empty string", + value: func() *string { + s := "a" + return &s + }(), + expect: true, + }, + { + desc: "zero int", + value: int(0), + expect: false, + }, + { + desc: "non-zero int", + value: int(1), + expect: true, + }, + { + desc: "nil map", + value: map[string]string(nil), + expect: false, + }, + { + desc: "empty map", + value: map[string]string{}, + expect: false, + }, + { + desc: "map with empty value", + value: map[string]string{ + "a": "", + }, + expect: false, + }, + { + desc: "map with non-empty value", + value: map[string]string{ + "a": "b", + }, + expect: true, + }, + { + desc: "map with mix of empty and non-empty values", + value: map[string]string{ + "a": "", + "b": "c", + }, + expect: true, + }, + { + desc: "zero time", + value: time.Time{}, + expect: false, + }, + { + desc: "non-zero time", + value: time.Now(), + expect: true, + }, + } + + for _, tt := range tts { + t.Run(tt.desc, func(t *testing.T) { + require.Equal(t, tt.expect, ExhaustiveNonEmpty(tt.value), "value=%+v", tt.value) + }) + } +} + +// TestExhaustiveNonEmptyStruct tests the basic functionality of ExhaustiveNonEmpty using different struct field/nesting +// scenarios. This test also covers the behavior of struct field ignore options. +func TestExhaustiveNonEmptyStruct(t *testing.T) { + t.Parallel() + type Inner struct { + Field string + } + + type Outer struct { + Inner + Slice []Inner + Pointer *Inner + Value Inner + Map map[string]Inner + } + + newNonEmpty := func() Outer { + return Outer{ + Inner: Inner{ + Field: "a", + }, + Slice: []Inner{ + {Field: "b"}, + }, + Pointer: &Inner{Field: "c"}, + Value: Inner{Field: "d"}, + Map: map[string]Inner{ + "e": {Field: "f"}, + }, + } + } + + tts := []struct { + desc string + value any + ignore []string + expect bool + }{ + { + desc: "empty struct", + value: Outer{}, + expect: false, + }, + { + desc: "non-empty struct", + value: newNonEmpty(), + expect: true, + }, + { + desc: "pointer to empty struct", + value: new(Outer), + expect: false, + }, + { + desc: "pointer to non-empty struct", + value: func() *Outer { + v := newNonEmpty() + return &v + }(), + expect: true, + }, + { + desc: "struct with empty embed", + value: func() Outer { + v := newNonEmpty() + v.Inner = Inner{} + return v + }(), + expect: false, + }, + { + desc: "struct with nil slice", + value: func() Outer { + v := newNonEmpty() + v.Slice = nil + return v + }(), + expect: false, + }, + { + desc: "struct with empty slice", + value: func() Outer { + v := newNonEmpty() + v.Slice = []Inner{} + return v + }(), + expect: false, + }, + { + desc: "struct with empty slice element", + value: func() Outer { + v := newNonEmpty() + v.Slice = []Inner{{}} + return v + }(), + expect: false, + }, + { + desc: "struct with nil pointer", + value: func() Outer { + v := newNonEmpty() + v.Pointer = nil + return v + }(), + expect: false, + }, + { + desc: "struct with empty pointer", + value: func() Outer { + v := newNonEmpty() + v.Pointer = &Inner{} + return v + }(), + expect: false, + }, + { + desc: "struct with empty value", + value: func() Outer { + v := newNonEmpty() + v.Value = Inner{} + return v + }(), + expect: false, + }, + { + desc: "struct with nil map", + value: func() Outer { + v := newNonEmpty() + v.Map = nil + return v + }(), + expect: false, + }, + { + desc: "struct with empty map", + value: func() Outer { + v := newNonEmpty() + v.Map = map[string]Inner{} + return v + }(), + expect: false, + }, + { + desc: "struct with empty map value", + value: func() Outer { + v := newNonEmpty() + v.Map = map[string]Inner{"a": {}} + return v + }(), + expect: false, + }, + { + desc: "ignore top-level field", + value: func() Outer { + v := newNonEmpty() + v.Value = Inner{} + return v + }(), + ignore: []string{"Outer.Value"}, + expect: true, + }, + { + desc: "ignore embedded field", + value: func() Outer { + v := newNonEmpty() + v.Inner = Inner{} + return v + }(), + ignore: []string{"Outer.Field"}, // embedded ignores use the outer type name + expect: true, + }, + { + desc: "ignore slice element field", + value: func() Outer { + v := newNonEmpty() + v.Slice = []Inner{{}} + return v + }(), + ignore: []string{"Inner.Field"}, + expect: true, + }, + { + desc: "ignore pointer field", + value: func() Outer { + v := newNonEmpty() + v.Pointer = &Inner{} + return v + }(), + ignore: []string{"Inner.Field"}, + expect: true, + }, + { + desc: "ignore map value field", + value: func() Outer { + v := newNonEmpty() + v.Map = map[string]Inner{"a": {}} + return v + }(), + ignore: []string{"Inner.Field"}, + expect: true, + }, + } + + for _, tt := range tts { + t.Run(tt.desc, func(t *testing.T) { + require.Equal(t, tt.expect, ExhaustiveNonEmpty(tt.value, tt.ignore...), "value=%+v", tt.value) + }) + } +} + +// TestFindAllEmptyStruct tests the basic functionality of FindAllEmpty using different struct field/nesting +// scenarios. This test also covers the behavior of struct field ignore options. +func TestFindAllEmptyStruct(t *testing.T) { + t.Parallel() + type Inner struct { + Field string + } + + type Outer struct { + Inner + Slice []Inner + Pointer *Inner + Value Inner + Map map[string]Inner + } + + newNonEmpty := func() Outer { + return Outer{ + Inner: Inner{ + Field: "a", + }, + Slice: []Inner{ + {Field: "b"}, + }, + Pointer: &Inner{Field: "c"}, + Value: Inner{Field: "d"}, + Map: map[string]Inner{ + "e": {Field: "f"}, + }, + } + } + + tts := []struct { + desc string + value any + ignore []string + expect []string + }{ + { + desc: "empty struct", + value: Outer{}, + expect: []string{"Outer.Field", "Outer.Slice", "Outer.Pointer", "Outer.Value.Field", "Outer.Map"}, + }, + { + desc: "non-empty struct", + value: newNonEmpty(), + expect: nil, + }, + { + desc: "pointer to empty struct", + value: new(Outer), + expect: []string{"Outer.Field", "Outer.Slice", "Outer.Pointer", "Outer.Value.Field", "Outer.Map"}, + }, + { + desc: "pointer to non-empty struct", + value: func() *Outer { + v := newNonEmpty() + return &v + }(), + expect: nil, + }, + { + desc: "struct with empty embed", + value: func() Outer { + v := newNonEmpty() + v.Inner = Inner{} + return v + }(), + expect: []string{"Outer.Field"}, + }, + { + desc: "struct with nil slice", + value: func() Outer { + v := newNonEmpty() + v.Slice = nil + return v + }(), + expect: []string{"Outer.Slice"}, + }, + { + desc: "struct with empty slice", + value: func() Outer { + v := newNonEmpty() + v.Slice = []Inner{} + return v + }(), + expect: []string{"Outer.Slice"}, + }, + { + desc: "struct with empty slice element", + value: func() Outer { + v := newNonEmpty() + v.Slice = []Inner{{}} + return v + }(), + expect: []string{"Outer.Slice.0.Field"}, + }, + { + desc: "struct with nil pointer", + value: func() Outer { + v := newNonEmpty() + v.Pointer = nil + return v + }(), + expect: []string{"Outer.Pointer"}, + }, + { + desc: "struct with empty pointer", + value: func() Outer { + v := newNonEmpty() + v.Pointer = &Inner{} + return v + }(), + expect: []string{"Outer.Pointer.Field"}, + }, + { + desc: "struct with empty value", + value: func() Outer { + v := newNonEmpty() + v.Value = Inner{} + return v + }(), + expect: []string{"Outer.Value.Field"}, + }, + { + desc: "struct with nil map", + value: func() Outer { + v := newNonEmpty() + v.Map = nil + return v + }(), + expect: []string{"Outer.Map"}, + }, + { + desc: "struct with empty map", + value: func() Outer { + v := newNonEmpty() + v.Map = map[string]Inner{} + return v + }(), + expect: []string{"Outer.Map"}, + }, + { + desc: "struct with empty map value", + value: func() Outer { + v := newNonEmpty() + v.Map = map[string]Inner{"a": {}} + return v + }(), + expect: []string{"Outer.Map.a.Field"}, + }, + { + desc: "ignore top-level field", + value: func() Outer { + v := newNonEmpty() + v.Value = Inner{} + return v + }(), + ignore: []string{"Outer.Value"}, + expect: nil, + }, + { + desc: "ignore embedded field", + value: func() Outer { + v := newNonEmpty() + v.Inner = Inner{} + return v + }(), + ignore: []string{"Outer.Field"}, // embedded ignores use the outer type name + expect: nil, + }, + { + desc: "ignore slice element field", + value: func() Outer { + v := newNonEmpty() + v.Slice = []Inner{{}} + return v + }(), + ignore: []string{"Inner.Field"}, + expect: nil, + }, + { + desc: "ignore pointer field", + value: func() Outer { + v := newNonEmpty() + v.Pointer = &Inner{} + return v + }(), + ignore: []string{"Inner.Field"}, + expect: nil, + }, + { + desc: "ignore map value field", + value: func() Outer { + v := newNonEmpty() + v.Map = map[string]Inner{"a": {}} + return v + }(), + ignore: []string{"Inner.Field"}, + expect: nil, + }, + } + + for _, tt := range tts { + t.Run(tt.desc, func(t *testing.T) { + require.ElementsMatch(t, tt.expect, FindAllEmpty(tt.value, tt.ignore...), "value=%+v", tt.value) + }) + } +} diff --git a/lib/utils/unpack.go b/lib/utils/unpack.go index 351a8ecd691c3..6d51b56a3c172 100644 --- a/lib/utils/unpack.go +++ b/lib/utils/unpack.go @@ -21,6 +21,7 @@ import ( "errors" "io" "os" + "path" "path/filepath" "strings" @@ -34,7 +35,10 @@ import ( // resulting files and directories are created using the current user context. // Extract will only unarchive files into dir, and will fail if the tarball // tries to write files outside of dir. -func Extract(r io.Reader, dir string) error { +// +// If any paths are specified, only the specified paths are extracted. +// The destination specified in the first matching path is selected. +func Extract(r io.Reader, dir string, paths ...ExtractPath) error { tarball := tar.NewReader(r) for { @@ -44,32 +48,95 @@ func Extract(r io.Reader, dir string) error { } else if err != nil { return trace.Wrap(err) } - + dirMode, ok := filterHeader(header, paths) + if !ok { + continue + } err = sanitizeTarPath(header, dir) if err != nil { return trace.Wrap(err) } - if err := extractFile(tarball, header, dir); err != nil { + if err := extractFile(tarball, header, dir, dirMode); err != nil { return trace.Wrap(err) } } return nil } +// ExtractPath specifies a path to be extracted. +type ExtractPath struct { + // Src path and Dst path within the archive to extract files to. + // Directories in the Src path are not included in the extraction dir. + // For example, given foo/bar/file.txt with Src=foo/bar Dst=baz, baz/file.txt results. + // Trailing slashes are always ignored. + Src, Dst string + // Skip extracting the Src path and ignore Dst. + Skip bool + // DirMode is the file mode for implicit parent directories in Dst. + DirMode os.FileMode +} + +// filterHeader modifies the tar header by filtering it through the ExtractPaths. +// filterHeader returns false if the tar header should be skipped. +// If no paths are provided, filterHeader assumes the header should be included, and sets +// the mode for implicit parent directories to teleport.DirMaskSharedGroup. +func filterHeader(hdr *tar.Header, paths []ExtractPath) (dirMode os.FileMode, include bool) { + name := path.Clean(hdr.Name) + for _, p := range paths { + src := path.Clean(p.Src) + switch hdr.Typeflag { + case tar.TypeDir: + // If name is a directory, then + // assume src is a directory prefix, or the directory itself, + // and replace that prefix with dst. + if src != "/" { + src += "/" // ensure HasPrefix does not match partial names + } + if !strings.HasPrefix(name, src) { + continue + } + dst := path.Join(p.Dst, strings.TrimPrefix(name, src)) + if dst != "/" { + dst += "/" // tar directory headers end in / + } + hdr.Name = dst + return p.DirMode, !p.Skip + default: + // If name is a file, then + // if src is an exact match to the file name, assume src is a file and write directly to dst, + // otherwise, assume src is a directory prefix, and replace that prefix with dst. + if src == name { + hdr.Name = path.Clean(p.Dst) + return p.DirMode, !p.Skip + } + if src != "/" { + src += "/" // ensure HasPrefix does not match partial names + } + if !strings.HasPrefix(name, src) { + continue + } + hdr.Name = path.Join(p.Dst, strings.TrimPrefix(name, src)) + return p.DirMode, !p.Skip + + } + } + return teleport.DirMaskSharedGroup, len(paths) == 0 +} + // extractFile extracts a single file or directory from tarball into dir. // Uses header to determine the type of item to create // Based on https://github.com/mholt/archiver -func extractFile(tarball *tar.Reader, header *tar.Header, dir string) error { +func extractFile(tarball *tar.Reader, header *tar.Header, dir string, dirMode os.FileMode) error { switch header.Typeflag { case tar.TypeDir: - return withDir(filepath.Join(dir, header.Name), nil) + return withDir(filepath.Join(dir, header.Name), dirMode, nil) case tar.TypeBlock, tar.TypeChar, tar.TypeReg, tar.TypeFifo: - return writeFile(filepath.Join(dir, header.Name), tarball, header.FileInfo().Mode()) + return writeFile(filepath.Join(dir, header.Name), tarball, header.FileInfo().Mode(), dirMode) case tar.TypeLink: - return writeHardLink(filepath.Join(dir, header.Name), filepath.Join(dir, header.Linkname)) + return writeHardLink(filepath.Join(dir, header.Name), filepath.Join(dir, header.Linkname), dirMode) case tar.TypeSymlink: - return writeSymbolicLink(filepath.Join(dir, header.Name), header.Linkname) + return writeSymbolicLink(filepath.Join(dir, header.Name), header.Linkname, dirMode) default: log.Warnf("Unsupported type flag %v for %v.", header.Typeflag, header.Name) } @@ -104,8 +171,8 @@ func sanitizeTarPath(header *tar.Header, dir string) error { return nil } -func writeFile(path string, r io.Reader, mode os.FileMode) error { - err := withDir(path, func() error { +func writeFile(path string, r io.Reader, mode, dirMode os.FileMode) error { + err := withDir(path, dirMode, func() error { // Create file only if it does not exist to prevent overwriting existing // files (like session recordings). out, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, mode) @@ -118,24 +185,24 @@ func writeFile(path string, r io.Reader, mode os.FileMode) error { return trace.Wrap(err) } -func writeSymbolicLink(path string, target string) error { - err := withDir(path, func() error { +func writeSymbolicLink(path, target string, dirMode os.FileMode) error { + err := withDir(path, dirMode, func() error { err := os.Symlink(target, path) return trace.ConvertSystemError(err) }) return trace.Wrap(err) } -func writeHardLink(path string, target string) error { - err := withDir(path, func() error { +func writeHardLink(path, target string, dirMode os.FileMode) error { + err := withDir(path, dirMode, func() error { err := os.Link(target, path) return trace.ConvertSystemError(err) }) return trace.Wrap(err) } -func withDir(path string, fn func() error) error { - err := os.MkdirAll(filepath.Dir(path), teleport.DirMaskSharedGroup) +func withDir(path string, mode os.FileMode, fn func() error) error { + err := os.MkdirAll(filepath.Dir(path), mode) if err != nil { return trace.ConvertSystemError(err) } diff --git a/lib/versioncontrol/upgradewindow/upgradewindow.go b/lib/versioncontrol/upgradewindow/upgradewindow.go index 53c2905a6dccd..133e7e9219e12 100644 --- a/lib/versioncontrol/upgradewindow/upgradewindow.go +++ b/lib/versioncontrol/upgradewindow/upgradewindow.go @@ -360,13 +360,17 @@ func (e *kubeDriver) Kind() string { } func (e *kubeDriver) Sync(ctx context.Context, rsp proto.ExportUpgradeWindowsResponse) error { - if rsp.KubeControllerSchedule == "" { + return trace.Wrap(e.setSchedule(ctx, rsp.KubeControllerSchedule)) +} + +func (e *kubeDriver) setSchedule(ctx context.Context, schedule string) error { + if schedule == "" { return e.Reset(ctx) } _, err := e.cfg.Backend.Put(ctx, backend.Item{ Key: []byte(kubeSchedKey), - Value: []byte(rsp.KubeControllerSchedule), + Value: []byte(schedule), }) return trace.Wrap(err) @@ -406,7 +410,11 @@ func (e *systemdDriver) Kind() string { } func (e *systemdDriver) Sync(ctx context.Context, rsp proto.ExportUpgradeWindowsResponse) error { - if len(rsp.SystemdUnitSchedule) == 0 { + return trace.Wrap(e.setSchedule(ctx, rsp.SystemdUnitSchedule)) +} + +func (e *systemdDriver) setSchedule(ctx context.Context, schedule string) error { + if len(schedule) == 0 { // treat an empty schedule value as equivalent to a reset return e.Reset(ctx) } @@ -418,7 +426,7 @@ func (e *systemdDriver) Sync(ctx context.Context, rsp proto.ExportUpgradeWindows } // export schedule file. if created it is set to 644, which is reasonable for a sensitive but non-secret config value. - if err := os.WriteFile(e.scheduleFile(), []byte(rsp.SystemdUnitSchedule), defaults.FilePermissions); err != nil { + if err := os.WriteFile(e.scheduleFile(), []byte(schedule), defaults.FilePermissions); err != nil { return trace.Errorf("failed to write schedule file: %v", err) } diff --git a/lib/web/apiserver.go b/lib/web/apiserver.go index dbd29a46f0c15..f687ea3ea35d2 100644 --- a/lib/web/apiserver.go +++ b/lib/web/apiserver.go @@ -55,22 +55,20 @@ import ( "google.golang.org/protobuf/encoding/protojson" "github.com/gravitational/teleport" - "github.com/gravitational/teleport/api" apiclient "github.com/gravitational/teleport/api/client" "github.com/gravitational/teleport/api/client/proto" "github.com/gravitational/teleport/api/client/webclient" "github.com/gravitational/teleport/api/constants" apidefaults "github.com/gravitational/teleport/api/defaults" - autoupdatepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" apitracing "github.com/gravitational/teleport/api/observability/tracing" "github.com/gravitational/teleport/api/types" - "github.com/gravitational/teleport/api/types/autoupdate" apievents "github.com/gravitational/teleport/api/types/events" "github.com/gravitational/teleport/api/types/installers" "github.com/gravitational/teleport/api/utils/keys" apisshutils "github.com/gravitational/teleport/api/utils/sshutils" "github.com/gravitational/teleport/lib/auth" "github.com/gravitational/teleport/lib/auth/authclient" + "github.com/gravitational/teleport/lib/auth/native" "github.com/gravitational/teleport/lib/auth/state" wantypes "github.com/gravitational/teleport/lib/auth/webauthntypes" "github.com/gravitational/teleport/lib/automaticupgrades" @@ -119,6 +117,8 @@ const ( // This cache is here to protect against accidental or intentional DDoS, the TTL must be low to quickly reflect // cluster configuration changes. findEndpointCacheTTL = 10 * time.Second + // DefaultAgentUpdateJitterSeconds is the default jitter agents should wait before updating. + DefaultAgentUpdateJitterSeconds = 60 ) // healthCheckAppServerFunc defines a function used to perform a health check @@ -171,6 +171,9 @@ type Handler struct { // rate-limits, each call must cause minimal work. The cached answer can be modulated after, for example if the // caller specified its Automatic Updates UUID or group. findEndpointCache *utils.FnCache + + // clusterMaintenanceConfig is used to cache the cluster maintenance config from the AUth Service. + clusterMaintenanceConfigCache *utils.FnCache } // HandlerOption is a functional argument - an option that can be passed @@ -411,6 +414,18 @@ func NewHandler(cfg Config, opts ...HandlerOption) (*APIHandler, error) { } h.findEndpointCache = findCache + // We create the cache after applying the options to make sure we use the fake clock if it was passed. + cmcCache, err := utils.NewFnCache(utils.FnCacheConfig{ + TTL: findEndpointCacheTTL, + Clock: h.clock, + Context: cfg.Context, + ReloadOnErr: false, + }) + if err != nil { + return nil, trace.Wrap(err, "creating /find cache") + } + h.clusterMaintenanceConfigCache = cmcCache + sessionLingeringThreshold := cachedSessionLingeringThreshold if cfg.CachedSessionLingeringThreshold != nil { sessionLingeringThreshold = *cfg.CachedSessionLingeringThreshold @@ -783,6 +798,11 @@ func (h *Handler) bindDefaultEndpoints() { // token generation h.POST("/webapi/token", h.WithAuth(h.createTokenHandle)) + // install script, the ':token' wildcard is a hack to make the router happy and support + // the token-less route "/scripts/install.sh". + // h.installScriptHandle Will reject any unknown sub-route. + h.GET("/scripts/:token", h.WithHighLimiter(h.installScriptHandle)) + // join scripts h.GET("/scripts/:token/install-node.sh", h.WithLimiter(h.getNodeJoinScriptHandle)) h.GET("/scripts/:token/install-app.sh", h.WithLimiter(h.getAppJoinScriptHandle)) @@ -976,7 +996,7 @@ func (h *Handler) bindDefaultEndpoints() { // Implements the agent version server. // Channel can contain "/", hence the use of a catch-all parameter - h.GET("/webapi/automaticupgrades/channel/*request", h.WithUnauthenticatedHighLimiter(h.automaticUpgrades)) + h.GET("/webapi/automaticupgrades/channel/*request", h.WithUnauthenticatedHighLimiter(h.automaticUpgrades109)) } // GetProxyClient returns authenticated auth server client @@ -1423,6 +1443,8 @@ func (h *Handler) ping(w http.ResponseWriter, r *http.Request, p httprouter.Para return nil, trace.Wrap(err) } + group := r.URL.Query().Get(webclient.AgentUpdateGroupParameter) + return webclient.PingResponse{ Auth: authSettings, Proxy: *proxyConfig, @@ -1430,13 +1452,21 @@ func (h *Handler) ping(w http.ResponseWriter, r *http.Request, p httprouter.Para MinClientVersion: teleport.MinClientVersion, ClusterName: h.auth.clusterName, AutomaticUpgrades: pr.ServerFeatures.GetAutomaticUpgrades(), - AutoUpdate: h.automaticUpdateSettings(r.Context()), + AutoUpdate: h.automaticUpdateSettings184(r.Context(), group, "" /* updater UUID */), + Edition: modules.GetModules().BuildType(), + FIPS: native.IsBoringBinary(), }, nil } func (h *Handler) find(w http.ResponseWriter, r *http.Request, p httprouter.Params) (interface{}, error) { + group := r.URL.Query().Get(webclient.AgentUpdateGroupParameter) + cacheKey := "find" + if group != "" { + cacheKey += "-" + group + } + // cache the generic answer to avoid doing work for each request - resp, err := utils.FnCacheGet[*webclient.PingResponse](r.Context(), h.findEndpointCache, "find", func(ctx context.Context) (*webclient.PingResponse, error) { + resp, err := utils.FnCacheGet[*webclient.PingResponse](r.Context(), h.findEndpointCache, cacheKey, func(ctx context.Context) (*webclient.PingResponse, error) { proxyConfig, err := h.cfg.ProxySettings.GetProxySettings(ctx) if err != nil { return nil, trace.Wrap(err) @@ -1447,33 +1477,12 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request, p httprouter.Para ServerVersion: teleport.Version, MinClientVersion: teleport.MinClientVersion, ClusterName: h.auth.clusterName, - AutoUpdate: h.automaticUpdateSettings(ctx), + Edition: modules.GetModules().BuildType(), + FIPS: native.IsBoringBinary(), + AutoUpdate: h.automaticUpdateSettings184(ctx, group, "" /* updater UUID */), }, nil }) - if err != nil { - return nil, trace.Wrap(err) - } - return resp, nil -} - -// TODO: add the request as a parameter when we'll need to modulate the content based on the UUID and group -func (h *Handler) automaticUpdateSettings(ctx context.Context) webclient.AutoUpdateSettings { - autoUpdateConfig, err := h.cfg.AccessPoint.GetAutoUpdateConfig(ctx) - // TODO(vapopov) DELETE IN v18.0.0 check of IsNotImplemented, must be backported to all latest supported versions. - if err != nil && !trace.IsNotFound(err) && !trace.IsNotImplemented(err) { - h.log.WithError(err).Warn("failed to receive AutoUpdateConfig") - } - - autoUpdateVersion, err := h.cfg.AccessPoint.GetAutoUpdateVersion(ctx) - // TODO(vapopov) DELETE IN v18.0.0 check of IsNotImplemented, must be backported to all latest supported versions. - if err != nil && !trace.IsNotFound(err) && !trace.IsNotImplemented(err) { - h.log.WithError(err).Warn("failed to receive AutoUpdateVersion") - } - - return webclient.AutoUpdateSettings{ - ToolsAutoUpdate: getToolsAutoUpdate(autoUpdateConfig), - ToolsVersion: getToolsVersion(autoUpdateVersion), - } + return resp, err } func (h *Handler) pingWithConnector(w http.ResponseWriter, r *http.Request, p httprouter.Params) (interface{}, error) { @@ -1687,9 +1696,13 @@ func (h *Handler) getWebConfig(w http.ResponseWriter, r *http.Request, p httprou automaticUpgradesEnabled := clusterFeatures.GetAutomaticUpgrades() var automaticUpgradesTargetVersion string if automaticUpgradesEnabled { - automaticUpgradesTargetVersion, err = h.cfg.AutomaticUpgradesChannels.DefaultVersion(r.Context()) + const group, updaterUUID = "", "" + agentVersion, err := h.autoUpdateAgentVersion(r.Context(), group, updaterUUID) if err != nil { - h.log.WithError(err).Error("Cannot read target version") + h.log.WithError(err).Error("Cannot read autoupdate target version") + } else { + // agentVersion doesn't have the leading "v" which is expected here. + automaticUpgradesTargetVersion = fmt.Sprintf("v%s", agentVersion) } } @@ -1941,7 +1954,7 @@ func (h *Handler) installer(w http.ResponseWriter, r *http.Request, p httprouter // https://updates.releases.teleport.dev/v1/stable/cloud/version installUpdater := automaticUpgrades(h.ClusterFeatures) if installUpdater { - repoChannel = stableCloudChannelRepo + repoChannel = automaticupgrades.DefaultCloudChannelName } azureClientID := r.URL.Query().Get("azure-client-id") @@ -4610,23 +4623,3 @@ func serveRobotsTxt(w http.ResponseWriter, r *http.Request, p httprouter.Params) w.Write([]byte(robots)) return nil, nil } - -func getToolsAutoUpdate(config *autoupdatepb.AutoUpdateConfig) bool { - // If we can't get the AU config or if AUs are not configured, we default to "disabled". - // This ensures we fail open and don't accidentally update agents if something is going wrong. - // If we want to enable AUs by default, it would be better to create a default "autoupdate_config" resource - // than changing this logic. - if config.GetSpec().GetTools() != nil { - return config.GetSpec().GetTools().GetMode() == autoupdate.ToolsUpdateModeEnabled - } - return false -} - -func getToolsVersion(version *autoupdatepb.AutoUpdateVersion) string { - // If we can't get the AU version or tools AU version is not specified, we default to the current proxy version. - // This ensures we always advertise a version compatible with the cluster. - if version.GetSpec().GetTools() == nil { - return api.Version - } - return version.GetSpec().GetTools().GetTargetVersion() -} diff --git a/lib/web/apiserver_ping_test.go b/lib/web/apiserver_ping_test.go index 02294751947c5..50d4e849620a7 100644 --- a/lib/web/apiserver_ping_test.go +++ b/lib/web/apiserver_ping_test.go @@ -255,49 +255,98 @@ func TestPing_autoUpdateResources(t *testing.T) { name string config *autoupdatev1pb.AutoUpdateConfigSpec version *autoupdatev1pb.AutoUpdateVersionSpec + rollout *autoupdatev1pb.AutoUpdateAgentRolloutSpec cleanup bool expected webclient.AutoUpdateSettings }{ { name: "resources not defined", expected: webclient.AutoUpdateSettings{ - ToolsVersion: api.Version, - ToolsAutoUpdate: false, + ToolsVersion: api.Version, + ToolsAutoUpdate: false, + AgentUpdateJitterSeconds: DefaultAgentUpdateJitterSeconds, + AgentAutoUpdate: false, + AgentVersion: api.Version, }, }, { - name: "enable auto update", + name: "enable tools auto update", config: &autoupdatev1pb.AutoUpdateConfigSpec{ Tools: &autoupdatev1pb.AutoUpdateConfigSpecTools{ Mode: autoupdate.ToolsUpdateModeEnabled, }, }, expected: webclient.AutoUpdateSettings{ - ToolsAutoUpdate: true, - ToolsVersion: api.Version, + ToolsAutoUpdate: true, + ToolsVersion: api.Version, + AgentUpdateJitterSeconds: DefaultAgentUpdateJitterSeconds, + AgentAutoUpdate: false, + AgentVersion: api.Version, }, cleanup: true, }, { - name: "empty config and version", + name: "enable agent auto update, immediate schedule", + rollout: &autoupdatev1pb.AutoUpdateAgentRolloutSpec{ + AutoupdateMode: autoupdate.AgentsUpdateModeEnabled, + Strategy: autoupdate.AgentsStrategyHaltOnError, + Schedule: autoupdate.AgentsScheduleImmediate, + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + }, + expected: webclient.AutoUpdateSettings{ + ToolsVersion: api.Version, + ToolsAutoUpdate: false, + AgentUpdateJitterSeconds: DefaultAgentUpdateJitterSeconds, + AgentAutoUpdate: true, + AgentVersion: "1.2.4", + }, + cleanup: true, + }, + { + name: "agent rollout present but AU mode is disabled", + rollout: &autoupdatev1pb.AutoUpdateAgentRolloutSpec{ + AutoupdateMode: autoupdate.AgentsUpdateModeDisabled, + Strategy: autoupdate.AgentsStrategyHaltOnError, + Schedule: autoupdate.AgentsScheduleImmediate, + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + }, + expected: webclient.AutoUpdateSettings{ + ToolsVersion: api.Version, + ToolsAutoUpdate: false, + AgentUpdateJitterSeconds: DefaultAgentUpdateJitterSeconds, + AgentAutoUpdate: false, + AgentVersion: "1.2.4", + }, + cleanup: true, + }, + { + name: "no autoupdate tool config nor version", config: &autoupdatev1pb.AutoUpdateConfigSpec{}, version: &autoupdatev1pb.AutoUpdateVersionSpec{}, expected: webclient.AutoUpdateSettings{ - ToolsVersion: api.Version, - ToolsAutoUpdate: false, + ToolsVersion: api.Version, + ToolsAutoUpdate: false, + AgentUpdateJitterSeconds: DefaultAgentUpdateJitterSeconds, + AgentAutoUpdate: false, + AgentVersion: api.Version, }, cleanup: true, }, { - name: "set tools auto update version", + name: "set auto update version", version: &autoupdatev1pb.AutoUpdateVersionSpec{ Tools: &autoupdatev1pb.AutoUpdateVersionSpecTools{ TargetVersion: "1.2.3", }, }, expected: webclient.AutoUpdateSettings{ - ToolsVersion: "1.2.3", - ToolsAutoUpdate: false, + ToolsVersion: "1.2.3", + ToolsAutoUpdate: false, + AgentUpdateJitterSeconds: DefaultAgentUpdateJitterSeconds, + AgentAutoUpdate: false, + AgentVersion: api.Version, }, cleanup: true, }, @@ -314,8 +363,11 @@ func TestPing_autoUpdateResources(t *testing.T) { }, }, expected: webclient.AutoUpdateSettings{ - ToolsAutoUpdate: true, - ToolsVersion: "1.2.3", + ToolsAutoUpdate: true, + ToolsVersion: "1.2.3", + AgentUpdateJitterSeconds: DefaultAgentUpdateJitterSeconds, + AgentAutoUpdate: false, + AgentVersion: api.Version, }, }, { @@ -331,8 +383,11 @@ func TestPing_autoUpdateResources(t *testing.T) { }, }, expected: webclient.AutoUpdateSettings{ - ToolsAutoUpdate: false, - ToolsVersion: "3.2.1", + ToolsAutoUpdate: false, + ToolsVersion: "3.2.1", + AgentUpdateJitterSeconds: DefaultAgentUpdateJitterSeconds, + AgentAutoUpdate: false, + AgentVersion: api.Version, }, }, } @@ -350,6 +405,12 @@ func TestPing_autoUpdateResources(t *testing.T) { _, err = env.server.Auth().UpsertAutoUpdateVersion(ctx, version) require.NoError(t, err) } + if tc.rollout != nil { + rollout, err := autoupdate.NewAutoUpdateAgentRollout(tc.rollout) + require.NoError(t, err) + _, err = env.server.Auth().UpsertAutoUpdateAgentRollout(ctx, rollout) + require.NoError(t, err) + } // expire the fn cache to force the next answer to be fresh for _, proxy := range env.proxies { @@ -368,6 +429,7 @@ func TestPing_autoUpdateResources(t *testing.T) { if tc.cleanup { require.NotErrorIs(t, env.server.Auth().DeleteAutoUpdateConfig(ctx), &trace.NotFoundError{}) require.NotErrorIs(t, env.server.Auth().DeleteAutoUpdateVersion(ctx), &trace.NotFoundError{}) + require.NotErrorIs(t, env.server.Auth().DeleteAutoUpdateAgentRollout(ctx), &trace.NotFoundError{}) } }) } diff --git a/lib/web/apiserver_test.go b/lib/web/apiserver_test.go index ef996f64db10b..79fae1e010ed0 100644 --- a/lib/web/apiserver_test.go +++ b/lib/web/apiserver_test.go @@ -3388,6 +3388,7 @@ func TestTokenGeneration(t *testing.T) { func TestInstallDatabaseScriptGeneration(t *testing.T) { const username = "test-user@example.com" + modules.SetTestModules(t, &modules.TestModules{TestBuildType: modules.BuildCommunity}) // Users should be able to create Tokens even if they can't update them roleTokenCRD, err := types.NewRole(services.RoleNameForUser(username), types.RoleSpecV6{ @@ -8100,9 +8101,9 @@ func createProxy(ctx context.Context, t *testing.T, proxyID string, node *regula }, ) handler.handler.cfg.ProxyKubeAddr = utils.FromAddr(kubeProxyAddr) + handler.handler.cfg.PublicProxyAddr = webServer.Listener.Addr().String() url, err := url.Parse("https://" + webServer.Listener.Addr().String()) require.NoError(t, err) - handler.handler.cfg.PublicProxyAddr = url.String() return &testProxy{ clock: clock, diff --git a/lib/web/autoupdate_common.go b/lib/web/autoupdate_common.go new file mode 100644 index 0000000000000..91e28a7993a18 --- /dev/null +++ b/lib/web/autoupdate_common.go @@ -0,0 +1,228 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package web + +import ( + "context" + "strings" + + "github.com/gravitational/trace" + + autoupdatepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/api/types/autoupdate" + "github.com/gravitational/teleport/lib/automaticupgrades" + "github.com/gravitational/teleport/lib/utils" +) + +// autoUpdateAgentVersion returns the version the agent should install/update to based on +// its group and updater UUID. +// If the cluster contains an autoupdate_agent_rollout resource from RFD184 it should take precedence. +// If the resource is not there, we fall back to RFD109-style updates with channels +// and maintenance window derived from the cluster_maintenance_config resource. +// Version returned follows semver without the leading "v". +func (h *Handler) autoUpdateAgentVersion(ctx context.Context, group, updaterUUID string) (string, error) { + rollout, err := h.cfg.AccessPoint.GetAutoUpdateAgentRollout(ctx) + if err != nil { + // Fallback to channels if there is no autoupdate_agent_rollout. + if trace.IsNotFound(err) || trace.IsNotImplemented(err) { + return getVersionFromChannel(ctx, h.cfg.AutomaticUpgradesChannels, group) + } + // Something is broken, we don't want to fallback to channels, this would be harmful. + return "", trace.Wrap(err, "getting autoupdate_agent_rollout") + } + + return getVersionFromRollout(rollout, group, updaterUUID) +} + +// autoUpdateAgentShouldUpdate returns if the agent should update now to based on its group +// and updater UUID. +// If the cluster contains an autoupdate_agent_rollout resource from RFD184 it should take precedence. +// If the resource is not there, we fall back to RFD109-style updates with channels +// and maintenance window derived from the cluster_maintenance_config resource. +func (h *Handler) autoUpdateAgentShouldUpdate(ctx context.Context, group, updaterUUID string, windowLookup bool) (bool, error) { + rollout, err := h.cfg.AccessPoint.GetAutoUpdateAgentRollout(ctx) + if err != nil { + // Fallback to channels if there is no autoupdate_agent_rollout. + if trace.IsNotFound(err) || trace.IsNotImplemented(err) { + // Updaters using the RFD184 API are not aware of maintenance windows + // like RFD109 updaters are. To have both updaters adopt the same behavior + // we must do the CMC window lookup for them. + if windowLookup { + return h.getTriggerFromWindowThenChannel(ctx, group) + } + return getTriggerFromChannel(ctx, h.cfg.AutomaticUpgradesChannels, group) + } + // Something is broken, we don't want to fallback to channels, this would be harmful. + return false, trace.Wrap(err, "failed to get auto-update rollout") + } + + return getTriggerFromRollout(rollout, group, updaterUUID) +} + +// getVersionFromRollout returns the version we should serve to the agent based +// on the RFD184 agent rollout, the agent group name, and its UUID. +// This logic is pretty complex and described in RFD 184. +// The spec is summed up in the following table: +// https://github.com/gravitational/teleport/blob/master/rfd/0184-agent-auto-updates.md#rollout-status-disabled +// Version returned follows semver without the leading "v". +func getVersionFromRollout( + rollout *autoupdatepb.AutoUpdateAgentRollout, + groupName, updaterUUID string, +) (string, error) { + switch rollout.GetSpec().GetAutoupdateMode() { + case autoupdate.AgentsUpdateModeDisabled: + // If AUs are disabled, we always answer the target version + return rollout.GetSpec().GetTargetVersion(), nil + case autoupdate.AgentsUpdateModeSuspended, autoupdate.AgentsUpdateModeEnabled: + // If AUs are enabled or suspended, we modulate the response based on the schedule and agent group state + default: + return "", trace.BadParameter("unsupported agent update mode %q", rollout.GetSpec().GetAutoupdateMode()) + } + + // If the schedule is immediate, agents always update to the latest version + if rollout.GetSpec().GetSchedule() == autoupdate.AgentsScheduleImmediate { + return rollout.GetSpec().GetTargetVersion(), nil + } + + // Else we follow the regular schedule and answer based on the agent group state + group, err := getGroup(rollout, groupName) + if err != nil { + return "", trace.Wrap(err, "getting group %q", groupName) + } + + switch group.GetState() { + case autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK: + return rollout.GetSpec().GetStartVersion(), nil + case autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE: + return rollout.GetSpec().GetTargetVersion(), nil + default: + return "", trace.NotImplemented("unsupported group state %q", group.GetState()) + } +} + +// getTriggerFromRollout returns the version we should serve to the agent based +// on the RFD184 agent rollout, the agent group name, and its UUID. +// This logic is pretty complex and described in RFD 184. +// The spec is summed up in the following table: +// https://github.com/gravitational/teleport/blob/master/rfd/0184-agent-auto-updates.md#rollout-status-disabled +func getTriggerFromRollout(rollout *autoupdatepb.AutoUpdateAgentRollout, groupName, updaterUUID string) (bool, error) { + // If the mode is "paused" or "disabled", we never tell to update + switch rollout.GetSpec().GetAutoupdateMode() { + case autoupdate.AgentsUpdateModeDisabled, autoupdate.AgentsUpdateModeSuspended: + // If AUs are disabled or suspended, never tell to update + return false, nil + case autoupdate.AgentsUpdateModeEnabled: + // If AUs are enabled, we modulate the response based on the schedule and agent group state + default: + return false, trace.BadParameter("unsupported agent update mode %q", rollout.GetSpec().GetAutoupdateMode()) + } + + // If the schedule is immediate, agents always update to the latest version + if rollout.GetSpec().GetSchedule() == autoupdate.AgentsScheduleImmediate { + return true, nil + } + + // Else we follow the regular schedule and answer based on the agent group state + group, err := getGroup(rollout, groupName) + if err != nil { + return false, trace.Wrap(err, "getting group %q", groupName) + } + + switch group.GetState() { + case autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED: + return false, nil + case autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK: + return true, nil + case autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE: + return rollout.GetSpec().GetStrategy() == autoupdate.AgentsStrategyHaltOnError, nil + default: + return false, trace.NotImplemented("Unsupported group state %q", group.GetState()) + } +} + +// getGroup returns the agent rollout group the requesting agent belongs to. +// If a group matches the agent-provided group name, this group is returned. +// Else the default group is returned. The default group currently is the last +// one. This might change in the future. +func getGroup( + rollout *autoupdatepb.AutoUpdateAgentRollout, + groupName string, +) (*autoupdatepb.AutoUpdateAgentRolloutStatusGroup, error) { + groups := rollout.GetStatus().GetGroups() + if len(groups) == 0 { + return nil, trace.BadParameter("no groups found") + } + + // Try to find a group with our name + for _, group := range groups { + if group.Name == groupName { + return group, nil + } + } + + // Fallback to the default group (currently the last one but this might change). + return groups[len(groups)-1], nil +} + +// getVersionFromChannel gets the target version from the RFD109 channels. +// Version returned follows semver without the leading "v". +func getVersionFromChannel(ctx context.Context, channels automaticupgrades.Channels, groupName string) (version string, err error) { + // RFD109 channels return the version with the 'v' prefix. + // We can't change the internals for backward compatibility, so we must trim the prefix if it's here. + defer func() { + version = strings.TrimPrefix(version, "v") + }() + + if channel, ok := channels[groupName]; ok { + return channel.GetVersion(ctx) + } + return channels.DefaultVersion(ctx) +} + +// getTriggerFromWindowThenChannel gets the target version from the RFD109 maintenance window and channels. +func (h *Handler) getTriggerFromWindowThenChannel(ctx context.Context, groupName string) (bool, error) { + // Caching the CMC for 10 seconds because this resource is cached neither by the auth nor the proxy. + // And this function can be accessed via unauthenticated endpoints. + cmc, err := utils.FnCacheGet[types.ClusterMaintenanceConfig](ctx, h.clusterMaintenanceConfigCache, "cmc", func(ctx context.Context) (types.ClusterMaintenanceConfig, error) { + return h.cfg.ProxyClient.GetClusterMaintenanceConfig(ctx) + }) + + // If we have a CMC, we check if the window is active, else we just check if the update is critical. + if err == nil && cmc.WithinUpgradeWindow(h.clock.Now()) { + return true, nil + } + + return getTriggerFromChannel(ctx, h.cfg.AutomaticUpgradesChannels, groupName) +} + +// getTriggerFromWindowThenChannel gets the target version from the RFD109 channels. +func getTriggerFromChannel(ctx context.Context, channels automaticupgrades.Channels, groupName string) (bool, error) { + if channel, ok := channels[groupName]; ok { + return channel.GetCritical(ctx) + } + defaultChannel, err := channels.DefaultChannel() + if err != nil { + return false, trace.Wrap(err, "creating new default channel") + } + return defaultChannel.GetCritical(ctx) +} diff --git a/lib/web/autoupdate_common_test.go b/lib/web/autoupdate_common_test.go new file mode 100644 index 0000000000000..e0a1a31719586 --- /dev/null +++ b/lib/web/autoupdate_common_test.go @@ -0,0 +1,799 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package web + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gravitational/trace" + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + autoupdatepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/api/types/autoupdate" + "github.com/gravitational/teleport/lib/auth/authclient" + "github.com/gravitational/teleport/lib/automaticupgrades" + "github.com/gravitational/teleport/lib/automaticupgrades/constants" + "github.com/gravitational/teleport/lib/utils" +) + +const ( + testVersionHigh = "2.3.4" + testVersionLow = "2.0.4" +) + +// fakeRolloutAccessPoint allows us to mock the ProxyAccessPoint in autoupdate +// tests. +type fakeRolloutAccessPoint struct { + authclient.ProxyAccessPoint + + rollout *autoupdatepb.AutoUpdateAgentRollout + err error +} + +func (ap *fakeRolloutAccessPoint) GetAutoUpdateAgentRollout(_ context.Context) (*autoupdatepb.AutoUpdateAgentRollout, error) { + return ap.rollout, ap.err +} + +// fakeRolloutAccessPoint allows us to mock the proxy's auth client in autoupdate +// tests. +type fakeCMCAuthClient struct { + authclient.ClientI + + cmc types.ClusterMaintenanceConfig + err error +} + +func (c *fakeCMCAuthClient) GetClusterMaintenanceConfig(_ context.Context) (types.ClusterMaintenanceConfig, error) { + return c.cmc, c.err +} + +func TestAutoUpdateAgentVersion(t *testing.T) { + t.Parallel() + groupName := "test-group" + ctx := context.Background() + + // brokenChannelUpstream is a buggy upstream version server. + // This allows us to craft version channels returning errors. + brokenChannelUpstream := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + })) + t.Cleanup(brokenChannelUpstream.Close) + + tests := []struct { + name string + rollout *autoupdatepb.AutoUpdateAgentRollout + rolloutErr error + channel *automaticupgrades.Channel + expectedVersion string + expectError require.ErrorAssertionFunc + }{ + { + name: "version is looked up from rollout if it is here", + rollout: &autoupdatepb.AutoUpdateAgentRollout{ + Spec: &autoupdatepb.AutoUpdateAgentRolloutSpec{ + AutoupdateMode: autoupdate.AgentsUpdateModeEnabled, + TargetVersion: testVersionHigh, + Schedule: autoupdate.AgentsScheduleImmediate, + }, + }, + channel: &automaticupgrades.Channel{StaticVersion: testVersionLow}, + expectError: require.NoError, + expectedVersion: testVersionHigh, + }, + { + name: "version is looked up from channel if rollout is not here", + rolloutErr: trace.NotFound("rollout is not here"), + channel: &automaticupgrades.Channel{StaticVersion: testVersionLow}, + expectError: require.NoError, + expectedVersion: testVersionLow, + }, + { + name: "hard error getting rollout should not fallback to version channels", + rolloutErr: trace.AccessDenied("something is very broken"), + channel: &automaticupgrades.Channel{ + StaticVersion: testVersionLow, + }, + expectError: require.Error, + }, + { + name: "no rollout, error checking channel", + rolloutErr: trace.NotFound("rollout is not here"), + channel: &automaticupgrades.Channel{ForwardURL: brokenChannelUpstream.URL}, + expectError: require.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test setup: building the channel, mock client, and handler with test config. + require.NoError(t, tt.channel.CheckAndSetDefaults()) + h := &Handler{ + cfg: Config{ + AccessPoint: &fakeRolloutAccessPoint{ + rollout: tt.rollout, + err: tt.rolloutErr, + }, + AutomaticUpgradesChannels: map[string]*automaticupgrades.Channel{ + groupName: tt.channel, + }, + }, + } + + // Test execution + result, err := h.autoUpdateAgentVersion(ctx, groupName, "") + tt.expectError(t, err) + require.Equal(t, tt.expectedVersion, result) + }) + } +} + +// TestAutoUpdateAgentShouldUpdate also accidentally tests getTriggerFromWindowThenChannel. +func TestAutoUpdateAgentShouldUpdate(t *testing.T) { + t.Parallel() + + groupName := "test-group" + ctx := context.Background() + + // brokenChannelUpstream is a buggy upstream version server. + // This allows us to craft version channels returning errors. + brokenChannelUpstream := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + })) + t.Cleanup(brokenChannelUpstream.Close) + + cacheClock := clockwork.NewFakeClock() + cmcCache, err := utils.NewFnCache(utils.FnCacheConfig{ + TTL: findEndpointCacheTTL, + Clock: cacheClock, + Context: ctx, + ReloadOnErr: false, + }) + require.NoError(t, err) + t.Cleanup(func() { + cmcCache.Shutdown(ctx) + }) + + // We don't use the cache clock because we are advancing it to invalidate the cmc cache and + // this would interfere with the test logic + clock := clockwork.NewFakeClock() + activeUpgradeWindow := types.AgentUpgradeWindow{UTCStartHour: uint32(clock.Now().Hour())} + inactiveUpgradeWindow := types.AgentUpgradeWindow{UTCStartHour: uint32(clock.Now().Add(2 * time.Hour).Hour())} + tests := []struct { + name string + rollout *autoupdatepb.AutoUpdateAgentRollout + rolloutErr error + channel *automaticupgrades.Channel + upgradeWindow types.AgentUpgradeWindow + cmcErr error + windowLookup bool + expectedTrigger bool + expectError require.ErrorAssertionFunc + }{ + { + name: "trigger is looked up from rollout if it is here, trigger firing", + rollout: &autoupdatepb.AutoUpdateAgentRollout{ + Spec: &autoupdatepb.AutoUpdateAgentRolloutSpec{ + AutoupdateMode: autoupdate.AgentsUpdateModeEnabled, + TargetVersion: testVersionHigh, + Schedule: autoupdate.AgentsScheduleImmediate, + }, + }, + channel: &automaticupgrades.Channel{StaticVersion: testVersionLow}, + expectError: require.NoError, + expectedTrigger: true, + }, + { + name: "trigger is looked up from rollout if it is here, trigger not firing", + rollout: &autoupdatepb.AutoUpdateAgentRollout{ + Spec: &autoupdatepb.AutoUpdateAgentRolloutSpec{ + AutoupdateMode: autoupdate.AgentsUpdateModeDisabled, + TargetVersion: testVersionHigh, + Schedule: autoupdate.AgentsScheduleImmediate, + }, + }, + channel: &automaticupgrades.Channel{StaticVersion: testVersionLow}, + expectError: require.NoError, + expectedTrigger: false, + }, + { + name: "trigger is looked up from channel if rollout is not here and window lookup is disabled, trigger not firing", + rolloutErr: trace.NotFound("rollout is not here"), + channel: &automaticupgrades.Channel{ + StaticVersion: testVersionLow, + Critical: false, + }, + expectError: require.NoError, + expectedTrigger: false, + }, + { + name: "trigger is looked up from channel if rollout is not here and window lookup is disabled, trigger firing", + rolloutErr: trace.NotFound("rollout is not here"), + channel: &automaticupgrades.Channel{ + StaticVersion: testVersionLow, + Critical: true, + }, + expectError: require.NoError, + expectedTrigger: true, + }, + { + name: "trigger is looked up from cmc, then channel if rollout is not here and window lookup is enabled, cmc firing", + rolloutErr: trace.NotFound("rollout is not here"), + channel: &automaticupgrades.Channel{ + StaticVersion: testVersionLow, + Critical: false, + }, + upgradeWindow: activeUpgradeWindow, + windowLookup: true, + expectError: require.NoError, + expectedTrigger: true, + }, + { + name: "trigger is looked up from cmc, then channel if rollout is not here and window lookup is enabled, cmc not firing", + rolloutErr: trace.NotFound("rollout is not here"), + channel: &automaticupgrades.Channel{ + StaticVersion: testVersionLow, + Critical: false, + }, + upgradeWindow: inactiveUpgradeWindow, + windowLookup: true, + expectError: require.NoError, + expectedTrigger: false, + }, + { + name: "trigger is looked up from cmc, then channel if rollout is not here and window lookup is enabled, cmc not firing but channel firing", + rolloutErr: trace.NotFound("rollout is not here"), + channel: &automaticupgrades.Channel{ + StaticVersion: testVersionLow, + Critical: true, + }, + upgradeWindow: inactiveUpgradeWindow, + windowLookup: true, + expectError: require.NoError, + expectedTrigger: true, + }, + { + name: "trigger is looked up from cmc, then channel if rollout is not here and window lookup is enabled, no cmc and channel not firing", + rolloutErr: trace.NotFound("rollout is not here"), + channel: &automaticupgrades.Channel{ + StaticVersion: testVersionLow, + Critical: false, + }, + cmcErr: trace.NotFound("no cmc for this cluster"), + windowLookup: true, + expectError: require.NoError, + expectedTrigger: false, + }, + { + name: "trigger is looked up from cmc, then channel if rollout is not here and window lookup is enabled, no cmc and channel firing", + rolloutErr: trace.NotFound("rollout is not here"), + channel: &automaticupgrades.Channel{ + StaticVersion: testVersionLow, + Critical: true, + }, + cmcErr: trace.NotFound("no cmc for this cluster"), + windowLookup: true, + expectError: require.NoError, + expectedTrigger: true, + }, + { + name: "hard error getting rollout should not fallback to RFD109 trigger", + rolloutErr: trace.AccessDenied("something is very broken"), + channel: &automaticupgrades.Channel{ + StaticVersion: testVersionLow, + }, + expectError: require.Error, + }, + { + name: "no rollout, error checking channel", + rolloutErr: trace.NotFound("rollout is not here"), + channel: &automaticupgrades.Channel{ + ForwardURL: brokenChannelUpstream.URL, + }, + expectError: require.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test setup: building the channel, mock clients, and handler with test config. + cmc := types.NewClusterMaintenanceConfig() + cmc.SetAgentUpgradeWindow(tt.upgradeWindow) + require.NoError(t, tt.channel.CheckAndSetDefaults()) + // Advance cache clock to expire cached cmc + cacheClock.Advance(2 * findEndpointCacheTTL) + h := &Handler{ + cfg: Config{ + AccessPoint: &fakeRolloutAccessPoint{ + rollout: tt.rollout, + err: tt.rolloutErr, + }, + ProxyClient: &fakeCMCAuthClient{ + cmc: cmc, + err: tt.cmcErr, + }, + AutomaticUpgradesChannels: map[string]*automaticupgrades.Channel{ + groupName: tt.channel, + }, + }, + clock: clock, + clusterMaintenanceConfigCache: cmcCache, + } + + // Test execution + result, err := h.autoUpdateAgentShouldUpdate(ctx, groupName, "", tt.windowLookup) + tt.expectError(t, err) + require.Equal(t, tt.expectedTrigger, result) + }) + } +} + +func TestGetVersionFromRollout(t *testing.T) { + t.Parallel() + groupName := "test-group" + + // This test matrix is written based on: + // https://github.com/gravitational/teleport/blob/master/rfd/0184-agent-auto-updates.md#rollout-status-disabled + latestAllTheTime := map[autoupdatepb.AutoUpdateAgentGroupState]string{ + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED: testVersionHigh, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE: testVersionHigh, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE: testVersionHigh, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK: testVersionHigh, + } + + activeDoneOnly := map[autoupdatepb.AutoUpdateAgentGroupState]string{ + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED: testVersionLow, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE: testVersionHigh, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE: testVersionHigh, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK: testVersionLow, + } + + tests := map[string]map[string]map[autoupdatepb.AutoUpdateAgentGroupState]string{ + autoupdate.AgentsUpdateModeDisabled: { + autoupdate.AgentsScheduleImmediate: latestAllTheTime, + autoupdate.AgentsScheduleRegular: latestAllTheTime, + }, + autoupdate.AgentsUpdateModeSuspended: { + autoupdate.AgentsScheduleImmediate: latestAllTheTime, + autoupdate.AgentsScheduleRegular: activeDoneOnly, + }, + autoupdate.AgentsUpdateModeEnabled: { + autoupdate.AgentsScheduleImmediate: latestAllTheTime, + autoupdate.AgentsScheduleRegular: activeDoneOnly, + }, + } + for mode, scheduleCases := range tests { + for schedule, stateCases := range scheduleCases { + for state, expectedVersion := range stateCases { + t.Run(fmt.Sprintf("%s/%s/%s", mode, schedule, state), func(t *testing.T) { + rollout := &autoupdatepb.AutoUpdateAgentRollout{ + Spec: &autoupdatepb.AutoUpdateAgentRolloutSpec{ + StartVersion: testVersionLow, + TargetVersion: testVersionHigh, + Schedule: schedule, + AutoupdateMode: mode, + // Strategy does not affect which version are served + Strategy: autoupdate.AgentsStrategyTimeBased, + }, + Status: &autoupdatepb.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdatepb.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: state, + }, + }, + }, + } + version, err := getVersionFromRollout(rollout, groupName, "") + require.NoError(t, err) + require.Equal(t, expectedVersion, version) + }) + } + } + } +} + +func TestGetTriggerFromRollout(t *testing.T) { + t.Parallel() + groupName := "test-group" + + // This test matrix is written based on: + // https://github.com/gravitational/teleport/blob/master/rfd/0184-agent-auto-updates.md#rollout-status-disabled + neverUpdate := map[autoupdatepb.AutoUpdateAgentGroupState]bool{ + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED: false, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE: false, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE: false, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK: false, + } + alwaysUpdate := map[autoupdatepb.AutoUpdateAgentGroupState]bool{ + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED: true, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE: true, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE: true, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK: true, + } + + tests := map[string]map[string]map[string]map[autoupdatepb.AutoUpdateAgentGroupState]bool{ + autoupdate.AgentsUpdateModeDisabled: { + autoupdate.AgentsStrategyTimeBased: { + autoupdate.AgentsScheduleImmediate: neverUpdate, + autoupdate.AgentsScheduleRegular: neverUpdate, + }, + autoupdate.AgentsStrategyHaltOnError: { + autoupdate.AgentsScheduleImmediate: neverUpdate, + autoupdate.AgentsScheduleRegular: neverUpdate, + }, + }, + autoupdate.AgentsUpdateModeSuspended: { + autoupdate.AgentsStrategyTimeBased: { + autoupdate.AgentsScheduleImmediate: neverUpdate, + autoupdate.AgentsScheduleRegular: neverUpdate, + }, + autoupdate.AgentsStrategyHaltOnError: { + autoupdate.AgentsScheduleImmediate: neverUpdate, + autoupdate.AgentsScheduleRegular: neverUpdate, + }, + }, + autoupdate.AgentsUpdateModeEnabled: { + autoupdate.AgentsStrategyTimeBased: { + autoupdate.AgentsScheduleImmediate: alwaysUpdate, + autoupdate.AgentsScheduleRegular: { + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED: false, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE: false, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE: true, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK: true, + }, + }, + autoupdate.AgentsStrategyHaltOnError: { + autoupdate.AgentsScheduleImmediate: alwaysUpdate, + autoupdate.AgentsScheduleRegular: { + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED: false, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE: true, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE: true, + autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ROLLEDBACK: true, + }, + }, + }, + } + for mode, strategyCases := range tests { + for strategy, scheduleCases := range strategyCases { + for schedule, stateCases := range scheduleCases { + for state, expectedTrigger := range stateCases { + t.Run(fmt.Sprintf("%s/%s/%s/%s", mode, strategy, schedule, state), func(t *testing.T) { + rollout := &autoupdatepb.AutoUpdateAgentRollout{ + Spec: &autoupdatepb.AutoUpdateAgentRolloutSpec{ + StartVersion: testVersionLow, + TargetVersion: testVersionHigh, + Schedule: schedule, + AutoupdateMode: mode, + Strategy: strategy, + }, + Status: &autoupdatepb.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdatepb.AutoUpdateAgentRolloutStatusGroup{ + { + Name: groupName, + State: state, + }, + }, + }, + } + shouldUpdate, err := getTriggerFromRollout(rollout, groupName, "") + require.NoError(t, err) + require.Equal(t, expectedTrigger, shouldUpdate) + }) + } + } + } + } +} + +func TestGetGroup(t *testing.T) { + groupName := "test-group" + t.Parallel() + tests := []struct { + name string + rollout *autoupdatepb.AutoUpdateAgentRollout + expectedResult *autoupdatepb.AutoUpdateAgentRolloutStatusGroup + expectError require.ErrorAssertionFunc + }{ + { + name: "nil", + expectError: require.Error, + }, + { + name: "nil status", + rollout: &autoupdatepb.AutoUpdateAgentRollout{}, + expectError: require.Error, + }, + { + name: "nil status groups", + rollout: &autoupdatepb.AutoUpdateAgentRollout{Status: &autoupdatepb.AutoUpdateAgentRolloutStatus{}}, + expectError: require.Error, + }, + { + name: "empty status groups", + rollout: &autoupdatepb.AutoUpdateAgentRollout{ + Status: &autoupdatepb.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdatepb.AutoUpdateAgentRolloutStatusGroup{}, + }, + }, + expectError: require.Error, + }, + { + name: "group matching name", + rollout: &autoupdatepb.AutoUpdateAgentRollout{ + Status: &autoupdatepb.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdatepb.AutoUpdateAgentRolloutStatusGroup{ + {Name: "foo", State: 1}, + {Name: "bar", State: 1}, + {Name: groupName, State: 2}, + {Name: "baz", State: 1}, + }, + }, + }, + expectedResult: &autoupdatepb.AutoUpdateAgentRolloutStatusGroup{ + Name: groupName, + State: 2, + }, + expectError: require.NoError, + }, + { + name: "no group matching name, should fallback to default", + rollout: &autoupdatepb.AutoUpdateAgentRollout{ + Status: &autoupdatepb.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdatepb.AutoUpdateAgentRolloutStatusGroup{ + {Name: "foo", State: 1}, + {Name: "bar", State: 1}, + {Name: "baz", State: 1}, + }, + }, + }, + expectedResult: &autoupdatepb.AutoUpdateAgentRolloutStatusGroup{ + Name: "baz", + State: 1, + }, + expectError: require.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := getGroup(tt.rollout, groupName) + tt.expectError(t, err) + require.Equal(t, tt.expectedResult, result) + }) + } +} + +type mockRFD109VersionServer struct { + t *testing.T + channels map[string]channelStub +} + +type channelStub struct { + // with our without the leading "v" + version string + critical bool + fail bool +} + +func (m *mockRFD109VersionServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var path string + var writeResp func(w http.ResponseWriter, stub channelStub) error + + switch { + case strings.HasSuffix(r.URL.Path, constants.VersionPath): + path = strings.Trim(strings.TrimSuffix(r.URL.Path, constants.VersionPath), "/") + writeResp = func(w http.ResponseWriter, stub channelStub) error { + _, err := w.Write([]byte(stub.version)) + return err + } + case strings.HasSuffix(r.URL.Path, constants.MaintenancePath): + path = strings.Trim(strings.TrimSuffix(r.URL.Path, constants.MaintenancePath), "/") + writeResp = func(w http.ResponseWriter, stub channelStub) error { + response := "no" + if stub.critical { + response = "yes" + } + _, err := w.Write([]byte(response)) + return err + } + default: + assert.Fail(m.t, "unsupported path %q", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + + channel, ok := m.channels[path] + if !ok { + w.WriteHeader(http.StatusNotFound) + assert.Fail(m.t, "channel %q not found", path) + return + } + if channel.fail { + w.WriteHeader(http.StatusInternalServerError) + return + } + assert.NoError(m.t, writeResp(w, channel), "failed to write response") +} + +func TestGetVersionFromChannel(t *testing.T) { + t.Parallel() + ctx := context.Background() + + channelName := "test-channel" + + mock := mockRFD109VersionServer{ + t: t, + channels: map[string]channelStub{ + "broken": {fail: true}, + "with-leading-v": {version: "v" + testVersionHigh}, + "without-leading-v": {version: testVersionHigh}, + "low": {version: testVersionLow}, + }, + } + srv := httptest.NewServer(http.HandlerFunc(mock.ServeHTTP)) + t.Cleanup(srv.Close) + + tests := []struct { + name string + channels automaticupgrades.Channels + expectedResult string + expectError require.ErrorAssertionFunc + }{ + { + name: "channel with leading v", + channels: automaticupgrades.Channels{ + channelName: {ForwardURL: srv.URL + "/with-leading-v"}, + "default": {ForwardURL: srv.URL + "/low"}, + }, + expectedResult: testVersionHigh, + expectError: require.NoError, + }, + { + name: "channel without leading v", + channels: automaticupgrades.Channels{ + channelName: {ForwardURL: srv.URL + "/without-leading-v"}, + "default": {ForwardURL: srv.URL + "/low"}, + }, + expectedResult: testVersionHigh, + expectError: require.NoError, + }, + { + name: "fallback to default with leading v", + channels: automaticupgrades.Channels{ + "default": {ForwardURL: srv.URL + "/with-leading-v"}, + }, + expectedResult: testVersionHigh, + expectError: require.NoError, + }, + { + name: "fallback to default without leading v", + channels: automaticupgrades.Channels{ + "default": {ForwardURL: srv.URL + "/without-leading-v"}, + }, + expectedResult: testVersionHigh, + expectError: require.NoError, + }, + { + name: "broken channel", + channels: automaticupgrades.Channels{ + channelName: {ForwardURL: srv.URL + "/broken"}, + "default": {ForwardURL: srv.URL + "/without-leading-v"}, + }, + expectError: require.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test setup + require.NoError(t, tt.channels.CheckAndSetDefaults()) + + // Test execution + result, err := getVersionFromChannel(ctx, tt.channels, channelName) + tt.expectError(t, err) + require.Equal(t, tt.expectedResult, result) + }) + } +} + +func TestGetTriggerFromChannel(t *testing.T) { + t.Parallel() + ctx := context.Background() + + channelName := "test-channel" + + mock := mockRFD109VersionServer{ + t: t, + channels: map[string]channelStub{ + "broken": {fail: true}, + "critical": {critical: true}, + "non-critical": {critical: false}, + }, + } + srv := httptest.NewServer(http.HandlerFunc(mock.ServeHTTP)) + t.Cleanup(srv.Close) + + tests := []struct { + name string + channels automaticupgrades.Channels + expectedResult bool + expectError require.ErrorAssertionFunc + }{ + { + name: "critical channel", + channels: automaticupgrades.Channels{ + channelName: {ForwardURL: srv.URL + "/critical"}, + "default": {ForwardURL: srv.URL + "/non-critical"}, + }, + expectedResult: true, + expectError: require.NoError, + }, + { + name: "non-critical channel", + channels: automaticupgrades.Channels{ + channelName: {ForwardURL: srv.URL + "/non-critical"}, + "default": {ForwardURL: srv.URL + "/critical"}, + }, + expectedResult: false, + expectError: require.NoError, + }, + { + name: "fallback to default which is critical", + channels: automaticupgrades.Channels{ + "default": {ForwardURL: srv.URL + "/critical"}, + }, + expectedResult: true, + expectError: require.NoError, + }, + { + name: "fallback to default which is non-critical", + channels: automaticupgrades.Channels{ + "default": {ForwardURL: srv.URL + "/non-critical"}, + }, + expectedResult: false, + expectError: require.NoError, + }, + { + name: "broken channel", + channels: automaticupgrades.Channels{ + channelName: {ForwardURL: srv.URL + "/broken"}, + "default": {ForwardURL: srv.URL + "/critical"}, + }, + expectError: require.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test setup + require.NoError(t, tt.channels.CheckAndSetDefaults()) + + // Test execution + result, err := getTriggerFromChannel(ctx, tt.channels, channelName) + tt.expectError(t, err) + require.Equal(t, tt.expectedResult, result) + }) + } +} diff --git a/lib/web/automaticupgrades.go b/lib/web/autoupdate_rfd109.go similarity index 70% rename from lib/web/automaticupgrades.go rename to lib/web/autoupdate_rfd109.go index 6b7833dc629e2..d2dd43fdb6f3f 100644 --- a/lib/web/automaticupgrades.go +++ b/lib/web/autoupdate_rfd109.go @@ -1,6 +1,6 @@ /* * Teleport - * Copyright (C) 2023 Gravitational, Inc. + * Copyright (C) 2024 Gravitational, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by @@ -21,6 +21,7 @@ package web import ( "context" "errors" + "fmt" "net/http" "strings" "time" @@ -28,17 +29,16 @@ import ( "github.com/gravitational/trace" "github.com/julienschmidt/httprouter" - "github.com/gravitational/teleport/lib/automaticupgrades" "github.com/gravitational/teleport/lib/automaticupgrades/constants" "github.com/gravitational/teleport/lib/automaticupgrades/version" ) const defaultChannelTimeout = 5 * time.Second -// automaticUpgrades implements a version server in the Teleport Proxy. +// automaticUpgrades109 implements a version server in the Teleport Proxy following the RFD 109 spec. // It is configured through the Teleport Proxy configuration and tells agent updaters // which version they should install. -func (h *Handler) automaticUpgrades(w http.ResponseWriter, r *http.Request, p httprouter.Params) (interface{}, error) { +func (h *Handler) automaticUpgrades109(w http.ResponseWriter, r *http.Request, p httprouter.Params) (interface{}, error) { if h.cfg.AutomaticUpgradesChannels == nil { return nil, trace.AccessDenied("This proxy is not configured to serve automatic upgrades channels.") } @@ -59,31 +59,25 @@ func (h *Handler) automaticUpgrades(w http.ResponseWriter, r *http.Request, p ht return nil, trace.BadParameter("a channel name is required") } - // We check if the channel is configured - channel, ok := h.cfg.AutomaticUpgradesChannels[channelName] - if !ok { - return nil, trace.NotFound("channel %s not found", channelName) - } - // Finally, we treat the request based on its type switch requestType { case "version": h.log.Debugf("Agent requesting version for channel %s", channelName) - return h.automaticUpgradesVersion(w, r, channel) + return h.automaticUpgradesVersion109(w, r, channelName) case "critical": h.log.Debugf("Agent requesting criticality for channel %s", channelName) - return h.automaticUpgradesCritical(w, r, channel) + return h.automaticUpgradesCritical109(w, r, channelName) default: return nil, trace.BadParameter("requestType path must end with 'version' or 'critical'") } } -// automaticUpgradesVersion handles version requests from upgraders -func (h *Handler) automaticUpgradesVersion(w http.ResponseWriter, r *http.Request, channel *automaticupgrades.Channel) (interface{}, error) { +// automaticUpgradesVersion109 handles version requests from upgraders +func (h *Handler) automaticUpgradesVersion109(w http.ResponseWriter, r *http.Request, channelName string) (interface{}, error) { ctx, cancel := context.WithTimeout(r.Context(), defaultChannelTimeout) defer cancel() - targetVersion, err := channel.GetVersion(ctx) + targetVersion, err := h.autoUpdateAgentVersion(ctx, channelName, "" /* updater UUID */) if err != nil { // If the error is that the upstream channel has no version // We gracefully handle by serving "none" @@ -96,16 +90,20 @@ func (h *Handler) automaticUpgradesVersion(w http.ResponseWriter, r *http.Reques return nil, trace.Wrap(err) } - _, err = w.Write([]byte(targetVersion)) + // RFD 109 specifies that version from channels must have the leading "v". + // As h.autoUpdateAgentVersion doesn't, we must add it. + _, err = fmt.Fprintf(w, "v%s", targetVersion) return nil, trace.Wrap(err) } -// automaticUpgradesCritical handles criticality requests from upgraders -func (h *Handler) automaticUpgradesCritical(w http.ResponseWriter, r *http.Request, channel *automaticupgrades.Channel) (interface{}, error) { +// automaticUpgradesCritical109 handles criticality requests from upgraders +func (h *Handler) automaticUpgradesCritical109(w http.ResponseWriter, r *http.Request, channelName string) (interface{}, error) { ctx, cancel := context.WithTimeout(r.Context(), defaultChannelTimeout) defer cancel() - critical, err := channel.GetCritical(ctx) + // RFD109 agents already retrieve maintenance windows from the CMC, no need to + // do a maintenance window lookup for them. + critical, err := h.autoUpdateAgentShouldUpdate(ctx, channelName, "" /* updater UUID */, false /* window lookup */) if err != nil { return nil, trace.Wrap(err) } diff --git a/lib/web/autoupdate_rfd184.go b/lib/web/autoupdate_rfd184.go new file mode 100644 index 0000000000000..99d1189383fae --- /dev/null +++ b/lib/web/autoupdate_rfd184.go @@ -0,0 +1,93 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package web + +import ( + "context" + + "github.com/gravitational/trace" + + "github.com/gravitational/teleport" + "github.com/gravitational/teleport/api" + "github.com/gravitational/teleport/api/client/webclient" + autoupdatepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + "github.com/gravitational/teleport/api/types/autoupdate" +) + +// automaticUpdateSettings184 crafts the automatic updates part of the ping/find response +// as described in RFD-184 (agents) and RFD-144 (tools). +func (h *Handler) automaticUpdateSettings184(ctx context.Context, group, updaterUUID string) webclient.AutoUpdateSettings { + // Tools auto updates section. + autoUpdateConfig, err := h.cfg.AccessPoint.GetAutoUpdateConfig(ctx) + // TODO(vapopov) DELETE IN v18.0.0 check of IsNotImplemented, must be backported to all latest supported versions. + if err != nil && !trace.IsNotFound(err) && !trace.IsNotImplemented(err) { + h.log.WithError(err).Error("failed to receive AutoUpdateConfig") + } + + autoUpdateVersion, err := h.cfg.AccessPoint.GetAutoUpdateVersion(ctx) + // TODO(vapopov) DELETE IN v18.0.0 check of IsNotImplemented, must be backported to all latest supported versions. + if err != nil && !trace.IsNotFound(err) && !trace.IsNotImplemented(err) { + h.log.WithError(err).Error("failed to receive AutoUpdateVersion") + } + + // Agent auto updates section. + agentVersion, err := h.autoUpdateAgentVersion(ctx, group, updaterUUID) + if err != nil { + h.log.WithError(err).Error("failed to resolve AgentVersion") + // Defaulting to current version + agentVersion = teleport.Version + } + // If the source of truth is RFD 109 configuration (channels + CMC) we must emulate the + // RFD109 agent maintenance window behavior by looking up the CMC and checking if + // we are in a maintenance window. + shouldUpdate, err := h.autoUpdateAgentShouldUpdate(ctx, group, updaterUUID, true /* window lookup */) + if err != nil { + h.log.WithError(err).Error("failed to resolve AgentAutoUpdate") + // Failing open + shouldUpdate = false + } + + return webclient.AutoUpdateSettings{ + ToolsAutoUpdate: getToolsAutoUpdate(autoUpdateConfig), + ToolsVersion: getToolsVersion(autoUpdateVersion), + AgentUpdateJitterSeconds: DefaultAgentUpdateJitterSeconds, + AgentVersion: agentVersion, + AgentAutoUpdate: shouldUpdate, + } +} + +func getToolsAutoUpdate(config *autoupdatepb.AutoUpdateConfig) bool { + // If we can't get the AU config or if AUs are not configured, we default to "disabled". + // This ensures we fail open and don't accidentally update agents if something is going wrong. + // If we want to enable AUs by default, it would be better to create a default "autoupdate_config" resource + // than changing this logic. + if config.GetSpec().GetTools() != nil { + return config.GetSpec().GetTools().GetMode() == autoupdate.ToolsUpdateModeEnabled + } + return false +} + +func getToolsVersion(version *autoupdatepb.AutoUpdateVersion) string { + // If we can't get the AU version or tools AU version is not specified, we default to the current proxy version. + // This ensures we always advertise a version compatible with the cluster. + if version.GetSpec().GetTools() == nil { + return api.Version + } + return version.GetSpec().GetTools().GetTargetVersion() +} diff --git a/lib/web/integrations_awsoidc.go b/lib/web/integrations_awsoidc.go index e890d32c09da9..e5f52d1dbc4b5 100644 --- a/lib/web/integrations_awsoidc.go +++ b/lib/web/integrations_awsoidc.go @@ -118,13 +118,15 @@ func (h *Handler) awsOIDCDeployService(w http.ResponseWriter, r *http.Request, p teleportVersionTag := teleport.Version if automaticUpgrades(h.ClusterFeatures) { - cloudStableVersion, err := h.cfg.AutomaticUpgradesChannels.DefaultVersion(ctx) + const group, updaterUUID = "", "" + autoUpdateVersion, err := h.autoUpdateAgentVersion(r.Context(), group, updaterUUID) if err != nil { - return "", trace.Wrap(err) + h.log.WithError(err).WithField("version", teleport.Version).Warn( + "Cannot read autoupdate target version, falling back to our own version", + ) + } else { + teleportVersionTag = autoUpdateVersion } - - // cloudStableVersion has vX.Y.Z format, however the container image tag does not include the `v`. - teleportVersionTag = strings.TrimPrefix(cloudStableVersion, "v") } deployServiceResp, err := clt.IntegrationAWSOIDCClient().DeployService(ctx, &integrationv1.DeployServiceRequest{ @@ -171,13 +173,16 @@ func (h *Handler) awsOIDCDeployDatabaseServices(w http.ResponseWriter, r *http.R teleportVersionTag := teleport.Version if automaticUpgrades(h.ClusterFeatures) { - cloudStableVersion, err := h.cfg.AutomaticUpgradesChannels.DefaultVersion(ctx) + const group, updaterUUID = "", "" + autoUpdateVersion, err := h.autoUpdateAgentVersion(r.Context(), group, updaterUUID) if err != nil { - return "", trace.Wrap(err) + h.log.WithError(err).WithField("version", teleport.Version).Warn( + "Cannot read autoupdate target version, falling back to our own version", + ) + } else { + teleportVersionTag = autoUpdateVersion } - // cloudStableVersion has vX.Y.Z format, however the container image tag does not include the `v`. - teleportVersionTag = strings.TrimPrefix(cloudStableVersion, "v") } iamTokenName := deployserviceconfig.DefaultTeleportIAMTokenName @@ -271,8 +276,8 @@ func (h *Handler) awsOIDCConfigureDeployServiceIAM(w http.ResponseWriter, r *htt fmt.Sprintf("--task-role=%s", libutils.UnixShellQuote(taskRole)), } script, err := oneoff.BuildScript(oneoff.OneOffScriptParams{ - TeleportArgs: strings.Join(argsList, " "), - SuccessMessage: "Success! You can now go back to the browser to complete the database enrollment.", + EntrypointArgs: strings.Join(argsList, " "), + SuccessMessage: "Success! You can now go back to the Teleport Web UI to complete the database enrollment.", }) if err != nil { return nil, trace.Wrap(err) @@ -307,8 +312,8 @@ func (h *Handler) awsOIDCConfigureEICEIAM(w http.ResponseWriter, r *http.Request fmt.Sprintf("--role=%s", libutils.UnixShellQuote(role)), } script, err := oneoff.BuildScript(oneoff.OneOffScriptParams{ - TeleportArgs: strings.Join(argsList, " "), - SuccessMessage: "Success! You can now go back to the browser to complete the EC2 enrollment.", + EntrypointArgs: strings.Join(argsList, " "), + SuccessMessage: "Success! You can now go back to the Teleport Web UI to complete the EC2 enrollment.", }) if err != nil { return nil, trace.Wrap(err) @@ -785,8 +790,8 @@ func (h *Handler) awsOIDCConfigureIdP(w http.ResponseWriter, r *http.Request, p } script, err := oneoff.BuildScript(oneoff.OneOffScriptParams{ - TeleportArgs: strings.Join(argsList, " "), - SuccessMessage: "Success! You can now go back to the browser to use the integration with AWS.", + EntrypointArgs: strings.Join(argsList, " "), + SuccessMessage: "Success! You can now go back to the Teleport Web UI to use the integration with AWS.", }) if err != nil { return nil, trace.Wrap(err) @@ -820,8 +825,8 @@ func (h *Handler) awsOIDCConfigureListDatabasesIAM(w http.ResponseWriter, r *htt fmt.Sprintf("--role=%s", libutils.UnixShellQuote(role)), } script, err := oneoff.BuildScript(oneoff.OneOffScriptParams{ - TeleportArgs: strings.Join(argsList, " "), - SuccessMessage: "Success! You can now go back to the browser to complete the Database enrollment.", + EntrypointArgs: strings.Join(argsList, " "), + SuccessMessage: "Success! You can now go back to the Teleport Web UI to complete the Database enrollment.", }) if err != nil { return nil, trace.Wrap(err) @@ -860,8 +865,8 @@ func (h *Handler) awsAccessGraphOIDCSync(w http.ResponseWriter, r *http.Request, fmt.Sprintf("--role=%s", libutils.UnixShellQuote(role)), } script, err := oneoff.BuildScript(oneoff.OneOffScriptParams{ - TeleportArgs: strings.Join(argsList, " "), - SuccessMessage: "Success! You can now go back to the browser to complete the Access Graph AWS Sync enrollment.", + EntrypointArgs: strings.Join(argsList, " "), + SuccessMessage: "Success! You can now go back to the Teleport Web UI to complete the Access Graph AWS Sync enrollment.", }) if err != nil { return nil, trace.Wrap(err) diff --git a/lib/web/integrations_awsoidc_test.go b/lib/web/integrations_awsoidc_test.go index 601717192873f..03baa572c6088 100644 --- a/lib/web/integrations_awsoidc_test.go +++ b/lib/web/integrations_awsoidc_test.go @@ -147,7 +147,7 @@ func TestBuildDeployServiceConfigureIAMScript(t *testing.T) { } require.Contains(t, string(resp.Bytes()), - fmt.Sprintf("teleportArgs='%s'\n", tc.expectedTeleportArgs), + fmt.Sprintf("entrypointArgs='%s'\n", tc.expectedTeleportArgs), ) }) } @@ -235,7 +235,7 @@ func TestBuildEICEConfigureIAMScript(t *testing.T) { } require.Contains(t, string(resp.Bytes()), - fmt.Sprintf("teleportArgs='%s'\n", tc.expectedTeleportArgs), + fmt.Sprintf("entrypointArgs='%s'\n", tc.expectedTeleportArgs), ) }) } @@ -380,7 +380,7 @@ func TestBuildAWSOIDCIdPConfigureScript(t *testing.T) { } require.Contains(t, string(resp.Bytes()), - fmt.Sprintf("teleportArgs='%s'\n", tc.expectedTeleportArgs), + fmt.Sprintf("entrypointArgs='%s'\n", tc.expectedTeleportArgs), ) }) } @@ -468,7 +468,7 @@ func TestBuildListDatabasesConfigureIAMScript(t *testing.T) { } require.Contains(t, string(resp.Bytes()), - fmt.Sprintf("teleportArgs='%s'\n", tc.expectedTeleportArgs), + fmt.Sprintf("entrypointArgs='%s'\n", tc.expectedTeleportArgs), ) }) } diff --git a/lib/web/join_tokens.go b/lib/web/join_tokens.go index 6eae2795253e2..d42e349b75a4e 100644 --- a/lib/web/join_tokens.go +++ b/lib/web/join_tokens.go @@ -17,7 +17,6 @@ limitations under the License. package web import ( - "bytes" "context" "encoding/hex" "fmt" @@ -25,33 +24,25 @@ import ( "net/http" "net/url" "reflect" - "regexp" "sort" - "strconv" "strings" "time" "github.com/google/uuid" "github.com/gravitational/trace" "github.com/julienschmidt/httprouter" - "k8s.io/apimachinery/pkg/util/validation" "github.com/gravitational/teleport/api/client/proto" "github.com/gravitational/teleport/api/types" apiutils "github.com/gravitational/teleport/api/utils" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/httplib" - "github.com/gravitational/teleport/lib/modules" "github.com/gravitational/teleport/lib/tlsca" "github.com/gravitational/teleport/lib/utils" "github.com/gravitational/teleport/lib/web/scripts" "github.com/gravitational/teleport/lib/web/ui" ) -const ( - stableCloudChannelRepo = "stable/cloud" -) - // nodeJoinToken contains node token fields for the UI. type nodeJoinToken struct { // ID is token ID. @@ -73,15 +64,9 @@ type scriptSettings struct { appURI string joinMethod string databaseInstallMode bool - installUpdater bool discoveryInstallMode bool discoveryGroup string - - // automaticUpgradesVersion is the target automatic upgrades version. - // The version must be valid semver, with the leading 'v'. e.g. v15.0.0-dev - // Required when installUpdater is true. - automaticUpgradesVersion string } // automaticUpgrades returns whether automaticUpgrades should be enabled. @@ -208,41 +193,16 @@ func (h *Handler) createTokenHandle(w http.ResponseWriter, r *http.Request, para }, nil } -// getAutoUpgrades checks if automaticUpgrades are enabled and returns the -// version that should be used according to auto upgrades default channel. -func (h *Handler) getAutoUpgrades(ctx context.Context) (bool, string, error) { - var autoUpgradesVersion string - var err error - autoUpgrades := automaticUpgrades(h.ClusterFeatures) - if autoUpgrades { - autoUpgradesVersion, err = h.cfg.AutomaticUpgradesChannels.DefaultVersion(ctx) - if err != nil { - log.WithError(err).Info("Failed to get auto upgrades version.") - return false, "", trace.Wrap(err) - } - } - return autoUpgrades, autoUpgradesVersion, nil - -} - func (h *Handler) getNodeJoinScriptHandle(w http.ResponseWriter, r *http.Request, params httprouter.Params) (interface{}, error) { httplib.SetScriptHeaders(w.Header()) - autoUpgrades, autoUpgradesVersion, err := h.getAutoUpgrades(r.Context()) - if err != nil { - w.Write(scripts.ErrorBashScript) - return nil, nil - } - settings := scriptSettings{ - token: params.ByName("token"), - appInstallMode: false, - joinMethod: r.URL.Query().Get("method"), - installUpdater: autoUpgrades, - automaticUpgradesVersion: autoUpgradesVersion, + token: params.ByName("token"), + appInstallMode: false, + joinMethod: r.URL.Query().Get("method"), } - script, err := getJoinScript(r.Context(), settings, h.GetProxyClient()) + script, err := h.getJoinScript(r.Context(), settings) if err != nil { log.WithError(err).Info("Failed to return the node install script.") w.Write(scripts.ErrorBashScript) @@ -276,22 +236,14 @@ func (h *Handler) getAppJoinScriptHandle(w http.ResponseWriter, r *http.Request, return nil, nil } - autoUpgrades, autoUpgradesVersion, err := h.getAutoUpgrades(r.Context()) - if err != nil { - w.Write(scripts.ErrorBashScript) - return nil, nil - } - settings := scriptSettings{ - token: params.ByName("token"), - appInstallMode: true, - appName: name, - appURI: uri, - installUpdater: autoUpgrades, - automaticUpgradesVersion: autoUpgradesVersion, + token: params.ByName("token"), + appInstallMode: true, + appName: name, + appURI: uri, } - script, err := getJoinScript(r.Context(), settings, h.GetProxyClient()) + script, err := h.getJoinScript(r.Context(), settings) if err != nil { log.WithError(err).Info("Failed to return the app install script.") w.Write(scripts.ErrorBashScript) @@ -310,20 +262,12 @@ func (h *Handler) getAppJoinScriptHandle(w http.ResponseWriter, r *http.Request, func (h *Handler) getDatabaseJoinScriptHandle(w http.ResponseWriter, r *http.Request, params httprouter.Params) (interface{}, error) { httplib.SetScriptHeaders(w.Header()) - autoUpgrades, autoUpgradesVersion, err := h.getAutoUpgrades(r.Context()) - if err != nil { - w.Write(scripts.ErrorBashScript) - return nil, nil - } - settings := scriptSettings{ - token: params.ByName("token"), - databaseInstallMode: true, - installUpdater: autoUpgrades, - automaticUpgradesVersion: autoUpgradesVersion, + token: params.ByName("token"), + databaseInstallMode: true, } - script, err := getJoinScript(r.Context(), settings, h.GetProxyClient()) + script, err := h.getJoinScript(r.Context(), settings) if err != nil { log.WithError(err).Info("Failed to return the database install script.") w.Write(scripts.ErrorBashScript) @@ -362,7 +306,7 @@ func (h *Handler) getDiscoveryJoinScriptHandle(w http.ResponseWriter, r *http.Re discoveryGroup: discoveryGroup, } - script, err := getJoinScript(r.Context(), settings, h.GetProxyClient()) + script, err := h.getJoinScript(r.Context(), settings) if err != nil { log.WithError(err).Info("Failed to return the discovery install script.") w.Write(scripts.ErrorBashScript) @@ -378,8 +322,9 @@ func (h *Handler) getDiscoveryJoinScriptHandle(w http.ResponseWriter, r *http.Re return nil, nil } -func getJoinScript(ctx context.Context, settings scriptSettings, m nodeAPIGetter) (string, error) { - switch types.JoinMethod(settings.joinMethod) { +func (h *Handler) getJoinScript(ctx context.Context, settings scriptSettings) (string, error) { + joinMethod := types.JoinMethod(settings.joinMethod) + switch joinMethod { case types.JoinMethodUnspecified, types.JoinMethodToken: if err := validateJoinToken(settings.token); err != nil { return "", trace.Wrap(err) @@ -389,133 +334,55 @@ func getJoinScript(ctx context.Context, settings scriptSettings, m nodeAPIGetter return "", trace.BadParameter("join method %q is not supported via script", settings.joinMethod) } + clt := h.GetProxyClient() + // The provided token can be attacker controlled, so we must validate // it with the backend before using it to generate the script. - token, err := m.GetToken(ctx, settings.token) + token, err := clt.GetToken(ctx, settings.token) if err != nil { return "", trace.BadParameter("invalid token") } - // Get hostname and port from proxy server address. - proxyServers, err := m.GetProxies() - if err != nil { - return "", trace.Wrap(err) - } - - if len(proxyServers) == 0 { - return "", trace.NotFound("no proxy servers found") - } - - version := proxyServers[0].GetTeleportVersion() - - publicAddr := proxyServers[0].GetPublicAddr() - if publicAddr == "" { - return "", trace.Errorf("proxy public_addr is not set, you must set proxy_service.public_addr to the publicly reachable address of the proxy before you can generate a node join script") - } - - hostname, portStr, err := utils.SplitHostPort(publicAddr) - if err != nil { - return "", trace.Wrap(err) - } + // TODO(hugoShaka): hit the local accesspoint which has a cache instead of asking the auth every time. // Get the CA pin hashes of the cluster to join. - localCAResponse, err := m.GetClusterCACert(ctx) + localCAResponse, err := clt.GetClusterCACert(ctx) if err != nil { return "", trace.Wrap(err) } + caPins, err := tlsca.CalculatePins(localCAResponse.TLSCA) if err != nil { return "", trace.Wrap(err) } - labelsList := []string{} - for labelKey, labelValues := range token.GetSuggestedLabels() { - labels := strings.Join(labelValues, " ") - labelsList = append(labelsList, fmt.Sprintf("%s=%s", labelKey, labels)) - } - - var dbServiceResourceLabels []string - if settings.databaseInstallMode { - suggestedAgentMatcherLabels := token.GetSuggestedAgentMatcherLabels() - dbServiceResourceLabels, err = scripts.MarshalLabelsYAML(suggestedAgentMatcherLabels, 6) - if err != nil { - return "", trace.Wrap(err) - } - } - - var buf bytes.Buffer - // If app install mode is requested but parameters are blank for some reason, - // we need to return an error. - if settings.appInstallMode { - if errs := validation.IsDNS1035Label(settings.appName); len(errs) > 0 { - return "", trace.BadParameter("appName %q must be a valid DNS subdomain: https://goteleport.com/docs/application-access/guides/connecting-apps/#application-name", settings.appName) - } - if !appURIPattern.MatchString(settings.appURI) { - return "", trace.BadParameter("appURI %q contains invalid characters", settings.appURI) - } - } - - if settings.discoveryInstallMode { - if settings.discoveryGroup == "" { - return "", trace.BadParameter("discovery group is required") - } - } - - packageName := types.PackageNameOSS - if modules.GetModules().BuildType() == modules.BuildEnterprise { - packageName = types.PackageNameEnt - } - - // By default, it will use `stable/v`, eg stable/v12 - repoChannel := "" - - // The install script will install the updater (teleport-ent-updater) for Cloud customers enrolled in Automatic Upgrades. - // The repo channel used must be `stable/cloud` which has the available packages for the Cloud Customer's agents. - // It pins the teleport version to the one specified by the default version channel - // This ensures the initial installed version is the same as the `teleport-ent-updater` would install. - if settings.installUpdater { - if settings.automaticUpgradesVersion == "" { - return "", trace.Wrap(err, "automatic upgrades version must be set when installUpdater is true") - } - - repoChannel = stableCloudChannelRepo - // automaticUpgradesVersion has vX.Y.Z format, however the script - // expects the version to not include the `v` so we strip it - version = strings.TrimPrefix(settings.automaticUpgradesVersion, "v") - } - - // This section relies on Go's default zero values to make sure that the settings - // are correct when not installing an app. - err = scripts.InstallNodeBashScript.Execute(&buf, map[string]interface{}{ - "token": settings.token, - "hostname": hostname, - "port": portStr, - // The install.sh script has some manually generated configs and some - // generated by the `teleport config` commands. The old bash - // version used space delimited values whereas the teleport command uses - // a comma delimeter. The Old version can be removed when the install.sh - // file has been completely converted over. - "caPinsOld": strings.Join(caPins, " "), - "caPins": strings.Join(caPins, ","), - "packageName": packageName, - "repoChannel": repoChannel, - "installUpdater": strconv.FormatBool(settings.installUpdater), - "version": utils.UnixShellQuote(version), - "appInstallMode": strconv.FormatBool(settings.appInstallMode), - "appName": utils.UnixShellQuote(settings.appName), - "appURI": utils.UnixShellQuote(settings.appURI), - "joinMethod": utils.UnixShellQuote(settings.joinMethod), - "labels": strings.Join(labelsList, ","), - "databaseInstallMode": strconv.FormatBool(settings.databaseInstallMode), - "db_service_resource_labels": dbServiceResourceLabels, - "discoveryInstallMode": settings.discoveryInstallMode, - "discoveryGroup": utils.UnixShellQuote(settings.discoveryGroup), - }) + installOpts, err := h.installScriptOptions(ctx) if err != nil { - return "", trace.Wrap(err) - } - - return buf.String(), nil + return "", trace.Wrap(err, "Building install script options") + } + + nodeInstallOpts := scripts.InstallNodeScriptOptions{ + InstallOptions: installOpts, + Token: token.GetName(), + CAPins: caPins, + // We are using the joinMethod from the script settings instead of the one from the token + // to reproduce the previous script behavior. I'm also afraid that using the + // join method from the token would provide an oracle for an attacker wanting to discover + // the join method. + // We might want to change this in the future to lookup the join method from the token + // to avoid potential mismatch and allow the caller to not care about the join method. + JoinMethod: joinMethod, + Labels: token.GetSuggestedLabels(), + LabelMatchers: token.GetSuggestedAgentMatcherLabels(), + AppServiceEnabled: settings.appInstallMode, + AppName: settings.appName, + AppURI: settings.appURI, + DatabaseServiceEnabled: settings.databaseInstallMode, + DiscoveryServiceEnabled: settings.discoveryInstallMode, + DiscoveryGroup: settings.discoveryGroup, + } + + return scripts.GetNodeInstallScript(ctx, nodeInstallOpts) } // validateJoinToken validate a join token. @@ -605,17 +472,3 @@ func isSameAzureRuleSet(r1, r2 []*types.ProvisionTokenSpecV2Azure_Rule) bool { sortAzureRules(r2) return reflect.DeepEqual(r1, r2) } - -type nodeAPIGetter interface { - // GetToken looks up a provisioning token. - GetToken(ctx context.Context, token string) (types.ProvisionToken, error) - - // GetClusterCACert returns the CAs for the local cluster without signing keys. - GetClusterCACert(ctx context.Context) (*proto.GetClusterCACertResponse, error) - - // GetProxies returns a list of registered proxies. - GetProxies() ([]types.Server, error) -} - -// appURIPattern is a regexp excluding invalid characters from application URIs. -var appURIPattern = regexp.MustCompile(`^[-\w/:. ]+$`) diff --git a/lib/web/join_tokens_test.go b/lib/web/join_tokens_test.go index 30bff2cff3663..561035ca22644 100644 --- a/lib/web/join_tokens_test.go +++ b/lib/web/join_tokens_test.go @@ -20,18 +20,28 @@ import ( "context" "encoding/hex" "fmt" + "math/rand/v2" "regexp" + "strconv" "testing" + "time" "github.com/gravitational/trace" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/durationpb" "github.com/gravitational/teleport" "github.com/gravitational/teleport/api/client/proto" + autoupdatev1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" "github.com/gravitational/teleport/api/types" - "github.com/gravitational/teleport/api/utils" + "github.com/gravitational/teleport/api/types/autoupdate" + apiutils "github.com/gravitational/teleport/api/utils" + "github.com/gravitational/teleport/lib/auth/authclient" + "github.com/gravitational/teleport/lib/automaticupgrades" "github.com/gravitational/teleport/lib/fixtures" "github.com/gravitational/teleport/lib/modules" + "github.com/gravitational/teleport/lib/utils" ) func TestGenerateIAMTokenName(t *testing.T) { @@ -348,42 +358,19 @@ func toHex(s string) string { return hex.EncodeToString([]byte(s)) } func TestGetNodeJoinScript(t *testing.T) { validToken := "f18da1c9f6630a51e8daf121e7451daa" + invalidToken := "f18da1c9f6630a51e8daf121e7451dab" validIAMToken := "valid-iam-token" internalResourceID := "967d38ff-7a61-4f42-bd2d-c61965b44db0" - m := &mockedNodeAPIGetter{ - mockGetProxyServers: func() ([]types.Server, error) { - var s types.ServerV2 - s.SetPublicAddrs([]string{"test-host:12345678"}) - - return []types.Server{&s}, nil - }, - mockGetClusterCACert: func(context.Context) (*proto.GetClusterCACertResponse, error) { - fakeBytes := []byte(fixtures.SigningCertPEM) - return &proto.GetClusterCACertResponse{TLSCA: fakeBytes}, nil - }, - mockGetToken: func(_ context.Context, token string) (types.ProvisionToken, error) { - if token == validToken || token == validIAMToken { - return &types.ProvisionTokenV2{ - Metadata: types.Metadata{ - Name: token, - }, - Spec: types.ProvisionTokenSpecV2{ - SuggestedLabels: types.Labels{ - types.InternalResourceIDLabel: utils.Strings{internalResourceID}, - }, - }, - }, nil - } - return nil, trace.NotFound("token does not exist") - }, - } + hostname := "proxy.example.com" + port := 1234 for _, test := range []struct { desc string settings scriptSettings errAssert require.ErrorAssertionFunc - extraAssertions func(script string) + token *types.ProvisionTokenV2 + extraAssertions func(t *testing.T, script string) }{ { desc: "zero value", @@ -392,22 +379,52 @@ func TestGetNodeJoinScript(t *testing.T) { }, { desc: "short token length", - settings: scriptSettings{token: toHex("f18da1c9f6630a51e8daf121e7451d")}, + settings: scriptSettings{token: toHex(validToken[:30])}, errAssert: require.Error, + token: &types.ProvisionTokenV2{ + Metadata: types.Metadata{ + Name: validToken[:30], + }, + Spec: types.ProvisionTokenSpecV2{ + SuggestedLabels: types.Labels{ + types.InternalResourceIDLabel: apiutils.Strings{internalResourceID}, + }, + }, + }, }, { desc: "valid length but does not exist", - settings: scriptSettings{token: toHex("xxxxxxx9f6630a51e8daf121exxxxxxx")}, + settings: scriptSettings{token: toHex(invalidToken)}, errAssert: require.Error, + token: &types.ProvisionTokenV2{ + Metadata: types.Metadata{ + Name: validToken, + }, + Spec: types.ProvisionTokenSpecV2{ + SuggestedLabels: types.Labels{ + types.InternalResourceIDLabel: apiutils.Strings{internalResourceID}, + }, + }, + }, }, { desc: "valid", settings: scriptSettings{token: validToken}, errAssert: require.NoError, - extraAssertions: func(script string) { + token: &types.ProvisionTokenV2{ + Metadata: types.Metadata{ + Name: validToken, + }, + Spec: types.ProvisionTokenSpecV2{ + SuggestedLabels: types.Labels{ + types.InternalResourceIDLabel: apiutils.Strings{internalResourceID}, + }, + }, + }, + extraAssertions: func(t *testing.T, script string) { require.Contains(t, script, validToken) - require.Contains(t, script, "test-host") - require.Contains(t, script, "12345678") + require.Contains(t, script, hostname) + require.Contains(t, script, strconv.Itoa(port)) require.Contains(t, script, "sha256:") require.NotContains(t, script, "JOIN_METHOD='iam'") }, @@ -426,8 +443,18 @@ func TestGetNodeJoinScript(t *testing.T) { token: validIAMToken, joinMethod: string(types.JoinMethodIAM), }, + token: &types.ProvisionTokenV2{ + Metadata: types.Metadata{ + Name: validIAMToken, + }, + Spec: types.ProvisionTokenSpecV2{ + SuggestedLabels: types.Labels{ + types.InternalResourceIDLabel: apiutils.Strings{internalResourceID}, + }, + }, + }, errAssert: require.NoError, - extraAssertions: func(script string) { + extraAssertions: func(t *testing.T, script string) { require.Contains(t, script, "JOIN_METHOD='iam'") }, }, @@ -435,48 +462,151 @@ func TestGetNodeJoinScript(t *testing.T) { desc: "internal resourceid label", settings: scriptSettings{token: validToken}, errAssert: require.NoError, - extraAssertions: func(script string) { + token: &types.ProvisionTokenV2{ + Metadata: types.Metadata{ + Name: validToken, + }, + Spec: types.ProvisionTokenSpecV2{ + SuggestedLabels: types.Labels{ + types.InternalResourceIDLabel: apiutils.Strings{internalResourceID}, + }, + }, + }, + extraAssertions: func(t *testing.T, script string) { require.Contains(t, script, "--labels ") require.Contains(t, script, fmt.Sprintf("%s=%s", types.InternalResourceIDLabel, internalResourceID)) }, }, + { + desc: "attempt to shell injection using suggested labels", + settings: scriptSettings{token: validToken}, + token: &types.ProvisionTokenV2{ + Metadata: types.Metadata{ + Name: validToken, + }, + Spec: types.ProvisionTokenSpecV2{ + SuggestedLabels: types.Labels{ + types.InternalResourceIDLabel: apiutils.Strings{internalResourceID}, + "env": []string{"bad label value | ; & $ > < ' !"}, + "bad label key | ; & $ > < ' !": []string{"env"}, + }, + }, + }, + errAssert: require.NoError, + extraAssertions: func(t *testing.T, script string) { + require.Contains(t, script, `bad\ label\ key\ \|\ \;\ \&\ \$\ \>\ \<\ \'\ \!=env`) + require.Contains(t, script, `env=bad\ label\ value\ \|\ \;\ \&\ \$\ \>\ \<\ \'\ \!`) + }, + }, } { t.Run(test.desc, func(t *testing.T) { - script, err := getJoinScript(context.Background(), test.settings, m) + h := newAutoupdateTestHandler(t, autoupdateTestHandlerConfig{ + hostname: hostname, + port: port, + token: test.token, + }) + script, err := h.getJoinScript(context.Background(), test.settings) test.errAssert(t, err) if err != nil { require.Empty(t, script) } if test.extraAssertions != nil { - test.extraAssertions(script) + test.extraAssertions(t, script) } }) } } -func TestGetAppJoinScript(t *testing.T) { - testTokenID := "f18da1c9f6630a51e8daf121e7451daa" - m := &mockedNodeAPIGetter{ - mockGetToken: func(_ context.Context, token string) (types.ProvisionToken, error) { - if token == testTokenID { - return &types.ProvisionTokenV2{ - Metadata: types.Metadata{ - Name: token, - }, - }, nil - } - return nil, trace.NotFound("token does not exist") - }, - mockGetProxyServers: func() ([]types.Server, error) { - var s types.ServerV2 - s.SetPublicAddrs([]string{"test-host:12345678"}) +type autoupdateAccessPointMock struct { + authclient.ProxyAccessPoint + mock.Mock +} + +func (a *autoupdateAccessPointMock) GetAutoUpdateAgentRollout(ctx context.Context) (*autoupdatev1pb.AutoUpdateAgentRollout, error) { + args := a.Called(ctx) + return args.Get(0).(*autoupdatev1pb.AutoUpdateAgentRollout), args.Error(1) +} - return []types.Server{&s}, nil +type autoupdateProxyClientMock struct { + authclient.ClientI + mock.Mock +} + +func (a *autoupdateProxyClientMock) GetToken(ctx context.Context, token string) (types.ProvisionToken, error) { + args := a.Called(ctx, token) + return args.Get(0).(types.ProvisionToken), args.Error(1) +} + +func (a *autoupdateProxyClientMock) GetClusterCACert(ctx context.Context) (*proto.GetClusterCACertResponse, error) { + args := a.Called(ctx) + return args.Get(0).(*proto.GetClusterCACertResponse), args.Error(1) +} + +type autoupdateTestHandlerConfig struct { + testModules *modules.TestModules + hostname string + port int + channels automaticupgrades.Channels + rollout *autoupdatev1pb.AutoUpdateAgentRollout + token *types.ProvisionTokenV2 +} + +func newAutoupdateTestHandler(t *testing.T, config autoupdateTestHandlerConfig) *Handler { + if config.hostname == "" { + config.hostname = fmt.Sprintf("proxy-%d.example.com", rand.Int()) + } + if config.port == 0 { + config.port = rand.IntN(65535) + } + addr := config.hostname + ":" + strconv.Itoa(config.port) + + if config.channels == nil { + config.channels = automaticupgrades.Channels{} + } + require.NoError(t, config.channels.CheckAndSetDefaults()) + + ap := &autoupdateAccessPointMock{} + if config.rollout == nil { + ap.On("GetAutoUpdateAgentRollout", mock.Anything).Return(config.rollout, trace.NotFound("rollout does not exist")) + } else { + ap.On("GetAutoUpdateAgentRollout", mock.Anything).Return(config.rollout, nil) + } + + clt := &autoupdateProxyClientMock{} + if config.token == nil { + clt.On("GetToken", mock.Anything, mock.Anything).Return(config.token, trace.NotFound("token does not exist")) + } else { + clt.On("GetToken", mock.Anything, config.token.GetName()).Return(config.token, nil) + } + + clt.On("GetClusterCACert", mock.Anything).Return(&proto.GetClusterCACertResponse{TLSCA: []byte(fixtures.SigningCertPEM)}, nil) + + if config.testModules == nil { + config.testModules = &modules.TestModules{ + TestBuildType: modules.BuildOSS, + } + } + modules.SetTestModules(t, config.testModules) + h := &Handler{ + ClusterFeatures: *config.testModules.Features().ToProto(), + cfg: Config{ + AutomaticUpgradesChannels: config.channels, + AccessPoint: ap, + PublicProxyAddr: addr, + ProxyClient: clt, }, - mockGetClusterCACert: func(context.Context) (*proto.GetClusterCACertResponse, error) { - fakeBytes := []byte(fixtures.SigningCertPEM) - return &proto.GetClusterCACertResponse{TLSCA: fakeBytes}, nil + log: utils.NewLoggerForTests(), + } + h.PublicProxyAddr() + return h +} + +func TestGetAppJoinScript(t *testing.T) { + testTokenID := "f18da1c9f6630a51e8daf121e7451daa" + token := &types.ProvisionTokenV2{ + Metadata: types.Metadata{ + Name: testTokenID, }, } badAppName := scriptSettings{ @@ -493,20 +623,24 @@ func TestGetAppJoinScript(t *testing.T) { appURI: "", } + h := newAutoupdateTestHandler(t, autoupdateTestHandlerConfig{token: token}) + hostname, port, err := utils.SplitHostPort(h.PublicProxyAddr()) + require.NoError(t, err) + // Test invalid app data. - script, err := getJoinScript(context.Background(), badAppName, m) + script, err := h.getJoinScript(context.Background(), badAppName) require.Empty(t, script) require.True(t, trace.IsBadParameter(err)) - script, err = getJoinScript(context.Background(), badAppURI, m) + script, err = h.getJoinScript(context.Background(), badAppURI) require.Empty(t, script) require.True(t, trace.IsBadParameter(err)) // Test various 'good' cases. expectedOutputs := []string{ testTokenID, - "test-host", - "12345678", + hostname, + port, "sha256:", } @@ -627,7 +761,7 @@ func TestGetAppJoinScript(t *testing.T) { for _, tc := range tests { tc := tc t.Run(tc.desc, func(t *testing.T) { - script, err = getJoinScript(context.Background(), tc.settings, m) + script, err = h.getJoinScript(context.Background(), tc.settings) if tc.shouldError { require.NotNil(t, err) require.Equal(t, script, "") @@ -643,55 +777,47 @@ func TestGetAppJoinScript(t *testing.T) { func TestGetDatabaseJoinScript(t *testing.T) { validToken := "f18da1c9f6630a51e8daf121e7451daa" - emptySuggestedAgentMatcherLabelsToken := "f18da1c9f6630a51e8daf121e7451000" internalResourceID := "967d38ff-7a61-4f42-bd2d-c61965b44db0" + hostname := "test.example.com" + port := 1234 - m := &mockedNodeAPIGetter{ - mockGetProxyServers: func() ([]types.Server, error) { - var s types.ServerV2 - s.SetPublicAddrs([]string{"test-host:12345678"}) - - return []types.Server{&s}, nil + token := &types.ProvisionTokenV2{ + Metadata: types.Metadata{ + Name: validToken, }, - mockGetClusterCACert: func(context.Context) (*proto.GetClusterCACertResponse, error) { - fakeBytes := []byte(fixtures.SigningCertPEM) - return &proto.GetClusterCACertResponse{TLSCA: fakeBytes}, nil + Spec: types.ProvisionTokenSpecV2{ + SuggestedLabels: types.Labels{ + types.InternalResourceIDLabel: apiutils.Strings{internalResourceID}, + }, + SuggestedAgentMatcherLabels: types.Labels{ + "env": apiutils.Strings{"prod"}, + "product": apiutils.Strings{"*"}, + "os": apiutils.Strings{"mac", "linux"}, + }, }, - mockGetToken: func(_ context.Context, token string) (types.ProvisionToken, error) { - provisionToken := &types.ProvisionTokenV2{ - Metadata: types.Metadata{ - Name: token, - }, - Spec: types.ProvisionTokenSpecV2{ - SuggestedLabels: types.Labels{ - types.InternalResourceIDLabel: utils.Strings{internalResourceID}, - }, - SuggestedAgentMatcherLabels: types.Labels{ - "env": utils.Strings{"prod"}, - "product": utils.Strings{"*"}, - "os": utils.Strings{"mac", "linux"}, - }, - }, - } - if token == validToken { - return provisionToken, nil - } - if token == emptySuggestedAgentMatcherLabelsToken { - provisionToken.Spec.SuggestedAgentMatcherLabels = types.Labels{} - return provisionToken, nil - } - return nil, trace.NotFound("token does not exist") + } + + noMatcherToken := &types.ProvisionTokenV2{ + Metadata: types.Metadata{ + Name: validToken, + }, + Spec: types.ProvisionTokenSpecV2{ + SuggestedLabels: types.Labels{ + types.InternalResourceIDLabel: apiutils.Strings{internalResourceID}, + }, }, } for _, test := range []struct { desc string settings scriptSettings + token *types.ProvisionTokenV2 errAssert require.ErrorAssertionFunc - extraAssertions func(script string) + extraAssertions func(t *testing.T, script string) }{ { - desc: "two installation methods", + desc: "two installation methods", + token: token, settings: scriptSettings{ token: validToken, databaseInstallMode: true, @@ -700,15 +826,17 @@ func TestGetDatabaseJoinScript(t *testing.T) { errAssert: require.Error, }, { - desc: "valid", + desc: "valid", + token: token, settings: scriptSettings{ databaseInstallMode: true, token: validToken, }, errAssert: require.NoError, - extraAssertions: func(script string) { + extraAssertions: func(t *testing.T, script string) { require.Contains(t, script, validToken) - require.Contains(t, script, "test-host") + require.Contains(t, script, hostname) + require.Contains(t, script, strconv.Itoa(port)) require.Contains(t, script, "sha256:") require.Contains(t, script, "--labels ") require.Contains(t, script, fmt.Sprintf("%s=%s", types.InternalResourceIDLabel, internalResourceID)) @@ -726,15 +854,97 @@ db_service: }, }, { - desc: "empty suggestedAgentMatcherLabels", + desc: "discover flow with wildcard label matcher", settings: scriptSettings{ databaseInstallMode: true, - token: emptySuggestedAgentMatcherLabelsToken, + token: validToken, + }, + token: &types.ProvisionTokenV2{ + Metadata: types.Metadata{ + Name: validToken, + }, + Spec: types.ProvisionTokenSpecV2{ + SuggestedLabels: types.Labels{ + types.InternalResourceIDLabel: apiutils.Strings{internalResourceID}, + }, + SuggestedAgentMatcherLabels: types.Labels{ + "*": apiutils.Strings{"*"}, + }, + }, }, errAssert: require.NoError, - extraAssertions: func(script string) { - require.Contains(t, script, emptySuggestedAgentMatcherLabelsToken) - require.Contains(t, script, "test-host") + extraAssertions: func(t *testing.T, script string) { + require.Contains(t, script, validToken) + require.Contains(t, script, hostname) + require.Contains(t, script, strconv.Itoa(port)) + require.Contains(t, script, "sha256:") + require.Contains(t, script, "--labels ") + require.Contains(t, script, fmt.Sprintf("%s=%s", types.InternalResourceIDLabel, internalResourceID)) + require.Contains(t, script, ` + - labels: + '*': '*' +`) + }, + }, + { + desc: "discover flow with shell injection attempt in resource matcher labels", + token: &types.ProvisionTokenV2{ + Metadata: types.Metadata{ + Name: validToken, + }, + Spec: types.ProvisionTokenSpecV2{ + SuggestedLabels: types.Labels{ + types.InternalResourceIDLabel: apiutils.Strings{internalResourceID}, + }, + SuggestedAgentMatcherLabels: types.Labels{ + "*": apiutils.Strings{"*"}, + "spa ces": apiutils.Strings{"spa ces"}, + "EOF": apiutils.Strings{"test heredoc"}, + `"EOF"`: apiutils.Strings{"test quoted heredoc"}, + "#'; <>\\#": apiutils.Strings{"try to escape yaml"}, + "&<>'\"$A,./;'BCD ${ABCD}": apiutils.Strings{"key with special characters"}, + "value with special characters": apiutils.Strings{"&<>'\"$A,./;'BCD ${ABCD}", "#&<>'\"$A,./;'BCD ${ABCD}"}, + }, + }, + }, + settings: scriptSettings{ + databaseInstallMode: true, + token: validToken, + }, + errAssert: require.NoError, + extraAssertions: func(t *testing.T, script string) { + require.Contains(t, script, validToken) + require.Contains(t, script, hostname) + require.Contains(t, script, strconv.Itoa(port)) + require.Contains(t, script, "sha256:") + require.Contains(t, script, "--labels ") + require.Contains(t, script, fmt.Sprintf("%s=%s", types.InternalResourceIDLabel, internalResourceID)) + require.Contains(t, script, ` + - labels: + '"EOF"': test quoted heredoc + '#''; <>\#': try to escape yaml + '&<>''"$A,./;''BCD ${ABCD}': key with special characters + '*': '*' + EOF: test heredoc + spa ces: spa ces + value with special characters: + - '&<>''"$A,./;''BCD ${ABCD}' + - '#&<>''"$A,./;''BCD ${ABCD}' +`) + }, + }, + { + desc: "empty suggestedAgentMatcherLabels", + token: noMatcherToken, + settings: scriptSettings{ + databaseInstallMode: true, + token: validToken, + }, + errAssert: require.NoError, + extraAssertions: func(t *testing.T, script string) { + require.Contains(t, script, validToken) + require.Contains(t, script, hostname) + require.Contains(t, script, strconv.Itoa(port)) require.Contains(t, script, "sha256:") require.Contains(t, script, "--labels ") require.Contains(t, script, fmt.Sprintf("%s=%s", types.InternalResourceIDLabel, internalResourceID)) @@ -749,14 +959,20 @@ db_service: }, } { t.Run(test.desc, func(t *testing.T) { - script, err := getJoinScript(context.Background(), test.settings, m) + h := newAutoupdateTestHandler(t, autoupdateTestHandlerConfig{ + hostname: hostname, + port: port, + token: test.token, + }) + + script, err := h.getJoinScript(context.Background(), test.settings) test.errAssert(t, err) if err != nil { require.Empty(t, script) } if test.extraAssertions != nil { - test.extraAssertions(script) + test.extraAssertions(t, script) } }) } @@ -764,30 +980,13 @@ db_service: func TestGetDiscoveryJoinScript(t *testing.T) { const validToken = "f18da1c9f6630a51e8daf121e7451daa" - - m := &mockedNodeAPIGetter{ - mockGetProxyServers: func() ([]types.Server, error) { - var s types.ServerV2 - s.SetPublicAddrs([]string{"test-host:12345678"}) - - return []types.Server{&s}, nil - }, - mockGetClusterCACert: func(context.Context) (*proto.GetClusterCACertResponse, error) { - fakeBytes := []byte(fixtures.SigningCertPEM) - return &proto.GetClusterCACertResponse{TLSCA: fakeBytes}, nil - }, - mockGetToken: func(_ context.Context, token string) (types.ProvisionToken, error) { - provisionToken := &types.ProvisionTokenV2{ - Metadata: types.Metadata{ - Name: token, - }, - Spec: types.ProvisionTokenSpecV2{}, - } - if token == validToken { - return provisionToken, nil - } - return nil, trace.NotFound("token does not exist") + hostname := "test.example.com" + port := 1234 + token := &types.ProvisionTokenV2{ + Metadata: types.Metadata{ + Name: validToken, }, + Spec: types.ProvisionTokenSpecV2{}, } for _, test := range []struct { @@ -806,7 +1005,8 @@ func TestGetDiscoveryJoinScript(t *testing.T) { errAssert: require.NoError, extraAssertions: func(t *testing.T, script string) { require.Contains(t, script, validToken) - require.Contains(t, script, "test-host") + require.Contains(t, script, hostname) + require.Contains(t, script, strconv.Itoa(port)) require.Contains(t, script, "sha256:") require.Contains(t, script, "--labels ") require.Contains(t, script, ` @@ -825,7 +1025,12 @@ discovery_service: }, } { t.Run(test.desc, func(t *testing.T) { - script, err := getJoinScript(context.Background(), test.settings, m) + h := newAutoupdateTestHandler(t, autoupdateTestHandlerConfig{ + hostname: hostname, + port: port, + token: token, + }) + script, err := h.getJoinScript(context.Background(), test.settings) test.errAssert(t, err) if err != nil { require.Empty(t, script) @@ -944,28 +1149,9 @@ func TestIsSameRuleSet(t *testing.T) { func TestJoinScript(t *testing.T) { validToken := "f18da1c9f6630a51e8daf121e7451daa" - - m := &mockedNodeAPIGetter{ - mockGetProxyServers: func() ([]types.Server, error) { - return []types.Server{ - &types.ServerV2{ - Spec: types.ServerSpecV2{ - PublicAddrs: []string{"test-host:12345678"}, - Version: teleport.Version, - }, - }, - }, nil - }, - mockGetClusterCACert: func(context.Context) (*proto.GetClusterCACertResponse, error) { - fakeBytes := []byte(fixtures.SigningCertPEM) - return &proto.GetClusterCACertResponse{TLSCA: fakeBytes}, nil - }, - mockGetToken: func(_ context.Context, token string) (types.ProvisionToken, error) { - return &types.ProvisionTokenV2{ - Metadata: types.Metadata{ - Name: token, - }, - }, nil + token := &types.ProvisionTokenV2{ + Metadata: types.Metadata{ + Name: validToken, }, } @@ -973,8 +1159,11 @@ func TestJoinScript(t *testing.T) { getGravitationalTeleportLinkRegex := regexp.MustCompile(`https://cdn\.teleport\.dev/\${TELEPORT_PACKAGE_NAME}[-_]v?\${TELEPORT_VERSION}`) t.Run("oss", func(t *testing.T) { + h := newAutoupdateTestHandler(t, autoupdateTestHandlerConfig{ + token: token, + }) // Using the OSS Version, all the links must contain only teleport as package name. - script, err := getJoinScript(context.Background(), scriptSettings{token: validToken}, m) + script, err := h.getJoinScript(context.Background(), scriptSettings{token: validToken}) require.NoError(t, err) matches := getGravitationalTeleportLinkRegex.FindAllString(script, -1) @@ -989,8 +1178,11 @@ func TestJoinScript(t *testing.T) { t.Run("ent", func(t *testing.T) { // Using the Enterprise Version, the package name must be teleport-ent - modules.SetTestModules(t, &modules.TestModules{TestBuildType: modules.BuildEnterprise}) - script, err := getJoinScript(context.Background(), scriptSettings{token: validToken}, m) + h := newAutoupdateTestHandler(t, autoupdateTestHandlerConfig{ + testModules: &modules.TestModules{TestBuildType: modules.BuildEnterprise}, + token: token, + }) + script, err := h.getJoinScript(context.Background(), scriptSettings{token: validToken}) require.NoError(t, err) matches := getGravitationalTeleportLinkRegex.FindAllString(script, -1) @@ -1006,45 +1198,76 @@ func TestJoinScript(t *testing.T) { t.Run("using repo", func(t *testing.T) { t.Run("installUpdater is true", func(t *testing.T) { - currentStableCloudVersion := "v99.1.1" - script, err := getJoinScript(context.Background(), scriptSettings{token: validToken, installUpdater: true, automaticUpgradesVersion: currentStableCloudVersion}, m) + currentStableCloudVersion := "1.2.3" + h := newAutoupdateTestHandler(t, autoupdateTestHandlerConfig{ + testModules: &modules.TestModules{TestFeatures: modules.Features{Cloud: true, AutomaticUpgrades: true}}, + token: token, + channels: automaticupgrades.Channels{ + automaticupgrades.DefaultChannelName: &automaticupgrades.Channel{StaticVersion: currentStableCloudVersion}, + }, + }) + + script, err := h.getJoinScript(context.Background(), scriptSettings{token: validToken}) require.NoError(t, err) - // list of packages must include the updater - require.Contains(t, script, ""+ - " PACKAGE_LIST=${TELEPORT_PACKAGE_PIN_VERSION}\n"+ - " # (warning): This expression is constant. Did you forget the $ on a variable?\n"+ - " # Disabling the warning above because expression is templated.\n"+ - " # shellcheck disable=SC2050\n"+ - " if is_using_systemd && [[ \"true\" == \"true\" ]]; then\n"+ - " # Teleport Updater requires systemd.\n"+ - " PACKAGE_LIST+=\" ${TELEPORT_UPDATER_PIN_VERSION}\"\n"+ - " fi\n", - ) + require.Contains(t, script, "UPDATER_STYLE='package'") // Repo channel is stable/cloud require.Contains(t, script, "REPO_CHANNEL='stable/cloud'") // TELEPORT_VERSION is the one provided by https://updates.releases.teleport.dev/v1/stable/cloud/version - require.Contains(t, script, "TELEPORT_VERSION='99.1.1'") + require.Contains(t, script, fmt.Sprintf("TELEPORT_VERSION='%s'", currentStableCloudVersion)) }) t.Run("installUpdater is false", func(t *testing.T) { - script, err := getJoinScript(context.Background(), scriptSettings{token: validToken, installUpdater: false}, m) + h := newAutoupdateTestHandler(t, autoupdateTestHandlerConfig{ + token: token, + }) + script, err := h.getJoinScript(context.Background(), scriptSettings{token: validToken}) require.NoError(t, err) - require.Contains(t, script, ""+ - " PACKAGE_LIST=${TELEPORT_PACKAGE_PIN_VERSION}\n"+ - " # (warning): This expression is constant. Did you forget the $ on a variable?\n"+ - " # Disabling the warning above because expression is templated.\n"+ - " # shellcheck disable=SC2050\n"+ - " if is_using_systemd && [[ \"false\" == \"true\" ]]; then\n"+ - " # Teleport Updater requires systemd.\n"+ - " PACKAGE_LIST+=\" ${TELEPORT_UPDATER_PIN_VERSION}\"\n"+ - " fi\n", - ) + require.Contains(t, script, "UPDATER_STYLE='none'") // Default based on current version is used instead require.Contains(t, script, "REPO_CHANNEL=''") // Current version must be used require.Contains(t, script, fmt.Sprintf("TELEPORT_VERSION='%s'", teleport.Version)) }) }) + t.Run("using teleport-update", func(t *testing.T) { + testRollout := &autoupdatev1pb.AutoUpdateAgentRollout{Spec: &autoupdatev1pb.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.2", + TargetVersion: "1.2.3", + Schedule: autoupdate.AgentsScheduleImmediate, + AutoupdateMode: autoupdate.AgentsUpdateModeEnabled, + Strategy: autoupdate.AgentsStrategyTimeBased, + MaintenanceWindowDuration: durationpb.New(1 * time.Hour), + }} + t.Run("rollout exists and autoupdates are on", func(t *testing.T) { + currentStableCloudVersion := "1.1.1" + config := autoupdateTestHandlerConfig{ + testModules: &modules.TestModules{TestFeatures: modules.Features{Cloud: true, AutomaticUpgrades: true}}, + channels: automaticupgrades.Channels{ + automaticupgrades.DefaultChannelName: &automaticupgrades.Channel{StaticVersion: currentStableCloudVersion}, + }, + rollout: testRollout, + token: token, + } + h := newAutoupdateTestHandler(t, config) + + script, err := h.getJoinScript(context.Background(), scriptSettings{token: validToken}) + require.NoError(t, err) + + // list of packages must include the updater + require.Contains(t, script, "UPDATER_STYLE='binary'") + require.Contains(t, script, fmt.Sprintf("TELEPORT_VERSION='%s'", testRollout.Spec.TargetVersion)) + }) + t.Run("rollout exists and autoupdates are off", func(t *testing.T) { + h := newAutoupdateTestHandler(t, autoupdateTestHandlerConfig{ + rollout: testRollout, + token: token, + }) + script, err := h.getJoinScript(context.Background(), scriptSettings{token: validToken}) + require.NoError(t, err) + require.Contains(t, script, "UPDATER_STYLE='binary'") + require.Contains(t, script, fmt.Sprintf("TELEPORT_VERSION='%s'", testRollout.Spec.TargetVersion)) + }) + }) } func TestAutomaticUpgrades(t *testing.T) { @@ -1152,32 +1375,3 @@ func TestIsSameAzureRuleSet(t *testing.T) { }) } } - -type mockedNodeAPIGetter struct { - mockGetProxyServers func() ([]types.Server, error) - mockGetClusterCACert func(ctx context.Context) (*proto.GetClusterCACertResponse, error) - mockGetToken func(ctx context.Context, token string) (types.ProvisionToken, error) -} - -func (m *mockedNodeAPIGetter) GetProxies() ([]types.Server, error) { - if m.mockGetProxyServers != nil { - return m.mockGetProxyServers() - } - - return nil, trace.NotImplemented("mockGetProxyServers not implemented") -} - -func (m *mockedNodeAPIGetter) GetClusterCACert(ctx context.Context) (*proto.GetClusterCACertResponse, error) { - if m.mockGetClusterCACert != nil { - return m.mockGetClusterCACert(ctx) - } - - return nil, trace.NotImplemented("mockGetClusterCACert not implemented") -} - -func (m *mockedNodeAPIGetter) GetToken(ctx context.Context, token string) (types.ProvisionToken, error) { - if m.mockGetToken != nil { - return m.mockGetToken(ctx, token) - } - return nil, trace.NotImplemented("mockGetToken not implemented") -} diff --git a/lib/web/scripts.go b/lib/web/scripts.go new file mode 100644 index 0000000000000..1652388a92de1 --- /dev/null +++ b/lib/web/scripts.go @@ -0,0 +1,166 @@ +/* + * Teleport + * Copyright (C) 2025 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package web + +import ( + "context" + "fmt" + "net/http" + "os" + "strconv" + + "github.com/coreos/go-semver/semver" + "github.com/gravitational/trace" + "github.com/julienschmidt/httprouter" + + "github.com/gravitational/teleport" + "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/lib/auth/native" + "github.com/gravitational/teleport/lib/modules" + "github.com/gravitational/teleport/lib/utils/teleportassets" + "github.com/gravitational/teleport/lib/web/scripts" +) + +const insecureParamName = "insecure" + +// installScriptHandle handles calls for "/scripts/install.sh" and responds with a bash script installing Teleport +// by downloading and running `teleport-update`. This installation script does not start the agent, join it, +// or configure its services. This is handled by the "/scripts/:token/install-*.sh" scripts. +func (h *Handler) installScriptHandle(w http.ResponseWriter, r *http.Request, params httprouter.Params) (any, error) { + // This is a hack because the router is not allowing us to register "/scripts/install.sh", so we use + // the parameter ":token" to match the script name. + // Currently, only "install.sh" is supported. + if params.ByName("token") != "install.sh" { + return nil, trace.NotFound(`Route not found, query "/scripts/install.sh" for the install-only script, or "/scripts/:token/install-node.sh" for the install + join script.`) + } + + // TODO(hugoShaka): cache function + opts, err := h.installScriptOptions(r.Context()) + if err != nil { + return nil, trace.Wrap(err, "Failed to build install script options") + } + + if insecure := r.URL.Query().Get(insecureParamName); insecure != "" { + v, err := strconv.ParseBool(insecure) + if err != nil { + return nil, trace.BadParameter("failed to parse insecure flag %q: %v", insecure, err) + } + opts.Insecure = v + } + + script, err := scripts.GetInstallScript(r.Context(), opts) + if err != nil { + h.log.WithError(err).Warn("Failed to get install script") + return nil, trace.Wrap(err, "getting script") + } + + w.WriteHeader(http.StatusOK) + if _, err := fmt.Fprintln(w, script); err != nil { + h.log.WithError(err).Warn("Failed to write install script") + } + + return nil, nil +} + +// installScriptOptions computes the agent installation options based on the proxy configuration and the cluster status. +// This includes: +// - the type of automatic updates +// - the desired version +// - the proxy address (used for updates). +// - the Teleport artifact name and CDN +func (h *Handler) installScriptOptions(ctx context.Context) (scripts.InstallScriptOptions, error) { + const defaultGroup, defaultUpdater = "", "" + + version, err := h.autoUpdateAgentVersion(ctx, defaultGroup, defaultUpdater) + if err != nil { + h.log.WithError(err).Warn("Failed to get intended agent version") + version = teleport.Version + } + + // if there's a rollout, we do new autoupdates + _, rolloutErr := h.cfg.AccessPoint.GetAutoUpdateAgentRollout(ctx) + if rolloutErr != nil && !(trace.IsNotFound(rolloutErr) || trace.IsNotImplemented(rolloutErr)) { + h.log.WithError(err).Warn("Failed to get rollout") + return scripts.InstallScriptOptions{}, trace.Wrap(err, "failed to check the autoupdate agent rollout state") + } + + var autoupdateStyle scripts.AutoupdateStyle + switch { + case rolloutErr == nil: + autoupdateStyle = scripts.UpdaterBinaryAutoupdate + case automaticUpgrades(h.ClusterFeatures): + autoupdateStyle = scripts.PackageManagerAutoupdate + default: + autoupdateStyle = scripts.NoAutoupdate + } + + var teleportFlavor string + switch modules.GetModules().BuildType() { + case modules.BuildEnterprise: + teleportFlavor = types.PackageNameEnt + case modules.BuildOSS, modules.BuildCommunity: + teleportFlavor = types.PackageNameOSS + default: + h.log.Warnf("Unknown built type %q, defaulting to the 'teleport' package.", modules.GetModules().BuildType()) + teleportFlavor = types.PackageNameOSS + } + + cdnBaseURL, err := getCDNBaseURL(version) + if err != nil { + h.log.WithError(err).Warn("Failed to get CDN base URL") + return scripts.InstallScriptOptions{}, trace.Wrap(err) + } + + return scripts.InstallScriptOptions{ + AutoupdateStyle: autoupdateStyle, + TeleportVersion: version, + CDNBaseURL: cdnBaseURL, + ProxyAddr: h.PublicProxyAddr(), + TeleportFlavor: teleportFlavor, + FIPS: native.IsBoringBinary(), + }, nil + +} + +// EnvVarCDNBaseURL is the environment variable that allows users to override the Teleport base CDN url used in the installation script. +// Setting this value is required for testing (make production builds install from the dev CDN, and vice versa). +// As we (the Teleport company) don't distribute AGPL binaries, this must be set when using a Teleport OSS build. +// Example values: +// - "https://cdn.teleport.dev" (prod) +// - "https://cdn.cloud.gravitational.io" (dev builds/staging) +const EnvVarCDNBaseURL = "TELEPORT_CDN_BASE_URL" + +func getCDNBaseURL(version string) (string, error) { + // If the user explicitly overrides the CDN base URL, we use it. + if override := os.Getenv(EnvVarCDNBaseURL); override != "" { + return override, nil + } + + v, err := semver.NewVersion(version) + if err != nil { + return "", trace.Wrap(err) + } + + // For backward compatibility we don't fail if the user is running AGPL and + // did not specify the CDN URL. However we will fail in v18 for this as we + // cannot automatically install binaries subject to a license the user has + // not agreed to. + + return teleportassets.CDNBaseURLForVersion(v), nil +} diff --git a/lib/web/scripts/install.go b/lib/web/scripts/install.go new file mode 100644 index 0000000000000..0e6d956be514a --- /dev/null +++ b/lib/web/scripts/install.go @@ -0,0 +1,220 @@ +/* + * Teleport + * Copyright (C) 2025 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package scripts + +import ( + "context" + _ "embed" + "fmt" + "net/url" + "regexp" + "strings" + + "github.com/google/safetext/shsprintf" + "github.com/gravitational/trace" + + "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/lib/utils/teleportassets" + "github.com/gravitational/teleport/lib/web/scripts/oneoff" +) + +// AutoupdateStyle represents the kind of autoupdate mechanism the script should use. +type AutoupdateStyle int + +const ( + // NoAutoupdate means the installed Teleport should not autoupdate. + NoAutoupdate AutoupdateStyle = iota + // PackageManagerAutoupdate means the installed Teleport should update via a script triggering package manager + // updates. The script lives in the 'teleport-ent-update' package and was our original attempt at automatic updates. + // See RFD-109 for more details: https://github.com/gravitational/teleport/blob/master/rfd/0109-cloud-agent-upgrades.md + PackageManagerAutoupdate + // UpdaterBinaryAutoupdate means the installed Teleport should update via the teleport-update binary. + // This update style does not depend on any package manager (although it has a system dependency to wake up the + // updater). + // See RFD-184 for more details: https://github.com/gravitational/teleport/blob/master/rfd/0184-agent-auto-updates.md + UpdaterBinaryAutoupdate + + teleportUpdateDefaultCDN = teleportassets.TeleportReleaseCDN +) + +func (style AutoupdateStyle) String() string { + switch style { + case PackageManagerAutoupdate: + return "package" + case UpdaterBinaryAutoupdate: + return "binary" + case NoAutoupdate: + return "none" + default: + return "unknown" + } +} + +// InstallScriptOptions contains the Teleport installation options used to generate installation scripts. +type InstallScriptOptions struct { + AutoupdateStyle AutoupdateStyle + // TeleportVersion that should be installed. Without the leading "v". + TeleportVersion string + // CDNBaseURL is the URL of the CDN hosting teleport tarballs. + // If left empty, the 'teleport-update' installer will pick the one to use. + // For example: "https://cdn.example.com" + CDNBaseURL string + // ProxyAddr is the address of the Teleport Proxy service that will be used + // by the updater to fetch the desired version. Teleport Addrs are + // 'hostname:port' (no scheme nor path). + ProxyAddr string + // TeleportFlavor is the name of the Teleport artifact fetched from the CDN. + // Common values are "teleport" and "teleport-ent". + TeleportFlavor string + // FIPS represents if the installed Teleport version should use Teleport + // binaries built for FIPS compliance. + FIPS bool + // Insecure disables TLS certificate verification on the teleport-update command. + // This is meant for testing purposes. + // This does not disable the TLS certificate verification when downloading + // the artifacts from the CDN. + // The agent install in insecure mode will not be able to automatically update. + Insecure bool +} + +// Check validates that the minimal options are set. +func (o *InstallScriptOptions) Check() error { + switch o.AutoupdateStyle { + case NoAutoupdate, PackageManagerAutoupdate: + return nil + case UpdaterBinaryAutoupdate: + // We'll do the checks later. + default: + return trace.BadParameter("unsupported autoupdate style: %v", o.AutoupdateStyle) + } + if o.ProxyAddr == "" { + return trace.BadParameter("Proxy address is required") + } + + if o.TeleportVersion == "" { + return trace.BadParameter("Teleport version is required") + } + + if o.TeleportFlavor == "" { + return trace.BadParameter("Teleport flavor is required") + } + + if o.CDNBaseURL != "" { + url, err := url.Parse(o.CDNBaseURL) + if err != nil { + return trace.Wrap(err, "failed to parse CDN base URL") + } + if url.Scheme != "https" { + return trace.BadParameter("CDNBaseURL's scheme must be 'https://'") + } + } + return nil +} + +// oneOffParams returns the oneoff.OneOffScriptParams that will install Teleport +// using the oneoff.sh script to download and execute 'teleport-update'. +func (o *InstallScriptOptions) oneOffParams() (params oneoff.OneOffScriptParams) { + // We add the leading v if it's not here + version := o.TeleportVersion + if o.TeleportVersion[0] != 'v' { + version = "v" + o.TeleportVersion + } + + args := []string{"enable", "--proxy", shsprintf.EscapeDefaultContext(o.ProxyAddr)} + // Pass the base-url override if the base url is set and is not the default one. + if o.CDNBaseURL != "" && o.CDNBaseURL != teleportUpdateDefaultCDN { + args = append(args, "--base-url", shsprintf.EscapeDefaultContext(o.CDNBaseURL)) + } + + successMessage := "Teleport successfully installed." + if o.Insecure { + args = append(args, "--insecure") + successMessage += " --insecure was used during installation, automatic updates will not work unless the Proxy Service presents a certificate trusted by the system." + } + + return oneoff.OneOffScriptParams{ + Entrypoint: "teleport-update", + EntrypointArgs: strings.Join(args, " "), + CDNBaseURL: o.CDNBaseURL, + TeleportVersion: version, + TeleportFlavor: o.TeleportFlavor, + SuccessMessage: successMessage, + TeleportFIPS: o.FIPS, + } +} + +// GetInstallScript returns a Teleport installation script. +// This script only installs Teleport, it does not start the agent, join it, nor configure its services. +// See the InstallNodeBashScript if you need a more complete setup. +func GetInstallScript(ctx context.Context, opts InstallScriptOptions) (string, error) { + switch opts.AutoupdateStyle { + case NoAutoupdate, PackageManagerAutoupdate: + return getLegacyInstallScript(ctx, opts) + case UpdaterBinaryAutoupdate: + return getUpdaterInstallScript(ctx, opts) + default: + return "", trace.BadParameter("unsupported autoupdate style: %v", opts.AutoupdateStyle) + } +} + +//go:embed install/install.sh +var legacyInstallScript string + +var ( + versionVar = regexp.MustCompile(`(?m)^TELEPORT_VERSION=""$`) + suffixVar = regexp.MustCompile(`(?m)^TELEPORT_SUFFIX=""$`) + editionVar = regexp.MustCompile(`(?m)^TELEPORT_EDITION=""$`) +) + +// getLegacyInstallScript returns the installation script that we have been serving at +// "https://cdn.teleport.dev/install.sh". This script installs teleport via package manager +// or by unpacking the tarball. Its usage should be phased out in favor of the updater-based +// installation script served by getUpdaterInstallScript. +func getLegacyInstallScript(ctx context.Context, opts InstallScriptOptions) (string, error) { + tunedScript := versionVar.ReplaceAllString(legacyInstallScript, fmt.Sprintf(`TELEPORT_VERSION="%s"`, opts.TeleportVersion)) + if opts.TeleportFlavor == types.PackageNameEnt { + tunedScript = suffixVar.ReplaceAllString(tunedScript, `TELEPORT_SUFFIX="-ent"`) + } + + var edition string + if opts.AutoupdateStyle == PackageManagerAutoupdate { + edition = "cloud" + } else if opts.TeleportFlavor == types.PackageNameEnt { + edition = "enterprise" + } else { + edition = "oss" + } + tunedScript = editionVar.ReplaceAllString(tunedScript, fmt.Sprintf(`TELEPORT_EDITION="%s"`, edition)) + + return tunedScript, nil +} + +// getUpdaterInstallScript returns an installation script that downloads teleport-update +// and uses it to install a self-updating version of Teleport. +// This installation script is based on the oneoff.sh script and will become the standard +// way of installing Teleport. +func getUpdaterInstallScript(ctx context.Context, opts InstallScriptOptions) (string, error) { + if err := opts.Check(); err != nil { + return "", trace.Wrap(err, "invalid install script parameters") + } + + scriptParams := opts.oneOffParams() + + return oneoff.BuildScript(scriptParams) +} diff --git a/lib/web/scripts/install/install.sh b/lib/web/scripts/install/install.sh new file mode 100755 index 0000000000000..720bbec0ec624 --- /dev/null +++ b/lib/web/scripts/install/install.sh @@ -0,0 +1,429 @@ +#!/bin/bash +# Copyright 2022 Gravitational, Inc + +# This script detects the current Linux distribution and installs Teleport +# through its package manager, if supported, or downloading a tarball otherwise. +# We'll download Teleport from the official website and checksum it to make sure it was properly +# downloaded before executing. + +# The script is wrapped inside a function to protect against the connection being interrupted +# in the middle of the stream. + +# For more download options, head to https://goteleport.com/download/ + +set -euo pipefail + +# download uses curl or wget to download a teleport binary +download() { + URL=$1 + TMP_PATH=$2 + + echo "Downloading $URL" + if type curl &>/dev/null; then + set -x + # shellcheck disable=SC2086 + $SUDO $CURL -o "$TMP_PATH" "$URL" + else + set -x + # shellcheck disable=SC2086 + $SUDO $CURL -O "$TMP_PATH" "$URL" + fi + set +x +} + +install_via_apt_get() { + echo "Installing Teleport v$TELEPORT_VERSION via apt-get" + add_apt_key + set -x + $SUDO apt-get install -y "teleport$TELEPORT_SUFFIX=$TELEPORT_VERSION" + set +x + if [ "$TELEPORT_EDITION" = "cloud" ]; then + set -x + $SUDO apt-get install -y teleport-ent-updater + set +x + fi +} + +add_apt_key() { + APT_REPO_ID=$ID + APT_REPO_VERSION_CODENAME=$VERSION_CODENAME + IS_LEGACY=0 + + # check if we must use legacy .asc key + case "$ID" in + ubuntu | pop | neon | zorin) + if ! expr "$VERSION_ID" : "2.*" >/dev/null; then + IS_LEGACY=1 + fi + ;; + debian | raspbian) + if [ "$VERSION_ID" -lt 11 ]; then + IS_LEGACY=1 + fi + ;; + linuxmint | parrot) + if [ "$VERSION_ID" -lt 5 ]; then + IS_LEGACY=1 + fi + ;; + elementary) + if [ "$VERSION_ID" -lt 6 ]; then + IS_LEGACY=1 + fi + ;; + kali) + YEAR="$(echo "$VERSION_ID" | cut -f1 -d.)" + if [ "$YEAR" -lt 2021 ]; then + IS_LEGACY=1 + fi + ;; + esac + + if [[ "$IS_LEGACY" == 0 ]]; then + # set APT_REPO_ID if necessary + case "$ID" in + linuxmint | kali | elementary | pop | raspbian | neon | zorin | parrot) + APT_REPO_ID=$ID_LIKE + ;; + esac + + # set APT_REPO_VERSION_CODENAME if necessary + case "$ID" in + linuxmint | elementary | pop | neon | zorin) + APT_REPO_VERSION_CODENAME=$UBUNTU_CODENAME + ;; + kali) + APT_REPO_VERSION_CODENAME="bullseye" + ;; + parrot) + APT_REPO_VERSION_CODENAME="buster" + ;; + esac + fi + + echo "Downloading Teleport's PGP public key..." + TEMP_DIR=$(mktemp -d -t teleport-XXXXXXXXXX) + MAJOR=$(echo "$TELEPORT_VERSION" | cut -f1 -d.) + TELEPORT_REPO="" + + CHANNEL="stable/v${MAJOR}" + if [ "$TELEPORT_EDITION" = "cloud" ]; then + CHANNEL="stable/cloud" + fi + + if [[ "$IS_LEGACY" == 1 ]]; then + if ! type gpg >/dev/null; then + echo "Installing gnupg" + set -x + $SUDO apt-get update + $SUDO apt-get install -y gnupg + set +x + fi + TMP_KEY="$TEMP_DIR/teleport-pubkey.asc" + download "https://deb.releases.teleport.dev/teleport-pubkey.asc" "$TMP_KEY" + set -x + $SUDO apt-key add "$TMP_KEY" + set +x + TELEPORT_REPO="deb https://apt.releases.teleport.dev/${APT_REPO_ID?} ${APT_REPO_VERSION_CODENAME?} ${CHANNEL}" + else + TMP_KEY="$TEMP_DIR/teleport-pubkey.gpg" + download "https://apt.releases.teleport.dev/gpg" "$TMP_KEY" + set -x + $SUDO mkdir -p /etc/apt/keyrings + $SUDO cp "$TMP_KEY" /etc/apt/keyrings/teleport-archive-keyring.asc + set +x + TELEPORT_REPO="deb [signed-by=/etc/apt/keyrings/teleport-archive-keyring.asc] https://apt.releases.teleport.dev/${APT_REPO_ID?} ${APT_REPO_VERSION_CODENAME?} ${CHANNEL}" + fi + + set -x + echo "$TELEPORT_REPO" | $SUDO tee /etc/apt/sources.list.d/teleport.list >/dev/null + set +x + + set -x + $SUDO apt-get update + set +x +} + +# $1 is the value of the $ID path segment in the YUM repo URL. In +# /etc/os-release, this is either the value of $ID or $ID_LIKE. +install_via_yum() { + # shellcheck source=/dev/null + source /etc/os-release + + # Get the major version from the version ID. + VERSION_ID=$(echo "$VERSION_ID" | grep -Eo "^[0-9]+") + TELEPORT_MAJOR_VERSION="v$(echo "$TELEPORT_VERSION" | grep -Eo "^[0-9]+")" + + CHANNEL="stable/${TELEPORT_MAJOR_VERSION}" + if [ "$TELEPORT_EDITION" = "cloud" ]; then + CHANNEL="stable/cloud" + fi + + if type dnf &>/dev/null; then + echo "Installing Teleport v$TELEPORT_VERSION through dnf" + $SUDO dnf install -y 'dnf-command(config-manager)' + $SUDO dnf config-manager --add-repo "$(rpm --eval "https://yum.releases.teleport.dev/$1/$VERSION_ID/Teleport/%{_arch}/$CHANNEL/teleport-yum.repo")" + $SUDO dnf install -y "teleport$TELEPORT_SUFFIX-$TELEPORT_VERSION" + + if [ "$TELEPORT_EDITION" = "cloud" ]; then + $SUDO dnf install -y teleport-ent-updater + fi + + else + echo "Installing Teleport v$TELEPORT_VERSION through yum" + $SUDO yum install -y yum-utils + $SUDO yum-config-manager --add-repo "$(rpm --eval "https://yum.releases.teleport.dev/$1/$VERSION_ID/Teleport/%{_arch}/$CHANNEL/teleport-yum.repo")" + $SUDO yum install -y "teleport$TELEPORT_SUFFIX-$TELEPORT_VERSION" + + if [ "$TELEPORT_EDITION" = "cloud" ]; then + $SUDO yum install -y teleport-ent-updater + fi + fi + set +x +} + +install_via_zypper() { + # shellcheck source=/dev/null + source /etc/os-release + + # Get the major version from the version ID. + VERSION_ID=$(echo "$VERSION_ID" | grep -Eo "^[0-9]+") + TELEPORT_MAJOR_VERSION="v$(echo "$TELEPORT_VERSION" | grep -Eo "^[0-9]+")" + + CHANNEL="stable/${TELEPORT_MAJOR_VERSION}" + if [ "$TELEPORT_EDITION" = "cloud" ]; then + CHANNEL="stable/cloud" + fi + + $SUDO rpm --import https://zypper.releases.teleport.dev/gpg + $SUDO zypper addrepo --refresh --repo "$(rpm --eval "https://zypper.releases.teleport.dev/$ID/$VERSION_ID/Teleport/%{_arch}/$CHANNEL/teleport-zypper.repo")" + $SUDO zypper --gpg-auto-import-keys refresh teleport + $SUDO zypper install -y "teleport$TELEPORT_SUFFIX" + + if [ "$TELEPORT_EDITION" = "cloud" ]; then + $SUDO zypper install -y teleport-ent-updater + fi + + set +x +} + + +# download .tar.gz file via curl/wget, unzip it and run the install script +install_via_curl() { + TEMP_DIR=$(mktemp -d -t teleport-XXXXXXXXXX) + + TELEPORT_FILENAME="teleport$TELEPORT_SUFFIX-v$TELEPORT_VERSION-linux-$ARCH-bin.tar.gz" + URL="https://cdn.teleport.dev/${TELEPORT_FILENAME}" + download "${URL}" "${TEMP_DIR}/${TELEPORT_FILENAME}" + + TMP_CHECKSUM="${TEMP_DIR}/${TELEPORT_FILENAME}.sha256" + download "${URL}.sha256" "$TMP_CHECKSUM" + + set -x + cd "$TEMP_DIR" + # shellcheck disable=SC2086 + $SUDO $SHA_COMMAND -c "$TMP_CHECKSUM" + cd - + + $SUDO tar -xzf "${TEMP_DIR}/${TELEPORT_FILENAME}" -C "$TEMP_DIR" + $SUDO "$TEMP_DIR/teleport${TELEPORT_SUFFIX}/install" + set +x +} + +# wrap script in a function so a partially downloaded script +# doesn't execute +install_teleport() { + # exit if not on Linux + if [[ $(uname) != "Linux" ]]; then + echo "ERROR: This script works only for Linux. Please go to the downloads page to find the proper installation method for your operating system:" + echo "https://goteleport.com/download/" + exit 1 + fi + + KERNEL_VERSION=$(uname -r) + MIN_VERSION="2.6.23" + if [ $MIN_VERSION != "$(echo -e "$MIN_VERSION\n$KERNEL_VERSION" | sort -V | head -n1)" ]; then + echo "ERROR: Teleport requires Linux kernel version $MIN_VERSION+" + exit 1 + fi + + # check if can run as admin either by running as root or by + # having 'sudo' or 'doas' installed + IS_ROOT="" + SUDO="" + if [ "$(id -u)" = 0 ]; then + # running as root, no need for sudo/doas + IS_ROOT="YES" + SUDO="" + elif type sudo &>/dev/null; then + SUDO="sudo" + elif type doas &>/dev/null; then + SUDO="doas" + fi + + if [ -z "$SUDO" ] && [ -z "$IS_ROOT" ]; then + echo "ERROR: The installer requires a way to run commands as root." + echo "Either run this script as root or install sudo/doas." + exit 1 + fi + + # require curl/wget + CURL="" + if type curl &>/dev/null; then + CURL="curl -fL" + elif type wget &>/dev/null; then + CURL="wget" + fi + if [ -z "$CURL" ]; then + echo "ERROR: This script requires either curl or wget in order to download files. Please install one of them and try again." + exit 1 + fi + + # require shasum/sha256sum + SHA_COMMAND="" + if type shasum &>/dev/null; then + SHA_COMMAND="shasum -a 256" + elif type sha256sum &>/dev/null; then + SHA_COMMAND="sha256sum" + else + echo "ERROR: This script requires sha256sum or shasum to validate the download. Please install it and try again." + exit 1 + fi + + # detect distro + OS_RELEASE=/etc/os-release + ID="" + ID_LIKE="" + VERSION_CODENAME="" + UBUNTU_CODENAME="" + if [[ -f "$OS_RELEASE" ]]; then + # shellcheck source=/dev/null + . $OS_RELEASE + fi + # Some $ID_LIKE values include multiple distro names in an arbitrary order, so + # evaluate the first one. + ID_LIKE="${ID_LIKE%% *}" + + # detect architecture + ARCH="" + case $(uname -m) in + x86_64) + ARCH="amd64" + ;; + i386) + ARCH="386" + ;; + armv7l) + ARCH="arm" + ;; + aarch64) + ARCH="arm64" + ;; + **) + echo "ERROR: Your system's architecture isn't officially supported or couldn't be determined." + echo "Please refer to the installation guide for more information:" + echo "https://goteleport.com/docs/installation/" + exit 1 + ;; + esac + + # select install method based on distribution + # if ID is debian derivate, run apt-get + case "$ID" in + debian | ubuntu | kali | linuxmint | pop | raspbian | neon | zorin | parrot | elementary) + install_via_apt_get + ;; + # if ID is amazon Linux 2/RHEL/etc, run yum + centos | rhel | amzn) + install_via_yum "$ID" + ;; + sles) + install_via_zypper + ;; + *) + # before downloading manually, double check if we didn't miss any debian or + # rh/fedora derived distros using the ID_LIKE var. + case "${ID_LIKE}" in + ubuntu | debian) + install_via_apt_get + ;; + centos | fedora | rhel) + # There is no repository for "fedora", and there is no difference + # between the repositories for "centos" and "rhel", so pick an arbitrary + # one. + install_via_yum rhel + ;; + *) + if [ "$TELEPORT_EDITION" = "cloud" ]; then + echo "The system does not support a package manager, which is required for Teleport Enterprise Cloud." + exit 1 + fi + + # if ID and ID_LIKE didn't return a supported distro, download through curl + echo "There is no officially supported package for your package manager. Downloading and installing Teleport via curl." + install_via_curl + ;; + esac + ;; + esac + + GREEN='\033[0;32m' + COLOR_OFF='\033[0m' + + echo "" + echo -e "${GREEN}$(teleport version) installed successfully!${COLOR_OFF}" + echo "" + echo "The following commands are now available:" + if type teleport &>/dev/null; then + echo " teleport - The daemon that runs the Auth Service, Proxy Service, and other Teleport services." + fi + if type tsh &>/dev/null; then + echo " tsh - A tool that lets end users interact with Teleport." + fi + if type tctl &>/dev/null; then + echo " tctl - An administrative tool that can configure the Teleport Auth Service." + fi + if type tbot &>/dev/null; then + echo " tbot - Teleport Machine ID client." + fi + if type teleport-update &>/dev/null; then + echo " teleport-update - Teleport auto-update agent." + fi +} + +# The suffix is "-ent" if we are installing a commercial edition of Teleport and +# empty for Teleport Community Edition. +TELEPORT_SUFFIX="" +TELEPORT_VERSION="" +TELEPORT_EDITION="" +if [ $# -ge 1 ] && [ -n "$1" ]; then + TELEPORT_VERSION=$1 +else + if [ -z "$TELEPORT_VERSION" ]; then + echo "ERROR: Please provide the version you want to install (e.g., 10.1.9)." + exit 1 + fi +fi + +if ! echo "$TELEPORT_VERSION" | grep -qE "[0-9]+\.[0-9]+\.[0-9]+"; then + echo "ERROR: The first parameter must be a version number, e.g., 10.1.9." + exit 1 +fi + +if [ $# -ge 2 ] && [ -n "$2" ]; then + TELEPORT_EDITION=$2 + + case $TELEPORT_EDITION in + enterprise | cloud) + TELEPORT_SUFFIX="-ent" + ;; + # An empty edition defaults to OSS. + oss | "" ) + ;; + *) + echo 'ERROR: The second parameter must be "oss", "cloud", or "enterprise".' + exit 1 + ;; + esac +fi +install_teleport diff --git a/lib/web/scripts/install_node.go b/lib/web/scripts/install_node.go index 64cbd0ac88776..eeb91435bef22 100644 --- a/lib/web/scripts/install_node.go +++ b/lib/web/scripts/install_node.go @@ -17,19 +17,30 @@ limitations under the License. package scripts import ( + "bytes" + "context" _ "embed" "fmt" + regexp "regexp" "sort" + "strconv" "strings" "text/template" + "github.com/google/safetext/shsprintf" "github.com/gravitational/trace" "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/util/validation" "github.com/gravitational/teleport/api/types" - "github.com/gravitational/teleport/api/utils" + apiutils "github.com/gravitational/teleport/api/utils" + "github.com/gravitational/teleport/lib/automaticupgrades" + "github.com/gravitational/teleport/lib/utils" ) +// appURIPattern is a regexp excluding invalid characters from application URIs. +var appURIPattern = regexp.MustCompile(`^[-\w/:. ]+$`) + // ErrorBashScript is used to display friendly error message when // there is an error prepping the actual script. var ErrorBashScript = []byte(` @@ -42,11 +53,150 @@ exit 1 // to install teleport and join a teleport cluster. // //go:embed node-join/install.sh -var installNodeBashScript string +var installNodeBashScriptRaw string + +var installNodeBashScript = template.Must(template.New("nodejoin").Parse(installNodeBashScriptRaw)) + +// InstallNodeScriptOptions contains the options configuring the install-node script. +type InstallNodeScriptOptions struct { + // Required for installation + InstallOptions InstallScriptOptions + + // Required for joining + Token string + CAPins []string + JoinMethod types.JoinMethod + + // Required for service configuration + Labels types.Labels + LabelMatchers types.Labels + + AppServiceEnabled bool + AppName string + AppURI string + + DatabaseServiceEnabled bool + DiscoveryServiceEnabled bool + DiscoveryGroup string +} + +// GetNodeInstallScript generates an agent installation script which will: +// - install Teleport +// - configure the Teleport agent joining +// - configure the Teleport agent services (currently support ssh, app, database, and discovery) +// - start the agent +func GetNodeInstallScript(ctx context.Context, opts InstallNodeScriptOptions) (string, error) { + // Computing installation-related values + + // By default, it will use `stable/v`, eg stable/v12 + repoChannel := "" + + switch opts.InstallOptions.AutoupdateStyle { + case NoAutoupdate, UpdaterBinaryAutoupdate: + case PackageManagerAutoupdate: + // Note: This is a cloud-specific repo. We could use the new stable/rolling + // repo in non-cloud case, but the script has never support enabling autoupdates + // in a non-cloud cluster. + // We will prefer using the new updater binary for autoupdates in self-hosted setups. + repoChannel = automaticupgrades.DefaultCloudChannelName + default: + return "", trace.BadParameter("unsupported autoupdate style: %v", opts.InstallOptions.AutoupdateStyle) + } + + // Computing joining-related values + hostname, portStr, err := utils.SplitHostPort(opts.InstallOptions.ProxyAddr) + if err != nil { + return "", trace.Wrap(err) + } + + // Computing service configuration-related values + labelsList := []string{} + for labelKey, labelValues := range opts.Labels { + labelKey = shsprintf.EscapeDefaultContext(labelKey) + for i := range labelValues { + labelValues[i] = shsprintf.EscapeDefaultContext(labelValues[i]) + } + labels := strings.Join(labelValues, " ") + labelsList = append(labelsList, fmt.Sprintf("%s=%s", labelKey, labels)) + } + + var dbServiceResourceLabels []string + if opts.DatabaseServiceEnabled { + dbServiceResourceLabels, err = marshalLabelsYAML(opts.LabelMatchers, 6) + if err != nil { + return "", trace.Wrap(err) + } + } + + var appServerResourceLabels []string + + if opts.AppServiceEnabled { + if errs := validation.IsDNS1035Label(opts.AppName); len(errs) > 0 { + return "", trace.BadParameter("appName %q must be a valid DNS subdomain: https://goteleport.com/docs/enroll-resources/application-access/guides/connecting-apps/#application-name", opts.AppName) + } + if !appURIPattern.MatchString(opts.AppURI) { + return "", trace.BadParameter("appURI %q contains invalid characters", opts.AppURI) + } + + appServerResourceLabels, err = marshalLabelsYAML(opts.Labels, 4) + if err != nil { + return "", trace.Wrap(err) + } + } + + if opts.DiscoveryServiceEnabled { + if opts.DiscoveryGroup == "" { + return "", trace.BadParameter("discovery group is required") + } + } + + var buf bytes.Buffer + + // TODO(hugoShaka): burn this map and replace it by something saner in a future PR. + + // This section relies on Go's default zero values to make sure that the settings + // are correct when not installing an app. + err = installNodeBashScript.Execute(&buf, map[string]interface{}{ + "token": opts.Token, + "hostname": hostname, + "port": portStr, + // The install.sh script has some manually generated configs and some + // generated by the `teleport config` commands. The old bash + // version used space delimited values whereas the teleport command uses + // a comma delimeter. The Old version can be removed when the install.sh + // file has been completely converted over. + "caPinsOld": strings.Join(opts.CAPins, " "), + "caPins": strings.Join(opts.CAPins, ","), + "packageName": opts.InstallOptions.TeleportFlavor, + "repoChannel": repoChannel, + "installUpdater": opts.InstallOptions.AutoupdateStyle.String(), + "version": shsprintf.EscapeDefaultContext(opts.InstallOptions.TeleportVersion), + "appInstallMode": strconv.FormatBool(opts.AppServiceEnabled), + "appServerResourceLabels": appServerResourceLabels, + "appName": shsprintf.EscapeDefaultContext(opts.AppName), + "appURI": shsprintf.EscapeDefaultContext(opts.AppURI), + "joinMethod": shsprintf.EscapeDefaultContext(string(opts.JoinMethod)), + "labels": strings.Join(labelsList, ","), + "databaseInstallMode": strconv.FormatBool(opts.DatabaseServiceEnabled), + // No one knows why this field is in snake case ¯\_(ツ)_/¯ + // Also, even if the name is similar to appServerResourceLabels, they must not be confused. + // appServerResourceLabels are labels to apply on the declared app, while + // db_service_resource_labels are labels matchers for the service to select resources to serve. + "db_service_resource_labels": dbServiceResourceLabels, + "discoveryInstallMode": strconv.FormatBool(opts.DiscoveryServiceEnabled), + "discoveryGroup": shsprintf.EscapeDefaultContext(opts.DiscoveryGroup), + }) + if err != nil { + return "", trace.Wrap(err) + } + + return buf.String(), nil +} -var InstallNodeBashScript = template.Must(template.New("nodejoin").Parse(installNodeBashScript)) +// TODO(hugoShaka): burn the indentation thing, this is too fragile and show be handled +// by the template itself. -// MarshalLabelsYAML returns a list of strings, each one containing a +// marshalLabelsYAML returns a list of strings, each one containing a // label key and list of value's pair. // This is used to create yaml sections within the join scripts. // @@ -54,7 +204,7 @@ var InstallNodeBashScript = template.Must(template.New("nodejoin").Parse(install // top of the default space already used, for the default yaml listing // format (the listing values with the dashes). If `extraListIndent` // is zero, it's equivalent to using default space only (which is 4 spaces). -func MarshalLabelsYAML(resourceMatcherLabels types.Labels, extraListIndent int) ([]string, error) { +func marshalLabelsYAML(resourceMatcherLabels types.Labels, extraListIndent int) ([]string, error) { if len(resourceMatcherLabels) == 0 { return []string{"{}"}, nil } @@ -71,7 +221,7 @@ func MarshalLabelsYAML(resourceMatcherLabels types.Labels, extraListIndent int) for _, labelName := range labelKeys { labelValues := resourceMatcherLabels[labelName] - bs, err := yaml.Marshal(map[string]utils.Strings{labelName: labelValues}) + bs, err := yaml.Marshal(map[string]apiutils.Strings{labelName: labelValues}) if err != nil { return nil, trace.Wrap(err) } diff --git a/lib/web/scripts/install_node_test.go b/lib/web/scripts/install_node_test.go index de005af077c98..3f3107718a544 100644 --- a/lib/web/scripts/install_node_test.go +++ b/lib/web/scripts/install_node_test.go @@ -63,7 +63,7 @@ func TestMarshalLabelsYAML(t *testing.T) { numExtraIndent: 2, }, } { - got, err := MarshalLabelsYAML(tt.labels, tt.numExtraIndent) + got, err := marshalLabelsYAML(tt.labels, tt.numExtraIndent) require.NoError(t, err) require.YAMLEq(t, strings.Join(tt.expected, "\n"), strings.Join(got, "\n")) diff --git a/lib/web/scripts/install_test.go b/lib/web/scripts/install_test.go new file mode 100644 index 0000000000000..5061f5d02bc37 --- /dev/null +++ b/lib/web/scripts/install_test.go @@ -0,0 +1,181 @@ +/* + * Teleport + * Copyright (C) 2025 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package scripts + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/lib/utils/teleportassets" +) + +func TestGetInstallScript(t *testing.T) { + ctx := context.Background() + testVersion := "1.2.3" + testProxyAddr := "proxy.example.com:443" + + tests := []struct { + name string + opts InstallScriptOptions + assertFn func(t *testing.T, script string) + }{ + { + name: "Legacy install, oss", + opts: InstallScriptOptions{ + AutoupdateStyle: NoAutoupdate, + TeleportVersion: testVersion, + TeleportFlavor: types.PackageNameOSS, + }, + assertFn: func(t *testing.T, script string) { + require.Contains(t, script, fmt.Sprintf(`TELEPORT_VERSION="%s"`, testVersion)) + require.Contains(t, script, `TELEPORT_SUFFIX=""`) + require.Contains(t, script, `TELEPORT_EDITION="oss"`) + }, + }, + { + name: "Legacy install, enterprise", + opts: InstallScriptOptions{ + AutoupdateStyle: NoAutoupdate, + TeleportVersion: testVersion, + TeleportFlavor: types.PackageNameEnt, + }, + assertFn: func(t *testing.T, script string) { + require.Contains(t, script, fmt.Sprintf(`TELEPORT_VERSION="%s"`, testVersion)) + require.Contains(t, script, `TELEPORT_SUFFIX="-ent"`) + require.Contains(t, script, `TELEPORT_EDITION="enterprise"`) + }, + }, + { + name: "Legacy install, cloud", + opts: InstallScriptOptions{ + AutoupdateStyle: PackageManagerAutoupdate, + TeleportVersion: testVersion, + TeleportFlavor: types.PackageNameEnt, + }, + assertFn: func(t *testing.T, script string) { + require.Contains(t, script, fmt.Sprintf(`TELEPORT_VERSION="%s"`, testVersion)) + require.Contains(t, script, `TELEPORT_SUFFIX="-ent"`) + require.Contains(t, script, `TELEPORT_EDITION="cloud"`) + }, + }, + { + name: "Oneoff install", + opts: InstallScriptOptions{ + AutoupdateStyle: UpdaterBinaryAutoupdate, + TeleportVersion: testVersion, + ProxyAddr: testProxyAddr, + TeleportFlavor: types.PackageNameOSS, + }, + assertFn: func(t *testing.T, script string) { + require.Contains(t, script, "entrypoint='teleport-update'") + require.Contains(t, script, fmt.Sprintf("teleportVersion='v%s'", testVersion)) + require.Contains(t, script, fmt.Sprintf("teleportFlavor='%s'", types.PackageNameOSS)) + require.Contains(t, script, fmt.Sprintf("cdnBaseURL='%s'", teleportassets.CDNBaseURL())) + require.Contains(t, script, fmt.Sprintf("entrypointArgs='enable --proxy %s'", testProxyAddr)) + require.Contains(t, script, "packageSuffix='bin.tar.gz'") + }, + }, + { + name: "Oneoff install custom CDN", + opts: InstallScriptOptions{ + AutoupdateStyle: UpdaterBinaryAutoupdate, + TeleportVersion: testVersion, + ProxyAddr: testProxyAddr, + TeleportFlavor: types.PackageNameOSS, + CDNBaseURL: "https://cdn.example.com", + }, + assertFn: func(t *testing.T, script string) { + require.Contains(t, script, "entrypoint='teleport-update'") + require.Contains(t, script, fmt.Sprintf("teleportVersion='v%s'", testVersion)) + require.Contains(t, script, fmt.Sprintf("teleportFlavor='%s'", types.PackageNameOSS)) + require.Contains(t, script, "cdnBaseURL='https://cdn.example.com'") + require.Contains(t, script, fmt.Sprintf("entrypointArgs='enable --proxy %s --base-url %s'", testProxyAddr, "https://cdn.example.com")) + require.Contains(t, script, "packageSuffix='bin.tar.gz'") + }, + }, + { + name: "Oneoff install default CDN", + opts: InstallScriptOptions{ + AutoupdateStyle: UpdaterBinaryAutoupdate, + TeleportVersion: testVersion, + ProxyAddr: testProxyAddr, + TeleportFlavor: types.PackageNameOSS, + CDNBaseURL: teleportassets.TeleportReleaseCDN, + }, + assertFn: func(t *testing.T, script string) { + require.Contains(t, script, "entrypoint='teleport-update'") + require.Contains(t, script, fmt.Sprintf("teleportVersion='v%s'", testVersion)) + require.Contains(t, script, fmt.Sprintf("teleportFlavor='%s'", types.PackageNameOSS)) + require.Contains(t, script, fmt.Sprintf("cdnBaseURL='%s'", teleportassets.TeleportReleaseCDN)) + require.Contains(t, script, fmt.Sprintf("entrypointArgs='enable --proxy %s'", testProxyAddr)) + require.Contains(t, script, "packageSuffix='bin.tar.gz'") + }, + }, + { + name: "Oneoff enterprise install", + opts: InstallScriptOptions{ + AutoupdateStyle: UpdaterBinaryAutoupdate, + TeleportVersion: testVersion, + ProxyAddr: testProxyAddr, + TeleportFlavor: types.PackageNameEnt, + }, + assertFn: func(t *testing.T, script string) { + require.Contains(t, script, "entrypoint='teleport-update'") + require.Contains(t, script, fmt.Sprintf("teleportVersion='v%s'", testVersion)) + require.Contains(t, script, fmt.Sprintf("teleportFlavor='%s'", types.PackageNameEnt)) + require.Contains(t, script, fmt.Sprintf("cdnBaseURL='%s'", teleportassets.CDNBaseURL())) + require.Contains(t, script, fmt.Sprintf("entrypointArgs='enable --proxy %s'", testProxyAddr)) + require.Contains(t, script, "packageSuffix='bin.tar.gz'") + }, + }, + { + name: "Oneoff enterprise FIPS install", + opts: InstallScriptOptions{ + AutoupdateStyle: UpdaterBinaryAutoupdate, + TeleportVersion: testVersion, + ProxyAddr: testProxyAddr, + TeleportFlavor: types.PackageNameEnt, + FIPS: true, + }, + assertFn: func(t *testing.T, script string) { + require.Contains(t, script, "entrypoint='teleport-update'") + require.Contains(t, script, fmt.Sprintf("teleportVersion='v%s'", testVersion)) + require.Contains(t, script, fmt.Sprintf("teleportFlavor='%s'", types.PackageNameEnt)) + require.Contains(t, script, fmt.Sprintf("cdnBaseURL='%s'", teleportassets.CDNBaseURL())) + require.Contains(t, script, fmt.Sprintf("entrypointArgs='enable --proxy %s'", testProxyAddr)) + require.Contains(t, script, "packageSuffix='fips-bin.tar.gz'") + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Sanity check, test input should be legal. + require.NoError(t, test.opts.Check()) + + // Test execution. + result, err := GetInstallScript(ctx, test.opts) + require.NoError(t, err) + test.assertFn(t, result) + }) + } +} diff --git a/lib/web/scripts/node-join/install.sh b/lib/web/scripts/node-join/install.sh index 9f19832c53e22..fdba964e96aa5 100755 --- a/lib/web/scripts/node-join/install.sh +++ b/lib/web/scripts/node-join/install.sh @@ -17,10 +17,17 @@ SYSTEMD_UNIT_PATH="/lib/systemd/system/teleport.service" TARGET_PORT_DEFAULT=443 TELEPORT_ARCHIVE_PATH='{{.packageName}}' TELEPORT_BINARY_DIR="/usr/local/bin" -TELEPORT_BINARY_LIST="teleport tctl tsh" +TELEPORT_BINARY_LIST="teleport tctl tsh teleport-update" TELEPORT_CONFIG_PATH="/etc/teleport.yaml" TELEPORT_DATA_DIR="/var/lib/teleport" TELEPORT_DOCS_URL="https://goteleport.com/docs/" +# TELEPORT_FORMAT contains the Teleport installation formats. +# The value is dynamically computed unless OVERRIDE_FORMAT it set. +# Possible values are: +# - "deb" +# - "rpm" +# - "tarball" +# - "updater" TELEPORT_FORMAT="" # initialise variables (because set -u disallows unbound variables) @@ -38,6 +45,9 @@ INTERACTIVE=false # optionally be replaced by the server before the script is served up TELEPORT_VERSION='{{.version}}' TELEPORT_PACKAGE_NAME='{{.packageName}}' +# UPDATER_STYLE holds the Teleport updater style. +# Supported values are "none", "" (same as "none"), "package", and "binary". +UPDATER_STYLE='{{.installUpdater}}' REPO_CHANNEL='{{.repoChannel}}' TARGET_HOSTNAME='{{.hostname}}' TARGET_PORT='{{.port}}' @@ -653,23 +663,30 @@ fi # use OSTYPE variable to figure out host type/arch if [[ "${OSTYPE}" == "linux"* ]]; then - # linux host, now detect arch - TELEPORT_BINARY_TYPE="linux" - ARCH=$(uname -m) - log "Detected host: ${OSTYPE}, using Teleport binary type ${TELEPORT_BINARY_TYPE}" - if [[ ${ARCH} == "armv7l" ]]; then - TELEPORT_ARCH="arm" - elif [[ ${ARCH} == "aarch64" ]]; then - TELEPORT_ARCH="arm64" - elif [[ ${ARCH} == "x86_64" ]]; then - TELEPORT_ARCH="amd64" - elif [[ ${ARCH} == "i686" ]]; then - TELEPORT_ARCH="386" + + if [[ "$UPDATER_STYLE" == "binary" ]]; then + # if we are using the new updater, we can bypass this detection dance + # and always use the updater. + TELEPORT_FORMAT="updater" else - log_important "Error: cannot detect architecture from uname -m: ${ARCH}" - exit 1 + # linux host, now detect arch + TELEPORT_BINARY_TYPE="linux" + ARCH=$(uname -m) + log "Detected host: ${OSTYPE}, using Teleport binary type ${TELEPORT_BINARY_TYPE}" + if [[ ${ARCH} == "armv7l" ]]; then + TELEPORT_ARCH="arm" + elif [[ ${ARCH} == "aarch64" ]]; then + TELEPORT_ARCH="arm64" + elif [[ ${ARCH} == "x86_64" ]]; then + TELEPORT_ARCH="amd64" + elif [[ ${ARCH} == "i686" ]]; then + TELEPORT_ARCH="386" + else + log_important "Error: cannot detect architecture from uname -m: ${ARCH}" + exit 1 + fi + log "Detected arch: ${ARCH}, using Teleport arch ${TELEPORT_ARCH}" fi - log "Detected arch: ${ARCH}, using Teleport arch ${TELEPORT_ARCH}" # if the download format is already set, we have no need to detect distro if [[ ${TELEPORT_FORMAT} == "" ]]; then # detect distro @@ -821,7 +838,9 @@ install_from_file() { tar -xzf "${TEMP_DIR}/${DOWNLOAD_FILENAME}" -C "${TEMP_DIR}" # install binaries to /usr/local/bin for BINARY in ${TELEPORT_BINARY_LIST}; do - ${COPY_COMMAND} "${TELEPORT_ARCHIVE_PATH}/${BINARY}" "${TELEPORT_BINARY_DIR}/" + if [ -e "${TELEPORT_ARCHIVE_PATH}/${BINARY}" ]; then + ${COPY_COMMAND} "${TELEPORT_ARCHIVE_PATH}/${BINARY}" "${TELEPORT_BINARY_DIR}/" + fi done elif [[ ${TELEPORT_FORMAT} == "deb" ]]; then # convert teleport arch to deb arch @@ -952,6 +971,26 @@ install_from_repo() { fi } +install_from_updater() { + SCRIPT_URL="https://$TARGET_HOSTNAME:$TARGET_PORT/scripts/install.sh" + CURL_COMMAND="curl -fsS" + if [[ ${DISABLE_TLS_VERIFICATION} == "true" ]]; then + CURL_COMMAND+=" -k" + SCRIPT_URL+="?insecure=true" + fi + + log "Requesting the install script: $SCRIPT_URL" + $CURL_COMMAND "$SCRIPT_URL" -o "$TEMP_DIR/install.sh" || (log "Failed to retrieve the install script." && exit 1) + + chmod +x "$TEMP_DIR/install.sh" + + log "Executing the install script" + # We execute the install script because it might be a bash or sh script depending on the install script served. + # This might cause issues if tmp is mounted with noexec, but the oneoff.sh script will also download and exec + # binaries from tmp + "$TEMP_DIR/install.sh" +} + # package_list returns the list of packages to install. # The list of packages can be fed into yum or apt because they already have the expected format when pinning versions. package_list() { @@ -972,7 +1011,7 @@ package_list() { # (warning): This expression is constant. Did you forget the $ on a variable? # Disabling the warning above because expression is templated. # shellcheck disable=SC2050 - if is_using_systemd && [[ "{{.installUpdater}}" == "true" ]]; then + if is_using_systemd && [[ "$UPDATER_STYLE" == "package" ]]; then # Teleport Updater requires systemd. PACKAGE_LIST+=" ${TELEPORT_UPDATER_PIN_VERSION}" fi @@ -1002,7 +1041,10 @@ is_repo_available() { return 1 } -if is_repo_available; then +if [[ "$TELEPORT_FORMAT" == "updater" ]]; then + log "Installing from updater binary." + install_from_updater +elif is_repo_available; then log "Installing repo for distro $ID." install_from_repo else diff --git a/lib/web/scripts/oneoff/oneoff.go b/lib/web/scripts/oneoff/oneoff.go index d1eb1cebe7fad..0c82b34ec7969 100644 --- a/lib/web/scripts/oneoff/oneoff.go +++ b/lib/web/scripts/oneoff/oneoff.go @@ -20,27 +20,31 @@ import ( "bytes" _ "embed" "slices" + "strings" "text/template" "github.com/gravitational/trace" - "github.com/gravitational/teleport" + "github.com/gravitational/teleport/api" "github.com/gravitational/teleport/api/types" "github.com/gravitational/teleport/lib/modules" + "github.com/gravitational/teleport/lib/utils/teleportassets" ) const ( - // teleportCDNLocation is the Teleport's CDN URL - // This is used to download the Teleport Binary - teleportCDNLocation = "https://cdn.teleport.dev" - // binUname is the default binary name for inspecting the host's OS. binUname = "uname" // binMktemp is the default binary name for creating temporary directories. binMktemp = "mktemp" + + // PrefixSUDO is a Teleport Command Prefix that executes with higher privileges + // Use with caution. + PrefixSUDO = "sudo" ) +var allowedCommandPrefix = []string{PrefixSUDO} + var ( //go:embed oneoff.sh oneoffScript string @@ -51,9 +55,19 @@ var ( // OneOffScriptParams contains the required params to create a script that downloads and executes teleport binary. type OneOffScriptParams struct { - // TeleportArgs is the arguments to pass to the teleport binary. + // TeleportCommandPrefix is a prefix command to use when calling teleport command. + // Acceptable values are: "sudo" + TeleportCommandPrefix string + // binSudo contains the location for the sudo binary. + // Used for testing. + binSudo string + + // Entrypoint is the name of the binary from the teleport package. Defaults to "teleport", but can be set to + // other binaries such as "teleport-update" or "tbot". + Entrypoint string + // EntrypointArgs is the arguments to pass to the Entrypoint binary. // Eg, 'version' - TeleportArgs string + EntrypointArgs string // BinUname is the binary used to get OS name and Architecture of the host. // Defaults to `uname`. @@ -76,6 +90,9 @@ type OneOffScriptParams struct { // - teleport-ent TeleportFlavor string + // TeleportFIPS represents if the script should install a FIPS build of Teleport. + TeleportFIPS bool + // SuccessMessage is a message shown to the user after the one off is completed. SuccessMessage string } @@ -84,10 +101,14 @@ var validPackageNames = []string{types.PackageNameOSS, types.PackageNameEnt} // CheckAndSetDefaults checks if the required params ara present. func (p *OneOffScriptParams) CheckAndSetDefaults() error { - if p.TeleportArgs == "" { + if p.EntrypointArgs == "" { return trace.BadParameter("missing teleport args") } + if p.Entrypoint == "" { + p.Entrypoint = "teleport" + } + if p.BinUname == "" { p.BinUname = binUname } @@ -96,13 +117,18 @@ func (p *OneOffScriptParams) CheckAndSetDefaults() error { p.BinMktemp = binMktemp } - if p.CDNBaseURL == "" { - p.CDNBaseURL = teleportCDNLocation + if p.binSudo == "" { + p.binSudo = PrefixSUDO } if p.TeleportVersion == "" { - p.TeleportVersion = "v" + teleport.Version + p.TeleportVersion = "v" + api.Version + } + + if p.CDNBaseURL == "" { + p.CDNBaseURL = teleportassets.CDNBaseURL() } + p.CDNBaseURL = strings.TrimRight(p.CDNBaseURL, "/") if p.TeleportFlavor == "" { p.TeleportFlavor = types.PackageNameOSS @@ -118,6 +144,14 @@ func (p *OneOffScriptParams) CheckAndSetDefaults() error { p.SuccessMessage = "Completed successfully." } + switch p.TeleportCommandPrefix { + case PrefixSUDO: + p.TeleportCommandPrefix = p.binSudo + case "": + default: + return trace.BadParameter("invalid command prefix %q, only %v are supported", p.TeleportCommandPrefix, allowedCommandPrefix) + } + return nil } diff --git a/lib/web/scripts/oneoff/oneoff.sh b/lib/web/scripts/oneoff/oneoff.sh index 0a256cae5df06..eaa15841be18b 100644 --- a/lib/web/scripts/oneoff/oneoff.sh +++ b/lib/web/scripts/oneoff/oneoff.sh @@ -1,54 +1,58 @@ -#!/usr/bin/env bash -set -euo pipefail +#!/usr/bin/env sh +set -eu cdnBaseURL='{{.CDNBaseURL}}' teleportVersion='{{.TeleportVersion}}' teleportFlavor='{{.TeleportFlavor}}' # teleport or teleport-ent successMessage='{{.SuccessMessage}}' +entrypointArgs='{{.EntrypointArgs}}' +entrypoint='{{.Entrypoint}}' +packageSuffix='{{ if .TeleportFIPS }}fips-{{ end }}bin.tar.gz' +fips='{{ if .TeleportFIPS }}true{{ end }}' # shellcheck disable=all -tempDir=$({{.BinMktemp}} -d) +# Use $HOME or / as base dir +tempDir=$({{.BinMktemp}} -d -p ${HOME:-}/) OS=$({{.BinUname}} -s) ARCH=$({{.BinUname}} -m) # shellcheck enable=all -teleportArgs='{{.TeleportArgs}}' +trap 'rm -rf -- "$tempDir"' EXIT -function teleportTarballName(){ - if [[ ${OS} == "Darwin" ]]; then - echo ${teleportFlavor}-${teleportVersion}-darwin-universal-bin.tar.gz +teleportTarballName() { + if [ "${OS}" = "Darwin" ]; then + if [ "$fips" = "true"]; then + echo "FIPS version of Teleport is not compatible with MacOS. Please run this script in a Linux machine." + return 1 + fi + echo "${teleportFlavor}-${teleportVersion}-darwin-universal-${packageSuffix}" return 0 fi; - if [[ ${OS} != "Linux" ]]; then + if [ "${OS}" != "Linux" ]; then echo "Only MacOS and Linux are supported." >&2 return 1 fi; - if [[ ${ARCH} == "armv7l" ]]; then echo "${teleportFlavor}-${teleportVersion}-linux-arm-bin.tar.gz" - elif [[ ${ARCH} == "aarch64" ]]; then echo "${teleportFlavor}-${teleportVersion}-linux-arm64-bin.tar.gz" - elif [[ ${ARCH} == "x86_64" ]]; then echo "${teleportFlavor}-${teleportVersion}-linux-amd64-bin.tar.gz" - elif [[ ${ARCH} == "i686" ]]; then echo "${teleportFlavor}-${teleportVersion}-linux-386-bin.tar.gz" + if [ ${ARCH} = "armv7l" ]; then echo "${teleportFlavor}-${teleportVersion}-linux-arm-${packageSuffix}" + elif [ ${ARCH} = "aarch64" ]; then echo "${teleportFlavor}-${teleportVersion}-linux-arm64-${packageSuffix}" + elif [ ${ARCH} = "x86_64" ]; then echo "${teleportFlavor}-${teleportVersion}-linux-amd64-${packageSuffix}" + elif [ ${ARCH} = "i686" ]; then echo "${teleportFlavor}-${teleportVersion}-linux-386-${packageSuffix}" else echo "Invalid Linux architecture ${ARCH}." >&2 return 1 fi; } -function main() { - pushd $tempDir > /dev/null - +main() { tarballName=$(teleportTarballName) - curl --show-error --fail --location --remote-name ${cdnBaseURL}/${tarballName} - echo "Extracting teleport to $tempDir ..." - tar -xzf ${tarballName} - - mkdir -p ./bin - mv ./${teleportFlavor}/teleport ./bin/teleport - echo "> ./bin/teleport ${teleportArgs}" - ./bin/teleport ${teleportArgs} && echo $successMessage + echo "Downloading from ${cdnBaseURL}/${tarballName} and extracting teleport to ${tempDir} ..." + curl --show-error --fail --location "${cdnBaseURL}/${tarballName}" | tar xzf - -C "${tempDir}" "${teleportFlavor}/${entrypoint}" - popd > /dev/null + mkdir -p "${tempDir}/bin" + mv "${tempDir}/${teleportFlavor}/${entrypoint}" "${tempDir}/bin/${entrypoint}" + echo "> ${tempDir}/bin/${entrypoint} ${entrypointArgs} $@" + {{.TeleportCommandPrefix}} "${tempDir}/bin/${entrypoint}" ${entrypointArgs} $@ && echo "$successMessage" } -main +main $@ diff --git a/lib/web/scripts/oneoff/oneoff_test.go b/lib/web/scripts/oneoff/oneoff_test.go index ec85b70821403..c6da7d96e2e21 100644 --- a/lib/web/scripts/oneoff/oneoff_test.go +++ b/lib/web/scripts/oneoff/oneoff_test.go @@ -1,18 +1,20 @@ /* -Copyright 2023 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ + * Teleport + * Copyright (C) 2023 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ package oneoff @@ -40,24 +42,34 @@ func TestOneOffScript(t *testing.T) { teleportVersionOutput := "Teleport v13.1.0 git:api/v13.1.0-0-gd83ec74 go1.20.4" scriptName := "oneoff.sh" + homeDir, err := os.UserHomeDir() + require.NoError(t, err) + homeDir = homeDir + "/" + unameMock, err := bintest.NewMock("uname") require.NoError(t, err) - defer func() { + t.Cleanup(func() { assert.NoError(t, unameMock.Close()) - }() + }) mktempMock, err := bintest.NewMock("mktemp") require.NoError(t, err) - defer func() { + t.Cleanup(func() { assert.NoError(t, mktempMock.Close()) - }() + }) + + sudoMock, err := bintest.NewMock("sudo") + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, sudoMock.Close()) + }) script, err := BuildScript(OneOffScriptParams{ BinUname: unameMock.Path, BinMktemp: mktempMock.Path, CDNBaseURL: "dummyURL", TeleportVersion: "v13.1.0", - TeleportArgs: "version", + EntrypointArgs: "version", }) require.NoError(t, err) @@ -69,9 +81,9 @@ func TestOneOffScript(t *testing.T) { teleportMock, err := bintest.NewMock(testWorkingDir + "/bin/teleport") require.NoError(t, err) - defer func() { + t.Cleanup(func() { assert.NoError(t, teleportMock.Close()) - }() + }) teleportBinTarball, err := utils.CompressTarGzArchive([]string{"teleport/teleport"}, singleFileFS{file: teleportMock.Path}) require.NoError(t, err) @@ -80,28 +92,87 @@ func TestOneOffScript(t *testing.T) { assert.Equal(t, "/teleport-v13.1.0-linux-amd64-bin.tar.gz", req.URL.Path) http.ServeContent(w, req, "teleport-v13.1.0-linux-amd64-bin.tar.gz", time.Now(), bytes.NewReader(teleportBinTarball.Bytes())) })) - defer func() { testServer.Close() }() + t.Cleanup(func() { testServer.Close() }) script, err := BuildScript(OneOffScriptParams{ BinUname: unameMock.Path, BinMktemp: mktempMock.Path, CDNBaseURL: testServer.URL, TeleportVersion: "v13.1.0", - TeleportArgs: "version", + EntrypointArgs: "version", SuccessMessage: "Test was a success.", }) require.NoError(t, err) unameMock.Expect("-s").AndWriteToStdout("Linux") unameMock.Expect("-m").AndWriteToStdout("x86_64") - mktempMock.Expect("-d").AndWriteToStdout(testWorkingDir) + mktempMock.Expect("-d", "-p", homeDir).AndWriteToStdout(testWorkingDir) teleportMock.Expect("version").AndWriteToStdout(teleportVersionOutput) err = os.WriteFile(scriptLocation, []byte(script), 0700) require.NoError(t, err) // execute script - out, err := exec.Command("bash", scriptLocation).CombinedOutput() + out, err := exec.Command("sh", scriptLocation).CombinedOutput() + + // validate + require.NoError(t, err, string(out)) + + require.True(t, unameMock.Check(t)) + require.True(t, mktempMock.Check(t)) + require.True(t, teleportMock.Check(t)) + + require.Contains(t, string(out), "teleport version") + require.Contains(t, string(out), teleportVersionOutput) + require.Contains(t, string(out), "Test was a success.") + + // Script should remove the temporary directory. + require.NoDirExists(t, testWorkingDir) + }) + + t.Run("command with prefix can be executed", func(t *testing.T) { + // set up + testWorkingDir := t.TempDir() + require.NoError(t, os.Mkdir(testWorkingDir+"/bin/", 0o755)) + scriptLocation := testWorkingDir + "/" + scriptName + + teleportMock, err := bintest.NewMock(testWorkingDir + "/bin/teleport") + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, teleportMock.Close()) + }) + + teleportBinTarball, err := utils.CompressTarGzArchive([]string{"teleport/teleport"}, singleFileFS{file: teleportMock.Path}) + require.NoError(t, err) + + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + assert.Equal(t, "/teleport-v13.1.0-linux-amd64-bin.tar.gz", req.URL.Path) + http.ServeContent(w, req, "teleport-v13.1.0-linux-amd64-bin.tar.gz", time.Now(), bytes.NewReader(teleportBinTarball.Bytes())) + })) + t.Cleanup(func() { testServer.Close() }) + + script, err := BuildScript(OneOffScriptParams{ + BinUname: unameMock.Path, + BinMktemp: mktempMock.Path, + CDNBaseURL: testServer.URL, + TeleportVersion: "v13.1.0", + EntrypointArgs: "version", + SuccessMessage: "Test was a success.", + TeleportCommandPrefix: "sudo", + binSudo: sudoMock.Path, + }) + require.NoError(t, err) + + unameMock.Expect("-s").AndWriteToStdout("Linux") + unameMock.Expect("-m").AndWriteToStdout("x86_64") + mktempMock.Expect("-d", "-p", homeDir).AndWriteToStdout(testWorkingDir) + sudoMock.Expect(teleportMock.Path, "version").AndWriteToStdout(teleportVersionOutput) + + err = os.WriteFile(scriptLocation, []byte(script), 0700) + require.NoError(t, err) + + // execute script + out, err := exec.Command("sh", scriptLocation).CombinedOutput() // validate require.NoError(t, err, string(out)) @@ -110,9 +181,70 @@ func TestOneOffScript(t *testing.T) { require.True(t, mktempMock.Check(t)) require.True(t, teleportMock.Check(t)) - require.Contains(t, string(out), "> ./bin/teleport version") + require.Contains(t, string(out), "teleport version") require.Contains(t, string(out), teleportVersionOutput) require.Contains(t, string(out), "Test was a success.") + + // Script should remove the temporary directory. + require.NoDirExists(t, testWorkingDir) + }) + + t.Run("command can be executed with extra arguments", func(t *testing.T) { + teleportHelpStart := "Use teleport start --config teleport.yaml" + // set up + testWorkingDir := t.TempDir() + require.NoError(t, os.Mkdir(testWorkingDir+"/bin/", 0o755)) + scriptLocation := testWorkingDir + "/" + scriptName + + teleportMock, err := bintest.NewMock(testWorkingDir + "/bin/teleport") + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, teleportMock.Close()) + }) + + teleportBinTarball, err := utils.CompressTarGzArchive([]string{"teleport/teleport"}, singleFileFS{file: teleportMock.Path}) + require.NoError(t, err) + + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + assert.Equal(t, "/teleport-v13.1.0-linux-amd64-bin.tar.gz", req.URL.Path) + http.ServeContent(w, req, "teleport-v13.1.0-linux-amd64-bin.tar.gz", time.Now(), bytes.NewReader(teleportBinTarball.Bytes())) + })) + t.Cleanup(func() { testServer.Close() }) + + script, err := BuildScript(OneOffScriptParams{ + BinUname: unameMock.Path, + BinMktemp: mktempMock.Path, + CDNBaseURL: testServer.URL, + EntrypointArgs: "help", + TeleportVersion: "v13.1.0", + SuccessMessage: "Test was a success.", + }) + require.NoError(t, err) + + unameMock.Expect("-s").AndWriteToStdout("Linux") + unameMock.Expect("-m").AndWriteToStdout("x86_64") + mktempMock.Expect("-d", "-p", homeDir).AndWriteToStdout(testWorkingDir) + teleportMock.Expect("help", "start").AndWriteToStdout(teleportHelpStart) + + err = os.WriteFile(scriptLocation, []byte(script), 0700) + require.NoError(t, err) + + // execute script + out, err := exec.Command("sh", scriptLocation, "start").CombinedOutput() + + // validate + require.NoError(t, err, string(out)) + + require.True(t, unameMock.Check(t)) + require.True(t, mktempMock.Check(t)) + require.True(t, teleportMock.Check(t)) + + require.Contains(t, string(out), "/bin/teleport help start") + require.Contains(t, string(out), teleportHelpStart) + require.Contains(t, string(out), "Test was a success.") + + // Script should remove the temporary directory. + require.NoDirExists(t, testWorkingDir) }) t.Run("invalid OS", func(t *testing.T) { @@ -122,13 +254,13 @@ func TestOneOffScript(t *testing.T) { unameMock.Expect("-s").AndWriteToStdout("Windows") unameMock.Expect("-m").AndWriteToStdout("x86_64") - mktempMock.Expect("-d").AndWriteToStdout(testWorkingDir) + mktempMock.Expect("-d", "-p", homeDir).AndWriteToStdout(testWorkingDir) err = os.WriteFile(scriptLocation, []byte(script), 0700) require.NoError(t, err) // execute script - out, err := exec.Command("bash", scriptLocation).CombinedOutput() + out, err := exec.Command("sh", scriptLocation).CombinedOutput() // validate require.Error(t, err, string(out)) @@ -142,13 +274,13 @@ func TestOneOffScript(t *testing.T) { unameMock.Expect("-s").AndWriteToStdout("Linux") unameMock.Expect("-m").AndWriteToStdout("apple-silicon") - mktempMock.Expect("-d").AndWriteToStdout(testWorkingDir) + mktempMock.Expect("-d", "-p", homeDir).AndWriteToStdout(testWorkingDir) err = os.WriteFile(scriptLocation, []byte(script), 0700) require.NoError(t, err) // execute script - out, err := exec.Command("bash", scriptLocation).CombinedOutput() + out, err := exec.Command("sh", scriptLocation).CombinedOutput() // validate require.Error(t, err, string(out)) @@ -161,13 +293,27 @@ func TestOneOffScript(t *testing.T) { BinMktemp: mktempMock.Path, CDNBaseURL: "dummyURL", TeleportVersion: "v13.1.0", - TeleportArgs: "version", + EntrypointArgs: "version", SuccessMessage: "Test was a success.", TeleportFlavor: "../not-teleport", }) require.True(t, trace.IsBadParameter(err), "expected BadParameter, got %+v", err) }) + t.Run("invalid command prefix should return an error", func(t *testing.T) { + _, err := BuildScript(OneOffScriptParams{ + BinUname: unameMock.Path, + BinMktemp: mktempMock.Path, + CDNBaseURL: "dummyURL", + TeleportVersion: "v13.1.0", + EntrypointArgs: "version", + SuccessMessage: "Test was a success.", + TeleportFlavor: "teleport", + TeleportCommandPrefix: "rm -rf thing", + }) + require.True(t, trace.IsBadParameter(err), "expected BadParameter, got %+v", err) + }) + t.Run("if enterprise build, it uses the enterprise package name", func(t *testing.T) { // set up testWorkingDir := t.TempDir() @@ -176,9 +322,9 @@ func TestOneOffScript(t *testing.T) { teleportMock, err := bintest.NewMock(testWorkingDir + "/bin/teleport") require.NoError(t, err) - defer func() { + t.Cleanup(func() { assert.NoError(t, teleportMock.Close()) - }() + }) modules.SetTestModules(t, &modules.TestModules{ TestBuildType: modules.BuildEnterprise, @@ -190,28 +336,28 @@ func TestOneOffScript(t *testing.T) { assert.Equal(t, "/teleport-ent-v13.1.0-linux-amd64-bin.tar.gz", req.URL.Path) http.ServeContent(w, req, "teleport-ent-v13.1.0-linux-amd64-bin.tar.gz", time.Now(), bytes.NewReader(teleportBinTarball.Bytes())) })) - defer func() { testServer.Close() }() + t.Cleanup(func() { testServer.Close() }) script, err := BuildScript(OneOffScriptParams{ BinUname: unameMock.Path, BinMktemp: mktempMock.Path, CDNBaseURL: testServer.URL, TeleportVersion: "v13.1.0", - TeleportArgs: "version", + EntrypointArgs: "version", SuccessMessage: "Test was a success.", }) require.NoError(t, err) unameMock.Expect("-s").AndWriteToStdout("Linux") unameMock.Expect("-m").AndWriteToStdout("x86_64") - mktempMock.Expect("-d").AndWriteToStdout(testWorkingDir) + mktempMock.Expect("-d", "-p", homeDir).AndWriteToStdout(testWorkingDir) teleportMock.Expect("version").AndWriteToStdout(teleportVersionOutput) err = os.WriteFile(scriptLocation, []byte(script), 0700) require.NoError(t, err) // execute script - out, err := exec.Command("bash", scriptLocation).CombinedOutput() + out, err := exec.Command("sh", scriptLocation).CombinedOutput() // validate require.NoError(t, err, string(out)) @@ -220,7 +366,7 @@ func TestOneOffScript(t *testing.T) { require.True(t, mktempMock.Check(t)) require.True(t, teleportMock.Check(t)) - require.Contains(t, string(out), "> ./bin/teleport version") + require.Contains(t, string(out), "/bin/teleport version") require.Contains(t, string(out), teleportVersionOutput) require.Contains(t, string(out), "Test was a success.") }) diff --git a/rfd/0184-agent-auto-updates.md b/rfd/0184-agent-auto-updates.md new file mode 100644 index 0000000000000..ae843482bb225 --- /dev/null +++ b/rfd/0184-agent-auto-updates.md @@ -0,0 +1,1969 @@ +--- +authors: Stephen Levine (stephen.levine@goteleport.com) & Hugo Hervieux (hugo.hervieux@goteleport.com) +state: draft +--- + +# RFD 0184 - Agent Automatic Updates + +## Required Approvers + +* Engineering: @russjones +* Product: @klizhentas || @xinding33 +* Security: Doyensec + +## What + +This RFD proposes a new mechanism for scheduled, automatic updates of Teleport agents. + +Users of Teleport will be able to use the tctl CLI to specify desired versions, update schedules, and a rollout strategy. + +Agents will be updated by a new `teleport-update` binary, built from `tools/teleport-update` in the Teleport repository. + +All agent installations are in-scope for this proposal, including agents installed on Linux servers and Kubernetes. + +The following anti-goals are out-of-scope for this proposal, but will be addressed in future RFDs: +- Signing of agent artifacts (e.g., via TUF) +- Teleport Cloud APIs for updating agents +- Improvements to the local functionality of the Kubernetes agent for better compatibility with FluxCD and ArgoCD +- Support for progressive rollouts of tbot, when not installed on the same system as a Teleport agent + +This RFD proposes a specific implementation of several sections in https://github.com/gravitational/teleport/pull/39217. + +Additionally, this RFD parallels the auto-update functionality for client tools proposed in https://github.com/gravitational/teleport/pull/39805. + +## Why + +1. We want customers to run the latest release of Teleport so that they are secure and have access to the latest + features. +2. We do not want customers to deal with the pain of updating agents installed on their own infrastructure. +3. We want to reduce the operational cost of customers running old agents. + For Cloud customers, this will allow us to support fewer simultaneous cluster versions and reduce support load. + For self-hosted customers, this will reduce support load associated with debugging old versions of Teleport. +4. Providing 99.99% availability for customers requires us to maintain high availability at the agent-level + as well as the cluster-level. + +The current systemd updater does not meet those requirements: +- The use of package managers (apt and yum) to apply updates leads users to accidentally upgrade Teleport. +- The installation process is complex, and users often end up installing the wrong version of Teleport. +- The update process does not provide sufficient safeties to protect against broken agent updates. +- Customers decline to adopt the existing updater because they want more control over when updates occur. +- We do not offer a nice user experience for self-hosted users. This results in a marginal automatic updates + adoption and does not reduce the support cost associated with upgrading self-hosted clusters. + +## How + +The new agent automatic updates system will rely on a separate `teleport-update` binary controlling which Teleport version is +installed. Automatic updates will be implemented incrementally: + +- Phase 1: Introduce a new, self-updating updater binary which does not rely on package managers. Allow tctl to roll out updates to all agents. +- Phase 2: Add the ability for the agent updater to immediately revert a faulty update. +- Phase 3: Introduce the concept of agent update groups and make users chose the order in which groups are updated. +- Phase 4: Add a feedback mechanism for the Teleport inventory to track the agents of each group and their update status. +- Phase 5: Add the canary deployment strategy: a few agents are updated first, if they don't die, the whole group is updated. +- Phase 6: Add the ability to perform slow and incremental version rollouts within an agent update group. +- Phase 7: If needed, backup local agent DB and restore during agent rollbacks. + +The updater will be usable after phase 1 and will gain new capabilities after each phase. +After phase 2, the new updater will have feature-parity with the existing updater script. +The existing auto-updates mechanism will remain unchanged and fully-functional throughout the process. +It will be deprecated in the future. + +Future phases might change as we are working on the implementation and collecting real-world feedback and experience. + +We will introduce two user-facing resources: + +1. The `autoupdate_config` resource, owned by the Teleport user. This resource allows Teleport users to configure: + - Whether automatic updates are enabled, disabled, or temporarily suspended + - The order in which agents should be updated (`dev` before `staging` before `prod`) + - Days and hours when agent updates should start + - Configuration for client auto-updates (e.g., `tsh` and `tctl`), which are out-of-scope for this RFD + + The resource will look like: + ```yaml + kind: autoupdate_config + spec: + # existing field, deprecated + tools_autoupdate: true/false + # new fields + tools: + mode: enabled/disabled + agents: + mode: enabled/disabled/suspended + schedules: + regular: + - name: dev + days: ["Mon", "Tue", "Wed", "Thu"] + start_hour: 0 + alert_after: 4h + canary_count: 5 # added in phase 5 + max_in_flight: 20% # added in phase 6 + - name: prod + days: ["Mon", "Tue", "Wed", "Thu"] + start_hour: 0 + wait_days: 1 # update this group at least 1 day after the previous one + alert_after: 4h + canary_count: 5 # added in phase 5 + max_in_flight: 20% # added in phase 6 + ``` + +2. The `autoupdate_version` resource, with `spec` owned by the Teleport cluster administrator (e.g. Teleport Cloud operators). + ```yaml + kind: autoupdate_version + spec: + tools: + target_version: vX + agents: + start_version: v1 + target_version: v2 + schedule: regular + strategy: halt-on-failure + mode: enabled + ``` + +We will also introduce an internal resource, tracking the agent rollout status. This resource is +owned by Teleport. Users and cluster operators can read its content but cannot create/update/upsert/delete it. +This resource is editable via select RPCs (e.g. start or rollback a group). + +The system will look like: + +```mermaid +flowchart TD + user(fa:fa-user User) + operator(fa:fa-user Operator) + auth[Auth Service] + proxy[Proxy Service] + updater[teleport-updater] + agent[Teleport Agent] + + autoupdate_config@{shape: notch-rect} + autoupdate_version@{shape: notch-rect} + autoupdate_rollout@{shape: notch-rect} + updater_status@{shape: notch-rect, label: "updater.yaml"} + + user -->|defines update schedule| autoupdate_config + operator -->|choses target version| autoupdate_version + autoupdate_config --> auth + autoupdate_version --> auth + auth -->|Describes desired state for each agent group| autoupdate_rollout + autoupdate_rollout --> proxy + proxy -->|Serves update instructions
via /find| updater + updater -->|Writes status| updater_status + updater_status --> agent + agent -->|Reports version and status via HelloMessage and InstanceHeartbeat| auth +``` + +You can find more details about each resource field [in the dedicated resource section](#teleport-resources). + +## Details + +This section contains the proposed implementation details and is mainly relevant for Teleport developers and curious +users who want to know the motivations behind this specific design. + +### Product requirements + +The following product requirements were defined by our leadership team: + +1. Phased rollout for Cloud tenants. We should be able to control the agent version per-tenant. + +2. Bucketed rollout that customers have control over. + - Control the bucket update day + - Control the bucket update hour + - Ability to pause a rollout + +3. Customers should be able to run "apt-get upgrade" without updating Teleport. + + Installation from a package manager should be possible, but the version should still be controlled by Teleport. + +4. Self-managed updates should be a first class citizen. Teleport must advertise the desired agent and client version. + +5. Self-hosted customers should be supported, for example, customers whose own internal customer is running a Teleport agent. + +6. Upgrading leaf clusters is out-of-scope. + +7. Rolling back after a broken update should be supported. Roll forward gets you 99.9%, we need rollback for 99.99%. + +8. We should have high quality metrics that report the version they are running and if they are running automatic + updates. For both users and us. + +9. Best effort should be made so automatic updates should be applied in a way that sessions are not terminated. (Currently only supported for SSH) + +10. All backends should be supported. + +11. Teleport Discover installation (curl one-liner) should be supported. + +12. We need to support Docker image repository mirrors and Teleport artifact mirrors. + +13. I should be able to install an auto-updating deployment of Teleport via whatever mechanism I want to, including OS packages such as apt and yum. + +14. If new instances join a bucket outside the upgrade window, and you are within your compatibility window, wait until your next group update start. + If you are not within your compatibility window, attempt to upgrade right away. + +15. If an agent comes back online after some period of time, and it is still compatible with + control plane, it should wait until the next upgrade window to be upgraded. + +16. Regular agent updates for Cloud tenants should complete in less than a week. + (Select tenants may support longer schedules, at the Cloud team's discretion.) + +17. A Cloud customer should be able to pause, resume, and rollback and existing rollout schedule. + A Cloud customer should not be able to create new rollout schedules. + + Teleport can create as many rollout schedules as it wants. + +18. A user logged-in to the agent host should be able to disable agent auto-updates and pin a version for that particular host. + +### User Stories + +#### As a Teleport Cloud operator I want to be able to update customers agents to a newer Teleport version + +
+Before + +```shell +tctl autoupdate agent status +# Rollout plan created the YYYY-MM-DD +# Previous version: v1 +# New version: v2 +# Status: enabled +# +# Group Name Status Update Start Time Connected Agents Up-to-date agents failed updates +# ---------- ----------------- ----------------- ---------------- ----------------- -------------- +# dev complete YYYY-MM-DD HHh 120 115 2 +# staging complete YYYY-MM-D2 HHh 20 20 0 +# prod not started 234 0 0 +``` +
+ +I run +```bash +tctl autoupdate agent-plan new-target v3 +# created new rollout from v2 to v3 +``` + +
+After + +```shell +tctl autoupdate agent status +# Rollout plan created the YYYY-MM-DD +# Previous version: v2 +# New version: v3 +# Status: enabled +# +# Group Name Status Update Start Time Connected Agents Up-to-date agents failed updates +# ---------- ----------------- ----------------- ---------------- ----------------- -------------- +# dev not started 120 0 0 +# staging not started 20 0 0 +# prod not started 234 0 0 +``` + +
+ +Now, new agents will install v2 by default, and v3 after the maintenance. + +> [!NOTE] +> If the previous maintenance was not finished, I will install v2 on new prod agents while the rest of prod is still running v1. +> This is expected as we don't want to keep track of an infinite number of versions. +> +> If this is an issue I can create a v1 -> v3 rollout instead. +> +> ```bash +> tctl autoupdate agent-plan new-target v3 --previous-version v1 +> # created new update plan from v1 to v3 +> ``` + +#### As a Teleport Cloud operator I want to minimize damage caused by broken versions to ensure we maintain 99.99% availability + +##### Failure mode 1(a): the new version crashes + +I create a new deployment with a broken version. The version is deployed to a few instances picked randomly. +Those instances are called the canaries. As the new version has an issue, one or many of those canary instances can't run the +new version and their updater has to revert to the previous one. The agents connect back online and +advertise they have failed to update. The maintenance is stuck until every instance that got selected to test the new version +is back online, and running the new version. + +
+Autoupdate agent rollout + +```yaml +kind: autoupdate_agent_rollout +spec: + version_config: + start_version: v1 + target_version: v2 + schedule: regular + strategy: halt-on-failure + mode: enabled +status: + groups: + - name: dev + start_time: 2020-12-09T16:09:53+00:00 + initial_count: 100 + present_count: 100 + failed_count: 0 + progress: 0 + state: canaries + canaries: + - updater_uuid: abc + host_uuid: def + hostname: foo.example.com + success: false + last_update_time: 2020-12-10T16:09:53+00:00 + last_update_reason: canaryTesting + - name: staging + start_time: 0000-00-00 + initial_count: 0 + present_count: 0 + failed_count: 0 + progress: 0 + state: unstarted + last_update_time: 2020-12-10T16:09:53+00:00 + last_update_reason: newAgentPlan +``` +
+ +I and the customer get an alert if the test instances are not running the expected version after an hour. +Teleport cloud operators and the customer can look up the hostname and host UUID of the test instances +to identify which one(s) failed to update and go troubleshoot. + +Customers receive cluster alerts, while Cloud receives alerts driven by Teleport metrics. + +The rollout resumes. + +If the issue is related to a specific instance and not the new Teleport version (e.g. VM out of disk space), +the user can instruct teleport to pick 5 new canary instances. + +##### Failure mode 1(b): the new version crashes, but not on the canaries + +This scenario is the same as the previous one but the Teleport agent bug only manifests on select agents. +For example: [the agent fails to read cloud-provider specific metadata and crashes](https://github.com/gravitational/teleport/issues/42312). +This can also be caused by a specific Teleport service crashing. For example, the discovery service is crashing but +all other services are OK. As most instances are running ssh_service, the discovery_service instances are less likely +to get picked. + +The version is deployed to a few instances picked randomly but none of them runs on the affected cloud provider. +The canary instances can update properly and the update is sent to every instance of the group. + +All agents are updated, and all agents hosted on the cloud provider affected by the bug crash. +The updaters of the affected agents will attempt to self-heal by reverting to the previous version. + +Once the previous Teleport version is running, the agents from the affected cloud platform will advertise the update +failed, and they had to rollback. + +If too many agents failed, this will block the group from transitioning from `active` to `done`, protecting the future +groups from the faulty updates. + +##### Failure mode 2(a): the new version crashes, and the old version cannot start + +I create a new deployment with a broken version. The version is deployed to a few instances picked randomly. +Those instances are called the canaries. As the new version has an issue, one or many of those canary instances can't +run the new version. Their updater also fails to revert to the previous version. + +The group update is stuck until the canary comes back online and runs the latest version. + +The customer and Teleport cloud receive an alert. Both customer and Teleport cloud can retrieve the +host id and hostname of the faulty canary instances. With this information they can go troubleshoot the failed agents. + +##### Failure mode 2(b): the new version crashes, and the old version cannot start, but not on the canaries + +This scenario is the same as the previous one but the Teleport agent bug only manifests on select agents. +For example: a clock drift blocks agents from re-connecting to Teleport. + +The canaries might not select one of the affected agent and allow the update to proceed. +All agents are updated, and all agents hosted on the cloud provider affected by the bug crash. +The updater fails to self-heal as the old version does not start anymore. + +If too many agents fail, this will block the group from transitioning from `active` to `done`, protecting the future +groups from the faulty updates. + +In this case, it's hard to identify which agent dropped. + +##### Failure mode 3: shadow failure + +Teleport cloud deploys a new version. Agents from the first group get updated. +The agents are seemingly running properly, but some functions are impaired. +For example, host user creation is failing. + +Some user tries to access a resource served by the agent, it fails and the user +notices the disruption. + +The customer can observe the agent update status and see that a recent update +might have caused this: + +```shell +tctl autoupdate agent status +# Rollout plan created the YYYY-MM-DD +# Previous version: v2 +# New version: v3 +# Status: enabled +# +# Group Name Status Update Start Time Connected Agents Up-to-date agents failed updates +# ---------- ----------------- ----------------- ---------------- ----------------- -------------- +# dev complete YYYY-MM-DD HHh 120 115 2 +# staging in progress (53%) YYYY-MM-D2 HHh 20 10 0 +# prod not started 234 0 0 +``` + +Then, the customer or Teleport Cloud team can suspend the rollout: + +```shell +tctl autoupdate agent suspend +# Automatic updates suspended +# No existing agent will get updated. New agents might install the new version +# depending on their group. +``` + +At this point, no new agent is updated to reduce the service disruption. +The customer can investigate, and get help from Teleport's support via a support ticket. +If the update is really the cause of the issue, the customer or Teleport cloud can perform a rollback: + +```shell +tctl autoupdate agent rollback +# Rolledback groups: [dev, staging] +# Warning: the automatic agent updates are suspended. +# Agents will not rollback until you run: +# $> tctl autoupdate agent resume +``` + +> [!NOTE] +> By default, all groups not in the "unstarted" state are rolledback. +> It is also possible to rollback only specific groups. + +
+After: + +```shell +tctl autoupdate agent status +# Rollout plan created the YYYY-MM-DD +# Previous version: v2 +# New version: v3 +# Status: suspended +# +# Group Name Status Update Start Time Connected Agents Up-to-date agents failed updates +# ---------- ----------------- ----------------- ---------------- ----------------- -------------- +# dev rolledback YYYY-MM-DD HHh 120 115 2 +# staging rolledback YYYY-MM-D2 HHh 20 10 0 +# prod not started 234 0 0 +``` +
+ +Finally, when the user is happy with the new plan, they can resume the updates. +This will trigger the rollback. + +```shell +tctl autoupdate agent resume +``` + +#### As a Teleport user and a Teleport on-call responder, I want to be able to pin a specific Teleport version of an agent to understand if a specific behaviour is caused by a specific Teleport version + +I connect to the server and lookup its status: +```shell +teleport-update status +# Running version v16.2.5 +# Automatic updates enabled. +# Proxy: example.teleport.sh +# Group: staging +``` + +I try to set a specific version: +```shell +teleport-update use-version v16.2.3 +# Error: the instance is enrolled into automatic updates. +# You must specify --disable-automatic-updates to opt this agent out of automatic updates and manually control the version. +``` + +I acknowledge that I am leaving automatic updates: +```shell +teleport-update use-version v16.2.3 --disable-automatic-updates +# Disabling automatic updates. You can re-enable them by running `teleport-update enable` +# Downloading version 16.2.3 +# Restarting teleport +# Cleaning up old binaries +``` + +When the issue is fixed, I can enroll back into automatic updates: + +```shell +teleport-update enable +# Enabling automatic updates +# Proxy: example.teleport.sh +# Group: staging +``` + +#### As a Teleport user I want to fast-track a group update + +I have a new rollout, completely unstarted, and my current maintenance schedule updates over several days. +However, the new version contains something that I need as soon as possible (e.g., a fix for a bug that affects me). + +
+Before: + +```shell +tctl autoupdate agent status +# Rollout plan created the YYYY-MM-DD +# Previous version: v2 +# New version: v3 +# Status: enabled +# +# Group Name Status Update Start Time Connected Agents Up-to-date agents failed updates +# ---------- ----------------- ----------------- ---------------- ----------------- -------------- +# dev not started 120 0 0 +# staging not started 20 0 0 +# prod not started 234 0 0 +``` +
+ +I can trigger the dev group immediately using the command: + +```shell +tctl autoupdate agent start-update dev [--force] +# Dev group update triggered. +``` + +The `--force` flag allows the user to skip progressive deployment mechanism such as canaries or backpressure. + +Alternatively +```shell +tctl autoupdate agent mark-done dev +``` + +
+After: + +```shell +tctl autoupdate agent status +# Rollout plan created the YYYY-MM-DD +# Previous version: v2 +# New version: v3 +# Status: enabled +# +# Group Name Status Update Start Time Connected Agents Up-to-date agents failed updates +# ---------- ----------------- ----------------- ---------------- ----------------- -------------- +# dev not started 120 0 0 +# staging not started 20 0 0 +# prod not started 234 0 0 +``` +
+ +#### As a Teleport user, I want to install a new agent automatically updated + +The manual way: + +```bash +wget https://cdn.teleport.dev/teleport-update-- +chmod +x teleport-update +./teleport-update enable --proxy example.teleport.sh --group production +# Detecting the Teleport version and edition used by cluster "example.teleport.sh" +# Installing the following teleport version: +# Version: 16.2.1 +# Edition: Enterprise +# OS: Linux +# Architecture: x86 +# Teleport installed +# Enabling automatic updates, the agent is part of the "production" update group. +# You can now configure the teleport agent with `teleport configure` or by writing your own `teleport.yaml`. +# When the configuration is done, enable and start teleport by running: +# `systemctl start teleport && systemctl enable teleport` +``` + +The one-liner: + +``` +curl https://cdn.teleport.dev/auto-install | bash -s example.teleport.sh +# Downloading the teleport updater +# Detecting the Teleport version and edition used by cluster "example.teleport.sh" +# Installing the following teleport version: +# Version: 16.2.1 +# Edition: Enterprise +# OS: Linux +# Architecture: x86 +# Teleport installed +# Enabling automatic updates, the agent is part of the "default" update group. +# You can now configure the teleport agent with `teleport configure` or by writing your own `teleport.yaml`. +# When the configuration is finished, enable and start teleport by running: +# `systemctl start teleport && systemctl enable teleport` +``` + +I can also install teleport using the package manager, then enroll the agent into AUs. See the section below: + +#### As a Teleport user I want to enroll my existing agent into AUs + +I have an agent, installed from a package manager or by manually unpacking the tarball. +This agent might or might not be enrolled in the previous automatic update mechanism (apt/yum-based). +I have the teleport updater installed and available in my path. +I run: + +```shell +teleport-update enable --group production +# Detecting the Teleport version and edition used by cluster "example.teleport.sh" +# Installing the following teleport version: +# Version: 16.2.1 +# Edition: Enterprise +# OS: Linux +# Architecture: x86 +# Teleport installed, reloading the service. +# Enabling automatic updates, the agent is part of the "production" update group. +``` + +> [!NOTE] +> The updater saw the teleport unit running and the existing teleport configuration. +> It used the configuration to pick the right proxy address. As teleport is already running, the teleport service is +> reloaded to use the new binary. + +If the agent was previously enrolled into AUs with the old teleport updater package, the `enable` command will also +remove the old package. + +### Teleport Resources + +#### Autoupdate Config + +This resource is owned by the Teleport cluster user. +This is how Teleport customers can specify their automatic update preferences. + +```yaml +kind: autoupdate_config +spec: + # existing field, deprecated + tools_autoupdate: true + tools: + mode: enabled/disabled + agents: + # agent_auto_update allows turning agent updates on or off at the + # cluster level. Only turn agent automatic updates off if self-managed + # agent updates are in place. Setting this to pause will temporarily halt the rollout. + mode: enabled/disabled/suspended + # strategy to use for the rollout + # Supported values are: + # - time-based + # - halt-on-failure + # - halt-on-failure-with-backpressure + # defaults to halt-on-failure, might default to halt-on-failure-with-backpressure after phase 6. + strategy: halt-on-failure + # agent_schedules specifies version rollout schedules for agents. + # The schedule used is determined by the schedule associated + # with the version in the autoupdate_version resource. + # For now, only the "regular" schedule is configurable. + schedules: + regular: + # name of the group. Must only contain valid backend / resource name characters. + - name: staging + # days specifies the days of the week when the group may be updated. + # mandatory value for most Cloud customers: ["Mon", "Tue", "Wed", "Thu"] + # default: ["*"] (all days) + days: [ “Sun”, “Mon”, ... | "*" ] + # start_hour specifies the hour when the group may start upgrading. + # default: 0 + start_hour: 0-23 + # wait_days specifies how many days to wait after the previous group finished before starting. + # This must be 0 when using the `time-based` strategy. + # default: 0 + wait_days: 0-1 + # canary_count specifies the desired number of canaries to update before any other agents + # are updated. + # default: 5 + canary_count: 0-10 + # max_in_flight specifies the maximum number of agents that may be updated at the same time. + # Only valid for the backpressure strategy. + # default: 20% + max_in_flight: 10-100% + # alert_after specifies the duration after which a cluster alert will be set if the group update has + # not completed. + # default: 4 + alert_after_hours: 1-8 + # ... +``` + +Default resource: +```yaml +kind: autoupdate_config +spec: + tools: + mode: enabled + agents: + mode: enabled + strategy: halt-on-failure + alert_after: 4h + schedules: + regular: + - name: default + days: ["Mon", "Tue", "Wed", "Thu"] + start_hour: 0 + canary_count: 5 + max_in_flight: 20% +``` + +#### Autoupdate version + +The `autoupdate_version` spec is owned by the Teleport cluster administrator. +In Teleport Cloud, this is the Cloud operations team. For self-hosted setups this is the user with access to the local +admin socket (tctl on local machine). + +> [!NOTE] +> This is currently an anti-pattern as we are trying to remove the use of the local administrator in Teleport. +> However, Teleport does not provide any role/permission that we can use for Teleport Cloud operations and cannot be +> granted to users. To part with local admin rights, we need a way to have Cloud or admin-only operations. +> This would also improve Cloud team operations by interacting with Teleport API rather than executing local tctl. +> +> Solving this problem is out of the scope of this RFD. + +```yaml +kind: autoupdate_version +spec: + tools: + target_version: vX + agents: + # start_version is the desired version for agents before their window. + start_version: v1 + # target_version is the desired version for agents after their window. + target_version: v2 + # schedule to use for the rollout + schedule: regular + # paused specifies whether the rollout is paused + # default: enabled + mode: enabled|disabled|suspended +``` + +#### Autoupdate agent rollout + +The `autoupdate_agent_rollout` resource is owned by Teleport. This resource can be read by users but not directly applied. +To create and reconcile this resource, the Auth service looks up bot `autoupdate_config` and `autoupdate_version` to know the desired mode, versions, and schedule. +Once the agent rollout is created, the auth uses its status to track the progress of the rollout through the different groups. + +```yaml +kind: autoupdate_agent_rollout +spec: + # content copied from the `autoupdate_version.spec.agents` + version_config: + start_version: v1 + target_version: v2 + schedule: regular + strategy: halt-on-failure + mode: enabled +status: + groups: + # name of group + - name: staging + # start_time is the time the upgrade will start + start_time: 2020-12-09T16:09:53+00:00 + # initial_count is the number of connected agents at the start of the window + initial_count: 432 + # missing_count is the number of agents disconnected since the start of the rollout + present_count: 53 + # failed_count is the number of agents rolled-back since the start of the rollout + failed_count: 23 + # canaries is a list of agents used for canary deployments + canaries: # part of phase 5 + # updater_uuid is the updater UUID + - updater_uuid: abc123-... + # host_uuid is the agent host UUID + host_uuid: def534-... + # hostname of the agent + hostname: foo.example.com + # success status + success: false + # progress is the current progress through the rollout + progress: 0.532 + # state is the current state of the rollout (unstarted, active, done, rollback) + state: active + # last_update_time is the time of the previous update for the group + last_update_time: 2020-12-09T16:09:53+00:00 + # last_update_reason is the trigger for the last update + last_update_reason: rollback +``` + +#### Protobuf + +```protobuf +syntax = "proto3"; + +package teleport.autoupdate.v1; + +import "teleport/header/v1/metadata.proto"; +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1;autoupdate"; + +// CONFIG + +// AutoUpdateConfig is a config singleton used to configure cluster +// autoupdate settings. +message AutoUpdateConfig { + string kind = 1; + string sub_kind = 2; + string version = 3; + teleport.header.v1.Metadata metadata = 4; + + AutoUpdateConfigSpec spec = 5; +} + +// AutoUpdateConfigSpec encodes the parameters of the autoupdate config object. +message AutoUpdateConfigSpec { + reserved 1; + AutoUpdateConfigSpecTools tools = 2; + AutoUpdateConfigSpecAgents agents = 3; +} + +// AutoUpdateConfigSpecTools encodes the parameters of automatic tools update. +message AutoUpdateConfigSpecTools { + // Mode encodes the feature flag to enable/disable tools autoupdates. + Mode mode = 1; +} + +// AutoUpdateConfigSpecTools encodes the parameters of automatic tools update. +message AutoUpdateConfigSpecAgents { + // mode specifies whether agent autoupdates are enabled, disabled, or paused. + Mode agent_auto_update_mode = 1; + // strategy to use for updating the agents. + Strategy strategy = 2; + // maintenance_window_minutes is the maintenance window duration in minutes. This can only be set if `strategy` is "time-based". + int64 maintenance_window_minutes = 3; + // alert_after_hours specifies the number of hours to wait before alerting that the rollout is not complete. + // This can only be set if `strategy` is "halt-on-failure". + int64 alert_after_hours = 5; + // agent_schedules specifies schedules for updates of grouped agents. + AgentAutoUpdateSchedules agent_schedules = 6; +} + +// Strategy type for the rollout +enum Strategy { + // UNSPECIFIED update strategy + STRATEGY_UNSPECIFIED = 0; + // PREVIOUS_MUST_SUCCEED update strategy with no backpressure + STRATEGY_HALT_ON_FAILURE = 1; + // TIME_BASED update strategy. + STRATEGY_TIME_BASED = 2; +} + +// AgentAutoUpdateSchedules specifies update scheduled for grouped agents. +message AgentAutoUpdateSchedules { + // regular schedules for non-critical versions. + repeated AgentAutoUpdateGroup regular = 1; +} + +// AgentAutoUpdateGroup specifies the update schedule for a group of agents. +message AgentAutoUpdateGroup { + // name of the group + string name = 1; + // days to run update + repeated Day days = 2; + // start_hour to initiate update + int32 start_hour = 3; + // wait_days after last group succeeds before this group can run. This can only be used when the strategy is "halt-on-failure". + int64 wait_days = 4; + // canary_count of agents to use in the canary deployment. + int64 canary_count = 5; + // max_in_flight specifies agents that can be updated at the same time, by percent. + string max_in_flight = 6; +} + +// Day of the week +enum Day { + DAY_UNSPECIFIED = 0; + DAY_ALL = 1; + DAY_SUNDAY = 2; + DAY_MONDAY = 3; + DAY_TUESDAY = 4; + DAY_WEDNESDAY = 5; + DAY_THURSDAY = 6; + DAY_FRIDAY = 7; + DAY_SATURDAY = 8; +} + +// Mode of operation +enum Mode { + // UNSPECIFIED update mode + MODE_UNSPECIFIED = 0; + // DISABLE updates + MODE_DISABLE = 1; + // ENABLE updates + MODE_ENABLE = 2; + // PAUSE updates + MODE_PAUSE = 3; +} + +// Schedule type for the rollout +enum Schedule { + // UNSPECIFIED update schedule + SCHEDULE_UNSPECIFIED = 0; + // REGULAR update schedule + SCHEDULE_REGULAR = 1; + // IMMEDIATE update schedule for updating all agents immediately + SCHEDULE_IMMEDIATE = 2; +} + +// VERSION + +// AutoUpdateVersion is a resource singleton with version required for +// tools autoupdate. +message AutoUpdateVersion { + string kind = 1; + string sub_kind = 2; + string version = 3; + teleport.header.v1.Metadata metadata = 4; + + AutoUpdateVersionSpec spec = 5; +} + +// AutoUpdateVersionSpec encodes the parameters of the autoupdate versions. +message AutoUpdateVersionSpec { + // ToolsVersion is the semantic version required for tools autoupdates. + reserved 1; + AutoUpdateVersionSpecTools tools = 2; + AutoUpdateVersionSpecAgents agents = 3; +} + +// AutoUpdateVersionSpecTools is the spec for the autoupdate version. +message AutoUpdateVersionSpecTools { + // target_version is the target tools version. + string target_version = 1; +} + +// AutoUpdateVersionSpecAgents is the spec for the autoupdate version. +message AutoUpdateVersionSpecAgents { + // start_version is the version to update from. + string start_version = 1; + // target_version is the version to update to. + string target_version = 2; + // schedule to use for the rollout + Schedule schedule = 3; + // autoupdate_mode to use for the rollout + Mode autoupdate_mode = 4; +} + +// AGENT ROLLOUT + +message AutoUpdateAgentRollout { + string kind = 1; + string sub_kind = 2; + string version = 3; + teleport.header.v1.Metadata metadata = 4; + AutoUpdateAgentRolloutSpec spec = 5; + AutoUpdateAgentRolloutStatus status = 6; +} + +message AutoUpdateAgentRolloutSpec { + // start_version is the version to update from. + string start_version = 1; + // target_version is the version to update to. + string target_version = 2; + // schedule to use for the rollout + Schedule schedule = 3; + // autoupdate_mode to use for the rollout + Mode autoupdate_mode = 4; + // strategy to use for updating the agents. + Strategy strategy = 5; +} + +message AutoUpdateAgentRolloutStatus { + repeated AutoUpdateAgentRolloutStatusGroup groups = 1; +} + +message AutoUpdateAgentRolloutStatusGroup { + // name of the group + string name = 1; + // start_time of the rollout + google.protobuf.Timestamp start_time = 2; + // initial_count is the number of connected agents at the start of the window. + int64 initial_count = 3; + // present_count is the current number of connected agents. + int64 present_count = 4; + // failed_count specifies the number of failed agents. + int64 failed_count = 5; + // canaries is a list of canary agents. + repeated Canary canaries = 6; + // progress is the current progress through the rollout. + float progress = 7; + // state is the current state of the rollout. + State state = 8; + // last_update_time is the time of the previous update for this group. + google.protobuf.Timestamp last_update_time = 9; + // last_update_reason is the trigger for the last update + string last_update_reason = 10; +} + +// Canary agent +message Canary { + // update_uuid of the canary agent + string update_uuid = 1; + // host_uuid of the canary agent + string host_uuid = 2; + // hostname of the canary agent + string hostname = 3; + // success state of the canary agent + bool success = 4; +} + +// State of the rollout +enum State { + // UNSPECIFIED state + STATE_UNSPECIFIED = 0; + // UNSTARTED state + STATE_UNSTARTED = 1; + // CANARY state + STATE_CANARY = 2; + // ACTIVE state + STATE_ACTIVE = 3; + // DONE state + STATE_DONE = 4; + // ROLLEDBACK state + STATE_ROLLEDBACK = 5; +} + +// AutoUpdateService provides an API to manage autoupdates. +service AutoUpdateService { + // GetAutoUpdateConfig gets the current autoupdate config singleton. + rpc GetAutoUpdateConfig(GetAutoUpdateConfigRequest) returns (AutoUpdateConfig); + + // CreateAutoUpdateConfig creates a new AutoUpdateConfig. + rpc CreateAutoUpdateConfig(CreateAutoUpdateConfigRequest) returns (AutoUpdateConfig); + + // CreateAutoUpdateConfig updates AutoUpdateConfig singleton. + rpc UpdateAutoUpdateConfig(UpdateAutoUpdateConfigRequest) returns (AutoUpdateConfig); + + // UpsertAutoUpdateConfig creates a new AutoUpdateConfig or replaces an existing AutoUpdateConfig. + rpc UpsertAutoUpdateConfig(UpsertAutoUpdateConfigRequest) returns (AutoUpdateConfig); + + // DeleteAutoUpdateConfig hard deletes the specified AutoUpdateConfig. + rpc DeleteAutoUpdateConfig(DeleteAutoUpdateConfigRequest) returns (google.protobuf.Empty); + + // GetAutoUpdateVersion gets the current autoupdate version singleton. + rpc GetAutoUpdateVersion(GetAutoUpdateVersionRequest) returns (AutoUpdateVersion); + + // CreateAutoUpdateVersion creates a new AutoUpdateVersion. + rpc CreateAutoUpdateVersion(CreateAutoUpdateVersionRequest) returns (AutoUpdateVersion); + + // UpdateAutoUpdateVersion updates AutoUpdateVersion singleton. + rpc UpdateAutoUpdateVersion(UpdateAutoUpdateVersionRequest) returns (AutoUpdateVersion); + + // UpsertAutoUpdateVersion creates a new AutoUpdateVersion or replaces an existing AutoUpdateVersion. + rpc UpsertAutoUpdateVersion(UpsertAutoUpdateVersionRequest) returns (AutoUpdateVersion); + + // DeleteAutoUpdateVersion hard deletes the specified AutoUpdateVersionRequest. + rpc DeleteAutoUpdateVersion(DeleteAutoUpdateVersionRequest) returns (google.protobuf.Empty); + + // GetAutoUpdateAgentRollout gets the current autoupdate version singleton. + rpc GetAutoUpdateAgentRollout(GetAutoUpdateAgentRolloutRequest) returns (AutoUpdateAgentRollout); + + // CreateAutoUpdateAgentRollout creates a new AutoUpdateAgentRollout. + rpc CreateAutoUpdateAgentRollout(CreateAutoUpdateAgentRolloutRequest) returns (AutoUpdateAgentRollout); + + // UpdateAutoUpdateAgentRollout updates AutoUpdateAgentRollout singleton. + rpc UpdateAutoUpdateAgentRollout(UpdateAutoUpdateAgentRolloutRequest) returns (AutoUpdateAgentRollout); + + // UpsertAutoUpdateAgentRollout creates a new AutoUpdateAgentRollout or replaces an existing AutoUpdateAgentRollout. + rpc UpsertAutoUpdateAgentRollout(UpsertAutoUpdateAgentRolloutRequest) returns (AutoUpdateAgentRollout); + + // DeleteAutoUpdateAgentRollout hard deletes the specified AutoUpdateAgentRolloutRequest. + rpc DeleteAutoUpdateAgentRollout(DeleteAutoUpdateAgentRolloutRequest) returns (google.protobuf.Empty); + + // TriggerAgentGroup changes the state of an agent group from `unstarted` to `active` or `canary`. + rpc TriggerAgentGroup(TriggerAgentGroupRequest) returns (AutoUpdateAgentRollout); + // ForceAgentGroup changes the state of an agent group from `unstarted`, `canary`, or `active` to the `done` state. + rpc ForceAgentGroup(ForceAgentGroupRequest) returns (AutoUpdateAgentRollout); + // ResetAgentGroup resets the state of an agent group. + // For `canary`, this means new canaries are picked + // For `active`, this means the initial instance count is computed again. + rpc ResetAgentGroup(ResetAgentGroupRequest) returns (AutoUpdateAgentRollout); + // RollbackAgentGroup changes the state of an agent group to `rolledback`. + rpc RollbackAgentGroup(RollbackAgentGroupRequest) returns (AutoUpdateAgentRollout); +} + +// Request for GetAutoUpdateConfig. +message GetAutoUpdateConfigRequest {} + +// Request for CreateAutoUpdateConfig. +message CreateAutoUpdateConfigRequest { + AutoUpdateConfig config = 1; +} + +// Request for UpdateAutoUpdateConfig. +message UpdateAutoUpdateConfigRequest { + AutoUpdateConfig config = 1; +} + +// Request for UpsertAutoUpdateConfig. +message UpsertAutoUpdateConfigRequest { + AutoUpdateConfig config = 1; +} + +// Request for DeleteAutoUpdateConfig. +message DeleteAutoUpdateConfigRequest {} + +// Request for GetAutoUpdateVersion. +message GetAutoUpdateVersionRequest {} + +// Request for CreateAutoUpdateVersion. +message CreateAutoUpdateVersionRequest { + AutoUpdateVersion version = 1; +} + +// Request for UpdateAutoUpdateConfig. +message UpdateAutoUpdateVersionRequest { + AutoUpdateVersion version = 1; +} + +// Request for UpsertAutoUpdateVersion. +message UpsertAutoUpdateVersionRequest { + AutoUpdateVersion version = 1; +} + +// Request for DeleteAutoUpdateVersion. +message DeleteAutoUpdateVersionRequest {} + +// Request for GetAutoUpdateAgentRollout. +message GetAutoUpdateAgentRolloutRequest {} + +// Request for CreateAutoUpdateAgentRollout. +message CreateAutoUpdateAgentRolloutRequest { + AutoUpdateAgentRollout plan = 1; +} + +// Request for UpdateAutoUpdateConfig. +message UpdateAutoUpdateAgentRolloutRequest { + AutoUpdateAgentRollout plan = 1; +} + +// Request for UpsertAutoUpdateAgentRollout. +message UpsertAutoUpdateAgentRolloutRequest { + AutoUpdateAgentRollout plan = 1; +} + +// Request for DeleteAutoUpdateAgentRollout. +message DeleteAutoUpdateAgentRolloutRequest {} + +message TriggerAgentGroupRequest { + // group is the agent update group name whose maintenance should be triggered. + string group = 1; + // desired_state describes the desired start state. + // Supported values are STATE_UNSPECIFIED, STATE_CANARY, and STATE_ACTIVE. + // When left empty, defaults to canary if they are supported. + State desired_state = 2; +} + +message ForceAgentGroupRequest { + // group is the agent update group name whose state should be forced to `done`. + string group = 1; +} + +message ResetAgentGroupRequest { + // group is the agent update group name whose state should be reset. + string group = 1; +} + +message RollbackAgentGroupRequest { + // group is the agent update group name whose state should change to `rolledback`. + string group = 1; +} +``` + +### Backend logic to progress the rollout + +#### Rollout strategies + +We support two rollout strategies, for two distinct use-cases: + +- `halt-on-failure` for damage reduction of a faulty update +- `time-based` for time-constrained maintenances + +In `halt-on-failure`, the update proceeds from the first group to the last group, ensuring that each group +successfully updates before allowing the next group to proceed. By default, only 5 agent groups are allowed. This +mitigates very long rollout plans. This is the strategy that offers the best availability. A group finishes its update +once most of its agents are running the correct version. Agents that missed the group update will try to catch +back as soon as possible. + +In `time-based` maintenances, agents update as soon as their maintenance window starts. There is no dependency +between groups. This strategy allows Teleport users to setup reliable follow-the-sun updates and enforce the +maintenance window more strictly. A group finishes its update at the end of the maintenance window, regardless +of the new version adoption rate. Agents that missed the maintenance window will not attempt to +update until the next maintenance window. + +After phase 6, a third strategy, `backpressure` will be added. This strategy will behave the same way `halt-on-failure` +does, except the agents will be progressively rolled-out within a group. + +#### Agent update mode + +The agent auto update mode is specified by both Cloud (via `autoupdate_version`) +and by the customer (via `autoupdate_config`). The agent update mode controls whether +the cluster in enrolled into automatic agent updates. + +The agent update mode can take 3 values: + +1. disabled: teleport should not manage agent updates +2. paused: the updates are temporarily suspended, we honour the existing rollout state +3. enabled: teleport can update agents + +The cluster agent rollout mode is computed by taking the lowest value. +For example: + +- Cloud says `enabled` and the customer says `enabled` -> the updates are `enabled` +- Cloud says `enabled` and the customer says `suspended` -> the updates are `suspended` +- Cloud says `disabled` and the customer says `suspended` -> the updates are `disabled` +- Cloud says `disabled` and the customer says `enabled` -> the updates are `disabled` + +The Teleport cluster only progresses the rollout if the mode is `enabled`. + +#### Group States + +Let `v1` be the previous version and `v2` the target version. + +A group can be in 5 states: +- `unstarted`: the group update has not been started yet. +- `canary`: a few canaries are getting updated. New agents should run `v1`. Existing agents should not attempt to update + and keep their existing version. +- `active`: the group is actively getting updated. New agents should run `v2`, existing agents are instructed to update + to `v2`. +- `done`: the group has been updated. New agents should run `v2`. +- `rolledback`: the group has been rolledback. New agents should run `v1`, existing agents should update to `v1`. + +The finite state machine for the `halt-on-failure` is the following: + +```mermaid +flowchart TD + unstarted((unstarted)) + canary((canary)) + active((active)) + done((done)) + rolledback((rolledback)) + + unstarted -->|TriggerGroupRPC
Start conditions are met| canary + canary -->|Canaries came back alive| active + canary -->|ForceGroupRPC| done + canary -->|RollbackGroupRPC| rolledback + active -->|ForceGroupRPC
Success criteria met| done + done -->|RollbackGroupRPC| rolledback + active -->|RollbackGroupRPC| rolledback + + canary -->|ResetGroupRPC| canary + active -->|ResetGroupRPC| active +``` + +The finite state machine for the `time-based` is the following: +```mermaid +flowchart TD + unstarted((unstarted)) + canary((canary)) + active((active)) + done((done)) + rolledback((rolledback)) + + unstarted -->|TriggerGroupRPC
Start conditions are met| canary + canary -->|Canaries came back alive and window is still active| active + canary -->|ForceGroupRPC
Canaries came back alive and window is over| done + canary -->|RollbackGroupRPC| rolledback + active -->|ForceGroupRPC
End of window| done + done -->|Beginning of window| active + done -->|RollbackGroupRPC| rolledback + active -->|RollbackGroupRPC| rolledback + + canary -->|ResetGroupRPC| canary +``` + + +> [!NOTE] +> Once we have a proper feedback mechanism (phase 5) we might introduce a new `unfinished` state, similar to done, but +> which indicates that not all agents got updated when using the `time-based` strategy. This does not change the update +> logic but might be clearer for the end user. + +#### Starting a group + +A group can be started if the following criteria are met +- for the `halt-on-failure` strategy: + - all of its previous group are in the `done` state + - it has been at least `wait_days` since the previous group update started + - the current week day is in the `days` list + - the current hour equals the `hour` field +- for the `time-based` strategy: + - the current week day is in the `days` list + - the current hour equals the `hour` field + +When all those criteria are met, the auth will transition the group into a new state. +If `canary_count` is not null, the group transitions to the `canary` state. +Else it transitions to the `active` state. + +In phase 4, at the start of a group rollout, the Teleport auth servers record the initial number connected agents. +The number of updated and non-updated agents is tracked by the auth servers. This will be used later to evaluate the +update success criteria. + +#### Canary testing (phase 5) + +A group in `canary` state will be randomly assigned `canary_count` canary agents. +Auth servers will select those canaries by reading them from the auth instance inventory and writing them to the `canaries` list in `agent_rollout_plan` status. +The proxies will instruct those canaries to update immediately. +During each reconciliation loop, the auth will lookup the instance heartbeat of each canary in the backend and update `agent_rollout_plan` status if needed. + +Once all canaries have a heartbeat containing the new version (the heartbeat must not be older than 20 minutes), +they successfully came back online and the group can transition to the `active` state. + +If canaries never update, report rollback, or disappear, the group will stay stuck in `canary` state. +An alert will eventually fire, warning the user about the stuck update. + +> [!NOTE] +> In the first version, canary selection will happen randomly. As most instances are running the ssh_service and not +> the other ones, we are less likely to catch an issue in a less common service. +> An optimisation would be to try to pick canaries maximizing the service coverage. +> This would make the test more robust and provide better availability guarantees. + +#### Updating a group + +A group in `active` mode is currently being updated. The conditions to leave `active` mode and transition to the +`done` mode will vary based on the phase and rollout strategy. + +- for the `halt-on-failure` strategy: + - Phase 3: we don't have any information about agents. The group transitions to `done` 60 minutes after its start. + - Phase 4: we know about the connected agent count and the connected agent versions. The group transitions to `done` if: + - at least `(100 - max_in_flight)%` of the agents are still connected + - at least `(100 - max_in_flight)%` of the agents are running the new version + - Phase 6: we incrementally update the progress, this adds a new criteria: the group progress is at 100% +- for the `time-based` strategy: + - the group transitions to the `done` state `maintenance_window_minutes` minutes after the `active` transition. + The rollout's `start_time` must be used to do this transition, not the schedule's `start_hour`. + This will allow the user to trigger manual out-of-maintenance updates if needed. + +The phase 6 backpressure calculations are covered in the Backpressure Calculations section below.. + +### Manually interacting with the rollout + +For user: +```shell +tctl autoupdate agent suspend/resume +tctl autoupdate agent enable/disable + +tctl autoupdate agent status +tctl autoupdate agent status + +tctl autoupdate agent start [--no-canary] +tctl autoupdate agent force +tctl autoupdate agent reset + +tctl autoupdate agent rollback [|--all] +``` + +For admin +```shell +tctl autoupdate agent-plan target [--previous-version ] +tctl autoupdate agent-plan enable/disable +tctl autoupdate agent-plan suspend/resume +``` + +### Editing the plan + +The updater will receive `agent_auto_update: true` from the time is it designated for update until the `target_version` in `autoupdate_version` (below) changes. +Changing the `target_version` resets the schedule immediately, clearing all progress. + +[TODO: What is the use-case for this? can we do like with target_version and reset all instead of trying to merge the state] +Changing the `start_version` in `autoupdate_version` changes the advertised `start_version` for all unfinished groups. + +Changing `agent_schedules` will preserve the `state` of groups that have the same name before and after the change. +However, any changes to `agent_schedules` that occur while a group is active will be rejected. + +Releasing new agent versions multiple times a week has the potential to starve dependent groups from updates. + +Note that the `default` group applies to agents that do not specify a group name. +If a `default` group is not present, the last group is treated as the default. + +### Updater APIs + +#### Update requests + +Teleport proxies will be updated to serve the desired agent version and edition from `/v1/webapi/find`. +The version served from that endpoint will be configured using new `autoupdate_version` resource. + +Whether the Teleport updater querying the endpoint is instructed to upgrade (via the `agent_auto_update` field) is +dependent on: +- The `host=[uuid]` parameter sent to `/v1/webapi/find` +- The `group=[name]` parameter sent to `/v1/webapi/find` +- The group state from the `autoupdate_agent_rollout` status (this also contains the version from `autoupdate_version`) + +To ensure that the updater is always able to retrieve the desired version, instructions to the updater are delivered via +unauthenticated requests to `/v1/webapi/find`. Teleport proxies modulate the `/v1/webapi/find` response given the host +UUID and group name. + +When the updater queries the proxy via `/v1/webapi/find?host=[uuid]&group=[name]`, the proxies query the +`autoupdate_agent_rollout` to determine the value of `agent_auto_update: true`. +The boolean is returned as `true` in the case that the provided `host` contains a UUID that is under the progress +percentage for the `group`: +`as_numeral(host_uuid) / as_numeral(max_uuid) < progress` + +The returned JSON looks like: + +`/v1/webapi/find?host=[uuid]&group=[name]` +```json +{ + "server_edition": "enterprise", + "auto_update": { + "agent_version": "15.1.1", + "agent_auto_update": true, + "agent_update_jitter_seconds": 10 + }, + // ... +} +``` + +Notes: + +- Agents will only update if `agent_auto_update` is `true`, but new installations will use `agent_version` regardless of + the value in `agent_auto_update`. +- The edition served is the cluster edition (enterprise, enterprise-fips, or oss) and cannot be configured. +- The group name is read from `/var/lib/teleport/versions/update.yaml` by the updater. +- The UUID is read from `/tmp/teleport_update_uuid`, which `teleport-update` regenerates when missing. +- the jitter is served by the teleport cluster and depends on the rollout strategy (60s by default, 10s when using + the backpressure strategy). + +Let `v1` be the previous version and `v2` the target version, the response matrix is the following: + +##### Rollout status: disabled + +| Group state | Version | Should update | +|-------------|---------|---------------| +| * | v2 | false | + +##### Rollout status: paused + +| Group state | Version | Should update | +|-------------|---------|---------------| +| unstarted | v1 | false | +| canary | v1 | false | +| active | v2 | false | +| done | v2 | false | +| rolledback | v1 | false | + +##### Rollout status: enabled + +| Group state | Version | Should update | +|-------------|---------|--------------------------------------------------| +| unstarted | v1 | false | +| canary | v1 | false, except for canaries | +| active | v2 | true if UUID <= progress | +| done | v2 | true if `halt-on-failure`, false if `time-based` | +| rolledback | v1 | true | + +#### Updater status reporting + +The updater reports status through the agent. The agent has two ways of reporting the update information: +- via instance heartbeats +- via the hello message, when registering against an auth server + +Instance heartbeat happen infrequently, based on the cluster size they can take up to 17 minutes to happen. +However, they are exposed to the user via existing `tctl inventory` method and will allow users to query which instance +is running which version and belongs to which group. + +Hello messages are sent on connection and are used to build the serve's local inventory. +This information is available almost instantaneously after the connection and can be cheaply queried by the auth ( +everything is in memory). The inventory is then used to count the local instances and drive the rollout. + +Both instance heartbeats and Hello merssages will be extended to incorporate and send data that is written to +`/var/lib/teleport/versions/update.yaml` and `/tmp/teleport_update_uuid` by the `teleport-update` binary. + +The following data related to the update is sent by the agent: +- `agent_update_start_time`: timestamp of individual agent's upgrade time +- `agent_update_start_version`: current agent version +- `agent_update_rollback`: whether the agent was rolled-back automatically +- `agent_update_uuid`: Auto-update UUID +- `agent_update_group`: Auto-update group name + +Auth servers use their local instance inventory to calculate rollout statistics and write them to `/autoupdate/[group]/[auth ID]` (e.g., `/autoupdate/staging/58526ba2-c12d-4a49-b5a4-1b694b82bf56`). + +Every minute, auth servers persist the version counts: +- `agent_data[group].stats[version]` + - `count`: number of currently connected agents at `version` in `group` + - `failed_count`: number of currently connected agents at `version` in `group` that experienced a rollback or inability to upgrade + - `lowest_uuid`: lowest UUID of all currently connected agents at `version` in `group` + - `count`: number of connected agents at `version` in `group` at start of window +- `agent_data[group]` + - `canaries`: list of updater UUIDs to use for canary deployments + +Expiration time of the persisted key is 1 hour. + +To progress the rollout, auth servers will range-read keys from `/autoupdate/[group]/*`, sum the counts, and write back to the `autoupdate_agent_rollout` status on a one-minute interval. +- To calculate the initial number of agents connected at the start of the window, each auth server will write the summed count of agents to `autoupdate_agent_rollout` status, if not already written. +- To calculate the canaries, each auth server will write a random selection of all canaries to `autoupdate_agent_rollout` status, if not already written. +- To determine the progress through the rollout, auth servers will write the calculated progress to the `autoupdate_agent_rollout` status using the formulas, declining to write if the current written progress is further ahead. + +If `/autoupdate/[group]/[auth ID]` is older than 1 minute, we do not consider its contents. +This prevents double-counting agents when auth servers are killed. + +#### Backpressure Calculations + +Given: +``` +initial_count[group] = sum(agent_data[group].stats[*]).count +``` + +Each auth server will calculate the progress as +`( max_in_flight * initial_count[group] + agent_data[group].stats[target_version].count ) / initial_count[group]` and +write the progress to `autoupdate_agent_rollout` status. This formula determines the progress percentage by adding a +`max_in_flight` percentage-window above the number of currently updated agents in the group. + +However, if `as_numeral(agent_data[group].stats[not(target_version)].lowest_uuid) / as_numeral(max_uuid)` is above the +calculated progress, that progress value will be used instead. This protects against a statistical deadlock, where no +UUIDs fall within the next `max_in_flight` window of UUID space, by always permitting the next non-updated agent to +update. + +To ensure that the rollout is halted if more than `max_in_flight` un-updated agents drop off, an addition restriction +must be imposed for the rollout to proceed: +`agent_data[group].stats[*].count > initial_count[group] - max_in_flight * initial_count[group]` + +To prevent double-counting of agents when considering all counts across all auth servers, only agents connected for one +minute will be considered in these formulas. + +### Linux Agents + +We will ship a new auto-updater binary for Linux servers written in Go that does not interface with the system package manager. +It will be distributed within the existing `teleport` packages, and additionally, in a dedicated `teleport-update-vX.Y.Z.tgz` tarball. +It will manage the installation of the correct Teleport agent version manually. + +It will read the unauthenticated `/v1/webapi/find` endpoint from the Teleport proxy, parse new fields on that endpoint, and install the specified agent version according to the specified update plan. +It will download the correct version of Teleport as a tarball, unpack it in `/var/lib/teleport`, and ensure it is symlinked from `/usr/local/bin`. + +Source code for the updater will live in the main Teleport repository, with the updater binary built from `tools/teleport-update`. + +#### Installation + +Package-initiated install: +```shell +$ apt-get install teleport +$ teleport-update enable --proxy example.teleport.sh + +# if not enabled already, configure teleport and: +$ systemctl enable teleport +``` + +Packageless install: +```shell +$ curl https://cdn.teleport.dev/teleport-update.tgz | tar xzf +$ ./teleport-update enable --proxy example.teleport.sh + +# if not enabled already, configure teleport and: +$ systemctl enable teleport +``` + +For grouped updates, a group identifier may be configured: +```shell +$ teleport-update enable --proxy example.teleport.sh --group staging +``` + +For air-gapped Teleport installs, the agent may be configured with a custom tarball path template: +```shell +$ teleport-update enable --proxy example.teleport.sh --template 'https://example.com/teleport-{{ .Edition }}-{{ .Version }}-{{ .Arch }}.tgz' +``` +(Checksum will use template path + `.sha256`) + +For Teleport installs with custom data directories, the data directory must be specified on each binary invocation: +```shell +$ teleport-update enable --proxy example.teleport.sh --data-dir /var/lib/teleport +``` + +For managing multiple Teleport installs, the install suffix must be specified on each binary invocation: +```shell +$ teleport-update enable --proxy example.teleport.sh --install-suffix clusterA +``` +This will create suffixed directories for binaries (`/usr/local/teleport/clusterA/bin`) and systemd units (`teleport-clusterA`). + + +#### Filesystem + +For a default install, without --install-suffix: +``` +$ tree /var/lib/teleport +/var/lib/teleport +└── versions + ├── 15.0.0 + │ ├── bin + │ │ ├── tsh + │ │ ├── tbot + │ │ ├── ... # other binaries + │ │ ├── teleport-update + │ │ └── teleport + │ ├── etc + │ │ └── systemd + │ │ └── teleport.service + │ └── backup + │ ├── sqlite.db + │ └── backup.yaml + ├── 15.1.1 + │ ├── bin + │ │ ├── tsh + │ │ ├── tbot + │ │ ├── ... # other binaries + │ │ ├── teleport-update + │ │ └── teleport + │ └── etc + │ └── systemd + │ └── teleport.service + └── update.yaml + +$ ls -l /usr/local/bin/tsh +/usr/local/bin/tsh -> /var/lib/teleport/versions/15.0.0/bin/tsh +$ ls -l /usr/local/bin/tbot +/usr/local/bin/tbot -> /var/lib/teleport/versions/15.0.0/bin/tbot +$ ls -l /usr/local/bin/teleport +/usr/local/bin/teleport -> /var/lib/teleport/versions/15.0.0/bin/teleport +$ ls -l /usr/local/bin/teleport-update +/usr/local/bin/teleport-update -> /var/lib/teleport/versions/15.0.0/bin/teleport-update +$ ls -l /usr/local/lib/systemd/system/teleport.service +/usr/local/lib/systemd/system/teleport.service -> /var/lib/teleport/versions/15.0.0/etc/systemd/teleport.service +``` + +With --install-suffix clusterA: +``` +$ tree /var/lib/teleport/install/clusterA +/var/lib/teleport/install/clusterA +└── versions + ├── 15.0.0 + │ ├── bin + │ │ ├── tsh + │ │ ├── tbot + │ │ ├── ... # other binaries + │ │ ├── teleport-update + │ │ └── teleport + │ ├── etc + │ │ └── systemd + │ │ └── teleport.service + │ └── backup + │ ├── sqlite.db + │ └── backup.yaml + ├── 15.1.1 + │ ├── bin + │ │ ├── tsh + │ │ ├── tbot + │ │ ├── ... # other binaries + │ │ ├── teleport-update + │ │ └── teleport + │ └── etc + │ └── systemd + │ └── teleport.service + └── update.yaml + +/var/lib/teleport +└── versions + ├── system # if installed via OS package + ├── bin + │ ├── tsh + │ ├── tbot + │ ├── ... # other binaries + │ ├── teleport-update + │ └── teleport + └── etc + └── systemd + └── teleport.service + +$ ls -l /usr/local/bin/tsh +/usr/local/teleport/clusterA/bin/tsh -> /var/lib/teleport/install/clusterA/versions/15.0.0/bin/tsh +$ ls -l /usr/local/bin/tbot +/usr/local/teleport/clusterA/bin/tbot -> /var/lib/teleport/install/clusterA/versions/15.0.0/bin/tbot +$ ls -l /usr/local/bin/teleport +/usr/local/teleport/clusterA/bin/teleport -> /var/lib/teleport/install/clusterA/versions/15.0.0/bin/teleport +$ ls -l /usr/local/bin/teleport-update +/usr/local/teleport/clusterA/bin/teleport-update -> /var/lib/teleport/install/clusterA/versions/15.0.0/bin/teleport-update +$ ls -l /usr/local/lib/systemd/system/teleport-clusterA.service +/usr/local/lib/systemd/system/teleport-clutserA.service -> /var/lib/teleport/install/clusterA/versions/15.0.0/etc/systemd/teleport.service +``` + +##### update.yaml + +This file stores configuration for `teleport-update`. + +All updates are applied atomically using renameio. + +``` +version: v1 +kind: update_config +spec: + # proxy specifies the Teleport proxy address to retrieve the agent version and update configuration from. + proxy: mytenant.teleport.sh + # group specifies the update group + group: staging + # url_template specifies a custom URL template for downloading Teleport. + # url_template: "" + # enabled specifies whether auto-updates are enabled, i.e., whether teleport-update update is allowed to update the agent. + enabled: true +status: + # start_time specifies the start time of the most recent update. + start_time: 2020-12-09T16:09:53+00:00 + # active_version specifies the active (symlinked) deployment of the teleport agent. + active_version: 15.1.1 + # version_history specifies the previous deployed versions, in order by recency. + version_history: ["15.1.3", "15.0.4"] + # rollback specifies whether the most recent version was deployed by an automated rollback. + rollback: true + # error specifies the last error encounted + error: "" +``` + +##### backup.yaml + +This file stores metadata about an individual backup of the Teleport agent's sqlite DB. + +``` +version: v1 +kind: db_backup +spec: + # proxy address from the backup + proxy: mytenant.teleport.sh + # version from the backup + version: 15.1.0 + # time the backup was created + creation_time: 2020-12-09T16:09:53+00:00 +``` + +#### Runtime + +The `teleport-update` binary will run as a periodically executing systemd service which runs every 10 minutes. +The systemd service will run: +```shell +$ teleport-update update +``` + +After it is installed, the `update` subcommand will no-op when executed until configured with the `teleport-update` command: +```shell +$ teleport-update enable --proxy mytenant.teleport.sh --group staging +``` + +If the proxy address is not provided with `--proxy`, the current proxy address from `teleport.yaml` is used, if present. + +The `enable` subcommand will change the behavior of `teleport-update update` to update teleport and restart the existing agent, if running. +It will also run update teleport immediately, to ensure that subsequent executions succeed. + +Both `update` and `enable` will maintain a shared lock file preventing any re-entrant executions. + +The `enable` subcommand will: +1. If an updater-incompatible version of the Teleport package is installed, fail immediately. +2. Query the `/v1/webapi/find` endpoint. +3. If the current updater-managed version of Teleport is the latest, jump to (15). +4. Ensure there is enough free disk space to update Teleport via `unix.Statfs()` and `content-length` header from `HEAD` request. +5. Download the desired Teleport tarball specified by `agent_version` and `server_edition`. +6. Download and verify the checksum (tarball URL suffixed with `.sha256`). +7. Extract the tarball to `/var/lib/teleport/versions/VERSION` and write the SHA to `/var/lib/teleport/versions/VERSION/sha256`. +8. Verify that the downloaded binaries are valid executables on the host. +9. Replace any existing binaries or symlinks with symlinks to the current version. +10. Backup `/var/lib/teleport/proc/sqlite.db` into `/var/lib/teleport/versions/OLD-VERSION/backup/sqlite.db` and create `backup.yaml`. +11. Restart the agent if the systemd service is already enabled. +12. Set `active_version` in `update.yaml` if successful or not enabled. +13. Replace the symlinks/binaries and `/var/lib/teleport/proc/sqlite.db` and quit (exit 1) if unsuccessful. +14. Remove all stored versions of the agent except the current version and last working version. +15. Configure `update.yaml` with the current proxy address and group, and set `enabled` to true. + +The `disable` subcommand will: +1. Configure `update.yaml` to set `enabled` to false. + +When `update` subcommand is otherwise executed, it will: +1. Check `update.yaml`, and quit (exit 0) if `enabled` is false, or quit (exit 1) if `enabled` is true and no proxy address is set. +2. Query the `/v1/webapi/find` endpoint. +3. Check that `agent_auto_updates` is true, quit otherwise. +4. If the current version of Teleport is the latest, quit. +5. Wait `random(0, agent_update_jitter_seconds)` seconds. +6. Ensure there is enough free disk space to update Teleport via `unix.Statfs()` and `content-length` header from `HEAD` request. +7. Download the desired Teleport tarball specified by `agent_version` and `server_edition`. +8. Download and verify the checksum (tarball URL suffixed with `.sha256`). +9. Extract the tarball to `/var/lib/teleport/versions/VERSION` and write the SHA to `/var/lib/teleport/versions/VERSION/sha256`. +10. Verify that the downloaded binaries are valid executables on the host. +11. Update symlinks to point at the new version. +12. Backup `/var/lib/teleport/proc/sqlite.db` into `/var/lib/teleport/versions/OLD-VERSION/backup/sqlite.db` and create `backup.yaml`. +13. Restart the agent if the systemd service is already enabled. +14. Set `active_version` in `update.yaml` if successful or not enabled. +15. Replace the old symlinks/binaries and `/var/lib/teleport/proc/sqlite.db` and quit (exit 1) if unsuccessful. +16. Remove all stored versions of the agent except the current version and last working version. + +To guarantee auto-updates of the updater itself, all commands will first check for an `active_version`, and reexec using the `teleport-update` at that version if present and different. +The `/usr/local/bin/teleport-update` symlink will take precedence to avoid reexec in most scenarios. + +To ensure that SELinux permissions do not prevent the `teleport-update` binary from installing/removing Teleport versions, the updater package will configure SELinux contexts to allow changes to all required paths. + +To ensure that backups are consistent, the updater will use the [SQLite backup API](https://www.sqlite.org/backup.html) to perform the backup. + +The `teleport` apt and yum packages will contain a system installation of Teleport in `/usr/local/teleport-system/`. +Post package installation, the `link` subcommand is executed automatically to link the system installation when no auto-updater-managed version of Teleport is linked: +``` +/usr/local/bin/teleport -> /usr/local/teleport-system/bin/teleport +/usr/local/bin/teleport-update -> /usr/local/teleport-system/bin/teleport-update +... +``` + +#### Failure Conditions + +If the new version of Teleport fails to start, the installation of Teleport is reverted as described above. + +If `teleport-update` itself fails with an error, and an older version of `teleport-update` is available, the update will retry with the older version. + +If the agent losses its connection to the proxy, `teleport-update` updates the agent to the group's current desired version immediately. + +Known failure conditions caused by intentional configuration (e.g., updates disabled) will not trigger retry logic. + +#### Status + +To retrieve known information about agent updates, the `status` subcommand will return the following: +```json +{ + "agent_version_installed": "15.1.1", + "agent_version_desired": "15.1.2", + "agent_version_previous": "15.1.0", + "agent_update_time_last": "2020-12-10T16:00:00+00:00", + "agent_update_time_jitter": 600, + "agent_updates_enabled": true +} +``` + +### Downgrades + +Downgrades may be necessary in cases where we have rolled out a bug or security vulnerability with critical impact. +To initiate a downgrade, `agent_version` is set to an older version than it was previously set to. + +Downgrades are challenging, because `sqlite.db` used by newer version of Teleport may not be valid for older versions of Teleport. + +When Teleport is downgraded to a previous version that has a backup of `sqlite.db` present in `/var/lib/teleport/versions/OLD-VERSION/backup/`: +1. `/var/lib/teleport/versions/OLD-VERSION/backup/backup.yaml` is validated to determine if the backup is usable (proxy and version must match, age must be less than cert lifetime, etc.) +2. If the backup is valid, Teleport is fully stopped, the backup is restored along with symlinks, and the downgraded version of Teleport is started. +3. If the backup is invalid, we refuse to downgrade. + +Downgrades are applied with `teleport-update update`, just like upgrades. +The above steps modulate the standard workflow in the section above. +If the downgraded version is already present, the uncompressed version is used to ensure fast recovery of the exact state before the failed upgrade. +To ensure that the target version is was not corrupted by incomplete extraction, the downgrade checks for the existence of `/var/lib/teleport/versions/TARGET-VERSION/sha256` before downgrading. +To ensure that the DB backup was not corrupted by incomplete copying, the downgrade checks for the existence of `/var/lib/teleport/versions/TARGET-VERSION/backup/backup.yaml` before restoring. + +Teleport must be fully-stopped to safely replace `sqlite.db`. +When restarting the agent during an upgrade, `SIGHUP` is used. +When restarting the agent during a downgrade, `systemd stop/start` are used before/after the downgrade. + +Teleport CA certificate rotations will break rollbacks. +In the future, this could be addressed with additional validation of the agent's client certificate issuer fingerprints. +For now, rolling forward will allow recovery from a broken rollback. + +Given that rollbacks may fail, we must maintain the following invariants: +1. Broken rollbacks can always be reverted by reversing the rollback exactly. +2. Broken versions can always be reverted by rolling back and then skipping the broken version. + +When rolling forward, the backup of the newer version's `sqlite.db` is only restored if that exact version is the roll-forward version. +Otherwise, the older, rollback version of `sqlite.db` is preserved (i.e., the newer version's backup is not used). +This ensures that a version update which broke the database can be recovered with a rollback and a new patch. +It also ensures that a broken rollback is always recoverable by reversing the rollback. + +Example: Given v1, v2, v3 versions of Teleport, where v2 is broken: +1. v1 -> v2 -> v1 -> v3 => DB from v1 is migrated directly to v3, avoiding v2 breakage. +2. v1 -> v2 -> v1 -> v2 -> v3 => DB from v2 is recovered, in case v1 database no longer has a valid certificate. + +### Manual Workflow + +For use cases that fall outside of the functionality provided by `teleport-update`, we provide an alternative manual workflow using the `/v1/webapi/find` endpoint. +This workflow supports customers that cannot use the auto-update mechanism provided by `teleport-update` because they use their own automation for updates (e.g., JamF or Ansible). + +Cluster administrators that want to self-manage agent updates may manually query the `/v1/webapi/find` endpoint using the host UUID, and implement auto-updates with their own automation. + +Cluster administrators that choose this path may use the `teleport` package without auto-updates enabled locally. + +### Installers + +The following install scripts will be updated to install the latest updater and run `teleport-update enable` with the proxy address: +- [/api/types/installers/agentless-installer.sh.tmpl](https://github.com/gravitational/teleport/blob/d0a68fd82412b48cb54f664ae8500f625fb91e48/api/types/installers/agentless-installer.sh.tmpl) +- [/api/types/installers/installer.sh.tmpl](https://github.com/gravitational/teleport/blob/d0a68fd82412b48cb54f664ae8500f625fb91e48/api/types/installers/installer.sh.tmpl) +- [/lib/web/scripts/oneoff/oneoff.sh](https://github.com/gravitational/teleport/blob/d0a68fd82412b48cb54f664ae8500f625fb91e48/lib/web/scripts/oneoff/oneoff.sh) +- [/lib/web/scripts/node-join/install.sh](https://github.com/gravitational/teleport/blob/d0a68fd82412b48cb54f664ae8500f625fb91e48/lib/web/scripts/node-join/install.sh) +- [/assets/aws/files/install-hardened.sh](https://github.com/gravitational/teleport/blob/d0a68fd82412b48cb54f664ae8500f625fb91e48/assets/aws/files/install-hardened.sh) + +Eventually, additional logic from the scripts could be added to `teleport-update`, such that `teleport-update` can configure teleport. + +Moving additional logic into the updater is out-of-scope for this proposal. + +To create pre-baked VM or container images that reduce the complexity of the cluster joining operation, two workflows are permitted: +- Install the `teleport` package and defer `teleport-update enable`, Teleport configuration, and `systemctl enable teleport` to cloud-init scripts. + This allows both the proxy address and token to be injected at VM initialization. The VM image may be used with any Teleport cluster. + Installers scripts will continue to function, as the package install operation will no-op. +- Install the `teleport` package and run `teleport-update enable` before the image is baked, but defer final Teleport configuration and `systemctl enable teleport` to cloud-init scripts. + This allows the proxy address to be pre-set in the image. `teleport.yaml` can be partially configured during image creation. At minimum, the token must be injected via cloud-init scripts. + Installers scripts would be skipped in favor of the `teleport configure` command. + +It is possible for a VM or container image to be created with a baked-in join token. +We should recommend against this workflow for security reasons, since a long-lived token improperly stored in an image could be leaked. + +Alternatively, users may prefer to skip pre-baked agent configuration, and run one of the script-based installers to join VMs to the cluster after the VM is started. + +Documentation should be created covering the above workflows. + +### Documentation + +The following documentation will need to be updated to cover the new updater workflow: +- https://goteleport.com/docs/choose-an-edition/teleport-cloud/downloads +- https://goteleport.com/docs/installation +- https://goteleport.com/docs/upgrading/self-hosted-linux +- https://goteleport.com/docs/upgrading/self-hosted-automatic-agent-updates + +Additionally, the Cloud dashboard tenants downloads tab will need to be updated to reference the new instructions. + +### Details - Kubernetes Agents + +The Kubernetes agent updater will be updated for compatibility with the new scheduling system. + +This means that it will stop reading update windows using the authenticated connection to the proxy, and instead update when indicated by the `/v1/webapi/find` endpoint. + +Rollbacks for the Kubernetes updater, as well as packaging changes to improve UX and compatibility, will be covered in a future RFD. + +## Migration + +The existing update system will remain in-place until the old auto-updater is fully deprecated. + +Both update systems can co-exist on the same machine. +The old auto-updater will update the system package, which will not affect the `teleport-update`-managed installation. + +Eventually, the `cluster_maintenance_config` resource and `teleport-ent-upgrader` package will be deprecated. + +## Security + +The initial version of automatic updates will rely on TLS to establish +connection authenticity to the Teleport download server. The authenticity of +assets served from the download server is out of scope for this RFD. Cluster +administrators concerned with the authenticity of assets served from the +download server can use self-managed updates with system package managers which +are signed. + +The Update Framework (TUF) will be used to implement secure updates in the future. + +Anyone who possesses an updater UUID can determine when that host is scheduled to update by repeatedly querying the public `/v1/webapi/find` endpoint. +It is not possible to discover the current version of that host, only the designated update window. + +## Logging + +All installation steps will be logged locally, such that they are viewable with `journalctl`. +Care will be taken to ensure that updater logs are sharable with Teleport Support for debugging and auditing purposes. + +When TUF is added, that events related to supply chain security may be sent to the Teleport cluster via the Teleport Agent. + +## Alternatives + +### `teleport update` Subcommand + +`teleport-update` is intended to be a minimal binary, with few dependencies, that is used to bootstrap initial Teleport agent installations. +It may be baked into AMIs or containers. + +If the entire `teleport` binary were used instead, security scanners would match vulnerabilities all Teleport dependencies, so customers would have to handle rebuilding artifacts (e.g., AMIs) more often. +Deploying these updates is often more disruptive than a soft restart of the agent triggered by the auto-updater. + +`teleport-update` will also handle `tbot` updates in the future, and it would be undesirable to distribute `teleport` with `tbot` just to enable automated updates. + +Finally, `teleport-update`'s API contract with the cluster must remain stable to ensure that outdated agent installations can always be recovered. +The first version of `teleport-update` will need to work with Teleport v14 and all future versions of Teleport. +This contract may be easier to manage with a separate artifact. + +### Mutually-Authenticated RPC for Update Boolean + +Agents will not always have a mutually-authenticated connection to auth to receive update instructions. +For example, the agent may be in a failed state due to a botched upgrade, may be temporarily stopped, or may be newly installed. +In the future, `tbot`-only installations may have expired certificates. + +Making the update boolean instruction available via the `/webapi/find` TLS endpoint reduces complexity as well as the risk of unrecoverable outages. + +## Execution Plan + +1. Implement Teleport APIs for new scheduling system (without backpressure strategy, canaries, or completion tracking) +2. Implement new Linux server auto-updater in Go, including systemd-based rollbacks. +3. Implement changes to Kubernetes auto-updater. +4. Test extensively on all supported Linux distributions. +5. Prep documentation changes. +6. Release via `teleport` package and script for package-less installation. +7. Release documentation changes. +8. Communicate to users that they should update to the new system. +9. Begin deprecation of old auto-updater resources, packages, and endpoints. +10. Add healthcheck endpoint to Teleport agents and incorporate into rollback logic. +11. Add progress and completion checking. +12. Add canary functionality. +13. Add backpressure functionality if necessary. +14. Add DB backups if necessary. diff --git a/tool/tctl/common/autoupdate_command.go b/tool/tctl/common/autoupdate_command.go index 76857b7d3b745..8085581f12f91 100644 --- a/tool/tctl/common/autoupdate_command.go +++ b/tool/tctl/common/autoupdate_command.go @@ -23,6 +23,8 @@ import ( "fmt" "io" "os" + "strings" + "time" "github.com/alecthomas/kingpin/v2" "github.com/coreos/go-semver/semver" @@ -32,6 +34,7 @@ import ( "github.com/gravitational/teleport/api/client/webclient" autoupdatev1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" "github.com/gravitational/teleport/api/types/autoupdate" + "github.com/gravitational/teleport/lib/asciitable" "github.com/gravitational/teleport/lib/auth/authclient" "github.com/gravitational/teleport/lib/service/servicecfg" "github.com/gravitational/teleport/lib/utils" @@ -45,10 +48,11 @@ const maxRetries = 3 type AutoUpdateCommand struct { app *kingpin.Application - targetCmd *kingpin.CmdClause - enableCmd *kingpin.CmdClause - disableCmd *kingpin.CmdClause - statusCmd *kingpin.CmdClause + toolsTargetCmd *kingpin.CmdClause + toolsEnableCmd *kingpin.CmdClause + toolsDisableCmd *kingpin.CmdClause + toolsStatusCmd *kingpin.CmdClause + agentsStatusCmd *kingpin.CmdClause toolsTargetVersion string proxy string @@ -68,16 +72,19 @@ func (c *AutoUpdateCommand) Initialize(app *kingpin.Application, _ *servicecfg.C clientToolsCmd := autoUpdateCmd.Command("client-tools", "Manage client tools auto update configuration.") - c.statusCmd = clientToolsCmd.Command("status", "Prints if the client tools updates are enabled/disabled, and the target version in specified format.") - c.statusCmd.Flag("proxy", "Address of the Teleport proxy. When defined this address will be used to retrieve client tools auto update configuration.").StringVar(&c.proxy) - c.statusCmd.Flag("format", "Output format: 'yaml' or 'json'").Default(teleport.YAML).StringVar(&c.format) + c.toolsStatusCmd = clientToolsCmd.Command("status", "Prints if the client tools updates are enabled/disabled, and the target version in specified format.") + c.toolsStatusCmd.Flag("proxy", "Address of the Teleport proxy. When defined this address will be used to retrieve client tools auto update configuration.").StringVar(&c.proxy) + c.toolsStatusCmd.Flag("format", "Output format: 'yaml' or 'json'").Default(teleport.YAML).StringVar(&c.format) - c.enableCmd = clientToolsCmd.Command("enable", "Enables client tools auto updates. Clients will be told to update to the target version.") - c.disableCmd = clientToolsCmd.Command("disable", "Disables client tools auto updates. Clients will not be told to update to the target version.") + c.toolsEnableCmd = clientToolsCmd.Command("enable", "Enables client tools auto updates. Clients will be told to update to the target version.") + c.toolsDisableCmd = clientToolsCmd.Command("disable", "Disables client tools auto updates. Clients will not be told to update to the target version.") - c.targetCmd = clientToolsCmd.Command("target", "Sets the client tools target version. This command is not supported on Teleport Cloud.") - c.targetCmd.Arg("version", "Client tools target version. Clients will be told to update to this version.").StringVar(&c.toolsTargetVersion) - c.targetCmd.Flag("clear", "removes the target version, Teleport will default to its current proxy version.").BoolVar(&c.clear) + c.toolsTargetCmd = clientToolsCmd.Command("target", "Sets the client tools target version. This command is not supported on Teleport Cloud.") + c.toolsTargetCmd.Arg("version", "Client tools target version. Clients will be told to update to this version.").StringVar(&c.toolsTargetVersion) + c.toolsTargetCmd.Flag("clear", "Removes the target version, Teleport will default to its current proxy version.").BoolVar(&c.clear) + + agentsCmd := autoUpdateCmd.Command("agents", "Manage agents auto update configuration.") + c.agentsStatusCmd = agentsCmd.Command("status", "Prints agents auto update status.") if c.stdout == nil { c.stdout = os.Stdout @@ -87,17 +94,19 @@ func (c *AutoUpdateCommand) Initialize(app *kingpin.Application, _ *servicecfg.C // TryRun takes the CLI command as an argument and executes it. func (c *AutoUpdateCommand) TryRun(ctx context.Context, cmd string, client *authclient.Client) (match bool, err error) { switch { - case cmd == c.targetCmd.FullCommand(): + case cmd == c.toolsTargetCmd.FullCommand(): err = c.TargetVersion(ctx, client) - case cmd == c.enableCmd.FullCommand(): + case cmd == c.toolsEnableCmd.FullCommand(): err = c.SetModeCommand(true)(ctx, client) - case cmd == c.disableCmd.FullCommand(): + case cmd == c.toolsDisableCmd.FullCommand(): err = c.SetModeCommand(false)(ctx, client) - case c.proxy == "" && cmd == c.statusCmd.FullCommand(): - err = c.Status(ctx, client) - case c.proxy != "" && cmd == c.statusCmd.FullCommand(): - err = c.StatusByProxy(ctx) + case c.proxy == "" && cmd == c.toolsStatusCmd.FullCommand(): + err = c.ToolsStatus(ctx, client) + case c.proxy != "" && cmd == c.toolsStatusCmd.FullCommand(): + err = c.ToolsStatusByProxy(ctx) return true, trace.Wrap(err) + case cmd == c.agentsStatusCmd.FullCommand(): + err = c.agentsStatusCommand(ctx, client) default: return false, nil } @@ -106,17 +115,17 @@ func (c *AutoUpdateCommand) TryRun(ctx context.Context, cmd string, client *auth } // TargetVersion creates or updates AutoUpdateVersion resource with client tools target version. -func (c *AutoUpdateCommand) TargetVersion(ctx context.Context, client *authclient.Client) error { +func (c *AutoUpdateCommand) TargetVersion(ctx context.Context, client autoupdateClient) error { var err error switch { case c.clear: - err = c.clearTargetVersion(ctx, client) + err = c.clearToolsTargetVersion(ctx, client) case c.toolsTargetVersion != "": // For parallel requests where we attempt to create a resource simultaneously, retries should be implemented. // The same approach applies to updates if the resource has been deleted during the process. // Second create request must return `AlreadyExists` error, update for deleted resource `NotFound` error. for i := 0; i < maxRetries; i++ { - err = c.setTargetVersion(ctx, client) + err = c.setToolsTargetVersion(ctx, client) if err == nil { break } @@ -129,13 +138,13 @@ func (c *AutoUpdateCommand) TargetVersion(ctx context.Context, client *authclien } // SetModeCommand returns a command to enable or disable client tools auto-updates in the cluster. -func (c *AutoUpdateCommand) SetModeCommand(enabled bool) func(ctx context.Context, client *authclient.Client) error { - return func(ctx context.Context, client *authclient.Client) error { +func (c *AutoUpdateCommand) SetModeCommand(enabled bool) func(ctx context.Context, client autoupdateClient) error { + return func(ctx context.Context, client autoupdateClient) error { // For parallel requests where we attempt to create a resource simultaneously, retries should be implemented. // The same approach applies to updates if the resource has been deleted during the process. // Second create request must return `AlreadyExists` error, update for deleted resource `NotFound` error. for i := 0; i < maxRetries; i++ { - err := c.setMode(ctx, client, enabled) + err := c.setToolsMode(ctx, client, enabled) if err == nil { break } @@ -153,8 +162,95 @@ type getResponse struct { TargetVersion string `json:"target_version"` } -// Status makes request to auth service to fetch client tools auto update version and mode. -func (c *AutoUpdateCommand) Status(ctx context.Context, client *authclient.Client) error { +// autoupdateClient is a subset of the Teleport client, with functions used to interact with automatic update resources. +// Not every AU function is part of the interface, we'll add them as we need. +type autoupdateClient interface { + GetAutoUpdateAgentRollout(context.Context) (*autoupdatev1pb.AutoUpdateAgentRollout, error) + GetAutoUpdateVersion(context.Context) (*autoupdatev1pb.AutoUpdateVersion, error) + GetAutoUpdateConfig(context.Context) (*autoupdatev1pb.AutoUpdateConfig, error) + CreateAutoUpdateConfig(context.Context, *autoupdatev1pb.AutoUpdateConfig) (*autoupdatev1pb.AutoUpdateConfig, error) + CreateAutoUpdateVersion(context.Context, *autoupdatev1pb.AutoUpdateVersion) (*autoupdatev1pb.AutoUpdateVersion, error) + UpdateAutoUpdateConfig(context.Context, *autoupdatev1pb.AutoUpdateConfig) (*autoupdatev1pb.AutoUpdateConfig, error) + UpdateAutoUpdateVersion(context.Context, *autoupdatev1pb.AutoUpdateVersion) (*autoupdatev1pb.AutoUpdateVersion, error) +} + +func (c *AutoUpdateCommand) agentsStatusCommand(ctx context.Context, client autoupdateClient) error { + rollout, err := client.GetAutoUpdateAgentRollout(ctx) + if err != nil && !trace.IsNotFound(err) { + return trace.Wrap(err) + } + + sb := strings.Builder{} + if rollout.GetSpec() == nil { + sb.WriteString("No active agent rollout (autoupdate_agent_rollout).\n") + } + if mode := rollout.GetSpec().GetAutoupdateMode(); mode != "" { + sb.WriteString("Agent autoupdate mode: " + mode + "\n") + } + if st := formatTimeIfNotEmpty(rollout.GetStatus().GetStartTime().AsTime(), time.DateTime); st != "" { + sb.WriteString("Rollout creation date: " + st + "\n") + } + if start := rollout.GetSpec().GetStartVersion(); start != "" { + sb.WriteString("Start version: " + start + "\n") + } + if target := rollout.GetSpec().GetTargetVersion(); target != "" { + sb.WriteString("Target version: " + target + "\n") + } + if state := rollout.GetStatus().GetState(); state != autoupdatev1pb.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_UNSPECIFIED { + sb.WriteString("Rollout state: " + userFriendlyState(state) + "\n") + } + if schedule := rollout.GetSpec().GetSchedule(); schedule == autoupdate.AgentsScheduleImmediate { + sb.WriteString("Schedule is immediate. Every group immediately updates to the target version.\n") + } + if strategy := rollout.GetSpec().GetStrategy(); strategy != "" { + sb.WriteString("Strategy: " + strategy + "\n") + } + + if groups := rollout.GetStatus().GetGroups(); len(groups) > 0 { + sb.WriteRune('\n') + headers := []string{"Group Name", "State", "Start Time", "State Reason"} + table := asciitable.MakeTable(headers) + for _, group := range groups { + table.AddRow([]string{ + group.GetName(), + userFriendlyState(group.GetState()), + formatTimeIfNotEmpty(group.GetStartTime().AsTime(), time.DateTime), + group.GetLastUpdateReason()}) + } + sb.Write(table.AsBuffer().Bytes()) + } + + fmt.Fprint(c.stdout, sb.String()) + return nil +} + +func formatTimeIfNotEmpty(t time.Time, format string) string { + if t.IsZero() || t.Unix() == 0 { + return "" + } + return t.Format(format) +} + +func userFriendlyState[T autoupdatev1pb.AutoUpdateAgentGroupState | autoupdatev1pb.AutoUpdateAgentRolloutState](state T) string { + switch state { + case 0: + return "Unknown" + case 1: + return "Unstarted" + case 2: + return "Active" + case 3: + return "Done" + case 4: + return "Rolledback" + default: + // If we don't know anything about this state, we display its integer + return fmt.Sprintf("Unknown state (%d)", state) + } +} + +// ToolsStatus makes request to auth service to fetch client tools auto update version and mode. +func (c *AutoUpdateCommand) ToolsStatus(ctx context.Context, client autoupdateClient) error { var response getResponse config, err := client.GetAutoUpdateConfig(ctx) if err != nil && !trace.IsNotFound(err) { @@ -172,12 +268,12 @@ func (c *AutoUpdateCommand) Status(ctx context.Context, client *authclient.Clien response.TargetVersion = version.Spec.Tools.TargetVersion } - return c.printResponse(response) + return c.printToolsResponse(response) } -// StatusByProxy makes request to `webapi/find` endpoint to fetch tools auto update version and mode +// ToolsStatusByProxy makes request to `webapi/find` endpoint to fetch tools auto update version and mode // without authentication. -func (c *AutoUpdateCommand) StatusByProxy(ctx context.Context) error { +func (c *AutoUpdateCommand) ToolsStatusByProxy(ctx context.Context) error { find, err := webclient.Find(&webclient.Config{ Context: ctx, ProxyAddr: c.proxy, @@ -190,13 +286,13 @@ func (c *AutoUpdateCommand) StatusByProxy(ctx context.Context) error { if find.AutoUpdate.ToolsAutoUpdate { mode = autoupdate.ToolsUpdateModeEnabled } - return c.printResponse(getResponse{ + return c.printToolsResponse(getResponse{ TargetVersion: find.AutoUpdate.ToolsVersion, Mode: mode, }) } -func (c *AutoUpdateCommand) setMode(ctx context.Context, client *authclient.Client, enabled bool) error { +func (c *AutoUpdateCommand) setToolsMode(ctx context.Context, client autoupdateClient, enabled bool) error { setMode := client.UpdateAutoUpdateConfig config, err := client.GetAutoUpdateConfig(ctx) if trace.IsNotFound(err) { @@ -224,7 +320,7 @@ func (c *AutoUpdateCommand) setMode(ctx context.Context, client *authclient.Clie return nil } -func (c *AutoUpdateCommand) setTargetVersion(ctx context.Context, client *authclient.Client) error { +func (c *AutoUpdateCommand) setToolsTargetVersion(ctx context.Context, client autoupdateClient) error { if _, err := semver.NewVersion(c.toolsTargetVersion); err != nil { return trace.WrapWithMessage(err, "not semantic version") } @@ -251,7 +347,7 @@ func (c *AutoUpdateCommand) setTargetVersion(ctx context.Context, client *authcl return nil } -func (c *AutoUpdateCommand) clearTargetVersion(ctx context.Context, client *authclient.Client) error { +func (c *AutoUpdateCommand) clearToolsTargetVersion(ctx context.Context, client autoupdateClient) error { version, err := client.GetAutoUpdateVersion(ctx) if trace.IsNotFound(err) { return nil @@ -268,7 +364,7 @@ func (c *AutoUpdateCommand) clearTargetVersion(ctx context.Context, client *auth return nil } -func (c *AutoUpdateCommand) printResponse(response getResponse) error { +func (c *AutoUpdateCommand) printToolsResponse(response getResponse) error { switch c.format { case teleport.JSON: if err := utils.WriteJSON(c.stdout, response); err != nil { diff --git a/tool/tctl/common/autoupdate_command_test.go b/tool/tctl/common/autoupdate_command_test.go index 4902593756f2e..cf649304359a6 100644 --- a/tool/tctl/common/autoupdate_command_test.go +++ b/tool/tctl/common/autoupdate_command_test.go @@ -22,12 +22,18 @@ import ( "bytes" "context" "testing" + "time" "github.com/gravitational/trace" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/gravitational/teleport/api/breaker" + autoupdatepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" + "github.com/gravitational/teleport/api/types/autoupdate" "github.com/gravitational/teleport/lib/auth/authclient" "github.com/gravitational/teleport/lib/service/servicecfg" "github.com/gravitational/teleport/lib/utils" @@ -115,3 +121,183 @@ func runAutoUpdateCommand(t *testing.T, client *authclient.Client, args []string _, err = command.TryRun(context.Background(), selectedCmd, client) return &stdoutBuff, err } + +type mockRolloutClient struct { + authclient.Client + mock.Mock +} + +func (m *mockRolloutClient) GetAutoUpdateAgentRollout(_ context.Context) (*autoupdatepb.AutoUpdateAgentRollout, error) { + args := m.Called() + return args.Get(0).(*autoupdatepb.AutoUpdateAgentRollout), args.Error(1) +} + +func TestAutoUpdateAgentStatusCommand(t *testing.T) { + ctx := context.Background() + + tests := []struct { + name string + fixture *autoupdatepb.AutoUpdateAgentRollout + fixtureErr error + expectedOutput string + }{ + { + name: "no rollout", + fixture: nil, + fixtureErr: trace.NotFound("no rollout found"), + expectedOutput: "No active agent rollout (autoupdate_agent_rollout).\n", + }, + { + name: "rollout immediate schedule", + fixture: &autoupdatepb.AutoUpdateAgentRollout{ + Spec: &autoupdatepb.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: autoupdate.AgentsScheduleImmediate, + AutoupdateMode: autoupdate.AgentsUpdateModeEnabled, + }, + }, + expectedOutput: `Agent autoupdate mode: enabled +Start version: 1.2.3 +Target version: 1.2.4 +Schedule is immediate. Every group immediately updates to the target version. +`, + }, + { + name: "rollout regular schedule time-based", + fixture: &autoupdatepb.AutoUpdateAgentRollout{ + Spec: &autoupdatepb.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: autoupdate.AgentsScheduleRegular, + AutoupdateMode: autoupdate.AgentsUpdateModeEnabled, + Strategy: autoupdate.AgentsStrategyTimeBased, + MaintenanceWindowDuration: durationpb.New(1 * time.Hour), + }, + Status: &autoupdatepb.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdatepb.AutoUpdateAgentRolloutStatusGroup{ + { + Name: "dev", + StartTime: timestamppb.New(time.Date(2025, 1, 15, 12, 00, 0, 0, time.UTC)), + State: autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + LastUpdateTime: nil, + LastUpdateReason: "outside_window", + ConfigDays: []string{"Mon", "Tue", "Wed", "Thu", "Fri"}, + ConfigStartHour: 8, + }, + { + Name: "stage", + StartTime: timestamppb.New(time.Date(2025, 1, 15, 14, 00, 0, 0, time.UTC)), + State: autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + LastUpdateReason: "in_window", + ConfigDays: []string{"Mon", "Tue", "Wed", "Thu", "Fri"}, + ConfigStartHour: 14, + }, + { + Name: "prod", + StartTime: nil, + State: autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateReason: "outside_window", + ConfigDays: []string{"Mon", "Tue", "Wed", "Thu", "Fri"}, + ConfigStartHour: 18, + }, + }, + State: autoupdatepb.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_ACTIVE, + StartTime: timestamppb.New(time.Date(2025, 1, 15, 2, 0, 0, 0, time.UTC)), + TimeOverride: nil, + }, + }, + expectedOutput: `Agent autoupdate mode: enabled +Rollout creation date: 2025-01-15 02:00:00 +Start version: 1.2.3 +Target version: 1.2.4 +Rollout state: Active +Strategy: time-based + +Group Name State Start Time State Reason +---------- --------- ------------------- -------------- +dev Done 2025-01-15 12:00:00 outside_window +stage Active 2025-01-15 14:00:00 in_window +prod Unstarted outside_window +`, + }, + { + name: "rollout regular schedule halt-on-error", + fixture: &autoupdatepb.AutoUpdateAgentRollout{ + Spec: &autoupdatepb.AutoUpdateAgentRolloutSpec{ + StartVersion: "1.2.3", + TargetVersion: "1.2.4", + Schedule: autoupdate.AgentsScheduleRegular, + AutoupdateMode: autoupdate.AgentsUpdateModeEnabled, + Strategy: autoupdate.AgentsStrategyHaltOnError, + }, + Status: &autoupdatepb.AutoUpdateAgentRolloutStatus{ + Groups: []*autoupdatepb.AutoUpdateAgentRolloutStatusGroup{ + { + Name: "dev", + StartTime: timestamppb.New(time.Date(2025, 1, 15, 12, 00, 0, 0, time.UTC)), + State: autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_DONE, + LastUpdateTime: nil, + LastUpdateReason: "outside_window", + ConfigDays: []string{"Mon", "Tue", "Wed", "Thu", "Fri"}, + ConfigStartHour: 8, + }, + { + Name: "stage", + StartTime: timestamppb.New(time.Date(2025, 1, 15, 14, 00, 0, 0, time.UTC)), + State: autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_ACTIVE, + LastUpdateReason: "in_window", + ConfigDays: []string{"Mon", "Tue", "Wed", "Thu", "Fri"}, + ConfigStartHour: 14, + }, + { + Name: "prod", + StartTime: nil, + State: autoupdatepb.AutoUpdateAgentGroupState_AUTO_UPDATE_AGENT_GROUP_STATE_UNSTARTED, + LastUpdateReason: "outside_window", + ConfigDays: []string{"Mon", "Tue", "Wed", "Thu", "Fri"}, + ConfigStartHour: 18, + }, + }, + State: autoupdatepb.AutoUpdateAgentRolloutState_AUTO_UPDATE_AGENT_ROLLOUT_STATE_ACTIVE, + StartTime: timestamppb.New(time.Date(2025, 1, 15, 2, 0, 0, 0, time.UTC)), + TimeOverride: nil, + }, + }, + expectedOutput: `Agent autoupdate mode: enabled +Rollout creation date: 2025-01-15 02:00:00 +Start version: 1.2.3 +Target version: 1.2.4 +Rollout state: Active +Strategy: halt-on-error + +Group Name State Start Time State Reason +---------- --------- ------------------- -------------- +dev Done 2025-01-15 12:00:00 outside_window +stage Active 2025-01-15 14:00:00 in_window +prod Unstarted outside_window +`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test setup: create mock client and load fixtures. + clt := &mockRolloutClient{} + clt.On("GetAutoUpdateAgentRollout", mock.Anything).Return(tt.fixture, tt.fixtureErr).Once() + + // Test execution: run command. + output := &bytes.Buffer{} + cmd := AutoUpdateCommand{stdout: output} + err := cmd.agentsStatusCommand(ctx, clt) + require.NoError(t, err) + + // Test validation: check the command output. + require.Equal(t, tt.expectedOutput, output.String()) + + // Test validation: check that the mock received the expected calls. + clt.AssertExpectations(t) + }) + } + +} diff --git a/tool/tctl/common/collection.go b/tool/tctl/common/collection.go index f92370cfd2033..fd07dca81b0cb 100644 --- a/tool/tctl/common/collection.go +++ b/tool/tctl/common/collection.go @@ -1391,7 +1391,7 @@ type autoUpdateConfigCollection struct { } func (c *autoUpdateConfigCollection) resources() []types.Resource { - return []types.Resource{types.Resource153ToLegacy(c.config)} + return []types.Resource{types.ProtoResource153ToLegacy(c.config)} } func (c *autoUpdateConfigCollection) writeText(w io.Writer, verbose bool) error { @@ -1409,7 +1409,7 @@ type autoUpdateVersionCollection struct { } func (c *autoUpdateVersionCollection) resources() []types.Resource { - return []types.Resource{types.Resource153ToLegacy(c.version)} + return []types.Resource{types.ProtoResource153ToLegacy(c.version)} } func (c *autoUpdateVersionCollection) writeText(w io.Writer, verbose bool) error { @@ -1421,3 +1421,25 @@ func (c *autoUpdateVersionCollection) writeText(w io.Writer, verbose bool) error _, err := t.AsBuffer().WriteTo(w) return trace.Wrap(err) } + +type autoUpdateAgentRolloutCollection struct { + rollout *autoupdatev1pb.AutoUpdateAgentRollout +} + +func (c *autoUpdateAgentRolloutCollection) resources() []types.Resource { + return []types.Resource{types.ProtoResource153ToLegacy(c.rollout)} +} + +func (c *autoUpdateAgentRolloutCollection) writeText(w io.Writer, verbose bool) error { + t := asciitable.MakeTable([]string{"Name", "Start Version", "Target Version", "Mode", "Schedule", "Strategy"}) + t.AddRow([]string{ + c.rollout.GetMetadata().GetName(), + fmt.Sprintf("%v", c.rollout.GetSpec().GetStartVersion()), + fmt.Sprintf("%v", c.rollout.GetSpec().GetTargetVersion()), + fmt.Sprintf("%v", c.rollout.GetSpec().GetAutoupdateMode()), + fmt.Sprintf("%v", c.rollout.GetSpec().GetSchedule()), + fmt.Sprintf("%v", c.rollout.GetSpec().GetStrategy()), + }) + _, err := t.AsBuffer().WriteTo(w) + return trace.Wrap(err) +} diff --git a/tool/tctl/common/collection_test.go b/tool/tctl/common/collection_test.go index 267e664ee3880..1cd8123477d75 100644 --- a/tool/tctl/common/collection_test.go +++ b/tool/tctl/common/collection_test.go @@ -24,10 +24,16 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/uuid" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/durationpb" + kyaml "k8s.io/apimachinery/pkg/util/yaml" "github.com/gravitational/teleport/api" + autoupdatev1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/api/types/autoupdate" "github.com/gravitational/teleport/lib/asciitable" + "github.com/gravitational/teleport/lib/defaults" + "github.com/gravitational/teleport/lib/services" "github.com/gravitational/teleport/tool/common" ) @@ -346,3 +352,66 @@ func makeTestLabels(extraStaticLabels map[string]string) map[string]string { } return labels } + +// autoUpdateConfigBrokenCollection is an intentionally broken version of the +// autoUpdateConfigCollection that is not marshaling resources properly because +// it's doing json marshaling instead of protojson marshaling. +type autoUpdateConfigBrokenCollection struct { + autoUpdateConfigCollection +} + +func (c *autoUpdateConfigBrokenCollection) resources() []types.Resource { + // We use Resource153ToLegacy instead of ProtoResource153ToLegacy. + return []types.Resource{types.Resource153ToLegacy(c.config)} +} + +// This test makes sure we marshal and unmarshal proto-based Resource153 properly. +// We had a bug where types.Resource153 implemented by protobuf structs were not +// marshaled properly (they should be marshaled using protojson). This test +// checks we can do a round-trip with one of those proto-struct resource. +func TestRoundTripProtoResource153(t *testing.T) { + // Test setup: generate fixture. + initial, err := autoupdate.NewAutoUpdateConfig(&autoupdatev1pb.AutoUpdateConfigSpec{ + Agents: &autoupdatev1pb.AutoUpdateConfigSpecAgents{ + Mode: autoupdate.AgentsUpdateModeEnabled, + Strategy: autoupdate.AgentsStrategyTimeBased, + MaintenanceWindowDuration: durationpb.New(1 * time.Hour), + Schedules: &autoupdatev1pb.AgentAutoUpdateSchedules{ + Regular: []*autoupdatev1pb.AgentAutoUpdateGroup{ + { + Name: "group1", + Days: []string{types.Wildcard}, + }, + }, + }, + }, + }) + require.NoError(t, err) + + // Test execution: dump the resource into a YAML manifest. + collection := &autoUpdateConfigCollection{config: initial} + buf := &bytes.Buffer{} + require.NoError(t, writeYAML(collection, buf)) + + // Test execution: load the YAML manifest back. + decoder := kyaml.NewYAMLOrJSONDecoder(buf, defaults.LookaheadBufSize) + var raw services.UnknownResource + require.NoError(t, decoder.Decode(&raw)) + result, err := services.UnmarshalProtoResource[*autoupdatev1pb.AutoUpdateConfig](raw.Raw) + require.NoError(t, err) + + // Test validation: check that the loaded content matches what we had before. + require.Equal(t, result, initial) + + // Test execution: now dump the resource into a YAML manifest with a + // collection using types.Resource153ToLegacy instead of types.ProtoResource153ToLegacy + brokenCollection := &autoUpdateConfigBrokenCollection{autoUpdateConfigCollection{initial}} + buf = &bytes.Buffer{} + require.NoError(t, writeYAML(brokenCollection, buf)) + + // Test execution: load the YAML manifest back and see that we can't unmarshal it. + decoder = kyaml.NewYAMLOrJSONDecoder(buf, defaults.LookaheadBufSize) + require.NoError(t, decoder.Decode(&raw)) + _, err = services.UnmarshalProtoResource[*autoupdatev1pb.AutoUpdateConfig](raw.Raw) + require.Error(t, err) +} diff --git a/tool/tctl/common/helpers_test.go b/tool/tctl/common/helpers_test.go index 5b115d8f763ab..81df51055877b 100644 --- a/tool/tctl/common/helpers_test.go +++ b/tool/tctl/common/helpers_test.go @@ -33,6 +33,7 @@ import ( "github.com/jonboulle/clockwork" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" + kyaml "k8s.io/apimachinery/pkg/util/yaml" "github.com/gravitational/teleport/api/breaker" apidefaults "github.com/gravitational/teleport/api/defaults" @@ -41,6 +42,7 @@ import ( "github.com/gravitational/teleport/lib/config" "github.com/gravitational/teleport/lib/service" "github.com/gravitational/teleport/lib/service/servicecfg" + "github.com/gravitational/teleport/lib/services" "github.com/gravitational/teleport/lib/utils" ) @@ -161,7 +163,15 @@ func mustDecodeJSON[T any](t *testing.T, r io.Reader) T { return out } +func mustTranscodeYAMLToJSON(t *testing.T, r io.Reader) []byte { + decoder := kyaml.NewYAMLToJSONDecoder(r) + var resource services.UnknownResource + require.NoError(t, decoder.Decode(&resource)) + return resource.Raw +} + func mustDecodeYAMLDocuments[T any](t *testing.T, r io.Reader, out *[]T) error { + t.Helper() decoder := yaml.NewDecoder(r) for { var entry T diff --git a/tool/tctl/common/resource_command.go b/tool/tctl/common/resource_command.go index ac7e5eeb68ffd..9ce9a1386c963 100644 --- a/tool/tctl/common/resource_command.go +++ b/tool/tctl/common/resource_command.go @@ -41,6 +41,7 @@ import ( apidefaults "github.com/gravitational/teleport/api/defaults" autoupdatev1pb "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" devicepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/devicetrust/v1" + headerv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/header/v1" loginrulepb "github.com/gravitational/teleport/api/gen/proto/go/teleport/loginrule/v1" "github.com/gravitational/teleport/api/internalutils/stream" "github.com/gravitational/teleport/api/types" @@ -144,6 +145,7 @@ func (rc *ResourceCommand) Initialize(app *kingpin.Application, config *servicec types.KindServerInfo: rc.createServerInfo, types.KindAutoUpdateConfig: rc.createAutoUpdateConfig, types.KindAutoUpdateVersion: rc.createAutoUpdateVersion, + types.KindAutoUpdateAgentRollout: rc.createAutoUpdateAgentRollout, } rc.config = config @@ -258,7 +260,7 @@ func (rc *ResourceCommand) GetMany(ctx context.Context, client *authclient.Clien } resources = append(resources, collection.resources()...) } - if err := utils.WriteYAML(os.Stdout, resources); err != nil { + if err := utils.WriteYAML(rc.stdout, resources); err != nil { return trace.Wrap(err) } return nil @@ -1147,6 +1149,7 @@ func (rc *ResourceCommand) Delete(ctx context.Context, client *authclient.Client types.KindUIConfig, types.KindAutoUpdateConfig, types.KindAutoUpdateVersion, + types.KindAutoUpdateAgentRollout, } if !slices.Contains(singletonResources, rc.ref.Kind) && (rc.ref.Kind == "" || rc.ref.Name == "") { return trace.BadParameter("provide a full resource name to delete, for example:\n$ tctl rm cluster/east\n") @@ -1506,6 +1509,11 @@ func (rc *ResourceCommand) Delete(ctx context.Context, client *authclient.Client return trace.Wrap(err) } fmt.Printf("AutoUpdateVersion has been deleted\n") + case types.KindAutoUpdateAgentRollout: + if err := client.DeleteAutoUpdateAgentRollout(ctx); err != nil { + return trace.Wrap(err) + } + fmt.Printf("AutoUpdateAgentRollout has been deleted\n") default: return trace.BadParameter("deleting resources of type %q is not supported", rc.ref.Kind) } @@ -2407,6 +2415,12 @@ func (rc *ResourceCommand) getCollection(ctx context.Context, client *authclient return nil, trace.Wrap(err) } return &autoUpdateVersionCollection{version}, nil + case types.KindAutoUpdateAgentRollout: + version, err := client.GetAutoUpdateAgentRollout(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + return &autoUpdateAgentRolloutCollection{version}, nil } return nil, trace.BadParameter("getting %q is not supported", rc.ref.String()) } @@ -2638,6 +2652,13 @@ func (rc *ResourceCommand) createAutoUpdateConfig(ctx context.Context, client *a return trace.Wrap(err) } + if config.GetMetadata() == nil { + config.Metadata = &headerv1.Metadata{} + } + if config.GetMetadata().GetName() == "" { + config.Metadata.Name = types.MetaNameAutoUpdateConfig + } + if rc.IsForced() { _, err = client.UpsertAutoUpdateConfig(ctx, config) } else { @@ -2657,6 +2678,13 @@ func (rc *ResourceCommand) createAutoUpdateVersion(ctx context.Context, client * return trace.Wrap(err) } + if version.GetMetadata() == nil { + version.Metadata = &headerv1.Metadata{} + } + if version.GetMetadata().GetName() == "" { + version.Metadata.Name = types.MetaNameAutoUpdateVersion + } + if rc.IsForced() { _, err = client.UpsertAutoUpdateVersion(ctx, version) } else { @@ -2669,3 +2697,29 @@ func (rc *ResourceCommand) createAutoUpdateVersion(ctx context.Context, client * fmt.Println("autoupdate_version has been created") return nil } + +func (rc *ResourceCommand) createAutoUpdateAgentRollout(ctx context.Context, client *authclient.Client, raw services.UnknownResource) error { + rollout, err := services.UnmarshalProtoResource[*autoupdatev1pb.AutoUpdateAgentRollout](raw.Raw) + if err != nil { + return trace.Wrap(err) + } + + if rollout.GetMetadata() == nil { + rollout.Metadata = &headerv1.Metadata{} + } + if rollout.GetMetadata().GetName() == "" { + rollout.Metadata.Name = types.MetaNameAutoUpdateAgentRollout + } + + if rc.IsForced() { + _, err = client.UpsertAutoUpdateAgentRollout(ctx, rollout) + } else { + _, err = client.CreateAutoUpdateAgentRollout(ctx, rollout) + } + if err != nil { + return trace.Wrap(err) + } + + fmt.Println("autoupdate_agent_rollout has been created") + return nil +} diff --git a/tool/tctl/common/resource_command_test.go b/tool/tctl/common/resource_command_test.go index d189c03e9c934..89e1c62f57a81 100644 --- a/tool/tctl/common/resource_command_test.go +++ b/tool/tctl/common/resource_command_test.go @@ -33,6 +33,7 @@ import ( "github.com/gravitational/trace" "github.com/jonboulle/clockwork" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/testing/protocmp" "k8s.io/apimachinery/pkg/util/yaml" @@ -1366,6 +1367,10 @@ func TestCreateResources(t *testing.T) { kind: types.KindAutoUpdateVersion, create: testCreateAutoUpdateVersion, }, + { + kind: types.KindAutoUpdateAgentRollout, + create: testCreateAutoUpdateAgentRollout, + }, } for _, test := range tests { @@ -1452,16 +1457,20 @@ version: v1 // Get the resource buf, err := runResourceCommand(t, fc, []string{"get", types.KindAutoUpdateConfig, "--format=json"}) require.NoError(t, err) - resources := mustDecodeJSON[[]*autoupdate.AutoUpdateConfig](t, buf) - require.Len(t, resources, 1) + + rawResources := mustDecodeJSON[[]services.UnknownResource](t, buf) + require.Len(t, rawResources, 1) + var resource autoupdate.AutoUpdateConfig + require.NoError(t, protojson.UnmarshalOptions{}.Unmarshal(rawResources[0].Raw, &resource)) var expected autoupdate.AutoUpdateConfig - require.NoError(t, yaml.Unmarshal([]byte(resourceYAML), &expected)) + expectedJSON := mustTranscodeYAMLToJSON(t, bytes.NewReader([]byte(resourceYAML))) + require.NoError(t, protojson.UnmarshalOptions{}.Unmarshal(expectedJSON, &expected)) require.Empty(t, cmp.Diff( - []*autoupdate.AutoUpdateConfig{&expected}, - resources, - protocmp.IgnoreFields(&headerv1.Metadata{}, "id", "revision"), + &expected, + &resource, + protocmp.IgnoreFields(&headerv1.Metadata{}, "revision", "id"), protocmp.Transform(), )) @@ -1494,16 +1503,20 @@ version: v1 // Get the resource buf, err := runResourceCommand(t, fc, []string{"get", types.KindAutoUpdateVersion, "--format=json"}) require.NoError(t, err) - resources := mustDecodeJSON[[]*autoupdate.AutoUpdateVersion](t, buf) - require.Len(t, resources, 1) + + rawResources := mustDecodeJSON[[]services.UnknownResource](t, buf) + require.Len(t, rawResources, 1) + var resource autoupdate.AutoUpdateVersion + require.NoError(t, protojson.UnmarshalOptions{}.Unmarshal(rawResources[0].Raw, &resource)) var expected autoupdate.AutoUpdateVersion - require.NoError(t, yaml.Unmarshal([]byte(resourceYAML), &expected)) + expectedJSON := mustTranscodeYAMLToJSON(t, bytes.NewReader([]byte(resourceYAML))) + require.NoError(t, protojson.UnmarshalOptions{}.Unmarshal(expectedJSON, &expected)) require.Empty(t, cmp.Diff( - []*autoupdate.AutoUpdateVersion{&expected}, - resources, - protocmp.IgnoreFields(&headerv1.Metadata{}, "id", "revision"), + &expected, + &resource, + protocmp.IgnoreFields(&headerv1.Metadata{}, "revision", "id"), protocmp.Transform(), )) @@ -1513,3 +1526,59 @@ version: v1 _, err = runResourceCommand(t, fc, []string{"get", types.KindAutoUpdateVersion}) require.ErrorContains(t, err, "autoupdate_version \"autoupdate-version\" doesn't exist") } + +func testCreateAutoUpdateAgentRollout(t *testing.T, fc *config.FileConfig) { + const resourceYAML = `kind: autoupdate_agent_rollout +metadata: + name: autoupdate-agent-rollout + revision: 3a43b44a-201e-4d7f-aef1-ae2f6d9811ed +spec: + start_version: 1.2.3 + target_version: 1.2.3 + autoupdate_mode: "suspended" + schedule: "regular" + strategy: "halt-on-error" +status: + groups: + - name: my-group + state: 1 + config_days: ["*"] + config_start_hour: 12 + config_wait_hours: 0 +version: v1 +` + _, err := runResourceCommand(t, fc, []string{"get", types.KindAutoUpdateAgentRollout, "--format=json"}) + require.ErrorContains(t, err, "doesn't exist") + + // Create the resource. + resourceYAMLPath := filepath.Join(t.TempDir(), "resource.yaml") + require.NoError(t, os.WriteFile(resourceYAMLPath, []byte(resourceYAML), 0644)) + _, err = runResourceCommand(t, fc, []string{"create", resourceYAMLPath}) + require.NoError(t, err) + + // Get the resource + buf, err := runResourceCommand(t, fc, []string{"get", types.KindAutoUpdateAgentRollout, "--format=json"}) + require.NoError(t, err) + + rawResources := mustDecodeJSON[[]services.UnknownResource](t, buf) + require.Len(t, rawResources, 1) + var resource autoupdate.AutoUpdateAgentRollout + require.NoError(t, protojson.UnmarshalOptions{}.Unmarshal(rawResources[0].Raw, &resource)) + + var expected autoupdate.AutoUpdateAgentRollout + expectedJSON := mustTranscodeYAMLToJSON(t, bytes.NewReader([]byte(resourceYAML))) + require.NoError(t, protojson.UnmarshalOptions{}.Unmarshal(expectedJSON, &expected)) + + require.Empty(t, cmp.Diff( + &expected, + &resource, + protocmp.IgnoreFields(&headerv1.Metadata{}, "revision", "id"), + protocmp.Transform(), + )) + + // Delete the resource + _, err = runResourceCommand(t, fc, []string{"rm", types.KindAutoUpdateAgentRollout}) + require.NoError(t, err) + _, err = runResourceCommand(t, fc, []string{"get", types.KindAutoUpdateAgentRollout}) + require.ErrorContains(t, err, "autoupdate_agent_rollout \"autoupdate-agent-rollout\" doesn't exist") +} diff --git a/tool/teleport-update/main.go b/tool/teleport-update/main.go new file mode 100644 index 0000000000000..2b42c4877f158 --- /dev/null +++ b/tool/teleport-update/main.go @@ -0,0 +1,492 @@ +/* + * Teleport + * Copyright (C) 2024 Gravitational, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + "time" + + "github.com/gravitational/trace" + "gopkg.in/yaml.v3" + + common "github.com/gravitational/teleport/lib/autoupdate" + autoupdate "github.com/gravitational/teleport/lib/autoupdate/agent" + "github.com/gravitational/teleport/lib/modules" + libutils "github.com/gravitational/teleport/lib/utils" +) + +const appHelp = `Teleport Updater + +The Teleport Updater applies Managed Updates to a Teleport agent installation. + +The Teleport Updater supports update scheduling and automated rollbacks. + +Find out more at https://goteleport.com/docs/upgrading/agent-managed-updates` + +const ( + // proxyServerEnvVar allows the proxy server address to be specified via env var. + proxyServerEnvVar = "TELEPORT_PROXY" + // updateGroupEnvVar allows the update group to be specified via env var. + updateGroupEnvVar = "TELEPORT_UPDATE_GROUP" + // updateVersionEnvVar forces the version to specified value. + updateVersionEnvVar = "TELEPORT_UPDATE_VERSION" + // updateLockTimeout is the duration commands will wait for update to complete before failing. + updateLockTimeout = 10 * time.Minute +) + +var ( + logLevel = slog.LevelVar{} + plog = slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ + AddSource: true, + Level: &logLevel, + })) +) + +func main() { + if code := Run(os.Args[1:]); code != 0 { + os.Exit(code) + } +} + +type cliConfig struct { + autoupdate.OverrideConfig + // Debug logs enabled + Debug bool + // InstallDir for Teleport (usually /opt/teleport) + InstallDir string + // InstallSuffix is the isolated suffix for the installation. + InstallSuffix string + // SelfSetup mode for using the current version of the teleport-update to setup the update service. + SelfSetup bool + // UpdateNow forces an immediate update. + UpdateNow bool + // Reload reloads Teleport. + Reload bool + // ForceUninstall allows Teleport to be completely removed. + ForceUninstall bool + // Insecure skips TLS certificate verification. + Insecure bool +} + +func Run(args []string) int { + var ccfg cliConfig + + ctx := context.Background() + ctx, _ = signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) + + app := libutils.InitCLIParser(autoupdate.BinaryName, appHelp).Interspersed(false) + app.Flag("debug", "Verbose logging to stdout."). + Short('d').BoolVar(&ccfg.Debug) + app.Flag("install-suffix", "Suffix for creating an agent installation outside of the default $PATH. Note: this changes the default data directory."). + Short('i').StringVar(&ccfg.InstallSuffix) + app.Flag("install-dir", "Directory containing Teleport installations."). + Hidden().StringVar(&ccfg.InstallDir) + app.Flag("insecure", "Insecure mode disables certificate verification. Do not use in production."). + BoolVar(&ccfg.Insecure) + + app.HelpFlag.Short('h') + + versionCmd := app.Command("version", fmt.Sprintf("Print the version of your %s binary.", autoupdate.BinaryName)) + + enableCmd := app.Command("enable", "Enable agent managed updates and perform initial installation or update. This creates a systemd timer that periodically runs the update subcommand.") + enableCmd.Flag("proxy", "Address of the Teleport Proxy."). + Short('p').Envar(proxyServerEnvVar).StringVar(&ccfg.Proxy) + enableCmd.Flag("group", "Update group for this agent installation."). + Short('g').Envar(updateGroupEnvVar).StringVar(&ccfg.Group) + enableCmd.Flag("base-url", "Base URL used to override the Teleport download URL."). + Short('b').Envar(common.BaseURLEnvVar).StringVar(&ccfg.BaseURL) + enableCmd.Flag("overwrite", "Allow existing installed Teleport binaries to be overwritten."). + Short('o').BoolVar(&ccfg.AllowOverwrite) + enableCmd.Flag("allow-proxy-conflict", "Allow proxy addresses in teleport.yaml and update.yaml to conflict."). + BoolVar(&ccfg.AllowProxyConflict) + enableCmd.Flag("force-version", "Force the provided version instead of using the version provided by the Teleport cluster."). + Hidden().Short('f').Envar(updateVersionEnvVar).StringVar(&ccfg.ForceVersion) + enableCmd.Flag("force-flag", "Force the provided version flags instead of using the version flags provided by the Teleport cluster."). + Hidden().StringsVar(&ccfg.ForceFlags) + enableCmd.Flag("self-setup", "Use the current teleport-update binary to create systemd service config for managed updates."). + Hidden().BoolVar(&ccfg.SelfSetup) + enableCmd.Flag("path", "Directory to link the active Teleport installation's binaries into."). + Hidden().StringVar(&ccfg.Path) + + disableCmd := app.Command("disable", "Disable agent managed updates. Does not affect the active installation of Teleport.") + + pinCmd := app.Command("pin", "Install Teleport and lock the updater to the installed version.") + pinCmd.Flag("proxy", "Address of the Teleport Proxy."). + Short('p').Envar(proxyServerEnvVar).StringVar(&ccfg.Proxy) + pinCmd.Flag("group", "Update group for this agent installation."). + Short('g').Envar(updateGroupEnvVar).StringVar(&ccfg.Group) + pinCmd.Flag("base-url", "Base URL used to override the Teleport download URL."). + Short('b').Envar(common.BaseURLEnvVar).StringVar(&ccfg.BaseURL) + pinCmd.Flag("overwrite", "Allow existing installed Teleport binaries to be overwritten."). + Short('o').BoolVar(&ccfg.AllowOverwrite) + pinCmd.Flag("allow-proxy-conflict", "Allow proxy addresses in teleport.yaml and update.yaml to conflict."). + BoolVar(&ccfg.AllowProxyConflict) + pinCmd.Flag("force-version", "Force the provided version instead of using the version provided by the Teleport cluster."). + Short('f').Envar(updateVersionEnvVar).StringVar(&ccfg.ForceVersion) + pinCmd.Flag("force-flag", "Force the provided version flags instead of using the version flags provided by the Teleport cluster."). + Hidden().StringsVar(&ccfg.ForceFlags) + pinCmd.Flag("self-setup", "Use the current teleport-update binary to create systemd service config for managed updates."). + Hidden().BoolVar(&ccfg.SelfSetup) + pinCmd.Flag("path", "Directory to link the active Teleport installation's binaries into."). + Hidden().StringVar(&ccfg.Path) + + unpinCmd := app.Command("unpin", "Unpin the current version, allowing it to be updated.") + + updateCmd := app.Command("update", "Update the agent to the latest version, if a new version is available.") + updateCmd.Flag("now", "Force immediate update even if update window is not active."). + Short('n').BoolVar(&ccfg.UpdateNow) + updateCmd.Flag("self-setup", "Use the current teleport-update binary to create systemd service config for managed updates and verify the Teleport installation."). + Hidden().BoolVar(&ccfg.SelfSetup) + + linkCmd := app.Command("link-package", "Link the system installation of Teleport from the Teleport package, if managed updates is disabled.") + unlinkCmd := app.Command("unlink-package", "Unlink the system installation of Teleport from the Teleport package.") + + setupCmd := app.Command("setup", "Write configuration files that run the update subcommand on a timer and verify the Teleport installation."). + Hidden() + setupCmd.Flag("reload", "Reload the Teleport agent. If not set, Teleport is not reloaded or restarted."). + BoolVar(&ccfg.Reload) + setupCmd.Flag("path", "Directory that the active Teleport installation's binaries are linked into."). + Required().StringVar(&ccfg.Path) + + statusCmd := app.Command("status", "Show Teleport agent auto-update status.") + + uninstallCmd := app.Command("uninstall", "Uninstall the updater-managed installation of Teleport. If the Teleport package is installed, it is restored as the primary installation.") + uninstallCmd.Flag("force", "Force complete uninstallation of Teleport, even if there is no packaged version of Teleport to revert to."). + Short('f').BoolVar(&ccfg.ForceUninstall) + + libutils.UpdateAppUsageTemplate(app, args) + command, err := app.Parse(args) + if err != nil { + app.Usage(args) + libutils.FatalError(err) + } + + // Logging must be configured as early as possible to ensure all log + // message are formatted correctly. + if err := setupLogger(ccfg.Debug); err != nil { + plog.ErrorContext(ctx, "Failed to set up logger.", "error", err) + return 1 + } + + switch command { + case statusCmd.FullCommand(), versionCmd.FullCommand(): + default: + if os.Geteuid() != 0 { + plog.ErrorContext(ctx, "This command must be run as root. Try running with sudo.") + return 1 + } + // Set required umask for commands that write files to system directories as root, and warn loudly if it changes. + autoupdate.SetRequiredUmask(ctx, plog) + } + + switch command { + case enableCmd.FullCommand(): + ccfg.Enabled = true + err = cmdInstall(ctx, &ccfg) + case pinCmd.FullCommand(): + ccfg.Pinned = true + err = cmdInstall(ctx, &ccfg) + case disableCmd.FullCommand(): + err = cmdDisable(ctx, &ccfg) + case unpinCmd.FullCommand(): + err = cmdUnpin(ctx, &ccfg) + case updateCmd.FullCommand(): + err = cmdUpdate(ctx, &ccfg) + case linkCmd.FullCommand(): + err = cmdLinkPackage(ctx, &ccfg) + case unlinkCmd.FullCommand(): + err = cmdUnlinkPackage(ctx, &ccfg) + case setupCmd.FullCommand(): + err = cmdSetup(ctx, &ccfg) + case uninstallCmd.FullCommand(): + err = cmdUninstall(ctx, &ccfg) + case versionCmd.FullCommand(): + modules.GetModules().PrintVersion() + case statusCmd.FullCommand(): + err = cmdStatus(ctx, &ccfg) + if errors.Is(err, autoupdate.ErrNotInstalled) { + plog.ErrorContext(ctx, "Teleport is not installed by teleport-update with this suffix.") + return 1 + } + default: + // This should only happen when there's a missing switch case above. + err = trace.Errorf("command %s not configured", command) + } + if err != nil { + plog.ErrorContext(ctx, "Command failed.", "error", err) + return 1 + } + return 0 +} + +func setupLogger(debug bool) error { + level := slog.LevelInfo + if debug { + level = slog.LevelDebug + } + + logLevel.Set(level) + return nil +} + +func initConfig(ctx context.Context, ccfg *cliConfig) (updater *autoupdate.Updater, lockFile string, err error) { + ns, err := autoupdate.NewNamespace(ctx, plog, ccfg.InstallSuffix, ccfg.InstallDir) + if err != nil { + return nil, "", trace.Wrap(err) + } + lockFile, err = ns.Init() + if err != nil { + return nil, "", trace.Wrap(err) + } + updater, err = autoupdate.NewLocalUpdater(autoupdate.LocalUpdaterConfig{ + SelfSetup: ccfg.SelfSetup, + Log: plog, + Debug: ccfg.Debug, + InsecureSkipVerify: ccfg.Insecure, + }, ns) + return updater, lockFile, trace.Wrap(err) +} + +func statusConfig(ctx context.Context, ccfg *cliConfig) (*autoupdate.Updater, error) { + ns, err := autoupdate.NewNamespace(ctx, plog, ccfg.InstallSuffix, ccfg.InstallDir) + if err != nil { + return nil, trace.Wrap(err) + } + updater, err := autoupdate.NewLocalUpdater(autoupdate.LocalUpdaterConfig{ + SelfSetup: ccfg.SelfSetup, + Log: plog, + Debug: ccfg.Debug, + InsecureSkipVerify: ccfg.Insecure, + }, ns) + return updater, trace.Wrap(err) +} + +// cmdDisable disables updates. +func cmdDisable(ctx context.Context, ccfg *cliConfig) error { + updater, lockFile, err := initConfig(ctx, ccfg) + if err != nil { + return trace.Wrap(err, "failed to initialize updater") + } + unlock, err := libutils.FSTryWriteLockTimeout(ctx, lockFile, updateLockTimeout) + if err != nil { + return trace.Wrap(err, "failed to grab concurrent execution lock %s", lockFile) + } + defer func() { + if err := unlock(); err != nil { + plog.DebugContext(ctx, "Failed to close lock file", "error", err) + } + }() + if err := updater.Disable(ctx); err != nil { + return trace.Wrap(err) + } + return nil +} + +// cmdUnpin unpins the current version. +func cmdUnpin(ctx context.Context, ccfg *cliConfig) error { + updater, lockFile, err := initConfig(ctx, ccfg) + if err != nil { + return trace.Wrap(err, "failed to setup updater") + } + unlock, err := libutils.FSTryWriteLockTimeout(ctx, lockFile, updateLockTimeout) + if err != nil { + return trace.Wrap(err, "failed to grab concurrent execution lock %n", lockFile) + } + defer func() { + if err := unlock(); err != nil { + plog.DebugContext(ctx, "Failed to close lock file", "error", err) + } + }() + if err := updater.Unpin(ctx); err != nil { + return trace.Wrap(err) + } + return nil +} + +// cmdInstall installs Teleport and sets configuration. +func cmdInstall(ctx context.Context, ccfg *cliConfig) error { + updater, lockFile, err := initConfig(ctx, ccfg) + if err != nil { + return trace.Wrap(err, "failed to initialize updater") + } + + // Ensure enable can't run concurrently. + unlock, err := libutils.FSTryWriteLockTimeout(ctx, lockFile, updateLockTimeout) + if err != nil { + return trace.Wrap(err, "failed to grab concurrent execution lock %s", lockFile) + } + defer func() { + if err := unlock(); err != nil { + plog.DebugContext(ctx, "Failed to close lock file", "error", err) + } + }() + if err := updater.Install(ctx, ccfg.OverrideConfig); err != nil { + return trace.Wrap(err) + } + return nil +} + +// cmdUpdate updates Teleport to the version specified by cluster reachable at the proxy address. +func cmdUpdate(ctx context.Context, ccfg *cliConfig) error { + updater, lockFile, err := initConfig(ctx, ccfg) + if err != nil { + return trace.Wrap(err, "failed to initialize updater") + } + // Ensure update can't run concurrently. + var unlock func() error + if ccfg.UpdateNow { + unlock, err = libutils.FSTryWriteLockTimeout(ctx, lockFile, updateLockTimeout) + } else { + unlock, err = libutils.FSWriteLock(lockFile) + } + if err != nil { + return trace.Wrap(err, "failed to grab concurrent execution lock %s", lockFile) + } + defer func() { + if err := unlock(); err != nil { + plog.DebugContext(ctx, "Failed to close lock file", "error", err) + } + }() + + if err := updater.Update(ctx, ccfg.UpdateNow); err != nil { + return trace.Wrap(err) + } + return nil +} + +// cmdLinkPackage creates system package links if no version is linked and managed updates is disabled. +func cmdLinkPackage(ctx context.Context, ccfg *cliConfig) error { + updater, lockFile, err := initConfig(ctx, ccfg) + if err != nil { + return trace.Wrap(err, "failed to initialize updater") + } + + // Skip operation and warn if the updater is currently running. + unlock, err := libutils.FSTryReadLock(lockFile) + if errors.Is(err, libutils.ErrUnsuccessfulLockTry) { + plog.WarnContext(ctx, "Updater is currently running. Skipping package linking.") + return nil + } + if err != nil { + return trace.Wrap(err, "failed to grab concurrent execution lock %q", lockFile) + } + defer func() { + if err := unlock(); err != nil { + plog.DebugContext(ctx, "Failed to close lock file", "error", err) + } + }() + + if err := updater.LinkPackage(ctx); err != nil { + return trace.Wrap(err) + } + return nil +} + +// cmdUnlinkPackage remove system package links. +func cmdUnlinkPackage(ctx context.Context, ccfg *cliConfig) error { + updater, lockFile, err := initConfig(ctx, ccfg) + if err != nil { + return trace.Wrap(err, "failed to setup updater") + } + + // Error if the updater is running. We could remove its links by accident. + unlock, err := libutils.FSTryWriteLock(lockFile) + if errors.Is(err, libutils.ErrUnsuccessfulLockTry) { + plog.WarnContext(ctx, "Updater is currently running. Skipping package unlinking.") + return nil + } + if err != nil { + return trace.Wrap(err, "failed to grab concurrent execution lock %q", lockFile) + } + defer func() { + if err := unlock(); err != nil { + plog.DebugContext(ctx, "Failed to close lock file", "error", err) + } + }() + + if err := updater.UnlinkPackage(ctx); err != nil { + return trace.Wrap(err) + } + return nil +} + +// cmdSetup writes configuration files that are needed to run teleport-update update. +func cmdSetup(ctx context.Context, ccfg *cliConfig) error { + ns, err := autoupdate.NewNamespace(ctx, plog, ccfg.InstallSuffix, ccfg.InstallDir) + if err != nil { + return trace.Wrap(err) + } + updater, err := autoupdate.NewLocalUpdater(autoupdate.LocalUpdaterConfig{ + SelfSetup: ccfg.SelfSetup, + Log: plog, + Debug: ccfg.Debug, + InsecureSkipVerify: ccfg.Insecure, + }, ns) + if err != nil { + return trace.Wrap(err) + } + err = updater.Setup(ctx, ccfg.Path, ccfg.Reload) + if err != nil { + return trace.Wrap(err) + } + return nil +} + +// cmdStatus displays auto-update status. +func cmdStatus(ctx context.Context, ccfg *cliConfig) error { + updater, err := statusConfig(ctx, ccfg) + if err != nil { + return trace.Wrap(err, "failed to initialize updater") + } + status, err := updater.Status(ctx) + if err != nil { + return trace.Wrap(err, "failed to get status") + } + enc := yaml.NewEncoder(os.Stdout) + return trace.Wrap(enc.Encode(status)) +} + +// cmdUninstall removes the updater-managed install of Teleport and gracefully reverts back to the Teleport package. +func cmdUninstall(ctx context.Context, ccfg *cliConfig) error { + updater, lockFile, err := initConfig(ctx, ccfg) + if err != nil { + return trace.Wrap(err, "failed to initialize updater") + } + // Ensure update can't run concurrently. + unlock, err := libutils.FSTryWriteLockTimeout(ctx, lockFile, updateLockTimeout) + if err != nil { + return trace.Wrap(err, "failed to grab concurrent execution lock %s", lockFile) + } + defer func() { + if err := unlock(); err != nil { + plog.DebugContext(ctx, "Failed to close lock file", "error", err) + } + }() + + if err := updater.Remove(ctx, ccfg.ForceUninstall); err != nil { + return trace.Wrap(err) + } + return nil +} diff --git a/web/packages/teleport/src/Audit/EventList/EventTypeCell.tsx b/web/packages/teleport/src/Audit/EventList/EventTypeCell.tsx index 12aa2d6b9baba..6ed201e3b0309 100644 --- a/web/packages/teleport/src/Audit/EventList/EventTypeCell.tsx +++ b/web/packages/teleport/src/Audit/EventList/EventTypeCell.tsx @@ -255,6 +255,12 @@ const EventIconMap: Record = { [eventCodes.INTEGRATION_UPDATE]: Icons.Info, [eventCodes.INTEGRATION_DELETE]: Icons.Info, [eventCodes.UNKNOWN]: Icons.Question, + [eventCodes.AUTOUPDATE_CONFIG_CREATE]: Icons.Info, + [eventCodes.AUTOUPDATE_CONFIG_UPDATE]: Icons.Info, + [eventCodes.AUTOUPDATE_CONFIG_DELETE]: Icons.Info, + [eventCodes.AUTOUPDATE_VERSION_CREATE]: Icons.Info, + [eventCodes.AUTOUPDATE_VERSION_UPDATE]: Icons.Info, + [eventCodes.AUTOUPDATE_VERSION_DELETE]: Icons.Info, }; export default function renderTypeCell(event: Event, clusterId: string) { diff --git a/web/packages/teleport/src/Audit/__snapshots__/Audit.story.test.tsx.snap b/web/packages/teleport/src/Audit/__snapshots__/Audit.story.test.tsx.snap index be45fbe4fb50d..f6fa499cf632d 100644 --- a/web/packages/teleport/src/Audit/__snapshots__/Audit.story.test.tsx.snap +++ b/web/packages/teleport/src/Audit/__snapshots__/Audit.story.test.tsx.snap @@ -429,12 +429,12 @@ exports[`list of all events 1`] = ` - - 222 + 228 of - 222 + 228 + + + + +
+ + + + + + + + Automatic Update Config Created +
+ + + User b6eae9ed-bfde-40ba-a880-948a2c598b2b.autest.cloud.gravitational.io created the Automatic Update Config + + + 2025-03-04T15:49:31.946Z + + + + + + + +
+ + + + + + + + Automatic Update Config Deleted +
+ + + User b6eae9ed-bfde-40ba-a880-948a2c598b2b.autest.cloud.gravitational.io deleted the Automatic Update Config + + + 2025-03-04T15:49:21.869Z + + + + + + + +
+ + + + + + + + Automatic Update Version Created +
+ + + User b6eae9ed-bfde-40ba-a880-948a2c598b2b.autest.cloud.gravitational.io created the Automatic Update Version + + + 2025-03-04T15:41:24.433Z + + + + + + + +
+ + + + + + + + Automatic Update Version Updated +
+ + + User b6eae9ed-bfde-40ba-a880-948a2c598b2b.autest.cloud.gravitational.io updated the Automatic Update Version + + + 2025-03-04T15:27:36.039Z + + + + + + + +
+ + + + + + + + Automatic Update Version Deleted +
+ + + User b6eae9ed-bfde-40ba-a880-948a2c598b2b.autest.cloud.gravitational.io deleted the Automatic Update Version + + + 2025-03-04T15:25:44.805Z + + + + + `Unknown '${unknown_type}' event (${unknown_code})`, }, + [eventCodes.AUTOUPDATE_CONFIG_CREATE]: { + type: 'auto_update_config.create', + desc: 'Automatic Update Config Created', + format: ({ user }) => { + return `User ${user} created the Automatic Update Config`; + }, + }, + [eventCodes.AUTOUPDATE_CONFIG_UPDATE]: { + type: 'auto_update_config.update', + desc: 'Automatic Update Config Updated', + format: ({ user }) => { + return `User ${user} updated the Automatic Update Config`; + }, + }, + [eventCodes.AUTOUPDATE_CONFIG_DELETE]: { + type: 'auto_update_config.delete', + desc: 'Automatic Update Config Deleted', + format: ({ user }) => { + return `User ${user} deleted the Automatic Update Config`; + }, + }, + [eventCodes.AUTOUPDATE_VERSION_CREATE]: { + type: 'auto_update_version.create', + desc: 'Automatic Update Version Created', + format: ({ user }) => { + return `User ${user} created the Automatic Update Version`; + }, + }, + [eventCodes.AUTOUPDATE_VERSION_UPDATE]: { + type: 'auto_update_version.update', + desc: 'Automatic Update Version Updated', + format: ({ user }) => { + return `User ${user} updated the Automatic Update Version`; + }, + }, + [eventCodes.AUTOUPDATE_VERSION_DELETE]: { + type: 'auto_update_version.delete', + desc: 'Automatic Update Version Deleted', + format: ({ user }) => { + return `User ${user} deleted the Automatic Update Version`; + }, + }, }; const unknownFormatter = { diff --git a/web/packages/teleport/src/services/audit/types.ts b/web/packages/teleport/src/services/audit/types.ts index c6ffb2f3a5e98..e2623ccd26885 100644 --- a/web/packages/teleport/src/services/audit/types.ts +++ b/web/packages/teleport/src/services/audit/types.ts @@ -272,6 +272,12 @@ export const eventCodes = { INTEGRATION_CREATE: 'IG001I', INTEGRATION_UPDATE: 'IG002I', INTEGRATION_DELETE: 'IG003I', + AUTOUPDATE_CONFIG_CREATE: 'AUC001I', + AUTOUPDATE_CONFIG_UPDATE: 'AUC002I', + AUTOUPDATE_CONFIG_DELETE: 'AUC003I', + AUTOUPDATE_VERSION_CREATE: 'AUV001I', + AUTOUPDATE_VERSION_UPDATE: 'AUV002I', + AUTOUPDATE_VERSION_DELETE: 'AUV003I', } as const; /** @@ -1512,6 +1518,42 @@ export type RawEvents = { typeof eventCodes.INTEGRATION_DELETE, HasName >; + [eventCodes.AUTOUPDATE_CONFIG_CREATE]: RawEvent< + typeof eventCodes.AUTOUPDATE_CONFIG_CREATE, + { + user: string; + } + >; + [eventCodes.AUTOUPDATE_CONFIG_UPDATE]: RawEvent< + typeof eventCodes.AUTOUPDATE_CONFIG_UPDATE, + { + user: string; + } + >; + [eventCodes.AUTOUPDATE_CONFIG_DELETE]: RawEvent< + typeof eventCodes.AUTOUPDATE_CONFIG_DELETE, + { + user: string; + } + >; + [eventCodes.AUTOUPDATE_VERSION_CREATE]: RawEvent< + typeof eventCodes.AUTOUPDATE_VERSION_CREATE, + { + user: string; + } + >; + [eventCodes.AUTOUPDATE_VERSION_UPDATE]: RawEvent< + typeof eventCodes.AUTOUPDATE_VERSION_UPDATE, + { + user: string; + } + >; + [eventCodes.AUTOUPDATE_VERSION_DELETE]: RawEvent< + typeof eventCodes.AUTOUPDATE_VERSION_DELETE, + { + user: string; + } + >; }; /**