From afae487f306e72f472ef25f0931602ff38e6bb3c Mon Sep 17 00:00:00 2001 From: Hugo Shaka Date: Tue, 31 Dec 2024 10:25:28 -0500 Subject: [PATCH] Reduce clock usage + add time and period override in rollout controller (#50634) --- .../teleport/autoupdate/v1/autoupdate.pb.go | 165 ++++++++++-------- .../teleport/autoupdate/v1/autoupdate.proto | 6 + lib/autoupdate/rollout/controller.go | 1 + lib/autoupdate/rollout/reconciler.go | 18 +- lib/autoupdate/rollout/reconciler_test.go | 2 +- lib/autoupdate/rollout/strategy.go | 2 +- .../rollout/strategy_haltonerror.go | 15 +- .../rollout/strategy_haltonerror_test.go | 4 +- lib/autoupdate/rollout/strategy_timebased.go | 16 +- .../rollout/strategy_timebased_test.go | 4 +- lib/config/configuration.go | 8 + 11 files changed, 136 insertions(+), 105 deletions(-) diff --git a/api/gen/proto/go/teleport/autoupdate/v1/autoupdate.pb.go b/api/gen/proto/go/teleport/autoupdate/v1/autoupdate.pb.go index e984e2ff98295..ebf89e4f8e971 100644 --- a/api/gen/proto/go/teleport/autoupdate/v1/autoupdate.pb.go +++ b/api/gen/proto/go/teleport/autoupdate/v1/autoupdate.pb.go @@ -986,6 +986,7 @@ func (x *AutoUpdateAgentRolloutSpec) GetMaintenanceWindowDuration() *durationpb. } // AutoUpdateAgentRolloutStatus tracks the current agent rollout status. +// The status is reset if any spec field changes except the mode. type AutoUpdateAgentRolloutStatus struct { state protoimpl.MessageState `protogen:"open.v1"` Groups []*AutoUpdateAgentRolloutStatusGroup `protobuf:"bytes,1,rep,name=groups,proto3" json:"groups,omitempty"` @@ -998,7 +999,11 @@ type AutoUpdateAgentRolloutStatus struct { // will not start updating to the new version directly. The controller sees that the group theoretical start time is // before the rollout start time and the maintenance window belongs to the previous rollout. // When the timestamp is nil, the controller will ignore the start time and check and allow groups to activate. - StartTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` + StartTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` + // Time override is an optional timestamp making the autoupdate_agent_rollout controller use a specific time instead + // of the system clock when evaluating time-based criteria. This field is used for testing and troubleshooting + // purposes. + TimeOverride *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=time_override,json=timeOverride,proto3" json:"time_override,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1054,6 +1059,13 @@ func (x *AutoUpdateAgentRolloutStatus) GetStartTime() *timestamppb.Timestamp { return nil } +func (x *AutoUpdateAgentRolloutStatus) GetTimeOverride() *timestamppb.Timestamp { + if x != nil { + return x.TimeOverride + } + return nil +} + // AutoUpdateAgentRolloutStatusGroup tracks the current agent rollout status of a specific group. type AutoUpdateAgentRolloutStatusGroup struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1316,7 +1328,7 @@ var file_teleport_autoupdate_v1_autoupdate_proto_rawDesc = []byte{ 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x19, 0x6d, 0x61, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x63, 0x65, 0x57, 0x69, 0x6e, 0x64, 0x6f, - 0x77, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xf7, 0x01, 0x0a, 0x1c, 0x41, 0x75, + 0x77, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xb8, 0x02, 0x0a, 0x1c, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x51, 0x0a, 0x06, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x74, 0x65, 0x6c, @@ -1332,73 +1344,77 @@ var file_teleport_autoupdate_v1_autoupdate_proto_rawDesc = []byte{ 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, - 0x69, 0x6d, 0x65, 0x22, 0xc0, 0x03, 0x0a, 0x21, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x39, 0x0a, - 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x47, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, - 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x31, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, - 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, - 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x44, 0x0a, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x6c, 0x61, 0x73, 0x74, 0x5f, - 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, - 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, - 0x64, 0x61, 0x79, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x44, 0x61, 0x79, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x68, 0x6f, 0x75, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x72, 0x74, 0x48, 0x6f, - 0x75, 0x72, 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x77, 0x61, 0x69, - 0x74, 0x5f, 0x68, 0x6f, 0x75, 0x72, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x57, 0x61, 0x69, 0x74, 0x48, 0x6f, 0x75, 0x72, 0x73, 0x4a, 0x04, - 0x08, 0x08, 0x10, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x77, 0x61, 0x69, - 0x74, 0x5f, 0x64, 0x61, 0x79, 0x73, 0x2a, 0xf7, 0x01, 0x0a, 0x19, 0x41, 0x75, 0x74, 0x6f, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x12, 0x2d, 0x0a, 0x29, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, + 0x69, 0x6d, 0x65, 0x12, 0x3f, 0x0a, 0x0d, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x4f, 0x76, 0x65, 0x72, + 0x72, 0x69, 0x64, 0x65, 0x22, 0xc0, 0x03, 0x0a, 0x21, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x39, + 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x47, 0x0a, 0x05, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x31, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x70, + 0x6f, 0x72, 0x74, 0x2e, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, + 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x12, 0x44, 0x0a, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x6c, 0x61, 0x73, 0x74, + 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x5f, 0x64, 0x61, 0x79, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x44, 0x61, 0x79, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x68, 0x6f, 0x75, 0x72, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x72, 0x74, 0x48, + 0x6f, 0x75, 0x72, 0x12, 0x2a, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x77, 0x61, + 0x69, 0x74, 0x5f, 0x68, 0x6f, 0x75, 0x72, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x57, 0x61, 0x69, 0x74, 0x48, 0x6f, 0x75, 0x72, 0x73, 0x4a, + 0x04, 0x08, 0x08, 0x10, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x77, 0x61, + 0x69, 0x74, 0x5f, 0x64, 0x61, 0x79, 0x73, 0x2a, 0xf7, 0x01, 0x0a, 0x19, 0x41, 0x75, 0x74, 0x6f, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x2d, 0x0a, 0x29, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, + 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, + 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x2b, 0x0a, 0x27, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, - 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, - 0x44, 0x10, 0x00, 0x12, 0x2b, 0x0a, 0x27, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, + 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x54, 0x41, 0x52, 0x54, 0x45, 0x44, 0x10, + 0x01, 0x12, 0x28, 0x0a, 0x24, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, + 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, 0x54, 0x41, + 0x54, 0x45, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x26, 0x0a, 0x22, 0x41, + 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, + 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x4f, 0x4e, + 0x45, 0x10, 0x03, 0x12, 0x2c, 0x0a, 0x28, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, - 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x54, 0x41, 0x52, 0x54, 0x45, 0x44, 0x10, 0x01, - 0x12, 0x28, 0x0a, 0x24, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, - 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, 0x54, 0x41, 0x54, - 0x45, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x26, 0x0a, 0x22, 0x41, 0x55, - 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, - 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x44, 0x4f, 0x4e, 0x45, - 0x10, 0x03, 0x12, 0x2c, 0x0a, 0x28, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, - 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x5f, 0x53, 0x54, - 0x41, 0x54, 0x45, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x45, 0x44, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x04, - 0x2a, 0x83, 0x02, 0x0a, 0x1b, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x41, - 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x2f, 0x0a, 0x2b, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, - 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x4f, 0x55, 0x54, 0x5f, 0x53, 0x54, - 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, - 0x00, 0x12, 0x2d, 0x0a, 0x29, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x45, 0x44, 0x42, 0x41, 0x43, 0x4b, 0x10, + 0x04, 0x2a, 0x83, 0x02, 0x0a, 0x1b, 0x41, 0x75, 0x74, 0x6f, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x6f, 0x6c, 0x6c, 0x6f, 0x75, 0x74, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x12, 0x2f, 0x0a, 0x2b, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, + 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x4f, 0x55, 0x54, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x2d, 0x0a, 0x29, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, + 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x4f, 0x55, 0x54, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x54, 0x41, 0x52, 0x54, 0x45, 0x44, 0x10, + 0x01, 0x12, 0x2a, 0x0a, 0x26, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x4f, 0x55, 0x54, 0x5f, 0x53, - 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x54, 0x41, 0x52, 0x54, 0x45, 0x44, 0x10, 0x01, - 0x12, 0x2a, 0x0a, 0x26, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, - 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x4f, 0x55, 0x54, 0x5f, 0x53, 0x54, - 0x41, 0x54, 0x45, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x28, 0x0a, 0x24, - 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, - 0x54, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x4f, 0x55, 0x54, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, - 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x03, 0x12, 0x2e, 0x0a, 0x2a, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, - 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, - 0x4f, 0x55, 0x54, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x45, 0x44, - 0x42, 0x41, 0x43, 0x4b, 0x10, 0x04, 0x42, 0x56, 0x5a, 0x54, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72, 0x61, 0x76, 0x69, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x61, 0x6c, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x70, 0x69, 0x2f, - 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x74, 0x65, 0x6c, - 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x43, 0x54, 0x49, 0x56, 0x45, 0x10, 0x02, 0x12, 0x28, 0x0a, + 0x24, 0x41, 0x55, 0x54, 0x4f, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, + 0x4e, 0x54, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x4f, 0x55, 0x54, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, + 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x03, 0x12, 0x2e, 0x0a, 0x2a, 0x41, 0x55, 0x54, 0x4f, 0x5f, + 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x52, 0x4f, 0x4c, + 0x4c, 0x4f, 0x55, 0x54, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x52, 0x4f, 0x4c, 0x4c, 0x45, + 0x44, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x04, 0x42, 0x56, 0x5a, 0x54, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72, 0x61, 0x76, 0x69, 0x74, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x61, 0x6c, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x70, 0x69, + 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x74, 0x65, + 0x6c, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2f, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x75, 0x74, 0x6f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1455,14 +1471,15 @@ var file_teleport_autoupdate_v1_autoupdate_proto_depIdxs = []int32{ 15, // 15: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatus.groups:type_name -> teleport.autoupdate.v1.AutoUpdateAgentRolloutStatusGroup 1, // 16: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatus.state:type_name -> teleport.autoupdate.v1.AutoUpdateAgentRolloutState 18, // 17: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatus.start_time:type_name -> google.protobuf.Timestamp - 18, // 18: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatusGroup.start_time:type_name -> google.protobuf.Timestamp - 0, // 19: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatusGroup.state:type_name -> teleport.autoupdate.v1.AutoUpdateAgentGroupState - 18, // 20: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatusGroup.last_update_time:type_name -> google.protobuf.Timestamp - 21, // [21:21] is the sub-list for method output_type - 21, // [21:21] is the sub-list for method input_type - 21, // [21:21] is the sub-list for extension type_name - 21, // [21:21] is the sub-list for extension extendee - 0, // [0:21] is the sub-list for field type_name + 18, // 18: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatus.time_override:type_name -> google.protobuf.Timestamp + 18, // 19: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatusGroup.start_time:type_name -> google.protobuf.Timestamp + 0, // 20: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatusGroup.state:type_name -> teleport.autoupdate.v1.AutoUpdateAgentGroupState + 18, // 21: teleport.autoupdate.v1.AutoUpdateAgentRolloutStatusGroup.last_update_time:type_name -> google.protobuf.Timestamp + 22, // [22:22] is the sub-list for method output_type + 22, // [22:22] is the sub-list for method input_type + 22, // [22:22] is the sub-list for extension type_name + 22, // [22:22] is the sub-list for extension extendee + 0, // [0:22] is the sub-list for field type_name } func init() { file_teleport_autoupdate_v1_autoupdate_proto_init() } diff --git a/api/proto/teleport/autoupdate/v1/autoupdate.proto b/api/proto/teleport/autoupdate/v1/autoupdate.proto index faf7ec93db129..73f6d440f998e 100644 --- a/api/proto/teleport/autoupdate/v1/autoupdate.proto +++ b/api/proto/teleport/autoupdate/v1/autoupdate.proto @@ -170,6 +170,7 @@ message AutoUpdateAgentRolloutSpec { } // AutoUpdateAgentRolloutStatus tracks the current agent rollout status. +// The status is reset if any spec field changes except the mode. message AutoUpdateAgentRolloutStatus { repeated AutoUpdateAgentRolloutStatusGroup groups = 1; AutoUpdateAgentRolloutState state = 2; @@ -182,6 +183,11 @@ message AutoUpdateAgentRolloutStatus { // before the rollout start time and the maintenance window belongs to the previous rollout. // When the timestamp is nil, the controller will ignore the start time and check and allow groups to activate. google.protobuf.Timestamp start_time = 3; + + // Time override is an optional timestamp making the autoupdate_agent_rollout controller use a specific time instead + // of the system clock when evaluating time-based criteria. This field is used for testing and troubleshooting + // purposes. + google.protobuf.Timestamp time_override = 4; } // AutoUpdateAgentRolloutStatusGroup tracks the current agent rollout status of a specific group. diff --git a/lib/autoupdate/rollout/controller.go b/lib/autoupdate/rollout/controller.go index eb253f366b6a7..7adbb1d009a15 100644 --- a/lib/autoupdate/rollout/controller.go +++ b/lib/autoupdate/rollout/controller.go @@ -91,6 +91,7 @@ func (c *Controller) Run(ctx context.Context) error { ticker := interval.New(config) defer ticker.Stop() + c.log.InfoContext(ctx, "Starting autoupdate_agent_rollout controller", "period", c.period) for { select { case <-ctx.Done(): diff --git a/lib/autoupdate/rollout/reconciler.go b/lib/autoupdate/rollout/reconciler.go index a6264ba3cbf27..02f393a58ff8d 100644 --- a/lib/autoupdate/rollout/reconciler.go +++ b/lib/autoupdate/rollout/reconciler.go @@ -166,6 +166,7 @@ func (r *reconciler) tryReconcile(ctx context.Context) error { // if there are no existing rollout, we create a new one and set the status if !rolloutExists { + r.log.DebugContext(ctx, "creating rollout") rollout, err := update.NewAutoUpdateAgentRollout(newSpec) rollout.Status = newStatus if err != nil { @@ -175,6 +176,7 @@ func (r *reconciler) tryReconcile(ctx context.Context) error { return trace.Wrap(err, "creating rollout") } + r.log.DebugContext(ctx, "updating rollout") // If there was a previous rollout, we update its spec and status and do an update. // We don't create a new resource to keep the metadata containing the revision ID. existingRollout.Spec = newSpec @@ -295,6 +297,16 @@ func (r *reconciler) computeStatus( // compute the group state changes now := r.clock.Now() + // If timeOverride is set to a non-zero value (we have two potential zeros, go time's zero and timestamppb's zero) + // we use this instead of the clock's time. + if timeOverride := status.GetTimeOverride().AsTime(); !(timeOverride.IsZero() || timeOverride.Unix() == 0) { + r.log.DebugContext(ctx, "reconciling with synthetic time instead of real time", + "time_override", timeOverride, + "real_time", now, + ) + now = timeOverride + } + // If this is a new rollout or the rollout has been reset, we create groups from the config groups := status.GetGroups() var err error @@ -306,7 +318,7 @@ func (r *reconciler) computeStatus( } status.Groups = groups - err = r.progressRollout(ctx, newSpec.GetStrategy(), status) + err = r.progressRollout(ctx, newSpec.GetStrategy(), status, now) // Failing to progress the update is not a hard failure. // We want to update the status even if something went wrong to surface the failed reconciliation and potential errors to the user. if err != nil { @@ -322,10 +334,10 @@ func (r *reconciler) computeStatus( // groups are updated in place. // If an error is returned, the groups should still be upserted, depending on the strategy, // failing to update a group might not be fatal (other groups can still progress independently). -func (r *reconciler) progressRollout(ctx context.Context, strategyName string, status *autoupdate.AutoUpdateAgentRolloutStatus) error { +func (r *reconciler) progressRollout(ctx context.Context, strategyName string, status *autoupdate.AutoUpdateAgentRolloutStatus, now time.Time) error { for _, strategy := range r.rolloutStrategies { if strategy.name() == strategyName { - return strategy.progressRollout(ctx, status) + return strategy.progressRollout(ctx, status, now) } } return trace.NotImplemented("rollout strategy %q not implemented", strategyName) diff --git a/lib/autoupdate/rollout/reconciler_test.go b/lib/autoupdate/rollout/reconciler_test.go index b5b77cd735edb..c2739685b7f72 100644 --- a/lib/autoupdate/rollout/reconciler_test.go +++ b/lib/autoupdate/rollout/reconciler_test.go @@ -714,7 +714,7 @@ func (f *fakeRolloutStrategy) name() string { return f.strategyName } -func (f *fakeRolloutStrategy) progressRollout(ctx context.Context, status *autoupdate.AutoUpdateAgentRolloutStatus) error { +func (f *fakeRolloutStrategy) progressRollout(ctx context.Context, status *autoupdate.AutoUpdateAgentRolloutStatus, now time.Time) error { f.calls++ return nil } diff --git a/lib/autoupdate/rollout/strategy.go b/lib/autoupdate/rollout/strategy.go index b3e214f6cebd7..d2f8c2da81f93 100644 --- a/lib/autoupdate/rollout/strategy.go +++ b/lib/autoupdate/rollout/strategy.go @@ -40,7 +40,7 @@ const ( // This interface allows us to inject dummy strategies for simpler testing. type rolloutStrategy interface { name() string - progressRollout(context.Context, *autoupdate.AutoUpdateAgentRolloutStatus) error + progressRollout(context.Context, *autoupdate.AutoUpdateAgentRolloutStatus, time.Time) error } func inWindow(group *autoupdate.AutoUpdateAgentRolloutStatusGroup, now time.Time) (bool, error) { diff --git a/lib/autoupdate/rollout/strategy_haltonerror.go b/lib/autoupdate/rollout/strategy_haltonerror.go index 0ab57a052768d..6ed1a4aae049d 100644 --- a/lib/autoupdate/rollout/strategy_haltonerror.go +++ b/lib/autoupdate/rollout/strategy_haltonerror.go @@ -24,7 +24,6 @@ import ( "time" "github.com/gravitational/trace" - "github.com/jonboulle/clockwork" "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" update "github.com/gravitational/teleport/api/types/autoupdate" @@ -39,29 +38,23 @@ const ( ) type haltOnErrorStrategy struct { - log *slog.Logger - clock clockwork.Clock + log *slog.Logger } func (h *haltOnErrorStrategy) name() string { return update.AgentsStrategyHaltOnError } -func newHaltOnErrorStrategy(log *slog.Logger, clock clockwork.Clock) (rolloutStrategy, error) { +func newHaltOnErrorStrategy(log *slog.Logger) (rolloutStrategy, error) { if log == nil { return nil, trace.BadParameter("missing log") } - if clock == nil { - return nil, trace.BadParameter("missing clock") - } return &haltOnErrorStrategy{ - log: log.With("strategy", update.AgentsStrategyHaltOnError), - clock: clock, + log: log.With("strategy", update.AgentsStrategyHaltOnError), }, nil } -func (h *haltOnErrorStrategy) progressRollout(ctx context.Context, status *autoupdate.AutoUpdateAgentRolloutStatus) error { - now := h.clock.Now() +func (h *haltOnErrorStrategy) progressRollout(ctx context.Context, status *autoupdate.AutoUpdateAgentRolloutStatus, now time.Time) error { // We process every group in order, all the previous groups must be in the DONE state // for the next group to become active. Even if some early groups are not DONE, // later groups might be ACTIVE and need to transition to DONE, so we cannot diff --git a/lib/autoupdate/rollout/strategy_haltonerror_test.go b/lib/autoupdate/rollout/strategy_haltonerror_test.go index 84d4de069efe3..ee3eb8e80ffca 100644 --- a/lib/autoupdate/rollout/strategy_haltonerror_test.go +++ b/lib/autoupdate/rollout/strategy_haltonerror_test.go @@ -134,7 +134,7 @@ func Test_canStartHaltOnError(t *testing.T) { func Test_progressGroupsHaltOnError(t *testing.T) { clock := clockwork.NewFakeClockAt(testSunday) log := utils.NewSlogLoggerForTests() - strategy, err := newHaltOnErrorStrategy(log, clock) + strategy, err := newHaltOnErrorStrategy(log) require.NoError(t, err) fewMinutesAgo := clock.Now().Add(-5 * time.Minute) @@ -500,7 +500,7 @@ func Test_progressGroupsHaltOnError(t *testing.T) { State: 0, StartTime: tt.rolloutStartTime, } - err := strategy.progressRollout(ctx, status) + err := strategy.progressRollout(ctx, status, clock.Now()) require.NoError(t, err) // We use require.Equal instead of Elements match because group order matters. // It's not super important for time-based, but is crucial for halt-on-error. diff --git a/lib/autoupdate/rollout/strategy_timebased.go b/lib/autoupdate/rollout/strategy_timebased.go index 13e844f0e4a5a..5d06adf0e5ace 100644 --- a/lib/autoupdate/rollout/strategy_timebased.go +++ b/lib/autoupdate/rollout/strategy_timebased.go @@ -21,9 +21,9 @@ package rollout import ( "context" "log/slog" + "time" "github.com/gravitational/trace" - "github.com/jonboulle/clockwork" "github.com/gravitational/teleport/api/gen/proto/go/teleport/autoupdate/v1" update "github.com/gravitational/teleport/api/types/autoupdate" @@ -35,29 +35,23 @@ const ( ) type timeBasedStrategy struct { - log *slog.Logger - clock clockwork.Clock + log *slog.Logger } func (h *timeBasedStrategy) name() string { return update.AgentsStrategyTimeBased } -func newTimeBasedStrategy(log *slog.Logger, clock clockwork.Clock) (rolloutStrategy, error) { +func newTimeBasedStrategy(log *slog.Logger) (rolloutStrategy, error) { if log == nil { return nil, trace.BadParameter("missing log") } - if clock == nil { - return nil, trace.BadParameter("missing clock") - } return &timeBasedStrategy{ - log: log.With("strategy", update.AgentsStrategyTimeBased), - clock: clock, + log: log.With("strategy", update.AgentsStrategyTimeBased), }, nil } -func (h *timeBasedStrategy) progressRollout(ctx context.Context, status *autoupdate.AutoUpdateAgentRolloutStatus) error { - now := h.clock.Now() +func (h *timeBasedStrategy) progressRollout(ctx context.Context, status *autoupdate.AutoUpdateAgentRolloutStatus, now time.Time) error { // We always process every group regardless of the order. var errs []error for _, group := range status.Groups { diff --git a/lib/autoupdate/rollout/strategy_timebased_test.go b/lib/autoupdate/rollout/strategy_timebased_test.go index 6402da3b21c44..84367f9927c04 100644 --- a/lib/autoupdate/rollout/strategy_timebased_test.go +++ b/lib/autoupdate/rollout/strategy_timebased_test.go @@ -34,7 +34,7 @@ import ( func Test_progressGroupsTimeBased(t *testing.T) { clock := clockwork.NewFakeClockAt(testSunday) log := utils.NewSlogLoggerForTests() - strategy, err := newTimeBasedStrategy(log, clock) + strategy, err := newTimeBasedStrategy(log) require.NoError(t, err) groupName := "test-group" @@ -332,7 +332,7 @@ func Test_progressGroupsTimeBased(t *testing.T) { State: 0, StartTime: tt.rolloutStartTime, } - err := strategy.progressRollout(ctx, status) + err := strategy.progressRollout(ctx, status, clock.Now()) require.NoError(t, err) // We use require.Equal instead of Elements match because group order matters. // It's not super important for time-based, but is crucial for halt-on-error. diff --git a/lib/config/configuration.go b/lib/config/configuration.go index 929bb757a85ca..5d42d9f4aa8a4 100644 --- a/lib/config/configuration.go +++ b/lib/config/configuration.go @@ -2753,6 +2753,14 @@ func Configure(clf *CommandLineFlags, cfg *servicecfg.Config, legacyAppFlags boo cfg.Proxy.QUICProxyPeering = true } + if rawPeriod := os.Getenv("TELEPORT_UNSTABLE_AGENT_ROLLOUT_SYNC_PERIOD"); rawPeriod != "" { + period, err := time.ParseDuration(rawPeriod) + if err != nil { + return trace.Wrap(err, "invalid agent rollout period %q", rawPeriod) + } + cfg.Auth.AgentRolloutControllerSyncPeriod = period + } + return nil }