diff --git a/apis/go/mlops/scheduler/scheduler.pb.go b/apis/go/mlops/scheduler/scheduler.pb.go index 1461692bc2..5d9705033d 100644 --- a/apis/go/mlops/scheduler/scheduler.pb.go +++ b/apis/go/mlops/scheduler/scheduler.pb.go @@ -533,15 +533,16 @@ func (PipelineStatusResponse_PipelineOperation) EnumDescriptor() ([]byte, []int) type PipelineVersionState_PipelineStatus int32 const ( - PipelineVersionState_PipelineStatusUnknown PipelineVersionState_PipelineStatus = 0 - PipelineVersionState_PipelineCreate PipelineVersionState_PipelineStatus = 1 - PipelineVersionState_PipelineCreating PipelineVersionState_PipelineStatus = 2 - PipelineVersionState_PipelineReady PipelineVersionState_PipelineStatus = 3 - PipelineVersionState_PipelineFailed PipelineVersionState_PipelineStatus = 4 - PipelineVersionState_PipelineTerminate PipelineVersionState_PipelineStatus = 5 - PipelineVersionState_PipelineTerminating PipelineVersionState_PipelineStatus = 6 - PipelineVersionState_PipelineTerminated PipelineVersionState_PipelineStatus = 7 - PipelineVersionState_PipelineRebalancing PipelineVersionState_PipelineStatus = 8 + PipelineVersionState_PipelineStatusUnknown PipelineVersionState_PipelineStatus = 0 + PipelineVersionState_PipelineCreate PipelineVersionState_PipelineStatus = 1 + PipelineVersionState_PipelineCreating PipelineVersionState_PipelineStatus = 2 + PipelineVersionState_PipelineReady PipelineVersionState_PipelineStatus = 3 + PipelineVersionState_PipelineFailed PipelineVersionState_PipelineStatus = 4 + PipelineVersionState_PipelineTerminate PipelineVersionState_PipelineStatus = 5 + PipelineVersionState_PipelineTerminating PipelineVersionState_PipelineStatus = 6 + PipelineVersionState_PipelineTerminated PipelineVersionState_PipelineStatus = 7 + PipelineVersionState_PipelineRebalancing PipelineVersionState_PipelineStatus = 8 + PipelineVersionState_PipelineFailedTerminating PipelineVersionState_PipelineStatus = 9 ) // Enum value maps for PipelineVersionState_PipelineStatus. @@ -556,17 +557,19 @@ var ( 6: "PipelineTerminating", 7: "PipelineTerminated", 8: "PipelineRebalancing", + 9: "PipelineFailedTerminating", } PipelineVersionState_PipelineStatus_value = map[string]int32{ - "PipelineStatusUnknown": 0, - "PipelineCreate": 1, - "PipelineCreating": 2, - "PipelineReady": 3, - "PipelineFailed": 4, - "PipelineTerminate": 5, - "PipelineTerminating": 6, - "PipelineTerminated": 7, - "PipelineRebalancing": 8, + "PipelineStatusUnknown": 0, + "PipelineCreate": 1, + "PipelineCreating": 2, + "PipelineReady": 3, + "PipelineFailed": 4, + "PipelineTerminate": 5, + "PipelineTerminating": 6, + "PipelineTerminated": 7, + "PipelineRebalancing": 8, + "PipelineFailedTerminating": 9, } ) @@ -5478,7 +5481,7 @@ var file_mlops_scheduler_scheduler_proto_rawDesc = []byte{ 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x22, 0x92, 0x05, 0x0a, 0x14, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x56, + 0x74, 0x65, 0x22, 0xb1, 0x05, 0x0a, 0x14, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x56, 0x65, @@ -5505,7 +5508,7 @@ var file_mlops_scheduler_scheduler_proto_rawDesc = []byte{ 0x69, 0x6e, 0x65, 0x47, 0x77, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x0a, 0x10, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x47, 0x77, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x47, - 0x77, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0xdd, 0x01, 0x0a, 0x0e, 0x50, 0x69, 0x70, 0x65, + 0x77, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0xfc, 0x01, 0x0a, 0x0e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x19, 0x0a, 0x15, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, @@ -5519,210 +5522,212 @@ var file_mlops_scheduler_scheduler_proto_rawDesc = []byte{ 0x74, 0x69, 0x6e, 0x67, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x10, 0x07, 0x12, 0x17, 0x0a, 0x13, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x62, 0x61, 0x6c, 0x61, - 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x10, 0x08, 0x22, 0x40, 0x0a, 0x16, 0x53, 0x63, 0x68, 0x65, 0x64, - 0x75, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x72, 0x4e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x75, 0x62, 0x73, 0x63, - 0x72, 0x69, 0x62, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x49, 0x0a, 0x17, 0x53, 0x63, 0x68, - 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x12, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x12, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x49, 0x0a, 0x1f, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, - 0x6c, 0x61, 0x6e, 0x65, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x75, 0x62, 0x73, 0x63, - 0x72, 0x69, 0x62, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0e, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x22, - 0xc9, 0x01, 0x0a, 0x14, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x32, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, - 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, - 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, - 0x6e, 0x74, 0x22, 0x67, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x11, 0x0a, 0x0d, 0x55, - 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x10, 0x00, 0x12, 0x10, - 0x0a, 0x0c, 0x53, 0x45, 0x4e, 0x44, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x53, 0x10, 0x01, - 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x45, 0x4e, 0x44, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x4c, 0x53, 0x10, - 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x45, 0x4e, 0x44, 0x5f, 0x50, 0x49, 0x50, 0x45, 0x4c, 0x49, - 0x4e, 0x45, 0x53, 0x10, 0x03, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x45, 0x4e, 0x44, 0x5f, 0x45, 0x58, - 0x50, 0x45, 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x53, 0x10, 0x04, 0x22, 0x8e, 0x02, 0x0a, 0x12, - 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x12, 0x49, 0x0a, 0x02, 0x6f, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x39, - 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, - 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, - 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x02, 0x6f, 0x70, 0x12, 0x14, 0x0a, - 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x6f, - 0x64, 0x65, 0x6c, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, - 0x03, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, - 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x16, 0x0a, - 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x22, 0x35, 0x0a, 0x0e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x4f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, - 0x77, 0x6e, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x10, 0x01, - 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x10, 0x02, 0x22, 0x90, 0x01, 0x0a, - 0x18, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x42, 0x0a, 0x06, 0x75, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x73, 0x65, 0x6c, 0x64, - 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, - 0x65, 0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x06, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, - 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, - 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, - 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, - 0x1b, 0x0a, 0x19, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x27, 0x0a, 0x0c, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, - 0x4d, 0x4f, 0x44, 0x45, 0x4c, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x50, 0x49, 0x50, 0x45, 0x4c, - 0x49, 0x4e, 0x45, 0x10, 0x01, 0x32, 0xd9, 0x11, 0x0a, 0x09, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, - 0x6c, 0x65, 0x72, 0x12, 0x6b, 0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4e, 0x6f, 0x74, - 0x69, 0x66, 0x79, 0x12, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, - 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x2c, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, - 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x62, 0x0a, 0x09, 0x4c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x12, 0x28, 0x2e, + 0x6e, 0x63, 0x69, 0x6e, 0x67, 0x10, 0x08, 0x12, 0x1d, 0x0a, 0x19, 0x50, 0x69, 0x70, 0x65, 0x6c, + 0x69, 0x6e, 0x65, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, + 0x74, 0x69, 0x6e, 0x67, 0x10, 0x09, 0x22, 0x40, 0x0a, 0x16, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, + 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x72, 0x4e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, + 0x69, 0x62, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x49, 0x0a, 0x17, 0x53, 0x63, 0x68, 0x65, + 0x64, 0x75, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x12, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x12, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x22, 0x49, 0x0a, 0x1f, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x6c, + 0x61, 0x6e, 0x65, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, + 0x69, 0x62, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, + 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xc9, + 0x01, 0x0a, 0x14, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x32, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, + 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, + 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x22, 0x67, 0x0a, 0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x11, 0x0a, 0x0d, 0x55, 0x4e, + 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x10, 0x00, 0x12, 0x10, 0x0a, + 0x0c, 0x53, 0x45, 0x4e, 0x44, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x53, 0x10, 0x01, 0x12, + 0x0f, 0x0a, 0x0b, 0x53, 0x45, 0x4e, 0x44, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x4c, 0x53, 0x10, 0x02, + 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x45, 0x4e, 0x44, 0x5f, 0x50, 0x49, 0x50, 0x45, 0x4c, 0x49, 0x4e, + 0x45, 0x53, 0x10, 0x03, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x45, 0x4e, 0x44, 0x5f, 0x45, 0x58, 0x50, + 0x45, 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x53, 0x10, 0x04, 0x22, 0x8e, 0x02, 0x0a, 0x12, 0x4d, + 0x6f, 0x64, 0x65, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x12, 0x49, 0x0a, 0x02, 0x6f, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x39, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, - 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, - 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, - 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x68, 0x0a, 0x0b, 0x55, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, - 0x64, 0x65, 0x6c, 0x12, 0x2a, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, - 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x6c, - 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, - 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4d, - 0x6f, 0x64, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6b, - 0x0a, 0x0c, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x2b, + 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x4f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x02, 0x6f, 0x70, 0x12, 0x14, 0x0a, 0x05, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, + 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x1c, + 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x16, 0x0a, 0x06, + 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x22, 0x35, 0x0a, 0x0e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x4f, 0x70, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, + 0x6e, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x10, 0x01, 0x12, + 0x0a, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x10, 0x02, 0x22, 0x90, 0x01, 0x0a, 0x18, + 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x42, 0x0a, 0x06, 0x75, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, + 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, + 0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x52, 0x06, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, + 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, + 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x1b, + 0x0a, 0x19, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x27, 0x0a, 0x0c, 0x52, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x4d, + 0x4f, 0x44, 0x45, 0x4c, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x50, 0x49, 0x50, 0x45, 0x4c, 0x49, + 0x4e, 0x45, 0x10, 0x01, 0x32, 0xd9, 0x11, 0x0a, 0x09, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, + 0x65, 0x72, 0x12, 0x6b, 0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4e, 0x6f, 0x74, 0x69, + 0x66, 0x79, 0x12, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, + 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x2c, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, + 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4e, + 0x6f, 0x74, 0x69, 0x66, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x62, 0x0a, 0x09, 0x4c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x12, 0x28, 0x2e, 0x73, + 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, + 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, + 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, + 0x4c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x68, 0x0a, 0x0b, 0x55, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, + 0x65, 0x6c, 0x12, 0x2a, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, + 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x6c, 0x6f, + 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, - 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65, - 0x6c, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x73, 0x65, - 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, - 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x71, 0x0a, 0x0e, 0x55, - 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x2d, 0x2e, + 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, + 0x64, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6b, 0x0a, + 0x0c, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, - 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, - 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x73, + 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65, 0x6c, + 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x73, 0x65, 0x6c, + 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, + 0x6c, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x71, 0x0a, 0x0e, 0x55, 0x6e, + 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x2d, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65, - 0x6c, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x74, - 0x0a, 0x0f, 0x53, 0x74, 0x61, 0x72, 0x74, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, - 0x74, 0x12, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, - 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, - 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x2f, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, - 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, - 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x71, 0x0a, 0x0e, 0x53, 0x74, 0x6f, 0x70, 0x45, 0x78, 0x70, 0x65, - 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, - 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, - 0x53, 0x74, 0x6f, 0x70, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, - 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, - 0x74, 0x6f, 0x70, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6d, 0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, - 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, - 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, - 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x6a, 0x0a, 0x0b, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, - 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, - 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, - 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x30, 0x01, 0x12, 0x73, 0x0a, 0x0e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x2d, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, - 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x50, 0x69, - 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, - 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x50, 0x69, 0x70, - 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x79, 0x0a, 0x10, 0x45, 0x78, 0x70, 0x65, 0x72, - 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x2e, 0x73, 0x65, + 0x6c, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, - 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x73, - 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, - 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x30, 0x01, 0x12, 0x74, 0x0a, 0x0f, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, + 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65, 0x6c, + 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x74, 0x0a, + 0x0f, 0x53, 0x74, 0x61, 0x72, 0x74, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, + 0x12, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, + 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x45, + 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x2f, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, + 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x45, + 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x71, 0x0a, 0x0e, 0x53, 0x74, 0x6f, 0x70, 0x45, 0x78, 0x70, 0x65, 0x72, + 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, - 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2f, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, - 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, - 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7c, 0x0a, 0x15, 0x53, 0x75, 0x62, 0x73, - 0x63, 0x72, 0x69, 0x62, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x31, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, - 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, - 0x72, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, - 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x79, 0x0a, 0x14, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, - 0x69, 0x62, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x30, - 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, - 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x75, 0x62, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x74, 0x6f, 0x70, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, + 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x74, + 0x6f, 0x70, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6d, 0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, + 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, + 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x6a, 0x0a, 0x0b, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, + 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, 0x6f, + 0x64, 0x65, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, - 0x01, 0x12, 0x88, 0x01, 0x0a, 0x19, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x45, - 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x35, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, - 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, - 0x65, 0x6e, 0x74, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, - 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, - 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x82, 0x01, 0x0a, - 0x17, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, - 0x6e, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x33, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, - 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, - 0x72, 0x2e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, - 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, - 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, + 0x01, 0x12, 0x73, 0x0a, 0x0e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x2d, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, + 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x50, 0x69, 0x70, + 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, + 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x50, 0x69, 0x70, 0x65, + 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x79, 0x0a, 0x10, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, + 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x2e, 0x73, 0x65, 0x6c, + 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, + 0x6c, 0x65, 0x72, 0x2e, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x73, 0x65, + 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, + 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, - 0x01, 0x12, 0x7e, 0x0a, 0x13, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x31, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, - 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x2e, - 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x32, 0x2e, 0x73, 0x65, - 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x63, 0x68, 0x61, 0x69, 0x6e, - 0x65, 0x72, 0x2e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x79, 0x0a, 0x10, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, - 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, - 0x6f, 0x64, 0x65, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x31, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, + 0x01, 0x12, 0x74, 0x0a, 0x0f, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, + 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x63, + 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2f, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, + 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x63, + 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7c, 0x0a, 0x15, 0x53, 0x75, 0x62, 0x73, 0x63, + 0x72, 0x69, 0x62, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x12, 0x31, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, + 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, + 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x79, 0x0a, 0x14, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, + 0x62, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x30, 0x2e, + 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, + 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x75, 0x62, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, + 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, + 0x12, 0x88, 0x01, 0x0a, 0x19, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x45, 0x78, + 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x35, + 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, + 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, + 0x6e, 0x74, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, + 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x45, + 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x82, 0x01, 0x0a, 0x17, + 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, + 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x33, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, - 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x82, 0x01, 0x0a, - 0x15, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x12, 0x37, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, + 0x2e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x73, + 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, + 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, + 0x12, 0x7e, 0x0a, 0x13, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x31, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, + 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x2e, 0x50, + 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x32, 0x2e, 0x73, 0x65, 0x6c, + 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x2e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x79, 0x0a, 0x10, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, + 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, 0x6f, + 0x64, 0x65, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x31, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, - 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x53, 0x75, 0x62, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x2c, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, - 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, - 0x01, 0x42, 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x69, 0x6f, 0x2f, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2d, - 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x32, 0x2f, - 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2f, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x82, 0x01, 0x0a, 0x15, + 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x12, 0x37, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, + 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x43, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x53, 0x75, 0x62, 0x73, 0x63, + 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, + 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, + 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, + 0x6c, 0x61, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, + 0x42, 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, + 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x69, 0x6f, 0x2f, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2d, 0x63, + 0x6f, 0x72, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x32, 0x2f, 0x6d, + 0x6c, 0x6f, 0x70, 0x73, 0x2f, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/apis/mlops/scheduler/scheduler.proto b/apis/mlops/scheduler/scheduler.proto index a5dfbfdb12..57a28ea846 100644 --- a/apis/mlops/scheduler/scheduler.proto +++ b/apis/mlops/scheduler/scheduler.proto @@ -77,9 +77,9 @@ message LlmSpec { message ModelRuntimeInfo { oneof modelRuntimeInfo { - MLServerModelSettings mlserver = 1; + MLServerModelSettings mlserver = 1; TritonModelConfig triton = 2; - } + } } message MLServerModelSettings { @@ -150,24 +150,24 @@ message ModelVersionStatus { uint32 version = 2; string serverName = 3; optional KubernetesMeta kubernetesMeta = 4; - map modelReplicaState = 5; + map modelReplicaState = 5; ModelStatus state = 6; optional Model modelDefn = 7; } message ModelStatus { enum ModelState { - ModelStateUnknown = 0; - ModelProgressing = 1; - ModelAvailable = 2; - ModelFailed = 3; - ModelTerminating = 4; - ModelTerminated = 5; - ModelTerminateFailed = 6; - ScheduleFailed = 7; - ModelScaledDown = 8; - ModelCreate = 9; - ModelTerminate = 10; + ModelStateUnknown = 0; + ModelProgressing = 1; + ModelAvailable = 2; + ModelFailed = 3; + ModelTerminating = 4; + ModelTerminated = 5; + ModelTerminateFailed = 6; + ScheduleFailed = 7; + ModelScaledDown = 8; + ModelCreate = 9; + ModelTerminate = 10; } ModelState state = 1; string reason = 2; @@ -180,19 +180,19 @@ message ModelStatus { message ModelReplicaStatus { enum ModelReplicaState { - ModelReplicaStateUnknown = 0; - LoadRequested = 1; - Loading = 2; - Loaded = 3; - LoadFailed = 4; - UnloadRequested = 5; - Unloading = 6; - Unloaded = 7; - UnloadFailed = 8; - Available = 9; - LoadedUnavailable = 10; - UnloadEnvoyRequested = 11; - Draining = 12; + ModelReplicaStateUnknown = 0; + LoadRequested = 1; + Loading = 2; + Loaded = 3; + LoadFailed = 4; + UnloadRequested = 5; + Unloading = 6; + Unloaded = 7; + UnloadFailed = 8; + Available = 9; + LoadedUnavailable = 10; + UnloadEnvoyRequested = 11; + Draining = 12; } ModelReplicaState state = 1; string reason = 2; @@ -356,7 +356,7 @@ message PipelineStep { string name = 1; repeated string inputs = 2; optional uint32 joinWindowMs = 3; // Join window millisecs, some nonzero default (TBD) - map tensorMap = 4; // optional map of tensor name mappings + map tensorMap = 4; // optional map of tensor name mappings JoinOp inputsJoin = 5; repeated string triggers = 6; JoinOp triggersJoin = 7; @@ -379,7 +379,7 @@ message PipelineInput { optional uint32 joinWindowMs = 3; // Join window millisecs for output, default 0 JoinOp joinType = 4; JoinOp triggersJoin = 5; - map tensorMap = 6; // optional map of tensor name mappings + map tensorMap = 6; // optional map of tensor name mappings } message PipelineOutput { @@ -391,7 +391,7 @@ message PipelineOutput { repeated string steps = 1; uint32 joinWindowMs = 2; // Join window millisecs for output, default 0 JoinOp stepsJoin = 3; - map tensorMap = 4; // optional map of tensor name mappings + map tensorMap = 4; // optional map of tensor name mappings } message LoadPipelineResponse { @@ -446,6 +446,7 @@ message PipelineVersionState { PipelineTerminating = 6; PipelineTerminated = 7; PipelineRebalancing = 8; + PipelineFailedTerminating = 9; } uint32 pipelineVersion = 1; PipelineStatus status = 2; @@ -481,17 +482,17 @@ message ControlPlaneResponse { message ModelUpdateMessage { - enum ModelOperation { - Unknown = 0; - Create = 1; - Delete = 2; - } - ModelOperation op = 1; - string model = 2; - uint32 version = 3; - string uid = 4; - uint64 timestamp = 5; - string stream = 6; + enum ModelOperation { + Unknown = 0; + Create = 1; + Delete = 2; + } + ModelOperation op = 1; + string model = 2; + uint32 version = 3; + string uid = 4; + uint64 timestamp = 5; + string stream = 6; } message ModelUpdateStatusMessage { diff --git a/tests/integration/godog/components/bootstrap.go b/tests/integration/godog/components/bootstrap.go new file mode 100644 index 0000000000..56800b47a1 --- /dev/null +++ b/tests/integration/godog/components/bootstrap.go @@ -0,0 +1,51 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + +package components + +import ( + "fmt" + + "github.com/seldonio/seldon-core/tests/integration/godog/k8sclient" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" +) + +const ( + KafkaNodePool ComponentName = "KafkaNodePool" +) + +func StartComponents(k8sClient *k8sclient.K8sClient, namespace string) (*EnvManager, error) { + var kafkaNodePoolGVK = schema.GroupVersionKind{ + Group: "kafka.strimzi.io", + Version: "v1beta2", + Kind: "KafkaNodePool", + } + + kafkaNodePool := NewK8sObjectComponent( + k8sClient, + KafkaNodePool, + kafkaNodePoolGVK, + types.NamespacedName{Namespace: namespace, Name: "kafka"}, + UnavailableByDeleting, + ) + + runtime := NewSeldonRuntimeComponent( + k8sClient, + namespace, + "seldon", + ) + + env, err := NewEnvManager(kafkaNodePool, runtime) + if err != nil { + return nil, fmt.Errorf("failed to bootstrap components: %v", err) + } + + return env, nil +} diff --git a/tests/integration/godog/components/component_manager.go b/tests/integration/godog/components/component_manager.go new file mode 100644 index 0000000000..34ede66b69 --- /dev/null +++ b/tests/integration/godog/components/component_manager.go @@ -0,0 +1,97 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + +package components + +import ( + "context" + "fmt" + "sync" +) + +type Component interface { + Name() ComponentName + Snapshot(ctx context.Context) error + Restore(ctx context.Context) error + MakeUnavailable(ctx context.Context) error + MakeAvailable(ctx context.Context) error + Scale(ctx context.Context, replicas int32) error // optional capability +} + +type ComponentName string + +type EnvManager struct { + mu sync.Mutex + components map[ComponentName]Component + order []ComponentName // deterministic snapshot/restore ordering +} + +func NewEnvManager(cs ...Component) (*EnvManager, error) { + m := make(map[ComponentName]Component, len(cs)) + order := make([]ComponentName, 0, len(cs)) + + for _, c := range cs { + if c == nil { + return nil, fmt.Errorf("nil component provided") + } + name := c.Name() + if name == "" { + return nil, fmt.Errorf("component has empty name: %T", c) + } + if _, exists := m[name]; exists { + return nil, fmt.Errorf("duplicate component name: %q", name) + } + m[name] = c + order = append(order, name) + } + + return &EnvManager{ + components: m, + order: order, + }, nil +} + +func (e *EnvManager) Component(name ComponentName) (Component, error) { + c, ok := e.components[name] + if !ok { + return nil, fmt.Errorf("unknown component: %q", name) + } + return c, nil +} + +func (e *EnvManager) SnapshotAll(ctx context.Context) error { + e.mu.Lock() + defer e.mu.Unlock() + for _, name := range e.order { + if err := e.components[name].Snapshot(ctx); err != nil { + return fmt.Errorf("snapshot %s: %w", name, err) + } + } + return nil +} + +func (e *EnvManager) RestoreAll(ctx context.Context) error { + e.mu.Lock() + defer e.mu.Unlock() + for _, name := range e.order { + if err := e.components[name].Restore(ctx); err != nil { + return fmt.Errorf("restore %s: %w", name, err) + } + } + return nil +} + +func (e *EnvManager) Runtime() *SeldonRuntimeComponent { + for _, comp := range e.components { + if c, ok := comp.(*SeldonRuntimeComponent); ok { + return c + } + } + return nil +} diff --git a/tests/integration/godog/components/generic.go b/tests/integration/godog/components/generic.go new file mode 100644 index 0000000000..d4195e4145 --- /dev/null +++ b/tests/integration/godog/components/generic.go @@ -0,0 +1,348 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + +package components + +import ( + "context" + "fmt" + "time" + + "github.com/seldonio/seldon-core/tests/integration/godog/k8sclient" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" +) + +type UnavailableMode string + +const ( + UnavailableByScaling UnavailableMode = "scale" + UnavailableByDeleting UnavailableMode = "delete" +) + +// K8sObjectComponent is a generic component manager for any K8s object (including CRDs) +// referenced by GVK + NamespacedName. +type K8sObjectComponent struct { + name ComponentName + gvk schema.GroupVersionKind + resource types.NamespacedName + k8s *k8sclient.K8sClient + + unavailableMode UnavailableMode + + // Snapshot + snapExists bool + objectSnap *unstructured.Unstructured // deep copy of baseline object + baseReplicas *int32 // if replicas exist + + dirty bool +} + +func NewK8sObjectComponent( + k8s *k8sclient.K8sClient, + name ComponentName, + gvk schema.GroupVersionKind, + resource types.NamespacedName, + unavailableMode UnavailableMode, +) *K8sObjectComponent { + return &K8sObjectComponent{ + name: name, + gvk: gvk, + resource: resource, + k8s: k8s, + unavailableMode: unavailableMode, + } +} + +func (c *K8sObjectComponent) Name() ComponentName { return c.name } + +func (c *K8sObjectComponent) Snapshot(ctx context.Context) error { + if c.objectSnap != nil || c.snapExists { + return nil + } + + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(c.gvk) + + err := c.k8s.KubeClient.Get(ctx, c.resource, obj) + if err != nil { + if apierrors.IsNotFound(err) { + // Baseline is "absent" + c.snapExists = false + return nil + } + return fmt.Errorf("%s snapshot: get %s/%s: %w", c.name, c.resource.Namespace, c.resource.Name, err) + } + + // Baseline is "present": deep copy + c.snapExists = true + c.objectSnap = obj.DeepCopy() + + // Try to snapshot replicas if present + if replicas64, found, _ := unstructured.NestedInt64(obj.Object, "spec", "replicas"); found { + r := int32(replicas64) + c.baseReplicas = &r + } + + return nil +} + +func (c *K8sObjectComponent) Restore(ctx context.Context) error { + if !c.dirty { + return nil + } + + // Baseline absent => ensure deleted (deleteIfExists now waits for NotFound) + if !c.snapExists { + if err := c.deleteIfExists(ctx); err != nil { + return err + } + c.dirty = false + return nil + } + + if c.objectSnap == nil { + return fmt.Errorf("%s restore: baseline snapshot missing", c.name) + } + + // 1) Ensure baseline object exists (create/update) + if err := c.applyBaseline(ctx); err != nil { + return err + } + + // 2) Wait until it exists (avoid races with subsequent steps) + if err := c.waitForExists(ctx); err != nil { + return err + } + + // 3) Restore replicas if we captured them, and wait until applied + if c.baseReplicas != nil { + if err := c.Scale(ctx, *c.baseReplicas); err != nil { + return err + } + if err := c.waitForReplicas(ctx, *c.baseReplicas); err != nil { + return err + } + } + + c.dirty = false + return nil +} + +func (c *K8sObjectComponent) waitForExists(ctx context.Context) error { + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return fmt.Errorf( + "timeout waiting for %s %s/%s to exist: %w", + c.gvk.Kind, c.resource.Namespace, c.resource.Name, ctx.Err(), + ) + case <-ticker.C: + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(c.gvk) + + err := c.k8s.KubeClient.Get(ctx, c.resource, obj) + if err == nil { + return nil + } + if apierrors.IsNotFound(err) { + continue + } + return fmt.Errorf( + "error checking existence of %s %s/%s: %w", + c.gvk.Kind, c.resource.Namespace, c.resource.Name, err, + ) + } + } +} + +func (c *K8sObjectComponent) waitForReplicas(ctx context.Context, want int32) error { + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return fmt.Errorf( + "timeout waiting for %s %s/%s replicas=%d: %w", + c.gvk.Kind, c.resource.Namespace, c.resource.Name, want, ctx.Err(), + ) + case <-ticker.C: + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(c.gvk) + + err := c.k8s.KubeClient.Get(ctx, c.resource, obj) + if err != nil { + if apierrors.IsNotFound(err) { + continue + } + return fmt.Errorf( + "error checking replicas for %s %s/%s: %w", + c.gvk.Kind, c.resource.Namespace, c.resource.Name, err, + ) + } + + r64, found, err := unstructured.NestedInt64(obj.Object, "spec", "replicas") + if err != nil { + return fmt.Errorf( + "error reading spec.replicas for %s %s/%s: %w", + c.gvk.Kind, c.resource.Namespace, c.resource.Name, err, + ) + } + if !found { + return fmt.Errorf( + "%s %s/%s does not have spec.replicas (cannot wait for replicas)", + c.gvk.Kind, c.resource.Namespace, c.resource.Name, + ) + } + + if int32(r64) == want { + return nil + } + } + } +} + +func (c *K8sObjectComponent) MakeUnavailable(ctx context.Context) error { + if err := c.Snapshot(ctx); err != nil { + return err + } + + switch c.unavailableMode { + case UnavailableByDeleting: + if err := c.deleteIfExists(ctx); err != nil { + return err + } + case UnavailableByScaling: + if err := c.Scale(ctx, 0); err != nil { + return err + } + default: + return fmt.Errorf("%s: unknown unavailable mode %q", c.name, c.unavailableMode) + } + + c.dirty = true + return nil +} + +func (c *K8sObjectComponent) MakeAvailable(ctx context.Context) error { + if err := c.Snapshot(ctx); err != nil { + return err + } + // MakeAvailable means "restore to baseline" + c.dirty = true + return c.Restore(ctx) +} + +// Scale attempts to update spec.replicas. If the object doesn't have that field, return a clear error. +func (c *K8sObjectComponent) Scale(ctx context.Context, replicas int32) error { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(c.gvk) + + if err := c.k8s.KubeClient.Get(ctx, c.resource, obj); err != nil { + return fmt.Errorf("%s scale: get %s/%s: %w", c.name, c.resource.Namespace, c.resource.Name, err) + } + + // Check the field exists or can be set + if _, found, _ := unstructured.NestedFieldNoCopy(obj.Object, "spec"); !found { + return fmt.Errorf("%s scale: object has no spec", c.name) + } + + if err := unstructured.SetNestedField(obj.Object, int64(replicas), "spec", "replicas"); err != nil { + return fmt.Errorf("%s scale: set spec.replicas: %w", c.name, err) + } + + if err := c.k8s.KubeClient.Update(ctx, obj); err != nil { + return fmt.Errorf("%s scale: update %s/%s: %w", c.name, c.resource.Namespace, c.resource.Name, err) + } + + c.dirty = true + return nil +} + +// ---- internals ---- + +func (c *K8sObjectComponent) deleteIfExists(ctx context.Context) error { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(c.gvk) + + // First: try to delete if it exists + if err := c.k8s.KubeClient.Get(ctx, c.resource, obj); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + + if err := c.k8s.KubeClient.Delete(ctx, obj); err != nil { + return err + } + + // Then: wait until it's actually gone + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return fmt.Errorf( + "timeout waiting for %s %s/%s to be deleted: %w", + c.gvk.Kind, c.resource.Namespace, c.resource.Name, ctx.Err(), + ) + case <-ticker.C: + check := &unstructured.Unstructured{} + check.SetGroupVersionKind(c.gvk) + + err := c.k8s.KubeClient.Get(ctx, c.resource, check) + if apierrors.IsNotFound(err) { + // ✅ deletion fully completed + return nil + } + if err != nil { + return fmt.Errorf( + "error checking deletion of %s %s/%s: %w", + c.gvk.Kind, c.resource.Namespace, c.resource.Name, err, + ) + } + } + } +} + +func (c *K8sObjectComponent) applyBaseline(ctx context.Context) error { + desired := c.objectSnap.DeepCopy() + + // Remove fields that should not be set on create/update. + // (Important when restoring full objects.) + unstructured.RemoveNestedField(desired.Object, "metadata", "resourceVersion") + unstructured.RemoveNestedField(desired.Object, "metadata", "uid") + unstructured.RemoveNestedField(desired.Object, "metadata", "generation") + unstructured.RemoveNestedField(desired.Object, "metadata", "managedFields") + unstructured.RemoveNestedField(desired.Object, "status") + + // Try update if exists, else create + existing := &unstructured.Unstructured{} + existing.SetGroupVersionKind(c.gvk) + + err := c.k8s.KubeClient.Get(ctx, c.resource, existing) + if err != nil { + if apierrors.IsNotFound(err) { + return c.k8s.KubeClient.Create(ctx, desired) + } + return fmt.Errorf("%s restore: get existing %s/%s: %w", c.name, c.resource.Namespace, c.resource.Name, err) + } + + // Preserve current resourceVersion for Update + desired.SetResourceVersion(existing.GetResourceVersion()) + return c.k8s.KubeClient.Update(ctx, desired) +} diff --git a/tests/integration/godog/components/runtime.go b/tests/integration/godog/components/runtime.go new file mode 100644 index 0000000000..e946c824b3 --- /dev/null +++ b/tests/integration/godog/components/runtime.go @@ -0,0 +1,485 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + +package components + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/seldonio/seldon-core/tests/integration/godog/k8sclient" + appsv1 "k8s.io/api/apps/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type SeldonRuntimeService string + +const ( + ServiceScheduler SeldonRuntimeService = "seldon-scheduler" + ServicePipelineGateway SeldonRuntimeService = "seldon-pipelinegateway" + ServiceModelGateway SeldonRuntimeService = "seldon-modelgateway" + ServiceDataflowEngine SeldonRuntimeService = "seldon-dataflow-engine" + ServiceEnvoy SeldonRuntimeService = "seldon-envoy" + ServiceHodometer SeldonRuntimeService = "hodometer" +) + +type workloadRef struct { + kind string // "Deployment" or "StatefulSet" + nn types.NamespacedName +} + +type SeldonRuntimeComponent struct { + name ComponentName + k8s *k8sclient.K8sClient + + gvk schema.GroupVersionKind + nn types.NamespacedName + + mu sync.Mutex + + // snapshot of whole runtime + baseline per-service replicas + snapExists bool + runtimeSnap *unstructured.Unstructured + baseReplicas map[SeldonRuntimeService]*int32 + + dirty bool +} + +// todo: this would be global components for the runtime which are not done or complete + +func (c *SeldonRuntimeComponent) MakeUnavailable(ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (c *SeldonRuntimeComponent) MakeAvailable(ctx context.Context) error { + //TODO implement me + panic("implement me") +} + +func (c *SeldonRuntimeComponent) Scale(ctx context.Context, replicas int32) error { + //TODO implement me + panic("implement me") +} + +func NewSeldonRuntimeComponent(k8s *k8sclient.K8sClient, namespace, runtimeName string) *SeldonRuntimeComponent { + return &SeldonRuntimeComponent{ + name: "SeldonRuntime", + k8s: k8s, + gvk: schema.GroupVersionKind{Group: "mlops.seldon.io", Version: "v1alpha1", Kind: "SeldonRuntime"}, + nn: types.NamespacedName{Namespace: namespace, Name: runtimeName}, + baseReplicas: map[SeldonRuntimeService]*int32{}, + } +} + +func (c *SeldonRuntimeComponent) Name() ComponentName { return c.name } + +func (c *SeldonRuntimeComponent) Snapshot(ctx context.Context) error { + if c.runtimeSnap != nil || c.snapExists { + return nil + } + + rt := &unstructured.Unstructured{} + rt.SetGroupVersionKind(c.gvk) + + if err := c.k8s.KubeClient.Get(ctx, c.nn, rt); err != nil { + if apierrors.IsNotFound(err) { + c.snapExists = false + return nil + } + return fmt.Errorf("runtime snapshot: get %s/%s: %w", c.nn.Namespace, c.nn.Name, err) + } + + c.snapExists = true + c.runtimeSnap = rt.DeepCopy() + + // read baseline overrides replicas for all overrides + overrides, found, err := unstructured.NestedSlice(rt.Object, "spec", "overrides") + if err != nil { + return fmt.Errorf("runtime snapshot: read spec.overrides: %w", err) + } + if found { + for _, v := range overrides { + m, ok := v.(map[string]any) + if !ok { + continue + } + name, _ := m["name"].(string) + if name == "" { + continue + } + // replicas might be missing (scheduler in your sample) + if r64, ok := m["replicas"].(int64); ok { + r := int32(r64) + svc := SeldonRuntimeService(name) + c.baseReplicas[svc] = &r + } + } + } + + return nil +} + +func (c *SeldonRuntimeComponent) Restore(ctx context.Context) error { + if !c.dirty { + return nil + } + + // baseline absent => ensure absent (optional) + if !c.snapExists { + _ = c.deleteRuntimeIfExists(ctx) + c.dirty = false + return nil + } + + if c.runtimeSnap == nil { + return fmt.Errorf("runtime restore: missing snapshot") + } + + // 1) Restore the runtime object spec (create/update) + if err := c.applyRuntimeBaseline(ctx); err != nil { + return err + } + + // 2) Restore baseline replicas for services that had replicas in baseline + // (If baseline did not specify replicas for a service, we don't force it.) + for svc, r := range c.baseReplicas { + if r == nil { + continue + } + if err := c.SetReplicas(ctx, svc, *r); err != nil { + return err + } + } + + c.dirty = false + return nil +} + +func (c *SeldonRuntimeComponent) mutateRuntime(ctx context.Context, fn func(rt *unstructured.Unstructured) error) error { + c.mu.Lock() + defer c.mu.Unlock() + + return retry.OnError(retry.DefaultBackoff, apierrors.IsConflict, func() error { + rt := &unstructured.Unstructured{} + rt.SetGroupVersionKind(c.gvk) + + if err := c.k8s.KubeClient.Get(ctx, c.nn, rt); err != nil { + return err + } + if err := fn(rt); err != nil { + return err + } + return c.k8s.KubeClient.Update(ctx, rt) + }) +} + +func (c *SeldonRuntimeComponent) SetReplicas(ctx context.Context, svc SeldonRuntimeService, replicas int32) error { + if err := c.Snapshot(ctx); err != nil { + return err + } + + c.dirty = true + + if err := c.mutateRuntime(ctx, func(rt *unstructured.Unstructured) error { + return setOverrideReplicas(rt, string(svc), replicas) + }); err != nil { + return err + } + + // wait for the managed workload to reflect it + return c.waitWorkloadAtReplicas(ctx, svc, replicas) +} + +func (c *SeldonRuntimeComponent) ScaleDown(ctx context.Context, svc SeldonRuntimeService) error { + return c.SetReplicas(ctx, svc, 0) +} + +func (c *SeldonRuntimeComponent) ScaleUpToBaseline(ctx context.Context, svc SeldonRuntimeService) error { + if err := c.Snapshot(ctx); err != nil { + return err + } + base := c.baseReplicas[svc] + if base == nil { + return fmt.Errorf("no baseline replicas recorded for %s", svc) + } + return c.SetReplicas(ctx, svc, *base) +} + +func (c *SeldonRuntimeComponent) RestartService(ctx context.Context, svc SeldonRuntimeService) error { + if err := c.Snapshot(ctx); err != nil { + return err + } + + // optional policy: pipeline-gw restart should ensure scheduler ready + if svc == ServicePipelineGateway { + if err := c.WaitServiceReady(ctx, ServiceScheduler); err != nil { + return fmt.Errorf("cannot restart %s while scheduler not ready: %w", svc, err) + } + } + + ref, err := c.discoverWorkload(ctx, svc) + if err != nil { + return err + } + + switch ref.kind { + case "Deployment": + if err := rolloutRestartDeployment(ctx, c.k8s.KubeClient, ref.nn); err != nil { + return err + } + return waitDeploymentReady(ctx, c.k8s.KubeClient, ref.nn) + + case "StatefulSet": + if err := rolloutRestartStatefulSet(ctx, c.k8s.KubeClient, ref.nn); err != nil { + return err + } + return waitStatefulSetReady(ctx, c.k8s.KubeClient, ref.nn) + + default: + return fmt.Errorf("unknown workload kind %q for %s", ref.kind, svc) + } +} + +func (c *SeldonRuntimeComponent) discoverWorkload(ctx context.Context, svc SeldonRuntimeService) (*workloadRef, error) { + appName := string(svc) // matches app.kubernetes.io/name in your manifests + + // Deployment first + var deps appsv1.DeploymentList + if err := c.k8s.KubeClient.List(ctx, &deps, + client.InNamespace(c.nn.Namespace), + client.MatchingLabels{"app.kubernetes.io/name": appName}, + ); err != nil { + return nil, err + } + if len(deps.Items) == 1 { + d := deps.Items[0] + return &workloadRef{kind: "Deployment", nn: types.NamespacedName{Namespace: d.Namespace, Name: d.Name}}, nil + } + if len(deps.Items) > 1 { + return nil, fmt.Errorf("multiple deployments matched app.kubernetes.io/name=%s", appName) + } + + // StatefulSet + var stss appsv1.StatefulSetList + if err := c.k8s.KubeClient.List(ctx, &stss, + client.InNamespace(c.nn.Namespace), + client.MatchingLabels{"app.kubernetes.io/name": appName}, + ); err != nil { + return nil, err + } + if len(stss.Items) == 1 { + s := stss.Items[0] + return &workloadRef{kind: "StatefulSet", nn: types.NamespacedName{Namespace: s.Namespace, Name: s.Name}}, nil + } + if len(stss.Items) > 1 { + return nil, fmt.Errorf("multiple statefulsets matched app.kubernetes.io/name=%s", appName) + } + + return nil, fmt.Errorf("no deployment/statefulset matched app.kubernetes.io/name=%s", appName) +} + +func (c *SeldonRuntimeComponent) WaitServiceReady(ctx context.Context, svc SeldonRuntimeService) error { + condType := runtimeReadyConditionType(svc) // e.g. "PipelineGatewayReady" + return c.waitRuntimeCondition(ctx, condType, "True") +} + +func runtimeReadyConditionType(svc SeldonRuntimeService) string { + switch svc { + case ServicePipelineGateway: + return "PipelineGatewayReady" + case ServiceScheduler: + return "SchedulerReady" + case ServiceModelGateway: + return "ModelGatewayReady" + case ServiceDataflowEngine: + return "DataflowEngineReady" + case ServiceEnvoy: + return "EnvoyReady" + case ServiceHodometer: + return "HodometerReady" + default: + return "" + } +} + +func findOverrideIndex(overrides []any, name string) int { + for i, v := range overrides { + m, ok := v.(map[string]any) + if !ok { + continue + } + if n, _ := m["name"].(string); n == name { + return i + } + } + return -1 +} + +func setOverrideReplicas(rt *unstructured.Unstructured, serviceName string, replicas int32) error { + overrides, found, err := unstructured.NestedSlice(rt.Object, "spec", "overrides") + if err != nil { + return fmt.Errorf("read spec.overrides: %w", err) + } + if !found { + overrides = []any{} + } + + idx := findOverrideIndex(overrides, serviceName) + if idx == -1 { + // Create override if missing + overrides = append(overrides, map[string]any{ + "name": serviceName, + "replicas": int64(replicas), + }) + } else { + m := overrides[idx].(map[string]any) + m["replicas"] = int64(replicas) + overrides[idx] = m + } + + if err := unstructured.SetNestedSlice(rt.Object, overrides, "spec", "overrides"); err != nil { + return fmt.Errorf("write spec.overrides: %w", err) + } + return nil +} + +func getOverrideReplicas(rt *unstructured.Unstructured, serviceName string) (*int32, error) { + overrides, found, err := unstructured.NestedSlice(rt.Object, "spec", "overrides") + if err != nil { + return nil, fmt.Errorf("read spec.overrides: %w", err) + } + if !found { + return nil, nil + } + + idx := findOverrideIndex(overrides, serviceName) + if idx == -1 { + return nil, nil + } + + m := overrides[idx].(map[string]any) + // replicas might be int64 (usual), but be tolerant + switch v := m["replicas"].(type) { + case int64: + r := int32(v) + return &r, nil + case float64: + r := int32(v) + return &r, nil + default: + return nil, nil + } +} + +func (c *SeldonRuntimeComponent) waitRuntimeCondition(ctx context.Context, condType string, wantStatus string) error { + if condType == "" { + return fmt.Errorf("runtime condition type is empty") + } + + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout waiting for runtime condition %q=%q: %w", condType, wantStatus, ctx.Err()) + case <-ticker.C: + rt := &unstructured.Unstructured{} + rt.SetGroupVersionKind(c.gvk) + + if err := c.k8s.KubeClient.Get(ctx, c.nn, rt); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return err + } + + conds, found, err := unstructured.NestedSlice(rt.Object, "status", "conditions") + if err != nil { + return fmt.Errorf("read status.conditions: %w", err) + } + if !found { + continue + } + + for _, v := range conds { + m, ok := v.(map[string]any) + if !ok { + continue + } + t, _ := m["type"].(string) + if t != condType { + continue + } + s, _ := m["status"].(string) + if s == wantStatus { + return nil + } + // Found condition but not in desired state yet. + break + } + } + } +} + +func (c *SeldonRuntimeComponent) deleteRuntimeIfExists(ctx context.Context) error { + rt := &unstructured.Unstructured{} + rt.SetGroupVersionKind(c.gvk) + + if err := c.k8s.KubeClient.Get(ctx, c.nn, rt); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + return c.k8s.KubeClient.Delete(ctx, rt) +} + +func (c *SeldonRuntimeComponent) applyRuntimeBaseline(ctx context.Context) error { + // Copy snapshot (don’t mutate stored snapshot) + base := c.runtimeSnap.DeepCopy() + + // Always ensure correct GVK/namespace/name + base.SetGroupVersionKind(c.gvk) + base.SetNamespace(c.nn.Namespace) + base.SetName(c.nn.Name) + + // Strip status to avoid update conflicts (unless you explicitly manage it) + unstructured.RemoveNestedField(base.Object, "status") + + // Strip server-managed metadata fields that can cause issues + base.SetResourceVersion("") // set later if updating existing + base.SetUID("") + base.SetGeneration(0) + base.SetManagedFields(nil) + + // Try create; if exists, update + current := &unstructured.Unstructured{} + current.SetGroupVersionKind(c.gvk) + + if err := c.k8s.KubeClient.Get(ctx, c.nn, current); err != nil { + if apierrors.IsNotFound(err) { + // Create fresh + return c.k8s.KubeClient.Create(ctx, base) + } + return err + } + + // Update existing: must carry resourceVersion + base.SetResourceVersion(current.GetResourceVersion()) + return c.k8s.KubeClient.Update(ctx, base) +} diff --git a/tests/integration/godog/components/util.go b/tests/integration/godog/components/util.go new file mode 100644 index 0000000000..085976d0b0 --- /dev/null +++ b/tests/integration/godog/components/util.go @@ -0,0 +1,178 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + +package components + +import ( + "context" + "fmt" + "time" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func rolloutRestartDeployment(ctx context.Context, c client.Client, nn types.NamespacedName) error { + var dep appsv1.Deployment + if err := c.Get(ctx, nn, &dep); err != nil { + return err + } + + if dep.Spec.Template.Annotations == nil { + dep.Spec.Template.Annotations = map[string]string{} + } + dep.Spec.Template.Annotations["tests.seldon.io/restartedAt"] = time.Now().UTC().Format(time.RFC3339Nano) + + return c.Update(ctx, &dep) +} + +func rolloutRestartStatefulSet(ctx context.Context, c client.Client, nn types.NamespacedName) error { + var sts appsv1.StatefulSet + if err := c.Get(ctx, nn, &sts); err != nil { + return err + } + + if sts.Spec.Template.Annotations == nil { + sts.Spec.Template.Annotations = map[string]string{} + } + sts.Spec.Template.Annotations["tests.seldon.io/restartedAt"] = time.Now().UTC().Format(time.RFC3339Nano) + + return c.Update(ctx, &sts) +} + +// wait helpers + +func waitDeploymentReady(ctx context.Context, c client.Client, nn types.NamespacedName) error { + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout waiting for deployment %s/%s ready: %w", nn.Namespace, nn.Name, ctx.Err()) + case <-ticker.C: + var dep appsv1.Deployment + if err := c.Get(ctx, nn, &dep); err != nil { + continue + } + want := int32(1) + if dep.Spec.Replicas != nil { + want = *dep.Spec.Replicas + } + if dep.Status.AvailableReplicas == want { + return nil + } + } + } +} + +func waitStatefulSetReady(ctx context.Context, c client.Client, nn types.NamespacedName) error { + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout waiting for statefulset %s/%s ready: %w", nn.Namespace, nn.Name, ctx.Err()) + case <-ticker.C: + var sts appsv1.StatefulSet + if err := c.Get(ctx, nn, &sts); err != nil { + continue + } + want := int32(1) + if sts.Spec.Replicas != nil { + want = *sts.Spec.Replicas + } + if sts.Status.ReadyReplicas == want { + return nil + } + } + } +} + +func (c *SeldonRuntimeComponent) waitWorkloadAtReplicas(ctx context.Context, svc SeldonRuntimeService, want int32) error { + ref, err := c.discoverWorkload(ctx, svc) + if err != nil { + // If scaling to 0, some operators might delete the workload; your runtime seems to keep it + // (it scales Deployments to 0). If that ever changes, you could treat NotFound as success when want==0. + return err + } + + switch ref.kind { + case "Deployment": + return waitDeploymentReplicas(ctx, c.k8s.KubeClient, ref.nn, want) + case "StatefulSet": + return waitStatefulSetReplicas(ctx, c.k8s.KubeClient, ref.nn, want) + default: + return fmt.Errorf("wait replicas: unknown workload kind %q for %s", ref.kind, svc) + } +} + +func waitDeploymentReplicas(ctx context.Context, c client.Client, nn types.NamespacedName, want int32) error { + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout waiting for deployment %s/%s replicas=%d: %w", nn.Namespace, nn.Name, want, ctx.Err()) + case <-ticker.C: + var dep appsv1.Deployment + if err := c.Get(ctx, nn, &dep); err != nil { + continue + } + cur := int32(1) + if dep.Spec.Replicas != nil { + cur = *dep.Spec.Replicas + } + if cur != want { + continue + } + // If scaling down, consider it done when desired replicas is 0. + if want == 0 { + return nil + } + // If scaling up, ensure it’s available too. + if dep.Status.AvailableReplicas == want { + return nil + } + } + } +} + +func waitStatefulSetReplicas(ctx context.Context, c client.Client, nn types.NamespacedName, want int32) error { + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout waiting for statefulset %s/%s replicas=%d: %w", nn.Namespace, nn.Name, want, ctx.Err()) + case <-ticker.C: + var sts appsv1.StatefulSet + if err := c.Get(ctx, nn, &sts); err != nil { + continue + } + cur := int32(1) + if sts.Spec.Replicas != nil { + cur = *sts.Spec.Replicas + } + if cur != want { + continue + } + if want == 0 { + return nil + } + if sts.Status.ReadyReplicas == want { + return nil + } + } + } +} diff --git a/tests/integration/godog/features/scheduler/pipeline_retries.feature b/tests/integration/godog/features/scheduler/pipeline_retries.feature new file mode 100644 index 0000000000..b49b73f8e2 --- /dev/null +++ b/tests/integration/godog/features/scheduler/pipeline_retries.feature @@ -0,0 +1,83 @@ +@SchedulerPipelineRetries @Functional @Scheduler +Feature: Scheduler retries failed pipelines + In order to ensure pipelines recover from transient failures + As a platform operator + I need the scheduler to retry creating and terminating pipelines that have previously failed + +# todo: this scenario seems to work but we currently don't assert on status pipeline terminate + Scenario: Retry creating a pipeline that failed while Kafka was unavailable + Given kafka-nodepool is unavailable for Core 2 with timeout "40s" + Then I restart scheduler with timeout "30s" + Then I restart dataflow-engine with timeout "30s" + Then I restart model-gw with timeout "30s" + Then I restart pipeline-gw with timeout "30s" + Given I create model spec with timeout "30s": + """ + apiVersion: mlops.seldon.io/v1alpha1 + kind: Model + metadata: + name: tfsimple1-hhk2 + spec: + storageUri: "gs://seldon-models/triton/simple" + requirements: + - tensorflow + apiVersion: mlops.seldon.io/v1alpha1 + """ + Then the model "tfsimple1-hhk2" should eventually become Ready with timeout "30s" + And I deploy a pipeline spec with timeout "30s": + """ + apiVersion: mlops.seldon.io/v1alpha1 + kind: Pipeline + metadata: + name: retry-pipe-hhk2 + spec: + steps: + - name: tfsimple1-hhk2 + output: + steps: + - tfsimple1-hhk2 + """ + And the pipeline should eventually become NotReady with timeout "30s" + When kafka-nodepool is available for Core 2 with timeout "40s" + Then the pipeline should eventually become Ready with timeout "180s" + +# todo: this second scenario needs to work fully as it seems that the pipeline doesn't fail to terminate when kafka is down + Scenario: Retry terminating a pipeline that previously failed to terminate + Given I create model spec with timeout "30s": + """ + apiVersion: mlops.seldon.io/v1alpha1 + kind: Model + metadata: + name: tfsimple1-hfk5 + spec: + dataflow: + cleanTopicsOnDelete: true + storageUri: "gs://seldon-models/triton/simple" + requirements: + - tensorflow + apiVersion: mlops.seldon.io/v1alpha1 + """ + Then the model "tfsimple1-hfk5" should eventually become Ready with timeout "30s" + Given I deploy a pipeline spec with timeout "30s": + """ + apiVersion: mlops.seldon.io/v1alpha1 + kind: Pipeline + metadata: + name: retry-pipe-hfk5 + spec: + dataflow: + cleanTopicsOnDelete: true + steps: + - name: tfsimple1-hfk5 + output: + steps: + - tfsimple1-hfk5 + """ + Then the pipeline should eventually become Ready with timeout "120s" + When kafka-nodepool is unavailable for Core 2 with timeout "40s" + And I wait for "5s" + Then I restart scheduler with timeout "30s" + And I delete pipeline the with timeout "30s" + When kafka-nodepool is available for Core 2 with timeout "40s" + Then the pipeline should eventually not exist with timeout "120s" + diff --git a/tests/integration/godog/godog-config.json b/tests/integration/godog/godog-config.json index 2c04056468..76ecf0c250 100644 --- a/tests/integration/godog/godog-config.json +++ b/tests/integration/godog/godog-config.json @@ -3,6 +3,7 @@ "log_level": "debug", "skip_cleanup" : false, "skip_cleanup_on_error": false, + "scenario_step_delay": "0s", "inference" : { "host": "localhost", "httpPort": 9000, diff --git a/tests/integration/godog/k8sclient/watcher_store.go b/tests/integration/godog/k8sclient/watcher_store.go index e8701b9d77..9f4866344b 100644 --- a/tests/integration/godog/k8sclient/watcher_store.go +++ b/tests/integration/godog/k8sclient/watcher_store.go @@ -141,6 +141,8 @@ func (s *WatcherStore) Start() { case <-s.doneChan: // Stop underlying watcher and exit s.modelWatcher.Stop() + s.pipelineWatcher.Stop() + s.experimentWatcher.Stop() return } } @@ -386,6 +388,18 @@ func (s *WatcherStore) waitForKey(ctx context.Context, key string, cond Conditio } } + //// Call cond even if !ok; when !ok, existing will be nil. + //if !ok { + // existing = nil + //} + //done, err := cond(existing) + //if err != nil { + // return err + //} + //if done { + // return nil + //} + // Slow path: register a waiter w := &waiter{ key: key, diff --git a/tests/integration/godog/k8sclient/watcher_store_test.go b/tests/integration/godog/k8sclient/watcher_store_test.go new file mode 100644 index 0000000000..ab1fdde9ae --- /dev/null +++ b/tests/integration/godog/k8sclient/watcher_store_test.go @@ -0,0 +1,203 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + +package k8sclient + +import ( + "context" + "testing" + "time" + + "github.com/seldonio/seldon-core/operator/v2/apis/mlops/v1alpha1" + mlopsscheme "github.com/seldonio/seldon-core/operator/v2/pkg/generated/clientset/versioned/scheme" + log "github.com/sirupsen/logrus" + "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// helper to build a minimal WatcherStore for unit tests (no real watchers) +func newTestWatcherStore(t *testing.T) *WatcherStore { + t.Helper() + + s := runtime.NewScheme() + _ = scheme.AddToScheme(s) + _ = mlopsscheme.AddToScheme(s) + + return &WatcherStore{ + namespace: "test-ns", + logger: log.New().WithField("test", t.Name()), + store: make(map[string]runtime.Object), + scheme: s, + // watchers and doneChan are unused in these tests + } +} + +func newPipeline(name, ns string) *v1alpha1.Pipeline { + return &v1alpha1.Pipeline{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + } +} + +// 1) WaitForPipelineCondition returns immediately if condition already satisfied +func TestWaitForPipelineConditionImmediate(t *testing.T) { + ws := newTestWatcherStore(t) + p := newPipeline("p1", "test-ns") + + // pre-populate store + ws.put(p) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + called := false + cond := func(obj runtime.Object) (bool, error) { + called = true + if obj == nil { + return false, nil + } + // we only care that it's non-nil for this test + return true, nil + } + + if err := ws.WaitForPipelineCondition(ctx, "p1", cond); err != nil { + t.Fatalf("expected no error, got %v", err) + } + if !called { + t.Fatalf("condition func was not called in fast path") + } +} + +// 2) WaitForPipelineCondition blocks until a matching put() arrives +func TestWaitForPipelineConditionOnAddEvent(t *testing.T) { + ws := newTestWatcherStore(t) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + done := make(chan error, 1) + + go func() { + cond := func(obj runtime.Object) (bool, error) { + return obj != nil, nil + } + done <- ws.WaitForPipelineCondition(ctx, "p2", cond) + }() + + // give the waiter a moment to register + time.Sleep(100 * time.Millisecond) + + // now simulate the watch ADD event + p := newPipeline("p2", "test-ns") + ws.put(p) + + err := <-done + if err != nil { + t.Fatalf("expected no error, got %v", err) + } +} + +// 3) WaitForPipelineCondition with a "deleted" condition unblocks on delete +// +// This assumes you have a PipelineDeleted condition that treats obj == nil as deleted. +func TestWaitForPipelineDeletedOnDeleteEvent(t *testing.T) { + ws := newTestWatcherStore(t) + p := newPipeline("p3", "test-ns") + + // object exists initially + ws.put(p) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + done := make(chan error, 1) + + go func() { + // Using a deleted-style condition: obj == nil => done + cond := func(obj runtime.Object) (bool, error) { + // pipeline considered deleted when obj is nil + return obj == nil, nil + } + done <- ws.WaitForPipelineCondition(ctx, "p3", cond) + }() + + time.Sleep(100 * time.Millisecond) + + // simulate watch DELETED event + ws.delete(p) + + err := <-done + if err != nil { + t.Fatalf("expected no error, got %v", err) + } +} + +// 4) WaitForPipelineCondition returns context error if condition not met in time +func TestWaitForPipelineConditionTimeout(t *testing.T) { + ws := newTestWatcherStore(t) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + + cond := func(obj runtime.Object) (bool, error) { + // condition that can never be satisfied in this test + return false, nil + } + + err := ws.WaitForPipelineCondition(ctx, "non-existent", cond) + if err == nil { + t.Fatalf("expected timeout error, got nil") + } + if ctx.Err() == nil { + t.Fatalf("expected context to be cancelled") + } +} + +// 5) notifyWaiters removes waiter once condition satisfied +func TestNotifyWaitersRemovesSatisfiedWaiter(t *testing.T) { + ws := newTestWatcherStore(t) + + // We'll wait on a specific key + key := "test-ns/Pipeline/p4" + + resultCh := make(chan error, 1) + w := &waiter{ + key: key, + cond: func(obj runtime.Object) (bool, error) { + return true, nil // immediately satisfied + }, + result: resultCh, + } + + ws.mu.Lock() + ws.waiters = append(ws.waiters, w) + ws.mu.Unlock() + + // notify with some obj (could be nil or not, condition returns true anyway) + ws.notifyWaiters(key, nil) + + // waiter should have been signalled and removed + select { + case err := <-resultCh: + if err != nil { + t.Fatalf("expected nil error from waiter, got %v", err) + } + case <-time.After(time.Second): + t.Fatalf("expected waiter to be signalled") + } + + ws.mu.RLock() + defer ws.mu.RUnlock() + if len(ws.waiters) != 0 { + t.Fatalf("expected no remaining waiters, got %d", len(ws.waiters)) + } +} diff --git a/tests/integration/godog/steps/assertions/pipeline.go b/tests/integration/godog/steps/assertions/pipeline.go index 77003b1926..8dc5f45df6 100644 --- a/tests/integration/godog/steps/assertions/pipeline.go +++ b/tests/integration/godog/steps/assertions/pipeline.go @@ -32,3 +32,63 @@ func PipelineReady(obj runtime.Object) (bool, error) { return false, nil } + +func PipelineNotReady(obj runtime.Object) (bool, error) { + if obj == nil { + // pipeline does not exist (yet) or has been deleted: treat as "not ready" + return false, nil // or true,nil if you want "absence = not ready" + } + + pipeline, ok := obj.(*v1alpha1.Pipeline) + if !ok { + return false, fmt.Errorf("unexpected type %T, expected *v1alpha1.Pipeline", obj) + } + + if pipeline.Status.IsReady() { + return false, nil + } + + return true, nil +} + +// PipelineDeleted returns done=true when the pipeline no longer exists in the store. +// When called with obj == nil (delete event or never present), we treat it as deleted. +func PipelineDeleted(obj runtime.Object) (bool, error) { + if obj == nil { + // The object is not present in the store: consider it deleted. + return true, nil + } + + // If you want to be extra strict and only consider it "deleted" when actually gone, + // you could also check DeletionTimestamp here, but it's usually unnecessary. + pipeline, ok := obj.(*v1alpha1.Pipeline) + if !ok { + return false, fmt.Errorf("unexpected type %T, expected *v1alpha1.Pipeline", obj) + } + + // If you want to assert it's *in the process* of being deleted first: + if pipeline.DeletionTimestamp != nil { + // still present but marked for deletion; NOT done yet if you want full delete + return false, nil + } + + // still present and not being deleted -> not done + return false, nil +} + +//func PipelineReadyMessageCondition(expectedMessage string) k8sclient.ConditionFunc { +// return func(obj runtime.Object) (bool, error) { +// if obj == nil { +// return false, nil +// } +// +// pipeline, ok := obj.(*v1alpha1.Pipeline) +// if !ok { +// return false, fmt.Errorf("unexpected type %T, expected *mlopsv1alpha1.Model", obj) +// } +// +// pipeline.Status.Status. +// +// return false, nil +// } +//} diff --git a/tests/integration/godog/steps/infra_steps.go b/tests/integration/godog/steps/infra_steps.go new file mode 100644 index 0000000000..6ebafa3a3d --- /dev/null +++ b/tests/integration/godog/steps/infra_steps.go @@ -0,0 +1,163 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + +package steps + +import ( + "context" + "fmt" + + "github.com/cucumber/godog" + "github.com/seldonio/seldon-core/tests/integration/godog/components" + "github.com/sirupsen/logrus" +) + +// Infrastructure todo: add attributes as we need +type Infrastructure struct { + env *components.EnvManager + log logrus.FieldLogger +} + +func newInfrastructure(env *components.EnvManager, log logrus.FieldLogger) *Infrastructure { + return &Infrastructure{env: env, log: log} +} + +func LoadInfrastructureSteps(scenario *godog.ScenarioContext, w *World) { + scenario.Step(`^(kafka-nodepool) is unavailable for Core 2 with timeout "([^"]+)"`, func(kind, timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + switch kind { + case "kafka-nodepool": + k, err := w.env.Component(components.KafkaNodePool) + if err != nil { + return err + } + + return k.MakeUnavailable(ctx) + default: + return fmt.Errorf("unknown target type: %s", kind) + } + }) + }) + + scenario.Step(`^(kafka-nodepool) is available for Core 2 with timeout "([^"]+)"`, func(kind, timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + switch kind { + case "kafka-nodepool": + k, err := w.env.Component(components.KafkaNodePool) + if err != nil { + return err + } + + return k.MakeAvailable(ctx) + default: + return fmt.Errorf("unknown target type: %s", kind) + } + }) + }) + scenario.Step(`^I restart (scheduler|dataflow-engine|model-gw|pipeline-gw) with timeout "([^"]+)"$`, + func(kind, timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + runtime := w.env.Runtime() + if runtime == nil { + return fmt.Errorf("runtime not defined") + } + + svc, err := w.infra.runtimeServiceFromStepKind(kind) + if err != nil { + return err + } + + return runtime.RestartService(ctx, svc) + }) + }, + ) + scenario.Step(`^I set replicas of (scheduler|dataflow-engine|model-gw|pipeline-gw) to "([^"]+)" with timeout "([^"]+)"$`, + func(kind string, replicaCount int32, timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + runtime := w.env.Runtime() + if runtime == nil { + return fmt.Errorf("runtime not defined") + } + + svc, err := w.infra.runtimeServiceFromStepKind(kind) + if err != nil { + return err + } + + return runtime.SetReplicas(ctx, svc, replicaCount) + }) + }, + ) + scenario.Step(`^I wait for (scheduler|dataflow-engine|model-gw|pipeline-gw) to be ready with timeout "([^"]+)"$`, + func(kind, timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + runtime := w.env.Runtime() + if runtime == nil { + return fmt.Errorf("runtime not defined") + } + + svc, err := w.infra.runtimeServiceFromStepKind(kind) + if err != nil { + return err + } + + return runtime.WaitServiceReady(ctx, svc) + }) + }, + ) + scenario.Step(`^I scale down (scheduler|dataflow-engine|model-gw|pipeline-gw) with timeout "([^"]+)"$`, + func(kind, timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + runtime := w.env.Runtime() + if runtime == nil { + return fmt.Errorf("runtime not defined") + } + + svc, err := w.infra.runtimeServiceFromStepKind(kind) + if err != nil { + return err + } + + return runtime.ScaleDown(ctx, svc) + }) + }, + ) + scenario.Step(`^I scale up to baseline (scheduler|dataflow-engine|model-gw|pipeline-gw) with timeout "([^"]+)"$`, + func(kind, timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + runtime := w.env.Runtime() + if runtime == nil { + return fmt.Errorf("runtime not defined") + } + + svc, err := w.infra.runtimeServiceFromStepKind(kind) + if err != nil { + return err + } + + return runtime.ScaleUpToBaseline(ctx, svc) + }) + }, + ) +} + +func (i *Infrastructure) runtimeServiceFromStepKind(kind string) (components.SeldonRuntimeService, error) { + switch kind { + case "scheduler": + return components.ServiceScheduler, nil + case "dataflow-engine": + return components.ServiceDataflowEngine, nil + case "model-gw": + return components.ServiceModelGateway, nil + case "pipeline-gw": + return components.ServicePipelineGateway, nil + default: + return "", fmt.Errorf("unknown target type: %s", kind) + } +} diff --git a/tests/integration/godog/steps/pipeline_steps.go b/tests/integration/godog/steps/pipeline_steps.go index ca63313b2b..d1c97caba4 100644 --- a/tests/integration/godog/steps/pipeline_steps.go +++ b/tests/integration/godog/steps/pipeline_steps.go @@ -44,15 +44,49 @@ func LoadCustomPipelineSteps(scenario *godog.ScenarioContext, w *World) { return w.currentPipeline.deployPipelineSpec(ctx, spec) }) }) - - scenario.Step(`^the pipeline "([^"]+)" (?:should )?eventually become Ready with timeout "([^"]+)"$`, func(name, timeout string) error { - ctx, cancel, err := timeoutToContext(timeout) - if err != nil { - return err - } - defer cancel() - - return w.currentPipeline.waitForPipelineNameReady(ctx, name) + scenario.Step(`^the pipeline "([^"]+)" (?:should )?eventually become (Ready|NotReady) with timeout "([^"]+)"$`, func(name, readiness, timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + switch readiness { + case "Ready": + return w.currentPipeline.waitForPipelineNameReady(ctx, name) + case "NotReady": + return w.currentPipeline.waitForPipelineNameNotReady(ctx, name) + default: + return fmt.Errorf("unknown readiness type: %s", readiness) + } + }) + }) + scenario.Step(`^the pipeline (?:should )?eventually become (Ready|NotReady) with timeout "([^"]+)"$`, func(readiness, timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + switch readiness { + case "Ready": + return w.currentPipeline.waitForPipelineReady(ctx) + case "NotReady": + return w.currentPipeline.waitForPipelineNotReady(ctx) + default: + return fmt.Errorf("unknown readiness type: %s", readiness) + } + }) + }) + scenario.Step(`^I delete pipeline "([^"]+)" with timeout "([^"]+)"$`, func(pipeline, timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + return w.currentPipeline.deletePipelineName(ctx, pipeline) + }) + }) + scenario.Step(`^I delete pipeline (?:the )?with timeout "([^"]+)"$`, func(timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + return w.currentPipeline.deletePipeline(ctx) + }) + }) + scenario.Step(`^the pipeline "([^"]+)" should eventually not exist with timeout "([^"]+)"$`, func(pipeline, timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + return w.currentPipeline.waitForPipelineNameIsDeleted(ctx, pipeline) + }) + }) + scenario.Step(`^the pipeline should eventually not exist with timeout "([^"]+)"$`, func(timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + return w.currentPipeline.waitForPipelineIsDeleted(ctx) + }) }) } @@ -86,6 +120,14 @@ func (p *Pipeline) applyScenarioLabel() { } } +func (p *Pipeline) deletePipeline(ctx context.Context) error { + return p.k8sClient.MlopsV1alpha1().Pipelines(p.namespace).Delete(ctx, p.pipelineName, metav1.DeleteOptions{}) +} + +func (p *Pipeline) deletePipelineName(ctx context.Context, pipeline string) error { + return p.k8sClient.MlopsV1alpha1().Pipelines(p.namespace).Delete(ctx, pipeline, metav1.DeleteOptions{}) +} + func (p *Pipeline) waitForPipelineNameReady(ctx context.Context, name string) error { return p.watcherStorage.WaitForPipelineCondition( ctx, @@ -94,6 +136,14 @@ func (p *Pipeline) waitForPipelineNameReady(ctx context.Context, name string) er ) } +func (p *Pipeline) waitForPipelineNameNotReady(ctx context.Context, name string) error { + return p.watcherStorage.WaitForPipelineCondition( + ctx, + name, + assertions.PipelineNotReady, + ) +} + func (p *Pipeline) waitForPipelineReady(ctx context.Context) error { return p.watcherStorage.WaitForObjectCondition( ctx, @@ -101,3 +151,27 @@ func (p *Pipeline) waitForPipelineReady(ctx context.Context) error { assertions.PipelineReady, ) } + +func (p *Pipeline) waitForPipelineNotReady(ctx context.Context) error { + return p.watcherStorage.WaitForObjectCondition( + ctx, + p.pipeline, + assertions.PipelineNotReady, + ) +} + +func (p *Pipeline) waitForPipelineNameIsDeleted(ctx context.Context, pipeline string) error { + return p.watcherStorage.WaitForPipelineCondition( + ctx, + pipeline, + assertions.PipelineDeleted, + ) +} + +func (p *Pipeline) waitForPipelineIsDeleted(ctx context.Context) error { + return p.watcherStorage.WaitForPipelineCondition( + ctx, + p.pipelineName, + assertions.PipelineDeleted, + ) +} diff --git a/tests/integration/godog/steps/world.go b/tests/integration/godog/steps/world.go index 3338e6e43a..5c19317a3d 100644 --- a/tests/integration/godog/steps/world.go +++ b/tests/integration/godog/steps/world.go @@ -14,23 +14,23 @@ import ( "github.com/seldonio/seldon-core/apis/go/v2/mlops/v2_dataplane" v "github.com/seldonio/seldon-core/operator/v2/pkg/generated/clientset/versioned" + "github.com/seldonio/seldon-core/tests/integration/godog/components" "github.com/seldonio/seldon-core/tests/integration/godog/k8sclient" log "github.com/sirupsen/logrus" ) type World struct { - namespace string - kubeClient *k8sclient.K8sClient - corek8sClient v.Interface - watcherStorage k8sclient.WatcherStorage - StartingClusterState string //todo: this will be a combination of starting state awareness of core 2 such as the - //todo: server config,seldon config and seldon runtime to be able to reconcile to starting state should we change - //todo: the state such as reducing replicas to 0 of scheduler to test unavailability + namespace string + kubeClient *k8sclient.K8sClient + corek8sClient v.Interface + watcherStorage k8sclient.WatcherStorage + env *components.EnvManager //this is a combination of components for the cluster currentModel *Model currentPipeline *Pipeline currentExperiment *Experiment server *server infer inference + infra *Infrastructure logger log.FieldLogger Label map[string]string } @@ -41,6 +41,7 @@ type Config struct { KubeClient *k8sclient.K8sClient K8sClient v.Interface WatcherStorage k8sclient.WatcherStorage + Env *components.EnvManager GRPC v2_dataplane.GRPCInferenceServiceClient IngressHost string HTTPPort uint @@ -62,6 +63,7 @@ func NewWorld(c Config) (*World, error) { namespace: c.Namespace, kubeClient: c.KubeClient, watcherStorage: c.WatcherStorage, + env: c.Env, currentModel: newModel(label, c.Namespace, c.K8sClient, c.Logger, c.WatcherStorage), currentExperiment: newExperiment(label, c.Namespace, c.K8sClient, c.Logger, c.WatcherStorage), currentPipeline: newPipeline(label, c.Namespace, c.K8sClient, c.Logger, c.WatcherStorage), @@ -73,6 +75,7 @@ func NewWorld(c Config) (*World, error) { httpPort: c.HTTPPort, log: c.Logger, ssl: c.SSL}, + infra: newInfrastructure(c.Env, c.Logger), Label: label, } diff --git a/tests/integration/godog/suite/config.go b/tests/integration/godog/suite/config.go index b6a92d5381..7f21dfed59 100644 --- a/tests/integration/godog/suite/config.go +++ b/tests/integration/godog/suite/config.go @@ -13,6 +13,8 @@ import ( "encoding/json" "fmt" "os" + "strings" + "time" ) type GodogConfig struct { @@ -20,6 +22,7 @@ type GodogConfig struct { LogLevel string `json:"log_level"` SkipCleanup bool `json:"skip_cleanup"` SkipCleanUpOnError bool `json:"skip_clean_up_on_error"` + ScenarioStepDelay Duration `json:"scenario_step_delay"` Inference Inference `json:"inference"` } @@ -55,3 +58,22 @@ func LoadConfig() (*GodogConfig, error) { return &config, nil } + +type Duration time.Duration + +func (d *Duration) UnmarshalJSON(b []byte) error { + // Trim quotes + s := strings.Trim(string(b), `"`) + if s == "null" || s == "" { + *d = 0 + return nil + } + + dur, err := time.ParseDuration(s) + if err != nil { + return err + } + + *d = Duration(dur) + return nil +} diff --git a/tests/integration/godog/suite/suite.go b/tests/integration/godog/suite/suite.go index 1f8d4779ce..e62b328865 100644 --- a/tests/integration/godog/suite/suite.go +++ b/tests/integration/godog/suite/suite.go @@ -13,10 +13,12 @@ import ( "context" "crypto/tls" "fmt" + "time" "github.com/cucumber/godog" v2dataplane "github.com/seldonio/seldon-core/apis/go/v2/mlops/v2_dataplane" v "github.com/seldonio/seldon-core/operator/v2/pkg/generated/clientset/versioned" + "github.com/seldonio/seldon-core/tests/integration/godog/components" "github.com/seldonio/seldon-core/tests/integration/godog/k8sclient" "github.com/seldonio/seldon-core/tests/integration/godog/steps" "github.com/sirupsen/logrus" @@ -32,6 +34,7 @@ type dependencies struct { mlopsClient *v.Clientset watcherStore k8sclient.WatcherStorage inferenceClient v2dataplane.GRPCInferenceServiceClient + infraManager *components.EnvManager Config *GodogConfig } @@ -53,6 +56,8 @@ type dependencies struct { var suiteDeps dependencies func InitializeTestSuite(ctx *godog.TestSuiteContext) { + realContext, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() // Load configuration from JSON file config, err := LoadConfig() if err != nil { @@ -101,12 +106,23 @@ func InitializeTestSuite(ctx *godog.TestSuiteContext) { } grpcClient := v2dataplane.NewGRPCInferenceServiceClient(conn) + infraManger, err := components.StartComponents(k8sClient, config.Namespace) + if err != nil { + panic(err) + } + + // Snapshot baseline state once per suite + if err := infraManger.SnapshotAll(realContext); err != nil { + panic(fmt.Errorf("failed to snapshot environment: %w", err)) + } + suiteDeps.logger = log suiteDeps.k8sClient = k8sClient suiteDeps.mlopsClient = clientSet // todo: this clientSet might get use for get requests or for the mlops interface and could be passed to the world might be split up by type suiteDeps.watcherStore = watchStore suiteDeps.Config = config suiteDeps.inferenceClient = grpcClient + suiteDeps.infraManager = infraManger ctx.BeforeSuite(func() { suiteDeps.watcherStore.Start() @@ -118,6 +134,7 @@ func InitializeTestSuite(ctx *godog.TestSuiteContext) { ctx.AfterSuite(func() { suiteDeps.watcherStore.Stop() + // e.g. clean namespace, close clients if needed delete servers }) } @@ -131,6 +148,7 @@ func InitializeScenario(scenarioCtx *godog.ScenarioContext) { KubeClient: suiteDeps.k8sClient, K8sClient: suiteDeps.mlopsClient, WatcherStorage: suiteDeps.watcherStore, + Env: suiteDeps.infraManager, IngressHost: suiteDeps.Config.Inference.Host, HTTPPort: suiteDeps.Config.Inference.HTTPPort, GRPCPort: suiteDeps.Config.Inference.GRPCPort, @@ -141,13 +159,27 @@ func InitializeScenario(scenarioCtx *godog.ScenarioContext) { panic(fmt.Errorf("failed to create world: %w", err)) } + var slowForThis bool // Before: reset state and prep cluster before each scenario scenarioCtx.Before(func(ctx context.Context, scenario *godog.Scenario) (context.Context, error) { + // Enable slow mode only for @slow scenarios recommended only for local testing + slowForThis = false + for _, t := range scenario.Tags { + if t.Name == "@slow" { + slowForThis = true + log.WithField("scenario", scenario.Name).Debugf("set to run slow scenario at %s per step", time.Duration(suiteDeps.Config.ScenarioStepDelay)) + break + } + } return ctx, nil }) // After: optional cleanup / rollback scenarioCtx.After(func(ctx context.Context, scenario *godog.Scenario, err error) (context.Context, error) { + err = suiteDeps.infraManager.RestoreAll(context.Background()) + if err != nil { + return ctx, err + } if err != nil && suiteDeps.Config.SkipCleanUpOnError { log.WithField("scenario", scenario.Name).Debugf("Skipping cleanup of resources for scenario with err %v", err) // don't clean up resources for scenarios that fail @@ -159,12 +191,21 @@ func InitializeScenario(scenarioCtx *godog.ScenarioContext) { } if err := suiteDeps.k8sClient.DeleteScenarioResources(ctx, world.Label); err != nil { - return ctx, fmt.Errorf("error when deleting models on before steps: %w", err) + return ctx, fmt.Errorf("error when deleting models on after scenario steps: %w", err) } return ctx, nil }) + // --- Step hook: delay after every step --- + scenarioCtx.StepContext().After(func(ctx context.Context, st *godog.Step, status godog.StepResultStatus, err error) (context.Context, error) { + // set delay for scenario steps when @slow tag is present + if slowForThis { + time.Sleep(time.Duration(suiteDeps.Config.ScenarioStepDelay)) + } + return ctx, nil + }) + // Register step definitions with access to world + k8sClient steps.LoadTemplateModelSteps(scenarioCtx, world) steps.LoadCustomModelSteps(scenarioCtx, world) @@ -173,4 +214,5 @@ func InitializeScenario(scenarioCtx *godog.ScenarioContext) { steps.LoadCustomPipelineSteps(scenarioCtx, world) steps.LoadExperimentSteps(scenarioCtx, world) steps.LoadUtilSteps(scenarioCtx, world) + steps.LoadInfrastructureSteps(scenarioCtx, world) }