diff --git a/pkg/cluster/condition.go b/pkg/cluster/condition.go index a5f1fbd9140..0c7eee820d6 100644 --- a/pkg/cluster/condition.go +++ b/pkg/cluster/condition.go @@ -28,6 +28,32 @@ func (m *manager) apiServersReady(ctx context.Context) (bool, error) { } func (m *manager) minimumWorkerNodesReady(ctx context.Context) (bool, error) { + machines, err := m.maocli.MachineV1beta1().Machines("openshift-machine-api").List(ctx, metav1.ListOptions{ + LabelSelector: "machine.openshift.io/cluster-api-machine-role=worker", + }) + if err != nil { + m.log.Error(err) + return false, nil + } + + readyWorkerMachines := 0 + for _, machine := range machines.Items { + if *machine.Status.Phase == "Running" { + readyWorkerMachines++ + } else { + if err == nil { + m.log.Errorf("Machine %s is %s; status: %v", machine.Name, *machine.Status.Phase, string(machine.Status.ProviderStatus.Raw)) + } else { + m.log.Errorf("Machine %s is %s; error decoding status: %s", machine.Name, *machine.Status.Phase, err) + } + } + } + + if readyWorkerMachines < minimumWorkerNodes { + m.log.Infof("%d machines out of %d machines ready", readyWorkerMachines, minimumWorkerNodes) + return false, nil + } + nodes, err := m.kubernetescli.CoreV1().Nodes().List(ctx, metav1.ListOptions{ LabelSelector: "node-role.kubernetes.io/worker", }) diff --git a/pkg/cluster/condition_test.go b/pkg/cluster/condition_test.go index a0515659204..f5ee3d84f4a 100644 --- a/pkg/cluster/condition_test.go +++ b/pkg/cluster/condition_test.go @@ -5,13 +5,16 @@ package cluster import ( "context" + "encoding/json" "errors" "testing" "time" configv1 "github.com/openshift/api/config/v1" + machinev1beta1 "github.com/openshift/api/machine/v1beta1" operatorv1 "github.com/openshift/api/operator/v1" configfake "github.com/openshift/client-go/config/clientset/versioned/fake" + machinefake "github.com/openshift/client-go/machine/clientset/versioned/fake" operatorfake "github.com/openshift/client-go/operator/clientset/versioned/fake" cloudcredentialv1 "github.com/openshift/cloud-credential-operator/pkg/apis/cloudcredential/v1" "github.com/sirupsen/logrus" @@ -29,6 +32,13 @@ import ( const errMustBeNilMsg = "err must be nil; condition is retried until timeout" +func marshalAzureMachineProviderStatus(t *testing.T, status *machinev1beta1.AzureMachineProviderStatus) *runtime.RawExtension { + buf, _ := json.Marshal(status) + return &runtime.RawExtension{ + Raw: buf, + } +} + func TestOperatorConsoleExists(t *testing.T) { ctx := context.Background() @@ -118,6 +128,8 @@ func TestIsOperatorAvailable(t *testing.T) { func TestMinimumWorkerNodesReady(t *testing.T) { ctx := context.Background() + machineStateRunning := "Running" + machineStateFailed := "Failed" for _, tt := range []struct { name string @@ -149,6 +161,50 @@ func TestMinimumWorkerNodesReady(t *testing.T) { }, } { m := &manager{ + log: logrus.NewEntry(logrus.StandardLogger()), + maocli: machinefake.NewSimpleClientset( + &machinev1beta1.Machine{ + ObjectMeta: metav1.ObjectMeta{Name: "node1", + Namespace: "openshift-machine-api", + Labels: map[string]string{ + "machine.openshift.io/cluster-api-machine-role": "worker", + "machine.openshift.io/cluster-api-machine-type": "worker", + }, + }, + Status: machinev1beta1.MachineStatus{ + Phase: &machineStateRunning, + ProviderStatus: marshalAzureMachineProviderStatus(t, &machinev1beta1.AzureMachineProviderStatus{}), + }, + }, + &machinev1beta1.Machine{ + ObjectMeta: metav1.ObjectMeta{Name: "node2", + Namespace: "openshift-machine-api", + Labels: map[string]string{ + "machine.openshift.io/cluster-api-machine-role": "worker", + "machine.openshift.io/cluster-api-machine-type": "worker", + }, + }, + Status: machinev1beta1.MachineStatus{ + Phase: &machineStateRunning, + ProviderStatus: marshalAzureMachineProviderStatus(t, &machinev1beta1.AzureMachineProviderStatus{}), + }, + }, + &machinev1beta1.Machine{ + ObjectMeta: metav1.ObjectMeta{Name: "node3", + Namespace: "openshift-machine-api", + Labels: map[string]string{ + "machine.openshift.io/cluster-api-machine-role": "worker", + "machine.openshift.io/cluster-api-machine-type": "worker", + }, + }, + Status: machinev1beta1.MachineStatus{ + Phase: &machineStateFailed, + ProviderStatus: marshalAzureMachineProviderStatus(t, &machinev1beta1.AzureMachineProviderStatus{}), + }, + }, + testMachine(t, "openshift-machine-api", "master1", &machinev1beta1.AzureMachineProviderSpec{}), + testMachine(t, "openshift-machine-api", "master2", &machinev1beta1.AzureMachineProviderSpec{}), + ), kubernetescli: fake.NewSimpleClientset(&corev1.NodeList{ Items: []corev1.Node{ { @@ -187,7 +243,7 @@ func TestMinimumWorkerNodesReady(t *testing.T) { t.Error(errMustBeNilMsg) } if ready != tt.want { - t.Error(ready) + t.Error(tt.name, ready) } } } diff --git a/pkg/util/mocks/azblob/azblob.go b/pkg/util/mocks/azblob/azblob.go deleted file mode 100644 index ddffccb2cfc..00000000000 --- a/pkg/util/mocks/azblob/azblob.go +++ /dev/null @@ -1,125 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/Azure/ARO-RP/pkg/util/azblob (interfaces: Manager,AZBlobClient) -// -// Generated by this command: -// -// mockgen -destination=../mocks/azblob/azblob.go github.com/Azure/ARO-RP/pkg/util/azblob Manager,AZBlobClient -// - -// Package mock_azblob is a generated GoMock package. -package mock_azblob - -import ( - context "context" - reflect "reflect" - - armstorage "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage" - azblob0 "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - gomock "go.uber.org/mock/gomock" - - azblob "github.com/Azure/ARO-RP/pkg/util/azblob" -) - -// MockManager is a mock of Manager interface. -type MockManager struct { - ctrl *gomock.Controller - recorder *MockManagerMockRecorder -} - -// MockManagerMockRecorder is the mock recorder for MockManager. -type MockManagerMockRecorder struct { - mock *MockManager -} - -// NewMockManager creates a new mock instance. -func NewMockManager(ctrl *gomock.Controller) *MockManager { - mock := &MockManager{ctrl: ctrl} - mock.recorder = &MockManagerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockManager) EXPECT() *MockManagerMockRecorder { - return m.recorder -} - -// GetAZBlobClient mocks base method. -func (m *MockManager) GetAZBlobClient(arg0 string, arg1 *azblob0.ClientOptions) (azblob.AZBlobClient, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAZBlobClient", arg0, arg1) - ret0, _ := ret[0].(azblob.AZBlobClient) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetAZBlobClient indicates an expected call of GetAZBlobClient. -func (mr *MockManagerMockRecorder) GetAZBlobClient(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAZBlobClient", reflect.TypeOf((*MockManager)(nil).GetAZBlobClient), arg0, arg1) -} - -// GetContainerProperties mocks base method. -func (m *MockManager) GetContainerProperties(arg0 context.Context, arg1, arg2, arg3 string) (armstorage.AccountsClientGetPropertiesResponse, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetContainerProperties", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(armstorage.AccountsClientGetPropertiesResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetContainerProperties indicates an expected call of GetContainerProperties. -func (mr *MockManagerMockRecorder) GetContainerProperties(arg0, arg1, arg2, arg3 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetContainerProperties", reflect.TypeOf((*MockManager)(nil).GetContainerProperties), arg0, arg1, arg2, arg3) -} - -// MockAZBlobClient is a mock of AZBlobClient interface. -type MockAZBlobClient struct { - ctrl *gomock.Controller - recorder *MockAZBlobClientMockRecorder -} - -// MockAZBlobClientMockRecorder is the mock recorder for MockAZBlobClient. -type MockAZBlobClientMockRecorder struct { - mock *MockAZBlobClient -} - -// NewMockAZBlobClient creates a new mock instance. -func NewMockAZBlobClient(ctrl *gomock.Controller) *MockAZBlobClient { - mock := &MockAZBlobClient{ctrl: ctrl} - mock.recorder = &MockAZBlobClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockAZBlobClient) EXPECT() *MockAZBlobClientMockRecorder { - return m.recorder -} - -// DeleteBlob mocks base method. -func (m *MockAZBlobClient) DeleteBlob(arg0 context.Context, arg1, arg2 string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteBlob", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteBlob indicates an expected call of DeleteBlob. -func (mr *MockAZBlobClientMockRecorder) DeleteBlob(arg0, arg1, arg2 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteBlob", reflect.TypeOf((*MockAZBlobClient)(nil).DeleteBlob), arg0, arg1, arg2) -} - -// UploadBuffer mocks base method. -func (m *MockAZBlobClient) UploadBuffer(arg0 context.Context, arg1, arg2 string, arg3 []byte) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UploadBuffer", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(error) - return ret0 -} - -// UploadBuffer indicates an expected call of UploadBuffer. -func (mr *MockAZBlobClientMockRecorder) UploadBuffer(arg0, arg1, arg2, arg3 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UploadBuffer", reflect.TypeOf((*MockAZBlobClient)(nil).UploadBuffer), arg0, arg1, arg2, arg3) -}