diff --git a/operator/src/main/java/oracle/kubernetes/operator/Main.java b/operator/src/main/java/oracle/kubernetes/operator/Main.java index ac12949612c..d015d072d46 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/Main.java +++ b/operator/src/main/java/oracle/kubernetes/operator/Main.java @@ -23,6 +23,7 @@ import javax.annotation.Nonnull; import io.kubernetes.client.openapi.models.CoreV1EventList; +import io.kubernetes.client.openapi.models.V1CustomResourceDefinition; import io.kubernetes.client.openapi.models.V1Namespace; import io.kubernetes.client.openapi.models.V1NamespaceList; import io.kubernetes.client.openapi.models.V1ObjectMeta; @@ -137,6 +138,7 @@ static class MainDelegateImpl implements MainDelegate, DomainProcessorDelegate { private final Engine engine; private final DomainProcessor domainProcessor; private final DomainNamespaces domainNamespaces; + private final AtomicReference crdRefernce; public MainDelegateImpl(Properties buildProps, ScheduledExecutorService scheduledExecutorService) { buildVersion = getBuildVersion(buildProps); @@ -152,6 +154,8 @@ public MainDelegateImpl(Properties buildProps, ScheduledExecutorService schedule domainNamespaces = new DomainNamespaces(productVersion); PodHelper.setProductVersion(productVersion.toString()); + + crdRefernce = new AtomicReference<>(); } private static String getBuildVersion(Properties buildProps) { @@ -248,6 +252,11 @@ public FiberGate createFiberGate() { public ScheduledFuture scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit) { return engine.getExecutor().scheduleWithFixedDelay(command, initialDelay, delay, unit); } + + @Override + public AtomicReference getCrdReference() { + return crdRefernce; + } } /** @@ -428,7 +437,7 @@ private Step createDomainRecheckSteps(OffsetDateTime now) { final DomainRecheck domainRecheck = new DomainRecheck(delegate, isFullRecheck); return Step.chain( domainRecheck.createOperatorNamespaceReview(), - CrdHelper.createDomainCrdStep(delegate.getKubernetesVersion(), delegate.getProductVersion()), + CrdHelper.createDomainCrdStep(delegate), createCRDPresenceCheck(), domainRecheck.createReadNamespacesStep()); } @@ -437,12 +446,30 @@ private Step createDomainRecheckSteps(OffsetDateTime now) { // domains in the operator's namespace. That should succeed (although usually returning an empty list) // if the CRD is present. Step createCRDPresenceCheck() { - return new CallBuilder().listDomainAsync(getOperatorNamespace(), new CrdPresenceResponseStep()); + return new CrdPresenceStep(); + } + + class CrdPresenceStep extends Step { + + @Override + public NextAction apply(Packet packet) { + V1CustomResourceDefinition existingCrd = delegate.getCrdReference().get(); + if (existingCrd != null) { + warnedOfCrdAbsence = false; + return doNext(packet); + } + return doNext(new CallBuilder().listDomainAsync(getOperatorNamespace(), + new CrdPresenceResponseStep(getNext())), packet); + } } // on failure, aborts the processing. class CrdPresenceResponseStep extends DefaultResponseStep { + CrdPresenceResponseStep(Step next) { + super(next); + } + @Override public NextAction onSuccess(Packet packet, CallResponse callResponse) { warnedOfCrdAbsence = false; diff --git a/operator/src/main/java/oracle/kubernetes/operator/MainDelegate.java b/operator/src/main/java/oracle/kubernetes/operator/MainDelegate.java index edfcdafb0cb..1bf9b64dbc0 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/MainDelegate.java +++ b/operator/src/main/java/oracle/kubernetes/operator/MainDelegate.java @@ -1,11 +1,13 @@ -// Copyright (c) 2020, 2021, Oracle and/or its affiliates. +// Copyright (c) 2020, 2022, Oracle and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. package oracle.kubernetes.operator; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import io.kubernetes.client.openapi.models.V1CustomResourceDefinition; import oracle.kubernetes.operator.helpers.KubernetesVersion; import oracle.kubernetes.operator.helpers.SemanticVersion; import oracle.kubernetes.operator.work.Packet; @@ -14,7 +16,7 @@ /** * Definition of an interface that returns values that the Main class requires. */ -interface MainDelegate { +public interface MainDelegate { SemanticVersion getProductVersion(); @@ -33,4 +35,6 @@ default void runSteps(Step firstStep) { KubernetesVersion getKubernetesVersion(); ScheduledFuture scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit); + + AtomicReference getCrdReference(); } diff --git a/operator/src/main/java/oracle/kubernetes/operator/helpers/CrdHelper.java b/operator/src/main/java/oracle/kubernetes/operator/helpers/CrdHelper.java index f741b9b84a8..90fd22cbd9f 100644 --- a/operator/src/main/java/oracle/kubernetes/operator/helpers/CrdHelper.java +++ b/operator/src/main/java/oracle/kubernetes/operator/helpers/CrdHelper.java @@ -1,4 +1,4 @@ -// Copyright (c) 2018, 2021, Oracle and/or its affiliates. +// Copyright (c) 2018, 2022, Oracle and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. package oracle.kubernetes.operator.helpers; @@ -17,6 +17,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -35,6 +36,7 @@ import okhttp3.internal.http2.StreamResetException; import oracle.kubernetes.operator.KubernetesConstants; import oracle.kubernetes.operator.LabelConstants; +import oracle.kubernetes.operator.MainDelegate; import oracle.kubernetes.operator.calls.CallResponse; import oracle.kubernetes.operator.logging.LoggingFacade; import oracle.kubernetes.operator.logging.LoggingFactory; @@ -74,7 +76,7 @@ public static void main(String[] args) throws URISyntaxException { } static void writeCrdFiles(String crdFileName) throws URISyntaxException { - CrdContext context = new CrdContext(null, null, null); + CrdContext context = new CrdContext(null, null); final URI outputFile = asFileURI(crdFileName); @@ -120,8 +122,8 @@ private static void dumpYaml(Writer writer, Object model) { // Map = gson.fromJson(Map.class) // yaml dump ? // ordering and format likely to change massively - public static Step createDomainCrdStep(KubernetesVersion version, SemanticVersion productVersion) { - return new CrdStep(version, productVersion); + public static Step createDomainCrdStep(MainDelegate mainDelegate) { + return new CrdStep(mainDelegate); } private static List getVersions(V1CustomResourceDefinition crd) { @@ -144,8 +146,8 @@ boolean isOutdatedCrd( static class CrdStep extends Step { final CrdContext context; - CrdStep(KubernetesVersion version, SemanticVersion productVersion) { - context = new CrdContext(version, productVersion, this); + CrdStep(MainDelegate mainDelegate) { + context = new CrdContext(mainDelegate, this); } @Override @@ -158,14 +160,12 @@ public NextAction apply(Packet packet) { static class CrdContext { private final Step conflictStep; private final V1CustomResourceDefinition model; - private final KubernetesVersion version; - private final SemanticVersion productVersion; + private final MainDelegate mainDelegate; - CrdContext(KubernetesVersion version, SemanticVersion productVersion, Step conflictStep) { - this.version = version; - this.productVersion = productVersion; + CrdContext(MainDelegate mainDelegate, Step conflictStep) { + this.mainDelegate = mainDelegate; this.conflictStep = conflictStep; - this.model = createModel(productVersion); + this.model = createModel(Optional.ofNullable(mainDelegate).map(MainDelegate::getProductVersion).orElse(null)); } static V1CustomResourceDefinition createModel(SemanticVersion productVersion) { @@ -294,7 +294,7 @@ ResponseStep createCreateResponseStep(Step next) { } private boolean isOutdatedCrd(V1CustomResourceDefinition existingCrd) { - return COMPARATOR.isOutdatedCrd(productVersion, existingCrd, this.model); + return COMPARATOR.isOutdatedCrd(mainDelegate.getProductVersion(), existingCrd, this.model); } private boolean existingCrdContainsVersion(V1CustomResourceDefinition existingCrd) { @@ -349,6 +349,8 @@ class ReadResponseStep extends DefaultResponseStep { public NextAction onSuccess( Packet packet, CallResponse callResponse) { V1CustomResourceDefinition existingCrd = callResponse.getResult(); + mainDelegate.getCrdReference().set(existingCrd); + if (existingCrd == null) { return doNext(createCrd(getNext()), packet); } else if (isOutdatedCrd(existingCrd)) { @@ -362,6 +364,7 @@ public NextAction onSuccess( @Override protected NextAction onFailureNoRetry(Packet packet, CallResponse callResponse) { + mainDelegate.getCrdReference().set(null); return isNotAuthorizedOrForbidden(callResponse) ? doNext(packet) : super.onFailureNoRetry(packet, callResponse); } @@ -381,7 +384,10 @@ public NextAction onFailure( @Override public NextAction onSuccess( Packet packet, CallResponse callResponse) { - LOGGER.info(MessageKeys.CREATING_CRD, callResponse.getResult().getMetadata().getName()); + V1CustomResourceDefinition existingCrd = callResponse.getResult(); + mainDelegate.getCrdReference().set(existingCrd); + + LOGGER.info(MessageKeys.CREATING_CRD, existingCrd.getMetadata().getName()); return doNext(packet); } @@ -407,7 +413,9 @@ public NextAction onFailure( @Override public NextAction onSuccess( Packet packet, CallResponse callResponse) { - LOGGER.info(MessageKeys.CREATING_CRD, callResponse.getResult().getMetadata().getName()); + V1CustomResourceDefinition existingCrd = callResponse.getResult(); + mainDelegate.getCrdReference().set(existingCrd); + LOGGER.info(MessageKeys.CREATING_CRD, existingCrd.getMetadata().getName()); return doNext(packet); } diff --git a/operator/src/test/java/oracle/kubernetes/operator/MainTest.java b/operator/src/test/java/oracle/kubernetes/operator/MainTest.java index 3eefd7d6923..c42b9d1bb97 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/MainTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/MainTest.java @@ -1,4 +1,4 @@ -// Copyright (c) 2018, 2021, Oracle and/or its affiliates. +// Copyright (c) 2018, 2022, Oracle and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. package oracle.kubernetes.operator; @@ -15,6 +15,7 @@ import java.util.Properties; import java.util.Random; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.stream.Collectors; @@ -24,6 +25,7 @@ import com.meterware.simplestub.StaticStubSupport; import io.kubernetes.client.openapi.models.CoreV1Event; import io.kubernetes.client.openapi.models.V1ConfigMap; +import io.kubernetes.client.openapi.models.V1CustomResourceDefinition; import io.kubernetes.client.openapi.models.V1Namespace; import io.kubernetes.client.openapi.models.V1ObjectMeta; import io.kubernetes.client.openapi.models.V1ObjectReference; @@ -358,12 +360,23 @@ private void runCreateReadNamespacesStep() { void whenNoCRD_logReasonForFailure() { loggerControl.withLogLevel(Level.SEVERE).collectLogMessages(logRecords, CRD_NOT_INSTALLED); simulateMissingCRD(); + delegate.hideCRD(); recheckDomains(); assertThat(logRecords, containsSevere(CRD_NOT_INSTALLED)); } + @Test + void whenCRDCreated_dontLogFailure() { + loggerControl.withLogLevel(Level.SEVERE).collectLogMessages(logRecords, CRD_NOT_INSTALLED); + simulateMissingCRD(); + + recheckDomains(); + + assertThat(logRecords, not(containsSevere(CRD_NOT_INSTALLED))); + } + @Test void afterLoggedCRDMissing_dontDoItASecondTime() { loggerControl.withLogLevel(Level.SEVERE).collectLogMessages(logRecords, CRD_NOT_INSTALLED); @@ -380,6 +393,7 @@ void afterLoggedCRDMissing_dontDoItASecondTime() { void afterMissingCRDdetected_correctionOfTheConditionAllowsProcessingToOccur() { defineSelectionStrategy(SelectionStrategy.Dedicated); simulateMissingCRD(); + delegate.hideCRD(); recheckDomains(); testSupport.cancelFailures(); @@ -397,6 +411,7 @@ void afterMissingCRDcorrected_subsequentFailureLogsReasonForFailure() { loggerControl.withLogLevel(Level.SEVERE).collectLogMessages(logRecords, CRD_NOT_INSTALLED); simulateMissingCRD(); + delegate.hideCRD(); recheckDomains(); assertThat(logRecords, containsSevere(CRD_NOT_INSTALLED)); @@ -1155,6 +1170,8 @@ void withNamespaceDedicated_changeToList_onCreateReadNamespaces_StartManagingNSE abstract static class MainDelegateStub implements MainDelegate { private final FiberTestSupport testSupport; private final DomainNamespaces domainNamespaces; + private final AtomicReference crdReference = new AtomicReference<>(); + private boolean hideCRD = false; public MainDelegateStub(FiberTestSupport testSupport, DomainNamespaces domainNamespaces) { this.testSupport = testSupport; @@ -1187,6 +1204,15 @@ public KubernetesVersion getKubernetesVersion() { public SemanticVersion getProductVersion() { return SemanticVersion.TEST_VERSION; } + + public void hideCRD() { + this.hideCRD = true; + } + + @Override + public AtomicReference getCrdReference() { + return hideCRD ? new AtomicReference<>() : crdReference; + } } static class TestStepFactory implements Main.NextStepFactory { diff --git a/operator/src/test/java/oracle/kubernetes/operator/helpers/CrdHelperTest.java b/operator/src/test/java/oracle/kubernetes/operator/helpers/CrdHelperTest.java index 05bf39ab2aa..9d524269d97 100644 --- a/operator/src/test/java/oracle/kubernetes/operator/helpers/CrdHelperTest.java +++ b/operator/src/test/java/oracle/kubernetes/operator/helpers/CrdHelperTest.java @@ -1,4 +1,4 @@ -// Copyright (c) 2018, 2021, Oracle and/or its affiliates. +// Copyright (c) 2018, 2022, Oracle and/or its affiliates. // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. package oracle.kubernetes.operator.helpers; @@ -11,6 +11,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.logging.Level; import java.util.logging.LogRecord; @@ -25,6 +26,7 @@ import io.kubernetes.client.openapi.models.V1ObjectMeta; import oracle.kubernetes.operator.KubernetesConstants; import oracle.kubernetes.operator.LabelConstants; +import oracle.kubernetes.operator.MainDelegate; import oracle.kubernetes.operator.utils.InMemoryFileSystem; import oracle.kubernetes.operator.work.Step; import oracle.kubernetes.operator.work.TerminalStep; @@ -144,7 +146,8 @@ Map getAdditionalPropertiesMap(String... pathElements) { @Test void whenCrdV1SupportedAndNoCrd_createIt() { - testSupport.runSteps(CrdHelper.createDomainCrdStep(KUBERNETES_VERSION_16, PRODUCT_VERSION)); + MainDelegateStub delegate = createStrictStub(MainDelegateStub.class, KUBERNETES_VERSION_16, PRODUCT_VERSION); + testSupport.runSteps(CrdHelper.createDomainCrdStep(delegate)); assertThat(logRecords, containsInfo(CREATING_CRD)); } @@ -154,7 +157,8 @@ void whenNoCrd_retryOnFailureAndLogFailedMessageInOnFailureNoRetry() { testSupport.addRetryStrategy(retryStrategy); testSupport.failOnCreate(CUSTOM_RESOURCE_DEFINITION, KubernetesConstants.CRD_NAME, null, HTTP_UNAUTHORIZED); - Step scriptCrdStep = CrdHelper.createDomainCrdStep(KUBERNETES_VERSION_16,PRODUCT_VERSION); + MainDelegateStub delegate = createStrictStub(MainDelegateStub.class, KUBERNETES_VERSION_16, PRODUCT_VERSION); + Step scriptCrdStep = CrdHelper.createDomainCrdStep(delegate); testSupport.runSteps(scriptCrdStep); assertThat(logRecords, containsInfo(CREATE_CRD_FAILED)); assertThat(retryStrategy.getConflictStep(), sameInstance(scriptCrdStep)); @@ -165,7 +169,8 @@ void whenNoCrd_proceedToNextStep() { testSupport.addRetryStrategy(retryStrategy); testSupport.failOnCreate(CUSTOM_RESOURCE_DEFINITION, KubernetesConstants.CRD_NAME, null, HTTP_UNAUTHORIZED); - Step scriptCrdStep = CrdHelper.createDomainCrdStep(KUBERNETES_VERSION_16,PRODUCT_VERSION); + MainDelegateStub delegate = createStrictStub(MainDelegateStub.class, KUBERNETES_VERSION_16, PRODUCT_VERSION); + Step scriptCrdStep = CrdHelper.createDomainCrdStep(delegate); testSupport.runSteps(Step.chain(scriptCrdStep, terminalStep)); assertThat(terminalStep.wasRun(), is(true)); @@ -176,7 +181,8 @@ void whenNoCrd_proceedToNextStep() { void whenExistingCrdHasCurrentApiVersionButOldProductVersion_replaceIt() { testSupport.defineResources(defineCrd(PRODUCT_VERSION)); - testSupport.runSteps(CrdHelper.createDomainCrdStep(KUBERNETES_VERSION_16, PRODUCT_VERSION_FUTURE)); + MainDelegateStub delegate = createStrictStub(MainDelegateStub.class, KUBERNETES_VERSION_16, PRODUCT_VERSION_FUTURE); + testSupport.runSteps(CrdHelper.createDomainCrdStep(delegate)); assertThat(logRecords, containsInfo(CREATING_CRD)); List crds = testSupport.getResources(CUSTOM_RESOURCE_DEFINITION); @@ -204,7 +210,8 @@ void whenExistingCrdHasFutureVersion_dontReplaceIt() { .name(KubernetesConstants.DOMAIN_VERSION)); testSupport.defineResources(existing); - testSupport.runSteps(CrdHelper.createDomainCrdStep(KUBERNETES_VERSION_16, PRODUCT_VERSION)); + MainDelegateStub delegate = createStrictStub(MainDelegateStub.class, KUBERNETES_VERSION_16, PRODUCT_VERSION); + testSupport.runSteps(CrdHelper.createDomainCrdStep(delegate)); } @Test @@ -219,7 +226,8 @@ void whenExistingCrdHasFutureVersionButNotCurrentStorage_updateIt() { .served(true) .name(KubernetesConstants.DOMAIN_VERSION)); - testSupport.runSteps(CrdHelper.createDomainCrdStep(KUBERNETES_VERSION_16, PRODUCT_VERSION)); + MainDelegateStub delegate = createStrictStub(MainDelegateStub.class, KUBERNETES_VERSION_16, PRODUCT_VERSION); + testSupport.runSteps(CrdHelper.createDomainCrdStep(delegate)); assertThat(logRecords, containsInfo(CREATING_CRD)); } @@ -230,7 +238,8 @@ void whenReplaceFails_scheduleRetryAndLogFailedMessageInOnFailureNoRetry() { testSupport.defineResources(defineCrd(PRODUCT_VERSION_OLD)); testSupport.failOnReplace(CUSTOM_RESOURCE_DEFINITION, KubernetesConstants.CRD_NAME, null, HTTP_UNAUTHORIZED); - Step scriptCrdStep = CrdHelper.createDomainCrdStep(KUBERNETES_VERSION_16, PRODUCT_VERSION); + MainDelegateStub delegate = createStrictStub(MainDelegateStub.class, KUBERNETES_VERSION_16, PRODUCT_VERSION); + Step scriptCrdStep = CrdHelper.createDomainCrdStep(delegate); testSupport.runSteps(scriptCrdStep); assertThat(logRecords, containsInfo(REPLACE_CRD_FAILED)); @@ -243,7 +252,8 @@ void whenReplaceFailsThrowsStreamException_scheduleRetryAndLogFailedMessageInOnF testSupport.defineResources(defineCrd(PRODUCT_VERSION_OLD)); testSupport.failOnReplaceWithStreamResetException(CUSTOM_RESOURCE_DEFINITION, KubernetesConstants.CRD_NAME, null); - Step scriptCrdStep = CrdHelper.createDomainCrdStep(KUBERNETES_VERSION_16, PRODUCT_VERSION); + MainDelegateStub delegate = createStrictStub(MainDelegateStub.class, KUBERNETES_VERSION_16, PRODUCT_VERSION); + Step scriptCrdStep = CrdHelper.createDomainCrdStep(delegate); testSupport.runSteps(scriptCrdStep); assertThat(logRecords, containsInfo(REPLACE_CRD_FAILED)); @@ -256,4 +266,31 @@ void whenCrdWritten_containsPreserveFieldsAnnotation() throws URISyntaxException assertThat(fileSystem.getContents("/crd.yaml"), containsString("x-kubernetes-preserve-unknown-fields")); } + + abstract static class MainDelegateStub implements MainDelegate { + private final KubernetesVersion kubernetesVersion; + private final SemanticVersion productVersion; + private final AtomicReference crdReference = new AtomicReference<>(); + + MainDelegateStub(KubernetesVersion kubernetesVersion, SemanticVersion productVersion) { + this.kubernetesVersion = kubernetesVersion; + this.productVersion = productVersion; + } + + @Override + public KubernetesVersion getKubernetesVersion() { + return kubernetesVersion; + } + + @Override + public SemanticVersion getProductVersion() { + return productVersion; + } + + @Override + public AtomicReference getCrdReference() { + return crdReference; + } + } + }