diff --git a/go.mod b/go.mod index eede7f628..1bd0d144d 100644 --- a/go.mod +++ b/go.mod @@ -119,4 +119,4 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) -replace k8s.io/apiserver => github.com/openshift/kubernetes-apiserver v0.0.0-20240410114447-9e7c11c45dab // points to openshift-apiserver-4.16-kubernetes-1.29.2 +replace k8s.io/apiserver => github.com/openshift/kubernetes-apiserver v0.0.0-20250915121356-f80f5359033a // points to openshift-apiserver-4.17-kubernetes-1.29.2 diff --git a/go.sum b/go.sum index ec80347e6..25f7e361a 100644 --- a/go.sum +++ b/go.sum @@ -157,8 +157,8 @@ github.com/openshift/build-machinery-go v0.0.0-20250602125535-1b6d00b8c37c h1:gJ github.com/openshift/build-machinery-go v0.0.0-20250602125535-1b6d00b8c37c/go.mod h1:8jcm8UPtg2mCAsxfqKil1xrmRMI3a+XU2TZ9fF8A7TE= github.com/openshift/client-go v0.0.0-20240408153607-64bd6feb83ae h1:WzEMf853apLG0ZgkfmKvYXBKBqhzx7nalP306yQP1ck= github.com/openshift/client-go v0.0.0-20240408153607-64bd6feb83ae/go.mod h1:YOfx7b9ieudQJImuo95BcVzifosCrCGBErbO/w/njBU= -github.com/openshift/kubernetes-apiserver v0.0.0-20240410114447-9e7c11c45dab h1:hVEUWx+0XiMkstOLlQ5BiBZTnA7WWJJf80Dc2L2FLyM= -github.com/openshift/kubernetes-apiserver v0.0.0-20240410114447-9e7c11c45dab/go.mod h1:B0LieKVoyU7ykQvPFm7XSdIHaCHSzCzQWPFa5bqbeMQ= +github.com/openshift/kubernetes-apiserver v0.0.0-20250915121356-f80f5359033a h1:NBEp4AKf+RXdvcHAPJUFnhOg7dwnNDkLY2PuJceC3B4= +github.com/openshift/kubernetes-apiserver v0.0.0-20250915121356-f80f5359033a/go.mod h1:B0LieKVoyU7ykQvPFm7XSdIHaCHSzCzQWPFa5bqbeMQ= github.com/openshift/library-go v0.0.0-20240408090355-0b298992f8cc h1:wYPToL2tSf8rey0YhdhqgVR1r8ES83o5YuUcJU4znMM= github.com/openshift/library-go v0.0.0-20240408090355-0b298992f8cc/go.mod h1:m/HsttSi90vSixwoy5mPUBHcZid2YRw/QbsLErLxF9s= github.com/openshift/osin v1.0.2-0.20220317075346-0f4d38c6e53f h1:4da9vH8eDlJo58703cADj3FlsdnFRgsnfuwj/4lYXfY= diff --git a/vendor/github.com/openshift/build-machinery-go/.gitignore b/vendor/github.com/openshift/build-machinery-go/.gitignore new file mode 100644 index 000000000..19607d9e6 --- /dev/null +++ b/vendor/github.com/openshift/build-machinery-go/.gitignore @@ -0,0 +1 @@ +*.log.raw diff --git a/vendor/github.com/openshift/build-machinery-go/make/targets/openshift/operator/mom.mk b/vendor/github.com/openshift/build-machinery-go/make/targets/openshift/operator/mom.mk new file mode 100644 index 000000000..21c81afe0 --- /dev/null +++ b/vendor/github.com/openshift/build-machinery-go/make/targets/openshift/operator/mom.mk @@ -0,0 +1,10 @@ +scripts_dir :=$(shell realpath $(dir $(lastword $(MAKEFILE_LIST)))../../../../scripts) + +test-operator-integration: build + bash $(scripts_dir)/test-operator-integration.sh +.PHONY: test-operator-integration + +update-test-operator-integration: build + REPLACE_TEST_OUTPUT=true bash $(scripts_dir)/test-operator-integration.sh + +.PHONY: update-test-operator-integration diff --git a/vendor/github.com/openshift/build-machinery-go/scripts/test-operator-integration.sh b/vendor/github.com/openshift/build-machinery-go/scripts/test-operator-integration.sh new file mode 100644 index 000000000..262bf6010 --- /dev/null +++ b/vendor/github.com/openshift/build-machinery-go/scripts/test-operator-integration.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail +set -x + +# Install multi-operator-manager. This will make sure the latest binary is installed +# If the installation failed, keep going, maybe the binary is available in the system +echo "Installing latest version of multi-operator-manager..." +if ! go install -mod=readonly github.com/openshift/multi-operator-manager/cmd/multi-operator-manager@latest; then + echo "Error: Failed to install multi-operator-manager." +fi + +# Check if the multi-operator-manager is installed; if not, fail +if ! command -v multi-operator-manager &> /dev/null; then + echo "Error: multi-operator-manager binary not available." + exit 1 +fi + +REPLACE_TEST_OUTPUT="${REPLACE_TEST_OUTPUT:-false}" + +# Define the path to the operator binary +MOM_CMD="${MOM_CMD:-multi-operator-manager}" + +# Define input and output directories (can be overridden if necessary) +APPLY_CONFIG_INPUT_DIR="${APPLY_CONFIG_INPUT_DIR:-./test-data/apply-configuration}" +APPLY_CONFIG_OUTPUT_DIR="${ARTIFACT_DIR:-./test-output}" + +# Make sure the output-dir is clean +if [ -d "${APPLY_CONFIG_OUTPUT_DIR}" ]; then + echo "Cleaning up existing ${APPLY_CONFIG_OUTPUT_DIR}" + rm -rf "${APPLY_CONFIG_OUTPUT_DIR}" +fi + +# Assemble the args +APPLY_CONFIG_ARGS=( + test + apply-configuration + --test-dir="$APPLY_CONFIG_INPUT_DIR" + --output-dir="$APPLY_CONFIG_OUTPUT_DIR" +) + +if [ "$REPLACE_TEST_OUTPUT" == "true" ] +then + APPLY_CONFIG_ARGS=("${APPLY_CONFIG_ARGS[@]}" "--replace-expected-output=true") +else + APPLY_CONFIG_ARGS=("${APPLY_CONFIG_ARGS[@]}" "--preserve-policy=KeepAlways") +fi + +# Run the apply-configuration command from the operator +"${MOM_CMD}" "${APPLY_CONFIG_ARGS[@]}" diff --git a/vendor/k8s.io/apiserver/pkg/storage/etcd3/etcd3retry/retry_etcdclient.go b/vendor/k8s.io/apiserver/pkg/storage/etcd3/etcd3retry/retry_etcdclient.go index 12bd733f3..b56fd2084 100644 --- a/vendor/k8s.io/apiserver/pkg/storage/etcd3/etcd3retry/retry_etcdclient.go +++ b/vendor/k8s.io/apiserver/pkg/storage/etcd3/etcd3retry/retry_etcdclient.go @@ -2,6 +2,9 @@ package etcd3retry import ( "context" + "fmt" + "regexp" + "strings" "time" etcdrpc "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" @@ -15,7 +18,7 @@ import ( "k8s.io/klog/v2" ) -var defaultRetry = wait.Backoff{ +var DefaultRetry = wait.Backoff{ Duration: 300 * time.Millisecond, Factor: 2, // double the timeout for every failure Jitter: 0.1, @@ -36,7 +39,7 @@ func NewRetryingEtcdStorage(delegate storage.Interface) storage.Interface { // in seconds (0 means forever). If no error is returned and out is not nil, out will be // set to the read value from database. func (c *retryClient) Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64) error { - return onError(ctx, defaultRetry, isRetriableEtcdError, func() error { + return OnError(ctx, DefaultRetry, IsRetriableErrorOnWrite, func() error { return c.Interface.Create(ctx, key, obj, out, ttl) }) } @@ -44,7 +47,7 @@ func (c *retryClient) Create(ctx context.Context, key string, obj, out runtime.O // Delete removes the specified key and returns the value that existed at that spot. // If key didn't exist, it will return NotFound storage error. func (c *retryClient) Delete(ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions, validateDeletion storage.ValidateObjectFunc, cachedExistingObject runtime.Object) error { - return onError(ctx, defaultRetry, isRetriableEtcdError, func() error { + return OnError(ctx, DefaultRetry, IsRetriableErrorOnWrite, func() error { return c.Interface.Delete(ctx, key, out, preconditions, validateDeletion, cachedExistingObject) }) } @@ -58,7 +61,7 @@ func (c *retryClient) Delete(ctx context.Context, key string, out runtime.Object // and send it in an "ADDED" event, before watch starts. func (c *retryClient) Watch(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) { var ret watch.Interface - err := onError(ctx, defaultRetry, isRetriableEtcdError, func() error { + err := OnError(ctx, DefaultRetry, IsRetriableErrorOnRead, func() error { var innerErr error ret, innerErr = c.Interface.Watch(ctx, key, opts) return innerErr @@ -72,7 +75,7 @@ func (c *retryClient) Watch(ctx context.Context, key string, opts storage.ListOp // The returned contents may be delayed, but it is guaranteed that they will // match 'opts.ResourceVersion' according 'opts.ResourceVersionMatch'. func (c *retryClient) Get(ctx context.Context, key string, opts storage.GetOptions, objPtr runtime.Object) error { - return onError(ctx, defaultRetry, isRetriableEtcdError, func() error { + return OnError(ctx, DefaultRetry, IsRetriableErrorOnRead, func() error { return c.Interface.Get(ctx, key, opts, objPtr) }) } @@ -84,7 +87,7 @@ func (c *retryClient) Get(ctx context.Context, key string, opts storage.GetOptio // The returned contents may be delayed, but it is guaranteed that they will // match 'opts.ResourceVersion' according 'opts.ResourceVersionMatch'. func (c *retryClient) GetList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error { - return onError(ctx, defaultRetry, isRetriableEtcdError, func() error { + return OnError(ctx, DefaultRetry, IsRetriableErrorOnRead, func() error { return c.Interface.GetList(ctx, key, opts, listObj) }) } @@ -125,29 +128,71 @@ func (c *retryClient) GetList(ctx context.Context, key string, opts storage.List // ) func (c *retryClient) GuaranteedUpdate(ctx context.Context, key string, destination runtime.Object, ignoreNotFound bool, preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, cachedExistingObject runtime.Object) error { - return onError(ctx, defaultRetry, isRetriableEtcdError, func() error { + return OnError(ctx, DefaultRetry, IsRetriableErrorOnWrite, func() error { return c.Interface.GuaranteedUpdate(ctx, key, destination, ignoreNotFound, preconditions, tryUpdate, cachedExistingObject) }) } -// isRetriableEtcdError returns true if a retry should be attempted, otherwise false. -// errorLabel is set to a non-empty value that reflects the type of error encountered. -func isRetriableEtcdError(err error) (errorLabel string, retry bool) { - if err != nil { - if etcdError, ok := etcdrpc.Error(err).(etcdrpc.EtcdError); ok { - if etcdError.Code() == codes.Unavailable { - errorLabel = "Unavailable" - retry = true - } - } +// These errors are coming back from the k8s.io/apiserver storage.Interface, not directly from an +// etcd client. Classifying them can be fragile since the storage methods may not return etcd client +// errors directly. +var errorLabelsBySuffix = map[string]string{ + "etcdserver: leader changed": "LeaderChanged", + "etcdserver: no leader": "NoLeader", + "raft proposal dropped": "ProposalDropped", + + "etcdserver: request timed out": "Timeout", + "etcdserver: request timed out, possibly due to previous leader failure": "Timeout", + "etcdserver: request timed out, possible due to connection lost": "Timeout", + "etcdserver: request timed out, waiting for the applied index took too long": "Timeout", + "etcdserver: server stopped": "Stopped", +} + +var retriableWriteErrorSuffixes = func() *regexp.Regexp { + // This list should include only errors the caller is certain have no side effects. + suffixes := []string{ + "etcdserver: leader changed", + "etcdserver: no leader", + "raft proposal dropped", + } + return regexp.MustCompile(fmt.Sprintf(`(%s)$`, strings.Join(suffixes, `|`))) +}() + +// IsRetriableErrorOnWrite returns true if and only if a retry should be attempted when the provided +// error is returned from a write attempt. If the error is retriable, a non-empty string classifying +// the error is also returned. +func IsRetriableErrorOnWrite(err error) (string, bool) { + if suffix := retriableWriteErrorSuffixes.FindString(err.Error()); suffix != "" { + return errorLabelsBySuffix[suffix], true + } + return "", false +} + +var retriableReadErrorSuffixes = func() *regexp.Regexp { + var suffixes []string + for suffix := range errorLabelsBySuffix { + suffixes = append(suffixes, suffix) + } + return regexp.MustCompile(fmt.Sprintf(`(%s)$`, strings.Join(suffixes, `|`))) +}() + +// IsRetriableErrorOnRead returns true if and only if a retry should be attempted when the provided +// error is returned from a read attempt. If the error is retriable, a non-empty string classifying +// the error is also returned. +func IsRetriableErrorOnRead(err error) (string, bool) { + if suffix := retriableReadErrorSuffixes.FindString(err.Error()); suffix != "" { + return errorLabelsBySuffix[suffix], true } - return + if etcdError, ok := etcdrpc.Error(err).(etcdrpc.EtcdError); ok && etcdError.Code() == codes.Unavailable { + return "Unavailable", true + } + return "", false } -// onError allows the caller to retry fn in case the error returned by fn is retriable +// OnError allows the caller to retry fn in case the error returned by fn is retriable // according to the provided function. backoff defines the maximum retries and the wait // interval between two retries. -func onError(ctx context.Context, backoff wait.Backoff, retriable func(error) (string, bool), fn func() error) error { +func OnError(ctx context.Context, backoff wait.Backoff, retriable func(error) (string, bool), fn func() error) error { var lastErr error var lastErrLabel string var retry bool @@ -163,6 +208,9 @@ func onError(ctx context.Context, backoff wait.Backoff, retriable func(error) (s } lastErrLabel, retry = retriable(err) + if klog.V(6).Enabled() { + klog.V(6).InfoS("observed storage error", "err", err, "retriable", retry) + } if retry { lastErr = err retryCounter++ diff --git a/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go b/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go index fbb97147c..9742a66f3 100644 --- a/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go +++ b/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go @@ -155,13 +155,13 @@ func newETCD3Check(c storagebackend.Config, timeout time.Duration, stopCh <-chan // retry in a loop in the background until we successfully create the client, storing the client or error encountered lock := sync.RWMutex{} - var prober *etcd3ProberMonitor + var prober *etcd3RetryingProberMonitor clientErr := fmt.Errorf("etcd client connection not yet established") go wait.PollImmediateUntil(time.Second, func() (bool, error) { lock.Lock() defer lock.Unlock() - newProber, err := newETCD3ProberMonitor(c) + newProber, err := newRetryingETCD3ProberMonitor(c) // Ensure that server is already not shutting down. select { case <-stopCh: diff --git a/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go b/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go index 2bf3727e8..0967a84cb 100644 --- a/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go +++ b/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go @@ -69,7 +69,7 @@ func CreateProber(c storagebackend.Config) (Prober, error) { case storagebackend.StorageTypeETCD2: return nil, fmt.Errorf("%s is no longer a supported storage backend", c.Type) case storagebackend.StorageTypeUnset, storagebackend.StorageTypeETCD3: - return newETCD3ProberMonitor(c) + return newRetryingETCD3ProberMonitor(c) default: return nil, fmt.Errorf("unknown storage type: %s", c.Type) } @@ -80,7 +80,7 @@ func CreateMonitor(c storagebackend.Config) (metrics.Monitor, error) { case storagebackend.StorageTypeETCD2: return nil, fmt.Errorf("%s is no longer a supported storage backend", c.Type) case storagebackend.StorageTypeUnset, storagebackend.StorageTypeETCD3: - return newETCD3ProberMonitor(c) + return newRetryingETCD3ProberMonitor(c) default: return nil, fmt.Errorf("unknown storage type: %s", c.Type) } diff --git a/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/retry_etcdprobemonitor.go b/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/retry_etcdprobemonitor.go new file mode 100644 index 000000000..0e6c19b45 --- /dev/null +++ b/vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/retry_etcdprobemonitor.go @@ -0,0 +1,46 @@ +package factory + +import ( + "context" + + "k8s.io/apiserver/pkg/storage/etcd3/etcd3retry" + "k8s.io/apiserver/pkg/storage/etcd3/metrics" + "k8s.io/apiserver/pkg/storage/storagebackend" +) + +type proberMonitor interface { + Prober + metrics.Monitor +} + +type etcd3RetryingProberMonitor struct { + delegate proberMonitor +} + +func newRetryingETCD3ProberMonitor(c storagebackend.Config) (*etcd3RetryingProberMonitor, error) { + delegate, err := newETCD3ProberMonitor(c) + if err != nil { + return nil, err + } + return &etcd3RetryingProberMonitor{delegate: delegate}, nil +} + +func (t *etcd3RetryingProberMonitor) Probe(ctx context.Context) error { + return etcd3retry.OnError(ctx, etcd3retry.DefaultRetry, etcd3retry.IsRetriableErrorOnRead, func() error { + return t.delegate.Probe(ctx) + }) +} + +func (t *etcd3RetryingProberMonitor) Monitor(ctx context.Context) (metrics.StorageMetrics, error) { + var ret metrics.StorageMetrics + err := etcd3retry.OnError(ctx, etcd3retry.DefaultRetry, etcd3retry.IsRetriableErrorOnRead, func() error { + var innerErr error + ret, innerErr = t.delegate.Monitor(ctx) + return innerErr + }) + return ret, err +} + +func (t *etcd3RetryingProberMonitor) Close() error { + return t.delegate.Close() +} diff --git a/vendor/modules.txt b/vendor/modules.txt index ec8386f1f..1d4e37332 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -709,7 +709,7 @@ k8s.io/apimachinery/pkg/version k8s.io/apimachinery/pkg/watch k8s.io/apimachinery/third_party/forked/golang/json k8s.io/apimachinery/third_party/forked/golang/reflect -# k8s.io/apiserver v0.29.2 => github.com/openshift/kubernetes-apiserver v0.0.0-20240410114447-9e7c11c45dab +# k8s.io/apiserver v0.29.2 => github.com/openshift/kubernetes-apiserver v0.0.0-20250915121356-f80f5359033a ## explicit; go 1.21 k8s.io/apiserver/pkg/admission k8s.io/apiserver/pkg/admission/cel @@ -1260,4 +1260,4 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# k8s.io/apiserver => github.com/openshift/kubernetes-apiserver v0.0.0-20240410114447-9e7c11c45dab +# k8s.io/apiserver => github.com/openshift/kubernetes-apiserver v0.0.0-20250915121356-f80f5359033a