From b206dc20bdb6f2b23904c2202df29199968aba8b Mon Sep 17 00:00:00 2001 From: gotjosh Date: Tue, 8 Oct 2024 16:50:36 +0100 Subject: [PATCH 1/4] When OOO native histograms are disabled it should be a client error When ingesters replay data from Kafka, if they receive a non-client error they will continue to attempt to push the batch to the TSDB. In this PR, I'm translating Native Histogram OOO errors into client errors to ensure that we can continue with the next set of write requests when replaying data. Signed-off-by: gotjosh --- pkg/ingester/ingester.go | 12 +++++ pkg/ingester/ingester_test.go | 93 ++++++++++++++++++++++++++++++++--- pkg/util/globalerror/user.go | 2 + 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 4adc37929ec..1cf6fdcc2c7 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1344,6 +1344,18 @@ func (i *Ingester) pushSamplesToAppender(userID string, timeseries []mimirpb.Pre return true // Map TSDB native histogram validation errors to soft errors. + case errors.Is(err, storage.ErrOOONativeHistogramsDisabled): + stats.invalidNativeHistogramCount++ + updateFirstPartial(i.errorSamplers.nativeHistogramValidationError, func() softError { + return newNativeHistogramValidationError(globalerror.NativeHistogramOOODisabled, err, model.Time(timestamp), labels) + }) + return true + case errors.Is(err, storage.ErrNativeHistogramsDisabled): + stats.invalidNativeHistogramCount++ + updateFirstPartial(i.errorSamplers.nativeHistogramValidationError, func() softError { + return newNativeHistogramValidationError(globalerror.NativeHistogramDisabled, err, model.Time(timestamp), labels) + }) + return true case errors.Is(err, histogram.ErrHistogramCountMismatch): stats.invalidNativeHistogramCount++ updateFirstPartial(i.errorSamplers.nativeHistogramValidationError, func() softError { diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index b4f74ba8440..117a823d9dc 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -386,6 +386,7 @@ func TestIngester_Push(t *testing.T) { maxMetadataPerUser int maxMetadataPerMetric int nativeHistograms bool + oooNativeHistograms bool ignoreOOOExemplars bool }{ "should succeed on valid series and metadata": { @@ -1915,6 +1916,77 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_tsdb_head_max_timestamp_seconds 0.01 `, }, + "should not fail if native histograms are disabled": { + nativeHistograms: false, + reqs: []*mimirpb.WriteRequest{ + mimirpb.NewWriteRequest(nil, mimirpb.API).AddHistogramSeries( + [][]mimirpb.LabelAdapter{metricLabelAdapters}, + []mimirpb.Histogram{mimirpb.FromHistogramToHistogramProto(10, util_test.GenerateTestHistogram(1))}, + nil, + ), + }, + expectedErr: nil, + }, + "should soft fail if OOO native histograms are disabled": { + nativeHistograms: true, + oooNativeHistograms: false, + reqs: []*mimirpb.WriteRequest{ + mimirpb.NewWriteRequest(nil, mimirpb.API).AddHistogramSeries( + [][]mimirpb.LabelAdapter{metricLabelAdapters}, + []mimirpb.Histogram{mimirpb.FromHistogramToHistogramProto(10, util_test.GenerateTestHistogram(1))}, + nil, + ), + mimirpb.NewWriteRequest(nil, mimirpb.API).AddHistogramSeries( + [][]mimirpb.LabelAdapter{metricLabelAdapters}, + []mimirpb.Histogram{mimirpb.FromHistogramToHistogramProto(-10, util_test.GenerateTestHistogram(1))}, + nil, + ), + }, + expectedErr: newErrorWithStatus(wrapOrAnnotateWithUser(newNativeHistogramValidationError(globalerror.NativeHistogramOOODisabled, fmt.Errorf("out-of-order native histogram ingestion is disabled"), model.Time(-10), []mimirpb.LabelAdapter{metricLabelAdapters[0]}), userID), codes.InvalidArgument), + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested per user. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total{user="test"} 1 + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + cortex_discarded_samples_total{group="",reason="invalid-native-histogram",user="test"} 1 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion per user. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total{user="test"} 1 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_ingester_tsdb_head_min_timestamp_seconds Minimum timestamp of the head block across all tenants. + # TYPE cortex_ingester_tsdb_head_min_timestamp_seconds gauge + cortex_ingester_tsdb_head_min_timestamp_seconds 0.01 + # HELP cortex_ingester_tsdb_head_max_timestamp_seconds Maximum timestamp of the head block across all tenants. + # TYPE cortex_ingester_tsdb_head_max_timestamp_seconds gauge + cortex_ingester_tsdb_head_max_timestamp_seconds 0.01 + # HELP cortex_ingester_active_native_histogram_buckets Number of currently active native histogram buckets per user. + # TYPE cortex_ingester_active_native_histogram_buckets gauge + cortex_ingester_active_native_histogram_buckets{user="test"} 8 + # HELP cortex_ingester_active_native_histogram_series Number of currently active native histogram series per user. + # TYPE cortex_ingester_active_native_histogram_series gauge + cortex_ingester_active_native_histogram_series{user="test"} 1 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + `, + expectedIngested: model.Matrix{ + &model.SampleStream{Metric: metricLabelSet, Histograms: []model.SampleHistogramPair{ + {Histogram: mimirpb.FromHistogramToPromHistogram(util_test.GenerateTestHistogram(1)), Timestamp: 10}}, + }, + }, + }, "should soft fail if histogram has a negative bucket count": { nativeHistograms: true, reqs: []*mimirpb.WriteRequest{ @@ -3194,7 +3266,13 @@ func TestIngester_Push(t *testing.T) { limits.MaxGlobalMetricsWithMetadataPerUser = testData.maxMetadataPerUser limits.MaxGlobalMetadataPerMetric = testData.maxMetadataPerMetric limits.NativeHistogramsIngestionEnabled = testData.nativeHistograms + limits.OOONativeHistogramsIngestionEnabled = testData.oooNativeHistograms limits.IgnoreOOOExemplars = testData.ignoreOOOExemplars + var oooTimeWindow int64 + if testData.nativeHistograms && !testData.oooNativeHistograms { + oooTimeWindow = int64(1 * time.Hour.Seconds()) + limits.OutOfOrderTimeWindow = model.Duration(1 * time.Hour) + } i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", registry) require.NoError(t, err) @@ -3222,10 +3300,11 @@ func TestIngester_Push(t *testing.T) { if testData.expectedErr == nil { assert.NoError(t, err) } else { + require.Error(t, err) handledErr := mapPushErrorToErrorWithStatus(err) errWithStatus, ok := handledErr.(globalerror.ErrorWithStatus) - assert.True(t, ok) - assert.True(t, errWithStatus.Equals(testData.expectedErr)) + require.True(t, ok) + require.Truef(t, errWithStatus.Equals(testData.expectedErr), "errors don't match \nactual: '%v'\nexpected: '%v'", errWithStatus, testData.expectedErr) } } } @@ -3244,7 +3323,7 @@ func TestIngester_Push(t *testing.T) { if len(res) == 0 { res = nil } - assert.Equal(t, testData.expectedIngested, res) + require.Equal(t, testData.expectedIngested, res) // Read back samples to see what has been really ingested exemplarRes, err := i.QueryExemplars(ctx, &client.ExemplarQueryRequest{ @@ -3307,9 +3386,11 @@ func TestIngester_Push(t *testing.T) { assert.Equal(t, int64(expectedTenantsCount), usagestats.GetInt(memoryTenantsStatsName).Value()) assert.Equal(t, int64(expectedSamplesCount)+appendedSamplesStatsBefore, usagestats.GetCounter(appendedSamplesStatsName).Total()) assert.Equal(t, int64(expectedExemplarsCount)+appendedExemplarsStatsBefore, usagestats.GetCounter(appendedExemplarsStatsName).Total()) - assert.Equal(t, int64(0), usagestats.GetInt(tenantsWithOutOfOrderEnabledStatName).Value()) - assert.Equal(t, int64(0), usagestats.GetInt(minOutOfOrderTimeWindowSecondsStatName).Value()) - assert.Equal(t, int64(0), usagestats.GetInt(maxOutOfOrderTimeWindowSecondsStatName).Value()) + + assert.Equal(t, testData.oooNativeHistograms, usagestats.GetInt(tenantsWithOutOfOrderEnabledStatName).Value() == int64(0)) + assert.Equal(t, oooTimeWindow, usagestats.GetInt(minOutOfOrderTimeWindowSecondsStatName).Value()) + assert.Equal(t, oooTimeWindow, usagestats.GetInt(maxOutOfOrderTimeWindowSecondsStatName).Value()) + }) } } diff --git a/pkg/util/globalerror/user.go b/pkg/util/globalerror/user.go index ac2ec555420..41f6cd37b4d 100644 --- a/pkg/util/globalerror/user.go +++ b/pkg/util/globalerror/user.go @@ -86,6 +86,8 @@ const ( NativeHistogramNegativeBucketCount ID = "native-histogram-negative-bucket-count" NativeHistogramSpanNegativeOffset ID = "native-histogram-span-negative-offset" NativeHistogramSpansBucketsMismatch ID = "native-histogram-spans-buckets-mismatch" + NativeHistogramDisabled ID = "native-histogram-disabled" + NativeHistogramOOODisabled ID = "native-histogram-ooo-disabled" // Alertmanager errors AlertmanagerMaxGrafanaConfigSize ID = "alertmanager-max-grafana-config-size" From 911760a898e2a029d472577254bd2e764e3f0ca7 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Tue, 8 Oct 2024 18:08:42 +0100 Subject: [PATCH 2/4] Address review feedback Signed-off-by: gotjosh --- .../mimir/manage/mimir-runbooks/_index.md | 23 ++++++-- pkg/ingester/ingester.go | 8 +-- pkg/ingester/ingester_test.go | 54 ++++++++----------- pkg/util/globalerror/user.go | 1 - 4 files changed, 40 insertions(+), 46 deletions(-) diff --git a/docs/sources/mimir/manage/mimir-runbooks/_index.md b/docs/sources/mimir/manage/mimir-runbooks/_index.md index 37fcd2a8ef5..224e9796a40 100644 --- a/docs/sources/mimir/manage/mimir-runbooks/_index.md +++ b/docs/sources/mimir/manage/mimir-runbooks/_index.md @@ -1671,7 +1671,7 @@ The series containing such samples are skipped during ingestion, and valid serie ### err-mimir-native-histogram-count-mismatch -This non-critical error occures when Mimir receives a write request that contains a sample that is a native histogram +This non-critical error occurs when Mimir receives a write request that contains a sample that is a native histogram where the buckets counts don't add up to the overall count recorded in the native histogram, provided that the overall sum is a regular float number. @@ -1685,7 +1685,7 @@ When `-ingester.error-sample-rate` is configured to a value greater than `0`, in ### err-mimir-native-histogram-count-not-big-enough -This non-critical error occures when Mimir receives a write request that contains a sample that is a native histogram +This non-critical error occurs when Mimir receives a write request that contains a sample that is a native histogram where the buckets counts add up to a higher number than the overall count recorded in the native histogram, provided that the overall sum is not a float number (NaN). @@ -1699,7 +1699,7 @@ When `-ingester.error-sample-rate` is configured to a value greater than `0`, in ### err-mimir-native-histogram-negative-bucket-count -This non-critical error occures when Mimir receives a write request that contains a sample that is a native histogram +This non-critical error occurs when Mimir receives a write request that contains a sample that is a native histogram where some bucket count is negative. {{< admonition type="note" >}} @@ -1712,7 +1712,7 @@ When `-ingester.error-sample-rate` is configured to a value greater than `0`, in ### err-mimir-native-histogram-span-negative-offset -This non-critical error occures when Mimir receives a write request that contains a sample that is a native histogram +This non-critical error occurs when Mimir receives a write request that contains a sample that is a native histogram where a bucket span has a negative offset. {{< admonition type="note" >}} @@ -1725,7 +1725,7 @@ When `-ingester.error-sample-rate` is configured to a value greater than `0`, in ### err-mimir-native-histogram-spans-buckets-mismatch -This non-critical error occures when Mimir receives a write request that contains a sample that is a native histogram +This non-critical error occurs when Mimir receives a write request that contains a sample that is a native histogram where the number of bucket counts does not agree with the number of buckets encoded in the bucket spans. {{< admonition type="note" >}} @@ -1736,6 +1736,19 @@ The series containing such samples are skipped during ingestion, and valid serie When `-ingester.error-sample-rate` is configured to a value greater than `0`, invalid native histogram errors are logged only once every `-ingester.error-sample-rate` times. {{< /admonition >}} +### err-mimir-native-histogram-ooo-disabled + +This non-critical error occurs when Mimir receives a write request that contains a sample that is a native histogram +where another sample with a more recent timestamp has already been ingested. + +{{< admonition type="note" >}} +The series containing such samples are skipped during ingestion, and valid series within the same request are ingested. +{{< /admonition >}} + +{{< admonition type="note" >}} +When `-ingester.error-sample-rate` is configured to a value greater than `0`, invalid native histogram errors are logged only once every `-ingester.error-sample-rate` times. +{{< /admonition >}} + ### err-mimir-label-invalid This non-critical error occurs when Mimir receives a write request that contains a series with an invalid label name. diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 1cf6fdcc2c7..a8a27cfbaf0 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1345,17 +1345,11 @@ func (i *Ingester) pushSamplesToAppender(userID string, timeseries []mimirpb.Pre // Map TSDB native histogram validation errors to soft errors. case errors.Is(err, storage.ErrOOONativeHistogramsDisabled): - stats.invalidNativeHistogramCount++ + stats.sampleOutOfOrderCount++ updateFirstPartial(i.errorSamplers.nativeHistogramValidationError, func() softError { return newNativeHistogramValidationError(globalerror.NativeHistogramOOODisabled, err, model.Time(timestamp), labels) }) return true - case errors.Is(err, storage.ErrNativeHistogramsDisabled): - stats.invalidNativeHistogramCount++ - updateFirstPartial(i.errorSamplers.nativeHistogramValidationError, func() softError { - return newNativeHistogramValidationError(globalerror.NativeHistogramDisabled, err, model.Time(timestamp), labels) - }) - return true case errors.Is(err, histogram.ErrHistogramCountMismatch): stats.invalidNativeHistogramCount++ updateFirstPartial(i.errorSamplers.nativeHistogramValidationError, func() softError { diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 117a823d9dc..588433fbef0 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -373,21 +373,21 @@ func TestIngester_Push(t *testing.T) { histogramWithSpansBucketsMismatch.PositiveSpans[1].Length++ tests := map[string]struct { - reqs []*mimirpb.WriteRequest - expectedErr error - expectedIngested model.Matrix - expectedMetadataIngested []*mimirpb.MetricMetadata - expectedExemplarsIngested []mimirpb.TimeSeries - expectedExemplarsDropped []mimirpb.TimeSeries - expectedMetrics string - additionalMetrics []string - disableActiveSeries bool - maxExemplars int - maxMetadataPerUser int - maxMetadataPerMetric int - nativeHistograms bool - oooNativeHistograms bool - ignoreOOOExemplars bool + reqs []*mimirpb.WriteRequest + expectedErr error + expectedIngested model.Matrix + expectedMetadataIngested []*mimirpb.MetricMetadata + expectedExemplarsIngested []mimirpb.TimeSeries + expectedExemplarsDropped []mimirpb.TimeSeries + expectedMetrics string + additionalMetrics []string + disableActiveSeries bool + maxExemplars int + maxMetadataPerUser int + maxMetadataPerMetric int + nativeHistograms bool + disableOOONativeHistograms bool + ignoreOOOExemplars bool }{ "should succeed on valid series and metadata": { reqs: []*mimirpb.WriteRequest{ @@ -1916,20 +1916,9 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_tsdb_head_max_timestamp_seconds 0.01 `, }, - "should not fail if native histograms are disabled": { - nativeHistograms: false, - reqs: []*mimirpb.WriteRequest{ - mimirpb.NewWriteRequest(nil, mimirpb.API).AddHistogramSeries( - [][]mimirpb.LabelAdapter{metricLabelAdapters}, - []mimirpb.Histogram{mimirpb.FromHistogramToHistogramProto(10, util_test.GenerateTestHistogram(1))}, - nil, - ), - }, - expectedErr: nil, - }, "should soft fail if OOO native histograms are disabled": { - nativeHistograms: true, - oooNativeHistograms: false, + nativeHistograms: true, + disableOOONativeHistograms: true, reqs: []*mimirpb.WriteRequest{ mimirpb.NewWriteRequest(nil, mimirpb.API).AddHistogramSeries( [][]mimirpb.LabelAdapter{metricLabelAdapters}, @@ -1949,7 +1938,7 @@ func TestIngester_Push(t *testing.T) { cortex_ingester_ingested_samples_total{user="test"} 1 # HELP cortex_discarded_samples_total The total number of samples that were discarded. # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{group="",reason="invalid-native-histogram",user="test"} 1 + cortex_discarded_samples_total{group="",reason="sample-out-of-order",user="test"} 1 # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion per user. # TYPE cortex_ingester_ingested_samples_failures_total counter cortex_ingester_ingested_samples_failures_total{user="test"} 1 @@ -3266,12 +3255,12 @@ func TestIngester_Push(t *testing.T) { limits.MaxGlobalMetricsWithMetadataPerUser = testData.maxMetadataPerUser limits.MaxGlobalMetadataPerMetric = testData.maxMetadataPerMetric limits.NativeHistogramsIngestionEnabled = testData.nativeHistograms - limits.OOONativeHistogramsIngestionEnabled = testData.oooNativeHistograms limits.IgnoreOOOExemplars = testData.ignoreOOOExemplars var oooTimeWindow int64 - if testData.nativeHistograms && !testData.oooNativeHistograms { + if testData.disableOOONativeHistograms { oooTimeWindow = int64(1 * time.Hour.Seconds()) limits.OutOfOrderTimeWindow = model.Duration(1 * time.Hour) + limits.OOONativeHistogramsIngestionEnabled = false } i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", registry) @@ -3387,10 +3376,9 @@ func TestIngester_Push(t *testing.T) { assert.Equal(t, int64(expectedSamplesCount)+appendedSamplesStatsBefore, usagestats.GetCounter(appendedSamplesStatsName).Total()) assert.Equal(t, int64(expectedExemplarsCount)+appendedExemplarsStatsBefore, usagestats.GetCounter(appendedExemplarsStatsName).Total()) - assert.Equal(t, testData.oooNativeHistograms, usagestats.GetInt(tenantsWithOutOfOrderEnabledStatName).Value() == int64(0)) + assert.Equal(t, testData.disableOOONativeHistograms, usagestats.GetInt(tenantsWithOutOfOrderEnabledStatName).Value() == int64(1)) assert.Equal(t, oooTimeWindow, usagestats.GetInt(minOutOfOrderTimeWindowSecondsStatName).Value()) assert.Equal(t, oooTimeWindow, usagestats.GetInt(maxOutOfOrderTimeWindowSecondsStatName).Value()) - }) } } diff --git a/pkg/util/globalerror/user.go b/pkg/util/globalerror/user.go index 41f6cd37b4d..8a378e5b3b3 100644 --- a/pkg/util/globalerror/user.go +++ b/pkg/util/globalerror/user.go @@ -86,7 +86,6 @@ const ( NativeHistogramNegativeBucketCount ID = "native-histogram-negative-bucket-count" NativeHistogramSpanNegativeOffset ID = "native-histogram-span-negative-offset" NativeHistogramSpansBucketsMismatch ID = "native-histogram-spans-buckets-mismatch" - NativeHistogramDisabled ID = "native-histogram-disabled" NativeHistogramOOODisabled ID = "native-histogram-ooo-disabled" // Alertmanager errors From 459ecd6efeac23d94243c665bbd39e2a77b8cde0 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Tue, 8 Oct 2024 18:46:00 +0100 Subject: [PATCH 3/4] Update docs/sources/mimir/manage/mimir-runbooks/_index.md Co-authored-by: Fiona Liao --- docs/sources/mimir/manage/mimir-runbooks/_index.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/sources/mimir/manage/mimir-runbooks/_index.md b/docs/sources/mimir/manage/mimir-runbooks/_index.md index 224e9796a40..f7cc5de5955 100644 --- a/docs/sources/mimir/manage/mimir-runbooks/_index.md +++ b/docs/sources/mimir/manage/mimir-runbooks/_index.md @@ -1739,7 +1739,8 @@ When `-ingester.error-sample-rate` is configured to a value greater than `0`, in ### err-mimir-native-histogram-ooo-disabled This non-critical error occurs when Mimir receives a write request that contains a sample that is a native histogram -where another sample with a more recent timestamp has already been ingested. +where another sample with a more recent timestamp has already been ingested and +`-ingester.ooo-native-histograms-ingestion-enabled` is set to `false`. {{< admonition type="note" >}} The series containing such samples are skipped during ingestion, and valid series within the same request are ingested. From d96562bbd7c78315db4877c3ab3b37d683f2294f Mon Sep 17 00:00:00 2001 From: gotjosh Date: Tue, 8 Oct 2024 19:14:03 +0100 Subject: [PATCH 4/4] lint Signed-off-by: gotjosh --- docs/sources/mimir/manage/mimir-runbooks/_index.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/sources/mimir/manage/mimir-runbooks/_index.md b/docs/sources/mimir/manage/mimir-runbooks/_index.md index f7cc5de5955..ab23693a68c 100644 --- a/docs/sources/mimir/manage/mimir-runbooks/_index.md +++ b/docs/sources/mimir/manage/mimir-runbooks/_index.md @@ -1739,8 +1739,7 @@ When `-ingester.error-sample-rate` is configured to a value greater than `0`, in ### err-mimir-native-histogram-ooo-disabled This non-critical error occurs when Mimir receives a write request that contains a sample that is a native histogram -where another sample with a more recent timestamp has already been ingested and -`-ingester.ooo-native-histograms-ingestion-enabled` is set to `false`. +where another sample with a more recent timestamp has already been ingested and `-ingester.ooo-native-histograms-ingestion-enabled` is set to `false`. {{< admonition type="note" >}} The series containing such samples are skipped during ingestion, and valid series within the same request are ingested.