diff --git a/pkg/receive/multitsdb.go b/pkg/receive/multitsdb.go index 2ac9d10ee67..b53f05d2213 100644 --- a/pkg/receive/multitsdb.go +++ b/pkg/receive/multitsdb.go @@ -141,7 +141,7 @@ func (l *localClient) String() string { mint, maxt := l.store.TimeRange() return fmt.Sprintf( "LabelSets: %v MinTime: %d MaxTime: %d", - labelpb.PromLabelSetsToStringN(l.LabelSets(), 10), mint, maxt, + labelpb.PromLabelSetsToStringN(l.LabelSets(), 1000), mint, maxt, ) } diff --git a/pkg/store/labelpb/label.go b/pkg/store/labelpb/label.go index 64312b4d061..909c4c41db6 100644 --- a/pkg/store/labelpb/label.go +++ b/pkg/store/labelpb/label.go @@ -303,17 +303,17 @@ func ExtendSortedLabels(lset, extend labels.Labels) labels.Labels { } func PromLabelSetsToString(lsets []labels.Labels) string { - return PromLabelSetsToStringN(lsets, 1000000) + return PromLabelSetsToStringN(lsets, 1000) } -func PromLabelSetsToStringN(lsets []labels.Labels, maxNumLabels int) string { +func PromLabelSetsToStringN(lsets []labels.Labels, maxLength int) string { + if len(lsets) == 0 { + return "" + } s := []string{} + maxLabelLength := (maxLength + min(10, len(lsets)) - 1) / min(10, len(lsets)) for _, ls := range lsets { - s = append(s, ls.String()) - maxNumLabels-- - if maxNumLabels <= 0 { - break - } + s = append(s, ls.String()[:min(maxLabelLength, len(ls.String()))]) } sort.Strings(s) return strings.Join(s, ",") diff --git a/pkg/store/proxy.go b/pkg/store/proxy.go index 99365c0963d..f929961b442 100644 --- a/pkg/store/proxy.go +++ b/pkg/store/proxy.go @@ -464,14 +464,16 @@ func (s *ProxyStore) Series(originalRequest *storepb.SeriesRequest, srv storepb. if resp.GetWarning() != "" { totalFailedStores++ - level.Error(s.logger).Log("msg", "Series: warning from store", "warning", resp.GetWarning()) + maxWarningBytes := 2000 + warning := resp.GetWarning()[:min(maxWarningBytes, len(resp.GetWarning()))] + level.Error(s.logger).Log("msg", "Series: warning from store", "warning", warning) if r.PartialResponseStrategy == storepb.PartialResponseStrategy_GROUP_REPLICA { // TODO: attribute the warning to the store(group key and replica key) that produced it. // Each client streams a sequence of time series, so it's not trivial to attribute the warning to a specific client. if totalFailedStores > 1 { level.Error(reqLogger).Log("msg", "more than one stores have failed") // If we don't know which store has failed, we can tolerate at most one failed store. - return status.Error(codes.Aborted, resp.GetWarning()) + return status.Error(codes.Aborted, warning) } } else if r.PartialResponseDisabled || r.PartialResponseStrategy == storepb.PartialResponseStrategy_ABORT { return status.Error(codes.Aborted, resp.GetWarning())