Skip to content

Commit fdaf626

Browse files
56quarters56quarters
committed
Hash object names used as cache keys for CachingBucket
When caching rule groups, the base64 encoded name of the rule group is used as part of the key for caching its contents. For long rule group names, this can exceed the max key length of Memcached. To solve this we use the sha256 of the object name in the cache key instead of the object name itself. Related #9386
1 parent 545a40b commit fdaf626

File tree

3 files changed

+55
-39
lines changed

3 files changed

+55
-39
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636
* [FEATURE] Alertmanager: limit added for maximum size of the Grafana configuration (`-alertmanager.max-config-size-bytes`). #9402
3737
* [FEATURE] Ingester: Experimental support for ingesting out-of-order native histograms. This is disabled by default and can be enabled by setting `-ingester.ooo-native-histograms-ingestion-enabled` to `true`. #7175
3838
* [FEATURE] Distributor: Added `-api.skip-label-count-validation-header-enabled` option to allow skipping label count validation on the HTTP write path based on `X-Mimir-SkipLabelCountValidation` header being `true` or not. #9576
39-
* [FEATURE] Ruler: Add experimental support for caching the contents of rule groups. This is disabled by default and can be enabled by setting `-ruler-storage.cache.rule-group-enabled`. #9595
39+
* [FEATURE] Ruler: Add experimental support for caching the contents of rule groups. This is disabled by default and can be enabled by setting `-ruler-storage.cache.rule-group-enabled`. #9595 #9968
4040
* [FEATURE] PromQL: Add experimental `info` function. Experimental functions are disabled by default, but can be enabled setting `-querier.promql-experimental-functions-enabled=true` in the query-frontend and querier. #9879
4141
* [ENHANCEMENT] mimirtool: Adds bearer token support for mimirtool's analyze ruler/prometheus commands. #9587
4242
* [ENHANCEMENT] Ruler: Support `exclude_alerts` parameter in `<prometheus-http-prefix>/api/v1/rules` endpoint. #9300

‎pkg/storage/tsdb/bucketcache/caching_bucket.go

Lines changed: 41 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ package bucketcache
88
import (
99
"bytes"
1010
"context"
11+
"crypto"
12+
"encoding/hex"
1113
"encoding/json"
1214
"io"
1315
"strconv"
@@ -165,20 +167,22 @@ func NewCachingBucket(bucketID string, bucketClient objstore.Bucket, cfg *Cachin
165167
}
166168

167169
func (cb *CachingBucket) Upload(ctx context.Context, name string, r io.Reader) error {
168-
cb.invalidation.start(ctx, name)
170+
hashedName := cachingKeyHash(name)
171+
cb.invalidation.start(ctx, name, hashedName)
169172
err := cb.Bucket.Upload(ctx, name, r)
170173
if err == nil {
171-
cb.invalidation.finish(ctx, name)
174+
cb.invalidation.finish(ctx, name, hashedName)
172175
}
173176

174177
return err
175178
}
176179

177180
func (cb *CachingBucket) Delete(ctx context.Context, name string) error {
178-
cb.invalidation.start(ctx, name)
181+
hashedName := cachingKeyHash(name)
182+
cb.invalidation.start(ctx, name, hashedName)
179183
err := cb.Bucket.Delete(ctx, name)
180184
if err == nil {
181-
cb.invalidation.finish(ctx, name)
185+
cb.invalidation.finish(ctx, name, hashedName)
182186
}
183187

184188
return err
@@ -259,8 +263,9 @@ func (cb *CachingBucket) Exists(ctx context.Context, name string) (bool, error)
259263
return cb.Bucket.Exists(ctx, name)
260264
}
261265

262-
key := cachingKeyExists(cb.bucketID, name)
263-
lockKey := cachingKeyExistsLock(cb.bucketID, name)
266+
hashedName := cachingKeyHash(name)
267+
key := cachingKeyExists(cb.bucketID, hashedName)
268+
lockKey := cachingKeyExistsLock(cb.bucketID, hashedName)
264269

265270
// Lookup the cache.
266271
if isCacheLookupEnabled(ctx) {
@@ -308,10 +313,11 @@ func (cb *CachingBucket) Get(ctx context.Context, name string) (io.ReadCloser, e
308313
return cb.Bucket.Get(ctx, name)
309314
}
310315

311-
contentLockKey := cachingKeyContentLock(cb.bucketID, name)
312-
contentKey := cachingKeyContent(cb.bucketID, name)
313-
existsLockKey := cachingKeyExistsLock(cb.bucketID, name)
314-
existsKey := cachingKeyExists(cb.bucketID, name)
316+
hashedName := cachingKeyHash(name)
317+
contentLockKey := cachingKeyContentLock(cb.bucketID, hashedName)
318+
contentKey := cachingKeyContent(cb.bucketID, hashedName)
319+
existsLockKey := cachingKeyExistsLock(cb.bucketID, hashedName)
320+
existsKey := cachingKeyExists(cb.bucketID, hashedName)
315321

316322
// Lookup the cache.
317323
if isCacheLookupEnabled(ctx) {
@@ -393,7 +399,8 @@ func (cb *CachingBucket) GetRange(ctx context.Context, name string, off, length
393399
return cb.Bucket.GetRange(ctx, name, off, length)
394400
}
395401

396-
return cb.cachedGetRange(ctx, name, off, length, cfgName, cfg)
402+
hashedName := cachingKeyHash(name)
403+
return cb.cachedGetRange(ctx, name, hashedName, off, length, cfgName, cfg)
397404
}
398405

399406
func (cb *CachingBucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error) {
@@ -402,12 +409,13 @@ func (cb *CachingBucket) Attributes(ctx context.Context, name string) (objstore.
402409
return cb.Bucket.Attributes(ctx, name)
403410
}
404411

405-
return cb.cachedAttributes(ctx, name, cfgName, cfg.cache, cfg.ttl)
412+
hashedName := cachingKeyHash(name)
413+
return cb.cachedAttributes(ctx, name, hashedName, cfgName, cfg.cache, cfg.ttl)
406414
}
407415

408-
func (cb *CachingBucket) cachedAttributes(ctx context.Context, name, cfgName string, cache cache.Cache, ttl time.Duration) (objstore.ObjectAttributes, error) {
409-
lockKey := cachingKeyAttributesLock(cb.bucketID, name)
410-
key := cachingKeyAttributes(cb.bucketID, name)
416+
func (cb *CachingBucket) cachedAttributes(ctx context.Context, name, hashedName, cfgName string, cache cache.Cache, ttl time.Duration) (objstore.ObjectAttributes, error) {
417+
lockKey := cachingKeyAttributesLock(cb.bucketID, hashedName)
418+
key := cachingKeyAttributes(cb.bucketID, hashedName)
411419

412420
// Lookup the cache.
413421
if isCacheLookupEnabled(ctx) {
@@ -445,8 +453,8 @@ func (cb *CachingBucket) cachedAttributes(ctx context.Context, name, cfgName str
445453
return attrs, nil
446454
}
447455

448-
func (cb *CachingBucket) cachedGetRange(ctx context.Context, name string, offset, length int64, cfgName string, cfg *getRangeConfig) (io.ReadCloser, error) {
449-
attrs, err := cb.cachedAttributes(ctx, name, cfgName, cfg.attributes.cache, cfg.attributes.ttl)
456+
func (cb *CachingBucket) cachedGetRange(ctx context.Context, name, hashedName string, offset, length int64, cfgName string, cfg *getRangeConfig) (io.ReadCloser, error) {
457+
attrs, err := cb.cachedAttributes(ctx, name, hashedName, cfgName, cfg.attributes.cache, cfg.attributes.ttl)
450458
if err != nil {
451459
return nil, errors.Wrapf(err, "failed to get object attributes: %s", name)
452460
}
@@ -484,7 +492,7 @@ func (cb *CachingBucket) cachedGetRange(ctx context.Context, name string, offset
484492
}
485493
totalRequestedBytes += (end - off)
486494

487-
k := cachingKeyObjectSubrange(cb.bucketID, name, off, end)
495+
k := cachingKeyObjectSubrange(cb.bucketID, hashedName, off, end)
488496
keys = append(keys, k)
489497
offsetKeys[off] = k
490498
}
@@ -663,7 +671,7 @@ func newCacheInvalidation(bucketID string, cfg *CachingBucketConfig, logger log.
663671
// prevent new cache entries for that item from being stored. This ensures that when the
664672
// cache entries for the item are deleted after it is mutated, reads which try to "add"
665673
// the lock key cannot and will go directly to object storage for a short period of time.
666-
func (i *cacheInvalidation) start(ctx context.Context, name string) {
674+
func (i *cacheInvalidation) start(ctx context.Context, name, hashedName string) {
667675
logger := spanlogger.FromContext(ctx, i.logger)
668676

669677
_, attrCfg := i.cfg.findAttributesConfig(name)
@@ -673,9 +681,9 @@ func (i *cacheInvalidation) start(ctx context.Context, name string) {
673681
existCfg = &getCfg.existsConfig
674682
}
675683

676-
attrLockKey := cachingKeyAttributesLock(i.bucketID, name)
677-
contentLockKey := cachingKeyContentLock(i.bucketID, name)
678-
existsLockKey := cachingKeyExistsLock(i.bucketID, name)
684+
attrLockKey := cachingKeyAttributesLock(i.bucketID, hashedName)
685+
contentLockKey := cachingKeyContentLock(i.bucketID, hashedName)
686+
existsLockKey := cachingKeyExistsLock(i.bucketID, hashedName)
679687

680688
if attrCfg != nil || getCfg != nil || existCfg != nil {
681689
err := i.runWithRetries(ctx, func() error {
@@ -703,7 +711,7 @@ func (i *cacheInvalidation) start(ctx context.Context, name string) {
703711
// finish removes attribute, existence, and content entries in a cache associated with
704712
// a given item. Note that it does not remove the "lock" entries in the cache to ensure
705713
// that other requests must read directly from object storage until the lock expires.
706-
func (i *cacheInvalidation) finish(ctx context.Context, name string) {
714+
func (i *cacheInvalidation) finish(ctx context.Context, name, hashedName string) {
707715
logger := spanlogger.FromContext(ctx, i.logger)
708716

709717
_, attrCfg := i.cfg.findAttributesConfig(name)
@@ -713,9 +721,9 @@ func (i *cacheInvalidation) finish(ctx context.Context, name string) {
713721
existCfg = &getCfg.existsConfig
714722
}
715723

716-
attrKey := cachingKeyAttributes(i.bucketID, name)
717-
contentKey := cachingKeyContent(i.bucketID, name)
718-
existsKey := cachingKeyExists(i.bucketID, name)
724+
attrKey := cachingKeyAttributes(i.bucketID, hashedName)
725+
contentKey := cachingKeyContent(i.bucketID, hashedName)
726+
existsKey := cachingKeyExists(i.bucketID, hashedName)
719727

720728
if attrCfg != nil || getCfg != nil || existCfg != nil {
721729
err := i.runWithRetries(ctx, func() error {
@@ -772,6 +780,13 @@ func (i *cacheInvalidation) runWithRetries(ctx context.Context, f func() error)
772780
return retry.Err()
773781
}
774782

783+
func cachingKeyHash(name string) string {
784+
hasher := crypto.SHA3_256.New()
785+
_, _ = hasher.Write([]byte(name)) // This will never error.
786+
// Hex because memcache keys must be non-whitespace non-control ASCII
787+
return hex.EncodeToString(hasher.Sum(nil))
788+
}
789+
775790
func cachingKeyAttributes(bucketID, name string) string {
776791
return composeCachingKey("attrs", bucketID, name)
777792
}

‎pkg/storage/tsdb/bucketcache/caching_bucket_test.go

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ func TestChunksCaching(t *testing.T) {
4040
}
4141

4242
name := "/test/chunks/000001"
43+
hashedName := cachingKeyHash(name)
4344

4445
inmem := objstore.NewInMemBucket()
4546
assert.NoError(t, inmem.Upload(context.Background(), name, bytes.NewReader(data)))
@@ -149,9 +150,9 @@ func TestChunksCaching(t *testing.T) {
149150
init: func() {
150151
ctx := context.Background()
151152
// Delete first 3 subranges.
152-
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, name, 0*subrangeSize, 1*subrangeSize)))
153-
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, name, 1*subrangeSize, 2*subrangeSize)))
154-
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, name, 2*subrangeSize, 3*subrangeSize)))
153+
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, hashedName, 0*subrangeSize, 1*subrangeSize)))
154+
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, hashedName, 1*subrangeSize, 2*subrangeSize)))
155+
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, hashedName, 2*subrangeSize, 3*subrangeSize)))
155156
},
156157
},
157158

@@ -167,9 +168,9 @@ func TestChunksCaching(t *testing.T) {
167168
init: func() {
168169
ctx := context.Background()
169170
// Delete last 3 subranges.
170-
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, name, 7*subrangeSize, 8*subrangeSize)))
171-
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, name, 8*subrangeSize, 9*subrangeSize)))
172-
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, name, 9*subrangeSize, 10*subrangeSize)))
171+
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, hashedName, 7*subrangeSize, 8*subrangeSize)))
172+
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, hashedName, 8*subrangeSize, 9*subrangeSize)))
173+
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, hashedName, 9*subrangeSize, 10*subrangeSize)))
173174
},
174175
},
175176

@@ -185,9 +186,9 @@ func TestChunksCaching(t *testing.T) {
185186
init: func() {
186187
ctx := context.Background()
187188
// Delete 3 subranges in the middle.
188-
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, name, 3*subrangeSize, 4*subrangeSize)))
189-
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, name, 4*subrangeSize, 5*subrangeSize)))
190-
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, name, 5*subrangeSize, 6*subrangeSize)))
189+
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, hashedName, 3*subrangeSize, 4*subrangeSize)))
190+
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, hashedName, 4*subrangeSize, 5*subrangeSize)))
191+
require.NoError(t, cache.Delete(ctx, cachingKeyObjectSubrange(bucketID, hashedName, 5*subrangeSize, 6*subrangeSize)))
191192
},
192193
},
193194

@@ -206,7 +207,7 @@ func TestChunksCaching(t *testing.T) {
206207
if i > 0 && i%3 == 0 {
207208
continue
208209
}
209-
require.NoError(t, cache.Delete(context.Background(), cachingKeyObjectSubrange(bucketID, name, i*subrangeSize, (i+1)*subrangeSize)))
210+
require.NoError(t, cache.Delete(context.Background(), cachingKeyObjectSubrange(bucketID, hashedName, i*subrangeSize, (i+1)*subrangeSize)))
210211
}
211212
},
212213
},
@@ -228,7 +229,7 @@ func TestChunksCaching(t *testing.T) {
228229
if i == 3 || i == 5 || i == 7 {
229230
continue
230231
}
231-
require.NoError(t, cache.Delete(context.Background(), cachingKeyObjectSubrange(bucketID, name, i*subrangeSize, (i+1)*subrangeSize)))
232+
require.NoError(t, cache.Delete(context.Background(), cachingKeyObjectSubrange(bucketID, hashedName, i*subrangeSize, (i+1)*subrangeSize)))
232233
}
233234
},
234235
},
@@ -249,7 +250,7 @@ func TestChunksCaching(t *testing.T) {
249250
if i == 5 || i == 6 || i == 7 {
250251
continue
251252
}
252-
require.NoError(t, cache.Delete(context.Background(), cachingKeyObjectSubrange(bucketID, name, i*subrangeSize, (i+1)*subrangeSize)))
253+
require.NoError(t, cache.Delete(context.Background(), cachingKeyObjectSubrange(bucketID, hashedName, i*subrangeSize, (i+1)*subrangeSize)))
253254
}
254255
},
255256
},

0 commit comments

Comments
 (0)