ruler: conditionally enable caching of rule group contents

Add a new experimental flag to enable caching of rule group contents
using the rule store cache. Rule groups are cached using the same TTL
as rule group listings: one evaluation interval.

Part of #9386

Signed-off-by: Nick Pillitteri <nick.pillitteri@grafana.com>

cmd/mimir/config-descriptor.json

@@ -13669,6 +13669,17 @@
           "required": false,
           "desc": "",
           "blockEntries": [
+            {
+              "kind": "field",
+              "name": "cache_rule_groups",
+              "required": false,
+              "desc": "Enabling caching of rule group contents if a cache backend is configured.",
+              "fieldValue": null,
+              "fieldDefaultValue": false,
+              "fieldFlag": "ruler-storage.cache.cache-rule-groups",
+              "fieldType": "boolean",
+              "fieldCategory": "experimental"
+            },
             {
               "kind": "field",
               "name": "backend",

cmd/mimir/help-all.txt.tmpl

@@ -2399,6 +2399,8 @@ Usage of ./cmd/mimir/mimir:
     	Backend storage to use. Supported backends are: s3, gcs, azure, swift, filesystem, local. (default "filesystem")
   -ruler-storage.cache.backend string
     	Backend for ruler storage cache, if not empty. The cache is supported for any storage backend except "local". Supported values: memcached, redis.
+  -ruler-storage.cache.cache-rule-groups
+    	[experimental] Enabling caching of rule group contents if a cache backend is configured.
   -ruler-storage.cache.memcached.addresses comma-separated-list-of-strings
     	Comma-separated list of memcached addresses. Each address can be an IP address, hostname, or an entry specified in the DNS Service Discovery format.
   -ruler-storage.cache.memcached.connect-timeout duration

docs/sources/mimir/configure/configuration-parameters/index.md

@@ -2166,6 +2166,11 @@ local:
   [directory: <string> | default = ""]
 
 cache:
+  # (experimental) Enabling caching of rule group contents if a cache backend is
+  # configured.
+  # CLI flag: -ruler-storage.cache.cache-rule-groups
+  [cache_rule_groups: <boolean> | default = false]
+
   # Backend for ruler storage cache, if not empty. The cache is supported for
   # any storage backend except "local". Supported values: memcached, redis.
   # CLI flag: -ruler-storage.cache.backend

pkg/ruler/rulestore/config.go

@@ -29,8 +29,21 @@ type Config struct {
 	bucket.Config `yaml:",inline"`
 	Local         LocalStoreConfig `yaml:"local"`
 
+	// RulerCache holds the configuration used for the ruler storage cache.
+	RulerCache RulerCacheConfig `yaml:"cache"`
+}
+
+// RulerCacheConfig is configuration for the cache used by ruler storage as well as
+// additional ruler storage specific configuration.
+//
+// NOTE: This is temporary while caching of rule groups is being tested. This will be removed
+// in the future and cache.BackendConfig will be moved back to the Config struct above.
+type RulerCacheConfig struct {
+	// CacheRuleGroups enables caching of rule group contents
+	CacheRuleGroups bool `yaml:"cache_rule_groups" category:"experimental"`
+
 	// Cache holds the configuration used for the ruler storage cache.
-	Cache cache.BackendConfig `yaml:"cache"`
+	Cache cache.BackendConfig `yaml:",inline"`
 }
 
 // RegisterFlags registers the backend storage config.
@@ -41,17 +54,18 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	cfg.Local.RegisterFlagsWithPrefix(prefix, f)
 	cfg.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, "ruler", f)
 
-	f.StringVar(&cfg.Cache.Backend, prefix+"cache.backend", "", fmt.Sprintf("Backend for ruler storage cache, if not empty. The cache is supported for any storage backend except %q. Supported values: %s.", BackendLocal, strings.Join(supportedCacheBackends, ", ")))
-	cfg.Cache.Memcached.RegisterFlagsWithPrefix(prefix+"cache.memcached.", f)
-	cfg.Cache.Redis.RegisterFlagsWithPrefix(prefix+"cache.redis.", f)
+	f.BoolVar(&cfg.RulerCache.CacheRuleGroups, prefix+"cache.cache-rule-groups", false, "Enabling caching of rule group contents if a cache backend is configured.")
+	f.StringVar(&cfg.RulerCache.Cache.Backend, prefix+"cache.backend", "", fmt.Sprintf("Backend for ruler storage cache, if not empty. The cache is supported for any storage backend except %q. Supported values: %s.", BackendLocal, strings.Join(supportedCacheBackends, ", ")))
+	cfg.RulerCache.Cache.Memcached.RegisterFlagsWithPrefix(prefix+"cache.memcached.", f)
+	cfg.RulerCache.Cache.Redis.RegisterFlagsWithPrefix(prefix+"cache.redis.", f)
 }
 
 func (cfg *Config) Validate() error {
 	if err := cfg.Config.Validate(); err != nil {
 		return err
 	}
 
-	return cfg.Cache.Validate()
+	return cfg.RulerCache.Cache.Validate()
 }
 
 // IsDefaults returns true if the storage options have not been set.

pkg/ruler/storage.go

@@ -60,7 +60,7 @@ func wrapBucketWithCache(bkt objstore.Bucket, cfg rulestore.Config, cacheTTL tim
 
 	cacheCfg := bucketcache.NewCachingBucketConfig()
 
-	cacheClient, err := cache.CreateClient("ruler-storage-cache", cfg.Cache, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
+	cacheClient, err := cache.CreateClient("ruler-storage-cache", cfg.RulerCache.Cache, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
 	if err != nil {
 		return nil, errors.Wrapf(err, "ruler-storage-cache")
 	}
@@ -77,6 +77,12 @@ func wrapBucketWithCache(bkt objstore.Bucket, cfg rulestore.Config, cacheTTL tim
 	codec := bucketcache.SnappyIterCodec{IterCodec: bucketcache.JSONIterCodec{}}
 	cacheCfg.CacheIter("iter", cacheClient, isNotTenantsDir, cacheTTL, codec)
 
+	// Only cache the contents of rule groups if enabled. This is an experimental feature and we need to be able
+	// to disable it. Once this feature is validated, it will be enabled unconditionally.
+	if cfg.RulerCache.CacheRuleGroups {
+		cacheCfg.CacheGet("rule-groups", cacheClient, isRuleGroup, maxItemSize(cfg.RulerCache.Cache), cacheTTL, cacheTTL, cacheTTL)
+	}
+
 	return bucketcache.NewCachingBucket("ruler", bkt, cacheCfg, logger, reg)
 }
 
@@ -87,3 +93,14 @@ func isNotTenantsDir(name string) bool {
 func isRuleGroup(name string) bool {
 	return strings.HasPrefix(name, "rules/")
 }
+
+func maxItemSize(cfg cache.BackendConfig) int {
+	switch cfg.Backend {
+	case cache.BackendMemcached:
+		return cfg.Memcached.MaxItemSize
+	case cache.BackendRedis:
+		return cfg.Redis.MaxItemSize
+	default:
+		return 0
+	}
+}