Support gzipped runtime configurations

Some runtime configs can be too large to fit in one kubernetes
configmap. However, Kubernetes configmaps accept binary data, so we can
gzip the yaml, which offers up to 10x compression.

This adds a new flag to runtimeconfig.Manager's config that enabled gzip
uncompression of the files, disabled by default.

Signed-off-by: Oleg Zaytsev <mail@olegzaytsev.com>

runtimeconfig/manager.go

@@ -2,6 +2,7 @@ package runtimeconfig
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"crypto/sha256"
 	"flag"
@@ -29,6 +30,7 @@ type Loader func(r io.Reader) (interface{}, error)
 // It holds config related to loading per-tenant config.
 type Config struct {
 	ReloadPeriod time.Duration `yaml:"period" category:"advanced"`
+	AllowGzip    bool          `yaml:"allow_gzip" category:"advanced"`
 	// LoadPath contains the path to the runtime config files.
 	// Requires a non-empty value
 	LoadPath flagext.StringSliceCSV `yaml:"file"`
@@ -39,6 +41,7 @@ type Config struct {
 func (mc *Config) RegisterFlags(f *flag.FlagSet) {
 	f.Var(&mc.LoadPath, "runtime-config.file", "Comma separated list of yaml files with the configuration that can be updated at runtime. Runtime config files will be merged from left to right.")
 	f.DurationVar(&mc.ReloadPeriod, "runtime-config.reload-period", 10*time.Second, "How often to check runtime config files.")
+	f.BoolVar(&mc.AllowGzip, "runtime-config.allow-gzip", false, "Allow runtime config files to be gzipped.")
 }
 
 // Manager periodically reloads the configuration from specified files, and keeps this
@@ -183,8 +186,8 @@ func (om *Manager) loadConfig() error {
 
 	mergedConfig := map[string]interface{}{}
 	for _, f := range om.cfg.LoadPath {
-		yamlFile := map[string]interface{}{}
-		err := yaml.Unmarshal(rawData[f], &yamlFile)
+		data := rawData[f]
+		yamlFile, err := om.unmarshalMaybeGzipped(data)
 		if err != nil {
 			om.configLoadSuccess.Set(0)
 			return errors.Wrapf(err, "unmarshal file %q", f)
@@ -218,6 +221,33 @@ func (om *Manager) loadConfig() error {
 	return nil
 }
 
+func (om *Manager) unmarshalMaybeGzipped(data []byte) (map[string]any, error) {
+	yamlFile := map[string]any{}
+	if om.cfg.AllowGzip && isGzip(data) {
+		r, err := gzip.NewReader(bytes.NewReader(data))
+		if err != nil {
+			return nil, err
+		}
+		defer r.Close()
+
+		err = yaml.NewDecoder(r).Decode(&yamlFile)
+		return yamlFile, errors.Wrap(err, "uncompress/unmarshal gzipped file")
+	}
+
+	if err := yaml.Unmarshal(data, &yamlFile); err != nil {
+		// Give a hint if we think that file is gzipped.
+		if !om.cfg.AllowGzip && isGzip(data) {
+			return nil, errors.Wrap(err, "file looks gzipped but gzip is disabled")
+		}
+		return nil, err
+	}
+	return yamlFile, nil
+}
+
+func isGzip(data []byte) bool {
+	return len(data) > 2 && data[0] == 0x1f && data[1] == 0x8b
+}
+
 func mergeConfigMaps(a, b map[string]interface{}) map[string]interface{} {
 	out := make(map[string]interface{}, len(a))
 	for k, v := range a {

runtimeconfig/manager_test.go

@@ -1,11 +1,13 @@
 package runtimeconfig
 
 import (
+	"compress/gzip"
 	"context"
 	"crypto/sha256"
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -176,6 +178,70 @@ func TestNewOverridesManager(t *testing.T) {
 	require.Equal(t, 150, conf.Overrides["user1"].Limit2)
 }
 
+func TestManagerGzip(t *testing.T) {
+	writeConfig := func(gzipped bool) string {
+		dir := t.TempDir()
+		filename := filepath.Join(dir, "overrides.yaml")
+		f, err := os.Create(filename)
+		require.NoError(t, err)
+		defer f.Close()
+		w := io.Writer(f)
+		if gzipped {
+			gw := gzip.NewWriter(f)
+			defer gw.Close()
+			w = gw
+		}
+		require.NoError(t, yaml.NewEncoder(w).Encode(map[string]any{
+			"overrides": map[string]any{
+				"user1": map[string]any{
+					"limit2": 150,
+				},
+			},
+		}))
+		return filename
+	}
+
+	defaultTestLimits = &TestLimits{Limit1: 100}
+
+	t.Run("allowed", func(t *testing.T) {
+		for _, gzipped := range []bool{true, false} {
+			t.Run(fmt.Sprintf("gzipped=%t", gzipped), func(t *testing.T) {
+				cfg := Config{
+					ReloadPeriod: time.Second,
+					LoadPath:     []string{writeConfig(gzipped)},
+					Loader:       testLoadOverrides,
+					AllowGzip:    true,
+				}
+				manager, err := New(cfg, "overrides", nil, log.NewNopLogger())
+				require.NoError(t, err)
+				require.NoError(t, services.StartAndAwaitRunning(context.Background(), manager))
+				t.Cleanup(func() { require.NoError(t, services.StopAndAwaitTerminated(context.Background(), manager)) })
+
+				// Make sure test limits were loaded.
+				require.NotNil(t, manager.GetConfig())
+				conf := manager.GetConfig().(*testOverrides)
+				require.NotNil(t, conf)
+				require.Equal(t, 150, conf.Overrides["user1"].Limit2)
+
+			})
+		}
+	})
+
+	t.Run("disallowed", func(t *testing.T) {
+		cfg := Config{
+			ReloadPeriod: time.Second,
+			LoadPath:     []string{writeConfig(true)},
+			Loader:       testLoadOverrides,
+			AllowGzip:    false,
+		}
+		manager, err := New(cfg, "overrides", nil, log.NewNopLogger())
+		require.NoError(t, err)
+		err = services.StartAndAwaitRunning(context.Background(), manager)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "file looks gzipped but gzip is disabled")
+	})
+}
+
 func TestOverridesManagerMultipleFilesAppend(t *testing.T) {
 	tempFiles, err := generateRuntimeFiles(t,
 		[]string{`overrides: