From 94897265ed64f9efad4670193e90cc906cd05bd5 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Tue, 8 Aug 2023 18:32:28 +0200 Subject: [PATCH 01/43] Add new runnable thread to execute redis bgsave --- api/v1alpha1/sentinel_types.go | 33 ++++++ api/v1alpha1/sentinel_types_test.go | 110 ++++++++++++++++++ controllers/redisshard_controller.go | 2 +- controllers/sentinel_controller.go | 4 +- pkg/generators/twemproxyconfig/generator.go | 2 +- pkg/redis/backup/backup.go | 64 ++++++++++ pkg/redis/backup/manager.go | 70 +++++++++++ pkg/redis/client/fake_client.go | 10 ++ pkg/redis/client/goredis_client.go | 10 +- pkg/redis/client/interface.go | 2 + pkg/redis/server/server.go | 8 ++ pkg/redis/sharded/redis_shard.go | 12 +- pkg/redis/sharded/redis_shard_test.go | 4 +- pkg/redis/sharded/redis_sharded_cluster.go | 25 +++- .../sharded/redis_sharded_cluster_test.go | 2 +- pkg/redis/sharded/sentinel_server.go | 5 +- pkg/redis/sharded/sentinel_server_test.go | 11 +- 17 files changed, 356 insertions(+), 18 deletions(-) create mode 100644 api/v1alpha1/sentinel_types_test.go create mode 100644 pkg/redis/backup/backup.go create mode 100644 pkg/redis/backup/manager.go diff --git a/api/v1alpha1/sentinel_types.go b/api/v1alpha1/sentinel_types.go index f383c533..7408f6ec 100644 --- a/api/v1alpha1/sentinel_types.go +++ b/api/v1alpha1/sentinel_types.go @@ -17,9 +17,12 @@ limitations under the License. package v1alpha1 import ( + "context" "time" "github.com/3scale/saas-operator/pkg/redis/client" + redis "github.com/3scale/saas-operator/pkg/redis/server" + "github.com/3scale/saas-operator/pkg/redis/sharded" "github.com/3scale/saas-operator/pkg/util" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -180,6 +183,36 @@ type SentinelStatus struct { MonitoredShards MonitoredShards `json:"monitoredShards,omitempty"` } +func (ss *SentinelStatus) ShardedCluster(ctx context.Context, pool *redis.ServerPool) (*sharded.Cluster, error) { + + // have a list of sentinels but must provide a map + // TODO: at some point change the SentinelStatus.Sentinels to also have a map and avoid this + msentinel := make(map[string]string, len(ss.Sentinels)) + for _, s := range ss.Sentinels { + msentinel[s] = s + } + + shards := make([]*sharded.Shard, 0, len(ss.MonitoredShards)) + // TODO: generate slice of shards from status + for _, s := range ss.MonitoredShards { + servers := make([]*sharded.RedisServer, 0, len(s.Servers)) + for _, rsd := range s.Servers { + srv, err := pool.GetServer("redis://"+rsd.Address, nil) + if err != nil { + return nil, err + } + servers = append(servers, sharded.NewRedisServerFromParams(srv, rsd.Role, rsd.Config)) + } + shards = append(shards, sharded.NewShardFromServers(s.Name, pool, servers...)) + } + + cluster, err := sharded.NewShardedCluster(ctx, pool, msentinel, shards...) + if err != nil { + return nil, err + } + return cluster, nil +} + type MonitoredShards []MonitoredShard // MonitoredShards implements sort.Interface based on the Name field. diff --git a/api/v1alpha1/sentinel_types_test.go b/api/v1alpha1/sentinel_types_test.go new file mode 100644 index 00000000..1b409a87 --- /dev/null +++ b/api/v1alpha1/sentinel_types_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "testing" + + "github.com/3scale/saas-operator/pkg/redis/client" + redis "github.com/3scale/saas-operator/pkg/redis/server" + "github.com/3scale/saas-operator/pkg/redis/sharded" + "github.com/3scale/saas-operator/pkg/util" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +func TestSentinelStatus_ShardedCluster(t *testing.T) { + type fields struct { + Sentinels []string + MonitoredShards MonitoredShards + } + type args struct { + ctx context.Context + pool *redis.ServerPool + } + tests := []struct { + name string + fields fields + args args + want *sharded.Cluster + wantErr bool + }{ + { + name: "Generates a sharded.Cluster resource from the sentinel status", + fields: fields{ + Sentinels: []string{"redis://127.0.0.1:26379"}, + MonitoredShards: []MonitoredShard{ + {Name: "shard01", + Servers: map[string]RedisServerDetails{ + "srv1": {Role: client.Master, Address: "127.0.0.1:1000", Config: map[string]string{"save": ""}}, + "srv2": {Role: client.Slave, Address: "127.0.0.1:2000", Config: map[string]string{"slave-read-only": "yes"}}, + }}, + {Name: "shard02", + Servers: map[string]RedisServerDetails{ + "srv3": {Role: client.Master, Address: "127.0.0.1:3000", Config: map[string]string{}}, + "srv4": {Role: client.Slave, Address: "127.0.0.1:4000", Config: map[string]string{}}, + }}, + }, + }, + args: args{ + ctx: context.TODO(), + pool: redis.NewServerPool( + redis.MustNewServer("redis://127.0.0.1:1000", util.Pointer("srv1")), + redis.MustNewServer("redis://127.0.0.1:2000", util.Pointer("srv2")), + redis.MustNewServer("redis://127.0.0.1:3000", util.Pointer("srv3")), + redis.MustNewServer("redis://127.0.0.1:4000", util.Pointer("srv4")), + redis.MustNewServer("redis://127.0.0.1:26379", util.Pointer("sentinel")), + ), + }, + want: &sharded.Cluster{ + Shards: []*sharded.Shard{ + {Name: "shard01", + Servers: []*sharded.RedisServer{ + sharded.NewRedisServerFromParams(redis.MustNewServer("redis://127.0.0.1:1000", util.Pointer("srv1")), client.Master, map[string]string{"save": ""}), + sharded.NewRedisServerFromParams(redis.MustNewServer("redis://127.0.0.1:2000", util.Pointer("srv2")), client.Slave, map[string]string{"slave-read-only": "yes"}), + }}, + {Name: "shard02", + Servers: []*sharded.RedisServer{ + sharded.NewRedisServerFromParams(redis.MustNewServer("redis://127.0.0.1:3000", util.Pointer("srv3")), client.Master, map[string]string{}), + sharded.NewRedisServerFromParams(redis.MustNewServer("redis://127.0.0.1:4000", util.Pointer("srv4")), client.Slave, map[string]string{}), + }}, + }, + Sentinels: []*sharded.SentinelServer{ + sharded.NewSentinelServerFromParams(redis.MustNewServer("redis://127.0.0.1:26379", util.Pointer("sentinel"))), + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ss := &SentinelStatus{ + Sentinels: tt.fields.Sentinels, + MonitoredShards: tt.fields.MonitoredShards, + } + got, err := ss.ShardedCluster(tt.args.ctx, tt.args.pool) + if (err != nil) != tt.wantErr { + t.Errorf("SentinelStatus.ShardedCluster() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(sharded.Cluster{}, sharded.Shard{}, redis.Server{})); len(diff) > 0 { + t.Errorf("SentinelStatus.ShardedCluster() = diff %s", diff) + } + }) + } +} diff --git a/controllers/redisshard_controller.go b/controllers/redisshard_controller.go index 14039e41..137562dd 100644 --- a/controllers/redisshard_controller.go +++ b/controllers/redisshard_controller.go @@ -128,7 +128,7 @@ func (r *RedisShardReconciler) setRedisRoles(ctx context.Context, key types.Name } } - shard, err := sharded.NewShard(key.Name, redisURLs, r.Pool) + shard, err := sharded.NewShardFromTopology(key.Name, redisURLs, r.Pool) if err != nil { return shard, &ctrl.Result{}, err } diff --git a/controllers/sentinel_controller.go b/controllers/sentinel_controller.go index f99f4f7f..2d4520db 100644 --- a/controllers/sentinel_controller.go +++ b/controllers/sentinel_controller.go @@ -105,7 +105,7 @@ func (r *SentinelReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c if err != nil { return ctrl.Result{}, err } - shardedCluster, err := sharded.NewShardedCluster(ctx, clustermap, r.Pool) + shardedCluster, err := sharded.NewShardedClusterFromTopology(ctx, clustermap, r.Pool) if err != nil { return ctrl.Result{}, err } @@ -120,7 +120,7 @@ func (r *SentinelReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c if err := shardedCluster.Discover(ctx); err != nil { return ctrl.Result{}, err } - if _, err := sentinel.Monitor(ctx, shardedCluster); err != nil { + if _, err := sentinel.Monitor(ctx, shardedCluster, saasv1alpha1.SentinelDefaultQuorum); err != nil { return ctrl.Result{}, err } } diff --git a/pkg/generators/twemproxyconfig/generator.go b/pkg/generators/twemproxyconfig/generator.go index 8b2fac3b..2fba42ee 100644 --- a/pkg/generators/twemproxyconfig/generator.go +++ b/pkg/generators/twemproxyconfig/generator.go @@ -86,7 +86,7 @@ func NewGenerator(ctx context.Context, instance *saasv1alpha1.TwemproxyConfig, c clustermap["sentinel"][alias] = u.String() } - shardedCluster, err := sharded.NewShardedCluster(ctx, clustermap, pool) + shardedCluster, err := sharded.NewShardedClusterFromTopology(ctx, clustermap, pool) if err != nil { return Generator{}, err } diff --git a/pkg/redis/backup/backup.go b/pkg/redis/backup/backup.go new file mode 100644 index 00000000..1a073349 --- /dev/null +++ b/pkg/redis/backup/backup.go @@ -0,0 +1,64 @@ +package backup + +import ( + "context" + "time" +) + +func (br BackupRunner) RunBackup(ctx context.Context) (chan bool, chan error) { + done := make(chan bool) + errCh := make(chan error) + + go func() { + if err := br.BackgroundSave(ctx); err != nil { + errCh <- err + } + close(done) + + }() + + return done, errCh +} + +func (br BackupRunner) BackgroundSave(ctx context.Context) error { + lastsave, err := br.server.RedisLastSave(ctx) + if err != nil { + return err + } + + err = br.server.RedisBGSave(ctx) + if err != nil { + // TODO: need to hanlde the case that a save is already running + return err + } + + ticker := time.NewTicker(60 * time.Second) + done := make(chan bool) + + // wait until BGSAVE completes + for { + select { + case <-ticker.C: + seconds, err := br.server.RedisLastSave(ctx) + if err != nil { + // retry in next ticker + continue + } + if seconds < lastsave { + // BGSAVE completed + close(done) + } + case <-done: + return nil + } + } + +} + +func (br BackupRunner) UploadBackup(ctx context.Context) error { + return nil +} + +func (br BackupRunner) TagBackup(ctx context.Context) error { + return nil +} diff --git a/pkg/redis/backup/manager.go b/pkg/redis/backup/manager.go new file mode 100644 index 00000000..053c87fe --- /dev/null +++ b/pkg/redis/backup/manager.go @@ -0,0 +1,70 @@ +package backup + +import ( + "context" + "fmt" + "time" + + "github.com/3scale/saas-operator/pkg/redis/sharded" + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/event" +) + +type BackupRunner struct { + shardName string + server *sharded.RedisServer + timeout time.Duration + // timestamp time.Time + eventsCh chan event.GenericEvent + started bool + cancel context.CancelFunc +} + +func (br *BackupRunner) GetID() string { + return br.shardName +} + +// IsStarted returns whether the backup runner is started or not +func (br *BackupRunner) IsStarted() bool { + return br.started +} + +func (br *BackupRunner) SetChannel(ch chan event.GenericEvent) { + br.eventsCh = ch +} + +// Start starts the backup runner +func (br *BackupRunner) Start(parentCtx context.Context, l logr.Logger) error { + log := l.WithValues("server", br.server.GetAlias()) + + var ctx context.Context + ctx, br.cancel = context.WithCancel(parentCtx) + defer br.cancel() + + var done chan bool + var errCh chan error + done, errCh = br.RunBackup(ctx) + log.Info("backup running") + + // apply a time boundary to the backup and listen for errors + timer := time.NewTimer(br.timeout) + for { + select { + case <-timer.C: + return fmt.Errorf("timeout reached (%v)", br.timeout) + + case <-done: + return nil + + case err := <-errCh: + return err + + } + } + +} + +// Stop stops the sentinel event watcher +func (br *BackupRunner) Stop() { + br.cancel() +} diff --git a/pkg/redis/client/fake_client.go b/pkg/redis/client/fake_client.go index 7c95a09e..c6434d83 100644 --- a/pkg/redis/client/fake_client.go +++ b/pkg/redis/client/fake_client.go @@ -135,6 +135,16 @@ func (fc *FakeClient) RedisDo(ctx context.Context, args ...interface{}) (interfa return rsp.InjectResponse(), rsp.InjectError() } +func (fc *FakeClient) RedisBGSave(ctx context.Context) error { + rsp := fc.pop() + return rsp.InjectError() +} + +func (fc *FakeClient) RedisLastSave(ctx context.Context) (int64, error) { + rsp := fc.pop() + return rsp.InjectResponse().(int64), rsp.InjectError() +} + func (fc *FakeClient) pop() (fakeRsp FakeResponse) { fakeRsp, fc.Responses = fc.Responses[0], fc.Responses[1:] return fakeRsp diff --git a/pkg/redis/client/goredis_client.go b/pkg/redis/client/goredis_client.go index 7ce62115..0f2235eb 100644 --- a/pkg/redis/client/goredis_client.go +++ b/pkg/redis/client/goredis_client.go @@ -119,7 +119,6 @@ func (c *GoRedisClient) SentinelDo(ctx context.Context, args ...interface{}) (in } func (c *GoRedisClient) RedisRole(ctx context.Context) (interface{}, error) { - val, err := c.redis.Do(ctx, "role").Result() return val, err } @@ -152,3 +151,12 @@ func (c *GoRedisClient) RedisDo(ctx context.Context, args ...interface{}) (inter val, err := c.redis.Do(ctx, args...).Result() return val, err } + +func (c *GoRedisClient) RedisBGSave(ctx context.Context) error { + _, err := c.redis.BgSave(ctx).Result() + return err +} + +func (c *GoRedisClient) RedisLastSave(ctx context.Context) (int64, error) { + return c.redis.LastSave(ctx).Result() +} diff --git a/pkg/redis/client/interface.go b/pkg/redis/client/interface.go index 12123674..11b2c06e 100644 --- a/pkg/redis/client/interface.go +++ b/pkg/redis/client/interface.go @@ -26,6 +26,8 @@ type TestableInterface interface { RedisSlaveOf(context.Context, string, string) error RedisDebugSleep(context.Context, time.Duration) error RedisDo(context.Context, ...interface{}) (interface{}, error) + RedisBGSave(context.Context) error + RedisLastSave(context.Context) (int64, error) Close() error } diff --git a/pkg/redis/server/server.go b/pkg/redis/server/server.go index 09d44b00..54c0992e 100644 --- a/pkg/redis/server/server.go +++ b/pkg/redis/server/server.go @@ -223,6 +223,14 @@ func (srv *Server) RedisDebugSleep(ctx context.Context, duration time.Duration) return srv.client.RedisDebugSleep(ctx, duration) } +func (srv *Server) RedisBGSave(ctx context.Context) error { + return srv.client.RedisBGSave(ctx) +} + +func (srv *Server) RedisLastSave(ctx context.Context) (int64, error) { + return srv.client.RedisLastSave(ctx) +} + // This is a horrible function to parse the horrible structs that the go-redis // client returns for administrative commands. I swear it's not my fault ... func sliceCmdToStruct(in interface{}, out interface{}) error { diff --git a/pkg/redis/sharded/redis_shard.go b/pkg/redis/sharded/redis_shard.go index 68a952d1..42382b5d 100644 --- a/pkg/redis/sharded/redis_shard.go +++ b/pkg/redis/sharded/redis_shard.go @@ -19,8 +19,16 @@ type Shard struct { pool *redis.ServerPool } -// NewShard returns a Shard object given the passed redis server URLs -func NewShard(name string, servers map[string]string, pool *redis.ServerPool) (*Shard, error) { +func NewShardFromServers(name string, pool *redis.ServerPool, servers ...*RedisServer) *Shard { + shard := &Shard{} + shard.Name = name + shard.Servers = append(shard.Servers, servers...) + shard.pool = pool + return shard +} + +// NewShardFromTopology returns a Shard object given the passed redis server URLs +func NewShardFromTopology(name string, servers map[string]string, pool *redis.ServerPool) (*Shard, error) { var merr util.MultiError shard := &Shard{Name: name, pool: pool} shard.Servers = make([]*RedisServer, 0, len(servers)) diff --git a/pkg/redis/sharded/redis_shard_test.go b/pkg/redis/sharded/redis_shard_test.go index 874ff9eb..d40ac28a 100644 --- a/pkg/redis/sharded/redis_shard_test.go +++ b/pkg/redis/sharded/redis_shard_test.go @@ -156,7 +156,7 @@ func TestNewShard(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := NewShard(tt.args.name, tt.args.servers, tt.args.pool) + got, err := NewShardFromTopology(tt.args.name, tt.args.servers, tt.args.pool) if (error(err) != nil) != tt.wantErr { t.Errorf("NewShard() error = %v, wantErr %v", err, tt.wantErr) return @@ -1033,7 +1033,7 @@ func TestShard_GetServerByID(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - shard, _ := NewShard("test", tt.servers, redis.NewServerPool()) + shard, _ := NewShardFromTopology("test", tt.servers, redis.NewServerPool()) got, err := shard.GetServerByID(tt.args.hostport) if (err != nil) != tt.wantErr { t.Errorf("Shard.GetServerByID() error = %v, wantErr %v", err, tt.wantErr) diff --git a/pkg/redis/sharded/redis_sharded_cluster.go b/pkg/redis/sharded/redis_sharded_cluster.go index 75698c17..43a817ba 100644 --- a/pkg/redis/sharded/redis_sharded_cluster.go +++ b/pkg/redis/sharded/redis_sharded_cluster.go @@ -18,9 +18,26 @@ type Cluster struct { pool *redis.ServerPool } -// NewShardedCluster returns a new ShardedCluster given the shard structure passed as a map[string][]string -func NewShardedCluster(ctx context.Context, serverList map[string]map[string]string, pool *redis.ServerPool) (*Cluster, error) { - logger := log.FromContext(ctx, "function", "NewShardedCluster") +func NewShardedCluster(ctx context.Context, pool *redis.ServerPool, sentinels map[string]string, shards ...*Shard) (*Cluster, error) { + cluster := &Cluster{pool: pool} + cluster.Shards = make([]*Shard, 0, len(shards)) + + // populate sentinel servers + has, err := NewHighAvailableSentinel(sentinels, pool) + if err != nil { + return nil, err + } + cluster.Sentinels = has + + // populate shards + cluster.Shards = append(cluster.Shards, shards...) + + return cluster, nil +} + +// NewShardedClusterFromTopology returns a new ShardedCluster given the shard structure passed as a map[string][]string +func NewShardedClusterFromTopology(ctx context.Context, serverList map[string]map[string]string, pool *redis.ServerPool) (*Cluster, error) { + logger := log.FromContext(ctx, "function", "NewShardedClusterFromTopology") cluster := Cluster{pool: pool} cluster.Shards = make([]*Shard, 0, len(serverList)) @@ -36,7 +53,7 @@ func NewShardedCluster(ctx context.Context, serverList map[string]map[string]str cluster.Sentinels = sentinels default: - shard, err := NewShard(shardName, shardServers, pool) + shard, err := NewShardFromTopology(shardName, shardServers, pool) if err != nil { logger.Error(err, "unable to create sharded cluster") return nil, err diff --git a/pkg/redis/sharded/redis_sharded_cluster_test.go b/pkg/redis/sharded/redis_sharded_cluster_test.go index aea2d64c..6fe3a18e 100644 --- a/pkg/redis/sharded/redis_sharded_cluster_test.go +++ b/pkg/redis/sharded/redis_sharded_cluster_test.go @@ -115,7 +115,7 @@ func TestNewShardedCluster(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := NewShardedCluster(tt.args.ctx, tt.args.serverList, tt.args.pool) + got, err := NewShardedClusterFromTopology(tt.args.ctx, tt.args.serverList, tt.args.pool) if (err != nil) != tt.wantErr { t.Errorf("NewShardedCluster() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/redis/sharded/sentinel_server.go b/pkg/redis/sharded/sentinel_server.go index e93534bb..581d5b78 100644 --- a/pkg/redis/sharded/sentinel_server.go +++ b/pkg/redis/sharded/sentinel_server.go @@ -4,7 +4,6 @@ import ( "context" "sort" - saasv1alpha1 "github.com/3scale/saas-operator/api/v1alpha1" redis "github.com/3scale/saas-operator/pkg/redis/server" "github.com/3scale/saas-operator/pkg/util" ) @@ -87,7 +86,7 @@ func (sentinel *SentinelServer) IsMonitoringShards(ctx context.Context, shards [ } // Monitor ensures that all the shards in the ShardedCluster object are monitored by the SentinelServer -func (sentinel *SentinelServer) Monitor(ctx context.Context, cluster *Cluster) ([]string, error) { +func (sentinel *SentinelServer) Monitor(ctx context.Context, cluster *Cluster, quorum int) ([]string, error) { changed := []string{} // Initialize unmonitored shards @@ -104,7 +103,7 @@ func (sentinel *SentinelServer) Monitor(ctx context.Context, cluster *Cluster) ( return changed, err } - err = sentinel.SentinelMonitor(ctx, name, master.GetHost(), master.GetPort(), saasv1alpha1.SentinelDefaultQuorum) + err = sentinel.SentinelMonitor(ctx, name, master.GetHost(), master.GetPort(), quorum) if err != nil { return changed, util.WrapError("redis-sentinel/SentinelServer.Monitor", err) } diff --git a/pkg/redis/sharded/sentinel_server_test.go b/pkg/redis/sharded/sentinel_server_test.go index 0ebcec46..f35f3ac0 100644 --- a/pkg/redis/sharded/sentinel_server_test.go +++ b/pkg/redis/sharded/sentinel_server_test.go @@ -271,6 +271,7 @@ func TestSentinelServer_Monitor(t *testing.T) { type args struct { ctx context.Context shards *Cluster + quorum int } tests := []struct { name string @@ -319,6 +320,7 @@ func TestSentinelServer_Monitor(t *testing.T) { args: args{ ctx: context.TODO(), shards: testShardedCluster, + quorum: 2, }, want: []string{}, wantErr: false, @@ -367,6 +369,7 @@ func TestSentinelServer_Monitor(t *testing.T) { args: args{ ctx: context.TODO(), shards: testShardedCluster, + quorum: 2, }, want: []string{"shard01"}, wantErr: false, @@ -423,6 +426,7 @@ func TestSentinelServer_Monitor(t *testing.T) { args: args{ ctx: context.TODO(), shards: testShardedCluster, + quorum: 2, }, want: []string{"shard00", "shard01", "shard02"}, wantErr: false, @@ -468,6 +472,7 @@ func TestSentinelServer_Monitor(t *testing.T) { args: args{ ctx: context.TODO(), shards: testShardedCluster, + quorum: 2, }, want: []string{"shard00"}, wantErr: true, @@ -495,6 +500,7 @@ func TestSentinelServer_Monitor(t *testing.T) { args: args{ ctx: context.TODO(), shards: testShardedCluster, + quorum: 2, }, want: []string{}, wantErr: true, @@ -516,6 +522,7 @@ func TestSentinelServer_Monitor(t *testing.T) { args: args{ ctx: context.TODO(), shards: testShardedCluster, + quorum: 2, }, want: []string{}, wantErr: true, @@ -542,6 +549,7 @@ func TestSentinelServer_Monitor(t *testing.T) { args: args{ ctx: context.TODO(), shards: testShardedCluster, + quorum: 2, }, want: []string{"shard00"}, wantErr: true, @@ -581,6 +589,7 @@ func TestSentinelServer_Monitor(t *testing.T) { }, }, }, + quorum: 2, }, want: []string{}, wantErr: true, @@ -588,7 +597,7 @@ func TestSentinelServer_Monitor(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := tt.ss.Monitor(tt.args.ctx, tt.args.shards) + got, err := tt.ss.Monitor(tt.args.ctx, tt.args.shards, tt.args.quorum) if (err != nil) != tt.wantErr { t.Errorf("SentinelServer.Monitor() error = %v, wantErr %v", err, tt.wantErr) return From 71023d133ed7b64db0cf7cfc51c40ca4ad41bf78 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Wed, 9 Aug 2023 15:38:49 +0200 Subject: [PATCH 02/43] Performs a BGSave and updates status accordingly --- PROJECT | 9 + api/v1alpha1/sentinel_types.go | 2 +- api/v1alpha1/shardedredisbackup_types.go | 166 ++++++++++++ api/v1alpha1/shardedredisbackup_types_test.go | 187 ++++++++++++++ api/v1alpha1/twemproxyconfig_types.go | 1 + api/v1alpha1/zz_generated.deepcopy.go | 152 +++++++++++ .../saas.3scale.net_shardedredisbackups.yaml | 87 +++++++ .../saas.3scale.net_twemproxyconfigs.yaml | 3 +- config/crd/kustomization.yaml | 3 + .../cainjection_in_shardedredisbackups.yaml | 7 + .../webhook_in_shardedredisbackups.yaml | 16 ++ config/rbac/role.yaml | 26 ++ .../rbac/shardedredisbackup_editor_role.yaml | 31 +++ .../rbac/shardedredisbackup_viewer_role.yaml | 27 ++ config/samples/kustomization.yaml | 1 + .../saas_v1alpha1_shardedredisbackup.yaml | 12 + config/staging/kustomization.yaml | 25 ++ controllers/shardedredisbackup_controller.go | 241 ++++++++++++++++++ .../shardedredisbackup_controller_test.go | 209 +++++++++++++++ go.mod | 1 + go.sum | 2 + main.go | 25 +- pkg/reconcilers/threads/manager.go | 65 +++-- pkg/reconcilers/threads/manager_test.go | 9 +- pkg/redis/backup/backup.go | 50 ++-- pkg/redis/backup/errors.go | 11 + pkg/redis/backup/manager.go | 116 ++++++--- pkg/redis/events/watcher.go | 4 + pkg/redis/metrics/sentinel_metrics.go | 4 + pkg/redis/sharded/redis_sharded_cluster.go | 15 +- .../sharded/redis_sharded_cluster_test.go | 6 +- pkg/redis/sharded/sentinel_server.go | 2 +- pkg/util/slice.go | 18 ++ pkg/util/time.go | 11 + 34 files changed, 1447 insertions(+), 97 deletions(-) create mode 100644 api/v1alpha1/shardedredisbackup_types.go create mode 100644 api/v1alpha1/shardedredisbackup_types_test.go create mode 100644 config/crd/bases/saas.3scale.net_shardedredisbackups.yaml create mode 100644 config/crd/patches/cainjection_in_shardedredisbackups.yaml create mode 100644 config/crd/patches/webhook_in_shardedredisbackups.yaml create mode 100644 config/rbac/shardedredisbackup_editor_role.yaml create mode 100644 config/rbac/shardedredisbackup_viewer_role.yaml create mode 100644 config/samples/saas_v1alpha1_shardedredisbackup.yaml create mode 100644 config/staging/kustomization.yaml create mode 100644 controllers/shardedredisbackup_controller.go create mode 100644 controllers/shardedredisbackup_controller_test.go create mode 100644 pkg/redis/backup/errors.go create mode 100644 pkg/util/time.go diff --git a/PROJECT b/PROJECT index c1480d3b..cc568257 100644 --- a/PROJECT +++ b/PROJECT @@ -106,4 +106,13 @@ resources: kind: TwemproxyConfig path: github.com/3scale/saas-operator/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: 3scale.net + group: saas + kind: ShardedRedisBackup + path: github.com/3scale/saas-operator/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/api/v1alpha1/sentinel_types.go b/api/v1alpha1/sentinel_types.go index 7408f6ec..ebf6dcf8 100644 --- a/api/v1alpha1/sentinel_types.go +++ b/api/v1alpha1/sentinel_types.go @@ -189,7 +189,7 @@ func (ss *SentinelStatus) ShardedCluster(ctx context.Context, pool *redis.Server // TODO: at some point change the SentinelStatus.Sentinels to also have a map and avoid this msentinel := make(map[string]string, len(ss.Sentinels)) for _, s := range ss.Sentinels { - msentinel[s] = s + msentinel[s] = "redis://" + s } shards := make([]*sharded.Shard, 0, len(ss.MonitoredShards)) diff --git a/api/v1alpha1/shardedredisbackup_types.go b/api/v1alpha1/shardedredisbackup_types.go new file mode 100644 index 00000000..1f350d0b --- /dev/null +++ b/api/v1alpha1/shardedredisbackup_types.go @@ -0,0 +1,166 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "fmt" + "reflect" + "sort" + + "github.com/3scale/saas-operator/pkg/util" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + backupHistoryLimit int32 = 10 +) + +// ShardedRedisBackupSpec defines the desired state of ShardedRedisBackup +type ShardedRedisBackupSpec struct { + SentinelRef string `json:"sentinelRef"` + Schedule string `json:"schedule"` + Timeout string `json:"timeout"` + //+optional + HistoryLimit *int32 `json:"historyLimit,omitempty"` +} + +// Default implements defaulting for ShardedRedisBackuppec +func (spec *ShardedRedisBackupSpec) Default() { + + spec.HistoryLimit = intOrDefault(spec.HistoryLimit, &backupHistoryLimit) +} + +// ShardedRedisBackupStatus defines the observed state of ShardedRedisBackup +type ShardedRedisBackupStatus struct { + //+optional + Backups BackupStatusList `json:"backups,omitempty"` +} + +func (status *ShardedRedisBackupStatus) AddBackup(b BackupStatus) { + status.Backups = append(status.Backups, b) + sort.Sort(sort.Reverse(status.Backups)) +} + +func (status *ShardedRedisBackupStatus) FindLastBackup(shardName string, state BackupState) *BackupStatus { + // backups expected to be ordered from newer to oldest + for i, b := range status.Backups { + if b.Shard == shardName && b.State == state { + return &status.Backups[i] + } + } + return nil +} + +func (status *ShardedRedisBackupStatus) GetRunningBackups() []*BackupStatus { + list := []*BackupStatus{} + for i, b := range status.Backups { + if b.State == BackupRunningState { + list = append(list, &status.Backups[i]) + } + } + return list +} + +func (status *ShardedRedisBackupStatus) ApplyHistoryLimit(limit int32, shards []string) bool { + truncated := make([][]BackupStatus, len(shards)) + for idx, shard := range shards { + var count int32 = 0 + truncated[idx] = make([]BackupStatus, 0, limit) + for _, bs := range status.Backups { + if count == limit { + break + } + if bs.Shard == shard { + truncated[idx] = append(truncated[idx], bs) + count++ + } + } + } + + var joint BackupStatusList = util.ConcatSlices(truncated) + sort.Sort(sort.Reverse(joint)) + + if !reflect.DeepEqual(joint, status.Backups) { + status.Backups = joint + return true + } + + return false +} + +type BackupStatusList []BackupStatus + +func (bsl BackupStatusList) Len() int { return len(bsl) } +func (bsl BackupStatusList) Less(i, j int) bool { + a := fmt.Sprintf("%d-%s", bsl[i].ScheduledFor.UTC().UnixMilli(), bsl[i].Shard) + b := fmt.Sprintf("%d-%s", bsl[j].ScheduledFor.UTC().UnixMilli(), bsl[j].Shard) + return a < b +} +func (bsl BackupStatusList) Swap(i, j int) { bsl[i], bsl[j] = bsl[j], bsl[i] } + +type BackupState string + +type BackupStatus struct { + Shard string `json:"shard"` + //+optional + ServerAlias *string `json:"serverAlias,omitempty"` + //+optional + ServerID *string `json:"serverID,omitempty"` + ScheduledFor metav1.Time `json:"scheduledFor"` + //+optional + StartedAt *metav1.Time `json:"startedAt,omitempty"` + Message string `json:"message"` + State BackupState `json:"state"` +} + +const ( + BackupPendingState BackupState = "Pending" + BackupRunningState BackupState = "Running" + BackupCompletedState BackupState = "Completed" + BackupFailedState BackupState = "Failed" + BackupUnknownState BackupState = "Unknown" +) + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// ShardedRedisBackup is the Schema for the shardedredisbackups API +type ShardedRedisBackup struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ShardedRedisBackupSpec `json:"spec,omitempty"` + Status ShardedRedisBackupStatus `json:"status,omitempty"` +} + +// Default implements defaulting for the Sentinel resource +func (srb *ShardedRedisBackup) Default() { + srb.Spec.Default() +} + +//+kubebuilder:object:root=true + +// ShardedRedisBackupList contains a list of ShardedRedisBackup +type ShardedRedisBackupList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ShardedRedisBackup `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ShardedRedisBackup{}, &ShardedRedisBackupList{}) +} diff --git a/api/v1alpha1/shardedredisbackup_types_test.go b/api/v1alpha1/shardedredisbackup_types_test.go new file mode 100644 index 00000000..ee6c2eb8 --- /dev/null +++ b/api/v1alpha1/shardedredisbackup_types_test.go @@ -0,0 +1,187 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "reflect" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestBackupStatusList_Less(t *testing.T) { + type args struct { + i int + j int + } + tests := []struct { + name string + bsl BackupStatusList + args args + want bool + }{ + { + name: "0 < 1", + bsl: []BackupStatus{ + { + Shard: "shard01", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 0, 0, 0, time.UTC), + }, + { + Shard: "shard02", + ScheduledFor: metav1.Date(2023, time.August, 2, 0, 0, 0, 0, time.UTC), + }, + }, + args: args{i: 0, j: 1}, + want: true, + }, + { + name: "0 < 1 (same timestamp)", + bsl: []BackupStatus{ + { + Shard: "shard01", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 0, 0, 0, time.UTC), + }, + { + Shard: "shard02", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 0, 0, 0, time.UTC), + }, + }, + args: args{i: 0, j: 1}, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.bsl.Less(tt.args.i, tt.args.j); got != tt.want { + t.Errorf("BackupStatusList.Less() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestShardedRedisBackupStatus_FindBackup(t *testing.T) { + type fields struct { + Backups BackupStatusList + } + type args struct { + shard string + state BackupState + } + tests := []struct { + name string + fields fields + args args + want *BackupStatus + }{ + { + name: "Returns latest pending backup", + fields: fields{ + Backups: []BackupStatus{ + { + Shard: "shard02", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 1, 0, 0, time.UTC), + State: BackupPendingState, + }, + { + Shard: "shard01", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 1, 0, 0, time.UTC), + State: BackupPendingState, + }, + { + Shard: "shard02", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 0, 0, 0, time.UTC), + State: BackupRunningState, + }, + { + Shard: "shard01", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 0, 0, 0, time.UTC), + State: BackupRunningState, + }, + }, + }, + args: args{shard: "shard02", state: BackupPendingState}, + want: &BackupStatus{ + Shard: "shard02", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 1, 0, 0, time.UTC), + State: BackupPendingState, + }, + }, + { + name: "Returns latest running backup", + fields: fields{ + Backups: []BackupStatus{ + { + Shard: "shard02", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 1, 0, 0, time.UTC), + State: BackupPendingState, + }, + { + Shard: "shard01", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 1, 0, 0, time.UTC), + State: BackupPendingState, + }, + { + Shard: "shard02", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 0, 0, 0, time.UTC), + State: BackupRunningState, + }, + { + Shard: "shard01", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 0, 0, 0, time.UTC), + State: BackupRunningState, + }, + }, + }, + args: args{shard: "shard02", state: BackupRunningState}, + want: &BackupStatus{ + Shard: "shard02", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 0, 0, 0, time.UTC), + State: BackupRunningState, + }, + }, + { + name: "Returns a nil", + fields: fields{ + Backups: []BackupStatus{ + { + Shard: "shard02", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 1, 0, 0, time.UTC), + State: BackupPendingState, + }, + }, + }, + args: args{shard: "shard02", state: BackupPendingState}, + want: &BackupStatus{ + Shard: "shard02", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 1, 0, 0, time.UTC), + State: BackupPendingState, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + status := &ShardedRedisBackupStatus{ + Backups: tt.fields.Backups, + } + if got := status.FindLastBackup(tt.args.shard, tt.args.state); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ShardedRedisBackupStatus.FindBackup() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/api/v1alpha1/twemproxyconfig_types.go b/api/v1alpha1/twemproxyconfig_types.go index 7fe6cfa6..a3ac8e6b 100644 --- a/api/v1alpha1/twemproxyconfig_types.go +++ b/api/v1alpha1/twemproxyconfig_types.go @@ -39,6 +39,7 @@ var ( type TwemproxyConfigSpec struct { // SentinelURI is the redis URI of sentinel. If not set, the controller // will try to autodiscover Sentinel within the namespace. + // TODO: remove, unused // +operator-sdk:csv:customresourcedefinitions:type=spec // +optional SentinelURIs []string `json:"sentinelURIs,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 56b34884..23d2668c 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -684,6 +684,57 @@ func (in *BackendStatus) DeepCopy() *BackendStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupStatus) DeepCopyInto(out *BackupStatus) { + *out = *in + if in.ServerAlias != nil { + in, out := &in.ServerAlias, &out.ServerAlias + *out = new(string) + **out = **in + } + if in.ServerID != nil { + in, out := &in.ServerID, &out.ServerID + *out = new(string) + **out = **in + } + in.ScheduledFor.DeepCopyInto(&out.ScheduledFor) + if in.StartedAt != nil { + in, out := &in.StartedAt, &out.StartedAt + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStatus. +func (in *BackupStatus) DeepCopy() *BackupStatus { + if in == nil { + return nil + } + out := new(BackupStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in BackupStatusList) DeepCopyInto(out *BackupStatusList) { + { + in := &in + *out = make(BackupStatusList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStatusList. +func (in BackupStatusList) DeepCopy() BackupStatusList { + if in == nil { + return nil + } + out := new(BackupStatusList) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BugsnagSpec) DeepCopyInto(out *BugsnagSpec) { *out = *in @@ -2677,6 +2728,107 @@ func (in *SentinelStatus) DeepCopy() *SentinelStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ShardedRedisBackup) DeepCopyInto(out *ShardedRedisBackup) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShardedRedisBackup. +func (in *ShardedRedisBackup) DeepCopy() *ShardedRedisBackup { + if in == nil { + return nil + } + out := new(ShardedRedisBackup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ShardedRedisBackup) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ShardedRedisBackupList) DeepCopyInto(out *ShardedRedisBackupList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ShardedRedisBackup, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShardedRedisBackupList. +func (in *ShardedRedisBackupList) DeepCopy() *ShardedRedisBackupList { + if in == nil { + return nil + } + out := new(ShardedRedisBackupList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ShardedRedisBackupList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ShardedRedisBackupSpec) DeepCopyInto(out *ShardedRedisBackupSpec) { + *out = *in + if in.HistoryLimit != nil { + in, out := &in.HistoryLimit, &out.HistoryLimit + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShardedRedisBackupSpec. +func (in *ShardedRedisBackupSpec) DeepCopy() *ShardedRedisBackupSpec { + if in == nil { + return nil + } + out := new(ShardedRedisBackupSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ShardedRedisBackupStatus) DeepCopyInto(out *ShardedRedisBackupStatus) { + *out = *in + if in.Backups != nil { + in, out := &in.Backups, &out.Backups + *out = make(BackupStatusList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShardedRedisBackupStatus. +func (in *ShardedRedisBackupStatus) DeepCopy() *ShardedRedisBackupStatus { + if in == nil { + return nil + } + out := new(ShardedRedisBackupStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ShardedRedisTopology) DeepCopyInto(out *ShardedRedisTopology) { *out = *in diff --git a/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml b/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml new file mode 100644 index 00000000..6a8e38e4 --- /dev/null +++ b/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml @@ -0,0 +1,87 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.10.0 + creationTimestamp: null + name: shardedredisbackups.saas.3scale.net +spec: + group: saas.3scale.net + names: + kind: ShardedRedisBackup + listKind: ShardedRedisBackupList + plural: shardedredisbackups + singular: shardedredisbackup + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ShardedRedisBackup is the Schema for the shardedredisbackups + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ShardedRedisBackupSpec defines the desired state of ShardedRedisBackup + properties: + historyLimit: + format: int32 + type: integer + schedule: + type: string + sentinelRef: + type: string + timeout: + type: string + required: + - schedule + - sentinelRef + - timeout + type: object + status: + description: ShardedRedisBackupStatus defines the observed state of ShardedRedisBackup + properties: + backups: + items: + properties: + message: + type: string + scheduledFor: + format: date-time + type: string + serverAlias: + type: string + serverID: + type: string + shard: + type: string + startedAt: + format: date-time + type: string + state: + type: string + required: + - message + - scheduledFor + - shard + - state + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/saas.3scale.net_twemproxyconfigs.yaml b/config/crd/bases/saas.3scale.net_twemproxyconfigs.yaml index b6a3b8d2..d7728df1 100644 --- a/config/crd/bases/saas.3scale.net_twemproxyconfigs.yaml +++ b/config/crd/bases/saas.3scale.net_twemproxyconfigs.yaml @@ -59,8 +59,9 @@ spec: even if they are manually changed. This switch defaults to "true". type: boolean sentinelURIs: - description: SentinelURI is the redis URI of sentinel. If not set, + description: 'SentinelURI is the redis URI of sentinel. If not set, the controller will try to autodiscover Sentinel within the namespace. + TODO: remove, unused' items: type: string type: array diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 84f248d6..fea632d1 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -13,6 +13,7 @@ resources: - bases/saas.3scale.net_sentinels.yaml - bases/saas.3scale.net_redisshards.yaml - bases/saas.3scale.net_twemproxyconfigs.yaml +- bases/saas.3scale.net_shardedredisbackups.yaml # +kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -29,6 +30,7 @@ patchesStrategicMerge: #- patches/webhook_in_sentinels.yaml #- patches/webhook_in_redisshards.yaml #- patches/webhook_in_twemproxyconfigs.yaml +#- patches/webhook_in_shardedredisbackups.yaml # +kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. @@ -44,6 +46,7 @@ patchesStrategicMerge: #- patches/cainjection_in_sentinels.yaml #- patches/cainjection_in_redisshards.yaml #- patches/cainjection_in_twemproxyconfigs.yaml +#- patches/cainjection_in_shardedredisbackups.yaml # +kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_shardedredisbackups.yaml b/config/crd/patches/cainjection_in_shardedredisbackups.yaml new file mode 100644 index 00000000..11e4730b --- /dev/null +++ b/config/crd/patches/cainjection_in_shardedredisbackups.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: shardedredisbackups.saas.3scale.net diff --git a/config/crd/patches/webhook_in_shardedredisbackups.yaml b/config/crd/patches/webhook_in_shardedredisbackups.yaml new file mode 100644 index 00000000..514d2ffd --- /dev/null +++ b/config/crd/patches/webhook_in_shardedredisbackups.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: shardedredisbackups.saas.3scale.net +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c4f4eb22..ed95dc77 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -351,6 +351,32 @@ rules: - get - patch - update +- apiGroups: + - saas.3scale.net + resources: + - shardedredisbackups + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - saas.3scale.net + resources: + - shardedredisbackups/finalizers + verbs: + - update +- apiGroups: + - saas.3scale.net + resources: + - shardedredisbackups/status + verbs: + - get + - patch + - update - apiGroups: - saas.3scale.net resources: diff --git a/config/rbac/shardedredisbackup_editor_role.yaml b/config/rbac/shardedredisbackup_editor_role.yaml new file mode 100644 index 00000000..10394217 --- /dev/null +++ b/config/rbac/shardedredisbackup_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit shardedredisbackups. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: shardedredisbackup-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: saas-operator + app.kubernetes.io/part-of: saas-operator + app.kubernetes.io/managed-by: kustomize + name: shardedredisbackup-editor-role +rules: +- apiGroups: + - saas.3scale.net + resources: + - shardedredisbackups + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - saas.3scale.net + resources: + - shardedredisbackups/status + verbs: + - get diff --git a/config/rbac/shardedredisbackup_viewer_role.yaml b/config/rbac/shardedredisbackup_viewer_role.yaml new file mode 100644 index 00000000..fbc76bb4 --- /dev/null +++ b/config/rbac/shardedredisbackup_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view shardedredisbackups. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: shardedredisbackup-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: saas-operator + app.kubernetes.io/part-of: saas-operator + app.kubernetes.io/managed-by: kustomize + name: shardedredisbackup-viewer-role +rules: +- apiGroups: + - saas.3scale.net + resources: + - shardedredisbackups + verbs: + - get + - list + - watch +- apiGroups: + - saas.3scale.net + resources: + - shardedredisbackups/status + verbs: + - get diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index ad7dad7c..2eee98f5 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -11,4 +11,5 @@ resources: - saas_v1alpha1_sentinel.yaml - saas_v1alpha1_redisshard.yaml - saas_v1alpha1_twemproxyconfig.yaml +- saas_v1alpha1_shardedredisbackup.yaml # +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/saas_v1alpha1_shardedredisbackup.yaml b/config/samples/saas_v1alpha1_shardedredisbackup.yaml new file mode 100644 index 00000000..59fa3d24 --- /dev/null +++ b/config/samples/saas_v1alpha1_shardedredisbackup.yaml @@ -0,0 +1,12 @@ +apiVersion: saas.3scale.net/v1alpha1 +kind: ShardedRedisBackup +metadata: + labels: + app.kubernetes.io/name: shardedredisbackup + app.kubernetes.io/instance: shardedredisbackup-sample + app.kubernetes.io/part-of: saas-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: saas-operator + name: shardedredisbackup-sample +spec: + # TODO(user): Add fields here diff --git a/config/staging/kustomization.yaml b/config/staging/kustomization.yaml new file mode 100644 index 00000000..44036021 --- /dev/null +++ b/config/staging/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: 3scale-saas +resources: +- ../default + +patches: + - target: + group: apps + version: v1 + kind: Deployment + name: controller-manager + patch: |- + - op: replace + path: /spec/template/spec/containers/0/env/0 + value: { "name": "WATCH_NAMESPACE", "value": "3scale-saas" } + - op: add + path: /spec/template/spec/containers/0/env/1 + value: { "name": "LOG_LEVEL", "value": "debug" } + - op: add + path: /spec/template/spec/containers/0/env/1 + value: { "name": "LOG_MODE", "value": "dev" } + - op: replace + path: "/spec/template/spec/containers/0/imagePullPolicy" + value: "Always" diff --git a/controllers/shardedredisbackup_controller.go b/controllers/shardedredisbackup_controller.go new file mode 100644 index 00000000..5f08d10a --- /dev/null +++ b/controllers/shardedredisbackup_controller.go @@ -0,0 +1,241 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + basereconciler "github.com/3scale-ops/basereconciler/reconciler" + saasv1alpha1 "github.com/3scale/saas-operator/api/v1alpha1" + "github.com/3scale/saas-operator/pkg/reconcilers/threads" + "github.com/3scale/saas-operator/pkg/redis/backup" + redis "github.com/3scale/saas-operator/pkg/redis/server" + "github.com/3scale/saas-operator/pkg/redis/sharded" + "github.com/3scale/saas-operator/pkg/util" + "github.com/go-logr/logr" + "github.com/robfig/cron/v3" +) + +// ShardedRedisBackupReconciler reconciles a ShardedRedisBackup object +type ShardedRedisBackupReconciler struct { + basereconciler.Reconciler + Log logr.Logger + BackupRunner threads.Manager + Pool *redis.ServerPool +} + +//+kubebuilder:rbac:groups=saas.3scale.net,namespace=placeholder,resources=shardedredisbackups,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=saas.3scale.net,namespace=placeholder,resources=shardedredisbackups/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=saas.3scale.net,namespace=placeholder,resources=shardedredisbackups/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the ShardedRedisBackup object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.0/pkg/reconcile +func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := r.Log.WithValues("name", req.Name, "namespace", req.Namespace) + ctx = log.IntoContext(ctx, logger) + now := time.Now() + + // ---------------------------------- + // ----- Phase 1: get instances ----- + // ---------------------------------- + + instance := &saasv1alpha1.ShardedRedisBackup{} + key := types.NamespacedName{Name: req.Name, Namespace: req.Namespace} + err := r.GetInstance(ctx, key, instance, pointer.String(saasv1alpha1.Finalizer), []func(){r.BackupRunner.CleanupThreads(instance)}) + if err != nil { + if apierrors.IsNotFound(err) { + // Return and don't requeue + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + + instance.Default() + + // Get Sentinel status + sentinel := &saasv1alpha1.Sentinel{ObjectMeta: metav1.ObjectMeta{Name: instance.Spec.SentinelRef, Namespace: req.Namespace}} + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(sentinel), sentinel); err != nil { + return ctrl.Result{}, err + } + + cluster, err := sentinel.Status.ShardedCluster(ctx, r.Pool) + if err != nil { + return ctrl.Result{}, err + } + + // ---------------------------------------- + // ----- Phase 2: run pending backups ----- + // ---------------------------------------- + + statusChanged := false + runners := make([]threads.RunnableThread, 0, len(cluster.Shards)) + for _, shard := range cluster.Shards { + scheduledBackup := instance.Status.FindLastBackup(shard.Name, saasv1alpha1.BackupPendingState) + if scheduledBackup != nil { + if scheduledBackup.State == saasv1alpha1.BackupPendingState && scheduledBackup.ScheduledFor.Time.Before(time.Now()) { + // add the backup runner thread + target := shard.GetSlavesRO()[0] + runners = append(runners, backup.NewBackupRunner(shard.Name, target, scheduledBackup.ScheduledFor.Time, 10*time.Minute, instance)) + scheduledBackup.ServerAlias = util.Pointer(target.GetAlias()) + scheduledBackup.ServerID = util.Pointer(target.ID()) + now := metav1.Now() + scheduledBackup.StartedAt = util.Pointer(now) + scheduledBackup.Message = "backup is running" + scheduledBackup.State = saasv1alpha1.BackupRunningState + statusChanged = true + } + } + } + + if err := r.BackupRunner.ReconcileThreads(ctx, instance, runners, logger.WithName("backup-runner")); err != nil { + return ctrl.Result{}, err + } + + if statusChanged { + logger.V(1).Info("backup runners started") + err := r.Client.Status().Update(ctx, instance) + return ctrl.Result{}, err + } + + // -------------------------------------------------------- + // ----- Phase 3: reconcile status of running backups ----- + // -------------------------------------------------------- + for _, b := range instance.Status.GetRunningBackups() { + var thread *backup.Runner + var srv *sharded.RedisServer + + if srv = cluster.LookupServerByID(*b.ServerID); srv == nil { + b.State = saasv1alpha1.BackupUnknownState + b.Message = "server not found in cluster" + statusChanged = true + continue + } + + if t := r.BackupRunner.GetThread(backup.ID(b.Shard, srv.GetAlias(), b.ScheduledFor.Time), instance, logger); t != nil { + thread = t.(*backup.Runner) + } else { + b.State = saasv1alpha1.BackupUnknownState + b.Message = "runner not found" + statusChanged = true + continue + } + + if thread.Status().Finished { + if err := thread.Status().Error; err != nil { + b.State = saasv1alpha1.BackupFailedState + b.Message = err.Error() + } else { + b.State = saasv1alpha1.BackupCompletedState + b.Message = "backup complete" + } + statusChanged = true + } + } + + if statusChanged { + logger.V(1).Info("backup schedule updated") + err := r.Client.Status().Update(ctx, instance) + return ctrl.Result{}, err + } + + // ------------------------------------- + // ----- Phase 4: schedule backups ----- + // ------------------------------------- + + schedule, err := cron.ParseStandard(instance.Spec.Schedule) + if err != nil { + return ctrl.Result{}, err + } + nextRun := schedule.Next(now) + + statusChanged, err = r.reconcileBackupList(ctx, instance, nextRun, cluster.GetShardNames()) + if err != nil { + return reconcile.Result{}, err + } + + if statusChanged { + logger.V(1).Info("backup schedule updated") + err := r.Client.Status().Update(ctx, instance) + return ctrl.Result{}, err + } + + // requeue for next schedule + return ctrl.Result{RequeueAfter: time.Until(nextRun.Add(1 * time.Second))}, nil +} + +func (r *ShardedRedisBackupReconciler) reconcileBackupList(ctx context.Context, instance *saasv1alpha1.ShardedRedisBackup, nextRun time.Time, shards []string) (bool, error) { + logger := log.FromContext(ctx, "function", "(r *ShardedRedisBackupReconciler) reconcileBackupList") + changed := false + + for _, shard := range shards { + // don't schedule if a backup is already running + if runningbackup := instance.Status.FindLastBackup(shard, saasv1alpha1.BackupRunningState); runningbackup != nil { + continue + } + if lastbackup := instance.Status.FindLastBackup(shard, saasv1alpha1.BackupPendingState); lastbackup != nil { + // found a pending backup for this shard + if nextRun == lastbackup.ScheduledFor.Time { + // already scheduled, do nothing + continue + } + } + + // add a new backup to the list + instance.Status.AddBackup(saasv1alpha1.BackupStatus{ + Shard: shard, + ScheduledFor: metav1.NewTime(nextRun), + Message: "backup scheduled", + State: saasv1alpha1.BackupPendingState, + }) + logger.V(1).Info("scheduled backup", "shard", shard, "scheduledFor", nextRun) + changed = true + } + + // apply historyLimit + if instance.Status.ApplyHistoryLimit(*instance.Spec.HistoryLimit, shards) { + changed = true + } + + return changed, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *ShardedRedisBackupReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&saasv1alpha1.ShardedRedisBackup{}). + Watches(&source.Channel{Source: r.BackupRunner.GetChannel()}, &handler.EnqueueRequestForObject{}). + Complete(r) +} diff --git a/controllers/shardedredisbackup_controller_test.go b/controllers/shardedredisbackup_controller_test.go new file mode 100644 index 00000000..236c83ca --- /dev/null +++ b/controllers/shardedredisbackup_controller_test.go @@ -0,0 +1,209 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "testing" + "time" + + basereconciler "github.com/3scale-ops/basereconciler/reconciler" + saasv1alpha1 "github.com/3scale/saas-operator/api/v1alpha1" + "github.com/3scale/saas-operator/pkg/util" + "github.com/go-logr/logr" + "github.com/google/go-cmp/cmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestShardedRedisBackupReconciler_reconcileBackupList(t *testing.T) { + type fields struct { + Reconciler basereconciler.Reconciler + Log logr.Logger + } + type args struct { + instance *saasv1alpha1.ShardedRedisBackup + nextRun time.Time + shards []string + } + tests := []struct { + name string + fields fields + args args + wantChanged bool + wantStatus saasv1alpha1.ShardedRedisBackupStatus + wantErr bool + }{ + { + name: "List is empty, adds a backup", + fields: fields{ + Reconciler: basereconciler.Reconciler{}, + Log: logr.Discard(), + }, + args: args{ + nextRun: util.MustParseRFC3339("2023-09-01T00:01:00Z"), + instance: &saasv1alpha1.ShardedRedisBackup{ + Spec: saasv1alpha1.ShardedRedisBackupSpec{HistoryLimit: util.Pointer(int32(10))}, + Status: saasv1alpha1.ShardedRedisBackupStatus{}, + }, + shards: []string{"shard01", "shard02"}, + }, + wantChanged: true, + wantStatus: saasv1alpha1.ShardedRedisBackupStatus{ + Backups: []saasv1alpha1.BackupStatus{ + { + Shard: "shard02", + ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), + Message: "Scheduled", + State: saasv1alpha1.BackupPendingState, + }, + { + Shard: "shard01", + ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), + Message: "Scheduled", + State: saasv1alpha1.BackupPendingState, + }, + }, + }, + wantErr: false, + }, + { + name: "No changes", + fields: fields{ + Reconciler: basereconciler.Reconciler{}, + Log: logr.Discard(), + }, + args: args{ + nextRun: util.MustParseRFC3339("2023-09-01T00:01:00Z"), + instance: &saasv1alpha1.ShardedRedisBackup{ + Spec: saasv1alpha1.ShardedRedisBackupSpec{HistoryLimit: util.Pointer(int32(10))}, + Status: saasv1alpha1.ShardedRedisBackupStatus{ + Backups: []saasv1alpha1.BackupStatus{ + { + Shard: "shard02", + ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), + Message: "Scheduled", + State: saasv1alpha1.BackupPendingState, + }, + { + Shard: "shard01", + ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), + Message: "Scheduled", + State: saasv1alpha1.BackupPendingState, + }, + }}, + }, + shards: []string{"shard01", "shard02"}, + }, + wantChanged: false, + wantStatus: saasv1alpha1.ShardedRedisBackupStatus{ + Backups: []saasv1alpha1.BackupStatus{ + { + Shard: "shard02", + ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), + Message: "Scheduled", + State: saasv1alpha1.BackupPendingState, + }, + { + Shard: "shard01", + ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), + Message: "Scheduled", + State: saasv1alpha1.BackupPendingState, + }, + }, + }, + wantErr: false, + }, + { + name: "Adds new backups", + fields: fields{ + Reconciler: basereconciler.Reconciler{}, + Log: logr.Discard(), + }, + args: args{ + nextRun: util.MustParseRFC3339("2023-09-01T00:02:00Z"), + instance: &saasv1alpha1.ShardedRedisBackup{ + Spec: saasv1alpha1.ShardedRedisBackupSpec{HistoryLimit: util.Pointer(int32(10))}, + Status: saasv1alpha1.ShardedRedisBackupStatus{ + Backups: []saasv1alpha1.BackupStatus{ + { + Shard: "shard02", + ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), + Message: "Scheduled", + State: saasv1alpha1.BackupPendingState, + }, + { + Shard: "shard01", + ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), + Message: "Scheduled", + State: saasv1alpha1.BackupPendingState, + }, + }}, + }, + shards: []string{"shard01", "shard02"}, + }, + wantChanged: true, + wantStatus: saasv1alpha1.ShardedRedisBackupStatus{ + Backups: []saasv1alpha1.BackupStatus{ + { + Shard: "shard02", + ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:02:00Z")), + Message: "Scheduled", + State: saasv1alpha1.BackupPendingState, + }, + { + Shard: "shard01", + ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:02:00Z")), + Message: "Scheduled", + State: saasv1alpha1.BackupPendingState, + }, + { + Shard: "shard02", + ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), + Message: "Scheduled", + State: saasv1alpha1.BackupPendingState, + }, + { + Shard: "shard01", + ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), + Message: "Scheduled", + State: saasv1alpha1.BackupPendingState, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &ShardedRedisBackupReconciler{ + Reconciler: tt.fields.Reconciler, + Log: tt.fields.Log, + } + got, err := r.reconcileBackupList(context.TODO(), tt.args.instance, tt.args.nextRun, tt.args.shards) + if (err != nil) != tt.wantErr { + t.Errorf("ShardedRedisBackupReconciler.reconcileBackupList() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(tt.args.instance.Status, tt.wantStatus); len(diff) > 0 { + t.Errorf("ShardedRedisBackupReconciler.reconcileBackupList() = diff %v", diff) + } + if got != tt.wantChanged { + t.Errorf("ShardedRedisBackupReconciler.reconcileBackupList() = %v, want %v", got, tt.wantChanged) + } + }) + } +} diff --git a/go.mod b/go.mod index 80693a5b..9fa268f6 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.42.1 github.com/prometheus/client_golang v1.14.0 + github.com/robfig/cron/v3 v3.0.1 github.com/tektoncd/pipeline v0.49.0 go.uber.org/zap v1.24.0 golang.org/x/time v0.3.0 diff --git a/go.sum b/go.sum index 55c114e9..cf547e0f 100644 --- a/go.sum +++ b/go.sum @@ -425,6 +425,8 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/statsd_exporter v0.21.0 h1:hA05Q5RFeIjgwKIYEdFd59xu5Wwaznf33yKI+pyX6T8= github.com/prometheus/statsd_exporter v0.21.0/go.mod h1:rbT83sZq2V+p73lHhPZfMc3MLCHmSHelCh9hSGYNLTQ= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= diff --git a/main.go b/main.go index 8721a1d8..8673084a 100644 --- a/main.go +++ b/main.go @@ -30,13 +30,6 @@ import ( basereconciler "github.com/3scale-ops/basereconciler/reconciler" marin3rv1alpha1 "github.com/3scale-ops/marin3r/apis/marin3r/v1alpha1" - saasv1alpha1 "github.com/3scale/saas-operator/api/v1alpha1" - "github.com/3scale/saas-operator/controllers" - "github.com/3scale/saas-operator/pkg/reconcilers/threads" - "github.com/3scale/saas-operator/pkg/reconcilers/workloads" - redis "github.com/3scale/saas-operator/pkg/redis/server" - "github.com/3scale/saas-operator/pkg/util" - "github.com/3scale/saas-operator/pkg/version" externalsecretsv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1" grafanav1alpha1 "github.com/grafana-operator/grafana-operator/v4/api/integreatly/v1alpha1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -47,6 +40,14 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/healthz" + + saasv1alpha1 "github.com/3scale/saas-operator/api/v1alpha1" + "github.com/3scale/saas-operator/controllers" + "github.com/3scale/saas-operator/pkg/reconcilers/threads" + "github.com/3scale/saas-operator/pkg/reconcilers/workloads" + redis "github.com/3scale/saas-operator/pkg/redis/server" + "github.com/3scale/saas-operator/pkg/util" + "github.com/3scale/saas-operator/pkg/version" // +kubebuilder:scaffold:imports ) @@ -158,6 +159,16 @@ func main() { os.Exit(1) } + if err = (&controllers.ShardedRedisBackupReconciler{ + Reconciler: basereconciler.NewFromManager(mgr), + BackupRunner: threads.NewManager(), + Log: ctrl.Log.WithName("controllers").WithName("ShardedRedisBackup"), + Pool: redisPool, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "ShardedRedisBackup") + os.Exit(1) + } + /* WORKLOADS RECONCILER BASED CONTROLLERS*/ if err = (&controllers.ApicastReconciler{ diff --git a/pkg/reconcilers/threads/manager.go b/pkg/reconcilers/threads/manager.go index effdca3f..ab20953d 100644 --- a/pkg/reconcilers/threads/manager.go +++ b/pkg/reconcilers/threads/manager.go @@ -17,6 +17,7 @@ type RunnableThread interface { Start(context.Context, logr.Logger) error Stop() IsStarted() bool + CanBeDeleted() bool } // Manager is a struct that holds configuration to @@ -35,23 +36,29 @@ func NewManager() Manager { } } -// RunThread runs thread and associates it with a given key so it can later be stopped -func (mgr *Manager) RunThread(ctx context.Context, key string, thread RunnableThread, log logr.Logger) error { +// runThread runs thread and associates it with a given key so it can later be stopped +func (mgr *Manager) runThread(ctx context.Context, key string, thread RunnableThread, log logr.Logger) error { thread.SetChannel(mgr.channel) - // run the exporter for this instance if it is not running, do nothing otherwise - if w, ok := mgr.threads[key]; !ok || !w.IsStarted() { - mgr.mu.Lock() + + t, ok := mgr.threads[key] + // do nothing if present and already started + if ok && t.IsStarted() { + return nil + } + + mgr.mu.Lock() + defer mgr.mu.Unlock() + if !ok { mgr.threads[key] = thread - if err := mgr.threads[key].Start(ctx, log); err != nil { - return err - } - mgr.mu.Unlock() + } + if err := mgr.threads[key].Start(ctx, log); err != nil { + return err } return nil } -// StopThread stops the thread identified by the given key -func (mgr *Manager) StopThread(key string) { +// stopThread stops the thread identified by the given key +func (mgr *Manager) stopThread(key string) { mgr.mu.Lock() if _, ok := mgr.threads[key]; !ok { return @@ -69,23 +76,23 @@ func (mgr *Manager) GetChannel() <-chan event.GenericEvent { // ReconcileThreads ensures that the threads identified by the provided keys are running. prefix() is used to identify // which threads belong to each resource. -func (mgr *Manager) ReconcileThreads(ctx context.Context, instance client.Object, threads []RunnableThread, log logr.Logger) error { +func (mgr *Manager) ReconcileThreads(ctx context.Context, owner client.Object, threads []RunnableThread, log logr.Logger) error { shouldRun := map[string]int{} for _, thread := range threads { - key := prefix(instance) + thread.GetID() + key := prefix(owner) + thread.GetID() shouldRun[key] = 1 - if err := mgr.RunThread(ctx, key, thread, log); err != nil { + if err := mgr.runThread(ctx, key, thread, log); err != nil { return err } } // Stop threads that should not be running anymore for key := range mgr.threads { - if strings.Contains(key, prefix(instance)) { - if _, ok := shouldRun[key]; !ok { - mgr.StopThread(key) + if strings.Contains(key, prefix(owner)) { + if _, ok := shouldRun[key]; !ok && mgr.threads[key].CanBeDeleted() { + mgr.stopThread(key) } } } @@ -96,16 +103,34 @@ func (mgr *Manager) ReconcileThreads(ctx context.Context, instance client.Object // CleanupThreads returns a function that cleans matching threads when invoked. // This is intended for use as a cleanup function in the finalize phase of a controller's // reconcile loop. -func (mgr *Manager) CleanupThreads(instance client.Object) func() { +func (mgr *Manager) CleanupThreads(owner client.Object) func() { return func() { for key := range mgr.threads { - if strings.Contains(key, prefix(instance)) { - mgr.StopThread(key) + if strings.Contains(key, prefix(owner)) { + mgr.stopThread(key) } } } } +// Returns a thread, typically for inspection by the caller (ie get status/errors) +func (mgr *Manager) GetThread(id string, owner client.Object, log logr.Logger) RunnableThread { + key := prefix(owner) + id + if _, ok := mgr.threads[key]; ok { + return mgr.threads[key] + } else { + return nil + } +} + +func (mgr *Manager) GetKeys() []string { + keys := make([]string, 0, len(mgr.threads)) + for k, _ := range mgr.threads { + keys = append(keys, k) + } + return keys +} + func prefix(o client.Object) string { return util.ObjectKey(o).String() + "_" } diff --git a/pkg/reconcilers/threads/manager_test.go b/pkg/reconcilers/threads/manager_test.go index 778046d9..b8f6cfee 100644 --- a/pkg/reconcilers/threads/manager_test.go +++ b/pkg/reconcilers/threads/manager_test.go @@ -28,8 +28,9 @@ func (trt *TestRunnableThread) Start(context.Context, logr.Logger) error { } return trt.TStartError } -func (trt *TestRunnableThread) Stop() { trt.started = false } -func (trt *TestRunnableThread) IsStarted() bool { return trt.started } +func (trt *TestRunnableThread) Stop() { trt.started = false } +func (trt *TestRunnableThread) IsStarted() bool { return trt.started } +func (trt *TestRunnableThread) CanBeDeleted() bool { return true } func TestManager_RunThread(t *testing.T) { type fields struct { @@ -87,7 +88,7 @@ func TestManager_RunThread(t *testing.T) { threads: tt.fields.threads, } - if err := mgr.RunThread(tt.args.ctx, tt.args.key, tt.args.thread, tt.args.log); (err != nil) != tt.wantErr { + if err := mgr.runThread(tt.args.ctx, tt.args.key, tt.args.thread, tt.args.log); (err != nil) != tt.wantErr { t.Errorf("Manager.RunThread() error = %v, wantErr %v", err, tt.wantErr) } else { if _, ok := mgr.threads[tt.args.key]; !ok { @@ -130,7 +131,7 @@ func TestManager_StopThread(t *testing.T) { channel: tt.fields.channel, threads: tt.fields.threads, } - mgr.StopThread(tt.args.key) + mgr.stopThread(tt.args.key) if _, ok := mgr.threads[tt.args.key]; ok { t.Errorf("Manager.RunThread() RunnableThread should have been deleted from manager") } diff --git a/pkg/redis/backup/backup.go b/pkg/redis/backup/backup.go index 1a073349..d6d7ef53 100644 --- a/pkg/redis/backup/backup.go +++ b/pkg/redis/backup/backup.go @@ -2,63 +2,55 @@ package backup import ( "context" + "fmt" "time" -) - -func (br BackupRunner) RunBackup(ctx context.Context) (chan bool, chan error) { - done := make(chan bool) - errCh := make(chan error) - - go func() { - if err := br.BackgroundSave(ctx); err != nil { - errCh <- err - } - close(done) - }() - - return done, errCh -} + "sigs.k8s.io/controller-runtime/pkg/log" +) -func (br BackupRunner) BackgroundSave(ctx context.Context) error { - lastsave, err := br.server.RedisLastSave(ctx) +func (br *Runner) BackgroundSave(ctx context.Context) error { + logger := log.FromContext(ctx, "function", "(br *Runner) BackgroundSave") + prevsave, err := br.server.RedisLastSave(ctx) if err != nil { - return err + logger.Error(errLastSave(err), "backup error") + return errLastSave(err) } err = br.server.RedisBGSave(ctx) if err != nil { // TODO: need to hanlde the case that a save is already running - return err + logger.Error(errBGSave(err), "backup error") + return errBGSave(err) } + logger.V(1).Info("BGSave running") ticker := time.NewTicker(60 * time.Second) - done := make(chan bool) // wait until BGSAVE completes for { select { case <-ticker.C: - seconds, err := br.server.RedisLastSave(ctx) + lastsave, err := br.server.RedisLastSave(ctx) if err != nil { - // retry in next ticker + // retry at next tick + logger.Error(errLastSave(err), "transient backup error") continue } - if seconds < lastsave { + if lastsave > prevsave { // BGSAVE completed - close(done) + logger.Info("BGSave finished") + return nil } - case <-done: - return nil + case <-ctx.Done(): + return fmt.Errorf("context cancelled") } } - } -func (br BackupRunner) UploadBackup(ctx context.Context) error { +func (br *Runner) UploadBackup(ctx context.Context) error { return nil } -func (br BackupRunner) TagBackup(ctx context.Context) error { +func (br *Runner) TagBackup(ctx context.Context) error { return nil } diff --git a/pkg/redis/backup/errors.go b/pkg/redis/backup/errors.go new file mode 100644 index 00000000..af5a7ef9 --- /dev/null +++ b/pkg/redis/backup/errors.go @@ -0,0 +1,11 @@ +package backup + +import "fmt" + +func errBGSave(err error) error { + return fmt.Errorf("redis cmd (BGSAVE) error: %w", err) +} + +func errLastSave(err error) error { + return fmt.Errorf("redis cmd (LASTSAVE) error: %w", err) +} diff --git a/pkg/redis/backup/manager.go b/pkg/redis/backup/manager.go index 053c87fe..c671499b 100644 --- a/pkg/redis/backup/manager.go +++ b/pkg/redis/backup/manager.go @@ -7,64 +7,122 @@ import ( "github.com/3scale/saas-operator/pkg/redis/sharded" "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/log" ) -type BackupRunner struct { +type Runner struct { shardName string server *sharded.RedisServer timeout time.Duration - // timestamp time.Time - eventsCh chan event.GenericEvent - started bool - cancel context.CancelFunc + timestamp time.Time + eventsCh chan event.GenericEvent + cancel context.CancelFunc + status RunnerStatus + instance client.Object } -func (br *BackupRunner) GetID() string { - return br.shardName +type RunnerStatus struct { + Started bool + Finished bool + Error error +} + +func NewBackupRunner(shardName string, server *sharded.RedisServer, timestamp time.Time, timeout time.Duration, instance client.Object) *Runner { + return &Runner{ + shardName: shardName, + server: server, + timestamp: timestamp, + timeout: timeout, + instance: instance, + } +} + +func ID(shard, alias string, ts time.Time) string { + return fmt.Sprintf("%s-%s-%d", shard, alias, ts.UTC().UnixMilli()) +} + +func (br *Runner) GetID() string { + return ID(br.shardName, br.server.GetAlias(), br.timestamp) } // IsStarted returns whether the backup runner is started or not -func (br *BackupRunner) IsStarted() bool { - return br.started +func (br *Runner) IsStarted() bool { + return br.status.Started } -func (br *BackupRunner) SetChannel(ch chan event.GenericEvent) { +func (br *Runner) CanBeDeleted() bool { + return time.Since(br.timestamp) > 1*time.Hour +} + +func (br *Runner) SetChannel(ch chan event.GenericEvent) { br.eventsCh = ch } // Start starts the backup runner -func (br *BackupRunner) Start(parentCtx context.Context, l logr.Logger) error { - log := l.WithValues("server", br.server.GetAlias()) +func (br *Runner) Start(parentCtx context.Context, l logr.Logger) error { + logger := l.WithValues("server", br.server.GetAlias(), "shard", br.shardName) var ctx context.Context ctx, br.cancel = context.WithCancel(parentCtx) - defer br.cancel() + ctx = log.IntoContext(ctx, logger) + + done := make(chan bool) + errCh := make(chan error) + + // this go routine runs the backup + go func() { + if err := br.BackgroundSave(ctx); err != nil { + errCh <- err + } + close(done) + }() - var done chan bool - var errCh chan error - done, errCh = br.RunBackup(ctx) - log.Info("backup running") + br.status = RunnerStatus{Started: true, Finished: false, Error: nil} + logger.Info("backup running") - // apply a time boundary to the backup and listen for errors - timer := time.NewTimer(br.timeout) - for { - select { - case <-timer.C: - return fmt.Errorf("timeout reached (%v)", br.timeout) + // this goroutine listens controls the max time execution of the backup + // and listens for status updates + go func() { + // apply a time boundary to the backup and listen for errors + timer := time.NewTimer(br.timeout) + for { + select { - case <-done: - return nil + case <-timer.C: + err := fmt.Errorf("timeout reached (%v)", br.timeout) + br.cancel() + logger.Error(err, "backup failed") + br.status.Finished = true + br.status.Error = err + br.eventsCh <- event.GenericEvent{Object: br.instance} + return - case err := <-errCh: - return err + case err := <-errCh: + logger.Error(err, "backup failed") + br.status.Finished = true + br.status.Error = err + br.eventsCh <- event.GenericEvent{Object: br.instance} + return + case <-done: + logger.V(1).Info("backup completed") + br.status.Finished = true + br.eventsCh <- event.GenericEvent{Object: br.instance} + return + } } - } + }() + return nil } // Stop stops the sentinel event watcher -func (br *BackupRunner) Stop() { +func (br *Runner) Stop() { br.cancel() } + +func (br *Runner) Status() RunnerStatus { + return br.status +} diff --git a/pkg/redis/events/watcher.go b/pkg/redis/events/watcher.go index f1d490db..daa2b3ae 100644 --- a/pkg/redis/events/watcher.go +++ b/pkg/redis/events/watcher.go @@ -111,6 +111,10 @@ func (sew *SentinelEventWatcher) IsStarted() bool { return sew.started } +func (sew *SentinelEventWatcher) CanBeDeleted() bool { + return true +} + func (sew *SentinelEventWatcher) SetChannel(ch chan event.GenericEvent) { sew.eventsCh = ch } diff --git a/pkg/redis/metrics/sentinel_metrics.go b/pkg/redis/metrics/sentinel_metrics.go index e32c0817..7dd50417 100644 --- a/pkg/redis/metrics/sentinel_metrics.go +++ b/pkg/redis/metrics/sentinel_metrics.go @@ -126,6 +126,10 @@ func (smg *SentinelMetricsGatherer) IsStarted() bool { return smg.started } +func (smg *SentinelMetricsGatherer) CanBeDeleted() bool { + return true +} + // SetChannel is required for SentinelMetricsGatherer to implement the RunnableThread // interface, but it actually does nothing with the channel. func (fw *SentinelMetricsGatherer) SetChannel(ch chan event.GenericEvent) {} diff --git a/pkg/redis/sharded/redis_sharded_cluster.go b/pkg/redis/sharded/redis_sharded_cluster.go index 43a817ba..bbf22ac8 100644 --- a/pkg/redis/sharded/redis_sharded_cluster.go +++ b/pkg/redis/sharded/redis_sharded_cluster.go @@ -83,7 +83,7 @@ func (cluster *Cluster) GetShardNames() []string { return shards } -func (cluster Cluster) GetShardByName(name string) *Shard { +func (cluster Cluster) LookupShardByName(name string) *Shard { for _, shard := range cluster.Shards { if shard.Name == name { return shard @@ -92,6 +92,17 @@ func (cluster Cluster) GetShardByName(name string) *Shard { return nil } +func (cluster Cluster) LookupServerByID(hostport string) *RedisServer { + for _, shard := range cluster.Shards { + for _, srv := range shard.Servers { + if hostport == srv.ID() { + return srv + } + } + } + return nil +} + func (cluster *Cluster) Discover(ctx context.Context, options ...DiscoveryOption) error { var merr util.MultiError @@ -122,7 +133,7 @@ func (cluster *Cluster) SentinelDiscover(ctx context.Context, opts ...DiscoveryO for _, master := range masters { // Get the corresponding shard - shard := cluster.GetShardByName(master.Name) + shard := cluster.LookupShardByName(master.Name) // Add the shard if not already present if shard == nil { diff --git a/pkg/redis/sharded/redis_sharded_cluster_test.go b/pkg/redis/sharded/redis_sharded_cluster_test.go index 6fe3a18e..6a97e930 100644 --- a/pkg/redis/sharded/redis_sharded_cluster_test.go +++ b/pkg/redis/sharded/redis_sharded_cluster_test.go @@ -163,7 +163,7 @@ func TestShardedCluster_GetShardNames(t *testing.T) { } } -func TestShardedCluster_GetShardByName(t *testing.T) { +func TestShardedCluster_LookupShardByName(t *testing.T) { type args struct { name string } @@ -224,8 +224,8 @@ func TestShardedCluster_GetShardByName(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if diff := deep.Equal(tt.sc.GetShardByName(tt.args.name), tt.want); len(diff) > 0 { - t.Errorf("ShardedCluster.GetShardByName() got diff: %v", diff) + if diff := deep.Equal(tt.sc.LookupShardByName(tt.args.name), tt.want); len(diff) > 0 { + t.Errorf("ShardedCluster.LookupShardByName() got diff: %v", diff) } }) } diff --git a/pkg/redis/sharded/sentinel_server.go b/pkg/redis/sharded/sentinel_server.go index 581d5b78..d60fcc54 100644 --- a/pkg/redis/sharded/sentinel_server.go +++ b/pkg/redis/sharded/sentinel_server.go @@ -97,7 +97,7 @@ func (sentinel *SentinelServer) Monitor(ctx context.Context, cluster *Cluster, q if err != nil { if err.Error() == shardNotInitializedError { - shard := cluster.GetShardByName(name) + shard := cluster.LookupShardByName(name) master, err := shard.GetMaster() if err != nil { return changed, err diff --git a/pkg/util/slice.go b/pkg/util/slice.go index e865bae6..d529a591 100644 --- a/pkg/util/slice.go +++ b/pkg/util/slice.go @@ -14,3 +14,21 @@ func Unique(stringSlice []string) []string { sort.Strings(list) return list } + +func ConcatSlices[T any](slices [][]T) []T { + var totalLen int + + for _, s := range slices { + totalLen += len(s) + } + + result := make([]T, totalLen) + + var i int + + for _, s := range slices { + i += copy(result[i:], s) + } + + return result +} diff --git a/pkg/util/time.go b/pkg/util/time.go new file mode 100644 index 00000000..459c3140 --- /dev/null +++ b/pkg/util/time.go @@ -0,0 +1,11 @@ +package util + +import "time" + +func MustParseRFC3339(in string) time.Time { + t, err := time.Parse(time.RFC3339, in) + if err != nil { + panic(err) + } + return t +} From 37cb8bde231a10358ca9c87c7da6cac59efcf2b2 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Fri, 11 Aug 2023 18:07:42 +0200 Subject: [PATCH 03/43] Upload to s3 --- .gitignore | 2 +- .gitleaks.toml | 5 + api/v1alpha1/redisshard_types.go | 12 +- api/v1alpha1/shardedredisbackup_types.go | 70 ++++++++++- .../bases/saas.3scale.net_redisshards.yaml | 3 + .../saas.3scale.net_shardedredisbackups.yaml | 53 ++++++++- config/test/redis-backups/kustomization.yaml | 16 +++ controllers/shardedredisbackup_controller.go | 66 ++++++++--- go.mod | 21 ++++ go.sum | 87 +++++++++----- pkg/generators/redisshard/configmaps.go | 4 +- pkg/generators/redisshard/generator.go | 2 + pkg/generators/redisshard/statefulset.go | 9 +- pkg/redis/backup/{backup.go => bgsave.go} | 16 +-- pkg/redis/backup/manager.go | 62 +++++----- pkg/redis/backup/s3upload.go | 110 ++++++++++++++++++ pkg/redis/backup/tag.go | 94 +++++++++++++++ 17 files changed, 528 insertions(+), 104 deletions(-) create mode 100644 .gitleaks.toml create mode 100644 config/test/redis-backups/kustomization.yaml rename pkg/redis/backup/{backup.go => bgsave.go} (73%) create mode 100644 pkg/redis/backup/s3upload.go create mode 100644 pkg/redis/backup/tag.go diff --git a/.gitignore b/.gitignore index b6f0e97d..5d437d60 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ *.dylib bin testbin/* - +*.env # Test binary, build with `go test -c` *.test diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 00000000..a5ec056c --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,5 @@ +[allowlist] +description = "global allow lists" +paths = [ + '''examples/backend/assets/redis-with-ssh/test-ssh-key''', +] \ No newline at end of file diff --git a/api/v1alpha1/redisshard_types.go b/api/v1alpha1/redisshard_types.go index 30ef3b9d..a3a1e62a 100644 --- a/api/v1alpha1/redisshard_types.go +++ b/api/v1alpha1/redisshard_types.go @@ -32,8 +32,9 @@ var ( Tag: pointer.String("4.0.11-alpine"), PullPolicy: (*corev1.PullPolicy)(pointer.String(string(corev1.PullIfNotPresent))), } - redisShardDefaultMasterIndex int32 = 0 - RedisShardDefaultReplicas int32 = 3 + redisShardDefaultMasterIndex int32 = 0 + redisShardDefaultCommand string = "redis-server /redis/redis.conf" + RedisShardDefaultReplicas int32 = 3 ) // RedisShardSpec defines the desired state of RedisShard @@ -51,6 +52,10 @@ type RedisShardSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec // +optional SlaveCount *int32 `json:"slaveCount,omitempty"` + // Command overrides the redis container command + // +operator-sdk:csv:customresourcedefinitions:type=spec + // +optional + Command *string `json:"command,omitempty"` } // Default implements defaulting for RedisShardSpec @@ -58,7 +63,8 @@ func (spec *RedisShardSpec) Default() { spec.Image = InitializeImageSpec(spec.Image, redisShardDefaultImage) spec.MasterIndex = intOrDefault(spec.MasterIndex, &redisShardDefaultMasterIndex) - spec.SlaveCount = intOrDefault(spec.SlaveCount, pointer.Int32(RedisShardDefaultReplicas-1)) + spec.SlaveCount = intOrDefault(spec.SlaveCount, util.Pointer(RedisShardDefaultReplicas-1)) + spec.Command = stringOrDefault(spec.Command, &redisShardDefaultCommand) } type RedisShardNodes struct { diff --git a/api/v1alpha1/shardedredisbackup_types.go b/api/v1alpha1/shardedredisbackup_types.go index 1f350d0b..484807ef 100644 --- a/api/v1alpha1/shardedredisbackup_types.go +++ b/api/v1alpha1/shardedredisbackup_types.go @@ -20,28 +20,86 @@ import ( "fmt" "reflect" "sort" + "time" "github.com/3scale/saas-operator/pkg/util" + "github.com/dustin/go-humanize" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var ( - backupHistoryLimit int32 = 10 +const ( + AWSAccessKeyID_SecretKey string = "AWS_ACCESS_KEY_ID" + AWSSecretAccessKey_SecretKey string = "AWS_SECRET_ACCESS_KEY" + BackupFile string = "redis_backup.rdb" + + // defaults + backupHistoryLimit int32 = 10 + backupDefaultTimeout string = "10m" + backupDefaultPollInterval string = "60s" + backupDefaultSSHPort uint32 = 22 + backupDefaultMinSize string = "1 GB" ) // ShardedRedisBackupSpec defines the desired state of ShardedRedisBackup type ShardedRedisBackupSpec struct { - SentinelRef string `json:"sentinelRef"` - Schedule string `json:"schedule"` - Timeout string `json:"timeout"` + SentinelRef string `json:"sentinelRef"` + Schedule string `json:"schedule"` + DBFile string `json:"dbFile"` + SSHOptions SSHOptions `json:"sshOptions"` + S3Options S3Options `json:"s3Options"` + //+optional + Timeout *metav1.Duration `json:"timeout"` //+optional HistoryLimit *int32 `json:"historyLimit,omitempty"` + //+optional + PollInterval *metav1.Duration `json:"pollInterval,omitempty"` + // +optional + MinSize *string `json:"minSize,omitempty"` } // Default implements defaulting for ShardedRedisBackuppec func (spec *ShardedRedisBackupSpec) Default() { - spec.HistoryLimit = intOrDefault(spec.HistoryLimit, &backupHistoryLimit) + if spec.Timeout == nil { + d, _ := time.ParseDuration(backupDefaultTimeout) + spec.Timeout = &metav1.Duration{Duration: d} + } + if spec.PollInterval == nil { + d, _ := time.ParseDuration(backupDefaultPollInterval) + spec.PollInterval = &metav1.Duration{Duration: d} + } + spec.HistoryLimit = intOrDefault(spec.HistoryLimit, util.Pointer(backupHistoryLimit)) + spec.MinSize = stringOrDefault(spec.MinSize, util.Pointer(backupDefaultMinSize)) + spec.SSHOptions.Default() +} + +func (spec *ShardedRedisBackupSpec) GetMinSize() (uint64, error) { + if spec.MinSize == nil { + return humanize.ParseBytes(backupDefaultMinSize) + } else { + return humanize.ParseBytes(*spec.MinSize) + } +} + +type SSHOptions struct { + User string `json:"user"` + PrivateKeySecretRef corev1.LocalObjectReference `json:"privateKeySecretRef"` + // +optional + Port *uint32 `json:"port,omitempty"` +} + +func (opts *SSHOptions) Default() { + if opts.Port == nil { + opts.Port = util.Pointer(backupDefaultSSHPort) + } +} + +type S3Options struct { + Bucket string `json:"bucket"` + Path string `json:"path"` + Region string `json:"region"` + CredentialsSecretRef corev1.LocalObjectReference `json:"credentialsSecretRef"` } // ShardedRedisBackupStatus defines the observed state of ShardedRedisBackup diff --git a/config/crd/bases/saas.3scale.net_redisshards.yaml b/config/crd/bases/saas.3scale.net_redisshards.yaml index 3c4ef4d3..98a2479f 100644 --- a/config/crd/bases/saas.3scale.net_redisshards.yaml +++ b/config/crd/bases/saas.3scale.net_redisshards.yaml @@ -42,6 +42,9 @@ spec: spec: description: RedisShardSpec defines the desired state of RedisShard properties: + command: + description: Command overrides the redis container command + type: string image: description: Image specification for the component properties: diff --git a/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml b/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml index 6a8e38e4..7e11975b 100644 --- a/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml +++ b/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml @@ -36,19 +36,70 @@ spec: spec: description: ShardedRedisBackupSpec defines the desired state of ShardedRedisBackup properties: + dbFile: + type: string historyLimit: format: int32 type: integer + pollInterval: + type: string + s3Options: + properties: + bucket: + type: string + credentialsSecretRef: + description: LocalObjectReference contains enough information + to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + path: + type: string + region: + type: string + required: + - bucket + - credentialsSecretRef + - path + - region + type: object schedule: type: string sentinelRef: type: string + sshOptions: + properties: + port: + format: int32 + type: integer + privateKeySecretRef: + description: LocalObjectReference contains enough information + to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - privateKeySecretRef + - user + type: object timeout: type: string required: + - dbFile + - s3Options - schedule - sentinelRef - - timeout + - sshOptions type: object status: description: ShardedRedisBackupStatus defines the observed state of ShardedRedisBackup diff --git a/config/test/redis-backups/kustomization.yaml b/config/test/redis-backups/kustomization.yaml new file mode 100644 index 00000000..7853314e --- /dev/null +++ b/config/test/redis-backups/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default + + +generatorOptions: + disableNameSuffixHash: true + +secretGenerator: +- name: redis-backup-ssh-private-key + type: kubernetes.io/ssh-auth + files: + - ssh-privatekey=../../../examples/backend/assets/redis-with-ssh/test-ssh-key +- name: aws-credentials + envs: + - ../../../examples/backend/assets/s3/credentials.env diff --git a/controllers/shardedredisbackup_controller.go b/controllers/shardedredisbackup_controller.go index 5f08d10a..ec92dabf 100644 --- a/controllers/shardedredisbackup_controller.go +++ b/controllers/shardedredisbackup_controller.go @@ -18,8 +18,10 @@ package controllers import ( "context" + "fmt" "time" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -96,6 +98,23 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R return ctrl.Result{}, err } + // Get SSH key + sshPrivateKey := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: instance.Spec.SSHOptions.PrivateKeySecretRef.Name, Namespace: req.Namespace}} + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(sshPrivateKey), sshPrivateKey); err != nil { + return ctrl.Result{}, err + } + if sshPrivateKey.Type != corev1.SecretTypeSSHAuth { + return ctrl.Result{}, fmt.Errorf("secret %s must be of 'kubernetes.io/ssh-auth' type", sshPrivateKey.GetName()) + } + + // Get AWS credentials + awsCredentials := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: instance.Spec.S3Options.CredentialsSecretRef.Name, Namespace: req.Namespace}} + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(awsCredentials), awsCredentials); err != nil { + return ctrl.Result{}, err + } + // ---------------------------------------- // ----- Phase 2: run pending backups ----- // ---------------------------------------- @@ -104,19 +123,38 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R runners := make([]threads.RunnableThread, 0, len(cluster.Shards)) for _, shard := range cluster.Shards { scheduledBackup := instance.Status.FindLastBackup(shard.Name, saasv1alpha1.BackupPendingState) - if scheduledBackup != nil { - if scheduledBackup.State == saasv1alpha1.BackupPendingState && scheduledBackup.ScheduledFor.Time.Before(time.Now()) { - // add the backup runner thread - target := shard.GetSlavesRO()[0] - runners = append(runners, backup.NewBackupRunner(shard.Name, target, scheduledBackup.ScheduledFor.Time, 10*time.Minute, instance)) - scheduledBackup.ServerAlias = util.Pointer(target.GetAlias()) - scheduledBackup.ServerID = util.Pointer(target.ID()) - now := metav1.Now() - scheduledBackup.StartedAt = util.Pointer(now) - scheduledBackup.Message = "backup is running" - scheduledBackup.State = saasv1alpha1.BackupRunningState - statusChanged = true + if scheduledBackup != nil && scheduledBackup.ScheduledFor.Time.Before(time.Now()) { + // add the backup runner thread + minSize, err := instance.Spec.GetMinSize() + if err != nil { + return ctrl.Result{}, fmt.Errorf("invalid 'spec.minSize' specification: %w", err) } + target := shard.GetSlavesRO()[0] + runners = append(runners, &backup.Runner{ + ShardName: shard.Name, + Server: target, + Timestamp: scheduledBackup.ScheduledFor.Time, + Timeout: instance.Spec.Timeout.Duration, + PollInterval: instance.Spec.PollInterval.Duration, + MinSize: minSize, + RedisDBFile: instance.Spec.DBFile, + Instance: instance, + SSHUser: instance.Spec.SSHOptions.User, + SSHKey: string(sshPrivateKey.Data[corev1.SSHAuthPrivateKey]), + SSHPort: *instance.Spec.SSHOptions.Port, + S3Bucket: instance.Spec.S3Options.Bucket, + S3Path: instance.Spec.S3Options.Path, + AWSAccessKeyID: string(awsCredentials.Data[saasv1alpha1.AWSAccessKeyID_SecretKey]), + AWSSecretAccessKey: string(awsCredentials.Data[saasv1alpha1.AWSSecretAccessKey_SecretKey]), + AWSRegion: instance.Spec.S3Options.Region, + }) + scheduledBackup.ServerAlias = util.Pointer(target.GetAlias()) + scheduledBackup.ServerID = util.Pointer(target.ID()) + now := metav1.Now() + scheduledBackup.StartedAt = util.Pointer(now) + scheduledBackup.Message = "backup is running" + scheduledBackup.State = saasv1alpha1.BackupRunningState + statusChanged = true } } @@ -125,7 +163,6 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R } if statusChanged { - logger.V(1).Info("backup runners started") err := r.Client.Status().Update(ctx, instance) return ctrl.Result{}, err } @@ -133,6 +170,7 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R // -------------------------------------------------------- // ----- Phase 3: reconcile status of running backups ----- // -------------------------------------------------------- + for _, b := range instance.Status.GetRunningBackups() { var thread *backup.Runner var srv *sharded.RedisServer @@ -166,7 +204,6 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R } if statusChanged { - logger.V(1).Info("backup schedule updated") err := r.Client.Status().Update(ctx, instance) return ctrl.Result{}, err } @@ -187,7 +224,6 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R } if statusChanged { - logger.V(1).Info("backup schedule updated") err := r.Client.Status().Update(ctx, instance) return ctrl.Result{}, err } diff --git a/go.mod b/go.mod index 9fa268f6..03177e60 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,11 @@ require ( github.com/3scale-ops/basereconciler v0.3.1 github.com/3scale-ops/marin3r v0.12.2 github.com/MakeNowJust/heredoc v1.0.0 + github.com/aws/aws-sdk-go v1.44.220 + github.com/aws/aws-sdk-go-v2/config v1.18.37 + github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 github.com/davecgh/go-spew v1.1.1 + github.com/dustin/go-humanize v1.0.0 github.com/envoyproxy/go-control-plane v0.11.0 github.com/evanphx/json-patch v5.6.0+incompatible github.com/external-secrets/external-secrets v0.8.1 @@ -26,6 +30,7 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/tektoncd/pipeline v0.49.0 go.uber.org/zap v1.24.0 + golang.org/x/crypto v0.9.0 golang.org/x/time v0.3.0 google.golang.org/protobuf v1.30.0 k8s.io/api v0.26.2 @@ -39,6 +44,22 @@ require ( require ( contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.21.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.35 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect + github.com/aws/smithy-go v1.14.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect github.com/blendle/zapdriver v1.3.1 // indirect diff --git a/go.sum b/go.sum index cf547e0f..021bb203 100644 --- a/go.sum +++ b/go.sum @@ -35,23 +35,15 @@ contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d/g contrib.go.opencensus.io/exporter/prometheus v0.4.0 h1:0QfIkj9z/iVZgK31D9H9ohjjIDApI2GOPScCKwxedbs= contrib.go.opencensus.io/exporter/prometheus v0.4.0/go.mod h1:o7cosnyfuPVK0tB8q0QmaQNhGnptITnPQB+z1+qeFB0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/3scale-ops/basereconciler v0.3.0 h1:gcN2fcSY+Sh2tG4xxvzFRqE/rimIivLOWiosxqP65sg= -github.com/3scale-ops/basereconciler v0.3.0/go.mod h1:P1N9tWurRFAxxx5FCWHH6XcVHLMiNKxXWkH5KtP6ywQ= github.com/3scale-ops/basereconciler v0.3.1 h1:+otkLsodzu/k4MPWm+fAd9OwznpUCuwiSDMEed7+Q1k= github.com/3scale-ops/basereconciler v0.3.1/go.mod h1:P1N9tWurRFAxxx5FCWHH6XcVHLMiNKxXWkH5KtP6ywQ= github.com/3scale-ops/marin3r v0.12.2 h1:sU4N7RZ5AQzfejrfP0KgpagmL/XD8a5NdSIv1qvaAnU= github.com/3scale-ops/marin3r v0.12.2/go.mod h1:BOU7EFv58PgPMgKgJFDd4ingfBhH6KC2AGJoD7i9SaU= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -66,6 +58,44 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.44.220 h1:yAj99qAt0Htjle9Up3DglgHfOP77lmFPrElA4jKnrBo= +github.com/aws/aws-sdk-go v1.44.220/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= +github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM= +github.com/aws/aws-sdk-go-v2/config v1.18.37 h1:RNAfbPqw1CstCooHaTPhScz7z1PyocQj0UL+l95CgzI= +github.com/aws/aws-sdk-go-v2/config v1.18.37/go.mod h1:8AnEFxW9/XGKCbjYDCJy7iltVNyEI9Iu9qC21UzhhgQ= +github.com/aws/aws-sdk-go-v2/credentials v1.13.35 h1:QpsNitYJu0GgvMBLUIYu9H4yryA5kMksjeIVQfgXrt8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.35/go.mod h1:o7rCaLtvK0hUggAGclf76mNGGkaG5a9KWlp+d9IpcV8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 h1:GPUcE/Yq7Ur8YSUk6lVkoIMWnJNO0HT18GUzCWCgCI0= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42/go.mod h1:rzfdUlfA+jdgLDmPKjd3Chq9V7LVLYo1Nz++Wb91aRo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 h1:6lJvvkQ9HmbHZ4h/IEwclwv2mrTW8Uq1SOB/kXy0mfw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4/go.mod h1:1PrKYwxTM+zjpw9Y41KFtoJCQrJ34Z47Y4VgVbfndjo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 h1:m0QTSI6pZYJTk5WSKx3fm5cNW/DCicVzULBgU/6IyD0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14/go.mod h1:dDilntgHy9WnHXsh7dDtUPgHKEfTJIBUTHM8OWm0f/0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 h1:eev2yZX7esGRjqRbnVk1UxMLw4CyVZDpZXRCcy75oQk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36/go.mod h1:lGnOkH9NJATw0XEPcAknFBj3zzNTEGRHtSw+CwC1YTg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 h1:oCvTFSDi67AX0pOX3PuPdGFewvLRU2zzFSrTsgURNo0= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.5/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 h1:dnInJb4S0oy8aQuri1mV6ipLlnZPfnsDNB9BGO9PDNY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5/go.mod h1:yygr8ACQRY2PrEcy3xsUI357stq2AxnFM6DIsR9lij4= +github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 h1:CQBFElb0LS8RojMJlxRSo/HXipvTZW2S44Lt9Mk2aYQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.21.5/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU= +github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= +github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -74,7 +104,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE= github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= @@ -99,8 +128,6 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195 h1:58f1tJ1ra+zFINPlwLWvQsR9CzAKt2e+EWV2yX9oXQ4= github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= @@ -114,6 +141,7 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= @@ -142,7 +170,6 @@ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2Vvl github.com/external-secrets/external-secrets v0.8.1 h1:LI7lYmR04Zi2gMVdgifTtyGKfBtYrCA380ePgds2gsY= github.com/external-secrets/external-secrets v0.8.1/go.mod h1:N5TxTxHLbCK2vVmcUAbUUorwuZiKxJqd/j8I65+44Zc= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -236,7 +263,6 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= @@ -251,6 +277,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE= @@ -286,9 +313,6 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grafana-operator/grafana-operator/v4 v4.10.0 h1:+AVEPP/wflmx5ySdzt1mIw+q63ZYVInxQhF3XKNhJv4= github.com/grafana-operator/grafana-operator/v4 v4.10.0/go.mod h1:k69wJcXVrqAcZBoGuh5LSqz0ak8LlVOxxqp0W3f/4V8= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= @@ -303,12 +327,13 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -347,7 +372,6 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= @@ -367,8 +391,6 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -388,7 +410,6 @@ github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3I github.com/openshift/api v0.0.0-20220715133027-dab5b363ebd1 h1:FzCXZdnkGLus4hHu7/d/utr3ELPiwNt2ffAqSspi6U8= github.com/openshift/api v0.0.0-20220715133027-dab5b363ebd1/go.mod h1:LEnw1IVscIxyDnltE3Wi7bQb/QzIM8BfPNKoGA1Qlxw= github.com/openshift/build-machinery-go v0.0.0-20211213093930-7e33a7eb4ce3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -430,15 +451,12 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -459,18 +477,16 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/tektoncd/pipeline v0.49.0 h1:LxpgoPZvIDiOvPj6vtInnGG0uzuQ5CPA+h8FdJdklh4= github.com/tektoncd/pipeline v0.49.0/go.mod h1:R3Qn/oTTf1SCLrj+rCg4sqUbpx7vE+6D8Z81+zKUdqQ= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -504,6 +520,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -528,7 +546,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -539,6 +556,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -582,6 +600,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -604,6 +624,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -656,11 +677,15 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -672,6 +697,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -728,6 +754,7 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -844,7 +871,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -863,7 +889,6 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/generators/redisshard/configmaps.go b/pkg/generators/redisshard/configmaps.go index f70a6eb9..82e61ee5 100644 --- a/pkg/generators/redisshard/configmaps.go +++ b/pkg/generators/redisshard/configmaps.go @@ -20,8 +20,8 @@ func (gen *Generator) redisConfigConfigMap() func() *corev1.ConfigMap { "redis.conf": heredoc.Doc(` slaveof 127.0.0.1 6379 tcp-keepalive 60 - save 900 1 - save 300 10 + daemonize yes + logfile /dev/stdout `), }, } diff --git a/pkg/generators/redisshard/generator.go b/pkg/generators/redisshard/generator.go index e7c7235a..a1ff2e90 100644 --- a/pkg/generators/redisshard/generator.go +++ b/pkg/generators/redisshard/generator.go @@ -19,6 +19,7 @@ type Generator struct { Image saasv1alpha1.ImageSpec MasterIndex int32 Replicas int32 + Command string } // Override the GetSelector function as it needs to be different in this case @@ -42,6 +43,7 @@ func NewGenerator(instance, namespace string, spec saasv1alpha1.RedisShardSpec) Image: *spec.Image, MasterIndex: *spec.MasterIndex, Replicas: *spec.SlaveCount + 1, + Command: *spec.Command, } } diff --git a/pkg/generators/redisshard/statefulset.go b/pkg/generators/redisshard/statefulset.go index d2850bb7..92857562 100644 --- a/pkg/generators/redisshard/statefulset.go +++ b/pkg/generators/redisshard/statefulset.go @@ -43,7 +43,7 @@ func (gen *Generator) statefulSet() func() *appsv1.StatefulSet { RestartPolicy: corev1.RestartPolicyAlways, Containers: []corev1.Container{ { - Command: []string{"redis-server", "/redis/redis.conf"}, + Command: strings.Split(gen.Command, " "), Image: fmt.Sprintf("%s:%s", *gen.Image.Name, *gen.Image.Tag), Name: "redis-server", Ports: pod.ContainerPorts( @@ -69,12 +69,7 @@ func (gen *Generator) statefulSet() func() *appsv1.StatefulSet { }, }, }, - SecurityContext: &corev1.PodSecurityContext{ - FSGroup: pointer.Int64(1000), - RunAsGroup: pointer.Int64(1000), - RunAsNonRoot: pointer.Bool(true), - RunAsUser: pointer.Int64(1000), - }, + SecurityContext: &corev1.PodSecurityContext{}, TerminationGracePeriodSeconds: pointer.Int64(0), Volumes: []corev1.Volume{ { diff --git a/pkg/redis/backup/backup.go b/pkg/redis/backup/bgsave.go similarity index 73% rename from pkg/redis/backup/backup.go rename to pkg/redis/backup/bgsave.go index d6d7ef53..f35c127c 100644 --- a/pkg/redis/backup/backup.go +++ b/pkg/redis/backup/bgsave.go @@ -10,13 +10,13 @@ import ( func (br *Runner) BackgroundSave(ctx context.Context) error { logger := log.FromContext(ctx, "function", "(br *Runner) BackgroundSave") - prevsave, err := br.server.RedisLastSave(ctx) + prevsave, err := br.Server.RedisLastSave(ctx) if err != nil { logger.Error(errLastSave(err), "backup error") return errLastSave(err) } - err = br.server.RedisBGSave(ctx) + err = br.Server.RedisBGSave(ctx) if err != nil { // TODO: need to hanlde the case that a save is already running logger.Error(errBGSave(err), "backup error") @@ -24,13 +24,13 @@ func (br *Runner) BackgroundSave(ctx context.Context) error { } logger.V(1).Info("BGSave running") - ticker := time.NewTicker(60 * time.Second) + ticker := time.NewTicker(br.PollInterval) // wait until BGSAVE completes for { select { case <-ticker.C: - lastsave, err := br.server.RedisLastSave(ctx) + lastsave, err := br.Server.RedisLastSave(ctx) if err != nil { // retry at next tick logger.Error(errLastSave(err), "transient backup error") @@ -46,11 +46,3 @@ func (br *Runner) BackgroundSave(ctx context.Context) error { } } } - -func (br *Runner) UploadBackup(ctx context.Context) error { - return nil -} - -func (br *Runner) TagBackup(ctx context.Context) error { - return nil -} diff --git a/pkg/redis/backup/manager.go b/pkg/redis/backup/manager.go index c671499b..3a4ff6c9 100644 --- a/pkg/redis/backup/manager.go +++ b/pkg/redis/backup/manager.go @@ -13,14 +13,25 @@ import ( ) type Runner struct { - shardName string - server *sharded.RedisServer - timeout time.Duration - timestamp time.Time - eventsCh chan event.GenericEvent - cancel context.CancelFunc - status RunnerStatus - instance client.Object + Instance client.Object + ShardName string + Server *sharded.RedisServer + Timeout time.Duration + PollInterval time.Duration + MinSize uint64 + Timestamp time.Time + RedisDBFile string + SSHUser string + SSHKey string + SSHPort uint32 + S3Bucket string + S3Path string + AWSAccessKeyID string + AWSSecretAccessKey string + AWSRegion string + eventsCh chan event.GenericEvent + cancel context.CancelFunc + status RunnerStatus } type RunnerStatus struct { @@ -29,22 +40,12 @@ type RunnerStatus struct { Error error } -func NewBackupRunner(shardName string, server *sharded.RedisServer, timestamp time.Time, timeout time.Duration, instance client.Object) *Runner { - return &Runner{ - shardName: shardName, - server: server, - timestamp: timestamp, - timeout: timeout, - instance: instance, - } -} - func ID(shard, alias string, ts time.Time) string { return fmt.Sprintf("%s-%s-%d", shard, alias, ts.UTC().UnixMilli()) } func (br *Runner) GetID() string { - return ID(br.shardName, br.server.GetAlias(), br.timestamp) + return ID(br.ShardName, br.Server.GetAlias(), br.Timestamp) } // IsStarted returns whether the backup runner is started or not @@ -53,7 +54,7 @@ func (br *Runner) IsStarted() bool { } func (br *Runner) CanBeDeleted() bool { - return time.Since(br.timestamp) > 1*time.Hour + return time.Since(br.Timestamp) > 1*time.Hour } func (br *Runner) SetChannel(ch chan event.GenericEvent) { @@ -62,7 +63,7 @@ func (br *Runner) SetChannel(ch chan event.GenericEvent) { // Start starts the backup runner func (br *Runner) Start(parentCtx context.Context, l logr.Logger) error { - logger := l.WithValues("server", br.server.GetAlias(), "shard", br.shardName) + logger := l.WithValues("server", br.Server.GetAlias(), "shard", br.ShardName) var ctx context.Context ctx, br.cancel = context.WithCancel(parentCtx) @@ -75,6 +76,15 @@ func (br *Runner) Start(parentCtx context.Context, l logr.Logger) error { go func() { if err := br.BackgroundSave(ctx); err != nil { errCh <- err + return + } + if err := br.UploadBackup(ctx); err != nil { + errCh <- err + return + } + if err := br.TagBackup(ctx); err != nil { + errCh <- err + return } close(done) }() @@ -86,30 +96,30 @@ func (br *Runner) Start(parentCtx context.Context, l logr.Logger) error { // and listens for status updates go func() { // apply a time boundary to the backup and listen for errors - timer := time.NewTimer(br.timeout) + timer := time.NewTimer(br.Timeout) for { select { case <-timer.C: - err := fmt.Errorf("timeout reached (%v)", br.timeout) + err := fmt.Errorf("timeout reached (%v)", br.Timeout) br.cancel() logger.Error(err, "backup failed") br.status.Finished = true br.status.Error = err - br.eventsCh <- event.GenericEvent{Object: br.instance} + br.eventsCh <- event.GenericEvent{Object: br.Instance} return case err := <-errCh: logger.Error(err, "backup failed") br.status.Finished = true br.status.Error = err - br.eventsCh <- event.GenericEvent{Object: br.instance} + br.eventsCh <- event.GenericEvent{Object: br.Instance} return case <-done: logger.V(1).Info("backup completed") br.status.Finished = true - br.eventsCh <- event.GenericEvent{Object: br.instance} + br.eventsCh <- event.GenericEvent{Object: br.Instance} return } } diff --git a/pkg/redis/backup/s3upload.go b/pkg/redis/backup/s3upload.go new file mode 100644 index 00000000..c5960f94 --- /dev/null +++ b/pkg/redis/backup/s3upload.go @@ -0,0 +1,110 @@ +package backup + +import ( + "context" + "fmt" + "net" + "path" + "strconv" + "time" + + "golang.org/x/crypto/ssh" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + backupFilePrefix string = "redis-backup" + backupFileExtension string = "rdb" + sshKeyFile string = "ssh-private-key" + awsAccessKeyEnvvar string = "AWS_ACCESS_KEY_ID" + awsSecretKeyEnvvar string = "AWS_SECRET_ACCESS_KEY" + awsRegionEnvvar string = "AWS_REGION" +) + +func (br *Runner) BackupFileBaseName() string { + return fmt.Sprintf("%s_%s", backupFilePrefix, br.ShardName) +} + +func (br *Runner) BackupFileBaseNameWithTimeSuffix(timeSuffix string) string { + return fmt.Sprintf("%s_%s", br.BackupFileBaseName(), timeSuffix) +} + +// BackupFile returns the backup file as "redis-backup---.rdb" +func (br *Runner) BackupFile() string { + return fmt.Sprintf("%s.%s", + br.BackupFileBaseNameWithTimeSuffix(br.Timestamp.Format(time.RFC3339)), + backupFileExtension) +} + +func (br *Runner) BackupFileCompressed() string { + return fmt.Sprintf("%s.gz", br.BackupFile()) +} + +func (br *Runner) BackupFileS3Path() string { + return fmt.Sprintf("%s/%s", br.S3Path, br.BackupFileCompressed()) +} + +func (br *Runner) UploadBackup(ctx context.Context) error { + logger := log.FromContext(ctx, "function", "(br *Runner) UploadBackup()") + + var commands = []string{ + // mv /data/dump.rdb /data/redis-backup---.rdb + fmt.Sprintf("mv %s %s/%s", + br.RedisDBFile, + path.Dir(br.RedisDBFile), br.BackupFile(), + ), + // gzip /data/redis-backup---.rdb + fmt.Sprintf("gzip %s/%s", path.Dir(br.RedisDBFile), br.BackupFile()), + // AWS_ACCESS_KEY_ID=*** AWS_SECRET_ACCESS_KEY=*** s3cmd put /data/redis-backup---.rdb s3:////redis-backup---.rdb + fmt.Sprintf("%s=%s %s=%s s3cmd put %s/%s s3://%s/%s/%s", + awsAccessKeyEnvvar, br.AWSAccessKeyID, + awsSecretKeyEnvvar, br.AWSSecretAccessKey, + path.Dir(br.RedisDBFile), br.BackupFileCompressed(), + br.S3Bucket, br.S3Path, br.BackupFileCompressed(), + ), + } + + for _, command := range commands { + logger.V(1).Info(fmt.Sprintf("running command '%s' on %s:%d", command, br.Server.GetHost(), br.SSHPort)) + output, err := remoteRun(ctx, br.SSHUser, br.Server.GetHost(), strconv.Itoa(int(br.SSHPort)), br.SSHKey, command) + if output != "" { + logger.V(1).Info(fmt.Sprintf("remote ssh command output: %s", output)) + } + if err != nil { + logger.V(1).Info(fmt.Sprintf("remote ssh command error: %s", err.Error())) + return fmt.Errorf("remote ssh command failed: %w (%s)", err, output) + } + } + + return nil +} + +// e.g. output, err := remoteRun(ctx, "root", "MY_IP", "MY_PORT", "PRIVATE_KEY", "ls") +func remoteRun(ctx context.Context, user, addr, port, privateKey, cmd string) (string, error) { + + key, err := ssh.ParsePrivateKey([]byte(privateKey)) + if err != nil { + return "", err + } + // Authentication + config := &ssh.ClientConfig{ + User: user, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(key), + }, + } + client, err := ssh.Dial("tcp", net.JoinHostPort(addr, port), config) + if err != nil { + return "", err + } + // Create a session. It is one session per command. + session, err := client.NewSession() + if err != nil { + return "", err + } + defer session.Close() + + output, err := session.CombinedOutput(cmd) + return string(output), err +} diff --git a/pkg/redis/backup/tag.go b/pkg/redis/backup/tag.go new file mode 100644 index 00000000..9af0a684 --- /dev/null +++ b/pkg/redis/backup/tag.go @@ -0,0 +1,94 @@ +package backup + +import ( + "context" + "fmt" + "os" + "sort" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/aws-sdk-go/aws" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +func (br *Runner) TagBackup(ctx context.Context) error { + logger := log.FromContext(ctx, "function", "(br *Runner) TagBackup()") + + // set AWS credentials + os.Setenv(awsAccessKeyEnvvar, br.AWSAccessKeyID) + os.Setenv(awsSecretKeyEnvvar, br.AWSSecretAccessKey) + os.Setenv(awsRegionEnvvar, br.AWSRegion) + + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return err + } + client := s3.NewFromConfig(cfg) + + // get backups of current day + dayResult, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(br.S3Bucket), + Prefix: aws.String(br.S3Path + "/" + br.BackupFileBaseNameWithTimeSuffix(br.Timestamp.Format("2006-01-02"))), + }) + if err != nil { + return err + } + sort.SliceStable(dayResult.Contents, func(i, j int) bool { + return dayResult.Contents[i].LastModified.Before(*dayResult.Contents[j].LastModified) + }) + + // get backups of current hour + hourResult, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(br.S3Bucket), + Prefix: aws.String(br.S3Path + "/" + br.BackupFileBaseNameWithTimeSuffix(br.Timestamp.Format("2006-01-02T15"))), + }) + if err != nil { + return err + } + sort.SliceStable(hourResult.Contents, func(i, j int) bool { + return hourResult.Contents[i].LastModified.Before(*hourResult.Contents[j].LastModified) + }) + + firstOfDay := dayResult.Contents[0] + firstOfHour := hourResult.Contents[0] + + last := hourResult.Contents[len(hourResult.Contents)-1] + if br.BackupFileS3Path() != *last.Key { + return fmt.Errorf("last backup %s has different key than expected (%s)", *last.Key, br.BackupFileS3Path()) + } + + // check backup size of last (given a size passed as threshold in the CR) + if last.Size < int64(br.MinSize) { + return fmt.Errorf("last backup %s is smaller that declared min size of %d", *last.Key, br.MinSize) + } + + tags := []types.Tag{ + {Key: aws.String("Layer"), Value: aws.String("bck-storage")}, + {Key: aws.String("App"), Value: aws.String("Backend")}, + } + + switch br.BackupFileS3Path() { + case *firstOfDay.Key: + tags = append(tags, types.Tag{Key: aws.String("Retention"), Value: aws.String("90d")}) + logger.V(1).Info("backup tagged with 90d retention") + case *firstOfHour.Key: + tags = append(tags, types.Tag{Key: aws.String("Retention"), Value: aws.String("7d")}) + logger.V(1).Info("backup tagged with 7d retention") + default: + tags = append(tags, types.Tag{Key: aws.String("Retention"), Value: aws.String("24h")}) + logger.V(1).Info("backup tagged with 24h retention") + } + + _, err = client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{ + Bucket: &br.S3Bucket, + Key: aws.String(br.BackupFileS3Path()), + Tagging: &types.Tagging{TagSet: tags}, + }) + if err != nil { + return err + } + + return nil +} From cc5f849c4501eeafecf00050a6564a756f2b5ed4 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Fri, 1 Sep 2023 16:28:34 +0200 Subject: [PATCH 04/43] remove file --- pkg/redis/backup/bgsave.go | 8 ++++++++ pkg/redis/backup/errors.go | 11 ----------- 2 files changed, 8 insertions(+), 11 deletions(-) delete mode 100644 pkg/redis/backup/errors.go diff --git a/pkg/redis/backup/bgsave.go b/pkg/redis/backup/bgsave.go index f35c127c..43653c9c 100644 --- a/pkg/redis/backup/bgsave.go +++ b/pkg/redis/backup/bgsave.go @@ -46,3 +46,11 @@ func (br *Runner) BackgroundSave(ctx context.Context) error { } } } + +func errBGSave(err error) error { + return fmt.Errorf("redis cmd (BGSAVE) error: %w", err) +} + +func errLastSave(err error) error { + return fmt.Errorf("redis cmd (LASTSAVE) error: %w", err) +} diff --git a/pkg/redis/backup/errors.go b/pkg/redis/backup/errors.go deleted file mode 100644 index af5a7ef9..00000000 --- a/pkg/redis/backup/errors.go +++ /dev/null @@ -1,11 +0,0 @@ -package backup - -import "fmt" - -func errBGSave(err error) error { - return fmt.Errorf("redis cmd (BGSAVE) error: %w", err) -} - -func errLastSave(err error) error { - return fmt.Errorf("redis cmd (LASTSAVE) error: %w", err) -} From 5d599f3f4841a484e593a9fb650a7cc36526eac7 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Fri, 1 Sep 2023 16:29:53 +0200 Subject: [PATCH 05/43] fix comment --- pkg/redis/backup/manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/redis/backup/manager.go b/pkg/redis/backup/manager.go index 3a4ff6c9..2859972c 100644 --- a/pkg/redis/backup/manager.go +++ b/pkg/redis/backup/manager.go @@ -92,7 +92,7 @@ func (br *Runner) Start(parentCtx context.Context, l logr.Logger) error { br.status = RunnerStatus{Started: true, Finished: false, Error: nil} logger.Info("backup running") - // this goroutine listens controls the max time execution of the backup + // this goroutine controls the max time execution of the backup // and listens for status updates go func() { // apply a time boundary to the backup and listen for errors From 31898c237e5776b653e4bdf7188f50e92ba8799e Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Fri, 1 Sep 2023 16:31:44 +0200 Subject: [PATCH 06/43] add TODO comment --- pkg/redis/backup/s3upload.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/redis/backup/s3upload.go b/pkg/redis/backup/s3upload.go index c5960f94..ac14964a 100644 --- a/pkg/redis/backup/s3upload.go +++ b/pkg/redis/backup/s3upload.go @@ -55,6 +55,7 @@ func (br *Runner) UploadBackup(ctx context.Context) error { ), // gzip /data/redis-backup---.rdb fmt.Sprintf("gzip %s/%s", path.Dir(br.RedisDBFile), br.BackupFile()), + // TODO: use awscli instead // AWS_ACCESS_KEY_ID=*** AWS_SECRET_ACCESS_KEY=*** s3cmd put /data/redis-backup---.rdb s3:////redis-backup---.rdb fmt.Sprintf("%s=%s %s=%s s3cmd put %s/%s s3://%s/%s/%s", awsAccessKeyEnvvar, br.AWSAccessKeyID, From 6da3964103d9d2cc5785d899a04a84b79814072f Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Mon, 4 Sep 2023 16:27:09 +0200 Subject: [PATCH 07/43] Remove coverage file --- controllers/coverage/coverage.html | 1776 ---------------------------- 1 file changed, 1776 deletions(-) delete mode 100644 controllers/coverage/coverage.html diff --git a/controllers/coverage/coverage.html b/controllers/coverage/coverage.html deleted file mode 100644 index ef33ae63..00000000 --- a/controllers/coverage/coverage.html +++ /dev/null @@ -1,1776 +0,0 @@ - - - - - controllers: Go Coverage Report - - - -
- -
- not tracked - - not covered - covered -
-
-
- - - - - - - - - - - - - - - - - - - - - - - -
- - - From fcb55a81b9a5d27a14e06f2ba46d9f8c4c5632e2 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Mon, 4 Sep 2023 17:41:42 +0200 Subject: [PATCH 08/43] Use latest kustomize --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 25b34d59..53f98eb1 100644 --- a/Makefile +++ b/Makefile @@ -294,7 +294,7 @@ KIND ?= $(LOCALBIN)/kind GOBINDATA ?= $(LOCALBIN)/go-bindata ## Tool Versions -KUSTOMIZE_VERSION ?= v3.8.7 +KUSTOMIZE_VERSION ?= v5.1.1 CONTROLLER_TOOLS_VERSION ?= v0.11.0 GINKGO_VERSION ?= v2.9.1 CRD_REFDOCS_VERSION ?= v0.0.8 From e8fa526c328959635c42256b9373960ac5b69b9d Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Mon, 4 Sep 2023 17:42:04 +0200 Subject: [PATCH 09/43] use awscli instead of s3cmd --- pkg/redis/backup/s3upload.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/redis/backup/s3upload.go b/pkg/redis/backup/s3upload.go index ac14964a..2aee3c9e 100644 --- a/pkg/redis/backup/s3upload.go +++ b/pkg/redis/backup/s3upload.go @@ -57,7 +57,8 @@ func (br *Runner) UploadBackup(ctx context.Context) error { fmt.Sprintf("gzip %s/%s", path.Dir(br.RedisDBFile), br.BackupFile()), // TODO: use awscli instead // AWS_ACCESS_KEY_ID=*** AWS_SECRET_ACCESS_KEY=*** s3cmd put /data/redis-backup---.rdb s3:////redis-backup---.rdb - fmt.Sprintf("%s=%s %s=%s s3cmd put %s/%s s3://%s/%s/%s", + fmt.Sprintf("%s=%s %s=%s %s=%s s3 cp %s/%s s3://%s/%s/%s", + awsRegionEnvvar, br.AWSRegion, awsAccessKeyEnvvar, br.AWSAccessKeyID, awsSecretKeyEnvvar, br.AWSSecretAccessKey, path.Dir(br.RedisDBFile), br.BackupFileCompressed(), From d150f2f6a800f8a9f8679444f57b573c38077381 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Tue, 5 Sep 2023 12:04:25 +0200 Subject: [PATCH 10/43] Add API descriptions --- api/v1alpha1/shardedredisbackup_types.go | 89 +++++++++++++++++++----- 1 file changed, 71 insertions(+), 18 deletions(-) diff --git a/api/v1alpha1/shardedredisbackup_types.go b/api/v1alpha1/shardedredisbackup_types.go index 484807ef..83e429d7 100644 --- a/api/v1alpha1/shardedredisbackup_types.go +++ b/api/v1alpha1/shardedredisbackup_types.go @@ -43,19 +43,41 @@ const ( // ShardedRedisBackupSpec defines the desired state of ShardedRedisBackup type ShardedRedisBackupSpec struct { - SentinelRef string `json:"sentinelRef"` - Schedule string `json:"schedule"` - DBFile string `json:"dbFile"` - SSHOptions SSHOptions `json:"sshOptions"` - S3Options S3Options `json:"s3Options"` - //+optional + // Reference to a sentinel instance + // +operator-sdk:csv:customresourcedefinitions:type=spec + SentinelRef string `json:"sentinelRef"` + // Cron-like schedule specification + // +operator-sdk:csv:customresourcedefinitions:type=spec + Schedule string `json:"schedule"` + // Name of the dbfile in the redis instances + // +operator-sdk:csv:customresourcedefinitions:type=spec + DBFile string `json:"dbFile"` + // SSH connection options + // +operator-sdk:csv:customresourcedefinitions:type=spec + SSHOptions SSHOptions `json:"sshOptions"` + // S3 storage options + // +operator-sdk:csv:customresourcedefinitions:type=spec + S3Options S3Options `json:"s3Options"` + // Max allowed time for a backup to complete + // +operator-sdk:csv:customresourcedefinitions:type=spec + // +optional Timeout *metav1.Duration `json:"timeout"` - //+optional + // Max number of backup history to keep + // +operator-sdk:csv:customresourcedefinitions:type=spec + // +optional HistoryLimit *int32 `json:"historyLimit,omitempty"` - //+optional + // How frequently redis is polled for the BGSave status + // +operator-sdk:csv:customresourcedefinitions:type=spec + // +optional PollInterval *metav1.Duration `json:"pollInterval,omitempty"` + // Min size the backup must have to be considered successful + // +operator-sdk:csv:customresourcedefinitions:type=spec // +optional MinSize *string `json:"minSize,omitempty"` + // If true, backup execution is stopped + // +operator-sdk:csv:customresourcedefinitions:type=spec + // +optional + Pause *bool `json:"pause,omitempty"` } // Default implements defaulting for ShardedRedisBackuppec @@ -83,8 +105,14 @@ func (spec *ShardedRedisBackupSpec) GetMinSize() (uint64, error) { } type SSHOptions struct { - User string `json:"user"` + // SSH user + // +operator-sdk:csv:customresourcedefinitions:type=spec + User string `json:"user"` + // Reference to a Secret that contains the SSH private key + // +operator-sdk:csv:customresourcedefinitions:type=spec PrivateKeySecretRef corev1.LocalObjectReference `json:"privateKeySecretRef"` + // SSH port (default is 22) + // +operator-sdk:csv:customresourcedefinitions:type=spec // +optional Port *uint32 `json:"port,omitempty"` } @@ -96,9 +124,19 @@ func (opts *SSHOptions) Default() { } type S3Options struct { - Bucket string `json:"bucket"` - Path string `json:"path"` - Region string `json:"region"` + // S3 bucket name + // +operator-sdk:csv:customresourcedefinitions:type=spec + Bucket string `json:"bucket"` + // S3 path where backups should be uploaded + // +operator-sdk:csv:customresourcedefinitions:type=spec + Path string `json:"path"` + // AWS region + // +operator-sdk:csv:customresourcedefinitions:type=spec + Region string `json:"region"` + // Reference to a Secret tha contains credentials to access S3 API. The credentials + // must have the following permissions: s3:GetObject, s3:PutObject, and s3:ListBucket, + // s3:ListObjects, s3:PutObjectTagging. + // +operator-sdk:csv:customresourcedefinitions:type=spec CredentialsSecretRef corev1.LocalObjectReference `json:"credentialsSecretRef"` } @@ -173,16 +211,31 @@ func (bsl BackupStatusList) Swap(i, j int) { bsl[i], bsl[j] = bsl[j], bsl[i] } type BackupState string type BackupStatus struct { + // Name of the shard + // +operator-sdk:csv:customresourcedefinitions:type=spec Shard string `json:"shard"` - //+optional + // Redis server alias + // +operator-sdk:csv:customresourcedefinitions:type=spec + // +operator-sdk:csv:customresourcedefinitions:type=spec + // +optional ServerAlias *string `json:"serverAlias,omitempty"` - //+optional - ServerID *string `json:"serverID,omitempty"` + // Server host:port + // +operator-sdk:csv:customresourcedefinitions:type=spec + // +optional + ServerID *string `json:"serverID,omitempty"` + // Scheduled time for the backup to start + // +operator-sdk:csv:customresourcedefinitions:type=spec ScheduledFor metav1.Time `json:"scheduledFor"` - //+optional + // Actual time the backup starts + // +operator-sdk:csv:customresourcedefinitions:type=spec + // +optional StartedAt *metav1.Time `json:"startedAt,omitempty"` - Message string `json:"message"` - State BackupState `json:"state"` + // Descriptive message of the backup status + // +operator-sdk:csv:customresourcedefinitions:type=spec + Message string `json:"message"` + // Backup status + // +operator-sdk:csv:customresourcedefinitions:type=spec + State BackupState `json:"state"` } const ( From 619f724aed1d5b017e554c5cb346957ffe33281e Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Tue, 5 Sep 2023 12:04:55 +0200 Subject: [PATCH 11/43] Regenerate manifests --- .../saas.3scale.net_shardedredisbackups.yaml | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml b/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml index 7e11975b..a21c8f9a 100644 --- a/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml +++ b/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.10.0 + controller-gen.kubebuilder.io/version: v0.11.0 creationTimestamp: null name: shardedredisbackups.saas.3scale.net spec: @@ -37,19 +37,32 @@ spec: description: ShardedRedisBackupSpec defines the desired state of ShardedRedisBackup properties: dbFile: + description: Name of the dbfile in the redis instances type: string historyLimit: + description: Max number of backup history to keep format: int32 type: integer + minSize: + description: Min size the backup must have to be considered successful + type: string + pause: + description: If true, backup execution is stopped + type: boolean pollInterval: + description: How frequently redis is polled for the BGSave status type: string s3Options: + description: S3 storage options properties: bucket: + description: S3 bucket name type: string credentialsSecretRef: - description: LocalObjectReference contains enough information - to let you locate the referenced object inside the same namespace. + description: 'Reference to a Secret tha contains credentials to + access S3 API. The credentials must have the following permissions: + s3:GetObject, s3:PutObject, and s3:ListBucket, s3:ListObjects, + s3:PutObjectTagging.' properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names @@ -58,8 +71,10 @@ spec: type: object x-kubernetes-map-type: atomic path: + description: S3 path where backups should be uploaded type: string region: + description: AWS region type: string required: - bucket @@ -68,17 +83,21 @@ spec: - region type: object schedule: + description: Cron-like schedule specification type: string sentinelRef: + description: Reference to a sentinel instance type: string sshOptions: + description: SSH connection options properties: port: + description: SSH port (default is 22) format: int32 type: integer privateKeySecretRef: - description: LocalObjectReference contains enough information - to let you locate the referenced object inside the same namespace. + description: Reference to a Secret that contains the SSH private + key properties: name: description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names @@ -87,12 +106,14 @@ spec: type: object x-kubernetes-map-type: atomic user: + description: SSH user type: string required: - privateKeySecretRef - user type: object timeout: + description: Max allowed time for a backup to complete type: string required: - dbFile @@ -108,20 +129,27 @@ spec: items: properties: message: + description: Descriptive message of the backup status type: string scheduledFor: + description: Scheduled time for the backup to start format: date-time type: string serverAlias: + description: Redis server alias type: string serverID: + description: Server host:port type: string shard: + description: Name of the shard type: string startedAt: + description: Actual time the backup starts format: date-time type: string state: + description: Backup status type: string required: - message From 42fd9ff35f2d79305e0a792a82b2b68b650f3645 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Tue, 5 Sep 2023 12:07:23 +0200 Subject: [PATCH 12/43] Add assets and examples for redis backup --- .../backend/assets/redis-with-ssh/Dockerfile | 27 ++++++++++ .../assets/redis-with-ssh/authorized_keys | 1 + .../assets/redis-with-ssh/entrypoint.sh | 9 ++++ .../assets/redis-with-ssh/test-ssh-key | 49 +++++++++++++++++++ .../assets/redis-with-ssh/test-ssh-key.pub | 1 + examples/backend/redis-v4/backup.yaml | 24 +++++++++ examples/backend/redis-v4/redis.yaml | 16 ++++-- examples/backend/redis-v4/sentinel.yaml | 14 +++--- pkg/generators/redisshard/configmaps.go | 2 - pkg/redis/backup/s3upload.go | 2 +- 10 files changed, 132 insertions(+), 13 deletions(-) create mode 100644 examples/backend/assets/redis-with-ssh/Dockerfile create mode 100644 examples/backend/assets/redis-with-ssh/authorized_keys create mode 100644 examples/backend/assets/redis-with-ssh/entrypoint.sh create mode 100644 examples/backend/assets/redis-with-ssh/test-ssh-key create mode 100644 examples/backend/assets/redis-with-ssh/test-ssh-key.pub create mode 100644 examples/backend/redis-v4/backup.yaml diff --git a/examples/backend/assets/redis-with-ssh/Dockerfile b/examples/backend/assets/redis-with-ssh/Dockerfile new file mode 100644 index 00000000..51fe6d90 --- /dev/null +++ b/examples/backend/assets/redis-with-ssh/Dockerfile @@ -0,0 +1,27 @@ +# Use the redis:4.0.11-alpine as the base image +FROM redis:4.0.11-alpine + +# update the image and install necessary tools +RUN apk update && apk upgrade +RUN apk add --no-cache openssh gzip python3 py3-pip +RUN pip3 install --upgrade pip +RUN pip3 install awscli + + +# configure SSH to run on port 2222 +RUN sed -i 's/#Port 22/Port 2222/' /etc/ssh/sshd_config +RUN ssh-keygen -A +RUN mkdir -p /root/.ssh +COPY authorized_keys /root/.ssh/ +RUN chmod 700 /root/.ssh && chmod 600 /root/.ssh/authorized_keys + +# entrypoint script +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +USER root + +# Start both Redis and SSH using the entrypoint script +CMD ["/entrypoint.sh"] + + diff --git a/examples/backend/assets/redis-with-ssh/authorized_keys b/examples/backend/assets/redis-with-ssh/authorized_keys new file mode 100644 index 00000000..89b91d87 --- /dev/null +++ b/examples/backend/assets/redis-with-ssh/authorized_keys @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDC/OAdOUAwUQR7LYoJ+9hASl6/rVUhz80m8qXrduYucbzNV3+ZeFjVu2S6UQlRYrbgQJn2p0zyDdn3KzOfKjOiH+BudlZmZjk1BTPPs7UOC+R8eRyFm9Moa5210tNSk2tZoyzJ8reH4/PU7VKE5mrZOCgqaGai+Aq6yzW6Js2+ttcP2kwF+JgK3kHgmSk5spwPH1ZQPpDHGdg60dJsLa/ZdKt1pxbUBI9Lpx3dbx9ttT1mIc+iBP1iXZuGwQk63gei4eJjAgAxx6o274r3pecKGw6WbPRZ+TqPqnEALwhAYhyjr4TIBhF30LIuWgx7ixlYoIG3Fttf2X0ruHHC3NI34G6Ir+oLDZnYgauvRKA1vmNFeGlCsZCnhL3/RIDxUkLYnyNdkOe0iw/RLMYpLLz23ZZXgXNkIc4jAZO2Jbitsohok0KbgsV/PhXPMgcAPwYx6gHnu5G6MkgCZVcjjCtD7LZM3cxWn0mimIZ7ZBCxvp50aI5R4oEje6wBEEmFDsxUjsk+xhvc8HlLXf41mFWHHDdQgOj/UpsQao2fRdarn7Km7gqgd/haT3CHfXSky+kq45TnVPh2aAlCNQ9UNRAlHCnJ2qqfeZdAQ334XrBF7Cz9piYTnt1pzSTPbY1ubPjgkfnVobZuPiaJcaoO9PrM4e0UKk1M69Gf3S4i2BAOlw== roi@localhost.localdomain diff --git a/examples/backend/assets/redis-with-ssh/entrypoint.sh b/examples/backend/assets/redis-with-ssh/entrypoint.sh new file mode 100644 index 00000000..dd5ba20a --- /dev/null +++ b/examples/backend/assets/redis-with-ssh/entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -x + +# Start SSH service in the background +/usr/sbin/sshd -D & + +# Redirect Redis logs to /dev/stdout +redis-server /redis/redis.conf \ No newline at end of file diff --git a/examples/backend/assets/redis-with-ssh/test-ssh-key b/examples/backend/assets/redis-with-ssh/test-ssh-key new file mode 100644 index 00000000..18939faf --- /dev/null +++ b/examples/backend/assets/redis-with-ssh/test-ssh-key @@ -0,0 +1,49 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn +NhAAAAAwEAAQAAAgEAwvzgHTlAMFEEey2KCfvYQEpev61VIc/NJvKl63bmLnG8zVd/mXhY +1btkulEJUWK24ECZ9qdM8g3Z9ysznyozoh/gbnZWZmY5NQUzz7O1DgvkfHkchZvTKGudtd +LTUpNrWaMsyfK3h+Pz1O1ShOZq2TgoKmhmovgKuss1uibNvrbXD9pMBfiYCt5B4JkpObKc +Dx9WUD6QxxnYOtHSbC2v2XSrdacW1ASPS6cd3W8fbbU9ZiHPogT9Yl2bhsEJOt4HouHiYw +IAMceqNu+K96XnChsOlmz0Wfk6j6pxAC8IQGIco6+EyAYRd9CyLloMe4sZWKCBtxbbX9l9 +K7hxwtzSN+BuiK/qCw2Z2IGrr0SgNb5jRXhpQrGQp4S9/0SA8VJC2J8jXZDntIsP0SzGKS +y89t2WV4FzZCHOIwGTtiW4rbKIaJNCm4LFfz4VzzIHAD8GMeoB57uRujJIAmVXI4wrQ+y2 +TN3MVp9JopiGe2QQsb6edGiOUeKBI3usARBJhQ7MVI7JPsYb3PB5S13+NZhVhxw3UIDo/1 +KbEGqNn0XWq5+ypu4KoHf4Wk9wh310pMvpKuOU51T4dmgJQjUPVDUQJRwpydqqn3mXQEN9 ++F6wRews/aYmE57dac0kz22Nbmz44JH51aG2bj4miXGqDvT6zOHtFCpNTOvRn90uItgQDp +cAAAdQBbFbOwWxWzsAAAAHc3NoLXJzYQAAAgEAwvzgHTlAMFEEey2KCfvYQEpev61VIc/N +JvKl63bmLnG8zVd/mXhY1btkulEJUWK24ECZ9qdM8g3Z9ysznyozoh/gbnZWZmY5NQUzz7 +O1DgvkfHkchZvTKGudtdLTUpNrWaMsyfK3h+Pz1O1ShOZq2TgoKmhmovgKuss1uibNvrbX +D9pMBfiYCt5B4JkpObKcDx9WUD6QxxnYOtHSbC2v2XSrdacW1ASPS6cd3W8fbbU9ZiHPog +T9Yl2bhsEJOt4HouHiYwIAMceqNu+K96XnChsOlmz0Wfk6j6pxAC8IQGIco6+EyAYRd9Cy +LloMe4sZWKCBtxbbX9l9K7hxwtzSN+BuiK/qCw2Z2IGrr0SgNb5jRXhpQrGQp4S9/0SA8V +JC2J8jXZDntIsP0SzGKSy89t2WV4FzZCHOIwGTtiW4rbKIaJNCm4LFfz4VzzIHAD8GMeoB +57uRujJIAmVXI4wrQ+y2TN3MVp9JopiGe2QQsb6edGiOUeKBI3usARBJhQ7MVI7JPsYb3P +B5S13+NZhVhxw3UIDo/1KbEGqNn0XWq5+ypu4KoHf4Wk9wh310pMvpKuOU51T4dmgJQjUP +VDUQJRwpydqqn3mXQEN9+F6wRews/aYmE57dac0kz22Nbmz44JH51aG2bj4miXGqDvT6zO +HtFCpNTOvRn90uItgQDpcAAAADAQABAAACAAKAS/A3OiA3vLW33XwXVxqIA+1nOivKuIg5 +OdTJQOsQCW6Jb3P1ndHIMyA7qmWDR6DQQgvkQuUI2XN96JhO71/wmCNCmfCZ9j4tKq/z6Q +jxWekErZIff0dotvGPCJ2sXndpET+C9G3ZT87PTtYCvrhoOw/8hH0A36TCwU43NEn8pMnc +2V31XGa4aHRcoKw1ndqTFelK/1yMNji/5E9HnBu1o5gLdRE/H2gnuVPEe/Xm13BScgcV1i +Pp74RoAw0Jo6FRJcSwCU1xNM2Rt2iHTiqwR1Jnkec8/pDuNSZY6dvDUzgzp+lQsSv9a7ZB +VtlzkNNoJmO7gRDX/NpFibZGNAZBSm12xn6BXNSsagfeFCg4Rfw2bPRjZWGGxAavweoxhs +FZJz0ZjG/MAsCfOaip16Vv/cu+hTcVxB9wl2Qffv41gphHjWEdBBMwHui0odk1V2IdZUtK +DnopoN+7xhqBFbtVREupCajeyxDfUwZ+li2dRRyGM+71VW+1dmqN5UfHmzBg7mbNNY7b1g +zKNHb9eTz0WieRb1AmrHDC0EzHhKsC98duNWbK5ppscVsSE+SnuXiqrjrpzwiEPHsJcLT1 +5fkbe6ppg8S0EJX9njP8KiLan5/OrEGPUvyKF/rAUQ1UycwMkabTT+l/Y9wdt2N9otvOWp +6U9NZAaHp429ycK4n5AAABAHMBewjECz7XLD1SbL1EWnGG2sUTgOROkZ3h+THf5ACJWDtu +v7+JXa3teDyRQL+BzP8koIBR7JjwB2hHwKWbsyuvLPHrUvVFnTDdYk8sFPhhI8sOvKXL/V +7oHmrzg+nzrm7L3y4bzreB3PLwST0Mvqh53jxxEdG5futZbxxoL3ym1zUEsqcjact3uWv/ +roIrLhVg8Hctw8P7rfKvtI+JwlcIrLWfnhJcd6Xftsc7D1oYsr0bC6tZgkUQS0wYr6o1+3 +5EQyTGGC7ugSPdFA7JtW3UYenHpXE28m2w01aQdMURZBEGmiNMM+FO35ie8GdHPG70JVSb +igbHF3BQtA9CgWMAAAEBAOYEsK+g51jYjB5InRePuhXNFQV6/YQ1FBgsIDkNJrwOU5SoNH +30tYjoeV8QqO8tdHx2X6ukdl0AbUhyLoTYmtYVwQA0zezQn4bNGbhC3Ju7HinSF3ih4Qa3 +dKKZsox8UKReW0zxeKmuxFAA83Pe1K5d9jpqdaVSyt0TL+nlMIjjXbA9aEgTYAy6aOEpdE +meyrPA6dcykX9slaCt5xOiX1bZlLALP9EckOf/g6brcLn0G+cDF2r40lJuK37IopGkiHNl +SL7zkpsOk8ZBt5Tzdez9Gm8E7b0GfpcR7eEDkX2e1aF1mpZUBR8P4oNlcNC5BZ3Tomi6Xv +0QqnBjeVyo1skAAAEBANkDOjJTT3FN89ZvMlMcK8qOxO5PFf9OFCTO6xDyCwkZULWMMQOF +7TdHIZr2jOiCs3zInTRNR5ixsSUZZQhPhwRspw34xvFJVDpFteCTJ00j6R4tkhrrsFT8o/ +OyKyaggR+jbsBE5j0Nn3aXKcnTwOkyvH/eWw7s+HB9nKWLPlVkDQhL1eFG9hdm95rSdFSq +8s+Z/+h2UbDrU/UnAqiWmEtu4nwPzAtF6ulyzr5TQy5nXei9ci6tupXx191HPFc7Eb5H/I +mJtRoSQixOrQA47iUInn3WrqY5aAqFcOOTgHg4Tvrv4x8ctWEcGeUxG5wNpLeVujcsxAde +SYXU1JSgil8AAAAZcm9pQGxvY2FsaG9zdC5sb2NhbGRvbWFpbgEC +-----END OPENSSH PRIVATE KEY----- diff --git a/examples/backend/assets/redis-with-ssh/test-ssh-key.pub b/examples/backend/assets/redis-with-ssh/test-ssh-key.pub new file mode 100644 index 00000000..89b91d87 --- /dev/null +++ b/examples/backend/assets/redis-with-ssh/test-ssh-key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDC/OAdOUAwUQR7LYoJ+9hASl6/rVUhz80m8qXrduYucbzNV3+ZeFjVu2S6UQlRYrbgQJn2p0zyDdn3KzOfKjOiH+BudlZmZjk1BTPPs7UOC+R8eRyFm9Moa5210tNSk2tZoyzJ8reH4/PU7VKE5mrZOCgqaGai+Aq6yzW6Js2+ttcP2kwF+JgK3kHgmSk5spwPH1ZQPpDHGdg60dJsLa/ZdKt1pxbUBI9Lpx3dbx9ttT1mIc+iBP1iXZuGwQk63gei4eJjAgAxx6o274r3pecKGw6WbPRZ+TqPqnEALwhAYhyjr4TIBhF30LIuWgx7ixlYoIG3Fttf2X0ruHHC3NI34G6Ir+oLDZnYgauvRKA1vmNFeGlCsZCnhL3/RIDxUkLYnyNdkOe0iw/RLMYpLLz23ZZXgXNkIc4jAZO2Jbitsohok0KbgsV/PhXPMgcAPwYx6gHnu5G6MkgCZVcjjCtD7LZM3cxWn0mimIZ7ZBCxvp50aI5R4oEje6wBEEmFDsxUjsk+xhvc8HlLXf41mFWHHDdQgOj/UpsQao2fRdarn7Km7gqgd/haT3CHfXSky+kq45TnVPh2aAlCNQ9UNRAlHCnJ2qqfeZdAQ334XrBF7Cz9piYTnt1pzSTPbY1ubPjgkfnVobZuPiaJcaoO9PrM4e0UKk1M69Gf3S4i2BAOlw== roi@localhost.localdomain diff --git a/examples/backend/redis-v4/backup.yaml b/examples/backend/redis-v4/backup.yaml new file mode 100644 index 00000000..0e63a2a4 --- /dev/null +++ b/examples/backend/redis-v4/backup.yaml @@ -0,0 +1,24 @@ +apiVersion: saas.3scale.net/v1alpha1 +kind: ShardedRedisBackup +metadata: + name: backup + namespace: default +spec: + timeout: 5m + schedule: "* * * * *" + sentinelRef: sentinel + historyLimit: 2 + pollInterval: 10s + dbFile: /data/dump.rdb + minSize: 0 MB + sshOptions: + privateKeySecretRef: + name: redis-backup-ssh-private-key + user: root + port: 2222 + s3Options: + bucket: 3scale-saas-operator-backups-test + path: backups + region: us-east-1 + credentialsSecretRef: + name: aws-credentials diff --git a/examples/backend/redis-v4/redis.yaml b/examples/backend/redis-v4/redis.yaml index 7bdb8a2d..b37cc6eb 100644 --- a/examples/backend/redis-v4/redis.yaml +++ b/examples/backend/redis-v4/redis.yaml @@ -2,14 +2,24 @@ apiVersion: saas.3scale.net/v1alpha1 kind: RedisShard metadata: name: shard01 -spec: {} +spec: + command: /entrypoint.sh + image: + name: quay.io/roivaz/redis-with-ssh + tag: 4.0.11-alpine + pullPolicy: Always --- apiVersion: saas.3scale.net/v1alpha1 kind: RedisShard metadata: name: shard02 -spec: {} +spec: + command: /entrypoint.sh + image: + name: quay.io/roivaz/redis-with-ssh + tag: 4.0.11-alpine + pullPolicy: Always --- apiVersion: saas.3scale.net/v1alpha1 @@ -17,4 +27,4 @@ kind: RedisShard metadata: name: resque spec: - slaveCount: 0 \ No newline at end of file + slaveCount: 0 diff --git a/examples/backend/redis-v4/sentinel.yaml b/examples/backend/redis-v4/sentinel.yaml index ec0994fb..3a3dc6cc 100644 --- a/examples/backend/redis-v4/sentinel.yaml +++ b/examples/backend/redis-v4/sentinel.yaml @@ -3,16 +3,16 @@ kind: Sentinel metadata: name: sentinel spec: - replicas: 3 + replicas: 1 config: clusterTopology: # DNS should not be used in production. DNS is # just convenient for testing purposes shard01: - shard01-0: redis://redis-shard-shard01-0.redis-shard-shard01:6379 - shard01-1: redis://redis-shard-shard01-1.redis-shard-shard01:6379 - shard01-2: redis://redis-shard-shard01-2.redis-shard-shard01:6379 + redis-shard-shard01-0: redis://redis-shard-shard01-0.redis-shard-shard01:6379 + redis-shard-shard01-1: redis://redis-shard-shard01-1.redis-shard-shard01:6379 + redis-shard-shard01-2: redis://redis-shard-shard01-2.redis-shard-shard01:6379 shard02: - shard02-0: redis://redis-shard-shard02-0.redis-shard-shard02:6379 - shard02-1: redis://redis-shard-shard02-1.redis-shard-shard02:6379 - shard02-2: redis://redis-shard-shard02-2.redis-shard-shard02:6379 + redis-shard-shard02-0: redis://redis-shard-shard02-0.redis-shard-shard02:6379 + redis-shard-shard02-1: redis://redis-shard-shard02-1.redis-shard-shard02:6379 + redis-shard-shard02-2: redis://redis-shard-shard02-2.redis-shard-shard02:6379 diff --git a/pkg/generators/redisshard/configmaps.go b/pkg/generators/redisshard/configmaps.go index 82e61ee5..ac770a81 100644 --- a/pkg/generators/redisshard/configmaps.go +++ b/pkg/generators/redisshard/configmaps.go @@ -20,8 +20,6 @@ func (gen *Generator) redisConfigConfigMap() func() *corev1.ConfigMap { "redis.conf": heredoc.Doc(` slaveof 127.0.0.1 6379 tcp-keepalive 60 - daemonize yes - logfile /dev/stdout `), }, } diff --git a/pkg/redis/backup/s3upload.go b/pkg/redis/backup/s3upload.go index 2aee3c9e..b8c424d6 100644 --- a/pkg/redis/backup/s3upload.go +++ b/pkg/redis/backup/s3upload.go @@ -57,7 +57,7 @@ func (br *Runner) UploadBackup(ctx context.Context) error { fmt.Sprintf("gzip %s/%s", path.Dir(br.RedisDBFile), br.BackupFile()), // TODO: use awscli instead // AWS_ACCESS_KEY_ID=*** AWS_SECRET_ACCESS_KEY=*** s3cmd put /data/redis-backup---.rdb s3:////redis-backup---.rdb - fmt.Sprintf("%s=%s %s=%s %s=%s s3 cp %s/%s s3://%s/%s/%s", + fmt.Sprintf("%s=%s %s=%s %s=%s aws s3 cp %s/%s s3://%s/%s/%s", awsRegionEnvvar, br.AWSRegion, awsAccessKeyEnvvar, br.AWSAccessKeyID, awsSecretKeyEnvvar, br.AWSSecretAccessKey, From b8170fcfcf17ebe693136d034dfedc8d1238ff07 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Tue, 5 Sep 2023 12:25:02 +0200 Subject: [PATCH 13/43] Implment the 'pause' flag --- Makefile | 3 +++ api/v1alpha1/shardedredisbackup_types.go | 7 +++++- .../saas.3scale.net_shardedredisbackups.yaml | 6 ++--- controllers/shardedredisbackup_controller.go | 22 +++++++++++-------- examples/backend/redis-v4/backup.yaml | 1 - pkg/redis/backup/manager.go | 8 ++++--- 6 files changed, 30 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 53f98eb1..add2cff6 100644 --- a/Makefile +++ b/Makefile @@ -277,6 +277,9 @@ kind-undeploy: export KUBECONFIG = $(PWD)/kubeconfig kind-undeploy: ## Undeploy controller from the Kind K8s cluster $(KUSTOMIZE) build config/test | kubectl delete -f - +kind-deploy-backup-assets: + $(KUSTOMIZE) build config/test/redis-backups --load-restrictor LoadRestrictionsNone | kubectl apply -f - + ##@ Build Dependencies ## Location to install dependencies to diff --git a/api/v1alpha1/shardedredisbackup_types.go b/api/v1alpha1/shardedredisbackup_types.go index 83e429d7..30432468 100644 --- a/api/v1alpha1/shardedredisbackup_types.go +++ b/api/v1alpha1/shardedredisbackup_types.go @@ -38,7 +38,7 @@ const ( backupDefaultTimeout string = "10m" backupDefaultPollInterval string = "60s" backupDefaultSSHPort uint32 = 22 - backupDefaultMinSize string = "1 GB" + backupDefaultPause bool = false ) // ShardedRedisBackupSpec defines the desired state of ShardedRedisBackup @@ -93,6 +93,7 @@ func (spec *ShardedRedisBackupSpec) Default() { } spec.HistoryLimit = intOrDefault(spec.HistoryLimit, util.Pointer(backupHistoryLimit)) spec.MinSize = stringOrDefault(spec.MinSize, util.Pointer(backupDefaultMinSize)) + spec.Pause = boolOrDefault(spec.Pause, util.Pointer(backupDefaultPause)) spec.SSHOptions.Default() } @@ -236,6 +237,10 @@ type BackupStatus struct { // Backup status // +operator-sdk:csv:customresourcedefinitions:type=spec State BackupState `json:"state"` + // Final storage location of the backup + // +operator-sdk:csv:customresourcedefinitions:type=spec + // +optional + BackupFile *string `json:"backupFile"` } const ( diff --git a/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml b/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml index a21c8f9a..5e9f4d72 100644 --- a/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml +++ b/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml @@ -43,9 +43,6 @@ spec: description: Max number of backup history to keep format: int32 type: integer - minSize: - description: Min size the backup must have to be considered successful - type: string pause: description: If true, backup execution is stopped type: boolean @@ -128,6 +125,9 @@ spec: backups: items: properties: + backupFile: + description: Final storage location of the backup + type: string message: description: Descriptive message of the backup status type: string diff --git a/controllers/shardedredisbackup_controller.go b/controllers/shardedredisbackup_controller.go index ec92dabf..48e67b80 100644 --- a/controllers/shardedredisbackup_controller.go +++ b/controllers/shardedredisbackup_controller.go @@ -191,13 +191,14 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R continue } - if thread.Status().Finished { - if err := thread.Status().Error; err != nil { + if status := thread.Status(); status.Finished { + if err := status.Error; err != nil { b.State = saasv1alpha1.BackupFailedState b.Message = err.Error() } else { b.State = saasv1alpha1.BackupCompletedState b.Message = "backup complete" + b.BackupFile = &status.BackupFile } statusChanged = true } @@ -218,14 +219,17 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R } nextRun := schedule.Next(now) - statusChanged, err = r.reconcileBackupList(ctx, instance, nextRun, cluster.GetShardNames()) - if err != nil { - return reconcile.Result{}, err - } + // only actually add the schedule if pause == false + if !*instance.Spec.Pause { + statusChanged, err = r.reconcileBackupList(ctx, instance, nextRun, cluster.GetShardNames()) + if err != nil { + return reconcile.Result{}, err + } - if statusChanged { - err := r.Client.Status().Update(ctx, instance) - return ctrl.Result{}, err + if statusChanged { + err := r.Client.Status().Update(ctx, instance) + return ctrl.Result{}, err + } } // requeue for next schedule diff --git a/examples/backend/redis-v4/backup.yaml b/examples/backend/redis-v4/backup.yaml index 0e63a2a4..e1057fd7 100644 --- a/examples/backend/redis-v4/backup.yaml +++ b/examples/backend/redis-v4/backup.yaml @@ -10,7 +10,6 @@ spec: historyLimit: 2 pollInterval: 10s dbFile: /data/dump.rdb - minSize: 0 MB sshOptions: privateKeySecretRef: name: redis-backup-ssh-private-key diff --git a/pkg/redis/backup/manager.go b/pkg/redis/backup/manager.go index 2859972c..96091f14 100644 --- a/pkg/redis/backup/manager.go +++ b/pkg/redis/backup/manager.go @@ -35,9 +35,10 @@ type Runner struct { } type RunnerStatus struct { - Started bool - Finished bool - Error error + Started bool + Finished bool + Error error + BackupFile string } func ID(shard, alias string, ts time.Time) string { @@ -119,6 +120,7 @@ func (br *Runner) Start(parentCtx context.Context, l logr.Logger) error { case <-done: logger.V(1).Info("backup completed") br.status.Finished = true + br.status.BackupFile = fmt.Sprintf("s3://%s/%s/%s", br.S3Bucket, br.S3Path, br.BackupFileCompressed()) br.eventsCh <- event.GenericEvent{Object: br.Instance} return } From d71cb11b47afee9ca930ed9b9e26818cab128ea4 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Tue, 5 Sep 2023 13:16:13 +0200 Subject: [PATCH 14/43] Remove minSize field --- api/v1alpha1/shardedredisbackup_types.go | 14 -------------- controllers/shardedredisbackup_controller.go | 5 ----- examples/backend/redis-v4/backup.yaml | 1 + pkg/redis/backup/manager.go | 1 - pkg/redis/backup/tag.go | 5 ----- 5 files changed, 1 insertion(+), 25 deletions(-) diff --git a/api/v1alpha1/shardedredisbackup_types.go b/api/v1alpha1/shardedredisbackup_types.go index 30432468..f3e6852d 100644 --- a/api/v1alpha1/shardedredisbackup_types.go +++ b/api/v1alpha1/shardedredisbackup_types.go @@ -23,7 +23,6 @@ import ( "time" "github.com/3scale/saas-operator/pkg/util" - "github.com/dustin/go-humanize" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -70,10 +69,6 @@ type ShardedRedisBackupSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec // +optional PollInterval *metav1.Duration `json:"pollInterval,omitempty"` - // Min size the backup must have to be considered successful - // +operator-sdk:csv:customresourcedefinitions:type=spec - // +optional - MinSize *string `json:"minSize,omitempty"` // If true, backup execution is stopped // +operator-sdk:csv:customresourcedefinitions:type=spec // +optional @@ -92,19 +87,10 @@ func (spec *ShardedRedisBackupSpec) Default() { spec.PollInterval = &metav1.Duration{Duration: d} } spec.HistoryLimit = intOrDefault(spec.HistoryLimit, util.Pointer(backupHistoryLimit)) - spec.MinSize = stringOrDefault(spec.MinSize, util.Pointer(backupDefaultMinSize)) spec.Pause = boolOrDefault(spec.Pause, util.Pointer(backupDefaultPause)) spec.SSHOptions.Default() } -func (spec *ShardedRedisBackupSpec) GetMinSize() (uint64, error) { - if spec.MinSize == nil { - return humanize.ParseBytes(backupDefaultMinSize) - } else { - return humanize.ParseBytes(*spec.MinSize) - } -} - type SSHOptions struct { // SSH user // +operator-sdk:csv:customresourcedefinitions:type=spec diff --git a/controllers/shardedredisbackup_controller.go b/controllers/shardedredisbackup_controller.go index 48e67b80..37e79b44 100644 --- a/controllers/shardedredisbackup_controller.go +++ b/controllers/shardedredisbackup_controller.go @@ -125,10 +125,6 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R scheduledBackup := instance.Status.FindLastBackup(shard.Name, saasv1alpha1.BackupPendingState) if scheduledBackup != nil && scheduledBackup.ScheduledFor.Time.Before(time.Now()) { // add the backup runner thread - minSize, err := instance.Spec.GetMinSize() - if err != nil { - return ctrl.Result{}, fmt.Errorf("invalid 'spec.minSize' specification: %w", err) - } target := shard.GetSlavesRO()[0] runners = append(runners, &backup.Runner{ ShardName: shard.Name, @@ -136,7 +132,6 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R Timestamp: scheduledBackup.ScheduledFor.Time, Timeout: instance.Spec.Timeout.Duration, PollInterval: instance.Spec.PollInterval.Duration, - MinSize: minSize, RedisDBFile: instance.Spec.DBFile, Instance: instance, SSHUser: instance.Spec.SSHOptions.User, diff --git a/examples/backend/redis-v4/backup.yaml b/examples/backend/redis-v4/backup.yaml index e1057fd7..bcb3478a 100644 --- a/examples/backend/redis-v4/backup.yaml +++ b/examples/backend/redis-v4/backup.yaml @@ -21,3 +21,4 @@ spec: region: us-east-1 credentialsSecretRef: name: aws-credentials + pause: false diff --git a/pkg/redis/backup/manager.go b/pkg/redis/backup/manager.go index 96091f14..4f8eaa85 100644 --- a/pkg/redis/backup/manager.go +++ b/pkg/redis/backup/manager.go @@ -18,7 +18,6 @@ type Runner struct { Server *sharded.RedisServer Timeout time.Duration PollInterval time.Duration - MinSize uint64 Timestamp time.Time RedisDBFile string SSHUser string diff --git a/pkg/redis/backup/tag.go b/pkg/redis/backup/tag.go index 9af0a684..91525287 100644 --- a/pkg/redis/backup/tag.go +++ b/pkg/redis/backup/tag.go @@ -59,11 +59,6 @@ func (br *Runner) TagBackup(ctx context.Context) error { return fmt.Errorf("last backup %s has different key than expected (%s)", *last.Key, br.BackupFileS3Path()) } - // check backup size of last (given a size passed as threshold in the CR) - if last.Size < int64(br.MinSize) { - return fmt.Errorf("last backup %s is smaller that declared min size of %d", *last.Key, br.MinSize) - } - tags := []types.Tag{ {Key: aws.String("Layer"), Value: aws.String("bck-storage")}, {Key: aws.String("App"), Value: aws.String("Backend")}, From de370aeed93800f01fd1322423d237b2eb40aaed Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Wed, 6 Sep 2023 11:46:13 +0200 Subject: [PATCH 15/43] go mod tidy --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index 03177e60..1eaa5831 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.18.37 github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 github.com/davecgh/go-spew v1.1.1 - github.com/dustin/go-humanize v1.0.0 github.com/envoyproxy/go-control-plane v0.11.0 github.com/evanphx/json-patch v5.6.0+incompatible github.com/external-secrets/external-secrets v0.8.1 diff --git a/go.sum b/go.sum index 021bb203..0bfa8aae 100644 --- a/go.sum +++ b/go.sum @@ -141,8 +141,6 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= From a5cd55320cc67be1899946ca737a690ee9ed0a37 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Thu, 7 Sep 2023 16:42:10 +0200 Subject: [PATCH 16/43] Add metrics and expose more info in the status --- api/v1alpha1/shardedredisbackup_types.go | 8 ++ api/v1alpha1/zz_generated.deepcopy.go | 73 +++++++++++++++++++ .../saas.3scale.net_shardedredisbackups.yaml | 8 ++ controllers/shardedredisbackup_controller.go | 2 + controllers/twemproxyconfig_controller.go | 3 +- pkg/redis/backup/manager.go | 4 + pkg/redis/backup/metrics.go | 56 ++++++++++++++ pkg/redis/backup/tag.go | 2 + 8 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 pkg/redis/backup/metrics.go diff --git a/api/v1alpha1/shardedredisbackup_types.go b/api/v1alpha1/shardedredisbackup_types.go index f3e6852d..b6be92a5 100644 --- a/api/v1alpha1/shardedredisbackup_types.go +++ b/api/v1alpha1/shardedredisbackup_types.go @@ -217,6 +217,10 @@ type BackupStatus struct { // +operator-sdk:csv:customresourcedefinitions:type=spec // +optional StartedAt *metav1.Time `json:"startedAt,omitempty"` + // when the backup was completed + // +operator-sdk:csv:customresourcedefinitions:type=spec + // +optional + FinishedAt *metav1.Time `json:"finishedAt,omitempty"` // Descriptive message of the backup status // +operator-sdk:csv:customresourcedefinitions:type=spec Message string `json:"message"` @@ -227,6 +231,10 @@ type BackupStatus struct { // +operator-sdk:csv:customresourcedefinitions:type=spec // +optional BackupFile *string `json:"backupFile"` + // Stored size of the backup in bytes + // +operator-sdk:csv:customresourcedefinitions:type=spec + // +optional + BackupSize *int64 `json:"backupSize"` } const ( diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 23d2668c..35aa6ab5 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -702,6 +702,20 @@ func (in *BackupStatus) DeepCopyInto(out *BackupStatus) { in, out := &in.StartedAt, &out.StartedAt *out = (*in).DeepCopy() } + if in.FinishedAt != nil { + in, out := &in.FinishedAt, &out.FinishedAt + *out = (*in).DeepCopy() + } + if in.BackupFile != nil { + in, out := &in.BackupFile, &out.BackupFile + *out = new(string) + **out = **in + } + if in.BackupSize != nil { + in, out := &in.BackupSize, &out.BackupSize + *out = new(int64) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStatus. @@ -2260,6 +2274,11 @@ func (in *RedisShardSpec) DeepCopyInto(out *RedisShardSpec) { *out = new(int32) **out = **in } + if in.Command != nil { + in, out := &in.Command, &out.Command + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RedisShardSpec. @@ -2383,6 +2402,22 @@ func (in *Runtime) DeepCopy() *Runtime { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *S3Options) DeepCopyInto(out *S3Options) { + *out = *in + out.CredentialsSecretRef = in.CredentialsSecretRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new S3Options. +func (in *S3Options) DeepCopy() *S3Options { + if in == nil { + return nil + } + out := new(S3Options) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SMTPSpec) DeepCopyInto(out *SMTPSpec) { *out = *in @@ -2410,6 +2445,27 @@ func (in *SMTPSpec) DeepCopy() *SMTPSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SSHOptions) DeepCopyInto(out *SSHOptions) { + *out = *in + out.PrivateKeySecretRef = in.PrivateKeySecretRef + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(uint32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SSHOptions. +func (in *SSHOptions) DeepCopy() *SSHOptions { + if in == nil { + return nil + } + out := new(SSHOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SearchServerSpec) DeepCopyInto(out *SearchServerSpec) { *out = *in @@ -2790,11 +2846,28 @@ func (in *ShardedRedisBackupList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ShardedRedisBackupSpec) DeepCopyInto(out *ShardedRedisBackupSpec) { *out = *in + in.SSHOptions.DeepCopyInto(&out.SSHOptions) + out.S3Options = in.S3Options + if in.Timeout != nil { + in, out := &in.Timeout, &out.Timeout + *out = new(metav1.Duration) + **out = **in + } if in.HistoryLimit != nil { in, out := &in.HistoryLimit, &out.HistoryLimit *out = new(int32) **out = **in } + if in.PollInterval != nil { + in, out := &in.PollInterval, &out.PollInterval + *out = new(metav1.Duration) + **out = **in + } + if in.Pause != nil { + in, out := &in.Pause, &out.Pause + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShardedRedisBackupSpec. diff --git a/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml b/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml index 5e9f4d72..759a17df 100644 --- a/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml +++ b/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml @@ -128,6 +128,14 @@ spec: backupFile: description: Final storage location of the backup type: string + backupSize: + description: Stored size of the backup in bytes + format: int64 + type: integer + finishedAt: + description: when the backup was completed + format: date-time + type: string message: description: Descriptive message of the backup status type: string diff --git a/controllers/shardedredisbackup_controller.go b/controllers/shardedredisbackup_controller.go index 37e79b44..1427c1f1 100644 --- a/controllers/shardedredisbackup_controller.go +++ b/controllers/shardedredisbackup_controller.go @@ -194,6 +194,8 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R b.State = saasv1alpha1.BackupCompletedState b.Message = "backup complete" b.BackupFile = &status.BackupFile + b.BackupSize = &status.BackupSize + b.FinishedAt = &metav1.Time{Time: status.FinishedAt} } statusChanged = true } diff --git a/controllers/twemproxyconfig_controller.go b/controllers/twemproxyconfig_controller.go index 94caf727..bf86ea96 100644 --- a/controllers/twemproxyconfig_controller.go +++ b/controllers/twemproxyconfig_controller.go @@ -34,7 +34,6 @@ import ( grafanav1alpha1 "github.com/grafana-operator/grafana-operator/v4/api/integreatly/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" @@ -145,7 +144,7 @@ func (r *TwemproxyConfigReconciler) reconcileConfigMap(ctx context.Context, owne current := &corev1.ConfigMap{} err := r.Client.Get(ctx, util.ObjectKey(desired), current) if err != nil { - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { // Create if err := controllerutil.SetControllerReference(owner, desired, r.Scheme); err != nil { return "", err diff --git a/pkg/redis/backup/manager.go b/pkg/redis/backup/manager.go index 4f8eaa85..9e8e3ec5 100644 --- a/pkg/redis/backup/manager.go +++ b/pkg/redis/backup/manager.go @@ -38,6 +38,8 @@ type RunnerStatus struct { Finished bool Error error BackupFile string + BackupSize int64 + FinishedAt time.Time } func ID(shard, alias string, ts time.Time) string { @@ -120,7 +122,9 @@ func (br *Runner) Start(parentCtx context.Context, l logr.Logger) error { logger.V(1).Info("backup completed") br.status.Finished = true br.status.BackupFile = fmt.Sprintf("s3://%s/%s/%s", br.S3Bucket, br.S3Path, br.BackupFileCompressed()) + br.status.FinishedAt = time.Now() br.eventsCh <- event.GenericEvent{Object: br.Instance} + br.publishMetrics() return } } diff --git a/pkg/redis/backup/metrics.go b/pkg/redis/backup/metrics.go new file mode 100644 index 00000000..2bc82d83 --- /dev/null +++ b/pkg/redis/backup/metrics.go @@ -0,0 +1,56 @@ +package backup + +import ( + "math" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +// metrics +var ( + backupSize = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "size", + Namespace: "saas_redis_backup", + Help: `"size of the latest backup in bytes"`, + }, + []string{"shard"}) + backupFailures = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "failures", + Namespace: "saas_redis_backup", + Help: `"total number of backup failures"`, + }, + []string{"shard"}) + + backupDuration = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "duration", + Namespace: "saas_redis_backup", + Help: `"seconds it took to complete the backup"`, + }, + []string{"shard"}) +) + +func init() { + // Register backup metrics with the global prometheus registry + metrics.Registry.MustRegister( + backupSize, backupFailures, backupDuration, + ) +} + +func (r *Runner) publishMetrics() { + if r.status.Error != nil { + backupSize.With(prometheus.Labels{"shard": r.ShardName}).Set(float64(0)) + backupFailures.With(prometheus.Labels{"shard": r.ShardName}).Inc() + } else { + backupSize.With(prometheus.Labels{"shard": r.ShardName}).Set(float64(r.status.BackupSize)) + backupDuration.With(prometheus.Labels{"shard": r.ShardName}).Set(math.Round(r.status.FinishedAt.Sub(r.Timestamp).Seconds())) + // ensure the failure counter is initialized + if err := backupFailures.With(prometheus.Labels{"shard": r.ShardName}).Write(&dto.Metric{}); err != nil { + backupFailures.With(prometheus.Labels{"shard": r.ShardName}).Add(0) + } + } +} diff --git a/pkg/redis/backup/tag.go b/pkg/redis/backup/tag.go index 91525287..dcaab692 100644 --- a/pkg/redis/backup/tag.go +++ b/pkg/redis/backup/tag.go @@ -58,6 +58,8 @@ func (br *Runner) TagBackup(ctx context.Context) error { if br.BackupFileS3Path() != *last.Key { return fmt.Errorf("last backup %s has different key than expected (%s)", *last.Key, br.BackupFileS3Path()) } + // store backup size + br.status.BackupSize = last.Size tags := []types.Tag{ {Key: aws.String("Layer"), Value: aws.String("bck-storage")}, From 4798a37a3556e68fa2a7f6b4fbe763b505fb5bf0 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Wed, 13 Sep 2023 17:47:35 +0200 Subject: [PATCH 17/43] Use minio to tests s3 calls in local dev env --- .gitignore | 3 ++ Makefile | 2 +- api/v1alpha1/shardedredisbackup_types.go | 4 +++ .../saas.3scale.net_shardedredisbackups.yaml | 4 +++ config/test/redis-backups/kustomization.yaml | 22 ++++++++++++++ controllers/shardedredisbackup_controller.go | 1 + examples/backend/redis-v4/backup.yaml | 3 +- go.mod | 5 ++-- go.sum | 14 --------- pkg/redis/backup/manager.go | 1 + pkg/redis/backup/s3upload.go | 11 ++++++- pkg/redis/backup/tag.go | 30 ++++++++++++++++--- 12 files changed, 76 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 5d437d60..96c4e21d 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ kubeconfig .vscode/configurationCache.log .vscode/dryrun.log .vscode/targets.log + +# helm charts +**/charts diff --git a/Makefile b/Makefile index add2cff6..c83d06fe 100644 --- a/Makefile +++ b/Makefile @@ -278,7 +278,7 @@ kind-undeploy: ## Undeploy controller from the Kind K8s cluster $(KUSTOMIZE) build config/test | kubectl delete -f - kind-deploy-backup-assets: - $(KUSTOMIZE) build config/test/redis-backups --load-restrictor LoadRestrictionsNone | kubectl apply -f - + $(KUSTOMIZE) build config/test/redis-backups --load-restrictor LoadRestrictionsNone --enable-helm | kubectl apply -f - ##@ Build Dependencies diff --git a/api/v1alpha1/shardedredisbackup_types.go b/api/v1alpha1/shardedredisbackup_types.go index b6be92a5..b602a2c4 100644 --- a/api/v1alpha1/shardedredisbackup_types.go +++ b/api/v1alpha1/shardedredisbackup_types.go @@ -125,6 +125,10 @@ type S3Options struct { // s3:ListObjects, s3:PutObjectTagging. // +operator-sdk:csv:customresourcedefinitions:type=spec CredentialsSecretRef corev1.LocalObjectReference `json:"credentialsSecretRef"` + // Optionally use a custom s3 service endpoint. Useful for testing with Minio. + // +optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + ServiceEndpoint *string `json:"serviceEndpoint"` } // ShardedRedisBackupStatus defines the observed state of ShardedRedisBackup diff --git a/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml b/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml index 759a17df..1075a23c 100644 --- a/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml +++ b/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml @@ -73,6 +73,10 @@ spec: region: description: AWS region type: string + serviceEndpoint: + description: Optionally use a custom s3 service endpoint. Useful + for testing with Minio. + type: string required: - bucket - credentialsSecretRef diff --git a/config/test/redis-backups/kustomization.yaml b/config/test/redis-backups/kustomization.yaml index 7853314e..ef124f5d 100644 --- a/config/test/redis-backups/kustomization.yaml +++ b/config/test/redis-backups/kustomization.yaml @@ -2,6 +2,25 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: default +helmCharts: + - name: minio + repo: https://charts.min.io + version: v5.0.13 + releaseName: minio + namespace: default + valuesInline: + mode: standalone + persistence: + enabled: false + resources: + requests: + memory: 1Mi + buckets: + - name: my-bucket + purge: true + versioning: false + rootUser: admin + rootPassword: admin123 generatorOptions: disableNameSuffixHash: true @@ -14,3 +33,6 @@ secretGenerator: - name: aws-credentials envs: - ../../../examples/backend/assets/s3/credentials.env + # literals: + # - AWS_ACCESS_KEY_ID=admin + # - AWS_SECRET_ACCESS_KEY=admin123 diff --git a/controllers/shardedredisbackup_controller.go b/controllers/shardedredisbackup_controller.go index 1427c1f1..4161e057 100644 --- a/controllers/shardedredisbackup_controller.go +++ b/controllers/shardedredisbackup_controller.go @@ -142,6 +142,7 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R AWSAccessKeyID: string(awsCredentials.Data[saasv1alpha1.AWSAccessKeyID_SecretKey]), AWSSecretAccessKey: string(awsCredentials.Data[saasv1alpha1.AWSSecretAccessKey_SecretKey]), AWSRegion: instance.Spec.S3Options.Region, + AWSS3Endpoint: instance.Spec.S3Options.ServiceEndpoint, }) scheduledBackup.ServerAlias = util.Pointer(target.GetAlias()) scheduledBackup.ServerID = util.Pointer(target.ID()) diff --git a/examples/backend/redis-v4/backup.yaml b/examples/backend/redis-v4/backup.yaml index bcb3478a..e39e6b2e 100644 --- a/examples/backend/redis-v4/backup.yaml +++ b/examples/backend/redis-v4/backup.yaml @@ -16,9 +16,10 @@ spec: user: root port: 2222 s3Options: - bucket: 3scale-saas-operator-backups-test + bucket: my-bucket path: backups region: us-east-1 credentialsSecretRef: name: aws-credentials + serviceEndpoint: http://minio.default.svc.cluster.local:9000 pause: false diff --git a/go.mod b/go.mod index 1eaa5831..c76a880c 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/3scale-ops/basereconciler v0.3.1 github.com/3scale-ops/marin3r v0.12.2 github.com/MakeNowJust/heredoc v1.0.0 - github.com/aws/aws-sdk-go v1.44.220 + github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/config v1.18.37 github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 github.com/davecgh/go-spew v1.1.1 @@ -26,6 +26,7 @@ require ( github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.42.1 github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_model v0.3.0 github.com/robfig/cron/v3 v3.0.1 github.com/tektoncd/pipeline v0.49.0 go.uber.org/zap v1.24.0 @@ -43,7 +44,6 @@ require ( require ( contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.21.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.35 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect @@ -100,7 +100,6 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/openshift/api v0.0.0-20220715133027-dab5b363ebd1 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/prometheus/statsd_exporter v0.21.0 // indirect diff --git a/go.sum b/go.sum index 0bfa8aae..774ff434 100644 --- a/go.sum +++ b/go.sum @@ -58,8 +58,6 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.44.220 h1:yAj99qAt0Htjle9Up3DglgHfOP77lmFPrElA4jKnrBo= -github.com/aws/aws-sdk-go v1.44.220/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= @@ -329,7 +327,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -484,7 +481,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -554,7 +550,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -598,8 +593,6 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -622,7 +615,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -675,15 +667,11 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -695,7 +683,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -752,7 +739,6 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/redis/backup/manager.go b/pkg/redis/backup/manager.go index 9e8e3ec5..fec7fbd8 100644 --- a/pkg/redis/backup/manager.go +++ b/pkg/redis/backup/manager.go @@ -28,6 +28,7 @@ type Runner struct { AWSAccessKeyID string AWSSecretAccessKey string AWSRegion string + AWSS3Endpoint *string eventsCh chan event.GenericEvent cancel context.CancelFunc status RunnerStatus diff --git a/pkg/redis/backup/s3upload.go b/pkg/redis/backup/s3upload.go index b8c424d6..1a5a8699 100644 --- a/pkg/redis/backup/s3upload.go +++ b/pkg/redis/backup/s3upload.go @@ -6,6 +6,7 @@ import ( "net" "path" "strconv" + "strings" "time" "golang.org/x/crypto/ssh" @@ -47,6 +48,13 @@ func (br *Runner) BackupFileS3Path() string { func (br *Runner) UploadBackup(ctx context.Context) error { logger := log.FromContext(ctx, "function", "(br *Runner) UploadBackup()") + var awsBaseCommand string + if br.AWSS3Endpoint != nil { + awsBaseCommand = strings.Join([]string{"aws", "--endpoint-url", *br.AWSS3Endpoint}, " ") + } else { + awsBaseCommand = "aws" + } + var commands = []string{ // mv /data/dump.rdb /data/redis-backup---.rdb fmt.Sprintf("mv %s %s/%s", @@ -57,10 +65,11 @@ func (br *Runner) UploadBackup(ctx context.Context) error { fmt.Sprintf("gzip %s/%s", path.Dir(br.RedisDBFile), br.BackupFile()), // TODO: use awscli instead // AWS_ACCESS_KEY_ID=*** AWS_SECRET_ACCESS_KEY=*** s3cmd put /data/redis-backup---.rdb s3:////redis-backup---.rdb - fmt.Sprintf("%s=%s %s=%s %s=%s aws s3 cp %s/%s s3://%s/%s/%s", + fmt.Sprintf("%s=%s %s=%s %s=%s %s s3 cp %s/%s s3://%s/%s/%s", awsRegionEnvvar, br.AWSRegion, awsAccessKeyEnvvar, br.AWSAccessKeyID, awsSecretKeyEnvvar, br.AWSSecretAccessKey, + awsBaseCommand, path.Dir(br.RedisDBFile), br.BackupFileCompressed(), br.S3Bucket, br.S3Path, br.BackupFileCompressed(), ), diff --git a/pkg/redis/backup/tag.go b/pkg/redis/backup/tag.go index dcaab692..1b8ad635 100644 --- a/pkg/redis/backup/tag.go +++ b/pkg/redis/backup/tag.go @@ -6,10 +6,10 @@ import ( "os" "sort" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/aws-sdk-go/aws" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -21,10 +21,32 @@ func (br *Runner) TagBackup(ctx context.Context) error { os.Setenv(awsSecretKeyEnvvar, br.AWSSecretAccessKey) os.Setenv(awsRegionEnvvar, br.AWSRegion) - cfg, err := config.LoadDefaultConfig(ctx) - if err != nil { - return err + var cfg aws.Config + var err error + + if br.AWSS3Endpoint != nil { + resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...any) (aws.Endpoint, error) { + return aws.Endpoint{ + PartitionID: "aws", + URL: *br.AWSS3Endpoint, + SigningRegion: br.AWSRegion, + HostnameImmutable: true, + }, nil + }) + + cfg, err = config.LoadDefaultConfig(ctx, + config.WithEndpointResolverWithOptions(resolver), + ) + if err != nil { + return err + } + } else { + cfg, err = config.LoadDefaultConfig(ctx) + if err != nil { + return err + } } + client := s3.NewFromConfig(cfg) // get backups of current day From b5795fcce4953b6d42ab71b7b8fdb3eb36fc5734 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Wed, 13 Sep 2023 17:48:41 +0200 Subject: [PATCH 18/43] Optionally use sudo --- Makefile | 8 +++++++- api/v1alpha1/shardedredisbackup_types.go | 7 +++++++ .../saas.3scale.net_shardedredisbackups.yaml | 3 +++ config/test/redis-backups/kustomization.yaml | 10 ++++------ controllers/shardedredisbackup_controller.go | 3 +++ examples/backend/redis-v4/backup.yaml | 3 ++- examples/backend/redis-v4/redis.yaml | 6 ++---- pkg/redis/backup/manager.go | 1 + pkg/redis/backup/s3upload.go | 3 +++ .../assets/redis-with-ssh/Dockerfile | 16 +++++++++------- .../assets/redis-with-ssh/authorized_keys | 0 .../assets/redis-with-ssh/entrypoint.sh | 0 .../assets/redis-with-ssh/test-ssh-key | 0 .../assets/redis-with-ssh/test-ssh-key.pub | 0 14 files changed, 41 insertions(+), 19 deletions(-) rename {examples/backend => test}/assets/redis-with-ssh/Dockerfile (54%) rename {examples/backend => test}/assets/redis-with-ssh/authorized_keys (100%) rename {examples/backend => test}/assets/redis-with-ssh/entrypoint.sh (100%) rename {examples/backend => test}/assets/redis-with-ssh/test-ssh-key (100%) rename {examples/backend => test}/assets/redis-with-ssh/test-ssh-key.pub (100%) diff --git a/Makefile b/Makefile index c83d06fe..8524ba95 100644 --- a/Makefile +++ b/Makefile @@ -277,9 +277,15 @@ kind-undeploy: export KUBECONFIG = $(PWD)/kubeconfig kind-undeploy: ## Undeploy controller from the Kind K8s cluster $(KUSTOMIZE) build config/test | kubectl delete -f - -kind-deploy-backup-assets: +kind-deploy-backup-assets: export KUBECONFIG = $(PWD)/kubeconfig +kind-deploy-backup-assets: kind-load-redis-with-ssh $(KUSTOMIZE) build config/test/redis-backups --load-restrictor LoadRestrictionsNone --enable-helm | kubectl apply -f - +REDIS_WITH_SSH_IMG = redis-with-ssh:4.0.11-alpine +kind-load-redis-with-ssh: + docker build -t $(REDIS_WITH_SSH_IMG) test/assets/redis-with-ssh + $(KIND) load docker-image $(REDIS_WITH_SSH_IMG) + ##@ Build Dependencies ## Location to install dependencies to diff --git a/api/v1alpha1/shardedredisbackup_types.go b/api/v1alpha1/shardedredisbackup_types.go index b602a2c4..0a14fe50 100644 --- a/api/v1alpha1/shardedredisbackup_types.go +++ b/api/v1alpha1/shardedredisbackup_types.go @@ -102,12 +102,19 @@ type SSHOptions struct { // +operator-sdk:csv:customresourcedefinitions:type=spec // +optional Port *uint32 `json:"port,omitempty"` + // Use sudo to execute commands agains the remote host + // +operator-sdk:csv:customresourcedefinitions:type=spec + // +optional + Sudo *bool `json:"sudo,omitempty"` } func (opts *SSHOptions) Default() { if opts.Port == nil { opts.Port = util.Pointer(backupDefaultSSHPort) } + if opts.Sudo == nil { + opts.Sudo = util.Pointer(false) + } } type S3Options struct { diff --git a/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml b/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml index 1075a23c..9467d7c1 100644 --- a/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml +++ b/config/crd/bases/saas.3scale.net_shardedredisbackups.yaml @@ -106,6 +106,9 @@ spec: type: string type: object x-kubernetes-map-type: atomic + sudo: + description: Use sudo to execute commands agains the remote host + type: boolean user: description: SSH user type: string diff --git a/config/test/redis-backups/kustomization.yaml b/config/test/redis-backups/kustomization.yaml index ef124f5d..20a1d39d 100644 --- a/config/test/redis-backups/kustomization.yaml +++ b/config/test/redis-backups/kustomization.yaml @@ -29,10 +29,8 @@ secretGenerator: - name: redis-backup-ssh-private-key type: kubernetes.io/ssh-auth files: - - ssh-privatekey=../../../examples/backend/assets/redis-with-ssh/test-ssh-key + - ssh-privatekey=../../../test/assets/redis-with-ssh/test-ssh-key - name: aws-credentials - envs: - - ../../../examples/backend/assets/s3/credentials.env - # literals: - # - AWS_ACCESS_KEY_ID=admin - # - AWS_SECRET_ACCESS_KEY=admin123 + literals: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=admin123 diff --git a/controllers/shardedredisbackup_controller.go b/controllers/shardedredisbackup_controller.go index 4161e057..c182c7b1 100644 --- a/controllers/shardedredisbackup_controller.go +++ b/controllers/shardedredisbackup_controller.go @@ -119,6 +119,8 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R // ----- Phase 2: run pending backups ----- // ---------------------------------------- + // TODO: hanlde error when no available RO slaves + statusChanged := false runners := make([]threads.RunnableThread, 0, len(cluster.Shards)) for _, shard := range cluster.Shards { @@ -137,6 +139,7 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R SSHUser: instance.Spec.SSHOptions.User, SSHKey: string(sshPrivateKey.Data[corev1.SSHAuthPrivateKey]), SSHPort: *instance.Spec.SSHOptions.Port, + SSHSudo: *instance.Spec.SSHOptions.Sudo, S3Bucket: instance.Spec.S3Options.Bucket, S3Path: instance.Spec.S3Options.Path, AWSAccessKeyID: string(awsCredentials.Data[saasv1alpha1.AWSAccessKeyID_SecretKey]), diff --git a/examples/backend/redis-v4/backup.yaml b/examples/backend/redis-v4/backup.yaml index e39e6b2e..30fb6dff 100644 --- a/examples/backend/redis-v4/backup.yaml +++ b/examples/backend/redis-v4/backup.yaml @@ -13,7 +13,8 @@ spec: sshOptions: privateKeySecretRef: name: redis-backup-ssh-private-key - user: root + user: docker + sudo: true port: 2222 s3Options: bucket: my-bucket diff --git a/examples/backend/redis-v4/redis.yaml b/examples/backend/redis-v4/redis.yaml index b37cc6eb..0fbdb65d 100644 --- a/examples/backend/redis-v4/redis.yaml +++ b/examples/backend/redis-v4/redis.yaml @@ -5,9 +5,8 @@ metadata: spec: command: /entrypoint.sh image: - name: quay.io/roivaz/redis-with-ssh + name: redis-with-ssh tag: 4.0.11-alpine - pullPolicy: Always --- apiVersion: saas.3scale.net/v1alpha1 @@ -17,9 +16,8 @@ metadata: spec: command: /entrypoint.sh image: - name: quay.io/roivaz/redis-with-ssh + name: redis-with-ssh tag: 4.0.11-alpine - pullPolicy: Always --- apiVersion: saas.3scale.net/v1alpha1 diff --git a/pkg/redis/backup/manager.go b/pkg/redis/backup/manager.go index fec7fbd8..915120f9 100644 --- a/pkg/redis/backup/manager.go +++ b/pkg/redis/backup/manager.go @@ -23,6 +23,7 @@ type Runner struct { SSHUser string SSHKey string SSHPort uint32 + SSHSudo bool S3Bucket string S3Path string AWSAccessKeyID string diff --git a/pkg/redis/backup/s3upload.go b/pkg/redis/backup/s3upload.go index 1a5a8699..e1fcf65e 100644 --- a/pkg/redis/backup/s3upload.go +++ b/pkg/redis/backup/s3upload.go @@ -76,6 +76,9 @@ func (br *Runner) UploadBackup(ctx context.Context) error { } for _, command := range commands { + if br.SSHSudo { + command = "sudo " + command + } logger.V(1).Info(fmt.Sprintf("running command '%s' on %s:%d", command, br.Server.GetHost(), br.SSHPort)) output, err := remoteRun(ctx, br.SSHUser, br.Server.GetHost(), strconv.Itoa(int(br.SSHPort)), br.SSHKey, command) if output != "" { diff --git a/examples/backend/assets/redis-with-ssh/Dockerfile b/test/assets/redis-with-ssh/Dockerfile similarity index 54% rename from examples/backend/assets/redis-with-ssh/Dockerfile rename to test/assets/redis-with-ssh/Dockerfile index 51fe6d90..c9b4310a 100644 --- a/examples/backend/assets/redis-with-ssh/Dockerfile +++ b/test/assets/redis-with-ssh/Dockerfile @@ -3,17 +3,21 @@ FROM redis:4.0.11-alpine # update the image and install necessary tools RUN apk update && apk upgrade -RUN apk add --no-cache openssh gzip python3 py3-pip +RUN apk add --no-cache openssh gzip python3 py3-pip sudo RUN pip3 install --upgrade pip RUN pip3 install awscli - # configure SSH to run on port 2222 RUN sed -i 's/#Port 22/Port 2222/' /etc/ssh/sshd_config + +# setup "docker" user with sudo +RUN adduser --disabled-password docker && passwd docker -d '' +RUN echo "docker ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers RUN ssh-keygen -A -RUN mkdir -p /root/.ssh -COPY authorized_keys /root/.ssh/ -RUN chmod 700 /root/.ssh && chmod 600 /root/.ssh/authorized_keys +RUN mkdir -p /home/docker/.ssh +COPY authorized_keys /home/docker/.ssh/ +RUN chmod 700 /home/docker/.ssh && chmod 600 /home/docker/.ssh/authorized_keys +RUN chown -R docker:docker /home/docker/.ssh # entrypoint script COPY entrypoint.sh /entrypoint.sh @@ -23,5 +27,3 @@ USER root # Start both Redis and SSH using the entrypoint script CMD ["/entrypoint.sh"] - - diff --git a/examples/backend/assets/redis-with-ssh/authorized_keys b/test/assets/redis-with-ssh/authorized_keys similarity index 100% rename from examples/backend/assets/redis-with-ssh/authorized_keys rename to test/assets/redis-with-ssh/authorized_keys diff --git a/examples/backend/assets/redis-with-ssh/entrypoint.sh b/test/assets/redis-with-ssh/entrypoint.sh similarity index 100% rename from examples/backend/assets/redis-with-ssh/entrypoint.sh rename to test/assets/redis-with-ssh/entrypoint.sh diff --git a/examples/backend/assets/redis-with-ssh/test-ssh-key b/test/assets/redis-with-ssh/test-ssh-key similarity index 100% rename from examples/backend/assets/redis-with-ssh/test-ssh-key rename to test/assets/redis-with-ssh/test-ssh-key diff --git a/examples/backend/assets/redis-with-ssh/test-ssh-key.pub b/test/assets/redis-with-ssh/test-ssh-key.pub similarity index 100% rename from examples/backend/assets/redis-with-ssh/test-ssh-key.pub rename to test/assets/redis-with-ssh/test-ssh-key.pub From 2428b3ac02e5fa8397d939c0effa1ccd429f3175 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Thu, 14 Sep 2023 11:46:02 +0200 Subject: [PATCH 19/43] .env finally not used --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 96c4e21d..a2c3fe06 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ *.dylib bin testbin/* -*.env + # Test binary, build with `go test -c` *.test From a116e9107de6e6abe09703146cdfbaa824cbd2a9 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Thu, 14 Sep 2023 11:47:05 +0200 Subject: [PATCH 20/43] Update gitleak ssh key path --- .gitleaks.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitleaks.toml b/.gitleaks.toml index a5ec056c..8bb1456b 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -1,5 +1,5 @@ [allowlist] description = "global allow lists" paths = [ - '''examples/backend/assets/redis-with-ssh/test-ssh-key''', + '''test/assets/redis-with-ssh/test-ssh-key''', ] \ No newline at end of file From e272af5aad85ad87f632db37804a16c34a8c8ec7 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Fri, 15 Sep 2023 12:22:14 +0200 Subject: [PATCH 21/43] Add e2e test for backups --- Makefile | 2 +- api/v1alpha1/zz_generated.deepcopy.go | 12 +- pkg/redis/client/fake_client.go | 5 + pkg/redis/client/goredis_client.go | 5 + pkg/redis/client/interface.go | 1 + pkg/redis/server/server.go | 4 + test/assets/redis-datasets/supernovas.csv | 1976 +++++++++++++++++++++ test/e2e/shardedredisbackup_sute_test.go | 247 +++ test/util/redis.go | 63 + 9 files changed, 2313 insertions(+), 2 deletions(-) create mode 100644 test/assets/redis-datasets/supernovas.csv create mode 100644 test/e2e/shardedredisbackup_sute_test.go diff --git a/Makefile b/Makefile index 8524ba95..bcea6f21 100644 --- a/Makefile +++ b/Makefile @@ -116,7 +116,7 @@ test-sequential: manifests generate fmt vet envtest assets ginkgo ## Run tests. KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) $(GINKGO) -v -r $(TEST_PKG) -coverprofile cover.out test-e2e: export KUBECONFIG = $(PWD)/kubeconfig -test-e2e: manifests ginkgo kind-create kind-deploy ## Runs e2e tests +test-e2e: manifests ginkgo kind-create kind-deploy kind-deploy-backup-assets ## Runs e2e tests $(GINKGO) -p -r ./test/e2e $(MAKE) kind-delete diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 35aa6ab5..fc0018fa 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -2406,6 +2406,11 @@ func (in *Runtime) DeepCopy() *Runtime { func (in *S3Options) DeepCopyInto(out *S3Options) { *out = *in out.CredentialsSecretRef = in.CredentialsSecretRef + if in.ServiceEndpoint != nil { + in, out := &in.ServiceEndpoint, &out.ServiceEndpoint + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new S3Options. @@ -2454,6 +2459,11 @@ func (in *SSHOptions) DeepCopyInto(out *SSHOptions) { *out = new(uint32) **out = **in } + if in.Sudo != nil { + in, out := &in.Sudo, &out.Sudo + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SSHOptions. @@ -2847,7 +2857,7 @@ func (in *ShardedRedisBackupList) DeepCopyObject() runtime.Object { func (in *ShardedRedisBackupSpec) DeepCopyInto(out *ShardedRedisBackupSpec) { *out = *in in.SSHOptions.DeepCopyInto(&out.SSHOptions) - out.S3Options = in.S3Options + in.S3Options.DeepCopyInto(&out.S3Options) if in.Timeout != nil { in, out := &in.Timeout, &out.Timeout *out = new(metav1.Duration) diff --git a/pkg/redis/client/fake_client.go b/pkg/redis/client/fake_client.go index c6434d83..a346dae2 100644 --- a/pkg/redis/client/fake_client.go +++ b/pkg/redis/client/fake_client.go @@ -145,6 +145,11 @@ func (fc *FakeClient) RedisLastSave(ctx context.Context) (int64, error) { return rsp.InjectResponse().(int64), rsp.InjectError() } +func (fc *FakeClient) RedisSet(ctx context.Context, key string, value interface{}) error { + rsp := fc.pop() + return rsp.InjectError() +} + func (fc *FakeClient) pop() (fakeRsp FakeResponse) { fakeRsp, fc.Responses = fc.Responses[0], fc.Responses[1:] return fakeRsp diff --git a/pkg/redis/client/goredis_client.go b/pkg/redis/client/goredis_client.go index 0f2235eb..bc51eb24 100644 --- a/pkg/redis/client/goredis_client.go +++ b/pkg/redis/client/goredis_client.go @@ -160,3 +160,8 @@ func (c *GoRedisClient) RedisBGSave(ctx context.Context) error { func (c *GoRedisClient) RedisLastSave(ctx context.Context) (int64, error) { return c.redis.LastSave(ctx).Result() } + +func (c *GoRedisClient) RedisSet(ctx context.Context, key string, value interface{}) error { + _, err := c.redis.Set(ctx, key, value, 0).Result() + return err +} diff --git a/pkg/redis/client/interface.go b/pkg/redis/client/interface.go index 11b2c06e..41805e57 100644 --- a/pkg/redis/client/interface.go +++ b/pkg/redis/client/interface.go @@ -28,6 +28,7 @@ type TestableInterface interface { RedisDo(context.Context, ...interface{}) (interface{}, error) RedisBGSave(context.Context) error RedisLastSave(context.Context) (int64, error) + RedisSet(context.Context, string, interface{}) error Close() error } diff --git a/pkg/redis/server/server.go b/pkg/redis/server/server.go index 54c0992e..5cd07eab 100644 --- a/pkg/redis/server/server.go +++ b/pkg/redis/server/server.go @@ -231,6 +231,10 @@ func (srv *Server) RedisLastSave(ctx context.Context) (int64, error) { return srv.client.RedisLastSave(ctx) } +func (srv *Server) RedisSet(ctx context.Context, key string, value interface{}) error { + return srv.client.RedisSet(ctx, key, value) +} + // This is a horrible function to parse the horrible structs that the go-redis // client returns for administrative commands. I swear it's not my fault ... func sliceCmdToStruct(in interface{}, out interface{}) error { diff --git a/test/assets/redis-datasets/supernovas.csv b/test/assets/redis-datasets/supernovas.csv new file mode 100644 index 00000000..59d455eb --- /dev/null +++ b/test/assets/redis-datasets/supernovas.csv @@ -0,0 +1,1976 @@ +Object,RA,Dec,Spec. Class,"Redshift, $z$",Prediction,Confidence,p_SNII,p_SNIa,p_SNIbc +2019lbi,190.088004,1.273998,SNII,0.038,SNIa,0.497,0.278,0.497,0.225 +2019pmd,49.599161,-1.930453,SNIa-norm,0.026,SNIa,0.972,0.002,0.972,0.026 +2019ppi,133.897475,49.160259,SNII,0.135,SNII,0.495,0.495,0.173,0.332 +2019szh,147.176208,-8.734392,SNIa-norm,0.053,SNIa,1.0,0.0,1.0,0.0 +2019tvv,177.772596,21.767258,SNIa-norm,0.059,SNIa,1.0,0.0,1.0,0.0 +2019ucc,49.642135,-4.429429,NA,0.066,SNIbc,0.87,0.005,0.125,0.87 +2019uev,142.706471,30.871843,NA,0.06,SNIbc,0.779,0.019,0.202,0.779 +2019uez,51.172538,-0.681409,SNII,0.022,SNII,0.78,0.78,0.053,0.167 +2019uit,192.567042,21.337646,SNIIn,0.086,SNII,0.457,0.457,0.101,0.442 +2019ulo,133.278015,-6.329666,NA,0.041,SNII,0.986,0.986,0.004,0.01 +2019unp,142.397468,35.289745,NA,0.145,SNIa,0.996,0.001,0.996,0.003 +2019vuz,112.309986,42.076944,NA,0.21,SNIa,0.466,0.218,0.466,0.316 +2019wbv,134.689142,-4.56562,NA,0.067,SNIbc,0.598,0.217,0.185,0.598 +2019wbw,150.67996,-8.641407,NA,0.138,SNIa,0.957,0.003,0.957,0.04 +2019wca,145.192026,-8.4785,NA,0.202,SNII,0.526,0.526,0.33,0.144 +2019wcb,114.503215,39.707421,NA,0.157,SNIa,0.967,0.015,0.967,0.018 +2019wcc,112.400389,42.108156,NA,0.131,SNIa,0.984,0.001,0.984,0.015 +2019wcf,140.301192,31.617246,NA,0.149,SNIa,0.991,0.0,0.991,0.009 +2019wka,48.667279,-0.871312,SNIa-norm,0.087,SNIa,1.0,0.0,1.0,0.0 +2019wkf,173.023848,13.3095,NA,0.207,SNIa,0.969,0.001,0.969,0.03 +2019wmr,146.514779,33.528265,SNII,0.03,SNIbc,0.952,0.001,0.047,0.952 +2019wqf,217.500587,39.945841,SNIa-norm,0.109,SNIa,1.0,0.0,1.0,0.0 +2019wyp,196.654579,16.591372,SNIa-norm,0.116,SNIa,0.976,0.0,0.976,0.024 +2019xbj,148.812743,3.846365,NA,0.214,SNIa,0.999,0.0,0.999,0.001 +2019yfu,191.401508,1.183637,SNIa-norm,0.06,SNIa,0.999,0.0,0.999,0.001 +2019yih,136.822285,-6.188904,NA,0.217,SNII,0.808,0.808,0.054,0.138 +2019yiz,54.894594,-5.28165,NA,0.156,SNIa,0.899,0.051,0.899,0.05 +2019yjl,172.617645,10.269092,NA,0.155,SNII,0.719,0.719,0.035,0.246 +2019ytc,144.635743,-10.177591,NA,0.121,SNIa,0.937,0.031,0.937,0.032 +2019ytn,145.546712,-11.938106,NA,0.201,SNIa,0.483,0.409,0.483,0.108 +2019yua,160.789985,-1.256824,NA,0.209,SNII,0.501,0.501,0.487,0.012 +2019yub,187.472324,11.675235,NA,0.076,SNII,0.95,0.95,0.044,0.006 +2019yuf,173.398792,10.980877,SNIc,0.202,SNIbc,0.713,0.182,0.105,0.713 +2019yuh,162.684432,-4.154778,NA,0.149,SNIa,0.999,0.0,0.999,0.001 +2019yuj,160.301097,-2.138911,NA,0.122,SNIa,0.534,0.452,0.534,0.014 +2019yvr,191.283892,-0.459091,SNIb,0.016,SNIbc,0.944,0.036,0.02,0.944 +2019yyb,135.953613,-5.566821,SNIa-norm,0.085,SNIa,1.0,0.0,1.0,0.0 +2019zag,128.194044,50.391618,NA,0.108,SNIa,1.0,0.0,1.0,0.0 +2019zcp,185.190779,9.335413,NA,0.077,SNIa,0.707,0.029,0.707,0.264 +2019zfl,50.295893,-1.008041,NA,0.211,SNIa,0.939,0.037,0.939,0.024 +2019zfv,162.840117,-4.281892,SNIa-norm,0.129,SNIa,1.0,0.0,1.0,0.0 +2019zgp,55.232488,-5.361982,NA,0.155,SNIa,0.992,0.0,0.992,0.008 +2019zha,154.395689,-4.012473,NA,0.182,SNII,0.645,0.645,0.231,0.124 +2019zit,185.986287,8.187872,NA,0.168,SNII,0.961,0.961,0.037,0.002 +2019zix,161.737351,-2.899353,NA,0.183,SNIa,0.956,0.004,0.956,0.04 +2019zja,48.660125,-1.340848,NA,0.022,SNIa,0.629,0.263,0.629,0.108 +2019zno,174.660766,13.696884,NA,0.194,SNIa,0.503,0.442,0.503,0.055 +2019zqa,153.303092,1.648057,NA,0.197,SNIa,0.793,0.159,0.793,0.048 +2019zre,170.549305,9.979284,NA,0.223,SNIa,0.945,0.036,0.945,0.019 +2019zrf,161.357527,-5.367437,NA,0.203,SNIa,0.529,0.31,0.529,0.161 +2019zsy,187.015927,8.458947,NA,0.201,SNIa,0.992,0.004,0.992,0.004 +2020aaaq,59.374364,-9.58168,NA,0.134,SNIa,0.991,0.0,0.991,0.009 +2020aabc,136.893487,27.980144,NA,0.114,SNIa,0.998,0.001,0.998,0.001 +2020aaca,324.314029,9.261819,NA,0.157,SNIa,0.532,0.441,0.532,0.027 +2020aaer,138.161633,52.713108,NA,0.185,SNIa,0.759,0.175,0.759,0.066 +2020aafg,16.883017,-18.292036,NA,0.089,SNIa,1.0,0.0,1.0,0.0 +2020aafm,138.394131,52.097229,SNII,0.264,SNIa,0.548,0.143,0.548,0.309 +2020aahb,155.739268,2.48982,NA,0.105,SNIa,0.999,0.001,0.999,0.0 +2020aahq,52.885264,-4.116037,NA,0.224,SNIa,1.0,0.0,1.0,0.0 +2020aajf,50.612436,-3.818666,SNIa-norm,0.069,SNIa,1.0,0.0,1.0,0.0 +2020aajt,0.761011,17.787973,NA,0.216,SNII,0.757,0.757,0.194,0.049 +2020aalu,0.524158,18.223875,NA,0.208,SNIa,0.478,0.342,0.478,0.18 +2020aamu,42.849787,-3.612322,NA,0.252,SNIa,0.952,0.034,0.952,0.014 +2020aapb,133.703639,28.427882,NA,0.229,SNIa,0.892,0.083,0.892,0.025 +2020aapc,138.596761,27.917182,NA,0.206,SNIa,0.801,0.105,0.801,0.094 +2020aaqu,39.806144,-5.086547,NA,0.217,SNIa,0.952,0.024,0.952,0.024 +2020aarz,38.429178,-4.257265,NA,0.155,SNIa,0.999,0.0,0.999,0.001 +2020aasl,155.764183,5.718256,NA,0.155,SNIa,0.877,0.049,0.877,0.074 +2020aatr,129.045076,-3.435099,SNIa-norm,0.041,SNIa,0.96,0.0,0.96,0.04 +2020aaua,189.815019,9.998724,NA,0.113,SNIa,1.0,0.0,1.0,0.0 +2020aaur,187.64705,7.438642,NA,0.097,SNIa,1.0,0.0,1.0,0.0 +2020aauu,187.660308,2.580164,NA,0.085,SNIa,1.0,0.0,1.0,0.0 +2020aavd,59.287429,-24.901844,SNIa-norm,0.172,SNIa,1.0,0.0,1.0,0.0 +2020aaws,193.328233,4.127956,NA,0.086,SNII,0.505,0.505,0.281,0.214 +2020aawu,192.975212,18.572608,NA,0.102,SNIa,0.99,0.01,0.99,0.0 +2020aawv,192.709433,18.348125,NA,0.105,SNIa,0.995,0.004,0.995,0.001 +2020aazk,139.699564,30.354678,SNIa-norm,0.112,SNIa,0.982,0.0,0.982,0.018 +2020aazr,314.144142,-0.176372,SNIa-norm,0.126,SNIa,0.887,0.038,0.887,0.075 +2020abah,174.222738,21.000272,SNII,0.039,SNIbc,0.536,0.46,0.004,0.536 +2020abbc,133.406751,27.617266,NA,0.097,SNIa,1.0,0.0,1.0,0.0 +2020abbj,188.7397,2.85675,NA,0.118,SNIa,1.0,0.0,1.0,0.0 +2020abci,143.607804,33.088121,NA,0.232,SNII,0.897,0.897,0.085,0.018 +2020abcn,171.925384,10.312841,SNIa-norm,0.038,SNIa,1.0,0.0,1.0,0.0 +2020abgb,139.737006,31.742684,NA,0.133,SNIbc,0.471,0.399,0.13,0.471 +2020abhk,134.56916,-5.815337,NA,0.138,SNIa,1.0,0.0,1.0,0.0 +2020abho,206.865427,34.909759,NA,0.198,SNII,0.756,0.756,0.159,0.085 +2020abhu,157.194724,3.115342,NA,0.197,SNIa,0.658,0.069,0.658,0.273 +2020abid,208.432971,33.796937,NA,0.232,SNII,0.588,0.588,0.251,0.161 +2020abim,193.373492,-5.431795,SNIa-norm,0.025,SNIa,0.779,0.008,0.779,0.213 +2020abip,204.885388,29.998022,SNIa-norm,0.039,SNIa,1.0,0.0,1.0,0.0 +2020abix,195.7633,-1.269897,NA,0.199,SNIa,1.0,0.0,1.0,0.0 +2020abjb,60.917925,-10.706406,NA,0.155,SNIa,0.998,0.002,0.998,0.0 +2020abji,14.614757,5.761791,NA,0.237,SNIa,1.0,0.0,1.0,0.0 +2020abjq,16.56095,3.57394,SNII,0.028,SNII,0.984,0.984,0.009,0.007 +2020abjy,334.353609,-0.514661,NA,0.166,SNIa,0.497,0.056,0.497,0.447 +2020abka,332.388388,5.940435,NA,0.116,SNIa,0.869,0.081,0.869,0.05 +2020abkb,52.921831,-4.474923,NA,0.148,SNIa,0.982,0.011,0.982,0.007 +2020abkf,194.861636,-2.246808,SNIa-norm,0.047,SNIa,0.832,0.071,0.832,0.097 +2020able,141.509455,24.521709,SNIbn,0.041,SNIa,0.964,0.0,0.964,0.036 +2020ablf,15.930636,4.575602,NA,0.225,SNIa,1.0,0.0,1.0,0.0 +2020abm,151.678005,3.175976,NA,0.179,SNIa,1.0,0.0,1.0,0.0 +2020abnt,10.352926,2.735525,NA,0.23,SNIa,0.992,0.003,0.992,0.005 +2020aboe,5.634387,23.692147,NA,0.102,SNIa,1.0,0.0,1.0,0.0 +2020abpt,40.1062,-1.196819,SNIa-norm,0.141,SNIa,0.989,0.007,0.989,0.004 +2020abpx,21.873683,13.841318,NA,0.182,SNIa,0.526,0.277,0.526,0.197 +2020abra,46.211351,-1.180595,SNIa-91T-like,0.12,SNIa,1.0,0.0,1.0,0.0 +2020abrb,111.41024,41.372929,NA,0.158,SNIa,0.989,0.001,0.989,0.01 +2020abrc,343.53056,34.877353,NA,0.23,SNIa,0.917,0.044,0.917,0.039 +2020abrd,21.074352,2.204799,NA,0.107,SNIa,1.0,0.0,1.0,0.0 +2020abre,150.308202,2.580315,NA,0.123,SNIa,0.884,0.058,0.884,0.058 +2020absk,136.003832,-0.088905,SNII,0.071,SNII,0.976,0.976,0.002,0.022 +2020absw,59.031933,-26.665264,NA,0.094,SNII,0.992,0.992,0.006,0.002 +2020abte,35.001031,-1.377143,NA,0.174,SNIa,0.943,0.032,0.943,0.025 +2020abtm,333.889204,31.452292,NA,0.065,SNIa,0.997,0.002,0.997,0.001 +2020abul,192.099108,-1.377222,NA,0.085,SNII,0.953,0.953,0.034,0.013 +2020abvg,17.901071,-15.214514,SNIa-norm,0.037,SNIa,0.847,0.011,0.847,0.142 +2020abvu,36.945921,40.935767,NA,0.153,SNIa,1.0,0.0,1.0,0.0 +2020abwx,21.00119,14.084061,NA,0.171,SNIa,0.997,0.0,0.997,0.003 +2020abxz,43.304387,-5.090522,NA,0.155,SNIa,0.993,0.0,0.993,0.007 +2020abya,50.070192,-3.742461,NA,0.155,SNIa,0.858,0.098,0.858,0.044 +2020abyb,2.088006,28.82653,NA,0.227,SNIbc,0.474,0.092,0.434,0.474 +2020abyv,20.068075,-17.438522,NA,0.155,SNII,0.548,0.548,0.447,0.005 +2020acas,0.376857,23.898236,NA,0.155,SNII,0.854,0.854,0.113,0.033 +2020acbb,93.368066,-22.437106,NA,0.143,SNIa,0.998,0.0,0.998,0.002 +2020acbc,64.171879,-24.834947,SNIa-norm,0.099,SNIa,0.999,0.001,0.999,0.0 +2020acct,146.233587,31.096033,SNIc,0.031,SNIa,0.687,0.005,0.687,0.308 +2020acds,65.100763,-24.920375,NA,0.18,SNIa,0.789,0.156,0.789,0.055 +2020acdu,60.044619,-11.330735,NA,0.089,SNIa,0.995,0.0,0.995,0.005 +2020acgk,178.573942,20.692314,NA,0.155,SNIa,0.584,0.022,0.584,0.394 +2020achi,65.951429,-10.687849,NA,0.18,SNII,0.963,0.963,0.026,0.011 +2020achj,62.243836,-8.768522,NA,0.214,SNII,0.542,0.542,0.396,0.062 +2020achk,66.508999,-7.729324,NA,0.15,SNIa,0.978,0.014,0.978,0.008 +2020achp,42.238333,41.034028,NA,0.145,SNIa,0.987,0.008,0.987,0.005 +2020acih,163.274097,-0.310258,NA,0.135,SNII,0.943,0.943,0.04,0.017 +2020aciz,150.840602,2.838874,SNIa-norm,0.107,SNIa,1.0,0.0,1.0,0.0 +2020acjk,127.481742,50.136336,NA,0.228,SNIa,0.787,0.184,0.787,0.029 +2020acjn,160.715659,-3.618176,NA,0.224,SNII,0.657,0.657,0.33,0.013 +2020acjy,159.724213,-0.941819,SNIa-norm,0.068,SNIa,1.0,0.0,1.0,0.0 +2020acmi,54.667625,-27.196842,SNIa-norm,0.111,SNIa,0.886,0.009,0.886,0.105 +2020acmm,9.21435,2.170323,SNIa-norm,0.092,SNIa,0.999,0.0,0.999,0.001 +2020acog,132.949646,-1.369528,NA,0.088,SNIbc,0.608,0.027,0.365,0.608 +2020acqi,117.788883,39.216494,NA,0.218,SNIa,0.947,0.03,0.947,0.023 +2020acsq,115.382678,42.088883,NA,0.245,SNIa,1.0,0.0,1.0,0.0 +2020actn,174.792659,10.661091,NA,0.09,SNII,0.976,0.976,0.012,0.012 +2020acts,89.469735,-21.552356,SNIa-norm,0.055,SNIa,1.0,0.0,1.0,0.0 +2020actt,116.231722,38.061715,NA,0.099,SNIa,1.0,0.0,1.0,0.0 +2020acty,140.649977,30.994661,SNIb,0.071,SNIbc,0.993,0.0,0.007,0.993 +2020acun,227.990837,-2.981891,SNII,0.024,SNIbc,0.993,0.0,0.007,0.993 +2020acvi,140.289678,41.378105,NA,0.23,SNIa,1.0,0.0,1.0,0.0 +2020acww,127.548529,49.967142,NA,0.181,SNIa,0.806,0.096,0.806,0.098 +2020acwx,39.497033,0.370972,NA,0.199,SNIa,0.616,0.345,0.616,0.039 +2020acwz,38.342934,40.379782,NA,0.149,SNII,0.992,0.992,0.007,0.001 +2020acxa,7.068329,16.911019,NA,0.097,SNIa,0.906,0.033,0.906,0.061 +2020acxb,52.719265,-2.014202,SNIa-norm,0.186,SNIa,0.793,0.121,0.793,0.086 +2020acxc,1.622433,25.054894,NA,0.071,SNII,0.992,0.992,0.008,0.0 +2020acxd,6.414608,27.564197,NA,0.233,SNIa,0.701,0.239,0.701,0.06 +2020acxe,131.494754,48.254211,NA,0.241,SNIa,0.899,0.059,0.899,0.042 +2020acxf,141.295976,50.860714,NA,0.155,SNII,0.505,0.505,0.467,0.028 +2020acxg,66.618154,-20.846117,SNIa-91T-like,0.219,SNIa,0.591,0.361,0.591,0.048 +2020acyo,49.337088,3.020039,NA,0.129,SNIa,0.937,0.039,0.937,0.024 +2020acyp,36.662262,-7.329814,NA,0.181,SNIa,0.991,0.001,0.991,0.008 +2020acyq,17.540558,5.154814,NA,0.17,SNII,0.563,0.563,0.34,0.097 +2020acyr,0.472342,20.607769,NA,0.112,SNIa,0.439,0.323,0.439,0.238 +2020acys,39.879286,38.387431,NA,0.18,SNIa,1.0,0.0,1.0,0.0 +2020acyt,129.315975,45.413275,NA,0.244,SNIa,0.866,0.043,0.866,0.091 +2020acyu,129.205127,45.303575,NA,0.254,SNII,0.559,0.559,0.242,0.199 +2020acyv,44.640123,-0.019158,NA,0.152,SNII,0.573,0.573,0.406,0.021 +2020aczd,68.2822,-7.527258,SNII,0.024,SNII,0.986,0.986,0.009,0.005 +2020aczf,189.413061,10.430539,NA,0.153,SNIa,0.995,0.0,0.995,0.005 +2020aczw,113.387647,41.763904,NA,0.2,SNIa,0.615,0.354,0.615,0.031 +2020adat,138.160832,42.494791,NA,0.181,SNIa,0.954,0.021,0.954,0.025 +2020adc,162.700753,-3.463972,NA,0.177,SNIa,0.914,0.004,0.914,0.082 +2020add,155.009404,6.631779,SNIa-norm,0.098,SNIa,1.0,0.0,1.0,0.0 +2020addt,52.718404,-1.520383,NA,0.225,SNIa,0.889,0.055,0.889,0.056 +2020adix,153.136925,6.528286,NA,0.098,SNIa,0.979,0.003,0.979,0.018 +2020adjs,140.660899,52.095925,NA,0.157,SNIa,0.684,0.09,0.684,0.226 +2020adkc,154.081739,-1.516317,SNIa-norm,0.097,SNIa,1.0,0.0,1.0,0.0 +2020adkf,222.8126,41.415279,NA,0.155,SNIa,1.0,0.0,1.0,0.0 +2020adky,68.501829,-6.879494,NA,0.158,SNIa,0.896,0.023,0.896,0.081 +2020adlp,121.756457,39.707651,SNIa-norm,0.083,SNIa,1.0,0.0,1.0,0.0 +2020adly,186.355295,12.160006,NA,0.177,SNIbc,0.702,0.073,0.225,0.702 +2020adlz,149.383727,3.075907,NA,0.075,SNIa,0.985,0.003,0.985,0.012 +2020adma,58.377592,-22.182542,NA,0.212,SNIa,0.973,0.01,0.973,0.017 +2020admp,40.868849,39.051539,NA,0.172,SNIa,0.93,0.026,0.93,0.044 +2020admv,135.161103,23.824267,NA,0.225,SNIa,0.971,0.002,0.971,0.027 +2020adob,137.547896,29.078247,NA,0.155,SNII,0.921,0.921,0.074,0.005 +2020ads,55.529574,-3.194178,NA,0.155,SNIa,1.0,0.0,1.0,0.0 +2020adti,16.999992,3.141931,NA,0.05,SNIbc,0.9,0.02,0.08,0.9 +2020advj,193.994179,-1.477796,NA,0.201,SNII,0.607,0.607,0.373,0.02 +2020advk,198.009668,-1.693936,NA,0.18,SNII,0.578,0.578,0.202,0.22 +2020advl,191.241352,-2.058703,NA,0.109,SNIa,0.995,0.002,0.995,0.003 +2020advq,210.398057,32.950108,NA,0.15,SNIa,0.52,0.413,0.52,0.067 +2020advr,146.111867,33.229083,NA,0.179,SNIa,0.956,0.026,0.956,0.018 +2020aect,174.129404,12.045621,NA,0.189,SNIa,0.722,0.057,0.722,0.221 +2020aecv,128.274221,49.507167,NA,0.194,SNIa,0.687,0.265,0.687,0.048 +2020aeeg,148.627102,6.255077,SNII,0.154,SNII,0.479,0.479,0.054,0.467 +2020aefy,109.608518,39.461544,NA,0.091,SNII,1.0,1.0,0.0,0.0 +2020aegj,36.622994,41.268259,NA,0.106,SNIa,1.0,0.0,1.0,0.0 +2020aehn,125.556656,43.967648,NA,0.16,SNII,0.794,0.794,0.188,0.018 +2020aehy,63.395806,-10.667209,NA,0.233,SNIa,0.764,0.162,0.764,0.074 +2020aeid,128.648304,44.430917,NA,0.233,SNII,0.411,0.411,0.267,0.322 +2020aejb,138.50919,24.737456,NA,0.163,SNIa,0.999,0.0,0.999,0.001 +2020aekp,235.797458,17.813111,SNIa-CSM,0.101,SNII,0.613,0.613,0.174,0.213 +2020aena,158.890602,-1.470212,NA,0.193,SNIa,0.998,0.0,0.998,0.002 +2020aenc,170.292136,11.412131,NA,0.191,SNIbc,0.716,0.163,0.121,0.716 +2020aepz,196.198926,-0.745206,NA,0.127,SNIbc,0.726,0.125,0.149,0.726 +2020aeql,155.158825,2.298366,NA,0.12,SNII,0.816,0.816,0.007,0.177 +2020aeqm,135.264454,23.286108,SNIa-norm,0.032,SNIa,0.91,0.002,0.91,0.088 +2020aeqp,187.34065,31.254522,SNIa-norm,0.119,SNIa,1.0,0.0,1.0,0.0 +2020aeqx,198.250104,6.170064,SNII,0.043,SNII,0.966,0.966,0.016,0.018 +2020aesh,184.446641,15.904445,NA,0.247,SNIa,0.965,0.029,0.965,0.006 +2020aesp,197.694808,-1.109991,NA,0.139,SNII,0.717,0.717,0.197,0.086 +2020aetg,44.665995,40.67045,NA,0.155,SNIa,0.803,0.041,0.803,0.156 +2020aeti,138.921903,25.660128,NA,0.153,SNIa,1.0,0.0,1.0,0.0 +2020aevg,174.312961,9.202487,NA,0.198,SNII,0.415,0.415,0.265,0.32 +2020aevl,202.970963,31.678672,NA,0.159,SNIa,0.748,0.122,0.748,0.13 +2020aevm,141.249225,1.827386,NA,0.194,SNIa,0.845,0.114,0.845,0.041 +2020aewd,111.157504,39.147206,NA,0.141,SNII,0.878,0.878,0.046,0.076 +2020aewm,194.105754,-4.615572,NA,0.112,SNII,0.528,0.528,0.11,0.362 +2020aeyo,220.357714,-7.54722,NA,0.113,SNIa,0.926,0.024,0.926,0.05 +2020agv,55.424835,-3.922991,NA,0.139,SNII,0.827,0.827,0.121,0.052 +2020ahd,194.248211,19.703956,NA,0.082,SNII,0.98,0.98,0.013,0.007 +2020akx,194.912848,-5.09655,NA,0.139,SNIa,1.0,0.0,1.0,0.0 +2020alx,174.646005,10.337293,NA,0.166,SNIa,0.923,0.023,0.923,0.054 +2020aly,208.041171,32.436945,NA,0.11,SNIa,0.992,0.0,0.992,0.008 +2020alz,200.79569,34.862117,NA,0.159,SNII,0.569,0.569,0.332,0.099 +2020ama,189.561937,9.230725,SNII,0.072,SNII,0.9,0.9,0.006,0.094 +2020amb,161.89008,-4.025333,NA,0.185,SNIa,0.522,0.466,0.522,0.012 +2020amc,160.806619,-3.703727,NA,0.155,SNIa,0.847,0.13,0.847,0.023 +2020amg,204.906366,30.805919,NA,0.148,SNIa,0.505,0.018,0.505,0.477 +2020anl,195.850142,-3.500935,NA,0.153,SNIa,0.528,0.447,0.528,0.025 +2020anm,193.137225,-5.533001,NA,0.132,SNIa,0.955,0.035,0.955,0.01 +2020ann,150.697742,3.599569,SNIa-norm,0.201,SNIa,1.0,0.0,1.0,0.0 +2020aot,195.898996,5.442134,NA,0.165,SNIa,0.914,0.029,0.914,0.057 +2020aov,180.898889,21.13069,NA,0.155,SNII,0.673,0.673,0.233,0.094 +2020apf,189.822344,-1.507323,NA,0.119,SNIa,1.0,0.0,1.0,0.0 +2020apu,190.320813,8.953855,NA,0.158,SNIa,1.0,0.0,1.0,0.0 +2020apw,141.012066,28.814982,SN,0.224,SNIa,0.998,0.002,0.998,0.0 +2020aqn,187.767952,7.97559,NA,0.245,SNIa,0.992,0.004,0.992,0.004 +2020aqz,133.119919,-6.712222,NA,0.189,SNIa,0.965,0.019,0.965,0.016 +2020ara,150.917211,-11.824694,NA,0.202,SNIa,0.786,0.136,0.786,0.078 +2020atv,151.274933,-12.126594,SNIa-norm,0.197,SNIa,1.0,0.0,1.0,0.0 +2020awg,237.260429,17.937869,SNII,0.034,SNII,0.984,0.984,0.003,0.013 +2020awu,171.348033,9.983899,SN,0.087,SNII,0.975,0.975,0.018,0.007 +2020awv,201.157096,31.832195,NA,0.226,SNIa,0.858,0.11,0.858,0.032 +2020aww,132.970646,50.411401,NA,0.11,SNIa,0.974,0.006,0.974,0.02 +2020aym,137.237001,28.632865,NA,0.19,SNIa,0.821,0.038,0.821,0.141 +2020azh,193.279267,-4.061596,SNII,0.029,SNII,0.683,0.683,0.028,0.289 +2020azk,195.699708,-5.041729,SNIb,0.082,SNII,0.789,0.789,0.122,0.089 +2020azn,141.918521,35.530075,SNIa-norm,0.153,SNIa,1.0,0.0,1.0,0.0 +2020azp,194.487917,-1.889886,NA,0.089,SNII,0.988,0.988,0.002,0.01 +2020azs,143.439971,33.028124,SNIa-norm,0.155,SNIa,0.993,0.001,0.993,0.006 +2020azt,193.425758,-4.441503,SNIa-norm,0.233,SNIa,0.994,0.004,0.994,0.002 +2020bam,121.307773,40.840103,NA,0.139,SNIa,0.995,0.003,0.995,0.002 +2020bcq,201.623554,36.008626,SNIb,0.018,SNIa,0.704,0.002,0.704,0.294 +2020bct,133.878949,1.313417,NA,0.181,SNIa,0.901,0.012,0.901,0.087 +2020bii,194.713203,6.509194,NA,0.196,SNIa,0.811,0.172,0.811,0.017 +2020bkv,195.064446,5.364786,NA,0.232,SNIa,0.78,0.175,0.78,0.045 +2020bnw,147.682843,35.688597,NA,0.15,SNIa,0.994,0.0,0.994,0.006 +2020bsk,189.356481,9.65036,NA,0.17,SNIa,0.914,0.072,0.914,0.014 +2020bso,183.934362,12.128966,NA,0.101,SNIa,0.443,0.33,0.443,0.227 +2020bta,203.43878,35.149021,NA,0.194,SNII,0.754,0.754,0.243,0.003 +2020bte,196.267572,4.756155,NA,0.153,SNIa,0.991,0.005,0.991,0.004 +2020btg,196.724096,3.894338,NA,0.243,SNII,0.473,0.473,0.386,0.141 +2020bwr,248.747221,36.207672,SNIIn,0.082,SNIbc,0.491,0.324,0.185,0.491 +2020byg,147.611669,-8.312454,NA,0.156,SNIa,0.899,0.049,0.899,0.052 +2020ciu,13.637694,5.228342,NA,0.08,SNII,0.572,0.572,0.008,0.42 +2020clh,150.818248,-11.250325,NA,0.073,SNIa,0.991,0.002,0.991,0.007 +2020cwg,190.781004,11.211258,NA,0.034,SNII,0.696,0.696,0.224,0.08 +2020cyz,213.114037,-5.82343,SNIa-norm,0.117,SNIa,1.0,0.0,1.0,0.0 +2020dbd,142.988688,33.209658,SNII,0.051,SNIa,0.83,0.0,0.83,0.17 +2020dec,187.140958,7.450765,NA,0.079,SNIa,0.499,0.01,0.499,0.491 +2020dow,137.983396,27.900481,SNIa-norm,0.051,SNIa,1.0,0.0,1.0,0.0 +2020dtk,197.866322,-4.360974,NA,0.206,SNIa,0.995,0.003,0.995,0.002 +2020duv,241.255767,27.260274,SNIa-91T-like,0.05,SNIa,1.0,0.0,1.0,0.0 +2020dvl,229.048095,0.07953,NA,0.109,SNIa,1.0,0.0,1.0,0.0 +2020dwg,239.603346,32.968919,SNIa-norm,0.035,SNIa,1.0,0.0,1.0,0.0 +2020dwq,241.739113,32.529268,NA,0.187,SNIa,1.0,0.0,1.0,0.0 +2020dxa,239.67955,26.819344,SNIa-norm,0.027,SNIa,0.985,0.0,0.985,0.015 +2020dyc,219.975025,-7.688293,SNIa-norm,0.052,SNIa,1.0,0.0,1.0,0.0 +2020eaf,44.583265,40.923169,SNII,0.032,SNIa,0.569,0.294,0.569,0.137 +2020ebc,155.228902,3.86409,NA,0.096,SNII,0.973,0.973,0.006,0.021 +2020ebm,169.814746,10.119853,NA,0.212,SNIa,1.0,0.0,1.0,0.0 +2020eci,230.81205,0.064954,SNIa-norm,0.068,SNIa,1.0,0.0,1.0,0.0 +2020ees,201.8133,32.03363,SNIa-norm,0.026,SNIa,1.0,0.0,1.0,0.0 +2020ej,241.696367,20.538001,SNII,0.037,SNII,0.814,0.814,0.014,0.172 +2020ejj,246.314055,25.382857,NA,0.153,SNII,0.877,0.877,0.019,0.104 +2020elt,185.441497,19.700595,NA,0.142,SNIa,0.999,0.001,0.999,0.0 +2020eol,137.678981,24.791669,NA,0.261,SNIa,0.576,0.385,0.576,0.039 +2020epi,185.536571,6.099735,NA,0.026,SNII,0.538,0.538,0.043,0.419 +2020eqz,207.470508,35.577866,NA,0.228,SNIa,0.88,0.091,0.88,0.029 +2020era,207.544847,34.863787,NA,0.21,SNIa,0.806,0.178,0.806,0.016 +2020eru,157.847207,-3.120265,NA,0.177,SNIa,1.0,0.0,1.0,0.0 +2020erx,155.23889,-2.876998,NA,0.223,SNIa,0.941,0.022,0.941,0.037 +2020esm,211.826088,-5.127117,SNIa-SC,0.016,SNIa,0.955,0.0,0.955,0.045 +2020eta,138.239953,36.186365,NA,0.104,SNIa,0.726,0.17,0.726,0.104 +2020evl,249.843639,35.31607,NA,0.103,SNII,0.717,0.717,0.26,0.023 +2020evp,192.826266,3.882172,NA,0.152,SNIa,0.996,0.002,0.996,0.002 +2020evq,197.711426,6.913442,NA,0.209,SNIa,1.0,0.0,1.0,0.0 +2020evr,192.387213,20.051608,NA,0.092,SNIbc,0.773,0.004,0.223,0.773 +2020evt,229.076709,0.100516,NA,0.115,SNIa,0.979,0.003,0.979,0.018 +2020evu,238.12045,24.494119,SNIa-norm,0.093,SNIa,1.0,0.0,1.0,0.0 +2020ewx,229.327104,13.02275,SNIa-norm,0.112,SNIa,1.0,0.0,1.0,0.0 +2020eyt,151.012946,3.129942,NA,0.193,SNIa,0.999,0.0,0.999,0.001 +2020eyv,148.57644,2.74105,NA,0.072,SNII,0.991,0.991,0.005,0.004 +2020eyx,227.809186,11.364686,NA,0.057,SNIa,1.0,0.0,1.0,0.0 +2020far,231.07864,10.470386,NA,0.184,SNIa,0.95,0.015,0.95,0.035 +2020fc,154.897321,6.328469,SNII,0.029,SNII,0.962,0.962,0.018,0.02 +2020fci,161.958417,-0.791753,NA,0.055,SNIa,1.0,0.0,1.0,0.0 +2020fhf,227.599858,8.789473,NA,0.082,SNIa,0.992,0.001,0.992,0.007 +2020fhj,242.118433,29.012867,SNIc-BL,0.087,SNIa,1.0,0.0,1.0,0.0 +2020fhs,240.276175,19.450186,SNIa-norm,0.026,SNIa,0.939,0.002,0.939,0.059 +2020fla,206.739956,30.885107,NA,0.166,SNII,0.957,0.957,0.035,0.008 +2020fle,236.522029,17.620927,NA,0.199,SNIa,1.0,0.0,1.0,0.0 +2020fo,136.87062,-7.315834,NA,0.173,SNIa,0.845,0.113,0.845,0.042 +2020fqv,189.138583,11.231655,SNIIb,0.008,SNII,0.985,0.985,0.001,0.014 +2020fsc,239.915417,26.54765,NA,0.094,SNIa,1.0,0.0,1.0,0.0 +2020fsd,194.681952,-2.286453,NA,0.114,SNIa,0.999,0.0,0.999,0.001 +2020ftl,185.015713,5.343329,SNIa-norm,0.024,SNIa,1.0,0.0,1.0,0.0 +2020fuq,219.818224,39.945771,NA,0.096,SNII,0.382,0.382,0.373,0.245 +2020fwj,232.292817,-3.110349,NA,0.147,SNIbc,0.619,0.063,0.318,0.619 +2020fwz,232.322849,-2.791815,NA,0.093,SNIa,0.8,0.145,0.8,0.055 +2020fxe,240.652142,34.135383,NA,0.094,SNIa,1.0,0.0,1.0,0.0 +2020fza,240.908173,19.837684,NA,0.059,SNII,0.958,0.958,0.025,0.017 +2020gcv,322.8403,9.42999,SNII,0.031,SNIa,0.525,0.001,0.525,0.474 +2020ghq,221.335112,38.738408,SNII,0.027,SNII,0.99,0.99,0.005,0.005 +2020ghr,236.155198,17.654452,NA,0.137,SNIa,0.836,0.012,0.836,0.152 +2020ghs,236.455116,17.225221,NA,0.156,SNII,0.8,0.8,0.084,0.116 +2020glj,237.234054,19.498212,NA,0.191,SNIa,0.851,0.132,0.851,0.017 +2020gll,236.922529,20.261092,NA,0.106,SNIa,0.999,0.0,0.999,0.001 +2020has,193.968279,6.873221,NA,0.163,SNIa,1.0,0.0,1.0,0.0 +2020hau,240.767221,27.655239,NA,0.145,SNIa,1.0,0.0,1.0,0.0 +2020hav,209.355964,31.210051,NA,0.156,SNII,0.522,0.522,0.457,0.021 +2020hdf,152.045359,5.286328,NA,0.2,SNII,0.56,0.56,0.43,0.01 +2020heg,209.063017,30.8073,NA,0.119,SNIa,1.0,0.0,1.0,0.0 +2020hep,229.900178,11.201748,NA,0.148,SNIa,0.883,0.001,0.883,0.116 +2020her,244.837754,23.520582,NA,0.101,SNIa,1.0,0.0,1.0,0.0 +2020hfm,154.838875,2.581282,NA,0.154,SNIa,1.0,0.0,1.0,0.0 +2020hgw,214.579712,38.929525,SNII,0.027,SNII,0.846,0.846,0.108,0.046 +2020hif,201.468186,34.609072,NA,0.204,SNIbc,0.492,0.047,0.461,0.492 +2020hik,191.875613,17.400744,NA,0.117,SNII,0.979,0.979,0.006,0.015 +2020hju,141.925665,31.80855,NA,0.179,SNIa,0.982,0.01,0.982,0.008 +2020hjv,147.497483,32.895236,NA,0.141,SNIa,1.0,0.0,1.0,0.0 +2020hmc,195.366992,18.012535,NA,0.206,SNIa,0.506,0.442,0.506,0.052 +2020hpu,205.130951,35.9968,NA,0.15,SNIa,0.668,0.035,0.668,0.297 +2020hr,139.623775,28.141627,SNIa-norm,0.083,SNIa,1.0,0.0,1.0,0.0 +2020hra,239.186226,19.500515,NA,0.103,SNIa,0.665,0.059,0.665,0.276 +2020hrb,135.018272,26.903592,NA,0.178,SNIa,1.0,0.0,1.0,0.0 +2020hrd,228.001145,-7.488296,NA,0.082,SNIa,0.689,0.052,0.689,0.259 +2020hro,228.801429,0.484342,NA,0.078,SNIa,1.0,0.0,1.0,0.0 +2020hrw,196.029629,5.561878,SNII,0.175,SNII,0.7,0.7,0.045,0.255 +2020hug,185.259328,19.015915,NA,0.053,SNIa,0.737,0.001,0.737,0.262 +2020hwg,191.670334,-5.291538,NA,0.144,SNII,0.988,0.988,0.003,0.009 +2020hwk,153.862058,2.335773,NA,0.229,SNIa,1.0,0.0,1.0,0.0 +2020hwu,155.990067,-5.071748,NA,0.241,SNIa,1.0,0.0,1.0,0.0 +2020icp,154.420683,-2.050779,NA,0.036,SNIa,0.748,0.004,0.748,0.248 +2020icw,150.382463,3.275423,NA,0.121,SNIa,1.0,0.0,1.0,0.0 +2020icx,155.550363,3.625249,NA,0.094,SNII,0.985,0.985,0.013,0.002 +2020iga,217.725875,38.225989,NA,0.13,SNIa,1.0,0.0,1.0,0.0 +2020igg,230.32357,10.284099,NA,0.07,SNII,0.976,0.976,0.018,0.006 +2020igh,193.956668,17.329173,NA,0.078,SNIa,0.652,0.024,0.652,0.324 +2020igy,244.532171,21.066292,SNIa-norm,0.028,SNIa,1.0,0.0,1.0,0.0 +2020iio,246.667379,25.533084,NA,0.12,SNIa,1.0,0.0,1.0,0.0 +2020iir,238.507026,25.005128,NA,0.208,SNIa,1.0,0.0,1.0,0.0 +2020iiv,226.547654,-6.994086,NA,0.181,SNIa,0.974,0.02,0.974,0.006 +2020ijc,248.47967,33.904975,NA,0.166,SNIa,0.777,0.098,0.777,0.125 +2020ije,135.226231,28.070519,NA,0.189,SNIa,0.976,0.007,0.976,0.017 +2020ika,169.817082,14.486441,NA,0.207,SNIa,1.0,0.0,1.0,0.0 +2020ikq,204.020875,28.983359,SNIIb,0.089,SNIa,1.0,0.0,1.0,0.0 +2020ikx,202.573062,32.157501,NA,0.224,SNIa,0.604,0.37,0.604,0.026 +2020iky,147.696504,31.221915,NA,0.188,SNII,0.67,0.67,0.316,0.014 +2020ikz,184.826718,13.724761,NA,0.094,SNIbc,0.646,0.005,0.349,0.646 +2020ilb,145.266392,34.73362,SNIa-norm,0.058,SNIa,1.0,0.0,1.0,0.0 +2020imb,178.165059,20.394621,NA,0.059,SNII,0.516,0.516,0.007,0.477 +2020imf,245.717623,32.838706,NA,0.152,SNIa,0.995,0.002,0.995,0.003 +2020imh,216.662688,42.829664,NA,0.193,SNIa,0.941,0.041,0.941,0.018 +2020imk,246.297503,24.260636,NA,0.185,SNIa,0.762,0.13,0.762,0.108 +2020imm,195.857487,-3.227978,NA,0.08,SNIbc,0.523,0.226,0.251,0.523 +2020imp,188.310297,10.664149,NA,0.173,SNIa,0.997,0.0,0.997,0.003 +2020inh,240.381199,34.256103,NA,0.094,SNII,0.933,0.933,0.051,0.016 +2020ino,243.294221,36.495664,NA,0.135,SNIa,1.0,0.0,1.0,0.0 +2020inp,226.622583,10.266889,SNIax,0.227,SNIa,0.993,0.007,0.993,0.0 +2020inw,201.671672,34.927709,NA,0.111,SNIa,0.648,0.084,0.648,0.268 +2020iow,193.269473,-4.640761,NA,0.073,SNII,0.993,0.993,0.004,0.003 +2020ioy,178.180455,17.111941,NA,0.091,SNIbc,0.899,0.014,0.087,0.899 +2020ioz,195.94825,20.902494,SNIa-norm,0.128,SNIa,1.0,0.0,1.0,0.0 +2020iqc,193.092865,17.870792,NA,0.266,SNIa,0.402,0.39,0.402,0.208 +2020iqk,208.956021,34.161655,NA,0.134,SNIa,0.994,0.0,0.994,0.006 +2020iqp,197.345878,6.846813,NA,0.23,SNIa,0.988,0.011,0.988,0.001 +2020iqr,206.662825,32.487047,NA,0.189,SNIa,0.982,0.008,0.982,0.01 +2020irb,235.845299,14.284179,NA,0.181,SNIa,0.986,0.006,0.986,0.008 +2020irg,196.37407,17.869565,NA,0.232,SNII,0.972,0.972,0.021,0.007 +2020irh,192.861143,6.589452,NA,0.132,SNII,0.992,0.992,0.003,0.005 +2020iri,196.956637,18.130372,NA,0.191,SNIa,1.0,0.0,1.0,0.0 +2020irn,186.270975,6.536647,NA,0.232,SNIa,0.849,0.084,0.849,0.067 +2020irx,236.480125,16.110519,NA,0.233,SNIa,0.835,0.125,0.835,0.04 +2020itc,239.235002,33.07754,NA,0.181,SNIa,0.997,0.003,0.997,0.0 +2020ite,186.504975,5.075445,NA,0.1,SNII,0.743,0.743,0.167,0.09 +2020ito,242.135315,31.774192,NA,0.102,SNII,0.974,0.974,0.017,0.009 +2020itp,239.704375,32.547414,NA,0.16,SNIa,0.815,0.001,0.815,0.184 +2020itq,158.199781,-1.092794,NA,0.166,SNIa,0.972,0.01,0.972,0.018 +2020itx,169.522303,13.459551,NA,0.173,SNIa,0.612,0.351,0.612,0.037 +2020itz,208.465438,34.829757,NA,0.2,SNII,0.761,0.761,0.217,0.022 +2020iul,181.488785,17.486452,NA,0.157,SNIa,0.999,0.0,0.999,0.001 +2020iuq,191.928688,11.254148,NA,0.215,SNIa,0.934,0.04,0.934,0.026 +2020ius,191.16535,11.840531,NA,0.249,SNIa,0.87,0.085,0.87,0.045 +2020iuu,208.44745,35.022333,NA,0.156,SNII,0.772,0.772,0.224,0.004 +2020iuv,210.113681,30.859596,NA,0.193,SNIa,0.783,0.091,0.783,0.126 +2020ivd,181.303483,16.641908,NA,0.165,SNIa,0.958,0.011,0.958,0.031 +2020ivf,231.702009,14.150076,NA,0.159,SNIa,0.769,0.135,0.769,0.096 +2020ivg,224.441625,41.238328,SNIIb,0.168,SNIbc,0.858,0.011,0.131,0.858 +2020ivj,228.053188,14.725389,NA,0.132,SNIa,0.995,0.001,0.995,0.004 +2020ivl,229.189008,13.630853,NA,0.204,SNIa,0.877,0.074,0.877,0.049 +2020ivq,231.617847,12.488358,NA,0.219,SNIa,0.763,0.188,0.763,0.049 +2020ivw,233.316658,12.890088,NA,0.051,SNII,0.963,0.963,0.03,0.007 +2020iwi,214.217232,42.09792,NA,0.152,SNII,0.816,0.816,0.138,0.046 +2020iwj,224.736974,38.354984,NA,0.076,SNII,0.384,0.384,0.3,0.316 +2020iwl,216.197607,37.625735,NA,0.199,SNIa,0.939,0.018,0.939,0.043 +2020iwv,205.758917,35.320089,NA,0.229,SNIa,0.897,0.044,0.897,0.059 +2020ixj,223.593608,42.159878,NA,0.179,SNIa,0.802,0.099,0.802,0.099 +2020ixk,237.680191,24.824251,NA,0.155,SNIa,1.0,0.0,1.0,0.0 +2020ixl,132.707246,27.809303,NA,0.211,SNIa,0.613,0.376,0.613,0.011 +2020ixm,241.725993,34.549838,NA,0.233,SNII,0.503,0.503,0.474,0.023 +2020ixs,237.3508,16.455015,NA,0.071,SNIbc,0.396,0.248,0.356,0.396 +2020iyt,227.440425,-7.260126,NA,0.176,SNIa,0.998,0.0,0.998,0.002 +2020iyu,216.78033,42.769976,NA,0.195,SNIa,0.999,0.001,0.999,0.0 +2020jba,240.823379,34.386947,NA,0.14,SNIa,1.0,0.0,1.0,0.0 +2020jcy,242.104157,23.604083,NA,0.186,SNII,0.735,0.735,0.262,0.003 +2020jde,134.959757,25.244659,NA,0.233,SNII,0.59,0.59,0.275,0.135 +2020jdf,137.691836,25.113898,NA,0.135,SNIa,0.479,0.082,0.479,0.439 +2020jdz,205.041646,29.465667,NA,0.091,SNII,0.761,0.761,0.188,0.051 +2020jed,174.978737,10.907668,NA,0.092,SNII,0.997,0.997,0.003,0.0 +2020jfo,185.460337,4.481695,SNII,0.019,SNII,0.839,0.839,0.04,0.121 +2020jfx,224.841338,39.780251,NA,0.142,SNIa,1.0,0.0,1.0,0.0 +2020jgl,142.243417,-14.805574,SNIa-norm,0.013,SNIa,0.994,0.001,0.994,0.005 +2020jhs,142.058708,25.670369,SNIIn,0.102,SNIbc,0.487,0.481,0.032,0.487 +2020jip,193.45608,2.108813,NA,0.063,SNIbc,0.932,0.017,0.051,0.932 +2020jjb,322.602721,11.602639,NA,0.038,SNIbc,0.936,0.001,0.063,0.936 +2020jjp,239.089454,33.030703,NA,0.136,SNIa,0.496,0.067,0.496,0.437 +2020jli,219.789168,-9.232294,NA,0.126,SNII,0.936,0.936,0.045,0.019 +2020jlj,228.059342,9.67316,NA,0.224,SNIa,0.999,0.0,0.999,0.001 +2020jlk,197.237008,-0.735595,NA,0.129,SNIbc,0.823,0.087,0.09,0.823 +2020jmm,223.298313,-12.473876,NA,0.125,SNII,0.966,0.966,0.033,0.001 +2020jnx,178.10772,21.302485,NA,0.096,SNII,0.999,0.999,0.001,0.0 +2020jny,180.74375,20.08544,SNIa-norm,0.038,SNIa,1.0,0.0,1.0,0.0 +2020jnz,181.460541,19.775107,NA,0.115,SNII,0.929,0.929,0.062,0.009 +2020jpm,218.490548,39.61688,NA,0.187,SNIa,1.0,0.0,1.0,0.0 +2020jpo,192.525846,13.026872,NA,0.213,SNII,0.668,0.668,0.314,0.018 +2020jpp,183.29202,13.416349,NA,0.213,SNII,0.561,0.561,0.416,0.023 +2020jpq,152.734575,4.299816,NA,0.177,SNIa,0.683,0.276,0.683,0.041 +2020jpw,142.603224,-14.516157,NA,0.17,SNIa,0.977,0.012,0.977,0.011 +2020jtj,242.668971,25.442064,NA,0.119,SNIa,0.805,0.041,0.805,0.154 +2020jtk,243.01015,24.502778,NA,0.14,SNIa,1.0,0.0,1.0,0.0 +2020jtq,146.770542,33.786916,NA,0.131,SNIa,0.997,0.0,0.997,0.003 +2020jtr,182.170759,19.874358,NA,0.165,SNIa,0.909,0.04,0.909,0.051 +2020jts,139.382816,34.741704,NA,0.142,SNIa,0.996,0.001,0.996,0.003 +2020juk,155.115481,-1.970072,NA,0.076,SNII,0.98,0.98,0.006,0.014 +2020jup,241.042038,27.57695,NA,0.147,SNIa,0.998,0.001,0.998,0.001 +2020juq,236.286792,21.878269,SNIa-norm,0.134,SNIa,1.0,0.0,1.0,0.0 +2020jur,226.142651,-11.410566,NA,0.149,SNIa,0.999,0.001,0.999,0.0 +2020juu,190.188348,13.316997,NA,0.145,SNIbc,0.585,0.019,0.396,0.585 +2020jvi,231.534765,10.009661,NA,0.192,SNIbc,0.69,0.01,0.3,0.69 +2020jvk,238.756296,17.050254,NA,0.075,SNIa,0.999,0.0,0.999,0.001 +2020jwb,221.227496,-7.073722,NA,0.166,SNIa,0.795,0.195,0.795,0.01 +2020jwr,190.418871,1.637389,NA,0.188,SNIa,1.0,0.0,1.0,0.0 +2020jww,242.714942,27.161696,SNII,0.04,SNII,0.99,0.99,0.008,0.002 +2020jxa,238.14381,25.576606,NA,0.147,SNII,0.907,0.907,0.038,0.055 +2020jyp,152.647468,1.77765,NA,0.108,SNII,0.928,0.928,0.048,0.024 +2020jzp,240.389175,36.292517,NA,0.262,SNIa,0.896,0.088,0.896,0.016 +2020jzs,232.457759,13.936722,NA,0.115,SNIa,1.0,0.0,1.0,0.0 +2020jzx,204.331299,27.970527,NA,0.131,SNII,0.915,0.915,0.046,0.039 +2020kaj,203.703579,28.402486,SNIa-norm,0.153,SNIa,0.995,0.0,0.995,0.005 +2020kak,202.903987,30.930326,NA,0.229,SNIa,0.773,0.038,0.773,0.189 +2020kax,230.901667,10.625135,NA,0.164,SNIa,1.0,0.0,1.0,0.0 +2020kbl,243.817746,21.093383,SNIa-norm,0.078,SNIa,1.0,0.0,1.0,0.0 +2020kcl,242.30452,32.861926,NA,0.168,SNIa,0.541,0.206,0.541,0.253 +2020kdc,231.853204,12.318747,NA,0.145,SNIa,0.939,0.042,0.939,0.019 +2020kfe,190.892351,2.645497,NA,0.21,SNIa,0.633,0.299,0.633,0.068 +2020kff,189.215376,-0.722258,NA,0.225,SNII,0.604,0.604,0.376,0.02 +2020kfg,193.121176,6.150437,NA,0.256,SNIa,0.987,0.007,0.987,0.006 +2020kfh,169.666929,9.325029,NA,0.242,SNIa,0.995,0.002,0.995,0.003 +2020kfi,190.248876,1.65585,NA,0.221,SNIa,0.475,0.424,0.475,0.101 +2020kfj,197.223353,-1.381662,NA,0.151,SNIa,0.986,0.003,0.986,0.011 +2020kfy,147.435779,30.371788,NA,0.115,SNIa,0.999,0.0,0.999,0.001 +2020kgr,194.959688,17.958518,NA,0.11,SNII,0.925,0.925,0.016,0.059 +2020kim,224.042388,40.420288,NA,0.23,SNIa,0.989,0.007,0.989,0.004 +2020kiq,185.48456,12.608834,NA,0.19,SNII,0.874,0.874,0.103,0.023 +2020kir,221.402108,-8.727239,NA,0.188,SNIa,0.987,0.011,0.987,0.002 +2020kit,200.767692,32.009264,NA,0.13,SNIa,0.996,0.004,0.996,0.0 +2020kld,154.512812,3.287436,NA,0.151,SNIa,1.0,0.0,1.0,0.0 +2020kmj,188.174896,6.972687,NA,0.091,SNII,0.512,0.512,0.025,0.463 +2020kml,225.774198,-2.679658,NA,0.092,SNII,0.567,0.567,0.366,0.067 +2020kmm,232.580043,-0.190944,NA,0.095,SNIa,0.985,0.011,0.985,0.004 +2020kmv,186.40696,7.392595,NA,0.12,SNIa,0.593,0.238,0.593,0.169 +2020knn,226.545637,14.185711,NA,0.26,SNIa,0.588,0.399,0.588,0.013 +2020kns,201.988042,31.143706,NA,0.181,SNIa,0.869,0.057,0.869,0.074 +2020knu,196.861242,17.713972,NA,0.212,SNIa,0.996,0.003,0.996,0.001 +2020knv,140.693465,32.354417,NA,0.153,SNIa,0.655,0.274,0.655,0.071 +2020knw,227.019309,-3.897698,NA,0.156,SNII,0.863,0.863,0.124,0.013 +2020knx,213.598175,-5.178336,NA,0.156,SNIa,0.558,0.381,0.558,0.061 +2020koa,186.358788,8.949467,NA,0.177,SNIa,0.926,0.045,0.926,0.029 +2020kob,240.518063,33.246751,NA,0.077,SNII,0.997,0.997,0.002,0.001 +2020koc,239.952154,27.260575,SNII,0.195,SNIbc,0.562,0.002,0.436,0.562 +2020koe,241.539613,34.114776,NA,0.101,SNII,0.721,0.721,0.227,0.052 +2020kof,190.152642,10.056707,NA,0.206,SNIa,0.998,0.002,0.998,0.0 +2020koh,196.595167,-3.690642,NA,0.177,SNIa,0.748,0.066,0.748,0.186 +2020kon,188.373828,10.390554,NA,0.232,SNIa,0.849,0.063,0.849,0.088 +2020kop,184.076993,10.35686,NA,0.155,SNII,0.991,0.991,0.007,0.002 +2020kpf,189.579417,-0.716635,NA,0.156,SNII,0.953,0.953,0.01,0.037 +2020kpk,228.273565,13.929952,NA,0.129,SNIa,0.997,0.001,0.997,0.002 +2020kpm,239.996958,32.542714,NA,0.067,SNII,0.985,0.985,0.013,0.002 +2020kpp,244.715892,23.726319,NA,0.235,SNIa,0.997,0.003,0.997,0.0 +2020kps,150.674231,3.012066,NA,0.134,SNII,0.569,0.569,0.323,0.108 +2020kpv,237.420154,27.484858,NA,0.264,SNIa,0.869,0.091,0.869,0.04 +2020kpz,187.731061,9.152848,SNII,0.045,SNIbc,0.855,0.039,0.106,0.855 +2020kqb,235.050798,14.28438,NA,0.045,SNII,0.999,0.999,0.001,0.0 +2020kqz,228.109198,-1.774139,NA,0.173,SNIa,0.442,0.301,0.442,0.257 +2020kre,197.624483,-1.314192,SNIa-CSM,0.173,SNIbc,0.627,0.022,0.351,0.627 +2020kry,196.714193,21.113931,NA,0.062,SNIa,0.721,0.079,0.721,0.2 +2020kuv,197.433209,-1.293389,NA,0.155,SNIa,0.637,0.259,0.637,0.104 +2020kuw,215.248938,39.242374,NA,0.208,SNII,0.539,0.539,0.431,0.03 +2020kux,141.59725,-13.931933,NA,0.213,SNIa,0.955,0.026,0.955,0.019 +2020kuy,194.135237,17.753161,NA,0.155,SNIa,0.993,0.007,0.993,0.0 +2020kva,184.466227,21.114014,NA,0.187,SNIa,0.962,0.0,0.962,0.038 +2020kvb,219.925062,38.421699,NA,0.195,SNII,0.789,0.789,0.188,0.023 +2020kvc,193.964064,-3.801126,NA,0.183,SNIa,0.811,0.106,0.811,0.083 +2020kvd,202.744516,34.916001,NA,0.169,SNII,0.623,0.623,0.302,0.075 +2020kvf,202.955902,30.542808,NA,0.156,SNIa,0.988,0.001,0.988,0.011 +2020kvg,221.428291,42.439943,NA,0.219,SNII,0.485,0.485,0.385,0.13 +2020kvl,193.361146,-2.445875,SNIa-norm,0.138,SNIa,0.906,0.0,0.906,0.094 +2020kxp,149.547925,7.526425,NA,0.183,SNIa,0.886,0.095,0.886,0.019 +2020kxr,197.764447,-3.975379,NA,0.239,SNII,0.676,0.676,0.317,0.007 +2020kxv,248.144563,31.955035,NA,0.129,SNIa,0.85,0.07,0.85,0.08 +2020kzy,237.957681,28.941416,NA,0.184,SNIa,0.992,0.006,0.992,0.002 +2020lac,227.499253,-0.536983,NA,0.231,SNIa,1.0,0.0,1.0,0.0 +2020lad,174.698471,12.453683,NA,0.155,SNIa,0.752,0.13,0.752,0.118 +2020lbf,310.934433,-1.238087,SNIa-norm,0.096,SNIa,1.0,0.0,1.0,0.0 +2020lbh,205.854225,35.184108,SNIc,0.133,SNIbc,0.714,0.17,0.116,0.714 +2020ldd,194.373081,2.484276,NA,0.169,SNIa,0.996,0.0,0.996,0.004 +2020lev,189.789546,-5.388393,NA,0.188,SNII,0.433,0.433,0.329,0.238 +2020lew,203.210486,32.539881,NA,0.236,SNIa,0.893,0.078,0.893,0.029 +2020lex,231.105458,9.115304,NA,0.227,SNIa,1.0,0.0,1.0,0.0 +2020ley,194.229458,2.347464,NA,0.262,SNIa,0.855,0.04,0.855,0.105 +2020lez,235.417635,12.779217,NA,0.156,SNIa,0.665,0.29,0.665,0.045 +2020lfi,315.370783,-0.285398,SNII,0.063,SNIa,0.582,0.0,0.582,0.418 +2020lgr,225.131766,39.553648,NA,0.156,SNIa,1.0,0.0,1.0,0.0 +2020lgy,210.077239,33.814065,NA,0.063,SNII,1.0,1.0,0.0,0.0 +2020lht,184.31585,5.201133,SNIa-norm,0.152,SNIa,0.997,0.0,0.997,0.003 +2020lhw,317.946421,10.615368,NA,0.111,SNII,0.999,0.999,0.0,0.001 +2020lhx,142.640784,-15.428755,NA,0.091,SNIa,0.729,0.103,0.729,0.168 +2020lhy,187.368784,11.196444,NA,0.171,SNIa,0.999,0.001,0.999,0.0 +2020lhz,195.637387,21.838819,NA,0.167,SNII,0.594,0.594,0.137,0.269 +2020lib,194.004793,-1.769824,NA,0.129,SNIa,0.976,0.003,0.976,0.021 +2020lis,333.761593,6.05595,NA,0.123,SNIa,0.997,0.0,0.997,0.003 +2020lks,209.672656,30.656105,NA,0.144,SNII,0.963,0.963,0.026,0.011 +2020lky,198.098517,-0.911736,NA,0.193,SNII,0.56,0.56,0.402,0.038 +2020lly,235.965013,13.681836,NA,0.13,SNII,0.977,0.977,0.023,0.0 +2020llz,225.399746,-2.938223,NA,0.156,SNIa,0.696,0.186,0.696,0.118 +2020lnm,192.872208,-5.429018,NA,0.07,SNIa,0.948,0.0,0.948,0.052 +2020lom,325.734179,4.305373,NA,0.128,SNII,0.367,0.367,0.285,0.348 +2020lrr,210.277981,34.580872,SNII,0.036,SNIa,0.987,0.0,0.987,0.013 +2020lsz,325.357794,13.148172,NA,0.143,SNIa,0.931,0.001,0.931,0.068 +2020ltj,240.771517,32.567802,NA,0.191,SNII,0.692,0.692,0.258,0.05 +2020ltk,143.179413,36.070522,NA,0.21,SNIa,0.499,0.492,0.499,0.009 +2020ltr,149.912008,3.069447,NA,0.14,SNIa,0.997,0.003,0.997,0.0 +2020lxc,244.216346,27.042083,NA,0.083,SNIbc,0.963,0.003,0.034,0.963 +2020lxd,172.541214,12.832518,NA,0.194,SNIa,0.931,0.032,0.931,0.037 +2020lxe,169.288808,13.357283,NA,0.103,SNIa,0.969,0.018,0.969,0.013 +2020lyb,160.426204,-0.846822,NA,0.266,SNIa,0.824,0.104,0.824,0.072 +2020lyv,150.501519,5.01633,NA,0.271,SNIa,0.66,0.093,0.66,0.247 +2020lyy,155.430713,2.442019,NA,0.17,SNIa,0.752,0.042,0.752,0.206 +2020lyz,186.449599,17.471221,NA,0.116,SNIa,0.494,0.424,0.494,0.082 +2020lzp,196.390233,-4.964514,NA,0.151,SNIa,0.461,0.459,0.461,0.08 +2020mbg,228.538921,-2.792643,NA,0.234,SNII,0.857,0.857,0.111,0.032 +2020mbh,249.382864,35.97594,NA,0.074,SNIa,0.929,0.014,0.929,0.057 +2020mbi,244.174947,36.699866,NA,0.16,SNIa,1.0,0.0,1.0,0.0 +2020mbn,191.333952,13.715857,NA,0.155,SNII,0.594,0.594,0.345,0.061 +2020mcd,212.036422,-4.051672,NA,0.122,SNIa,1.0,0.0,1.0,0.0 +2020mfa,158.599771,-5.442664,NA,0.156,SNIa,0.718,0.113,0.718,0.169 +2020mfb,192.512258,-4.711208,NA,0.156,SNIa,0.759,0.117,0.759,0.124 +2020mgm,231.678383,-1.352908,NA,0.076,SNII,0.658,0.658,0.155,0.187 +2020mhg,229.252788,-4.508728,NA,0.204,SNIa,0.942,0.037,0.942,0.021 +2020mjn,232.139556,13.010147,NA,0.081,SNIbc,0.859,0.012,0.129,0.859 +2020mju,243.514152,33.901439,NA,0.172,SNIa,1.0,0.0,1.0,0.0 +2020mka,230.591665,13.522553,NA,0.135,SNIa,0.819,0.033,0.819,0.148 +2020mks,236.454971,19.684578,SNII,0.043,SNII,0.996,0.996,0.0,0.004 +2020mms,246.469043,26.690541,NA,0.179,SNIa,0.995,0.002,0.995,0.003 +2020mnx,194.622887,-5.097069,NA,0.118,SNIa,0.529,0.312,0.529,0.159 +2020mor,185.686949,8.343828,NA,0.152,SNIa,0.999,0.001,0.999,0.0 +2020mqf,174.60475,8.992906,NA,0.2,SNIa,0.703,0.264,0.703,0.033 +2020mrv,227.614895,-3.015327,NA,0.136,SNIa,0.999,0.0,0.999,0.001 +2020mrw,328.623428,7.204962,NA,0.088,SNIa,1.0,0.0,1.0,0.0 +2020msu,219.869944,38.447405,SNIa-norm,0.114,SNIa,1.0,0.0,1.0,0.0 +2020mvu,183.04427,18.482788,NA,0.153,SNIa,1.0,0.0,1.0,0.0 +2020mvw,148.681364,2.702313,NA,0.074,SNIa,0.848,0.101,0.848,0.051 +2020mvx,187.668685,11.033212,NA,0.178,SNIbc,0.454,0.159,0.387,0.454 +2020mvy,226.762096,9.7272,NA,0.136,SNIa,1.0,0.0,1.0,0.0 +2020mwc,149.846896,6.497194,NA,0.077,SNII,0.856,0.856,0.095,0.049 +2020mye,2.460287,21.753127,NA,0.123,SNIa,0.999,0.0,0.999,0.001 +2020myi,0.650022,23.151742,SNIa-norm,0.036,SNIa,1.0,0.0,1.0,0.0 +2020mza,3.874453,17.324312,NA,0.023,SNII,0.687,0.687,0.166,0.147 +2020mzr,0.901667,19.985024,NA,0.199,SNIa,1.0,0.0,1.0,0.0 +2020nap,181.78675,16.921169,SNII,0.032,SNII,0.694,0.694,0.188,0.118 +2020nbo,239.558958,18.102814,SNIa-norm,0.1,SNIa,1.0,0.0,1.0,0.0 +2020nbv,195.038217,-4.438361,NA,0.214,SNIa,0.757,0.184,0.757,0.059 +2020nbw,193.294367,-1.821428,NA,0.15,SNII,0.498,0.498,0.323,0.179 +2020nbx,193.955091,21.076758,NA,0.186,SNIa,0.553,0.406,0.553,0.041 +2020ncg,232.74677,-3.664619,NA,0.055,SNII,0.994,0.994,0.006,0.0 +2020nde,230.264689,12.926658,NA,0.104,SNIa,1.0,0.0,1.0,0.0 +2020ndf,190.074187,11.454727,NA,0.21,SNIa,0.967,0.018,0.967,0.015 +2020ndh,233.781217,9.750806,NA,0.173,SNIa,0.921,0.035,0.921,0.044 +2020ndi,231.313134,8.971973,NA,0.23,SNIa,0.994,0.0,0.994,0.006 +2020ndj,195.902907,16.938673,NA,0.224,SNIa,0.999,0.001,0.999,0.0 +2020neh,230.333667,14.069628,TDE,0.065,SNIa,0.533,0.002,0.533,0.465 +2020nff,193.364317,-4.53585,NA,0.228,SNIa,0.49,0.324,0.49,0.186 +2020nfg,239.207454,32.237107,NA,0.238,SNIa,0.838,0.103,0.838,0.059 +2020nfh,236.793844,16.301581,NA,0.156,SNIa,0.436,0.306,0.436,0.258 +2020ngu,231.233313,12.436478,NA,0.212,SNII,0.594,0.594,0.387,0.019 +2020nhj,242.651448,20.101164,NA,0.194,SNIa,0.799,0.019,0.799,0.182 +2020nhl,221.132071,-7.860108,NA,0.132,SNIa,0.558,0.054,0.558,0.388 +2020nib,241.091275,34.299179,NA,0.23,SNII,0.76,0.76,0.235,0.005 +2020nic,243.828062,27.897536,NA,0.241,SNIa,0.944,0.035,0.944,0.021 +2020nie,222.764476,38.465525,NA,0.21,SNIa,0.953,0.03,0.953,0.017 +2020nim,244.218175,26.735868,SNII,0.127,SNIbc,0.842,0.005,0.153,0.842 +2020nis,227.965037,-11.183486,NA,0.221,SNIa,0.999,0.0,0.999,0.001 +2020njw,243.674733,31.047722,NA,0.199,SNIbc,0.573,0.041,0.386,0.573 +2020nki,193.837496,19.462228,NA,0.13,SNIa,0.999,0.0,0.999,0.001 +2020nlb,186.35075,18.203478,SNIa-norm,0.009,SNIa,0.999,0.001,0.999,0.0 +2020nlk,5.304707,28.148409,NA,0.099,SNIa,1.0,0.0,1.0,0.0 +2020nnk,241.607121,25.275023,NA,0.187,SNIa,1.0,0.0,1.0,0.0 +2020noq,227.298044,-10.546491,NA,0.162,SNIa,1.0,0.0,1.0,0.0 +2020nor,230.847391,14.167783,NA,0.172,SNIa,1.0,0.0,1.0,0.0 +2020nov,254.554085,2.117537,TDE,0.078,SNIbc,0.413,0.377,0.21,0.413 +2020noz,187.251065,7.849545,SNIIn,0.034,SNIbc,0.745,0.008,0.247,0.745 +2020npn,314.332508,-2.100167,NA,0.215,SNIa,0.955,0.025,0.955,0.02 +2020nqu,355.760619,24.50802,NA,0.117,SNII,0.998,0.998,0.001,0.001 +2020nrg,53.637179,-4.810514,NA,0.025,SNII,0.703,0.703,0.196,0.101 +2020nrh,51.971987,-3.802636,NA,0.056,SNIbc,0.722,0.1,0.178,0.722 +2020nrm,227.026443,8.945308,NA,0.218,SNIa,1.0,0.0,1.0,0.0 +2020nrp,185.777704,19.716094,NA,0.176,SNIa,1.0,0.0,1.0,0.0 +2020nsp,244.667354,30.832242,NA,0.156,SNII,0.999,0.999,0.001,0.0 +2020nuc,220.080042,41.408539,NA,0.064,SNII,1.0,1.0,0.0,0.0 +2020nud,6.622427,19.487148,NA,0.165,SNIa,0.992,0.002,0.992,0.006 +2020nue,313.462167,-0.183882,NA,0.198,SNIa,0.864,0.11,0.864,0.026 +2020nva,41.527297,37.972531,SNIa-norm,0.062,SNIa,0.998,0.0,0.998,0.002 +2020nvb,187.244935,3.572782,SNIa-norm,0.055,SNIbc,0.478,0.09,0.432,0.478 +2020nvd,6.541825,28.067598,NA,0.138,SNII,0.564,0.564,0.015,0.421 +2020nwa,332.777896,0.303206,NA,0.185,SNIa,1.0,0.0,1.0,0.0 +2020nxl,2.049388,19.932797,NA,0.112,SNIa,1.0,0.0,1.0,0.0 +2020nxt,339.400983,35.002122,SNIbn,0.029,SNIa,0.854,0.107,0.854,0.039 +2020oa,197.372716,7.545379,NA,0.076,SNIa,1.0,0.0,1.0,0.0 +2020oaf,224.032533,-8.159156,NA,0.131,SNII,0.517,0.517,0.398,0.085 +2020oay,193.706084,-4.578441,NA,0.09,SNIa,0.996,0.004,0.996,0.0 +2020odn,322.206636,9.907038,NA,0.239,SNIa,0.874,0.091,0.874,0.035 +2020odq,333.343463,5.467709,NA,0.113,SNII,0.992,0.992,0.003,0.005 +2020odr,239.266208,32.241553,NA,0.238,SNIa,0.692,0.101,0.692,0.207 +2020odt,228.757388,-1.320396,NA,0.204,SNII,0.903,0.903,0.093,0.004 +2020odu,242.863814,24.680614,NA,0.136,SNIa,0.555,0.213,0.555,0.232 +2020odw,226.607376,-10.516432,NA,0.238,SNIa,0.983,0.016,0.983,0.001 +2020oen,228.298975,9.889328,NA,0.2,SNII,0.519,0.519,0.412,0.069 +2020ofp,226.20246,-2.602367,NA,0.106,SNIa,0.899,0.086,0.899,0.015 +2020ofw,195.139466,27.504522,SNII,0.02,SNII,0.946,0.946,0.036,0.018 +2020ogv,196.393395,-2.586303,NA,0.052,SNIa,0.99,0.01,0.99,0.0 +2020ohy,229.492426,11.223258,NA,0.164,SNII,0.487,0.487,0.417,0.096 +2020oi,185.728875,15.823594,SNIc,0.022,SNIbc,0.853,0.011,0.136,0.853 +2020oiv,219.458381,43.205534,NA,0.226,SNIa,0.981,0.012,0.981,0.007 +2020oiw,219.289986,37.933119,NA,0.157,SNII,0.866,0.866,0.116,0.018 +2020oix,193.254805,6.625266,NA,0.217,SNIa,0.908,0.03,0.908,0.062 +2020oiy,218.439994,42.069234,NA,0.176,SNIa,0.962,0.031,0.962,0.007 +2020ojl,220.681743,-7.355098,NA,0.176,SNIa,0.911,0.057,0.911,0.032 +2020oku,195.756023,2.319084,NA,0.214,SNIa,0.736,0.235,0.736,0.029 +2020olk,236.214075,18.363897,NA,0.114,SNII,0.571,0.571,0.347,0.082 +2020onu,17.865782,-17.36828,SNIa-norm,0.049,SNIa,1.0,0.0,1.0,0.0 +2020ooo,333.435543,6.448099,NA,0.155,SNIa,0.991,0.0,0.991,0.009 +2020oos,193.784872,-2.557171,NA,0.151,SNIa,0.96,0.016,0.96,0.024 +2020oot,204.815066,29.319215,NA,0.144,SNII,0.762,0.762,0.22,0.018 +2020oou,196.474776,-4.140306,NA,0.09,SNII,0.92,0.92,0.079,0.001 +2020oov,216.306572,43.40218,NA,0.201,SNIa,0.988,0.011,0.988,0.001 +2020oow,196.626809,-4.517585,NA,0.195,SNIa,0.965,0.0,0.965,0.035 +2020oox,220.856116,42.729736,NA,0.212,SNIa,0.999,0.0,0.999,0.001 +2020opa,321.597025,11.473488,NA,0.094,SNIa,0.859,0.0,0.859,0.141 +2020opy,239.107211,23.372504,TDE,0.195,SNIa,0.765,0.0,0.765,0.235 +2020ore,234.064038,12.768706,NA,0.137,SNIa,1.0,0.0,1.0,0.0 +2020ovg,241.7722,28.788727,NA,0.107,SNII,0.996,0.996,0.004,0.0 +2020ovi,235.836019,20.863375,NA,0.289,SNIa,1.0,0.0,1.0,0.0 +2020ovk,358.573856,26.326704,SNII,0.033,SNII,1.0,1.0,0.0,0.0 +2020ovv,184.051336,9.507194,NA,0.115,SNIa,0.917,0.071,0.917,0.012 +2020owi,228.369575,-3.595869,NA,0.101,SNII,0.963,0.963,0.016,0.021 +2020oxt,317.841321,0.278,NA,0.206,SNII,0.705,0.705,0.276,0.019 +2020pcl,227.332963,9.791917,NA,0.131,SNII,0.791,0.791,0.024,0.185 +2020pcz,241.819758,20.396261,NA,0.191,SNII,0.938,0.938,0.052,0.01 +2020pf,186.519583,12.459903,SNIa-norm,0.222,SNIa,1.0,0.0,1.0,0.0 +2020pfs,247.254537,33.002233,NA,0.188,SNII,0.908,0.908,0.082,0.01 +2020pfy,240.775617,33.626858,NA,0.116,SNIa,0.473,0.095,0.473,0.432 +2020pga,246.955383,25.247406,NA,0.213,SNIa,0.782,0.177,0.782,0.041 +2020pip,193.289105,3.348241,NA,0.227,SNIa,0.971,0.021,0.971,0.008 +2020pix,238.502913,28.17263,NA,0.114,SNIa,0.996,0.0,0.996,0.004 +2020pki,318.101243,13.004849,SNIa-norm,0.042,SNIbc,0.698,0.01,0.292,0.698 +2020pmj,358.780208,26.457436,NA,0.14,SNIa,0.989,0.003,0.989,0.008 +2020pmq,5.565098,18.564451,NA,0.175,SNIa,0.828,0.032,0.828,0.14 +2020pni,225.958192,42.114033,SNII,0.028,SNII,0.775,0.775,0.088,0.137 +2020pog,212.511385,-6.139262,NA,0.194,SNIa,0.977,0.009,0.977,0.014 +2020poh,227.486485,-12.080703,NA,0.132,SNII,0.705,0.705,0.263,0.032 +2020poi,197.216802,-4.028892,NA,0.155,SNIa,0.849,0.081,0.849,0.07 +2020ppe,9.899893,6.664144,SNIa-norm,0.069,SNIa,0.999,0.0,0.999,0.001 +2020ppq,227.557798,12.283068,NA,0.186,SNIa,1.0,0.0,1.0,0.0 +2020pqw,193.836641,17.786313,NA,0.179,SNIa,0.908,0.061,0.908,0.031 +2020prg,12.594697,-20.215518,SNIa-norm,0.142,SNIa,1.0,0.0,1.0,0.0 +2020pru,240.316821,17.768019,NA,0.04,SNIbc,0.526,0.013,0.461,0.526 +2020pss,224.269508,-7.841858,NA,0.213,SNIbc,0.684,0.016,0.3,0.684 +2020pst,18.300139,2.283857,SNIa-norm,0.043,SNIa,1.0,0.0,1.0,0.0 +2020psu,16.857011,1.043718,NA,0.179,SNIa,0.985,0.015,0.985,0.0 +2020psv,5.324888,21.071029,SNIa-norm,0.116,SNIa,0.999,0.0,0.999,0.001 +2020pth,10.641202,3.90286,NA,0.163,SNIa,0.998,0.0,0.998,0.002 +2020ptm,20.755505,14.222452,NA,0.15,SNII,0.517,0.517,0.227,0.256 +2020pux,330.254,1.968832,NA,0.225,SNIa,0.859,0.092,0.859,0.049 +2020pvj,333.428391,0.0393,NA,0.131,SNIa,1.0,0.0,1.0,0.0 +2020pvt,333.712858,6.499519,NA,0.213,SNIa,1.0,0.0,1.0,0.0 +2020pvy,334.679971,6.259911,NA,0.223,SNIa,0.658,0.278,0.658,0.064 +2020pwl,226.559077,-12.766074,NA,0.164,SNIa,0.998,0.0,0.998,0.002 +2020pwr,236.366192,20.475472,NA,0.157,SNIa,0.818,0.048,0.818,0.134 +2020pyf,237.459943,27.077806,SNIa-norm,0.081,SNIa,1.0,0.0,1.0,0.0 +2020pyi,190.698518,13.347192,NA,0.187,SNIa,0.897,0.06,0.897,0.043 +2020pyk,7.506543,26.89885,NA,0.238,SNII,0.87,0.87,0.109,0.021 +2020pyx,219.218152,-8.066207,NA,0.204,SNIa,0.713,0.206,0.713,0.081 +2020pyy,1.827206,17.077117,SNIa-norm,0.101,SNIa,1.0,0.0,1.0,0.0 +2020pzl,0.710992,24.666197,NA,0.212,SNII,0.642,0.642,0.096,0.262 +2020qbi,208.80353,32.866709,NA,0.043,SNIbc,0.976,0.004,0.02,0.976 +2020qbj,192.365393,17.20996,NA,0.195,SNIa,0.567,0.405,0.567,0.028 +2020qbu,23.472496,-13.080354,SNIc,0.03,SNIa,0.826,0.0,0.826,0.174 +2020qda,36.939654,-1.555669,NA,0.063,SNIbc,0.847,0.026,0.127,0.847 +2020qfj,236.001276,18.342309,NA,0.162,SNIa,1.0,0.0,1.0,0.0 +2020qfv,240.666789,21.170503,NA,0.128,SNIa,0.969,0.009,0.969,0.022 +2020qfw,214.034051,-5.345502,NA,0.117,SNIa,0.565,0.393,0.565,0.042 +2020qfz,225.725763,-4.405699,NA,0.212,SNIa,0.991,0.006,0.991,0.003 +2020qgl,237.224638,27.044067,NA,0.2,SNIa,0.999,0.0,0.999,0.001 +2020qgm,224.76085,0.65195,NA,0.213,SNIa,0.845,0.133,0.845,0.022 +2020qgn,231.66632,9.7741,NA,0.155,SNIa,0.993,0.001,0.993,0.006 +2020qgo,232.204589,-2.840419,NA,0.116,SNII,0.636,0.636,0.325,0.039 +2020qgp,0.563179,23.329447,NA,0.157,SNIa,0.997,0.001,0.997,0.002 +2020qhn,240.584959,27.240502,NA,0.251,SNIa,1.0,0.0,1.0,0.0 +2020qho,241.208284,24.460303,NA,0.125,SNIbc,0.798,0.002,0.2,0.798 +2020qhp,4.497723,20.677978,NA,0.175,SNIa,0.556,0.062,0.556,0.382 +2020qht,3.491009,20.695553,NA,0.184,SNIa,0.958,0.021,0.958,0.021 +2020qkx,5.626834,26.702356,SNIa-norm,0.224,SNIa,1.0,0.0,1.0,0.0 +2020qlg,215.693008,37.528381,NA,0.17,SNIa,0.905,0.068,0.905,0.027 +2020qlp,325.102936,8.870062,NA,0.214,SNIa,1.0,0.0,1.0,0.0 +2020qlq,333.729537,31.77805,SNIc,0.06,SNII,0.503,0.503,0.12,0.377 +2020qmj,11.025013,5.259994,SNIIn,0.051,SNIa,0.69,0.003,0.69,0.307 +2020qmo,10.390393,1.437454,SNII,0.033,SNIbc,0.561,0.002,0.437,0.561 +2020qmv,36.3538,-7.987278,NA,0.153,SNIa,0.996,0.0,0.996,0.004 +2020qmy,318.030193,-0.728629,NA,0.066,SNII,0.991,0.991,0.008,0.001 +2020qnh,49.983467,2.561676,NA,0.224,SNII,0.52,0.52,0.303,0.177 +2020qpu,192.004132,20.371495,NA,0.082,SNIa,0.995,0.001,0.995,0.004 +2020qqe,47.847355,2.452321,NA,0.196,SNII,0.54,0.54,0.3,0.16 +2020qqg,35.272524,-3.581696,NA,0.187,SNIa,0.97,0.028,0.97,0.002 +2020qqi,322.305525,12.43867,NA,0.179,SNIbc,0.767,0.018,0.215,0.767 +2020qql,247.742152,32.30755,SNIa-norm,0.112,SNIa,1.0,0.0,1.0,0.0 +2020qrn,207.852396,32.038106,NA,0.238,SNIa,0.824,0.156,0.824,0.02 +2020qrq,210.386342,30.878517,NA,0.022,SNIa,0.894,0.0,0.894,0.106 +2020qrt,219.486216,39.900071,NA,0.115,SNIa,0.999,0.0,0.999,0.001 +2020qrv,317.347171,10.174267,NA,0.198,SNIa,0.809,0.087,0.809,0.104 +2020qsy,14.552975,2.917489,NA,0.228,SNIa,0.998,0.0,0.998,0.002 +2020qte,226.684923,-12.351915,NA,0.211,SNIa,0.995,0.002,0.995,0.003 +2020qtp,244.1864,24.627103,NA,0.146,SNIa,1.0,0.0,1.0,0.0 +2020qwb,327.317138,7.174453,SNIa-norm,0.071,SNIbc,0.807,0.007,0.186,0.807 +2020qxl,235.114417,12.996979,NA,0.088,SNIa,0.513,0.107,0.513,0.38 +2020qxm,229.553256,13.17831,NA,0.136,SNIa,0.994,0.001,0.994,0.005 +2020qyx,243.078107,27.472175,NA,0.156,SNIbc,0.884,0.012,0.104,0.884 +2020qzd,227.181977,-7.550817,NA,0.153,SNIa,0.976,0.002,0.976,0.022 +2020qze,227.900811,13.732519,NA,0.153,SNIbc,0.697,0.064,0.239,0.697 +2020qzf,239.474921,26.736018,NA,0.096,SNIa,0.468,0.285,0.468,0.247 +2020qzg,241.612783,29.077,NA,0.096,SNIbc,0.736,0.042,0.222,0.736 +2020qzo,21.2514,2.105078,NA,0.239,SNIa,0.876,0.093,0.876,0.031 +2020rcp,241.811242,32.108481,NA,0.177,SNIa,0.722,0.087,0.722,0.191 +2020rcr,242.260902,28.200275,NA,0.241,SNIa,0.976,0.008,0.976,0.016 +2020rcs,186.017128,8.074017,NA,0.157,SNIa,0.998,0.002,0.998,0.0 +2020rdu,239.260409,24.580782,SNIIn,0.122,SNII,0.498,0.498,0.061,0.441 +2020rfe,3.813684,16.57794,NA,0.142,SNIa,1.0,0.0,1.0,0.0 +2020rfx,359.797943,18.785526,NA,0.192,SNIa,0.994,0.002,0.994,0.004 +2020rhg,9.706767,3.403693,SNII,0.164,SNII,0.85,0.85,0.087,0.063 +2020rhw,248.232938,31.264266,NA,0.084,SNIbc,0.716,0.018,0.266,0.716 +2020rii,249.016723,33.209832,SNIa-norm,0.073,SNIa,1.0,0.0,1.0,0.0 +2020rij,17.454558,2.847616,NA,0.136,SNIa,1.0,0.0,1.0,0.0 +2020riu,12.008021,2.190448,NA,0.229,SNIa,0.927,0.056,0.927,0.017 +2020rjw,196.535371,-3.405419,NA,0.158,SNIa,0.953,0.036,0.953,0.011 +2020rki,7.175363,28.164519,NA,0.07,SNII,0.999,0.999,0.001,0.0 +2020rkp,220.089216,-7.852954,NA,0.156,SNIa,0.974,0.009,0.974,0.017 +2020rkq,234.667238,13.881159,NA,0.186,SNII,0.933,0.933,0.061,0.006 +2020rkr,331.011955,7.147636,NA,0.205,SNIa,0.603,0.103,0.603,0.294 +2020rkt,36.245288,-5.261758,SNIa-norm,0.206,SNIa,1.0,0.0,1.0,0.0 +2020rla,334.047913,34.716301,NA,0.14,SNII,0.977,0.977,0.021,0.002 +2020rmc,241.857837,26.932517,NA,0.085,SNIa,1.0,0.0,1.0,0.0 +2020rmg,17.280127,-15.997169,SNIa-norm,0.029,SNIa,1.0,0.0,1.0,0.0 +2020rmo,2.218086,20.338818,NA,0.22,SNII,0.959,0.959,0.034,0.007 +2020rmp,229.092546,-3.344525,NA,0.169,SNII,0.656,0.656,0.032,0.312 +2020rmy,310.156625,-3.521631,SNIa-norm,0.147,SNIa,0.999,0.0,0.999,0.001 +2020rnl,327.901275,5.340058,NA,0.245,SNIa,1.0,0.0,1.0,0.0 +2020rof,13.703043,4.19507,NA,0.132,SNII,0.885,0.885,0.087,0.028 +2020roi,244.539324,36.773681,NA,0.241,SNII,0.728,0.728,0.108,0.164 +2020roj,236.458575,21.977906,NA,0.194,SNIa,0.903,0.075,0.903,0.022 +2020rok,321.646509,10.176144,NA,0.223,SNII,0.816,0.816,0.17,0.014 +2020rol,218.962098,37.907698,NA,0.21,SNIa,0.999,0.0,0.999,0.001 +2020rom,317.558634,-3.406252,NA,0.212,SNII,0.75,0.75,0.214,0.036 +2020ron,228.368128,9.906137,NA,0.13,SNIa,1.0,0.0,1.0,0.0 +2020roo,230.964716,10.062399,NA,0.089,SNIa,1.0,0.0,1.0,0.0 +2020rop,7.384336,20.023649,NA,0.156,SNII,0.68,0.68,0.309,0.011 +2020row,331.63483,7.62478,SNIa-norm,0.163,SNIa,1.0,0.0,1.0,0.0 +2020rpl,357.077913,24.843103,NA,0.191,SNIa,0.52,0.293,0.52,0.187 +2020rrn,209.620998,31.891871,NA,0.133,SNIa,1.0,0.0,1.0,0.0 +2020rrq,211.308108,-5.687089,NA,0.174,SNIa,0.866,0.076,0.866,0.058 +2020rrr,325.82211,9.46804,NA,0.256,SNIa,0.727,0.246,0.727,0.027 +2020rsn,18.789217,1.539094,NA,0.188,SNIa,0.495,0.451,0.495,0.054 +2020rso,340.162804,31.510947,NA,0.195,SNIa,0.507,0.324,0.507,0.169 +2020rsp,208.89626,32.32081,NA,0.087,SNIa,0.564,0.011,0.564,0.425 +2020rsr,240.805488,31.620673,NA,0.261,SNIa,0.839,0.069,0.839,0.092 +2020rss,46.425109,-1.073238,NA,0.132,SNIa,0.997,0.0,0.997,0.003 +2020rst,66.050411,-26.649108,NA,0.071,SNIbc,0.655,0.047,0.298,0.655 +2020rsu,247.64906,32.529278,NA,0.148,SNIbc,0.67,0.188,0.142,0.67 +2020rsv,67.214375,-24.838527,NA,0.143,SNIa,0.999,0.001,0.999,0.0 +2020rsw,244.772254,33.069983,NA,0.254,SNIa,0.808,0.157,0.808,0.035 +2020rsx,324.686481,10.182861,NA,0.228,SNII,0.514,0.514,0.285,0.201 +2020rtb,36.953216,-8.388946,NA,0.19,SNIa,0.996,0.001,0.996,0.003 +2020rth,52.113556,-5.254542,SNII,0.028,SNII,0.997,0.997,0.0,0.003 +2020rvg,37.047011,-2.239155,NA,0.216,SNIa,0.992,0.006,0.992,0.002 +2020rvq,245.391867,32.567458,SNIa-norm,0.175,SNIa,1.0,0.0,1.0,0.0 +2020rw,194.004001,-0.054387,NA,0.258,SNII,0.674,0.674,0.255,0.071 +2020rxb,241.395858,24.577752,NA,0.092,SNIa,0.928,0.01,0.928,0.062 +2020rxy,317.167577,14.010653,NA,0.164,SNIa,0.9,0.084,0.9,0.016 +2020ryb,227.319526,9.81284,SNIa-norm,0.063,SNIa,0.984,0.001,0.984,0.015 +2020ryn,224.353606,-3.126411,SNIa-norm,0.186,SNIa,1.0,0.0,1.0,0.0 +2020sag,328.159773,7.230755,SNIa-norm,0.092,SNIa,1.0,0.0,1.0,0.0 +2020scj,1.126042,17.542932,SNIa-norm,0.119,SNIa,1.0,0.0,1.0,0.0 +2020sck,17.645135,2.113941,SNIax,0.015,SNIa,0.768,0.001,0.768,0.231 +2020scq,40.970106,-0.618525,SNII,0.038,SNIbc,0.696,0.004,0.3,0.696 +2020sdu,16.80732,0.670233,NA,0.156,SNIa,0.972,0.009,0.972,0.019 +2020sdy,44.461323,42.261693,NA,0.035,SNIa,0.994,0.005,0.994,0.001 +2020sej,229.883079,10.671256,NA,0.155,SNIa,0.824,0.03,0.824,0.146 +2020sex,322.0373,9.96722,NA,0.232,SNII,0.732,0.732,0.238,0.03 +2020sga,320.901253,13.817495,SNIa-norm,0.078,SNIa,0.996,0.0,0.996,0.004 +2020sgf,50.719866,42.556085,SNIc,0.027,SNIbc,0.969,0.024,0.007,0.969 +2020sgy,60.914308,-7.237555,NA,0.204,SNIa,1.0,0.0,1.0,0.0 +2020shc,121.552659,35.896817,NA,0.051,SNII,0.559,0.559,0.158,0.283 +2020shf,125.352362,43.035508,SNII,0.078,SNIbc,0.735,0.192,0.073,0.735 +2020sia,5.169135,28.82265,SNIa-norm,0.144,SNIa,1.0,0.0,1.0,0.0 +2020sjo,66.591442,-10.098813,SNIa-norm,0.024,SNIa,1.0,0.0,1.0,0.0 +2020skd,315.896372,-0.17288,SNIa-norm,0.132,SNIa,1.0,0.0,1.0,0.0 +2020skh,28.114089,38.393568,NA,0.094,SNIa,0.946,0.001,0.946,0.053 +2020ski,36.469367,-4.946623,NA,0.163,SNIa,0.998,0.0,0.998,0.002 +2020skj,227.844326,-0.1361,NA,0.103,SNIa,1.0,0.0,1.0,0.0 +2020skr,40.688247,38.083711,SNIa-norm,0.105,SNIa,1.0,0.0,1.0,0.0 +2020skv,247.259816,36.552908,NA,0.171,SNIa,1.0,0.0,1.0,0.0 +2020slb,37.019231,38.468971,NA,0.198,SNIa,0.997,0.0,0.997,0.003 +2020smi,314.585394,-3.705321,NA,0.228,SNII,0.471,0.471,0.12,0.409 +2020sss,313.402698,-4.853921,NA,0.167,SNIbc,0.598,0.013,0.389,0.598 +2020ssu,38.430775,37.664369,NA,0.03,SNIbc,0.839,0.018,0.143,0.839 +2020svk,315.563945,0.087686,NA,0.219,SNIa,0.89,0.048,0.89,0.062 +2020svn,59.544618,-8.816882,SNII,0.026,SNII,0.555,0.555,0.214,0.231 +2020svo,40.633464,-0.963143,SNIa-norm,0.039,SNIa,1.0,0.0,1.0,0.0 +2020sxp,246.219266,28.821113,NA,0.189,SNIa,0.996,0.003,0.996,0.001 +2020sxr,52.313313,-28.941932,NA,0.207,SNII,0.54,0.54,0.295,0.165 +2020sxs,66.908957,-23.100251,NA,0.218,SNIa,0.998,0.001,0.998,0.001 +2020sym,22.252909,-19.108914,SNIc,0.061,SNIbc,0.939,0.011,0.05,0.939 +2020szo,322.943742,8.667252,NA,0.215,SNIa,1.0,0.0,1.0,0.0 +2020szp,335.274412,7.079796,NA,0.199,SNIa,0.668,0.05,0.668,0.282 +2020tan,17.435836,0.340233,SNIIn,0.091,SNII,0.707,0.707,0.156,0.137 +2020tao,66.331404,-22.574728,NA,0.227,SNIa,0.915,0.041,0.915,0.044 +2020tbu,40.918546,-0.280542,NA,0.136,SNIa,0.796,0.019,0.796,0.185 +2020tdk,93.524064,-21.220791,NA,0.162,SNIbc,0.681,0.03,0.289,0.681 +2020teb,46.446988,-0.161556,SNIa-norm,0.105,SNIa,1.0,0.0,1.0,0.0 +2020ted,246.941008,28.168819,NA,0.176,SNII,0.97,0.97,0.027,0.003 +2020tee,13.885617,-19.9466,NA,0.189,SNIa,0.998,0.002,0.998,0.0 +2020tfb,92.21721,-26.412771,SNII,0.078,SNII,0.897,0.897,0.016,0.087 +2020tfc,334.25334,30.655857,SNIa-norm,0.04,SNIa,1.0,0.0,1.0,0.0 +2020tfw,240.975252,34.123255,NA,0.159,SNIa,0.999,0.0,0.999,0.001 +2020tfx,41.086711,-1.360175,NA,0.183,SNIa,0.862,0.068,0.862,0.07 +2020tfy,337.767296,36.685564,SNIa-norm,0.224,SNIa,0.496,0.15,0.496,0.354 +2020tga,34.874241,-5.055477,NA,0.174,SNII,0.797,0.797,0.168,0.035 +2020tgb,37.464205,-0.757202,NA,0.192,SNIa,0.789,0.107,0.789,0.104 +2020tgc,237.868899,27.573566,NA,0.164,SNIa,0.628,0.345,0.628,0.027 +2020tgd,240.463474,36.23834,NA,0.214,SNIa,0.65,0.318,0.65,0.032 +2020tgu,244.208743,32.335946,NA,0.225,SNIa,0.498,0.486,0.498,0.016 +2020tgv,52.695894,-2.10223,NA,0.138,SNIa,0.909,0.015,0.909,0.076 +2020tgw,12.566662,-15.35447,NA,0.129,SNIa,0.993,0.0,0.993,0.007 +2020tho,45.796331,-1.959766,NA,0.106,SNIbc,0.441,0.22,0.339,0.441 +2020tht,17.726796,-18.96394,SNIa-norm,0.038,SNIa,1.0,0.0,1.0,0.0 +2020thu,318.439046,8.41655,NA,0.196,SNIa,0.913,0.045,0.913,0.042 +2020thx,22.284891,-14.793961,SNII,0.12,SNII,0.995,0.995,0.003,0.002 +2020thy,359.652867,27.504478,NA,0.168,SNIa,0.997,0.0,0.997,0.003 +2020tic,35.313525,-4.711142,NA,0.216,SNIa,1.0,0.0,1.0,0.0 +2020tip,356.693933,25.558094,SNIa-norm,0.092,SNIa,1.0,0.0,1.0,0.0 +2020tjd,86.520769,-23.566018,SNIIb,0.064,SNII,0.885,0.885,0.023,0.092 +2020tkc,1.646159,28.309405,SNIIb,0.039,SNIbc,0.981,0.002,0.017,0.981 +2020tkd,35.064292,41.99512,NA,0.1,SNIa,1.0,0.0,1.0,0.0 +2020tkw,84.77709,-23.797423,NA,0.085,SNIa,1.0,0.0,1.0,0.0 +2020tle,244.152304,27.221513,NA,0.168,SNIa,1.0,0.0,1.0,0.0 +2020tlf,220.041805,42.77762,SNII,0.016,SNIa,0.877,0.006,0.877,0.117 +2020tln,237.570096,25.104622,NA,0.089,SNII,0.987,0.987,0.012,0.001 +2020tlo,311.333329,-0.864919,NA,0.231,SNIa,0.986,0.009,0.986,0.005 +2020tly,7.360825,17.485344,SNII,0.071,SNII,1.0,1.0,0.0,0.0 +2020tmi,239.609387,31.87554,NA,0.221,SNIa,0.602,0.327,0.602,0.071 +2020toi,57.893204,-23.669994,NA,0.211,SNIa,0.395,0.21,0.395,0.395 +2020toj,5.948229,19.2192,NA,0.218,SNII,0.678,0.678,0.231,0.091 +2020tok,220.227653,37.803552,NA,0.162,SNIa,0.503,0.455,0.503,0.042 +2020tol,241.130412,26.733786,NA,0.191,SNIa,0.619,0.227,0.619,0.154 +2020tom,229.311662,10.311965,NA,0.155,SNII,0.6,0.6,0.344,0.056 +2020ton,314.01705,-1.756875,NA,0.221,SNIa,0.711,0.235,0.711,0.054 +2020too,16.824608,1.934523,NA,0.149,SNIa,0.582,0.332,0.582,0.086 +2020tpv,63.473231,-12.338714,NA,0.226,SNIa,0.77,0.209,0.77,0.021 +2020tpw,63.665073,-9.506905,NA,0.233,SNIa,0.842,0.117,0.842,0.041 +2020tqy,10.079175,1.512551,NA,0.22,SNIa,0.725,0.252,0.725,0.023 +2020tqz,333.03605,0.123469,SNIa-norm,0.137,SNIa,0.997,0.0,0.997,0.003 +2020tra,21.917757,-18.553559,NA,0.152,SNIa,1.0,0.0,1.0,0.0 +2020trb,334.966772,5.802576,NA,0.198,SNIa,0.921,0.051,0.921,0.028 +2020trc,325.370173,2.84355,NA,0.22,SNII,0.578,0.578,0.389,0.033 +2020tre,328.98005,6.500369,NA,0.156,SNII,0.773,0.773,0.128,0.099 +2020trf,331.994046,2.861728,NA,0.223,SNIa,0.758,0.2,0.758,0.042 +2020trg,3.530083,16.954436,NA,0.219,SNIa,0.84,0.072,0.84,0.088 +2020trh,24.012785,-13.804411,NA,0.245,SNIa,0.946,0.029,0.946,0.025 +2020tri,238.422858,26.841053,NA,0.146,SNIa,0.998,0.002,0.998,0.0 +2020trj,219.189371,41.393517,NA,0.198,SNIa,0.663,0.271,0.663,0.066 +2020trk,21.207564,-17.236249,NA,0.219,SNIa,0.997,0.0,0.997,0.003 +2020trl,315.860585,-0.746292,NA,0.056,SNIbc,0.599,0.325,0.076,0.599 +2020tue,18.051429,-16.635999,NA,0.229,SNII,0.918,0.918,0.069,0.013 +2020tug,359.866177,17.865214,SNIa-norm,0.053,SNIa,1.0,0.0,1.0,0.0 +2020tuu,356.244837,25.399787,NA,0.11,SNII,0.999,0.999,0.001,0.0 +2020tuy,37.633948,-0.790968,NA,0.088,SNIa,0.498,0.088,0.498,0.414 +2020tvj,42.326887,-3.920502,NA,0.291,SNIa,1.0,0.0,1.0,0.0 +2020tvk,45.777206,0.241754,NA,0.105,SNIa,0.991,0.008,0.991,0.001 +2020tvl,35.706567,-1.775704,NA,0.232,SNIa,0.587,0.334,0.587,0.079 +2020tvt,62.710375,-9.078497,NA,0.164,SNIa,1.0,0.0,1.0,0.0 +2020twa,6.862206,25.683202,NA,0.109,SNIa,0.998,0.0,0.998,0.002 +2020twb,318.690648,14.368489,NA,0.157,SNIa,0.576,0.208,0.576,0.216 +2020two,323.958664,8.93979,NA,0.196,SNIa,0.704,0.126,0.704,0.17 +2020twp,334.647592,5.333682,SNIc,0.149,SNIa,0.575,0.0,0.575,0.425 +2020txk,7.130666,24.730581,NA,0.243,SNIa,0.986,0.012,0.986,0.002 +2020tyl,7.020375,24.867103,NA,0.208,SNIa,1.0,0.0,1.0,0.0 +2020tyw,18.206905,-16.732232,NA,0.187,SNIa,0.857,0.127,0.857,0.016 +2020tyx,315.310724,-1.687966,NA,0.194,SNIa,0.997,0.001,0.997,0.002 +2020tzs,333.759341,0.465092,SNII,0.083,SNII,1.0,1.0,0.0,0.0 +2020uaq,233.704292,10.977039,SNIIn,0.144,SNII,0.706,0.706,0.194,0.1 +2020uba,22.672596,-12.731006,NA,0.085,SNIa,0.805,0.008,0.805,0.187 +2020ue,190.694908,2.659503,SNIa-norm,0.012,SNIbc,0.657,0.067,0.276,0.657 +2020ufe,81.018854,-23.024201,SNIa-norm,0.18,SNIa,0.998,0.0,0.998,0.002 +2020uga,18.727142,1.916228,NA,0.163,SNIa,0.999,0.0,0.999,0.001 +2020uhm,120.361414,36.81921,SNII,0.092,SNII,0.685,0.685,0.161,0.154 +2020uig,91.171319,-21.575559,NA,0.213,SNIa,1.0,0.0,1.0,0.0 +2020ujp,61.927678,-11.122753,NA,0.196,SNIa,0.946,0.042,0.946,0.012 +2020ulf,240.420405,21.354899,SNIa-norm,0.033,SNIa,1.0,0.0,1.0,0.0 +2020ulp,49.519244,-0.99266,NA,0.103,SNIbc,0.558,0.136,0.306,0.558 +2020ulx,36.358613,-8.742534,NA,0.115,SNIa,0.83,0.051,0.83,0.119 +2020ulz,66.212746,-10.934253,SNIa-norm,0.077,SNIa,1.0,0.0,1.0,0.0 +2020umf,358.14123,25.751322,NA,0.155,SNII,0.997,0.997,0.0,0.003 +2020unn,126.289975,44.205277,NA,0.167,SNII,0.608,0.608,0.191,0.201 +2020urt,122.15035,40.459689,NA,0.188,SNII,0.511,0.511,0.129,0.36 +2020uru,117.243692,38.598531,NA,0.065,SNIbc,0.808,0.096,0.096,0.808 +2020usc,10.278771,2.618168,NA,0.219,SNIa,1.0,0.0,1.0,0.0 +2020usd,240.153312,19.755886,NA,0.18,SNIa,0.795,0.171,0.795,0.034 +2020use,53.361738,-29.075545,NA,0.112,SNIa,1.0,0.0,1.0,0.0 +2020usj,247.013472,25.229582,NA,0.222,SNIa,0.589,0.363,0.589,0.048 +2020usk,239.219867,26.941853,NA,0.095,SNIa,0.628,0.064,0.628,0.308 +2020usl,59.067719,-25.553784,NA,0.171,SNIa,0.799,0.181,0.799,0.02 +2020usm,359.671636,29.008657,SNIa-norm,0.146,SNIa,1.0,0.0,1.0,0.0 +2020usn,53.803208,-1.326246,NA,0.2,SNIa,0.985,0.0,0.985,0.015 +2020utd,59.469212,-26.847189,NA,0.254,SNII,0.65,0.65,0.323,0.027 +2020utj,357.6614,25.734175,NA,0.132,SNIa,0.893,0.093,0.893,0.014 +2020utk,317.240079,9.443497,NA,0.252,SNIa,0.986,0.009,0.986,0.005 +2020utl,58.569653,-23.548735,NA,0.155,SNIa,0.87,0.047,0.87,0.083 +2020utm,48.864375,-3.010598,SNIIn,0.056,SNII,0.793,0.793,0.027,0.18 +2020uwl,22.360856,-15.412621,SNII,0.118,SNIa,1.0,0.0,1.0,0.0 +2020uwr,359.264637,17.904025,SNIa-norm,0.088,SNIa,1.0,0.0,1.0,0.0 +2020uxw,314.53871,-4.261802,NA,0.157,SNIa,0.998,0.001,0.998,0.001 +2020uxz,21.028705,12.921477,SNIa-norm,0.003,SNIa,1.0,0.0,1.0,0.0 +2020uyc,311.118404,-1.046349,NA,0.111,SNIa,0.981,0.0,0.981,0.019 +2020uys,21.393391,-18.364051,NA,0.132,SNIa,1.0,0.0,1.0,0.0 +2020uzk,138.62795,51.22555,NA,0.126,SNIa,1.0,0.0,1.0,0.0 +2020van,356.055523,24.809262,NA,0.207,SNII,0.515,0.515,0.232,0.253 +2020vbj,334.635817,35.633286,SNIa-91T-like,0.125,SNIa,0.961,0.0,0.961,0.039 +2020vbs,15.290273,-15.251279,SNIa-norm,0.054,SNIa,0.999,0.0,0.999,0.001 +2020vdk,13.577809,-14.689888,SNII,0.169,SNIa,0.426,0.211,0.426,0.363 +2020vds,37.60221,-3.765058,NA,0.192,SNIa,0.96,0.023,0.96,0.017 +2020vdt,230.677019,9.687285,NA,0.083,SNIa,0.521,0.133,0.521,0.346 +2020vfc,244.707887,23.785211,NA,0.095,SNIa,0.846,0.007,0.846,0.147 +2020vfy,335.711613,5.332556,NA,0.202,SNIa,0.848,0.142,0.848,0.01 +2020vgc,244.619942,26.512217,NA,0.109,SNIa,0.63,0.319,0.63,0.051 +2020vjz,318.96238,11.060037,NA,0.132,SNII,0.987,0.987,0.01,0.003 +2020vkd,319.338099,12.635323,NA,0.248,SNIa,0.665,0.298,0.665,0.037 +2020vom,80.869569,-23.104404,NA,0.135,SNIa,0.646,0.057,0.646,0.297 +2020von,13.493559,-19.179208,NA,0.048,SNII,0.999,0.999,0.001,0.0 +2020voy,15.987501,-18.266958,NA,0.221,SNIa,0.849,0.1,0.849,0.051 +2020vpn,2.014104,16.437419,NA,0.136,SNIa,0.998,0.0,0.998,0.002 +2020vqr,310.050066,-4.064892,NA,0.153,SNII,0.611,0.611,0.198,0.191 +2020vqy,66.294137,-25.366863,NA,0.153,SNIa,0.456,0.35,0.456,0.194 +2020vtq,2.74895,27.336264,NA,0.221,SNIa,0.56,0.349,0.56,0.091 +2020vuc,38.942066,-3.628297,NA,0.189,SNIa,0.776,0.098,0.776,0.126 +2020vxe,20.335529,-19.737167,NA,0.104,SNIa,0.998,0.0,0.998,0.002 +2020vyu,241.793412,24.11915,NA,0.096,SNII,0.674,0.674,0.23,0.096 +2020vzb,22.053342,-14.585547,NA,0.155,SNIa,0.978,0.005,0.978,0.017 +2020vzc,13.998679,3.974644,NA,0.216,SNII,0.472,0.472,0.178,0.35 +2020vzd,48.222919,2.087866,NA,0.225,SNIa,0.908,0.016,0.908,0.076 +2020vze,11.764588,-20.44847,NA,0.198,SNIa,0.97,0.019,0.97,0.011 +2020vzf,90.229254,-23.966156,NA,0.215,SNII,0.682,0.682,0.238,0.08 +2020vzg,60.899681,-11.029332,NA,0.17,SNIbc,0.364,0.31,0.326,0.364 +2020vzh,57.827058,-26.729036,NA,0.19,SNIa,0.762,0.143,0.762,0.095 +2020war,343.196187,30.397492,NA,0.135,SNIa,0.985,0.005,0.985,0.01 +2020wbf,40.604319,-4.386342,NA,0.112,SNII,0.864,0.864,0.089,0.047 +2020wbh,16.827667,-14.695956,NA,0.168,SNIa,0.961,0.027,0.961,0.012 +2020wda,23.895087,-15.133741,NA,0.217,SNII,0.62,0.62,0.3,0.08 +2020wfg,126.442124,43.304141,SNIa-norm,0.131,SNIa,0.999,0.0,0.999,0.001 +2020wfo,13.375487,-20.24041,NA,0.236,SNIa,0.898,0.073,0.898,0.029 +2020whv,138.633361,31.633328,SNII,0.095,SNII,0.998,0.998,0.0,0.002 +2020wjd,131.515729,49.165466,NA,0.216,SNIa,1.0,0.0,1.0,0.0 +2020wlj,317.705392,-5.610689,NA,0.207,SNIa,0.798,0.161,0.798,0.041 +2020wmb,242.642478,23.814869,NA,0.135,SNIa,0.856,0.084,0.856,0.06 +2020wmg,11.720725,-20.355922,NA,0.248,SNIa,0.54,0.284,0.54,0.176 +2020wmz,6.537609,16.093509,NA,0.073,SNII,0.997,0.997,0.002,0.001 +2020wnz,65.934932,-21.257767,NA,0.164,SNIa,0.998,0.0,0.998,0.002 +2020woh,48.371229,-3.794534,NA,0.25,SNIa,0.882,0.089,0.882,0.029 +2020woq,334.841104,35.537298,NA,0.101,SNII,0.971,0.971,0.024,0.005 +2020wpm,52.805946,-2.785408,NA,0.188,SNIa,0.537,0.385,0.537,0.078 +2020wpn,0.967729,23.623597,NA,0.109,SNIa,0.513,0.448,0.513,0.039 +2020wpo,325.886471,2.478242,NA,0.223,SNIa,0.957,0.036,0.957,0.007 +2020wpp,45.794679,-0.257401,NA,0.247,SNIa,0.876,0.106,0.876,0.018 +2020wto,7.450614,24.123108,NA,0.166,SNIa,0.994,0.003,0.994,0.003 +2020wux,20.883361,-19.171032,SNIa-norm,0.114,SNIa,1.0,0.0,1.0,0.0 +2020wwa,7.901315,17.223695,NA,0.155,SNII,0.645,0.645,0.18,0.175 +2020wwr,334.68841,-0.165565,NA,0.169,SNIa,0.601,0.34,0.601,0.059 +2020wwt,7.303841,18.414719,NA,0.157,SNIa,1.0,0.0,1.0,0.0 +2020wxz,343.630744,30.442686,NA,0.242,SNIa,1.0,0.0,1.0,0.0 +2020wyx,41.128089,-3.910253,NA,0.143,SNII,0.999,0.999,0.001,0.0 +2020wyz,38.203098,41.09517,NA,0.116,SNIa,0.653,0.055,0.653,0.292 +2020wzd,7.162921,17.87846,NA,0.102,SNIa,1.0,0.0,1.0,0.0 +2020wzp,21.914854,-15.244873,NA,0.118,SNIa,0.782,0.09,0.782,0.128 +2020wzq,47.778689,1.642469,NA,0.183,SNIa,1.0,0.0,1.0,0.0 +2020xaq,16.243605,2.083931,NA,0.207,SNII,0.762,0.762,0.235,0.003 +2020xbi,10.23574,3.377909,NA,0.214,SNIa,0.837,0.09,0.837,0.073 +2020xef,13.465321,4.723497,NA,0.21,SNIa,0.774,0.162,0.774,0.064 +2020xek,13.641865,5.180209,NA,0.195,SNIa,0.497,0.216,0.497,0.287 +2020xez,13.376643,6.466387,NA,0.203,SNIa,0.488,0.46,0.488,0.052 +2020xft,10.801815,1.516023,NA,0.204,SNIa,0.983,0.007,0.983,0.01 +2020xgz,22.230783,-18.100128,NA,0.168,SNII,0.99,0.99,0.005,0.005 +2020xhr,63.22831,-8.564301,NA,0.195,SNII,0.944,0.944,0.029,0.027 +2020xju,339.527758,32.022583,NA,0.239,SNIa,0.644,0.271,0.644,0.085 +2020xkf,357.9435,24.545318,NA,0.136,SNIa,0.523,0.338,0.523,0.139 +2020xku,16.819971,-15.750263,NA,0.077,SNIa,0.764,0.0,0.764,0.236 +2020xlm,149.847554,6.497775,NA,0.077,SNIa,0.504,0.011,0.504,0.485 +2020xoc,23.043416,-15.228859,SNIa-norm,0.13,SNIa,0.999,0.0,0.999,0.001 +2020xpj,90.566792,-23.604427,NA,0.256,SNII,0.842,0.842,0.099,0.059 +2020xpx,94.674051,-26.84596,NA,0.149,SNIa,0.506,0.471,0.506,0.023 +2020xqb,326.338291,2.92999,SNIa-norm,0.124,SNIa,0.672,0.002,0.672,0.326 +2020xse,66.222397,-22.744988,NA,0.183,SNII,0.957,0.957,0.037,0.006 +2020xst,53.908412,-28.31909,NA,0.091,SNII,0.693,0.693,0.227,0.08 +2020xsy,163.766692,-1.539883,SLSN-II,0.121,SNII,0.492,0.492,0.185,0.323 +2020xua,342.842406,32.351311,SNII,0.029,SNII,0.984,0.984,0.011,0.005 +2020xur,313.409332,-5.06175,NA,0.193,SNIa,0.973,0.022,0.973,0.005 +2020xye,337.780529,36.7951,NA,0.197,SNIa,1.0,0.0,1.0,0.0 +2020yac,18.451826,-19.1395,NA,0.084,SNIa,0.999,0.0,0.999,0.001 +2020ybc,5.315379,17.953119,NA,0.214,SNIa,0.974,0.005,0.974,0.021 +2020ybn,92.759385,-22.297296,SNIIn,0.147,SNIa,0.693,0.0,0.693,0.307 +2020yda,90.443945,-23.19752,NA,0.229,SNIa,0.996,0.001,0.996,0.003 +2020yde,20.437567,2.227078,NA,0.208,SNIa,0.67,0.312,0.67,0.018 +2020ydf,18.0803,2.217775,NA,0.252,SNII,0.532,0.532,0.433,0.035 +2020yeg,109.094521,39.602367,NA,0.144,SNII,0.47,0.47,0.177,0.353 +2020yfm,95.159581,-26.627305,NA,0.15,SNIa,0.963,0.03,0.963,0.007 +2020yfx,52.630959,-0.817969,NA,0.153,SNIa,1.0,0.0,1.0,0.0 +2020yhi,127.31325,50.772733,NA,0.115,SNIa,0.902,0.002,0.902,0.096 +2020yhn,331.011154,3.284925,SNIa-norm,0.118,SNIa,1.0,0.0,1.0,0.0 +2020yis,37.099392,0.186314,NA,0.119,SNIa,0.887,0.092,0.887,0.021 +2020yit,343.21995,34.793206,NA,0.139,SNII,0.682,0.682,0.274,0.044 +2020yiu,48.665461,-1.808549,NA,0.134,SNII,0.918,0.918,0.06,0.022 +2020yiw,325.962762,10.165472,NA,0.212,SNIa,0.548,0.364,0.548,0.088 +2020yix,324.830904,12.2301,NA,0.212,SNIa,0.824,0.108,0.824,0.068 +2020yiy,358.946863,18.644733,NA,0.23,SNIa,0.898,0.043,0.898,0.059 +2020yja,58.904804,-26.061953,NA,0.155,SNIa,0.967,0.018,0.967,0.015 +2020yjb,238.689721,16.293772,NA,0.155,SNIbc,0.367,0.361,0.272,0.367 +2020yjc,337.460117,35.027764,NA,0.225,SNIa,0.525,0.424,0.525,0.051 +2020yjd,126.401467,44.687342,NA,0.251,SNIa,0.796,0.139,0.796,0.065 +2020ykb,64.038242,-25.474295,SNII,0.142,SNIa,0.977,0.001,0.977,0.022 +2020yko,155.914761,3.854317,NA,0.153,SNIbc,0.539,0.048,0.413,0.539 +2020ypn,315.486096,-4.59295,NA,0.153,SNIa,0.998,0.001,0.998,0.001 +2020yrj,111.130467,38.646978,NA,0.121,SNIa,0.543,0.341,0.543,0.116 +2020ytg,312.04165,-1.211144,SNII,0.053,SNII,0.512,0.512,0.307,0.181 +2020yuc,359.867979,28.559997,NA,0.14,SNIa,0.998,0.0,0.998,0.002 +2020yxr,340.105825,35.287486,NA,0.155,SNIa,1.0,0.0,1.0,0.0 +2020yyb,10.91413,4.202235,NA,0.271,SNIa,0.727,0.154,0.727,0.119 +2020yyc,58.840503,-9.131534,NA,0.205,SNIa,0.813,0.101,0.813,0.086 +2020yza,160.39654,-3.4415,SNIa-norm,0.105,SNIa,1.0,0.0,1.0,0.0 +2020zaj,309.675775,-1.217458,NA,0.06,SNIa,0.811,0.144,0.811,0.045 +2020zbr,59.981433,-12.237808,SNIa-norm,0.101,SNIa,1.0,0.0,1.0,0.0 +2020zcp,44.293914,-0.837063,SNIb,0.057,SNIbc,0.86,0.06,0.08,0.86 +2020zfn,326.084129,8.623478,SNIa-norm,0.048,SNIbc,0.549,0.002,0.449,0.549 +2020zfs,18.246996,1.040109,NA,0.163,SNIa,1.0,0.0,1.0,0.0 +2020zgc,21.070825,-19.431851,SNIa-norm,0.064,SNIa,1.0,0.0,1.0,0.0 +2020zhh,66.58267,-10.09911,SNIa-norm,0.024,SNIa,1.0,0.0,1.0,0.0 +2020zj,240.334933,16.677739,SNIa-norm,0.026,SNIa,1.0,0.0,1.0,0.0 +2020zji,57.128363,-26.483889,NA,0.097,SNIbc,0.674,0.013,0.313,0.674 +2020zjo,109.973488,37.558072,SNIa-norm,0.096,SNIa,1.0,0.0,1.0,0.0 +2020zjp,130.241965,50.303313,NA,0.128,SNIa,1.0,0.0,1.0,0.0 +2020zjt,116.942179,40.848888,NA,0.225,SNII,0.561,0.561,0.246,0.193 +2020zke,116.284621,38.404272,NA,0.106,SNIa,1.0,0.0,1.0,0.0 +2020zkg,131.57032,43.743714,NA,0.127,SNIa,1.0,0.0,1.0,0.0 +2020zmi,315.045875,-4.067011,SNIa-norm,0.037,SNIbc,0.848,0.006,0.146,0.848 +2020zmn,17.014616,1.144884,LBV,0.057,SNII,0.971,0.971,0.022,0.007 +2020zmv,120.198301,35.473794,NA,0.153,SNIa,0.999,0.0,0.999,0.001 +2020znh,66.468537,-12.236158,NA,0.112,SNIbc,0.488,0.052,0.46,0.488 +2020zoy,321.141783,9.195841,NA,0.176,SNIa,0.97,0.014,0.97,0.016 +2020zpi,40.085196,-1.446925,SNIa-norm,0.036,SNIa,1.0,0.0,1.0,0.0 +2020zql,20.296451,-17.36395,NA,0.227,SNIa,0.577,0.107,0.577,0.316 +2020zqm,23.061746,-15.133436,NA,0.105,SNIbc,0.859,0.006,0.135,0.859 +2020zqn,336.738242,31.714308,NA,0.189,SNIa,0.961,0.002,0.961,0.037 +2020zrf,96.074305,-24.848228,NA,0.155,SNIa,0.858,0.056,0.858,0.086 +2020zrg,18.377383,-18.897814,NA,0.237,SNIa,0.506,0.487,0.506,0.007 +2020zrj,53.087779,-29.100194,NA,0.157,SNIa,0.986,0.009,0.986,0.005 +2020zrv,358.574871,16.674097,NA,0.173,SNIa,0.456,0.411,0.456,0.133 +2020zti,60.226121,-12.012814,NA,0.103,SNIa,0.999,0.0,0.999,0.001 +2020zx,220.95745,41.228884,SNII,0.059,SNII,0.974,0.974,0.008,0.018 +2020zyj,4.12915,21.468139,NA,0.163,SNIa,0.946,0.008,0.946,0.046 +2020zyy,94.163516,-21.49979,SNIa-91T-like,0.106,SNIa,1.0,0.0,1.0,0.0 +2021J,186.61257,31.222369,SNIa-norm,0.009,SNIa,1.0,0.0,1.0,0.0 +2021aabh,131.012042,48.29475,NA,0.186,SNIa,0.988,0.0,0.988,0.012 +2021aabr,36.699179,-3.819589,NA,0.129,SNII,0.986,0.986,0.012,0.002 +2021aacm,310.836446,-5.556878,NA,0.153,SNIa,0.997,0.0,0.997,0.003 +2021aadc,1.136091,19.761508,SLSN-II,0.208,SNIbc,0.607,0.038,0.355,0.607 +2021aadd,319.447597,13.594622,SNIb,0.026,SNIbc,0.82,0.016,0.164,0.82 +2021aaej,11.586008,-19.558436,SNII,0.076,SNII,0.996,0.996,0.004,0.0 +2021aaez,7.328537,27.789011,SNIa-norm,0.094,SNIa,1.0,0.0,1.0,0.0 +2021aafe,6.343629,25.232447,NA,0.221,SNIa,0.999,0.0,0.999,0.001 +2021aagz,150.246512,7.138304,SNIa-SC,0.057,SNIa,1.0,0.0,1.0,0.0 +2021aahm,66.560325,-9.731069,NA,0.188,SNIa,0.871,0.109,0.871,0.02 +2021aahr,34.605046,-4.967106,NA,0.183,SNIa,0.997,0.001,0.997,0.002 +2021aaie,12.484879,-20.256844,NA,0.153,SNII,0.581,0.581,0.226,0.193 +2021aaiz,358.567692,27.958508,NA,0.069,SNIa,1.0,0.0,1.0,0.0 +2021aakk,48.031369,-1.695508,NA,0.129,SNIa,0.92,0.036,0.92,0.044 +2021aalv,40.514097,38.473249,NA,0.073,SNIbc,0.546,0.079,0.375,0.546 +2021aamn,7.273319,16.945806,NA,0.244,SNIa,0.811,0.125,0.811,0.064 +2021aamo,2.588705,19.801585,SNIa-norm,0.062,SNIbc,0.636,0.008,0.356,0.636 +2021aamp,5.580113,18.657319,NA,0.156,SNIa,0.937,0.045,0.937,0.018 +2021aaoa,63.885229,-12.092369,NA,0.171,SNIa,0.967,0.01,0.967,0.023 +2021aapa,62.712584,-9.629451,SNIIn,0.152,SNIbc,0.917,0.042,0.041,0.917 +2021aaqg,36.342687,-7.842681,SNIa-91T-like,0.044,SNIa,0.996,0.0,0.996,0.004 +2021aaqh,17.219303,-16.151483,NA,0.185,SNII,0.511,0.511,0.474,0.015 +2021aaqi,18.294374,-15.559626,SNIa-norm,0.165,SNII,0.569,0.569,0.409,0.022 +2021aaqn,39.492451,-1.831448,SNII,0.022,SNII,0.992,0.992,0.001,0.007 +2021aarr,3.49835,-0.074258,NA,0.21,SNIa,0.938,0.031,0.938,0.031 +2021aave,62.145024,-8.635113,NA,0.111,SNII,0.989,0.989,0.006,0.005 +2021aaws,63.063454,-11.506386,NA,0.189,SNIa,0.657,0.162,0.657,0.181 +2021aaxi,34.991346,-3.223703,SNIa-norm,0.161,SNIa,0.874,0.072,0.874,0.054 +2021aaxm,36.870629,-3.416906,NA,0.193,SNIa,0.443,0.382,0.443,0.175 +2021abam,17.050999,-19.790003,SNIa-norm,0.063,SNIa,0.999,0.0,0.999,0.001 +2021abax,5.464905,17.599534,NA,0.144,SNIa,1.0,0.0,1.0,0.0 +2021abbi,64.496189,-7.74814,NA,0.068,SNIa,0.886,0.079,0.886,0.035 +2021abbl,17.557494,2.302422,SNIa-91T-like,0.217,SNIa,0.999,0.0,0.999,0.001 +2021abeu,332.347796,5.306076,NA,0.095,SNIa,1.0,0.0,1.0,0.0 +2021abgw,51.290178,-3.711493,NA,0.153,SNIa,0.829,0.135,0.829,0.036 +2021abgx,3.347664,16.603888,SNIa-norm,0.126,SNIa,1.0,0.0,1.0,0.0 +2021abgy,213.153796,-4.297114,NA,0.118,SNIa,0.781,0.113,0.781,0.106 +2021abgz,16.464096,-18.510222,NA,0.122,SNII,0.839,0.839,0.108,0.053 +2021abha,64.617446,-24.719296,NA,0.184,SNIa,0.885,0.043,0.885,0.072 +2021abhb,54.432446,-27.524556,NA,0.112,SNII,0.603,0.603,0.33,0.067 +2021abhc,23.1875,-15.103906,NA,0.135,SNIa,0.854,0.071,0.854,0.075 +2021abhd,126.547056,43.054575,NA,0.171,SNIa,0.989,0.003,0.989,0.008 +2021abhe,59.823105,-10.985271,NA,0.235,SNIa,0.83,0.159,0.83,0.011 +2021abj,116.295883,39.394994,NA,0.14,SNIa,0.98,0.009,0.98,0.011 +2021abje,132.459426,47.930207,NA,0.123,SNIa,0.998,0.0,0.998,0.002 +2021abjy,17.296225,0.918444,NA,0.123,SNIa,0.808,0.154,0.808,0.038 +2021abmt,3.164821,20.169894,NA,0.059,SNIbc,0.537,0.152,0.311,0.537 +2021abng,3.148794,1.645729,NA,0.147,SNIa,0.966,0.014,0.966,0.02 +2021abor,58.899808,-8.105314,NA,0.191,SNII,0.711,0.711,0.278,0.011 +2021abpc,321.514703,10.405976,SNIa-norm,0.086,SNIa,0.999,0.0,0.999,0.001 +2021abwi,140.588831,51.113922,NA,0.094,SNII,0.758,0.758,0.033,0.209 +2021abwj,334.906251,6.527727,NA,0.113,SNIa,1.0,0.0,1.0,0.0 +2021abwk,319.811292,10.071314,SNIa-norm,0.086,SNIa,0.999,0.0,0.999,0.001 +2021abwl,59.064612,-26.480398,SNIa-norm,0.202,SNIa,0.868,0.113,0.868,0.019 +2021abwm,125.539875,49.000094,SNIa-norm,0.102,SNIa,0.996,0.0,0.996,0.004 +2021abym,138.149413,34.855041,SNIa-norm,0.072,SNIa,1.0,0.0,1.0,0.0 +2021accm,319.276312,-0.703256,SNIb,0.09,SNIa,0.594,0.054,0.594,0.352 +2021aceh,333.521576,6.824559,SNII,0.048,SNII,0.996,0.996,0.003,0.001 +2021acey,185.295167,32.697889,SNII,0.059,SNIbc,0.831,0.04,0.129,0.831 +2021acfc,41.423217,-0.560051,SNII,0.08,SNII,0.971,0.971,0.008,0.021 +2021acfi,62.23649,-9.043781,SNIa-norm,0.178,SNIa,0.997,0.0,0.997,0.003 +2021acfj,36.344339,-7.840844,SNIb,0.036,SNIbc,0.741,0.056,0.203,0.741 +2021acfu,335.729762,0.527131,NA,0.162,SNIa,0.532,0.236,0.532,0.232 +2021acfw,17.811505,1.812854,NA,0.162,SNIa,0.987,0.005,0.987,0.008 +2021acgo,7.531646,16.162881,SNIa-norm,0.14,SNIa,0.974,0.001,0.974,0.025 +2021acgu,40.304429,0.548281,SNIa-norm,0.146,SNIa,1.0,0.0,1.0,0.0 +2021achf,7.48218,20.563025,NA,0.105,SNIa,0.902,0.074,0.902,0.024 +2021achg,39.26762,42.505373,NA,0.072,SNII,0.789,0.789,0.159,0.052 +2021achh,357.014012,24.14052,NA,0.13,SNII,0.52,0.52,0.473,0.007 +2021achl,62.399121,-8.164694,NA,0.124,SNIa,0.999,0.0,0.999,0.001 +2021achm,16.786624,-15.605582,NA,0.132,SNIa,0.959,0.016,0.959,0.025 +2021achw,13.008004,-19.021617,NA,0.161,SNIa,0.533,0.302,0.533,0.165 +2021achy,329.867345,2.643973,SNII,0.094,SNIa,0.577,0.313,0.577,0.11 +2021aci,214.777232,42.457807,SNIa-norm,0.157,SNIa,0.997,0.0,0.997,0.003 +2021acjn,67.289967,-20.928317,NA,0.127,SNIa,0.372,0.265,0.372,0.363 +2021acjv,310.69046,-1.971131,SNII,0.035,SNIa,0.511,0.415,0.511,0.074 +2021acnq,39.740802,38.255254,NA,0.122,SNIa,0.768,0.097,0.768,0.135 +2021acns,320.367924,13.109678,NA,0.213,SNIa,0.934,0.056,0.934,0.01 +2021actw,17.963887,-19.481119,SNIa-norm,0.129,SNIa,0.982,0.0,0.982,0.018 +2021acvm,16.43797,-14.981891,NA,0.178,SNIa,0.643,0.336,0.643,0.021 +2021acvn,21.70686,-15.253473,NA,0.207,SNIbc,0.803,0.028,0.169,0.803 +2021acvo,13.228244,-14.740088,NA,0.198,SNIa,0.513,0.463,0.513,0.024 +2021acvp,317.70345,-4.375938,SNIa-norm,0.162,SNIa,0.914,0.016,0.914,0.07 +2021acwo,58.937983,-26.106492,NA,0.162,SNII,0.372,0.372,0.291,0.337 +2021acza,357.286924,24.684221,SNIa-norm,0.198,SNIa,0.785,0.176,0.785,0.039 +2021aczd,132.221155,49.349557,SNIa-norm,0.186,SNIa,0.999,0.001,0.999,0.0 +2021adew,309.778555,0.047498,SNIa-norm,0.097,SNIa,1.0,0.0,1.0,0.0 +2021adnv,51.629258,-2.120703,SNIa-norm,0.208,SNIa,0.995,0.0,0.995,0.005 +2021adnw,62.139254,-12.57885,SNII,0.218,SNIa,0.639,0.344,0.639,0.017 +2021adtd,139.077776,42.389612,SNIa-norm,0.056,SNIa,0.999,0.0,0.999,0.001 +2021adug,13.549121,-14.969836,SNIa-norm,0.096,SNIa,1.0,0.0,1.0,0.0 +2021adwh,150.243337,3.46727,SNIa-norm,0.196,SNIa,0.978,0.009,0.978,0.013 +2021adxd,9.845458,2.804036,SNII,0.089,SNIbc,0.741,0.016,0.243,0.741 +2021aegy,35.745769,35.66257,SNIa-norm,0.152,SNIa,1.0,0.0,1.0,0.0 +2021aeja,47.993887,2.846788,SNIa-norm,0.087,SNIa,1.0,0.0,1.0,0.0 +2021aele,53.8857,-28.514327,SNIa-norm,0.127,SNIa,1.0,0.0,1.0,0.0 +2021aepp,66.418272,-7.825537,SNII,0.031,SNII,0.606,0.606,0.343,0.051 +2021aeuw,321.253833,12.441965,SNIc,0.032,SNII,0.4,0.4,0.21,0.39 +2021afd,155.537,3.060089,NA,0.207,SNIa,0.701,0.23,0.701,0.069 +2021afe,155.613021,0.130675,NA,0.165,SNIa,0.931,0.038,0.931,0.031 +2021afeb,15.068499,-15.298335,SNIa-norm,0.045,SNIa,0.962,0.014,0.962,0.024 +2021aff,151.608937,6.912212,SNII,0.054,SNIbc,0.526,0.467,0.007,0.526 +2021afg,161.831662,-2.351689,NA,0.11,SNII,0.978,0.978,0.017,0.005 +2021afh,192.902036,5.790027,NA,0.197,SNIa,0.55,0.041,0.55,0.409 +2021afi,156.603161,6.000685,NA,0.25,SNIa,0.712,0.254,0.712,0.034 +2021afj,174.118653,13.221047,SNIa-norm,0.089,SNIa,1.0,0.0,1.0,0.0 +2021afkm,21.896704,3.489225,SNIa-norm,0.133,SNIa,0.946,0.003,0.946,0.051 +2021afra,325.107571,8.308136,SNIa-norm,0.086,SNIa,0.85,0.087,0.85,0.063 +2021afui,121.674125,38.878872,SNIa-norm,0.105,SNIa,0.879,0.051,0.879,0.07 +2021agt,196.714776,17.066447,NA,0.123,SNIbc,0.783,0.141,0.076,0.783 +2021agu,193.60077,7.042739,SNIa-norm,0.114,SNIa,1.0,0.0,1.0,0.0 +2021akq,81.704225,-23.325711,NA,0.178,SNIa,0.997,0.002,0.997,0.001 +2021alq,226.257858,12.641641,SNII,0.184,SNIbc,0.655,0.012,0.333,0.655 +2021amm,152.621242,5.624772,NA,0.153,SNIa,0.992,0.0,0.992,0.008 +2021amq,242.975246,21.008697,NA,0.163,SNIa,1.0,0.0,1.0,0.0 +2021aoi,174.93463,9.690697,NA,0.169,SNIa,0.823,0.16,0.823,0.017 +2021aon,115.702838,37.669742,NA,0.094,SNII,0.961,0.961,0.016,0.023 +2021aot,10.847421,2.030519,NA,0.135,SNIa,0.747,0.139,0.747,0.114 +2021aou,153.320129,2.961792,NA,0.221,SNIa,0.782,0.164,0.782,0.054 +2021aov,187.422487,13.607273,NA,0.199,SNIa,0.791,0.136,0.791,0.073 +2021aow,185.842023,5.953927,NA,0.193,SNIbc,0.472,0.275,0.253,0.472 +2021aox,161.730513,-4.394704,NA,0.209,SNIa,0.909,0.051,0.909,0.04 +2021aoy,9.960371,3.697658,NA,0.179,SNIa,0.648,0.302,0.648,0.05 +2021apd,16.37522,3.755237,SNIa-norm,0.12,SNIa,0.974,0.001,0.974,0.025 +2021arg,67.828255,-10.396371,SNII,0.026,SNII,0.986,0.986,0.01,0.004 +2021arl,185.750425,18.593578,SNIa-norm,0.076,SNIa,1.0,0.0,1.0,0.0 +2021aru,202.412123,32.593343,NA,0.201,SNIa,0.901,0.059,0.901,0.04 +2021at,116.785778,43.489215,SNIa-91T-like,0.142,SNIa,0.999,0.0,0.999,0.001 +2021atg,196.342383,-0.556042,NA,0.192,SNIa,0.985,0.007,0.985,0.008 +2021ath,208.201896,31.839411,NA,0.235,SNIa,0.97,0.019,0.97,0.011 +2021ati,202.773989,31.451535,NA,0.156,SNIa,0.508,0.461,0.508,0.031 +2021atj,188.271163,9.940645,NA,0.192,SNIa,0.997,0.002,0.997,0.001 +2021atk,189.773592,-5.567144,NA,0.226,SNIa,0.992,0.005,0.992,0.003 +2021avi,236.093536,17.033056,NA,0.062,SNIbc,0.834,0.011,0.155,0.834 +2021bbb,194.859229,-3.275639,NA,0.172,SNIa,0.969,0.019,0.969,0.012 +2021bbc,194.222908,-4.017644,NA,0.211,SNIa,0.892,0.083,0.892,0.025 +2021bbd,189.894438,-3.719044,NA,0.099,SNII,0.478,0.478,0.171,0.351 +2021bbp,42.148596,38.748753,NA,0.171,SNIa,0.93,0.04,0.93,0.03 +2021bbq,179.911375,19.959778,NA,0.208,SNII,0.492,0.492,0.49,0.018 +2021bbz,176.348311,20.324461,SNIa-norm,0.025,SNIa,1.0,0.0,1.0,0.0 +2021be,67.885923,-6.569593,NA,0.226,SNIa,0.559,0.422,0.559,0.019 +2021beb,112.001296,42.661794,NA,0.131,SNIa,0.986,0.008,0.986,0.006 +2021bf,17.135651,4.30361,NA,0.242,SNIa,0.827,0.169,0.827,0.004 +2021bg,17.364373,1.830296,NA,0.219,SNIa,0.985,0.003,0.985,0.012 +2021bh,60.73646,-8.469142,NA,0.238,SNIa,0.908,0.067,0.908,0.025 +2021biy,190.516756,32.535522,LRN,0.081,SNII,0.681,0.681,0.011,0.308 +2021bmb,224.325368,43.459729,SNIa-norm,0.069,SNIa,1.0,0.0,1.0,0.0 +2021bmt,93.136879,-21.12335,SNIa-norm,0.101,SNIa,1.0,0.0,1.0,0.0 +2021bmu,19.519088,14.553625,SNIa-91bg-like,0.079,SNIbc,0.956,0.013,0.031,0.956 +2021bmv,66.66115,-11.896042,SNIIn,0.131,SNII,0.488,0.488,0.312,0.2 +2021bpq,152.542304,6.750943,SNIa-norm,0.121,SNIa,0.986,0.001,0.986,0.013 +2021bsf,181.516804,20.537,SNIa-norm,0.052,SNIa,1.0,0.0,1.0,0.0 +2021bsr,135.088613,23.145503,SNIa-91T-like,0.083,SNIa,0.995,0.0,0.995,0.005 +2021bsv,162.923501,-0.432497,NA,0.095,SNIa,0.997,0.0,0.997,0.003 +2021btn,244.61772,13.774421,SNII,0.096,SNIbc,0.861,0.009,0.13,0.861 +2021bug,188.59485,2.317294,LBV,0.021,SNIbc,0.68,0.052,0.268,0.68 +2021buj,206.655896,30.371772,NA,0.193,SNIa,0.995,0.002,0.995,0.003 +2021bvy,186.745771,18.516925,NA,0.087,SNIa,1.0,0.0,1.0,0.0 +2021bwv,152.614297,2.228139,SNII,0.029,SNII,0.984,0.984,0.006,0.01 +2021bwy,194.991501,7.01177,NA,0.197,SNIa,0.98,0.009,0.98,0.011 +2021bxe,194.366864,29.20023,NA,0.192,SNIa,0.513,0.474,0.513,0.013 +2021bzk,122.502271,40.31267,NA,0.129,SNIa,0.764,0.065,0.764,0.171 +2021bzl,126.631967,48.38305,NA,0.176,SNIa,1.0,0.0,1.0,0.0 +2021bzq,68.395067,-7.476778,NA,0.162,SNIa,0.932,0.046,0.932,0.022 +2021cb,237.932029,27.795294,NA,0.185,SNIbc,0.633,0.006,0.361,0.633 +2021cbe,132.849867,-1.343767,NA,0.238,SNII,0.379,0.379,0.354,0.267 +2021cca,154.08046,-1.787467,SNIa-norm,0.201,SNIa,0.999,0.0,0.999,0.001 +2021ccl,196.609052,0.277946,SNIa-norm,0.101,SNIa,1.0,0.0,1.0,0.0 +2021cdn,234.724704,7.134738,NA,0.081,SNII,0.527,0.527,0.016,0.457 +2021cgf,133.127208,26.843975,NA,0.107,SNIa,0.557,0.05,0.557,0.393 +2021cgj,135.256653,45.474968,NA,0.112,SNIbc,0.847,0.011,0.142,0.847 +2021chy,183.692888,13.3314,NA,0.046,SNIbc,0.663,0.163,0.174,0.663 +2021cji,140.180847,50.543977,NA,0.097,SNII,0.872,0.872,0.013,0.115 +2021cjj,187.28008,7.274661,NA,0.131,SNIa,1.0,0.0,1.0,0.0 +2021cjl,66.835763,-10.698153,NA,0.235,SNIa,0.974,0.022,0.974,0.004 +2021cjn,67.989496,-7.487803,NA,0.143,SNIa,0.997,0.001,0.997,0.002 +2021cka,149.5664,2.406525,NA,0.11,SNIa,1.0,0.0,1.0,0.0 +2021cly,59.655212,-10.325842,SNIb,0.098,SNII,0.596,0.596,0.107,0.297 +2021cmo,246.559421,24.996966,NA,0.135,SNIa,1.0,0.0,1.0,0.0 +2021cob,187.605664,2.151898,SNIax,0.151,SNIa,0.791,0.018,0.791,0.191 +2021coc,190.915908,-0.484736,NA,0.088,SNII,0.999,0.999,0.001,0.0 +2021cox,245.119232,24.139062,NA,0.07,SNIa,0.903,0.001,0.903,0.096 +2021cqv,192.191883,31.835267,NA,0.143,SNIa,0.816,0.034,0.816,0.15 +2021cso,178.638587,20.201317,NA,0.123,SNII,0.42,0.42,0.378,0.202 +2021cta,130.362,44.146181,SNIa-norm,0.096,SNIa,1.0,0.0,1.0,0.0 +2021ctc,221.395568,42.369056,NA,0.213,SNIa,0.896,0.073,0.896,0.031 +2021ctd,189.632806,-1.675863,NA,0.193,SNIa,1.0,0.0,1.0,0.0 +2021cte,224.866328,37.707989,NA,0.149,SNIa,0.918,0.051,0.918,0.031 +2021ctf,184.742038,9.342553,NA,0.195,SNIa,0.846,0.114,0.846,0.04 +2021ctg,205.145271,28.928586,NA,0.095,SNIbc,0.531,0.269,0.2,0.531 +2021cth,185.78203,6.209207,NA,0.097,SNII,0.982,0.982,0.013,0.005 +2021ctn,163.226446,-0.772542,SNIa-91T-like,0.098,SNIbc,0.634,0.039,0.327,0.634 +2021cuw,146.034142,31.000214,NA,0.131,SNIa,0.993,0.005,0.993,0.002 +2021dbg,141.111658,-6.581422,SNII,0.022,SNII,0.974,0.974,0.015,0.011 +2021dcj,133.446604,-0.9822,NA,0.153,SNIa,0.985,0.001,0.985,0.014 +2021dcu,242.025953,25.483744,SNIa-norm,0.026,SNIa,1.0,0.0,1.0,0.0 +2021dcv,150.3565,5.044825,NA,0.104,SNIa,0.998,0.0,0.998,0.002 +2021dda,184.796183,6.148023,SNIa-91T-like,0.152,SNIa,1.0,0.0,1.0,0.0 +2021dgu,186.381315,3.437049,NA,0.031,SNIbc,0.475,0.339,0.186,0.475 +2021dha,172.910246,14.076292,SNIa-norm,0.065,SNIa,0.998,0.0,0.998,0.002 +2021dhc,140.629458,28.344397,NA,0.097,SNII,0.954,0.954,0.036,0.01 +2021dib,173.736132,9.038674,SNIc-BL,0.184,SNIbc,0.679,0.027,0.294,0.679 +2021dk,209.964462,35.756736,NA,0.116,SNIa,0.997,0.0,0.997,0.003 +2021dkj,66.983654,-10.035678,NA,0.147,SNIa,0.827,0.037,0.827,0.136 +2021dkz,193.090888,2.085117,NA,0.156,SNIa,1.0,0.0,1.0,0.0 +2021dl,64.113086,-7.644854,NA,0.073,SNIa,0.693,0.065,0.693,0.242 +2021dmu,184.540038,5.332036,NA,0.146,SNIa,1.0,0.0,1.0,0.0 +2021dnm,189.350886,23.093552,SNIa-norm,0.07,SNIa,1.0,0.0,1.0,0.0 +2021do,154.235544,73.397501,SNIc,0.014,SNIbc,0.736,0.009,0.255,0.736 +2021dov,134.077949,-0.442139,SNIa-norm,0.026,SNIa,1.0,0.0,1.0,0.0 +2021dow,214.748225,41.722536,NA,0.172,SNII,0.909,0.909,0.035,0.056 +2021dox,137.5816,41.001419,NA,0.217,SNIa,0.998,0.001,0.998,0.001 +2021doy,109.799858,37.249317,NA,0.18,SNIa,0.926,0.058,0.926,0.016 +2021doz,137.307546,43.017369,NA,0.237,SNIa,0.534,0.448,0.534,0.018 +2021dpa,140.392738,34.755003,NA,0.152,SNII,0.43,0.43,0.211,0.359 +2021dpb,128.538908,-4.138725,NA,0.23,SNIa,0.472,0.227,0.472,0.301 +2021dpc,128.806654,-5.066969,NA,0.177,SNII,0.576,0.576,0.342,0.082 +2021dpd,172.940442,13.600214,NA,0.17,SNIa,0.73,0.149,0.73,0.121 +2021dpe,217.444067,38.461933,NA,0.164,SNIa,0.856,0.015,0.856,0.129 +2021dpg,181.041291,18.485504,NA,0.187,SNII,0.535,0.535,0.236,0.229 +2021dph,217.868271,34.278678,NA,0.204,SNIa,0.725,0.226,0.725,0.049 +2021dpi,215.868083,33.537267,NA,0.207,SNIa,0.984,0.01,0.984,0.006 +2021dpj,129.938899,-3.272754,NA,0.161,SNIa,1.0,0.0,1.0,0.0 +2021dpw,186.239313,6.097828,SNIa-norm,0.127,SNIa,1.0,0.0,1.0,0.0 +2021dqo,132.912905,23.935953,SNIa-norm,0.125,SNIa,1.0,0.0,1.0,0.0 +2021dsj,117.2701,39.640414,NA,0.21,SNIa,0.984,0.013,0.984,0.003 +2021dsk,198.5973,17.935631,NA,0.201,SNIa,0.911,0.042,0.911,0.047 +2021dsp,224.374954,37.571956,NA,0.277,SNII,0.517,0.517,0.433,0.05 +2021dtf,59.618592,-12.268597,NA,0.185,SNIa,0.973,0.015,0.973,0.012 +2021efy,159.425129,-1.295557,NA,0.137,SNII,0.738,0.738,0.19,0.072 +2021ehb,46.949208,40.311269,TDE,0.082,SNII,0.444,0.444,0.222,0.334 +2021ehh,195.662156,27.629596,NA,0.166,SNIa,0.994,0.0,0.994,0.006 +2021eiz,170.217733,13.454508,NA,0.242,SNII,0.821,0.821,0.096,0.083 +2021ekf,65.610679,-25.140136,SNIa-91T-like,0.231,SNIa,0.848,0.136,0.848,0.016 +2021ekr,189.792721,2.805661,NA,0.161,SNIa,1.0,0.0,1.0,0.0 +2021enc,239.544087,27.359115,NA,0.098,SNIa,1.0,0.0,1.0,0.0 +2021eng,155.720327,-0.632753,SNIa-norm,0.12,SNIa,0.996,0.0,0.996,0.004 +2021epm,228.291911,0.775051,NA,0.103,SNIa,1.0,0.0,1.0,0.0 +2021fnt,203.247596,32.60959,SNIa-norm,0.025,SNIa,0.864,0.0,0.864,0.136 +2021fom,194.800298,-4.203983,SNIa-norm,0.194,SNIa,1.0,0.0,1.0,0.0 +2021foz,224.835024,38.515978,SNIa-norm,0.215,SNIa,1.0,0.0,1.0,0.0 +2021fpp,154.99366,-0.379567,NA,0.166,SNIa,0.908,0.051,0.908,0.041 +2021frk,189.840051,-0.414374,SNIa-norm,0.12,SNIa,0.998,0.0,0.998,0.002 +2021fsb,155.236112,-0.373685,SNIIb,0.083,SNIbc,0.887,0.032,0.081,0.887 +2021ftl,194.661991,-0.337939,NA,0.171,SNIa,0.96,0.027,0.96,0.013 +2021fui,162.129692,-3.5459,NA,0.242,SNII,0.794,0.794,0.184,0.022 +2021fvg,192.891824,-0.780257,NA,0.154,SNIa,0.838,0.086,0.838,0.076 +2021fwm,159.325711,-5.16066,SNIa-norm,0.12,SNIa,1.0,0.0,1.0,0.0 +2021fwo,194.808333,-4.098692,NA,0.211,SNIa,0.943,0.012,0.943,0.045 +2021gba,219.266186,-8.076434,SNIa-norm,0.084,SNIa,0.999,0.0,0.999,0.001 +2021gca,217.03053,33.497059,NA,0.214,SNIa,0.729,0.045,0.729,0.226 +2021gcv,203.0635,31.7204,SNIa-norm,0.096,SNIa,0.865,0.003,0.865,0.132 +2021gda,187.024988,18.6153,NA,0.081,SNIbc,0.499,0.003,0.498,0.499 +2021gdd,206.938339,28.855789,NA,0.084,SNIa,0.567,0.127,0.567,0.306 +2021gdv,226.355633,41.821497,NA,0.144,SNIa,0.989,0.0,0.989,0.011 +2021gdz,178.329255,20.108592,SNII,0.05,SNII,0.47,0.47,0.207,0.323 +2021gel,186.634531,31.428385,SNIa-91bg-like,0.056,SNIa,1.0,0.0,1.0,0.0 +2021gez,245.541794,21.222222,SNIa-norm,0.065,SNIa,0.98,0.0,0.98,0.02 +2021gfv,235.841889,12.960039,SNIa-norm,0.125,SNIa,0.994,0.001,0.994,0.005 +2021ghd,179.204031,20.0936,NA,0.177,SNIa,0.999,0.001,0.999,0.0 +2021ghe,181.729097,18.256116,NA,0.155,SNIa,1.0,0.0,1.0,0.0 +2021gik,229.224617,10.742633,NA,0.241,SNIa,0.999,0.0,0.999,0.001 +2021gis,248.347342,35.913331,NA,0.114,SNIa,0.995,0.0,0.995,0.005 +2021gji,209.166192,31.909219,NA,0.155,SNIa,0.743,0.102,0.743,0.155 +2021gmv,127.83746,-3.922906,NA,0.268,SNIa,0.939,0.051,0.939,0.01 +2021gnn,182.134754,17.839384,NA,0.207,SNIa,1.0,0.0,1.0,0.0 +2021gno,183.042898,13.249175,SNIb-pec,0.02,SNIbc,0.744,0.0,0.256,0.744 +2021gnt,187.757746,13.350981,NA,0.224,SNIa,0.898,0.033,0.898,0.069 +2021gpt,230.33524,13.965568,NA,0.167,SNII,0.551,0.551,0.098,0.351 +2021gqc,226.949936,-10.247193,SNIa-norm,0.07,SNIa,1.0,0.0,1.0,0.0 +2021gqu,190.002492,1.157033,NA,0.101,SNIbc,0.916,0.005,0.079,0.916 +2021guc,213.124812,-5.938878,NA,0.156,SNII,0.635,0.635,0.349,0.016 +2021gwa,235.654473,6.854538,SNIa-norm,0.078,SNIa,1.0,0.0,1.0,0.0 +2021gyh,195.966091,-3.451749,SNIa-norm,0.086,SNIa,1.0,0.0,1.0,0.0 +2021gyq,239.360079,27.063172,NA,0.128,SNIbc,0.895,0.01,0.095,0.895 +2021gyv,225.329595,38.057529,NA,0.245,SNIa,0.766,0.213,0.766,0.021 +2021gza,219.059475,33.764658,NA,0.191,SNIa,0.858,0.119,0.858,0.023 +2021gzc,194.015929,2.844003,NA,0.16,SNIa,0.596,0.049,0.596,0.355 +2021hbx,245.097896,24.122323,NA,0.073,SNII,0.963,0.963,0.016,0.021 +2021hby,184.944513,15.6406,NA,0.213,SNIa,0.981,0.012,0.981,0.007 +2021hca,170.195231,14.204458,SNIa-norm,0.193,SNIa,1.0,0.0,1.0,0.0 +2021hdj,187.704663,19.084642,NA,0.127,SNIa,0.869,0.052,0.869,0.079 +2021hdq,229.418227,-4.171753,NA,0.215,SNIa,1.0,0.0,1.0,0.0 +2021hem,245.316677,14.552684,SNIa-norm,0.045,SNIa,1.0,0.0,1.0,0.0 +2021hid,186.563821,30.1921,NA,0.134,SNIa,0.689,0.027,0.689,0.284 +2021hiz,186.42365,7.228405,SNIa-norm,0.005,SNIa,1.0,0.0,1.0,0.0 +2021hjj,183.786833,4.932469,NA,0.091,SNIa,1.0,0.0,1.0,0.0 +2021hky,211.619871,-6.097786,NA,0.185,SNII,0.594,0.594,0.379,0.027 +2021hlp,158.42065,-3.70665,SNIa-norm,0.058,SNIa,1.0,0.0,1.0,0.0 +2021hmf,235.795358,14.373172,NA,0.205,SNIa,0.772,0.194,0.772,0.034 +2021hol,219.274454,34.633047,SNIa-norm,0.11,SNIa,1.0,0.0,1.0,0.0 +2021hpr,154.160875,73.400497,SNIa-norm,0.016,SNIa,0.985,0.009,0.985,0.006 +2021hpz,154.473007,-1.772922,NA,0.156,SNII,0.585,0.585,0.334,0.081 +2021hqa,172.638308,9.837463,NA,0.179,SNIa,0.983,0.009,0.983,0.008 +2021hrj,156.923713,72.82693,SNIb,0.023,SNIbc,0.722,0.014,0.264,0.722 +2021hud,240.472104,20.258336,SNII,0.126,SNII,0.648,0.648,0.28,0.072 +2021huz,216.762812,33.558897,NA,0.107,SNIa,0.843,0.08,0.843,0.077 +2021hwj,237.663158,26.609737,NA,0.16,SNIa,1.0,0.0,1.0,0.0 +2021hxv,186.909273,1.917961,SNII,0.094,SNII,0.996,0.996,0.002,0.002 +2021hye,195.827143,-1.193397,NA,0.142,SNIa,0.969,0.014,0.969,0.017 +2021hyf,189.839183,1.652025,NA,0.036,SNIa,0.715,0.082,0.715,0.203 +2021hzh,192.918854,-4.768913,NA,0.118,SNII,0.999,0.999,0.001,0.0 +2021hzs,180.627375,17.812044,NA,0.127,SNIa,1.0,0.0,1.0,0.0 +2021iae,237.986613,25.631458,NA,0.165,SNIa,1.0,0.0,1.0,0.0 +2021ibg,208.645363,33.861492,NA,0.134,SNII,0.973,0.973,0.019,0.008 +2021ibh,182.810204,20.254842,NA,0.132,SNIa,1.0,0.0,1.0,0.0 +2021idh,155.827592,-0.429375,SNIa-norm,0.097,SNIa,1.0,0.0,1.0,0.0 +2021ied,245.720404,25.624883,NA,0.104,SNIa,0.999,0.0,0.999,0.001 +2021ifw,173.7069,11.745669,NA,0.128,SNIa,1.0,0.0,1.0,0.0 +2021ihk,209.380558,30.839017,NA,0.231,SNIa,0.681,0.053,0.681,0.266 +2021ihp,127.985958,-4.133819,SNII,0.048,SNII,0.84,0.84,0.055,0.105 +2021iig,216.584971,33.481078,NA,0.179,SNIa,0.969,0.019,0.969,0.012 +2021iik,131.954937,-2.051689,NA,0.257,SNIa,0.991,0.007,0.991,0.002 +2021ikk,245.017114,26.748962,NA,0.134,SNIa,1.0,0.0,1.0,0.0 +2021ilr,227.543639,-11.45898,NA,0.188,SNIa,0.997,0.0,0.997,0.003 +2021ils,173.70754,12.319347,NA,0.223,SNIa,0.997,0.002,0.997,0.001 +2021ilt,173.214433,13.022556,NA,0.156,SNIa,0.974,0.002,0.974,0.024 +2021im,12.407393,2.788204,NA,0.147,SNIa,0.648,0.305,0.648,0.047 +2021in,13.530453,5.019582,NA,0.21,SNIa,1.0,0.0,1.0,0.0 +2021ina,185.896877,18.792202,NA,0.208,SNIa,0.442,0.217,0.442,0.341 +2021inc,207.08837,28.268638,SNII,0.079,SNII,0.987,0.987,0.008,0.005 +2021inl,195.38852,27.831974,SNIb-pec,0.148,SNIa,0.889,0.017,0.889,0.094 +2021int,187.927604,23.064923,NA,0.179,SNII,0.593,0.593,0.376,0.031 +2021iok,226.887726,-12.214697,SNIa-norm,0.201,SNIa,1.0,0.0,1.0,0.0 +2021ipc,216.401371,43.388327,NA,0.188,SNII,0.787,0.787,0.156,0.057 +2021ipg,160.448576,-0.960628,NA,0.214,SNIa,1.0,0.0,1.0,0.0 +2021iqw,135.220544,27.217572,NA,0.129,SNIa,0.997,0.003,0.997,0.0 +2021isf,219.092013,34.643143,NA,0.198,SNIa,1.0,0.0,1.0,0.0 +2021isp,192.781748,5.270371,NA,0.19,SNIa,0.994,0.004,0.994,0.002 +2021ita,186.495088,7.235303,SNIa-norm,0.113,SNIa,0.731,0.045,0.731,0.224 +2021itd,221.212831,43.316798,SNIa-norm,0.08,SNIa,1.0,0.0,1.0,0.0 +2021itp,186.210229,32.025467,SNIa-91T-like,0.138,SNIa,1.0,0.0,1.0,0.0 +2021iuk,186.493583,6.740844,SNIa-norm,0.101,SNIa,1.0,0.0,1.0,0.0 +2021iux,154.641433,-0.648989,NA,0.121,SNIa,0.952,0.017,0.952,0.031 +2021iv,140.375203,31.3229,NA,0.164,SNIa,0.998,0.0,0.998,0.002 +2021iww,185.88553,19.014204,NA,0.068,SNIbc,0.89,0.014,0.096,0.89 +2021iwx,195.816754,28.453241,NA,0.148,SNIa,0.84,0.083,0.84,0.077 +2021iwy,189.593346,21.095426,NA,0.25,SNIa,0.587,0.405,0.587,0.008 +2021iyp,244.342462,32.521019,NA,0.194,SNIa,0.646,0.111,0.646,0.243 +2021iyq,245.300596,25.481484,NA,0.231,SNIa,0.765,0.19,0.765,0.045 +2021iyr,182.297388,16.867569,SNIa-norm,0.117,SNIa,1.0,0.0,1.0,0.0 +2021iys,182.297463,17.084211,NA,0.134,SNIa,0.429,0.288,0.429,0.283 +2021iyt,154.68505,0.055378,SNIbn,0.109,SNIa,0.984,0.005,0.984,0.011 +2021iyu,157.46962,-4.421075,NA,0.213,SNIa,0.742,0.204,0.742,0.054 +2021iyv,197.018457,-4.008406,NA,0.147,SNIa,0.866,0.068,0.866,0.066 +2021iyw,187.657471,18.754088,NA,0.15,SNIa,0.63,0.336,0.63,0.034 +2021jkd,232.531859,-0.565288,SNIa-norm,0.082,SNIa,1.0,0.0,1.0,0.0 +2021jkg,228.0133,-8.878572,NA,0.171,SNIa,0.621,0.039,0.621,0.34 +2021jlj,191.165518,-2.087485,NA,0.115,SNIa,1.0,0.0,1.0,0.0 +2021jna,190.497283,-4.248394,NA,0.227,SNIa,0.519,0.467,0.519,0.014 +2021jni,158.631401,-4.414556,NA,0.205,SNIa,0.89,0.028,0.89,0.082 +2021jof,242.843725,28.873099,SNIa-91T-like,0.116,SNIa,1.0,0.0,1.0,0.0 +2021jox,187.211175,19.331092,SNIa-norm,0.184,SNIa,0.998,0.0,0.998,0.002 +2021joy,186.369852,18.710429,NA,0.163,SNIa,0.99,0.0,0.99,0.01 +2021joz,174.228182,10.264546,SNII,0.138,SNII,0.77,0.77,0.166,0.064 +2021jr,39.951087,42.077631,NA,0.208,SNIa,0.937,0.021,0.937,0.042 +2021jv,59.643008,-8.373612,NA,0.204,SNII,0.817,0.817,0.17,0.013 +2021jvl,246.166642,23.934836,SNII,0.22,SNII,0.795,0.795,0.201,0.004 +2021jvm,241.152562,27.611231,NA,0.264,SNIa,0.954,0.023,0.954,0.023 +2021jvo,137.581329,27.568044,NA,0.174,SNIa,0.637,0.343,0.637,0.02 +2021jvt,222.9694,-11.360412,NA,0.21,SNII,0.894,0.894,0.071,0.035 +2021jvu,241.99573,32.021187,NA,0.161,SNIa,1.0,0.0,1.0,0.0 +2021jw,64.392609,-8.260536,SNIc-BL,0.079,SNIbc,0.738,0.005,0.257,0.738 +2021jze,154.972375,-3.7354,SNIa-norm,0.033,SNIa,1.0,0.0,1.0,0.0 +2021jzf,244.142,20.919511,SNIa-norm,0.09,SNIa,1.0,0.0,1.0,0.0 +2021jzg,313.626037,-1.081284,NA,0.154,SNIa,0.908,0.044,0.908,0.048 +2021jzh,225.151979,-4.873092,NA,0.219,SNIa,0.496,0.489,0.496,0.015 +2021kbi,311.639109,-4.084202,NA,0.108,SNIa,0.943,0.02,0.943,0.037 +2021kbj,238.592411,24.490423,SNIa-norm,0.14,SNIa,1.0,0.0,1.0,0.0 +2021kbk,241.85767,23.613894,NA,0.099,SNII,0.861,0.861,0.028,0.111 +2021kbl,216.543446,41.508063,NA,0.215,SNIa,0.928,0.055,0.928,0.017 +2021kbm,224.107503,43.041144,NA,0.192,SNIa,0.992,0.007,0.992,0.001 +2021kbn,224.441726,41.483741,NA,0.188,SNIa,0.454,0.201,0.454,0.345 +2021kbo,220.937639,41.546236,NA,0.124,SNII,0.859,0.859,0.096,0.045 +2021kbx,221.187707,40.915357,NA,0.24,SNII,0.775,0.775,0.196,0.029 +2021kcb,194.406801,28.254642,NA,0.171,SNIa,0.716,0.131,0.716,0.153 +2021kcc,194.0331,-4.960614,SNIa-norm,0.168,SNIa,0.96,0.009,0.96,0.031 +2021kcz,182.590692,16.789145,SNII,0.227,SNIa,0.987,0.0,0.987,0.013 +2021kdb,129.728768,-4.107634,NA,0.189,SNIa,0.7,0.279,0.7,0.021 +2021kdd,221.885254,42.111903,NA,0.243,SNIa,0.961,0.027,0.961,0.012 +2021kdv,205.421954,28.31073,NA,0.111,SNII,0.992,0.992,0.003,0.005 +2021key,193.657342,-0.460472,NA,0.114,SNIa,0.729,0.008,0.729,0.263 +2021kfy,196.519226,23.801101,SNIa-norm,0.177,SNIa,0.997,0.002,0.997,0.001 +2021kgn,320.750461,8.578501,NA,0.182,SNIa,0.994,0.001,0.994,0.005 +2021khl,244.473545,32.109015,SNIa-norm,0.054,SNIa,0.905,0.0,0.905,0.095 +2021kiw,216.846533,43.407823,NA,0.112,SNIa,1.0,0.0,1.0,0.0 +2021kkv,238.868783,29.060352,NA,0.223,SNIbc,0.47,0.202,0.328,0.47 +2021kmq,319.102924,12.224333,NA,0.2,SNII,0.548,0.548,0.158,0.294 +2021kms,321.817839,8.176533,NA,0.182,SNIa,0.994,0.005,0.994,0.001 +2021kng,185.349629,4.028072,NA,0.156,SNIa,0.516,0.433,0.516,0.051 +2021kni,357.731904,27.092927,NA,0.024,SNIa,0.593,0.028,0.593,0.379 +2021kor,157.463735,72.723745,NA,0.199,SNIa,0.998,0.0,0.998,0.002 +2021kpo,196.412014,-4.026711,NA,0.155,SNIa,1.0,0.0,1.0,0.0 +2021kpp,215.931686,39.587992,NA,0.146,SNIa,1.0,0.0,1.0,0.0 +2021kpq,161.661635,-3.239909,NA,0.191,SNIa,0.971,0.013,0.971,0.016 +2021kpr,173.954304,14.001516,NA,0.239,SNIa,0.684,0.013,0.684,0.303 +2021kps,195.351219,-3.968658,NA,0.161,SNIa,0.604,0.275,0.604,0.121 +2021kqp,223.794754,-6.985927,SN,0.095,SNIa,0.981,0.0,0.981,0.019 +2021krc,186.636875,16.847331,SNII,0.034,SNII,0.998,0.998,0.001,0.001 +2021krk,154.989094,74.177349,NA,0.169,SNIa,1.0,0.0,1.0,0.0 +2021kwa,187.640649,11.517883,SNIa-norm,0.08,SNIa,0.987,0.0,0.987,0.013 +2021kwg,358.098125,28.772908,SNIc,0.016,SNIbc,0.636,0.216,0.148,0.636 +2021kwh,1.338036,27.486205,NA,0.068,SNIa,0.998,0.0,0.998,0.002 +2021kwy,248.397626,34.685835,NA,0.125,SNIa,1.0,0.0,1.0,0.0 +2021kxk,190.091038,1.794301,NA,0.221,SNIa,0.947,0.034,0.947,0.019 +2021kyp,205.024564,29.447669,SNIa-norm,0.146,SNIa,1.0,0.0,1.0,0.0 +2021kyv,216.980358,33.00208,SN,0.105,SNII,0.997,0.997,0.002,0.001 +2021kzc,237.619004,20.015161,NA,0.155,SNII,0.667,0.667,0.257,0.076 +2021kzd,221.170478,43.253351,NA,0.194,SNII,0.679,0.679,0.273,0.048 +2021kzx,240.97705,27.863083,SNIa-91T-like,0.069,SNIa,1.0,0.0,1.0,0.0 +2021lby,181.363397,17.836367,NA,0.165,SNIa,0.956,0.008,0.956,0.036 +2021lcc,215.834829,42.997894,NA,0.215,SNIa,0.761,0.222,0.761,0.017 +2021lce,192.360038,5.865394,NA,0.086,SNIbc,0.477,0.07,0.453,0.477 +2021lel,161.534697,-3.356932,NA,0.091,SNII,0.785,0.785,0.178,0.037 +2021ley,182.29892,21.591367,NA,0.155,SNIa,0.846,0.109,0.846,0.045 +2021lfv,233.320616,10.271486,SNIa-norm,0.21,SNIa,0.999,0.0,0.999,0.001 +2021lgc,316.477615,-1.077201,NA,0.144,SNIbc,0.699,0.004,0.297,0.699 +2021lgq,231.536181,-0.457297,NA,0.097,SNIa,1.0,0.0,1.0,0.0 +2021lhf,241.631173,34.974971,NA,0.146,SNIa,0.998,0.0,0.998,0.002 +2021lhi,243.269342,14.226997,NA,0.252,SNII,0.684,0.684,0.238,0.078 +2021lii,245.823946,26.542694,NA,0.064,SNII,0.985,0.985,0.006,0.009 +2021lik,226.703553,10.37875,NA,0.218,SNIbc,0.614,0.087,0.299,0.614 +2021lit,233.3289,12.341049,NA,0.155,SNIa,0.984,0.0,0.984,0.016 +2021liv,230.844984,8.895926,NA,0.188,SNIa,1.0,0.0,1.0,0.0 +2021lix,190.111413,-4.67955,NA,0.152,SNIbc,0.583,0.169,0.248,0.583 +2021ljb,189.25018,-5.533179,SNIa-norm,0.128,SNIa,1.0,0.0,1.0,0.0 +2021lnv,197.819238,4.981872,NA,0.089,SNIbc,0.536,0.283,0.181,0.536 +2021log,229.866857,13.849511,SNIa-norm,0.14,SNIa,1.0,0.0,1.0,0.0 +2021low,176.192743,20.126208,SNIa-norm,0.037,SNIa,1.0,0.0,1.0,0.0 +2021lqv,240.031199,21.82813,NA,0.148,SNIa,1.0,0.0,1.0,0.0 +2021lqw,195.964936,28.727906,NA,0.168,SNIa,0.999,0.0,0.999,0.001 +2021lra,161.82658,-4.392798,NA,0.161,SNIa,0.509,0.445,0.509,0.046 +2021lrj,310.900131,-3.509296,NA,0.189,SNII,0.579,0.579,0.409,0.012 +2021lru,317.556644,-1.839234,NA,0.211,SNIa,0.988,0.01,0.988,0.002 +2021lsx,207.110158,27.909697,NA,0.09,SNII,0.991,0.991,0.006,0.003 +2021lta,246.502421,28.427454,SNIa-norm,0.169,SNIa,1.0,0.0,1.0,0.0 +2021lus,246.693044,34.968739,NA,0.166,SNIa,0.853,0.002,0.853,0.145 +2021lut,239.822743,32.757536,NA,0.157,SNIa,0.98,0.002,0.98,0.018 +2021luu,243.082161,26.783494,SNIa-norm,0.164,SNIa,0.999,0.0,0.999,0.001 +2021luv,245.994451,25.190191,NA,0.145,SNIa,1.0,0.0,1.0,0.0 +2021luw,227.229887,12.573655,NA,0.193,SNIa,0.865,0.03,0.865,0.105 +2021lux,243.884618,35.27602,NA,0.161,SNIa,0.997,0.002,0.997,0.001 +2021lxb,195.494387,28.008555,SNIa-norm,0.027,SNIa,0.973,0.001,0.973,0.026 +2021lya,194.400681,27.33915,NA,0.077,SNIbc,0.685,0.023,0.292,0.685 +2021lzb,213.486471,-4.410394,NA,0.249,SNIa,0.993,0.006,0.993,0.001 +2021lzc,227.340624,-11.119413,SNIa-norm,0.184,SNIa,1.0,0.0,1.0,0.0 +2021lzd,227.222221,-12.531928,NA,0.155,SNIa,0.978,0.008,0.978,0.014 +2021lze,186.939929,2.835572,NA,0.155,SNIa,0.498,0.256,0.498,0.246 +2021lzg,181.720142,17.986489,SNIa-91T-like,0.221,SNIa,1.0,0.0,1.0,0.0 +2021lzh,198.208259,-0.458491,NA,0.214,SNIa,0.725,0.24,0.725,0.035 +2021mba,314.173013,-3.008098,NA,0.202,SNIbc,0.497,0.483,0.02,0.497 +2021mfx,239.868333,28.565083,NA,0.253,SNIa,0.934,0.027,0.934,0.039 +2021mfy,176.079769,19.318463,SNIa-norm,0.102,SNIa,1.0,0.0,1.0,0.0 +2021mfz,331.97943,6.600532,NA,0.221,SNIa,0.914,0.079,0.914,0.007 +2021mga,192.917174,2.339764,NA,0.249,SNIa,0.94,0.042,0.94,0.018 +2021mgc,331.873994,3.096954,SNIa-norm,0.065,SNIa,0.969,0.0,0.969,0.031 +2021mgd,194.680401,3.60355,SNIa-norm,0.178,SNIa,1.0,0.0,1.0,0.0 +2021mge,170.883948,12.436812,NA,0.199,SNII,0.487,0.487,0.366,0.147 +2021mgl,3.775005,20.580952,NA,0.081,SNII,0.961,0.961,0.016,0.023 +2021mim,235.089034,7.281499,SNIa-norm,0.04,SNIa,0.999,0.0,0.999,0.001 +2021mms,186.091513,5.393417,NA,0.239,SNII,0.55,0.55,0.298,0.152 +2021mpm,189.796746,2.841119,NA,0.2,SNIa,0.999,0.0,0.999,0.001 +2021mqa,189.008262,1.224428,NA,0.19,SNII,0.96,0.96,0.012,0.028 +2021mqf,160.260883,-3.937256,SNIa-norm,0.208,SNIa,0.998,0.002,0.998,0.0 +2021mqr,240.308128,35.792272,NA,0.151,SNIa,1.0,0.0,1.0,0.0 +2021msx,245.356625,12.915306,NA,0.232,SNIa,0.605,0.368,0.605,0.027 +2021mwb,242.237984,35.421083,SNIa-norm,0.089,SNIa,1.0,0.0,1.0,0.0 +2021mxw,226.32469,9.741177,NA,0.202,SNIa,1.0,0.0,1.0,0.0 +2021mzk,327.803168,7.414039,NA,0.24,SNIa,0.533,0.057,0.533,0.41 +2021naw,195.778705,-1.263008,NA,0.199,SNIa,0.984,0.009,0.984,0.007 +2021naz,211.412338,35.616751,NA,0.194,SNIbc,0.731,0.15,0.119,0.731 +2021ndc,204.338971,28.535014,NA,0.094,SNII,0.907,0.907,0.063,0.03 +2021nfl,219.629383,-7.903629,NA,0.217,SNII,0.658,0.658,0.284,0.058 +2021nip,244.210033,12.642931,SNII,0.167,SNII,0.878,0.878,0.07,0.052 +2021nlj,187.862615,4.22653,SNIa-norm,0.164,SNIa,1.0,0.0,1.0,0.0 +2021nma,172.890751,8.928679,NA,0.202,SNII,0.982,0.982,0.014,0.004 +2021nmk,191.814323,-0.628231,NA,0.126,SNII,0.634,0.634,0.04,0.326 +2021nra,216.61523,33.82393,NA,0.199,SNIa,0.999,0.001,0.999,0.0 +2021nse,197.809587,-1.047188,NA,0.166,SNIa,0.999,0.001,0.999,0.0 +2021nsf,223.723345,42.98848,NA,0.2,SNIa,0.946,0.034,0.946,0.02 +2021nsg,321.97085,13.88283,NA,0.136,SNIa,0.651,0.057,0.651,0.292 +2021nsh,333.86416,5.544722,SNIa-norm,0.089,SNIa,0.998,0.0,0.998,0.002 +2021nsi,315.944858,-0.960106,SNIa-norm,0.129,SNIa,1.0,0.0,1.0,0.0 +2021nsk,244.289098,21.013725,NA,0.128,SNIa,0.437,0.305,0.437,0.258 +2021nsl,330.814842,6.176044,NA,0.084,SNII,0.792,0.792,0.183,0.025 +2021nsm,248.496476,33.094827,NA,0.235,SNIa,0.998,0.0,0.998,0.002 +2021nue,324.041717,10.632814,SNII,0.034,SNII,0.819,0.819,0.096,0.085 +2021nug,192.78738,3.479719,NA,0.155,SNIa,0.997,0.001,0.997,0.002 +2021nuh,194.221771,5.5698,NA,0.191,SNIa,0.514,0.466,0.514,0.02 +2021nxq,216.336179,37.762421,SLSN-I,0.132,SNIbc,0.415,0.345,0.24,0.415 +2021nyt,245.807564,26.47594,SNIa-norm,0.054,SNIa,1.0,0.0,1.0,0.0 +2021oal,237.166533,20.059356,SNIa-norm,0.091,SNIa,1.0,0.0,1.0,0.0 +2021oaq,174.61124,11.324337,NA,0.234,SNIa,0.946,0.044,0.946,0.01 +2021oar,210.5939,34.151972,NA,0.173,SNIa,0.511,0.246,0.511,0.243 +2021oas,194.203163,3.395239,NA,0.239,SNIbc,0.45,0.325,0.225,0.45 +2021oat,195.034431,28.170293,SNIa-norm,0.03,SNIa,1.0,0.0,1.0,0.0 +2021oau,194.38585,4.046697,NA,0.197,SNIa,0.733,0.058,0.733,0.209 +2021oav,194.285462,5.174736,NA,0.252,SNIa,0.973,0.017,0.973,0.01 +2021oaw,188.534174,3.824729,SNIa-norm,0.084,SNIa,0.999,0.0,0.999,0.001 +2021oax,244.441658,23.938949,NA,0.217,SNII,0.99,0.99,0.008,0.002 +2021obg,225.349115,-11.409288,NA,0.135,SNIbc,0.708,0.177,0.115,0.708 +2021oek,175.287968,13.78132,NA,0.115,SNIa,0.989,0.0,0.989,0.011 +2021oet,254.367258,2.142372,NA,0.148,SNIa,1.0,0.0,1.0,0.0 +2021oez,227.453641,0.533519,SNIa-norm,0.026,SNIa,1.0,0.0,1.0,0.0 +2021ofa,228.189522,-1.068756,NA,0.119,SNIa,0.99,0.002,0.99,0.008 +2021ofo,187.57662,11.185605,NA,0.243,SNIa,0.894,0.001,0.894,0.105 +2021ofq,209.871336,35.422781,SNIa-norm,0.168,SNIa,1.0,0.0,1.0,0.0 +2021ofr,194.700512,28.843597,NA,0.072,SNIa,0.818,0.001,0.818,0.181 +2021ofz,245.631507,31.851382,NA,0.229,SNII,0.628,0.628,0.353,0.019 +2021ogq,3.940342,22.387189,NA,0.151,SNIbc,0.541,0.042,0.417,0.541 +2021ojn,220.208679,42.253241,SNIa-norm,0.087,SNIa,1.0,0.0,1.0,0.0 +2021oov,238.705663,23.649058,NA,0.25,SNIa,0.818,0.031,0.818,0.151 +2021oqc,226.391523,-10.166304,NA,0.14,SNIa,0.901,0.039,0.901,0.06 +2021orm,239.183639,25.674094,SNIa-norm,0.092,SNIa,1.0,0.0,1.0,0.0 +2021ort,215.452767,38.509394,NA,0.129,SNII,0.57,0.57,0.101,0.329 +2021ovd,190.198946,32.033555,NA,0.208,SNIa,1.0,0.0,1.0,0.0 +2021owa,227.442248,0.018564,NA,0.276,SNIa,0.562,0.424,0.562,0.014 +2021pbf,222.86508,-11.900588,SNIa-norm,0.169,SNIa,1.0,0.0,1.0,0.0 +2021pby,322.402533,12.916377,SNIa-norm,0.036,SNIa,1.0,0.0,1.0,0.0 +2021pc,18.520066,-1.740137,SNIa-norm,0.027,SNIa,1.0,0.0,1.0,0.0 +2021pca,322.213425,10.404063,NA,0.161,SNIa,0.999,0.0,0.999,0.001 +2021pci,238.524109,24.594759,SNIa-norm,0.057,SNIa,1.0,0.0,1.0,0.0 +2021pcn,183.176408,16.506314,NA,0.21,SNIa,1.0,0.0,1.0,0.0 +2021pd,157.318402,-5.056958,NA,0.182,SNIa,0.995,0.0,0.995,0.005 +2021pde,244.244084,27.617732,SNIa-norm,0.074,SNIa,0.996,0.0,0.996,0.004 +2021pe,125.533605,43.310146,NA,0.103,SNII,0.983,0.983,0.004,0.013 +2021pfs,210.848253,-6.031638,SNIa-norm,0.022,SNIa,0.999,0.001,0.999,0.0 +2021pgm,41.282958,38.026081,NA,0.03,SNIa,0.824,0.004,0.824,0.172 +2021pgu,209.311658,36.039253,NA,0.144,SNIa,1.0,0.0,1.0,0.0 +2021phy,227.371689,-7.12761,NA,0.212,SNIa,0.999,0.0,0.999,0.001 +2021pi,221.590716,41.459,NA,0.103,SNIa,0.865,0.001,0.865,0.134 +2021pj,138.863343,31.807609,SNII,0.026,SNII,0.962,0.962,0.008,0.03 +2021pkz,193.821738,2.89731,SNIa-norm,0.017,SNIbc,0.666,0.016,0.318,0.666 +2021pl,182.908211,12.163049,SNIa-norm,0.1,SNIa,1.0,0.0,1.0,0.0 +2021pn,191.566041,-0.222028,NA,0.225,SNIa,0.88,0.086,0.88,0.034 +2021pnp,244.110919,36.740614,SNIIb,0.145,SNIbc,0.929,0.002,0.069,0.929 +2021pqw,186.677955,7.820379,NA,0.226,SNIa,1.0,0.0,1.0,0.0 +2021psj,187.521299,1.667966,NA,0.208,SNIa,0.982,0.004,0.982,0.014 +2021psl,208.835412,30.962497,NA,0.083,SNII,0.992,0.992,0.007,0.001 +2021pti,236.03302,19.987606,NA,0.216,SNII,0.456,0.456,0.259,0.285 +2021ptk,203.503871,30.806903,NA,0.133,SNIa,0.643,0.273,0.643,0.084 +2021ptl,227.502258,-4.047836,NA,0.144,SNIa,0.989,0.006,0.989,0.005 +2021puj,193.23415,4.795636,NA,0.168,SNIa,0.728,0.172,0.728,0.1 +2021put,208.800304,31.640661,NA,0.233,SNII,0.776,0.776,0.182,0.042 +2021pvw,186.904762,10.421361,NA,0.244,SNIa,0.582,0.32,0.582,0.098 +2021pwt,242.510342,32.188044,NA,0.205,SNIa,0.824,0.084,0.824,0.092 +2021pyk,216.614426,34.047745,NA,0.223,SNIa,0.989,0.008,0.989,0.003 +2021pzh,230.512478,9.72536,NA,0.177,SNIa,0.997,0.0,0.997,0.003 +2021pzl,209.043804,-6.080154,NA,0.215,SNIa,0.971,0.005,0.971,0.024 +2021pzs,233.481471,13.783994,NA,0.182,SNIa,0.553,0.411,0.553,0.036 +2021qas,4.281698,21.984934,SNIa-norm,0.093,SNIa,1.0,0.0,1.0,0.0 +2021qbv,234.960062,10.946186,NA,0.142,SNIa,0.996,0.003,0.996,0.001 +2021qbx,228.857902,-5.200016,NA,0.077,SNIbc,0.893,0.006,0.101,0.893 +2021qed,315.199729,-4.186786,NA,0.09,SNII,0.618,0.618,0.332,0.05 +2021qes,318.636612,-1.196278,SNII,0.042,SNII,0.958,0.958,0.002,0.04 +2021qgl,238.175901,23.163074,NA,0.078,SNIa,0.939,0.003,0.939,0.058 +2021qgw,244.365225,13.961614,NA,0.16,SNII,0.832,0.832,0.163,0.005 +2021qhg,237.554587,25.537125,NA,0.2,SNIa,0.991,0.004,0.991,0.005 +2021qht,21.763153,-13.708141,NA,0.048,SNII,0.651,0.651,0.006,0.343 +2021qjh,246.301946,26.698833,NA,0.182,SNIa,0.802,0.165,0.802,0.033 +2021qjs,223.8682,-6.950078,NA,0.109,SNIa,0.835,0.079,0.835,0.086 +2021qkr,237.233104,20.672897,NA,0.232,SNIa,0.472,0.362,0.472,0.166 +2021qls,231.488637,12.569806,NA,0.219,SNIa,0.967,0.017,0.967,0.016 +2021qol,223.983592,-11.415367,NA,0.123,SNII,0.947,0.947,0.041,0.012 +2021qqo,186.090868,9.292026,SNIa-norm,0.017,SNIa,1.0,0.0,1.0,0.0 +2021qrm,213.208587,-5.848886,NA,0.151,SNIa,0.923,0.032,0.923,0.045 +2021qsi,226.046395,-8.738482,NA,0.208,SNII,0.538,0.538,0.426,0.036 +2021qsm,311.577233,-0.995122,NA,0.211,SNIa,0.83,0.122,0.83,0.048 +2021qsr,222.945529,-9.404844,NA,0.202,SNIa,0.993,0.005,0.993,0.002 +2021qua,321.747779,10.128642,SNIa-91T-like,0.08,SNIa,1.0,0.0,1.0,0.0 +2021quj,233.337746,13.515464,NA,0.194,SNIa,0.999,0.001,0.999,0.0 +2021qve,203.276375,32.591869,SNIa-norm,0.026,SNIa,1.0,0.0,1.0,0.0 +2021qvo,332.050333,7.116883,SNIa-norm,0.093,SNIa,0.981,0.0,0.981,0.019 +2021qvr,352.111049,22.419995,SNII,0.022,SNIbc,0.764,0.003,0.233,0.764 +2021qxh,183.791004,14.936133,NA,0.194,SNII,0.967,0.967,0.03,0.003 +2021qxi,334.713829,1.293089,NA,0.198,SNIa,0.962,0.012,0.962,0.026 +2021qxj,193.687671,5.525061,NA,0.222,SNIa,0.861,0.043,0.861,0.096 +2021qxk,234.132937,5.738944,NA,0.136,SNIa,0.934,0.006,0.934,0.06 +2021qxl,233.313417,10.957825,NA,0.16,SNIa,0.982,0.007,0.982,0.011 +2021qxm,326.223233,3.241308,NA,0.137,SNII,0.972,0.972,0.011,0.017 +2021qxn,239.242333,24.553428,NA,0.229,SNIbc,0.489,0.039,0.472,0.489 +2021qxo,315.998658,-3.418508,NA,0.099,SNIa,1.0,0.0,1.0,0.0 +2021qxp,2.808804,20.3361,NA,0.095,SNII,0.995,0.995,0.002,0.003 +2021qxq,239.582625,27.376089,SNIa-norm,0.206,SNIa,1.0,0.0,1.0,0.0 +2021qxr,187.054862,10.295647,SNIa-norm,0.018,SNIa,0.982,0.013,0.982,0.005 +2021qxu,7.872368,17.383359,SNIa-norm,0.038,SNIa,1.0,0.0,1.0,0.0 +2021qxv,229.747063,-3.195855,TDE,0.168,SNII,0.842,0.842,0.018,0.14 +2021qyb,231.988743,10.275476,NA,0.094,SNII,0.799,0.799,0.139,0.062 +2021qzi,311.307798,-5.619432,SNII,0.064,SNII,1.0,1.0,0.0,0.0 +2021qzp,11.8985,-20.427819,SNIc-BL,0.025,SNIbc,0.964,0.006,0.03,0.964 +2021rbc,230.392999,12.162381,NA,0.036,SNII,0.472,0.472,0.367,0.161 +2021rbq,238.486027,27.688168,NA,0.161,SNII,0.766,0.766,0.229,0.005 +2021rdf,2.481958,19.976039,SNIa-norm,0.108,SNIa,1.0,0.0,1.0,0.0 +2021reh,207.360139,28.471353,NA,0.158,SNIa,0.914,0.018,0.914,0.068 +2021rem,229.948873,13.687621,SNII,0.037,SNII,0.44,0.44,0.138,0.422 +2021rfd,192.865804,6.934972,NA,0.183,SNIa,0.547,0.352,0.547,0.101 +2021rfz,7.261004,16.843205,NA,0.198,SNII,0.567,0.567,0.41,0.023 +2021rga,242.270681,20.213156,NA,0.164,SNIa,0.991,0.001,0.991,0.008 +2021rgc,229.516496,-4.138098,NA,0.149,SNIa,0.998,0.001,0.998,0.001 +2021rgd,193.629147,3.780436,NA,0.099,SNIbc,0.466,0.248,0.286,0.466 +2021rge,189.331557,3.286317,NA,0.17,SNIa,1.0,0.0,1.0,0.0 +2021rgf,194.979898,26.9678,NA,0.172,SNIa,0.747,0.197,0.747,0.056 +2021rgg,231.673426,10.048224,NA,0.244,SNIa,0.923,0.03,0.923,0.047 +2021rgh,193.363168,5.300149,NA,0.231,SNIa,0.971,0.018,0.971,0.011 +2021rgs,184.912354,11.997633,SNIa-norm,0.079,SNIa,0.848,0.007,0.848,0.145 +2021rh,134.712658,24.451639,NA,0.233,SNII,0.646,0.646,0.328,0.026 +2021rhu,0.814306,16.14569,SNIa-norm,0.009,SNIa,0.999,0.0,0.999,0.001 +2021rkc,193.849196,5.617408,NA,0.156,SNII,0.55,0.55,0.368,0.082 +2021rkd,321.912279,10.873578,NA,0.268,SNIbc,0.349,0.307,0.344,0.349 +2021rmi,222.583146,41.564625,NA,0.169,SNIa,0.955,0.011,0.955,0.034 +2021rmq,184.839139,32.249891,NA,0.131,SNII,0.41,0.41,0.285,0.305 +2021rpa,317.562023,-3.181326,NA,0.15,SNII,0.843,0.843,0.095,0.062 +2021rpe,317.387066,-5.366767,NA,0.138,SNIa,0.599,0.248,0.599,0.153 +2021rpr,314.095213,-0.225986,NA,0.189,SNIa,1.0,0.0,1.0,0.0 +2021rqo,316.97588,-5.591655,NA,0.164,SNII,0.931,0.931,0.042,0.027 +2021rrc,319.712781,10.103665,NA,0.241,SNII,0.415,0.415,0.268,0.317 +2021rui,226.588254,-11.169443,NA,0.238,SNIa,0.928,0.041,0.928,0.031 +2021rxa,331.054429,1.910881,NA,0.177,SNII,0.569,0.569,0.201,0.23 +2021rxk,226.544639,-7.488126,NA,0.185,SNIa,0.874,0.052,0.874,0.074 +2021ryz,228.004898,-3.222283,SNIa-norm,0.023,SNIa,1.0,0.0,1.0,0.0 +2021sbc,317.771975,13.078733,SNIa-norm,0.123,SNIa,1.0,0.0,1.0,0.0 +2021scl,213.994521,-5.125643,NA,0.245,SNIa,0.599,0.379,0.599,0.022 +2021scp,233.56379,14.410743,NA,0.097,SNII,0.97,0.97,0.019,0.011 +2021scq,356.399165,24.441086,NA,0.203,SNIa,0.992,0.007,0.992,0.001 +2021scr,2.793245,18.529856,NA,0.202,SNIa,0.929,0.055,0.929,0.016 +2021scs,6.845024,21.643156,NA,0.263,SNIa,0.971,0.023,0.971,0.006 +2021sdy,329.806933,2.995708,NA,0.103,SNIa,0.981,0.0,0.981,0.019 +2021seu,215.394287,37.90965,Other,0.077,SNII,0.378,0.378,0.274,0.348 +2021sev,202.145658,32.151214,SNII,0.087,SNII,0.761,0.761,0.202,0.037 +2021sfe,318.155421,-0.614875,NA,0.157,SNIa,0.996,0.0,0.996,0.004 +2021sff,10.65121,3.954375,NA,0.229,SNIa,1.0,0.0,1.0,0.0 +2021sft,317.747739,-1.547549,NA,0.181,SNIa,0.716,0.212,0.716,0.072 +2021sgk,224.054933,-7.441675,NA,0.181,SNIa,0.748,0.22,0.748,0.032 +2021sgo,223.676175,-9.529836,NA,0.204,SNIa,0.815,0.089,0.815,0.096 +2021sgt,233.409167,6.353347,NA,0.213,SNIa,0.996,0.001,0.996,0.003 +2021shc,230.026233,14.222381,NA,0.152,SNIa,0.998,0.001,0.998,0.001 +2021shd,240.789204,19.788949,NA,0.156,SNIa,0.861,0.065,0.861,0.074 +2021she,193.778393,5.700925,NA,0.125,SNIa,0.534,0.102,0.534,0.364 +2021shf,254.30574,2.947237,NA,0.155,SNIa,0.963,0.027,0.963,0.01 +2021sil,248.123959,35.933251,NA,0.164,SNIbc,0.514,0.037,0.449,0.514 +2021sje,234.758362,7.730531,SNIb,0.035,SNIa,0.938,0.038,0.938,0.024 +2021sjn,211.281195,-7.091107,NA,0.176,SNIa,0.986,0.008,0.986,0.006 +2021sjq,231.437955,10.907085,NA,0.22,SNIa,0.977,0.023,0.977,0.0 +2021sjr,172.780004,12.153433,NA,0.243,SNII,0.62,0.62,0.362,0.018 +2021sjv,317.559708,-3.663525,NA,0.183,SNIa,0.653,0.306,0.653,0.041 +2021sjw,246.404217,28.744611,NA,0.197,SNII,0.895,0.895,0.101,0.004 +2021sjx,227.005783,-8.107981,NA,0.229,SNIa,0.977,0.015,0.977,0.008 +2021sjz,14.730059,-14.228905,NA,0.169,SNII,0.797,0.797,0.032,0.171 +2021ski,184.527372,32.229806,NA,0.158,SNIa,0.854,0.1,0.854,0.046 +2021skm,244.233507,21.809983,SNII,0.035,SNII,0.57,0.57,0.017,0.413 +2021sld,225.157339,-8.283483,SNIa-norm,0.111,SNIa,1.0,0.0,1.0,0.0 +2021smj,186.69398,8.882682,SNIa-norm,0.019,SNIa,0.982,0.0,0.982,0.018 +2021smp,231.483919,-1.728976,NA,0.127,SNIa,0.998,0.0,0.998,0.002 +2021sne,230.896649,11.296208,NA,0.198,SNIa,0.997,0.002,0.997,0.001 +2021snh,232.95167,13.869117,NA,0.155,SNIa,0.989,0.007,0.989,0.004 +2021sou,194.397296,-1.426547,NA,0.153,SNII,0.535,0.535,0.43,0.035 +2021sox,311.905422,-3.992245,NA,0.193,SNIa,0.591,0.359,0.591,0.05 +2021sqj,233.031121,-0.685936,NA,0.096,SNII,0.936,0.936,0.018,0.046 +2021srs,328.039475,5.614856,NA,0.177,SNII,0.749,0.749,0.236,0.015 +2021sun,22.97566,-13.421816,SNIa-norm,0.071,SNIa,1.0,0.0,1.0,0.0 +2021suq,22.593839,-15.727367,NA,0.183,SNII,0.695,0.695,0.29,0.015 +2021sve,249.668514,32.548503,SNIa-norm,0.042,SNIa,1.0,0.0,1.0,0.0 +2021swc,182.578899,17.903038,NA,0.239,SNIa,0.567,0.41,0.567,0.023 +2021swd,169.821782,14.32841,NA,0.072,SNIbc,0.388,0.263,0.349,0.388 +2021tci,244.61412,28.602554,NA,0.143,SNIbc,0.826,0.096,0.078,0.826 +2021tex,237.935269,20.441748,NA,0.189,SNIa,0.989,0.0,0.989,0.011 +2021tfr,228.09597,-11.393252,NA,0.118,SNIbc,0.66,0.016,0.324,0.66 +2021tfw,7.290475,19.916686,NA,0.184,SNIa,0.982,0.008,0.982,0.01 +2021thj,220.092087,-8.415914,NA,0.124,SNIbc,0.565,0.088,0.347,0.565 +2021thk,5.561803,24.09003,NA,0.216,SNII,0.71,0.71,0.208,0.082 +2021tjh,210.449531,31.401045,NA,0.153,SNIbc,0.446,0.152,0.402,0.446 +2021tji,318.488332,0.01942,NA,0.232,SNIa,0.999,0.001,0.999,0.0 +2021tjj,227.971006,-3.139571,NA,0.156,SNII,0.547,0.547,0.378,0.075 +2021tjk,228.668279,-0.633583,NA,0.242,SNIa,0.899,0.054,0.899,0.047 +2021tjt,248.998912,34.127098,NA,0.227,SNIa,1.0,0.0,1.0,0.0 +2021tks,2.575069,1.416016,SNIa-norm,0.079,SNIa,1.0,0.0,1.0,0.0 +2021tol,19.502461,-1.579351,SNIa-norm,0.054,SNIa,1.0,0.0,1.0,0.0 +2021tom,152.668218,71.343075,NA,0.073,SNIa,0.787,0.053,0.787,0.16 +2021too,325.226184,10.325149,SNIc-BL,0.078,SNIbc,0.985,0.0,0.015,0.985 +2021tqc,310.866251,-1.539682,NA,0.09,SNIa,0.987,0.002,0.987,0.011 +2021tqq,37.147217,-4.615985,SNIa-norm,0.133,SNIa,0.662,0.042,0.662,0.296 +2021tsj,239.191859,28.758274,SNIa-norm,0.095,SNIa,0.988,0.0,0.988,0.012 +2021tua,239.176881,24.601762,NA,0.218,SNIa,0.985,0.008,0.985,0.007 +2021tup,334.273693,4.845726,SNIa-norm,0.135,SNIa,1.0,0.0,1.0,0.0 +2021tux,17.907889,-18.028508,SNIb,0.044,SNIbc,0.986,0.01,0.004,0.986 +2021tvn,324.232028,10.163723,NA,0.169,SNII,0.984,0.984,0.014,0.002 +2021tvz,322.51564,9.526492,NA,0.039,SNII,0.979,0.979,0.011,0.01 +2021txi,4.438534,27.243955,NA,0.105,SNIa,0.805,0.003,0.805,0.192 +2021udc,33.860312,34.740553,SNIIb,0.075,SNIa,0.997,0.0,0.997,0.003 +2021ueu,213.145311,-4.395458,NA,0.106,SNIa,0.998,0.001,0.998,0.001 +2021ufx,241.169173,27.330262,NA,0.178,SNIa,1.0,0.0,1.0,0.0 +2021uhf,233.222123,13.101972,NA,0.091,SNIa,0.98,0.003,0.98,0.017 +2021uhl,334.587883,6.122483,SNIa-norm,0.068,SNIa,1.0,0.0,1.0,0.0 +2021uil,243.564413,31.019894,NA,0.071,SNIbc,0.831,0.012,0.157,0.831 +2021uip,333.811622,-0.092265,NA,0.118,SNIa,0.959,0.004,0.959,0.037 +2021uiq,228.533721,1.126744,SNIa-CSM,0.177,SNIbc,0.569,0.018,0.413,0.569 +2021ukf,13.102699,-18.878573,SNII,0.059,SNII,0.983,0.983,0.011,0.006 +2021ukp,238.94179,25.441156,SNIa-norm,0.153,SNIa,1.0,0.0,1.0,0.0 +2021ulp,316.935771,-1.766339,NA,0.148,SNIa,0.983,0.011,0.983,0.006 +2021und,249.593683,32.853397,NA,0.164,SNII,0.808,0.808,0.181,0.011 +2021unv,39.725096,-5.270206,NA,0.166,SNIa,0.993,0.006,0.993,0.001 +2021uo,208.111091,31.579198,NA,0.18,SNII,0.719,0.719,0.277,0.004 +2021uou,40.43319,-2.247241,SNIa-norm,0.236,SNIbc,0.529,0.028,0.443,0.529 +2021uow,334.721422,5.833297,NA,0.195,SNII,0.758,0.758,0.165,0.077 +2021upd,42.218232,-3.809121,NA,0.143,SNII,0.796,0.796,0.147,0.057 +2021uqi,314.199012,-0.878739,NA,0.208,SNIa,0.643,0.351,0.643,0.006 +2021uqw,328.497613,6.697514,SNIIb,0.057,SNIa,0.558,0.0,0.558,0.442 +2021urb,323.608604,9.183878,SNIc,0.17,SNIa,0.486,0.051,0.486,0.463 +2021us,177.825,18.32665,SNIa-norm,0.14,SNIa,1.0,0.0,1.0,0.0 +2021utd,18.920468,1.19671,SNIc,0.054,SNIbc,0.944,0.009,0.047,0.944 +2021uwq,210.028476,-7.024125,NA,0.11,SNIa,0.993,0.001,0.993,0.006 +2021uww,330.372599,2.013666,NA,0.189,SNIa,0.97,0.013,0.97,0.017 +2021uwx,333.35587,6.782652,SLSN-I,0.155,SNII,0.542,0.542,0.302,0.156 +2021uxn,18.150779,-17.3961,NA,0.08,SNIbc,0.94,0.02,0.04,0.94 +2021uyq,330.911067,7.303261,SNIa-norm,0.079,SNIbc,0.661,0.054,0.285,0.661 +2021van,21.123408,-19.862139,SNIa-norm,0.073,SNIbc,0.522,0.006,0.472,0.522 +2021vaw,5.808477,16.176153,SNIa-norm,0.167,SNIa,1.0,0.0,1.0,0.0 +2021vcg,213.112348,-5.368836,NA,0.201,SNIa,0.991,0.007,0.991,0.002 +2021vcz,1.824732,0.838288,SNIa-norm,0.133,SNIa,1.0,0.0,1.0,0.0 +2021vgl,2.51042,28.211084,NA,0.027,SNII,0.727,0.727,0.017,0.256 +2021vgn,245.293814,36.061208,SNII,0.125,SNIbc,0.894,0.005,0.101,0.894 +2021vhk,357.826929,25.26525,NA,0.236,SNIa,0.587,0.388,0.587,0.025 +2021vjf,233.296002,12.995761,NA,0.21,SNIa,0.994,0.003,0.994,0.003 +2021vjk,245.666339,23.091294,NA,0.13,SNIa,0.666,0.282,0.666,0.052 +2021vjs,240.857947,25.295234,NA,0.193,SNIa,0.833,0.15,0.833,0.017 +2021vjt,212.533898,-4.307409,NA,0.206,SNIa,0.745,0.206,0.745,0.049 +2021vku,322.387057,8.079133,SNIa-norm,0.124,SNIa,0.997,0.0,0.997,0.003 +2021vml,237.715978,27.336812,NA,0.172,SNII,0.84,0.84,0.154,0.006 +2021vmm,228.002996,-11.405081,SNIa-norm,0.193,SNIa,1.0,0.0,1.0,0.0 +2021vqm,316.534533,-3.983964,NA,0.223,SNIa,0.788,0.074,0.788,0.138 +2021vqz,334.170122,-0.955667,NA,0.143,SNIa,0.997,0.001,0.997,0.002 +2021vri,35.757136,-0.419381,SNIa-norm,0.142,SNIa,1.0,0.0,1.0,0.0 +2021vrq,48.653565,0.934148,NA,0.125,SNIa,1.0,0.0,1.0,0.0 +2021vrt,223.891008,-9.264459,SNII,0.049,SNII,0.384,0.384,0.35,0.266 +2021vtq,11.912226,-20.523433,SNIa-norm,0.019,SNIa,0.999,0.0,0.999,0.001 +2021vue,235.188391,14.077381,NA,0.155,SNIa,0.895,0.067,0.895,0.038 +2021vut,37.346137,36.998717,NA,0.153,SNII,0.586,0.586,0.366,0.048 +2021vva,231.584466,14.085407,NA,0.156,SNII,0.538,0.538,0.4,0.062 +2021vvw,232.226404,-1.394456,NA,0.15,SNIa,0.608,0.02,0.608,0.372 +2021vwf,2.422892,0.06841,NA,0.156,SNII,0.546,0.546,0.402,0.052 +2021vwx,68.405046,-22.034011,SNIa-norm,0.08,SNIbc,0.886,0.031,0.083,0.886 +2021vwz,11.500265,-21.008176,NA,0.135,SNIbc,0.489,0.286,0.225,0.489 +2021vym,323.616967,10.344444,SNIa-norm,0.052,SNIa,0.974,0.0,0.974,0.026 +2021vzt,239.937599,33.930895,NA,0.217,SNIa,0.71,0.219,0.71,0.071 +2021wbk,240.210712,34.319136,NA,0.201,SNIa,0.922,0.008,0.922,0.07 +2021wbn,239.769683,33.916844,NA,0.128,SNIa,0.998,0.0,0.998,0.002 +2021wd,219.605436,41.215785,NA,0.158,SNIa,1.0,0.0,1.0,0.0 +2021wfg,325.105233,11.748978,SNIa-norm,0.084,SNIa,1.0,0.0,1.0,0.0 +2021wfn,330.71715,3.372794,SNIa-norm,0.153,SNIa,1.0,0.0,1.0,0.0 +2021whv,240.877369,24.620583,SNII,0.044,SNIa,0.526,0.348,0.526,0.126 +2021wjs,318.972508,10.690021,SNII,0.059,SNII,0.656,0.656,0.259,0.085 +2021wli,334.870529,1.000739,NA,0.186,SNIa,0.997,0.001,0.997,0.002 +2021wlm,7.169462,16.896186,NA,0.1,SNII,0.999,0.999,0.001,0.0 +2021wln,20.867171,-1.301364,NA,0.184,SNIa,0.998,0.0,0.998,0.002 +2021wlp,4.265809,18.06938,NA,0.262,SNIa,0.748,0.09,0.748,0.162 +2021wlq,18.643786,-15.874185,NA,0.105,SNII,0.659,0.659,0.23,0.111 +2021wlr,18.304326,1.281651,SNII,0.037,SNII,0.988,0.988,0.003,0.009 +2021wls,239.317467,24.859279,NA,0.207,SNIa,0.905,0.052,0.905,0.043 +2021wlt,16.446669,-19.294508,NA,0.073,SNII,0.807,0.807,0.026,0.167 +2021wlu,16.724731,0.770755,NA,0.153,SNIa,0.999,0.0,0.999,0.001 +2021wlv,359.223138,2.870367,NA,0.227,SNIa,0.656,0.206,0.656,0.138 +2021wlz,7.564136,27.459314,NA,0.131,SNIbc,0.679,0.007,0.314,0.679 +2021wmo,21.703596,2.5626,NA,0.2,SNIa,0.455,0.162,0.455,0.383 +2021wmp,332.265633,5.53565,NA,0.237,SNIa,0.861,0.104,0.861,0.035 +2021wmq,230.548029,10.570275,NA,0.261,SNIa,0.913,0.032,0.913,0.055 +2021wmr,234.34354,10.103534,NA,0.163,SNII,0.515,0.515,0.462,0.023 +2021wms,318.286254,9.217581,NA,0.199,SNII,0.949,0.949,0.046,0.005 +2021wmt,233.040222,10.413394,NA,0.205,SNIa,0.62,0.208,0.62,0.172 +2021wmu,318.177367,-3.230828,NA,0.209,SNIa,0.977,0.018,0.977,0.005 +2021wmv,247.337045,31.660969,NA,0.224,SNIa,0.822,0.16,0.822,0.018 +2021wmx,243.895046,36.465361,NA,0.212,SNIa,0.984,0.003,0.984,0.013 +2021wmy,225.82165,-4.557953,NA,0.21,SNIa,0.795,0.095,0.795,0.11 +2021wmz,318.437796,-4.141592,NA,0.203,SNII,0.667,0.667,0.295,0.038 +2021wnb,329.035275,7.121014,NA,0.232,SNIa,0.522,0.302,0.522,0.176 +2021wog,334.146342,4.907561,NA,0.121,SNIa,0.774,0.15,0.774,0.076 +2021wqr,318.208683,9.204792,NA,0.172,SNIa,0.935,0.013,0.935,0.052 +2021wr,115.760812,39.248655,NA,0.191,SNII,0.525,0.525,0.395,0.08 +2021wrt,38.419492,40.667472,SNIa-norm,0.106,SNIa,1.0,0.0,1.0,0.0 +2021wtv,353.18177,24.245497,NA,0.096,SNIa,0.978,0.015,0.978,0.007 +2021wub,210.55663,-7.259882,NA,0.218,SNIa,0.906,0.061,0.906,0.033 +2021wuc,6.577006,18.641929,NA,0.235,SNII,0.911,0.911,0.066,0.023 +2021wue,18.947387,1.505814,SNIa-norm,0.101,SNIa,1.0,0.0,1.0,0.0 +2021wug,245.398221,29.224736,NA,0.205,SNIa,0.678,0.188,0.678,0.134 +2021wvw,48.697477,40.263252,SNII,0.083,SNII,0.684,0.684,0.138,0.178 +2021xbd,239.747154,35.494747,NA,0.155,SNIa,1.0,0.0,1.0,0.0 +2021xbf,312.002502,-0.178494,SNII,0.022,SNIbc,0.916,0.006,0.078,0.916 +2021xbg,35.9852,-6.701072,SNIc,0.059,SNIa,0.768,0.002,0.768,0.23 +2021xbn,17.254741,2.431484,NA,0.186,SNII,0.525,0.525,0.403,0.072 +2021xct,242.125242,23.701142,SNIa-norm,0.082,SNIa,0.997,0.0,0.997,0.003 +2021xdd,239.685744,28.733692,NA,0.101,SNIa,0.989,0.005,0.989,0.006 +2021xei,320.955555,10.752985,NA,0.203,SNII,0.719,0.719,0.271,0.01 +2021xgi,10.90266,-20.530727,NA,0.185,SNIbc,0.557,0.119,0.324,0.557 +2021xhy,324.059102,8.898066,NA,0.188,SNIa,0.836,0.106,0.836,0.058 +2021xic,317.226246,9.657781,NA,0.199,SNIa,0.605,0.339,0.605,0.056 +2021xin,318.91743,14.572651,NA,0.099,SNII,0.853,0.853,0.099,0.048 +2021xix,224.322322,-3.863756,NA,0.222,SNIa,0.966,0.021,0.966,0.013 +2021xjc,212.531852,-5.744891,NA,0.116,SNIa,0.648,0.301,0.648,0.051 +2021xjh,325.689583,13.186776,SNIa-91T-like,0.174,SNIa,0.856,0.018,0.856,0.126 +2021xkm,18.960987,-15.905279,SNIa-norm,0.066,SNIa,1.0,0.0,1.0,0.0 +2021xmq,49.207292,3.660367,SNIa-norm,0.048,SNIa,0.499,0.018,0.499,0.483 +2021xox,131.006544,49.071764,NA,0.198,SNIbc,0.576,0.073,0.351,0.576 +2021xpq,33.690283,36.032778,NA,0.105,SNIa,0.999,0.0,0.999,0.001 +2021xqw,316.883407,-1.509456,NA,0.076,SNIa,1.0,0.0,1.0,0.0 +2021xre,3.779984,17.498727,SNIIn,0.153,SNIa,0.998,0.0,0.998,0.002 +2021xvu,16.710189,0.046778,SNIa-norm,0.09,SNII,0.464,0.464,0.34,0.196 +2021xyf,6.249771,27.610139,NA,0.049,SNII,0.983,0.983,0.006,0.011 +2021yab,20.149867,3.440495,SNIa-norm,0.107,SNIa,1.0,0.0,1.0,0.0 +2021ybc,37.329292,-4.681763,SNIIb,0.107,SNIa,0.574,0.0,0.574,0.426 +2021ycq,334.738124,6.725499,NA,0.069,SNIbc,0.95,0.01,0.04,0.95 +2021ycz,51.454477,-5.282832,NA,0.184,SNIa,0.999,0.0,0.999,0.001 +2021yfc,0.87286,2.097244,NA,0.198,SNIa,1.0,0.0,1.0,0.0 +2021yfd,330.12518,3.027246,NA,0.246,SNII,0.562,0.562,0.411,0.027 +2021yga,10.0279,3.859528,NA,0.22,SNIa,0.998,0.002,0.998,0.0 +2021ygi,37.096727,-0.816019,NA,0.158,SNIa,1.0,0.0,1.0,0.0 +2021ygj,43.138665,-3.109607,NA,0.167,SNII,0.692,0.692,0.226,0.082 +2021ygk,248.879832,36.143388,NA,0.169,SNIa,0.941,0.031,0.941,0.028 +2021ygl,240.543064,32.459818,NA,0.199,SNIa,0.948,0.022,0.948,0.03 +2021ygy,4.8257,24.645108,NA,0.111,SNIa,0.96,0.004,0.96,0.036 +2021ykz,16.669192,-19.797983,NA,0.155,SNIa,0.993,0.001,0.993,0.006 +2021ylc,14.007183,-14.630431,NA,0.191,SNIa,1.0,0.0,1.0,0.0 +2021ynu,37.035196,0.053392,NA,0.097,SNIa,0.836,0.001,0.836,0.163 +2021yny,3.001044,16.604242,NA,0.232,SNIa,0.939,0.025,0.939,0.036 +2021yow,2.318458,22.19877,NA,0.192,SNIa,0.57,0.154,0.57,0.276 +2021ypd,0.776856,17.832585,NA,0.105,SNII,0.68,0.68,0.306,0.014 +2021ype,331.358747,1.890758,NA,0.171,SNIa,0.996,0.002,0.996,0.002 +2021ypp,4.576284,20.723215,NA,0.083,SNII,0.579,0.579,0.288,0.133 +2021ypq,53.785264,-3.530638,NA,0.079,SNIa,0.821,0.166,0.821,0.013 +2021ypr,19.851019,2.876493,NA,0.129,SNII,0.887,0.887,0.111,0.002 +2021yps,19.913602,2.675678,NA,0.265,SNIa,0.807,0.101,0.807,0.092 +2021ypt,16.717842,1.063848,NA,0.224,SNIa,0.743,0.198,0.743,0.059 +2021ypu,44.707377,-0.438003,NA,0.174,SNIa,0.847,0.124,0.847,0.029 +2021yqa,21.712929,2.156575,NA,0.224,SNIa,0.985,0.005,0.985,0.01 +2021yrx,331.966214,3.073208,NA,0.122,SNIa,1.0,0.0,1.0,0.0 +2021yuc,327.493946,5.339619,NA,0.231,SNIa,0.77,0.183,0.77,0.047 +2021yxl,317.985029,-4.264739,NA,0.163,SNII,0.877,0.877,0.093,0.03 +2021yzm,6.501757,28.984146,NA,0.218,SNIa,0.781,0.124,0.781,0.095 +2021yzn,322.420395,13.875645,NA,0.128,SNII,0.461,0.461,0.287,0.252 +2021zcc,18.207325,-18.134628,NA,0.051,SNIa,0.885,0.002,0.885,0.113 +2021zcj,51.952995,-26.647118,NA,0.143,SNII,0.398,0.398,0.255,0.347 +2021zep,22.06545,-14.051486,SNIa-norm,0.104,SNIa,1.0,0.0,1.0,0.0 +2021zfw,322.906997,11.832946,SNIa-norm,0.025,SNIa,1.0,0.0,1.0,0.0 +2021zfx,317.348935,9.760885,SNIbn,0.08,SNIa,0.702,0.002,0.702,0.296 +2021zfy,54.830996,-27.554048,NA,0.155,SNIa,0.975,0.017,0.975,0.008 +2021zkm,245.871479,32.391108,NA,0.176,SNIa,0.837,0.092,0.837,0.071 +2021zmr,21.396228,-19.179478,NA,0.105,SNIa,1.0,0.0,1.0,0.0 +2021znf,67.314329,-21.892836,NA,0.109,SNII,0.641,0.641,0.287,0.072 +2021zqa,5.617929,16.736283,SNIa-norm,0.153,SNIa,1.0,0.0,1.0,0.0 +2021zqb,36.420642,-4.685683,NA,0.226,SNIa,0.637,0.157,0.637,0.206 +2021zqt,21.851168,-14.808537,NA,0.155,SNIa,1.0,0.0,1.0,0.0 +2021zqv,335.188258,7.04055,NA,0.139,SNIa,0.997,0.0,0.997,0.003 +2021zri,140.211901,42.272211,SNIa-norm,0.033,SNIa,1.0,0.0,1.0,0.0 +2021ztz,3.111527,17.197566,NA,0.218,SNIa,1.0,0.0,1.0,0.0 +2021zuf,359.456425,19.746603,NA,0.15,SNII,0.623,0.623,0.342,0.035 +2021zvo,16.923117,2.357817,NA,0.222,SNIa,0.992,0.003,0.992,0.005 +2021zvx,2.69685,17.396661,NA,0.196,SNIa,0.993,0.001,0.993,0.006 +2021zzv,68.583044,-8.624821,SNII,0.031,SNII,0.997,0.997,0.001,0.002 diff --git a/test/e2e/shardedredisbackup_sute_test.go b/test/e2e/shardedredisbackup_sute_test.go new file mode 100644 index 00000000..d96460d7 --- /dev/null +++ b/test/e2e/shardedredisbackup_sute_test.go @@ -0,0 +1,247 @@ +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + saasv1alpha1 "github.com/3scale/saas-operator/api/v1alpha1" + "github.com/3scale/saas-operator/pkg/util" + testutil "github.com/3scale/saas-operator/test/util" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + awsCredentials = "aws-credentials" + sshPrivateKey = "redis-backup-ssh-private-key" +) + +var _ = Describe("shardedredisbackup e2e suite", func() { + var ns string + var shards []saasv1alpha1.RedisShard + var sentinel saasv1alpha1.Sentinel + var backup saasv1alpha1.ShardedRedisBackup + + BeforeEach(func() { + // Create a namespace for each block + ns = "test-ns-" + nameGenerator.Generate() + + // Add any setup steps that needs to be executed before each test + testNamespace := &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Namespace"}, + ObjectMeta: metav1.ObjectMeta{Name: ns}, + } + + err := k8sClient.Create(context.Background(), testNamespace) + Expect(err).ToNot(HaveOccurred()) + + n := &corev1.Namespace{} + Eventually(func() error { + return k8sClient.Get(context.Background(), types.NamespacedName{Name: ns}, n) + }, timeout, poll).ShouldNot(HaveOccurred()) + + // create redis shards + shards = []saasv1alpha1.RedisShard{ + { + ObjectMeta: metav1.ObjectMeta{Name: "rs0", Namespace: ns}, + Spec: saasv1alpha1.RedisShardSpec{ + MasterIndex: pointer.Int32(0), + SlaveCount: pointer.Int32(2), + Command: util.Pointer("/entrypoint.sh"), + Image: &saasv1alpha1.ImageSpec{ + Name: util.Pointer("redis-with-ssh"), + Tag: util.Pointer("4.0.11-alpine"), + }, + }, + }, + // { + // ObjectMeta: metav1.ObjectMeta{Name: "rs1", Namespace: ns}, + // Spec: saasv1alpha1.RedisShardSpec{MasterIndex: pointer.Int32(2), SlaveCount: pointer.Int32(2)}, + // }, + } + + for i, shard := range shards { + err = k8sClient.Create(context.Background(), &shard) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() error { + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: shard.GetName(), Namespace: ns}, &shard) + if err != nil { + return err + } + if shard.Status.ShardNodes != nil && shard.Status.ShardNodes.Master != nil { + // store the resource for later use + shards[i] = shard + GinkgoWriter.Printf("[debug] Shard %s topology: %+v\n", shard.GetName(), *shard.Status.ShardNodes) + return nil + } else { + return fmt.Errorf("RedisShard %s not ready", shard.ObjectMeta.Name) + } + + }, timeout, poll).ShouldNot(HaveOccurred()) + } + + // create sentinel + sentinel = saasv1alpha1.Sentinel{ + ObjectMeta: metav1.ObjectMeta{Name: "sentinel", Namespace: ns}, + Spec: saasv1alpha1.SentinelSpec{ + Replicas: util.Pointer(int32(1)), + Config: &saasv1alpha1.SentinelConfig{ + MonitoredShards: map[string][]string{ + shards[0].GetName(): { + "redis://" + shards[0].Status.ShardNodes.GetHostPortByPodIndex(0), + "redis://" + shards[0].Status.ShardNodes.GetHostPortByPodIndex(1), + "redis://" + shards[0].Status.ShardNodes.GetHostPortByPodIndex(2), + }, + // shards[1].GetName(): { + // "redis://" + shards[1].Status.ShardNodes.GetHostPortByPodIndex(0), + // "redis://" + shards[1].Status.ShardNodes.GetHostPortByPodIndex(1), + // "redis://" + shards[1].Status.ShardNodes.GetHostPortByPodIndex(2), + // }, + }, + }, + }, + } + + err = k8sClient.Create(context.Background(), &sentinel) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() error { + + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: sentinel.GetName(), Namespace: ns}, &sentinel) + Expect(err).ToNot(HaveOccurred()) + + if len(sentinel.Status.MonitoredShards) != len(shards) { + return fmt.Errorf("sentinel not ready") + } + return nil + }, timeout, poll).ShouldNot(HaveOccurred()) + + // Load a test dataset into redis + rclient, stopCh, err := testutil.RedisClient(cfg, + types.NamespacedName{ + Name: "redis-shard-rs0-0", + Namespace: ns, + }) + Expect(err).ToNot(HaveOccurred()) + defer close(stopCh) + + dir, _ := os.Getwd() + // got the dataset from https://redis.com/blog/datasets-for-test-databases/ + err = testutil.LoadRedisDataset(context.Background(), rclient, filepath.Join(dir, "../assets/redis-datasets/supernovas.csv")) + Expect(err).ToNot(HaveOccurred()) + + // copy over required credentials from default namespace + for _, creds := range []string{awsCredentials, sshPrivateKey} { + secret := &corev1.Secret{} + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: creds, Namespace: "default"}, secret) + Expect(err).ToNot(HaveOccurred()) + + secret.ObjectMeta = metav1.ObjectMeta{Name: creds, Namespace: ns} + err = k8sClient.Create(context.Background(), secret) + Expect(err).ToNot(HaveOccurred()) + } + + // Create a shardedredisbackup resource + backup = saasv1alpha1.ShardedRedisBackup{ + ObjectMeta: metav1.ObjectMeta{Name: "backup", Namespace: ns}, + Spec: saasv1alpha1.ShardedRedisBackupSpec{ + SentinelRef: sentinel.GetName(), + Schedule: "* * * * *", + DBFile: "/data/dump.rdb", + SSHOptions: saasv1alpha1.SSHOptions{ + User: "docker", + PrivateKeySecretRef: corev1.LocalObjectReference{ + Name: "redis-backup-ssh-private-key", + }, + Port: util.Pointer(uint32(2222)), + Sudo: util.Pointer(true), + }, + S3Options: saasv1alpha1.S3Options{ + Bucket: "my-bucket", + Path: "backups", + Region: "us-east-1", + CredentialsSecretRef: corev1.LocalObjectReference{ + Name: "aws-credentials", + }, + ServiceEndpoint: util.Pointer("http://minio.default.svc.cluster.local:9000"), + }, + PollInterval: &metav1.Duration{Duration: 1 * time.Second}, + }, + } + + err = k8sClient.Create(context.Background(), &backup) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() error { + + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: backup.GetName(), Namespace: ns}, &backup) + Expect(err).ToNot(HaveOccurred()) + + if len(backup.Status.Backups) == 0 { + msg := "[debug] waiting for backup to be scheduled" + GinkgoWriter.Println(msg) + return fmt.Errorf(msg) + } + return nil + }, timeout, poll).ShouldNot(HaveOccurred()) + + }) + + AfterEach(func() { + + // Delete backup + err := k8sClient.Delete(context.Background(), &backup, client.PropagationPolicy(metav1.DeletePropagationForeground)) + Expect(err).ToNot(HaveOccurred()) + + // Delete sentinel + err = k8sClient.Delete(context.Background(), &sentinel, client.PropagationPolicy(metav1.DeletePropagationForeground)) + Expect(err).ToNot(HaveOccurred()) + + // Delete redis shards + for _, shard := range shards { + err := k8sClient.Delete(context.Background(), &shard, client.PropagationPolicy(metav1.DeletePropagationForeground)) + Expect(err).ToNot(HaveOccurred()) + } + + // Delete the namespace + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}} + err = k8sClient.Delete(context.Background(), ns, client.PropagationPolicy(metav1.DeletePropagationForeground)) + Expect(err).ToNot(HaveOccurred()) + }) + + It("runs a backup that completes successfully", func() { + + Eventually(func() error { + + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: backup.GetName(), Namespace: ns}, &backup) + Expect(err).ToNot(HaveOccurred()) + backupResult := backup.Status.Backups[len(backup.Status.Backups)-1] + + switch backupResult.State { + case saasv1alpha1.BackupPendingState: + GinkgoWriter.Printf("[debug %s] backup has not yet started\n", time.Now()) + return fmt.Errorf("") + case saasv1alpha1.BackupRunningState: + GinkgoWriter.Printf("[debug %s] backup is running\n", time.Now()) + return fmt.Errorf("") + case saasv1alpha1.BackupCompletedState: + GinkgoWriter.Println("[debug %s] backup completed successfully\n", time.Now()) + return nil + default: + GinkgoWriter.Printf("[debug %s] backup failed: '%s'\n", time.Now(), backupResult.Message) + return fmt.Errorf(backupResult.Message) + } + + }, timeout, poll).ShouldNot(HaveOccurred()) + + }) +}) diff --git a/test/util/redis.go b/test/util/redis.go index cad1a0a8..02bbe7f6 100644 --- a/test/util/redis.go +++ b/test/util/redis.go @@ -1,10 +1,17 @@ package util import ( + "context" + "encoding/csv" "fmt" + "io" + "net/url" + "os" + "strings" saasv1alpha1 "github.com/3scale/saas-operator/api/v1alpha1" redis "github.com/3scale/saas-operator/pkg/redis/server" + "github.com/3scale/saas-operator/pkg/util" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" ) @@ -36,3 +43,59 @@ func SentinelClient(cfg *rest.Config, podKey types.NamespacedName) (*redis.Serve return ss, stopCh, nil } + +func LoadRedisDataset(ctx context.Context, srv *redis.Server, file string) error { + + // read csv file + csvfile, err := os.Open(file) + if err != nil { + return err + } + defer csvfile.Close() + + data, err := CSVToMap(csvfile) + if err != nil { + return err + } + + for key, value := range data { + err := srv.RedisSet(ctx, key, value) + if err != nil { + return err + } + } + + return nil +} + +func CSVToMap(reader io.Reader) (map[string]string, error) { + r := csv.NewReader(reader) + m := map[string]string{} + var headers []string + for { + record, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if headers == nil { + headers = record + } else { + m = util.MergeMaps(m, RecordToKeyValues(headers, record)) + } + } + return m, nil +} + +func RecordToKeyValues(headers []string, record []string) map[string]string { + m := map[string]string{} + + //skip first header, which is the row key + for i := range headers[1:] { + key := url.QueryEscape(strings.Join([]string{record[0], headers[i]}, ".")) + m[key] = record[i] + } + return m +} From ee7ae24f318f6fa01983f64cd2704da218c2e1fe Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Mon, 18 Sep 2023 11:59:12 +0200 Subject: [PATCH 22/43] basreconciler v0.3.2 --- controllers/apicast_controller.go | 11 +++------ controllers/autossl_controller.go | 11 +++------ controllers/backend_controller.go | 11 +++------ controllers/corsproxy_controller.go | 11 +++------ controllers/echoapi_controller.go | 11 +++------ controllers/mappingservice_controller.go | 11 +++------ controllers/redisshard_controller.go | 11 +++------ controllers/sentinel_controller.go | 11 +++------ controllers/shardedredisbackup_controller.go | 11 +++------ controllers/system_controller.go | 11 +++------ controllers/twemproxyconfig_controller.go | 10 +++----- controllers/zync_controller.go | 11 +++------ go.mod | 2 +- go.sum | 4 ++-- pkg/assets/bindata.go | 12 ++++++---- .../workloads/test/test_controller.go | 6 ++--- test/e2e/sentinel_suite_test.go | 24 +++++++++---------- 17 files changed, 61 insertions(+), 118 deletions(-) diff --git a/controllers/apicast_controller.go b/controllers/apicast_controller.go index cb400c30..d33e89e9 100644 --- a/controllers/apicast_controller.go +++ b/controllers/apicast_controller.go @@ -30,7 +30,6 @@ import ( autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log" @@ -61,13 +60,9 @@ func (r *ApicastReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct instance := &saasv1alpha1.Apicast{} key := types.NamespacedName{Name: req.Name, Namespace: req.Namespace} - err := r.GetInstance(ctx, key, instance, nil, nil) - if err != nil { - if apierrors.IsNotFound(err) { - // Return and don't requeue - return ctrl.Result{}, nil - } - return ctrl.Result{}, err + result, err := r.GetInstance(ctx, key, instance, nil, nil) + if result != nil || err != nil { + return *result, err } // Apply defaults for reconcile but do not store them in the API diff --git a/controllers/autossl_controller.go b/controllers/autossl_controller.go index 4fe2f5e1..75f2d7ab 100644 --- a/controllers/autossl_controller.go +++ b/controllers/autossl_controller.go @@ -30,7 +30,6 @@ import ( autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log" @@ -60,13 +59,9 @@ func (r *AutoSSLReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct instance := &saasv1alpha1.AutoSSL{} key := types.NamespacedName{Name: req.Name, Namespace: req.Namespace} - err := r.GetInstance(ctx, key, instance, nil, nil) - if err != nil { - if apierrors.IsNotFound(err) { - // Return and don't requeue - return ctrl.Result{}, nil - } - return ctrl.Result{}, err + result, err := r.GetInstance(ctx, key, instance, nil, nil) + if result != nil || err != nil { + return *result, err } // Apply defaults for reconcile but do not store them in the API diff --git a/controllers/backend_controller.go b/controllers/backend_controller.go index 6b40a3b2..2335b63c 100644 --- a/controllers/backend_controller.go +++ b/controllers/backend_controller.go @@ -31,7 +31,6 @@ import ( autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -66,13 +65,9 @@ func (r *BackendReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct instance := &saasv1alpha1.Backend{} key := types.NamespacedName{Name: req.Name, Namespace: req.Namespace} - err := r.GetInstance(ctx, key, instance, nil, nil) - if err != nil { - if apierrors.IsNotFound(err) { - // Return and don't requeue - return ctrl.Result{}, err - } - return ctrl.Result{}, nil + result, err := r.GetInstance(ctx, key, instance, nil, nil) + if result != nil || err != nil { + return *result, err } // Apply defaults for reconcile but do not store them in the API diff --git a/controllers/corsproxy_controller.go b/controllers/corsproxy_controller.go index cea71363..c996ef70 100644 --- a/controllers/corsproxy_controller.go +++ b/controllers/corsproxy_controller.go @@ -31,7 +31,6 @@ import ( autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -65,13 +64,9 @@ func (r *CORSProxyReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( instance := &saasv1alpha1.CORSProxy{} key := types.NamespacedName{Name: req.Name, Namespace: req.Namespace} - err := r.GetInstance(ctx, key, instance, nil, nil) - if err != nil { - if apierrors.IsNotFound(err) { - // Return and don't requeue - return ctrl.Result{}, err - } - return ctrl.Result{}, nil + result, err := r.GetInstance(ctx, key, instance, nil, nil) + if result != nil || err != nil { + return *result, err } // Apply defaults for reconcile but do not store them in the API diff --git a/controllers/echoapi_controller.go b/controllers/echoapi_controller.go index 805030bc..c59015d6 100644 --- a/controllers/echoapi_controller.go +++ b/controllers/echoapi_controller.go @@ -30,7 +30,6 @@ import ( autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log" @@ -60,13 +59,9 @@ func (r *EchoAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct instance := &saasv1alpha1.EchoAPI{} key := types.NamespacedName{Name: req.Name, Namespace: req.Namespace} - err := r.GetInstance(ctx, key, instance, nil, nil) - if err != nil { - if apierrors.IsNotFound(err) { - // Return and don't requeue - return ctrl.Result{}, err - } - return ctrl.Result{}, nil + result, err := r.GetInstance(ctx, key, instance, nil, nil) + if result != nil || err != nil { + return *result, err } // Apply defaults for reconcile but do not store them in the API diff --git a/controllers/mappingservice_controller.go b/controllers/mappingservice_controller.go index 83c3a46d..f6933da4 100644 --- a/controllers/mappingservice_controller.go +++ b/controllers/mappingservice_controller.go @@ -31,7 +31,6 @@ import ( autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -65,13 +64,9 @@ func (r *MappingServiceReconciler) Reconcile(ctx context.Context, req ctrl.Reque instance := &saasv1alpha1.MappingService{} key := types.NamespacedName{Name: req.Name, Namespace: req.Namespace} - err := r.GetInstance(ctx, key, instance, nil, nil) - if err != nil { - if apierrors.IsNotFound(err) { - // Return and don't requeue - return ctrl.Result{}, nil - } - return ctrl.Result{}, err + result, err := r.GetInstance(ctx, key, instance, nil, nil) + if result != nil || err != nil { + return *result, err } // Apply defaults for reconcile but do not store them in the API diff --git a/controllers/redisshard_controller.go b/controllers/redisshard_controller.go index 137562dd..9b8709fb 100644 --- a/controllers/redisshard_controller.go +++ b/controllers/redisshard_controller.go @@ -31,7 +31,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log" @@ -59,13 +58,9 @@ func (r *RedisShardReconciler) Reconcile(ctx context.Context, req ctrl.Request) instance := &saasv1alpha1.RedisShard{} key := types.NamespacedName{Name: req.Name, Namespace: req.Namespace} - err := r.GetInstance(ctx, key, instance, nil, nil) - if err != nil { - if apierrors.IsNotFound(err) { - // Return and don't requeue - return ctrl.Result{}, nil - } - return ctrl.Result{}, err + result, err := r.GetInstance(ctx, key, instance, nil, nil) + if result != nil || err != nil { + return *result, err } // Apply defaults for reconcile but do not store them in the API diff --git a/controllers/sentinel_controller.go b/controllers/sentinel_controller.go index 2d4520db..9ec88a2b 100644 --- a/controllers/sentinel_controller.go +++ b/controllers/sentinel_controller.go @@ -36,7 +36,6 @@ import ( corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" "k8s.io/apimachinery/pkg/api/equality" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" "k8s.io/utils/pointer" @@ -74,17 +73,13 @@ func (r *SentinelReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c instance := &saasv1alpha1.Sentinel{} key := types.NamespacedName{Name: req.Name, Namespace: req.Namespace} - err := r.GetInstance(ctx, + result, err := r.GetInstance(ctx, key, instance, pointer.String(saasv1alpha1.Finalizer), []func(){r.SentinelEvents.CleanupThreads(instance), r.Metrics.CleanupThreads(instance)}) - if err != nil { - if apierrors.IsNotFound(err) { - // Return and don't requeue - return ctrl.Result{}, nil - } - return ctrl.Result{}, err + if result != nil || err != nil { + return *result, err } // Apply defaults for reconcile but do not store them in the API diff --git a/controllers/shardedredisbackup_controller.go b/controllers/shardedredisbackup_controller.go index c182c7b1..115e606f 100644 --- a/controllers/shardedredisbackup_controller.go +++ b/controllers/shardedredisbackup_controller.go @@ -22,7 +22,6 @@ import ( "time" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" @@ -76,13 +75,9 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R instance := &saasv1alpha1.ShardedRedisBackup{} key := types.NamespacedName{Name: req.Name, Namespace: req.Namespace} - err := r.GetInstance(ctx, key, instance, pointer.String(saasv1alpha1.Finalizer), []func(){r.BackupRunner.CleanupThreads(instance)}) - if err != nil { - if apierrors.IsNotFound(err) { - // Return and don't requeue - return ctrl.Result{}, err - } - return ctrl.Result{}, nil + result, err := r.GetInstance(ctx, key, instance, pointer.String(saasv1alpha1.Finalizer), []func(){r.BackupRunner.CleanupThreads(instance)}) + if result != nil || err != nil { + return *result, err } instance.Default() diff --git a/controllers/system_controller.go b/controllers/system_controller.go index 1dd78d3b..9ae529fd 100644 --- a/controllers/system_controller.go +++ b/controllers/system_controller.go @@ -31,7 +31,6 @@ import ( autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -67,13 +66,9 @@ func (r *SystemReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr instance := &saasv1alpha1.System{} key := types.NamespacedName{Name: req.Name, Namespace: req.Namespace} - err := r.GetInstance(ctx, key, instance, nil, nil) - if err != nil { - if apierrors.IsNotFound(err) { - // Return and don't requeue - return ctrl.Result{}, err - } - return ctrl.Result{}, nil + result, err := r.GetInstance(ctx, key, instance, nil, nil) + if result != nil || err != nil { + return *result, err } // Apply defaults for reconcile but do not store them in the API diff --git a/controllers/twemproxyconfig_controller.go b/controllers/twemproxyconfig_controller.go index bf86ea96..8c8fac58 100644 --- a/controllers/twemproxyconfig_controller.go +++ b/controllers/twemproxyconfig_controller.go @@ -69,15 +69,11 @@ func (r *TwemproxyConfigReconciler) Reconcile(ctx context.Context, req ctrl.Requ instance := &saasv1alpha1.TwemproxyConfig{} key := types.NamespacedName{Name: req.Name, Namespace: req.Namespace} - err := r.GetInstance(ctx, key, instance, + result, err := r.GetInstance(ctx, key, instance, pointer.String(saasv1alpha1.Finalizer), []func(){r.SentinelEvents.CleanupThreads(instance)}) - if err != nil { - if apierrors.IsNotFound(err) { - // Return and don't requeue - return ctrl.Result{}, nil - } - return ctrl.Result{}, err + if result != nil || err != nil { + return *result, err } // Apply defaults for reconcile but do not store them in the API diff --git a/controllers/zync_controller.go b/controllers/zync_controller.go index 21949ae7..37e4beb8 100644 --- a/controllers/zync_controller.go +++ b/controllers/zync_controller.go @@ -30,7 +30,6 @@ import ( autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -64,13 +63,9 @@ func (r *ZyncReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. instance := &saasv1alpha1.Zync{} key := types.NamespacedName{Name: req.Name, Namespace: req.Namespace} - err := r.GetInstance(ctx, key, instance, nil, nil) - if err != nil { - if apierrors.IsNotFound(err) { - // Return and don't requeue - return ctrl.Result{}, nil - } - return ctrl.Result{}, err + result, err := r.GetInstance(ctx, key, instance, nil, nil) + if result != nil || err != nil { + return *result, err } // Apply defaults for reconcile but do not store them in the API diff --git a/go.mod b/go.mod index c76a880c..a73a46aa 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/3scale/saas-operator go 1.20 require ( - github.com/3scale-ops/basereconciler v0.3.1 + github.com/3scale-ops/basereconciler v0.3.2 github.com/3scale-ops/marin3r v0.12.2 github.com/MakeNowJust/heredoc v1.0.0 github.com/aws/aws-sdk-go-v2 v1.21.0 diff --git a/go.sum b/go.sum index 774ff434..a2f56147 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,8 @@ contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d/g contrib.go.opencensus.io/exporter/prometheus v0.4.0 h1:0QfIkj9z/iVZgK31D9H9ohjjIDApI2GOPScCKwxedbs= contrib.go.opencensus.io/exporter/prometheus v0.4.0/go.mod h1:o7cosnyfuPVK0tB8q0QmaQNhGnptITnPQB+z1+qeFB0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/3scale-ops/basereconciler v0.3.1 h1:+otkLsodzu/k4MPWm+fAd9OwznpUCuwiSDMEed7+Q1k= -github.com/3scale-ops/basereconciler v0.3.1/go.mod h1:P1N9tWurRFAxxx5FCWHH6XcVHLMiNKxXWkH5KtP6ywQ= +github.com/3scale-ops/basereconciler v0.3.2 h1:mBekSSp4ccMK1inBWYnlbFmSGcEWtixq2yZrqdMYfQc= +github.com/3scale-ops/basereconciler v0.3.2/go.mod h1:SY0phdSkPSdE1BFR0MzS9Ze9y33+hGhuZGJIk2+AxFk= github.com/3scale-ops/marin3r v0.12.2 h1:sU4N7RZ5AQzfejrfP0KgpagmL/XD8a5NdSIv1qvaAnU= github.com/3scale-ops/marin3r v0.12.2/go.mod h1:BOU7EFv58PgPMgKgJFDd4ingfBhH6KC2AGJoD7i9SaU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= diff --git a/pkg/assets/bindata.go b/pkg/assets/bindata.go index 5bc7172d..c64b4955 100644 --- a/pkg/assets/bindata.go +++ b/pkg/assets/bindata.go @@ -354,11 +354,13 @@ var _bindata = map[string]func() (*asset, error){ // directory embedded in the file by go-bindata. // For example if you run go-bindata on data/... and data contains the // following hierarchy: -// data/ -// foo.txt -// img/ -// a.png -// b.png +// +// data/ +// foo.txt +// img/ +// a.png +// b.png +// // then AssetDir("data") would return []string{"foo.txt", "img"} // AssetDir("data/img") would return []string{"a.png", "b.png"} // AssetDir("foo.txt") and AssetDir("notexist") would return an error diff --git a/pkg/reconcilers/workloads/test/test_controller.go b/pkg/reconcilers/workloads/test/test_controller.go index 3b39bc35..761f17a5 100644 --- a/pkg/reconcilers/workloads/test/test_controller.go +++ b/pkg/reconcilers/workloads/test/test_controller.go @@ -70,9 +70,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu instance := &v1alpha1.Test{} key := types.NamespacedName{Name: req.Name, Namespace: req.Namespace} - err := r.GetInstance(ctx, key, instance, nil, nil) - if err != nil { - return ctrl.Result{}, err + result, err := r.GetInstance(ctx, key, instance, nil, nil) + if result != nil || err != nil { + return *result, err } main := &TestWorkloadGenerator{ diff --git a/test/e2e/sentinel_suite_test.go b/test/e2e/sentinel_suite_test.go index 76fcace8..334a16d2 100644 --- a/test/e2e/sentinel_suite_test.go +++ b/test/e2e/sentinel_suite_test.go @@ -164,17 +164,17 @@ var _ = Describe("sentinel e2e suite", func() { shards[0].Status.ShardNodes.GetHostPortByPodIndex(0): { Address: shards[0].Status.ShardNodes.GetHostPortByPodIndex(0), Role: redisclient.Master, - Config: map[string]string{"save": "900 1 300 10"}, + Config: map[string]string{"save": ""}, }, shards[0].Status.ShardNodes.GetHostPortByPodIndex(1): { Address: shards[0].Status.ShardNodes.GetHostPortByPodIndex(1), Role: redisclient.Slave, - Config: map[string]string{"save": "900 1 300 10", "slave-read-only": "yes"}, + Config: map[string]string{"save": "", "slave-read-only": "yes"}, }, shards[0].Status.ShardNodes.GetHostPortByPodIndex(2): { Address: shards[0].Status.ShardNodes.GetHostPortByPodIndex(2), Role: redisclient.Slave, - Config: map[string]string{"save": "900 1 300 10", "slave-read-only": "yes"}, + Config: map[string]string{"save": "", "slave-read-only": "yes"}, }, }, }, @@ -184,17 +184,17 @@ var _ = Describe("sentinel e2e suite", func() { shards[1].Status.ShardNodes.GetHostPortByPodIndex(0): { Address: shards[1].Status.ShardNodes.GetHostPortByPodIndex(0), Role: redisclient.Slave, - Config: map[string]string{"save": "900 1 300 10", "slave-read-only": "yes"}, + Config: map[string]string{"save": "", "slave-read-only": "yes"}, }, shards[1].Status.ShardNodes.GetHostPortByPodIndex(1): { Address: shards[1].Status.ShardNodes.GetHostPortByPodIndex(1), Role: redisclient.Slave, - Config: map[string]string{"save": "900 1 300 10", "slave-read-only": "yes"}, + Config: map[string]string{"save": "", "slave-read-only": "yes"}, }, shards[1].Status.ShardNodes.GetHostPortByPodIndex(2): { Address: shards[1].Status.ShardNodes.GetHostPortByPodIndex(2), Role: redisclient.Master, - Config: map[string]string{"save": "900 1 300 10"}, + Config: map[string]string{"save": ""}, }, }, }, @@ -289,17 +289,17 @@ var _ = Describe("sentinel e2e suite", func() { shards[0].Status.ShardNodes.GetHostPortByPodIndex(0): { Address: shards[0].Status.ShardNodes.GetHostPortByPodIndex(0), Role: redisclient.Slave, - Config: map[string]string{"save": "900 1 300 10", "slave-read-only": "yes"}, + Config: map[string]string{"save": "", "slave-read-only": "yes"}, }, shards[0].Status.ShardNodes.GetHostPortByPodIndex(1): { Address: shards[0].Status.ShardNodes.GetHostPortByPodIndex(1), Role: redisclient.Master, - Config: map[string]string{"save": "900 1 300 10"}, + Config: map[string]string{"save": ""}, }, shards[0].Status.ShardNodes.GetHostPortByPodIndex(2): { Address: shards[0].Status.ShardNodes.GetHostPortByPodIndex(2), Role: redisclient.Slave, - Config: map[string]string{"save": "900 1 300 10", "slave-read-only": "yes"}, + Config: map[string]string{"save": "", "slave-read-only": "yes"}, }, }, }, @@ -309,17 +309,17 @@ var _ = Describe("sentinel e2e suite", func() { shards[1].Status.ShardNodes.GetHostPortByPodIndex(0): { Address: shards[1].Status.ShardNodes.GetHostPortByPodIndex(0), Role: redisclient.Slave, - Config: map[string]string{"save": "900 1 300 10", "slave-read-only": "yes"}, + Config: map[string]string{"save": "", "slave-read-only": "yes"}, }, shards[1].Status.ShardNodes.GetHostPortByPodIndex(1): { Address: shards[1].Status.ShardNodes.GetHostPortByPodIndex(1), Role: redisclient.Slave, - Config: map[string]string{"save": "900 1 300 10", "slave-read-only": "yes"}, + Config: map[string]string{"save": "", "slave-read-only": "yes"}, }, shards[1].Status.ShardNodes.GetHostPortByPodIndex(2): { Address: shards[1].Status.ShardNodes.GetHostPortByPodIndex(2), Role: redisclient.Master, - Config: map[string]string{"save": "900 1 300 10"}, + Config: map[string]string{"save": ""}, }, }, }, From ba8db00ffa1ba8e519e12eba3e9e2c8c26d58cc8 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Mon, 18 Sep 2023 15:58:06 +0200 Subject: [PATCH 23/43] Handle error when no RO slaves are available. Use the time the backup actually starts for the backup file name --- controllers/shardedredisbackup_controller.go | 25 +++++++++++++------- pkg/redis/backup/manager.go | 10 +++++--- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/controllers/shardedredisbackup_controller.go b/controllers/shardedredisbackup_controller.go index 115e606f..c69e5207 100644 --- a/controllers/shardedredisbackup_controller.go +++ b/controllers/shardedredisbackup_controller.go @@ -114,19 +114,26 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R // ----- Phase 2: run pending backups ----- // ---------------------------------------- - // TODO: hanlde error when no available RO slaves - statusChanged := false + requeue := false runners := make([]threads.RunnableThread, 0, len(cluster.Shards)) for _, shard := range cluster.Shards { scheduledBackup := instance.Status.FindLastBackup(shard.Name, saasv1alpha1.BackupPendingState) - if scheduledBackup != nil && scheduledBackup.ScheduledFor.Time.Before(time.Now()) { + if scheduledBackup != nil && scheduledBackup.ScheduledFor.Time.Before(now) { + // hanlde error when no available RO slaves + var roSlaves []*sharded.RedisServer + if roSlaves = shard.GetSlavesRO(); len(roSlaves) == 0 { + logger.Error(fmt.Errorf("no available RO slaves in shard"), fmt.Sprintf("skipped shard %s, will be retried", shard.Name)) + requeue = true + } + // add the backup runner thread - target := shard.GetSlavesRO()[0] + target := roSlaves[0] runners = append(runners, &backup.Runner{ ShardName: shard.Name, Server: target, - Timestamp: scheduledBackup.ScheduledFor.Time, + ScheduledFor: scheduledBackup.ScheduledFor.Time, + Timestamp: now, Timeout: instance.Spec.Timeout.Duration, PollInterval: instance.Spec.PollInterval.Duration, RedisDBFile: instance.Spec.DBFile, @@ -144,8 +151,7 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R }) scheduledBackup.ServerAlias = util.Pointer(target.GetAlias()) scheduledBackup.ServerID = util.Pointer(target.ID()) - now := metav1.Now() - scheduledBackup.StartedAt = util.Pointer(now) + scheduledBackup.StartedAt = &metav1.Time{Time: now} scheduledBackup.Message = "backup is running" scheduledBackup.State = saasv1alpha1.BackupRunningState statusChanged = true @@ -155,11 +161,14 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R if err := r.BackupRunner.ReconcileThreads(ctx, instance, runners, logger.WithName("backup-runner")); err != nil { return ctrl.Result{}, err } - if statusChanged { err := r.Client.Status().Update(ctx, instance) return ctrl.Result{}, err } + // requeue if any of the shards had no available RO slaves + if requeue { + return ctrl.Result{RequeueAfter: 1 * time.Minute}, nil + } // -------------------------------------------------------- // ----- Phase 3: reconcile status of running backups ----- diff --git a/pkg/redis/backup/manager.go b/pkg/redis/backup/manager.go index 915120f9..a45957f2 100644 --- a/pkg/redis/backup/manager.go +++ b/pkg/redis/backup/manager.go @@ -16,9 +16,10 @@ type Runner struct { Instance client.Object ShardName string Server *sharded.RedisServer + ScheduledFor time.Time + Timestamp time.Time Timeout time.Duration PollInterval time.Duration - Timestamp time.Time RedisDBFile string SSHUser string SSHKey string @@ -49,7 +50,7 @@ func ID(shard, alias string, ts time.Time) string { } func (br *Runner) GetID() string { - return ID(br.ShardName, br.Server.GetAlias(), br.Timestamp) + return ID(br.ShardName, br.Server.GetAlias(), br.ScheduledFor) } // IsStarted returns whether the backup runner is started or not @@ -58,7 +59,10 @@ func (br *Runner) IsStarted() bool { } func (br *Runner) CanBeDeleted() bool { - return time.Since(br.Timestamp) > 1*time.Hour + // let the thread be deleted once the timeout has passed 2 times + // This gives enough time for the controller to update the status + // with the info of the thread once it has completed + return time.Since(br.Timestamp) > br.Timeout*2 } func (br *Runner) SetChannel(ch chan event.GenericEvent) { From d9afab8804f61ca15a4ad60bc75ebcdf96576691 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Mon, 18 Sep 2023 16:11:23 +0200 Subject: [PATCH 24/43] Hide sensitive info from showing up in logs --- pkg/redis/backup/s3upload.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/redis/backup/s3upload.go b/pkg/redis/backup/s3upload.go index e1fcf65e..6b4781be 100644 --- a/pkg/redis/backup/s3upload.go +++ b/pkg/redis/backup/s3upload.go @@ -79,7 +79,7 @@ func (br *Runner) UploadBackup(ctx context.Context) error { if br.SSHSudo { command = "sudo " + command } - logger.V(1).Info(fmt.Sprintf("running command '%s' on %s:%d", command, br.Server.GetHost(), br.SSHPort)) + logger.V(1).Info(br.hideSensitive(fmt.Sprintf("running command '%s' on %s:%d", command, br.Server.GetHost(), br.SSHPort))) output, err := remoteRun(ctx, br.SSHUser, br.Server.GetHost(), strconv.Itoa(int(br.SSHPort)), br.SSHKey, command) if output != "" { logger.V(1).Info(fmt.Sprintf("remote ssh command output: %s", output)) @@ -122,3 +122,10 @@ func remoteRun(ctx context.Context, user, addr, port, privateKey, cmd string) (s output, err := session.CombinedOutput(cmd) return string(output), err } + +func (br *Runner) hideSensitive(msg string) string { + for _, ss := range []string{br.AWSSecretAccessKey, br.SSHKey} { + msg = strings.ReplaceAll(msg, ss, "*****") + } + return msg +} From ec73500627ec69401e05e0e772a644f788a0cea0 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Mon, 18 Sep 2023 16:21:23 +0200 Subject: [PATCH 25/43] Clean/improve code comments --- api/v1alpha1/sentinel_types.go | 6 +++++- controllers/shardedredisbackup_controller.go | 7 +++---- pkg/redis/backup/bgsave.go | 1 - pkg/redis/backup/manager.go | 5 +++++ 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/sentinel_types.go b/api/v1alpha1/sentinel_types.go index ebf6dcf8..1e110e6b 100644 --- a/api/v1alpha1/sentinel_types.go +++ b/api/v1alpha1/sentinel_types.go @@ -183,6 +183,10 @@ type SentinelStatus struct { MonitoredShards MonitoredShards `json:"monitoredShards,omitempty"` } +// ShardedCluster returns a *sharded.Cluster struct from the information reported by the sentinel status instead +// of directly contacting sentinel/redis to gather the state of the cluster. This avoids calls to sentinel/redis +// but is less robust as it depends entirely on the Sentinel controller working properly and without delays. +// As of now, this is used in the SharededRedisBackup controller but not in the TwemproxyConfig controller. func (ss *SentinelStatus) ShardedCluster(ctx context.Context, pool *redis.ServerPool) (*sharded.Cluster, error) { // have a list of sentinels but must provide a map @@ -193,7 +197,7 @@ func (ss *SentinelStatus) ShardedCluster(ctx context.Context, pool *redis.Server } shards := make([]*sharded.Shard, 0, len(ss.MonitoredShards)) - // TODO: generate slice of shards from status + // generate slice of shards from status for _, s := range ss.MonitoredShards { servers := make([]*sharded.RedisServer, 0, len(s.Servers)) for _, rsd := range s.Servers { diff --git a/controllers/shardedredisbackup_controller.go b/controllers/shardedredisbackup_controller.go index c69e5207..36ea08b3 100644 --- a/controllers/shardedredisbackup_controller.go +++ b/controllers/shardedredisbackup_controller.go @@ -128,10 +128,9 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R } // add the backup runner thread - target := roSlaves[0] runners = append(runners, &backup.Runner{ ShardName: shard.Name, - Server: target, + Server: roSlaves[0], ScheduledFor: scheduledBackup.ScheduledFor.Time, Timestamp: now, Timeout: instance.Spec.Timeout.Duration, @@ -149,8 +148,8 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R AWSRegion: instance.Spec.S3Options.Region, AWSS3Endpoint: instance.Spec.S3Options.ServiceEndpoint, }) - scheduledBackup.ServerAlias = util.Pointer(target.GetAlias()) - scheduledBackup.ServerID = util.Pointer(target.ID()) + scheduledBackup.ServerAlias = util.Pointer(roSlaves[0].GetAlias()) + scheduledBackup.ServerID = util.Pointer(roSlaves[0].ID()) scheduledBackup.StartedAt = &metav1.Time{Time: now} scheduledBackup.Message = "backup is running" scheduledBackup.State = saasv1alpha1.BackupRunningState diff --git a/pkg/redis/backup/bgsave.go b/pkg/redis/backup/bgsave.go index 43653c9c..1aa5aa77 100644 --- a/pkg/redis/backup/bgsave.go +++ b/pkg/redis/backup/bgsave.go @@ -18,7 +18,6 @@ func (br *Runner) BackgroundSave(ctx context.Context) error { err = br.Server.RedisBGSave(ctx) if err != nil { - // TODO: need to hanlde the case that a save is already running logger.Error(errBGSave(err), "backup error") return errBGSave(err) } diff --git a/pkg/redis/backup/manager.go b/pkg/redis/backup/manager.go index a45957f2..b56a5cad 100644 --- a/pkg/redis/backup/manager.go +++ b/pkg/redis/backup/manager.go @@ -45,10 +45,12 @@ type RunnerStatus struct { FinishedAt time.Time } +// ID is the function that used to generate the ID of the backup runner func ID(shard, alias string, ts time.Time) string { return fmt.Sprintf("%s-%s-%d", shard, alias, ts.UTC().UnixMilli()) } +// GetID returns the ID of this backup runner func (br *Runner) GetID() string { return ID(br.ShardName, br.Server.GetAlias(), br.ScheduledFor) } @@ -58,6 +60,7 @@ func (br *Runner) IsStarted() bool { return br.status.Started } +// CanBeDeleted reports the reconciler if this backup runner key can be deleted from the map of threads func (br *Runner) CanBeDeleted() bool { // let the thread be deleted once the timeout has passed 2 times // This gives enough time for the controller to update the status @@ -65,6 +68,7 @@ func (br *Runner) CanBeDeleted() bool { return time.Since(br.Timestamp) > br.Timeout*2 } +// SetChannel created the communication channel for this backup runner func (br *Runner) SetChannel(ch chan event.GenericEvent) { br.eventsCh = ch } @@ -144,6 +148,7 @@ func (br *Runner) Stop() { br.cancel() } +// Status returns the RunnerStatus struct for this backup runner func (br *Runner) Status() RunnerStatus { return br.status } From fae070276e9859ece8a3e73052883d2ab425b30b Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Mon, 18 Sep 2023 16:59:30 +0200 Subject: [PATCH 26/43] Minor fixes --- .gitleaks.toml | 1 + config/staging/kustomization.yaml | 25 ------------------------- examples/backend/redis-v4/sentinel.yaml | 2 +- 3 files changed, 2 insertions(+), 26 deletions(-) delete mode 100644 config/staging/kustomization.yaml diff --git a/.gitleaks.toml b/.gitleaks.toml index 8bb1456b..de44544b 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -2,4 +2,5 @@ description = "global allow lists" paths = [ '''test/assets/redis-with-ssh/test-ssh-key''', + '''config/test/redis-backups/kustomization.yaml''', ] \ No newline at end of file diff --git a/config/staging/kustomization.yaml b/config/staging/kustomization.yaml deleted file mode 100644 index 44036021..00000000 --- a/config/staging/kustomization.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: 3scale-saas -resources: -- ../default - -patches: - - target: - group: apps - version: v1 - kind: Deployment - name: controller-manager - patch: |- - - op: replace - path: /spec/template/spec/containers/0/env/0 - value: { "name": "WATCH_NAMESPACE", "value": "3scale-saas" } - - op: add - path: /spec/template/spec/containers/0/env/1 - value: { "name": "LOG_LEVEL", "value": "debug" } - - op: add - path: /spec/template/spec/containers/0/env/1 - value: { "name": "LOG_MODE", "value": "dev" } - - op: replace - path: "/spec/template/spec/containers/0/imagePullPolicy" - value: "Always" diff --git a/examples/backend/redis-v4/sentinel.yaml b/examples/backend/redis-v4/sentinel.yaml index 3a3dc6cc..1a1aa66d 100644 --- a/examples/backend/redis-v4/sentinel.yaml +++ b/examples/backend/redis-v4/sentinel.yaml @@ -3,7 +3,7 @@ kind: Sentinel metadata: name: sentinel spec: - replicas: 1 + replicas: 3 config: clusterTopology: # DNS should not be used in production. DNS is From bf5f52eaeec32962b8167efaebd7412bfd7683ce Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Tue, 19 Sep 2023 12:02:41 +0200 Subject: [PATCH 27/43] Add a backup success counter --- pkg/redis/backup/metrics.go | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/pkg/redis/backup/metrics.go b/pkg/redis/backup/metrics.go index 2bc82d83..22cdb17f 100644 --- a/pkg/redis/backup/metrics.go +++ b/pkg/redis/backup/metrics.go @@ -17,14 +17,20 @@ var ( Help: `"size of the latest backup in bytes"`, }, []string{"shard"}) - backupFailures = prometheus.NewCounterVec( + backupFailureCount = prometheus.NewCounterVec( prometheus.CounterOpts{ - Name: "failures", + Name: "failure_count", Namespace: "saas_redis_backup", Help: `"total number of backup failures"`, }, []string{"shard"}) - + backupSuccessCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "success_count", + Namespace: "saas_redis_backup", + Help: `"total number of backup successes"`, + }, + []string{"shard"}) backupDuration = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "duration", @@ -37,20 +43,25 @@ var ( func init() { // Register backup metrics with the global prometheus registry metrics.Registry.MustRegister( - backupSize, backupFailures, backupDuration, + backupSize, backupFailureCount, backupDuration, backupSuccessCount, ) } func (r *Runner) publishMetrics() { + // ensure counters are initialized + if err := backupSuccessCount.With(prometheus.Labels{"shard": r.ShardName}).Write(&dto.Metric{}); err != nil { + backupFailureCount.With(prometheus.Labels{"shard": r.ShardName}).Add(0) + } + if err := backupFailureCount.With(prometheus.Labels{"shard": r.ShardName}).Write(&dto.Metric{}); err != nil { + backupFailureCount.With(prometheus.Labels{"shard": r.ShardName}).Add(0) + } + // update metrics if r.status.Error != nil { backupSize.With(prometheus.Labels{"shard": r.ShardName}).Set(float64(0)) - backupFailures.With(prometheus.Labels{"shard": r.ShardName}).Inc() + backupFailureCount.With(prometheus.Labels{"shard": r.ShardName}).Inc() } else { backupSize.With(prometheus.Labels{"shard": r.ShardName}).Set(float64(r.status.BackupSize)) backupDuration.With(prometheus.Labels{"shard": r.ShardName}).Set(math.Round(r.status.FinishedAt.Sub(r.Timestamp).Seconds())) - // ensure the failure counter is initialized - if err := backupFailures.With(prometheus.Labels{"shard": r.ShardName}).Write(&dto.Metric{}); err != nil { - backupFailures.With(prometheus.Labels{"shard": r.ShardName}).Add(0) - } + backupSuccessCount.With(prometheus.Labels{"shard": r.ShardName}).Add(1) } } From b908989a7fabb29e2db53af4cf95899a53652860 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Tue, 19 Sep 2023 18:01:53 +0200 Subject: [PATCH 28/43] Fix log levels for backup feature --- pkg/redis/backup/bgsave.go | 2 +- pkg/redis/backup/manager.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/redis/backup/bgsave.go b/pkg/redis/backup/bgsave.go index 1aa5aa77..a4ddb024 100644 --- a/pkg/redis/backup/bgsave.go +++ b/pkg/redis/backup/bgsave.go @@ -37,7 +37,7 @@ func (br *Runner) BackgroundSave(ctx context.Context) error { } if lastsave > prevsave { // BGSAVE completed - logger.Info("BGSave finished") + logger.V(1).Info("BGSave finished") return nil } case <-ctx.Done(): diff --git a/pkg/redis/backup/manager.go b/pkg/redis/backup/manager.go index b56a5cad..b53456fc 100644 --- a/pkg/redis/backup/manager.go +++ b/pkg/redis/backup/manager.go @@ -129,7 +129,7 @@ func (br *Runner) Start(parentCtx context.Context, l logr.Logger) error { return case <-done: - logger.V(1).Info("backup completed") + logger.Info("backup completed successfully") br.status.Finished = true br.status.BackupFile = fmt.Sprintf("s3://%s/%s/%s", br.S3Bucket, br.S3Path, br.BackupFileCompressed()) br.status.FinishedAt = time.Now() From 4029713621663b45afb62b19d34e256aeb3b5061 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Tue, 19 Sep 2023 18:02:09 +0200 Subject: [PATCH 29/43] Fix tests --- api/v1alpha1/sentinel_types.go | 4 ++++ api/v1alpha1/sentinel_types_test.go | 2 +- .../shardedredisbackup_controller_test.go | 24 +++++++++---------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/api/v1alpha1/sentinel_types.go b/api/v1alpha1/sentinel_types.go index 1e110e6b..c14279a1 100644 --- a/api/v1alpha1/sentinel_types.go +++ b/api/v1alpha1/sentinel_types.go @@ -18,6 +18,7 @@ package v1alpha1 import ( "context" + "sort" "time" "github.com/3scale/saas-operator/pkg/redis/client" @@ -207,6 +208,9 @@ func (ss *SentinelStatus) ShardedCluster(ctx context.Context, pool *redis.Server } servers = append(servers, sharded.NewRedisServerFromParams(srv, rsd.Role, rsd.Config)) } + sort.Slice(servers, func(i, j int) bool { + return servers[i].ID() < servers[j].ID() + }) shards = append(shards, sharded.NewShardFromServers(s.Name, pool, servers...)) } diff --git a/api/v1alpha1/sentinel_types_test.go b/api/v1alpha1/sentinel_types_test.go index 1b409a87..053c6c12 100644 --- a/api/v1alpha1/sentinel_types_test.go +++ b/api/v1alpha1/sentinel_types_test.go @@ -47,7 +47,7 @@ func TestSentinelStatus_ShardedCluster(t *testing.T) { { name: "Generates a sharded.Cluster resource from the sentinel status", fields: fields{ - Sentinels: []string{"redis://127.0.0.1:26379"}, + Sentinels: []string{"127.0.0.1:26379"}, MonitoredShards: []MonitoredShard{ {Name: "shard01", Servers: map[string]RedisServerDetails{ diff --git a/controllers/shardedredisbackup_controller_test.go b/controllers/shardedredisbackup_controller_test.go index 236c83ca..3dc6b858 100644 --- a/controllers/shardedredisbackup_controller_test.go +++ b/controllers/shardedredisbackup_controller_test.go @@ -67,13 +67,13 @@ func TestShardedRedisBackupReconciler_reconcileBackupList(t *testing.T) { { Shard: "shard02", ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), - Message: "Scheduled", + Message: "backup scheduled", State: saasv1alpha1.BackupPendingState, }, { Shard: "shard01", ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), - Message: "Scheduled", + Message: "backup scheduled", State: saasv1alpha1.BackupPendingState, }, }, @@ -95,13 +95,13 @@ func TestShardedRedisBackupReconciler_reconcileBackupList(t *testing.T) { { Shard: "shard02", ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), - Message: "Scheduled", + Message: "backup scheduled", State: saasv1alpha1.BackupPendingState, }, { Shard: "shard01", ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), - Message: "Scheduled", + Message: "backup scheduled", State: saasv1alpha1.BackupPendingState, }, }}, @@ -114,13 +114,13 @@ func TestShardedRedisBackupReconciler_reconcileBackupList(t *testing.T) { { Shard: "shard02", ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), - Message: "Scheduled", + Message: "backup scheduled", State: saasv1alpha1.BackupPendingState, }, { Shard: "shard01", ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), - Message: "Scheduled", + Message: "backup scheduled", State: saasv1alpha1.BackupPendingState, }, }, @@ -142,13 +142,13 @@ func TestShardedRedisBackupReconciler_reconcileBackupList(t *testing.T) { { Shard: "shard02", ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), - Message: "Scheduled", + Message: "backup scheduled", State: saasv1alpha1.BackupPendingState, }, { Shard: "shard01", ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), - Message: "Scheduled", + Message: "backup scheduled", State: saasv1alpha1.BackupPendingState, }, }}, @@ -161,25 +161,25 @@ func TestShardedRedisBackupReconciler_reconcileBackupList(t *testing.T) { { Shard: "shard02", ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:02:00Z")), - Message: "Scheduled", + Message: "backup scheduled", State: saasv1alpha1.BackupPendingState, }, { Shard: "shard01", ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:02:00Z")), - Message: "Scheduled", + Message: "backup scheduled", State: saasv1alpha1.BackupPendingState, }, { Shard: "shard02", ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), - Message: "Scheduled", + Message: "backup scheduled", State: saasv1alpha1.BackupPendingState, }, { Shard: "shard01", ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), - Message: "Scheduled", + Message: "backup scheduled", State: saasv1alpha1.BackupPendingState, }, }, From ff33e326251d15fdb8561d0de56041295b5a71e0 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Tue, 19 Sep 2023 18:21:21 +0200 Subject: [PATCH 30/43] Raise e2e tests timeout Sometimes Sentinel is not ready within the timeout, so raise it by just another 30 more seconds. --- test/e2e/suite_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index e7a056c8..0beff8fe 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -47,7 +47,7 @@ var ( cfg *rest.Config k8sClient client.Client testEnv *envtest.Environment - timeout time.Duration = 120 * time.Second + timeout time.Duration = 150 * time.Second poll time.Duration = 10 * time.Second nameGenerator namegenerator.Generator ) From b25707c90889df8dea19627ebdca99c3f80147af Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Tue, 19 Sep 2023 22:55:01 +0200 Subject: [PATCH 31/43] Check the backup file in the e2e test --- ...st.go => shardedredisbackup_suite_test.go} | 35 +++++++++++-- test/util/minio.go | 51 +++++++++++++++++++ 2 files changed, 81 insertions(+), 5 deletions(-) rename test/e2e/{shardedredisbackup_sute_test.go => shardedredisbackup_suite_test.go} (85%) create mode 100644 test/util/minio.go diff --git a/test/e2e/shardedredisbackup_sute_test.go b/test/e2e/shardedredisbackup_suite_test.go similarity index 85% rename from test/e2e/shardedredisbackup_sute_test.go rename to test/e2e/shardedredisbackup_suite_test.go index d96460d7..1b42e4f6 100644 --- a/test/e2e/shardedredisbackup_sute_test.go +++ b/test/e2e/shardedredisbackup_suite_test.go @@ -5,11 +5,14 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" saasv1alpha1 "github.com/3scale/saas-operator/api/v1alpha1" "github.com/3scale/saas-operator/pkg/util" testutil "github.com/3scale/saas-operator/test/util" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -22,6 +25,8 @@ import ( const ( awsCredentials = "aws-credentials" sshPrivateKey = "redis-backup-ssh-private-key" + bucketName = "my-bucket" + backupsPath = "backups" ) var _ = Describe("shardedredisbackup e2e suite", func() { @@ -166,8 +171,8 @@ var _ = Describe("shardedredisbackup e2e suite", func() { Sudo: util.Pointer(true), }, S3Options: saasv1alpha1.S3Options{ - Bucket: "my-bucket", - Path: "backups", + Bucket: bucketName, + Path: backupsPath, Region: "us-east-1", CredentialsSecretRef: corev1.LocalObjectReference{ Name: "aws-credentials", @@ -218,13 +223,14 @@ var _ = Describe("shardedredisbackup e2e suite", func() { Expect(err).ToNot(HaveOccurred()) }) - It("runs a backup that completes successfully", func() { + FIt("runs a backup that completes successfully", func() { + var backupResult saasv1alpha1.BackupStatus Eventually(func() error { err := k8sClient.Get(context.Background(), types.NamespacedName{Name: backup.GetName(), Namespace: ns}, &backup) Expect(err).ToNot(HaveOccurred()) - backupResult := backup.Status.Backups[len(backup.Status.Backups)-1] + backupResult = backup.Status.Backups[len(backup.Status.Backups)-1] switch backupResult.State { case saasv1alpha1.BackupPendingState: @@ -234,7 +240,7 @@ var _ = Describe("shardedredisbackup e2e suite", func() { GinkgoWriter.Printf("[debug %s] backup is running\n", time.Now()) return fmt.Errorf("") case saasv1alpha1.BackupCompletedState: - GinkgoWriter.Println("[debug %s] backup completed successfully\n", time.Now()) + GinkgoWriter.Printf("[debug %s] backup completed successfully\n", time.Now()) return nil default: GinkgoWriter.Printf("[debug %s] backup failed: '%s'\n", time.Now(), backupResult.Message) @@ -243,5 +249,24 @@ var _ = Describe("shardedredisbackup e2e suite", func() { }, timeout, poll).ShouldNot(HaveOccurred()) + By("checking that the backup is actually where the status says it is and has the reported size") + ctx := context.Background() + list := &corev1.PodList{} + err := k8sClient.List(context.Background(), list, + client.InNamespace("default"), + client.MatchingLabels{"app": "minio"}) + Expect(err).ToNot(HaveOccurred()) + Expect(len(list.Items)).To(Equal(1)) + + s3client, stopCh, err := testutil.MinioClient(ctx, cfg, client.ObjectKeyFromObject(&list.Items[0]), "admin", "admin123") + Expect(err).ToNot(HaveOccurred()) + defer close(stopCh) + + result, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(strings.TrimPrefix(*backupResult.BackupFile, fmt.Sprintf("s3://%s/", bucketName))), + }) + Expect(err).ToNot(HaveOccurred()) + Expect(result.ContentLength).To(Equal(*backupResult.BackupSize)) }) }) diff --git a/test/util/minio.go b/test/util/minio.go new file mode 100644 index 00000000..ce345b05 --- /dev/null +++ b/test/util/minio.go @@ -0,0 +1,51 @@ +package util + +import ( + "context" + "fmt" + "os" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" +) + +const ( + minioPort uint32 = 9000 + region string = "us-east-1" +) + +func MinioClient(ctx context.Context, cfg *rest.Config, podKey types.NamespacedName, user, passwd string) (*s3.Client, chan struct{}, error) { + + // set credentials + os.Setenv("AWS_ACCESS_KEY_ID", user) + os.Setenv("AWS_SECRET_ACCESS_KEY", passwd) + os.Setenv("AWS_REGION", region) + + localPort, stopCh, err := PortForward(cfg, podKey, minioPort) + if err != nil { + return nil, nil, err + } + + resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...any) (aws.Endpoint, error) { + return aws.Endpoint{ + PartitionID: "aws", + URL: fmt.Sprintf("http://localhost:%d", localPort), + SigningRegion: region, + HostnameImmutable: true, + }, nil + }) + + awsconfig, err := config.LoadDefaultConfig(ctx, + config.WithEndpointResolverWithOptions(resolver), + config.WithClientLogMode(aws.LogResponseWithBody), + ) + + if err != nil { + return nil, nil, err + } + client := s3.NewFromConfig(awsconfig) + return client, stopCh, nil +} From 6a8db1f01f9ffe967a598deae363a45290ea386b Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Tue, 19 Sep 2023 23:00:36 +0200 Subject: [PATCH 32/43] Fix bug when no RO slaves available --- controllers/shardedredisbackup_controller.go | 1 + test/e2e/shardedredisbackup_suite_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/controllers/shardedredisbackup_controller.go b/controllers/shardedredisbackup_controller.go index 36ea08b3..d8cfa57e 100644 --- a/controllers/shardedredisbackup_controller.go +++ b/controllers/shardedredisbackup_controller.go @@ -125,6 +125,7 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R if roSlaves = shard.GetSlavesRO(); len(roSlaves) == 0 { logger.Error(fmt.Errorf("no available RO slaves in shard"), fmt.Sprintf("skipped shard %s, will be retried", shard.Name)) requeue = true + continue } // add the backup runner thread diff --git a/test/e2e/shardedredisbackup_suite_test.go b/test/e2e/shardedredisbackup_suite_test.go index 1b42e4f6..a547509b 100644 --- a/test/e2e/shardedredisbackup_suite_test.go +++ b/test/e2e/shardedredisbackup_suite_test.go @@ -223,7 +223,7 @@ var _ = Describe("shardedredisbackup e2e suite", func() { Expect(err).ToNot(HaveOccurred()) }) - FIt("runs a backup that completes successfully", func() { + It("runs a backup that completes successfully", func() { var backupResult saasv1alpha1.BackupStatus Eventually(func() error { From a09ae99abe25d5ecb5eb0ff8462b212038a7da4a Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Fri, 22 Sep 2023 11:12:41 +0200 Subject: [PATCH 33/43] Add backup sample --- .../saas_v1alpha1_shardedredisbackup.yaml | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/config/samples/saas_v1alpha1_shardedredisbackup.yaml b/config/samples/saas_v1alpha1_shardedredisbackup.yaml index 59fa3d24..1f967e4b 100644 --- a/config/samples/saas_v1alpha1_shardedredisbackup.yaml +++ b/config/samples/saas_v1alpha1_shardedredisbackup.yaml @@ -1,12 +1,22 @@ apiVersion: saas.3scale.net/v1alpha1 kind: ShardedRedisBackup metadata: - labels: - app.kubernetes.io/name: shardedredisbackup - app.kubernetes.io/instance: shardedredisbackup-sample - app.kubernetes.io/part-of: saas-operator - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/created-by: saas-operator - name: shardedredisbackup-sample + name: backup + namespace: default spec: - # TODO(user): Add fields here + timeout: 5m + schedule: "*/30 * * * *" + sentinelRef: sentinel + historyLimit: 10 + pollInterval: 10s + dbFile: /data/dump.rdb + sshOptions: + privateKeySecretRef: + name: redis-ssh-private-key + user: root + s3Options: + bucket: my-bucket + path: backups + region: us-east-1 + credentialsSecretRef: + name: aws-credentials From 38b5d5e724b2ce6f07e4ee69b1ba870a93e09e36 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Tue, 26 Sep 2023 12:07:31 +0200 Subject: [PATCH 34/43] Use AWS static credential provider instead of envvars --- go.mod | 2 +- pkg/redis/backup/tag.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index a73a46aa..c95bc4fd 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/MakeNowJust/heredoc v1.0.0 github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/config v1.18.37 + github.com/aws/aws-sdk-go-v2/credentials v1.13.35 github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 github.com/davecgh/go-spew v1.1.1 github.com/envoyproxy/go-control-plane v0.11.0 @@ -45,7 +46,6 @@ require ( contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.35 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect diff --git a/pkg/redis/backup/tag.go b/pkg/redis/backup/tag.go index 1b8ad635..d0534895 100644 --- a/pkg/redis/backup/tag.go +++ b/pkg/redis/backup/tag.go @@ -3,11 +3,11 @@ package backup import ( "context" "fmt" - "os" "sort" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "sigs.k8s.io/controller-runtime/pkg/log" @@ -16,11 +16,6 @@ import ( func (br *Runner) TagBackup(ctx context.Context) error { logger := log.FromContext(ctx, "function", "(br *Runner) TagBackup()") - // set AWS credentials - os.Setenv(awsAccessKeyEnvvar, br.AWSAccessKeyID) - os.Setenv(awsSecretKeyEnvvar, br.AWSSecretAccessKey) - os.Setenv(awsRegionEnvvar, br.AWSRegion) - var cfg aws.Config var err error @@ -35,13 +30,18 @@ func (br *Runner) TagBackup(ctx context.Context) error { }) cfg, err = config.LoadDefaultConfig(ctx, + config.WithRegion(br.AWSRegion), config.WithEndpointResolverWithOptions(resolver), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(br.AWSAccessKeyID, br.AWSSecretAccessKey, "")), ) if err != nil { return err } } else { - cfg, err = config.LoadDefaultConfig(ctx) + cfg, err = config.LoadDefaultConfig(ctx, + config.WithRegion(br.AWSRegion), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(br.AWSAccessKeyID, br.AWSSecretAccessKey, "")), + ) if err != nil { return err } From 8d1f1fcdb984a17c8571562e3b6c479f8ad8fcb3 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Tue, 26 Sep 2023 16:08:59 +0200 Subject: [PATCH 35/43] Extract AWS client config to util package --- pkg/redis/backup/s3upload.go | 13 ++++----- pkg/redis/backup/tag.go | 37 ++++---------------------- pkg/util/aws_client.go | 51 ++++++++++++++++++++++++++++++++++++ test/util/minio.go | 26 +++--------------- 4 files changed, 64 insertions(+), 63 deletions(-) create mode 100644 pkg/util/aws_client.go diff --git a/pkg/redis/backup/s3upload.go b/pkg/redis/backup/s3upload.go index 6b4781be..e02fd391 100644 --- a/pkg/redis/backup/s3upload.go +++ b/pkg/redis/backup/s3upload.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/3scale/saas-operator/pkg/util" "golang.org/x/crypto/ssh" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -16,10 +17,6 @@ import ( const ( backupFilePrefix string = "redis-backup" backupFileExtension string = "rdb" - sshKeyFile string = "ssh-private-key" - awsAccessKeyEnvvar string = "AWS_ACCESS_KEY_ID" - awsSecretKeyEnvvar string = "AWS_SECRET_ACCESS_KEY" - awsRegionEnvvar string = "AWS_REGION" ) func (br *Runner) BackupFileBaseName() string { @@ -65,10 +62,10 @@ func (br *Runner) UploadBackup(ctx context.Context) error { fmt.Sprintf("gzip %s/%s", path.Dir(br.RedisDBFile), br.BackupFile()), // TODO: use awscli instead // AWS_ACCESS_KEY_ID=*** AWS_SECRET_ACCESS_KEY=*** s3cmd put /data/redis-backup---.rdb s3:////redis-backup---.rdb - fmt.Sprintf("%s=%s %s=%s %s=%s %s s3 cp %s/%s s3://%s/%s/%s", - awsRegionEnvvar, br.AWSRegion, - awsAccessKeyEnvvar, br.AWSAccessKeyID, - awsSecretKeyEnvvar, br.AWSSecretAccessKey, + fmt.Sprintf("%s=%s %s=%s %s=%s %s s3 cp --only-show-errors %s/%s s3://%s/%s/%s", + util.AWSRegionEnvvar, br.AWSRegion, + util.AWSAccessKeyEnvvar, br.AWSAccessKeyID, + util.AWSSecretKeyEnvvar, br.AWSSecretAccessKey, awsBaseCommand, path.Dir(br.RedisDBFile), br.BackupFileCompressed(), br.S3Bucket, br.S3Path, br.BackupFileCompressed(), diff --git a/pkg/redis/backup/tag.go b/pkg/redis/backup/tag.go index d0534895..d16c77a1 100644 --- a/pkg/redis/backup/tag.go +++ b/pkg/redis/backup/tag.go @@ -5,9 +5,8 @@ import ( "fmt" "sort" + "github.com/3scale/saas-operator/pkg/util" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "sigs.k8s.io/controller-runtime/pkg/log" @@ -16,38 +15,12 @@ import ( func (br *Runner) TagBackup(ctx context.Context) error { logger := log.FromContext(ctx, "function", "(br *Runner) TagBackup()") - var cfg aws.Config - var err error - - if br.AWSS3Endpoint != nil { - resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...any) (aws.Endpoint, error) { - return aws.Endpoint{ - PartitionID: "aws", - URL: *br.AWSS3Endpoint, - SigningRegion: br.AWSRegion, - HostnameImmutable: true, - }, nil - }) - - cfg, err = config.LoadDefaultConfig(ctx, - config.WithRegion(br.AWSRegion), - config.WithEndpointResolverWithOptions(resolver), - config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(br.AWSAccessKeyID, br.AWSSecretAccessKey, "")), - ) - if err != nil { - return err - } - } else { - cfg, err = config.LoadDefaultConfig(ctx, - config.WithRegion(br.AWSRegion), - config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(br.AWSAccessKeyID, br.AWSSecretAccessKey, "")), - ) - if err != nil { - return err - } + awsconfig, err := util.AWSConfig(ctx, br.AWSAccessKeyID, br.AWSSecretAccessKey, br.AWSRegion, br.AWSS3Endpoint) + if err != nil { + return err } - client := s3.NewFromConfig(cfg) + client := s3.NewFromConfig(*awsconfig) // get backups of current day dayResult, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ diff --git a/pkg/util/aws_client.go b/pkg/util/aws_client.go new file mode 100644 index 00000000..677cc727 --- /dev/null +++ b/pkg/util/aws_client.go @@ -0,0 +1,51 @@ +package util + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" +) + +const ( + AWSAccessKeyEnvvar string = "AWS_ACCESS_KEY_ID" + AWSSecretKeyEnvvar string = "AWS_SECRET_ACCESS_KEY" + AWSRegionEnvvar string = "AWS_REGION" +) + +func AWSConfig(ctx context.Context, accessKeyID, secretAccessKey, region string, serviceEndpoint *string) (*aws.Config, error) { + + var cfg aws.Config + var err error + + if serviceEndpoint != nil { + resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...any) (aws.Endpoint, error) { + return aws.Endpoint{ + PartitionID: "aws", + URL: *serviceEndpoint, + SigningRegion: region, + HostnameImmutable: true, + }, nil + }) + + cfg, err = config.LoadDefaultConfig(ctx, + config.WithRegion(region), + config.WithEndpointResolverWithOptions(resolver), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, "")), + ) + if err != nil { + return nil, err + } + } else { + cfg, err = config.LoadDefaultConfig(ctx, + config.WithRegion(region), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, "")), + ) + if err != nil { + return nil, err + } + } + + return &cfg, nil +} diff --git a/test/util/minio.go b/test/util/minio.go index ce345b05..6e085ad9 100644 --- a/test/util/minio.go +++ b/test/util/minio.go @@ -3,10 +3,8 @@ package util import ( "context" "fmt" - "os" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" + "github.com/3scale/saas-operator/pkg/util" "github.com/aws/aws-sdk-go-v2/service/s3" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" @@ -19,33 +17,15 @@ const ( func MinioClient(ctx context.Context, cfg *rest.Config, podKey types.NamespacedName, user, passwd string) (*s3.Client, chan struct{}, error) { - // set credentials - os.Setenv("AWS_ACCESS_KEY_ID", user) - os.Setenv("AWS_SECRET_ACCESS_KEY", passwd) - os.Setenv("AWS_REGION", region) - localPort, stopCh, err := PortForward(cfg, podKey, minioPort) if err != nil { return nil, nil, err } - resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...any) (aws.Endpoint, error) { - return aws.Endpoint{ - PartitionID: "aws", - URL: fmt.Sprintf("http://localhost:%d", localPort), - SigningRegion: region, - HostnameImmutable: true, - }, nil - }) - - awsconfig, err := config.LoadDefaultConfig(ctx, - config.WithEndpointResolverWithOptions(resolver), - config.WithClientLogMode(aws.LogResponseWithBody), - ) - + awsconfig, err := util.AWSConfig(ctx, user, passwd, region, util.Pointer(fmt.Sprintf("http://localhost:%d", localPort))) if err != nil { return nil, nil, err } - client := s3.NewFromConfig(awsconfig) + client := s3.NewFromConfig(*awsconfig) return client, stopCh, nil } From 021442fe4f61268cd244a572d642f801b44ad85d Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Tue, 26 Sep 2023 16:23:40 +0200 Subject: [PATCH 36/43] Add checks to ensure the AWS and SSH secrets have the required keys --- controllers/shardedredisbackup_controller.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/controllers/shardedredisbackup_controller.go b/controllers/shardedredisbackup_controller.go index d8cfa57e..174ba49e 100644 --- a/controllers/shardedredisbackup_controller.go +++ b/controllers/shardedredisbackup_controller.go @@ -102,6 +102,9 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R if sshPrivateKey.Type != corev1.SecretTypeSSHAuth { return ctrl.Result{}, fmt.Errorf("secret %s must be of 'kubernetes.io/ssh-auth' type", sshPrivateKey.GetName()) } + if _, ok := sshPrivateKey.Data[corev1.SSHAuthPrivateKey]; !ok { + return ctrl.Result{}, fmt.Errorf("secret %s is missing %s key", sshPrivateKey.GetName(), corev1.SSHAuthPrivateKey) + } // Get AWS credentials awsCredentials := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ @@ -109,6 +112,12 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R if err := r.Client.Get(ctx, client.ObjectKeyFromObject(awsCredentials), awsCredentials); err != nil { return ctrl.Result{}, err } + if _, ok := awsCredentials.Data[util.AWSAccessKeyEnvvar]; !ok { + return ctrl.Result{}, fmt.Errorf("secret %s is missing %s key", awsCredentials.GetName(), util.AWSAccessKeyEnvvar) + } + if _, ok := awsCredentials.Data[util.AWSSecretKeyEnvvar]; !ok { + return ctrl.Result{}, fmt.Errorf("secret %s is missing %s key", awsCredentials.GetName(), util.AWSSecretKeyEnvvar) + } // ---------------------------------------- // ----- Phase 2: run pending backups ----- From fc39200ad12905635b8e67bc0b6250871195cb9d Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Wed, 27 Sep 2023 10:48:29 +0200 Subject: [PATCH 37/43] Only 1 scheduled backup per shard --- api/v1alpha1/shardedredisbackup_types.go | 10 +- api/v1alpha1/shardedredisbackup_types_test.go | 100 ++++++++++++++++-- controllers/shardedredisbackup_controller.go | 9 +- 3 files changed, 102 insertions(+), 17 deletions(-) diff --git a/api/v1alpha1/shardedredisbackup_types.go b/api/v1alpha1/shardedredisbackup_types.go index 0a14fe50..f2e4400b 100644 --- a/api/v1alpha1/shardedredisbackup_types.go +++ b/api/v1alpha1/shardedredisbackup_types.go @@ -149,14 +149,18 @@ func (status *ShardedRedisBackupStatus) AddBackup(b BackupStatus) { sort.Sort(sort.Reverse(status.Backups)) } -func (status *ShardedRedisBackupStatus) FindLastBackup(shardName string, state BackupState) *BackupStatus { +func (status *ShardedRedisBackupStatus) DeleteBackup(pos int) { + status.Backups = append(status.Backups[:pos], status.Backups[pos+1:]...) +} + +func (status *ShardedRedisBackupStatus) FindLastBackup(shardName string, state BackupState) (*BackupStatus, int) { // backups expected to be ordered from newer to oldest for i, b := range status.Backups { if b.Shard == shardName && b.State == state { - return &status.Backups[i] + return &status.Backups[i], i } } - return nil + return nil, -1 } func (status *ShardedRedisBackupStatus) GetRunningBackups() []*BackupStatus { diff --git a/api/v1alpha1/shardedredisbackup_types_test.go b/api/v1alpha1/shardedredisbackup_types_test.go index ee6c2eb8..8171fe22 100644 --- a/api/v1alpha1/shardedredisbackup_types_test.go +++ b/api/v1alpha1/shardedredisbackup_types_test.go @@ -84,10 +84,11 @@ func TestShardedRedisBackupStatus_FindBackup(t *testing.T) { state BackupState } tests := []struct { - name string - fields fields - args args - want *BackupStatus + name string + fields fields + args args + want *BackupStatus + wantPos int }{ { name: "Returns latest pending backup", @@ -121,6 +122,7 @@ func TestShardedRedisBackupStatus_FindBackup(t *testing.T) { ScheduledFor: metav1.Date(2023, time.August, 1, 0, 1, 0, 0, time.UTC), State: BackupPendingState, }, + wantPos: 0, }, { name: "Returns latest running backup", @@ -154,6 +156,7 @@ func TestShardedRedisBackupStatus_FindBackup(t *testing.T) { ScheduledFor: metav1.Date(2023, time.August, 1, 0, 0, 0, 0, time.UTC), State: BackupRunningState, }, + wantPos: 2, }, { name: "Returns a nil", @@ -166,11 +169,85 @@ func TestShardedRedisBackupStatus_FindBackup(t *testing.T) { }, }, }, - args: args{shard: "shard02", state: BackupPendingState}, - want: &BackupStatus{ - Shard: "shard02", - ScheduledFor: metav1.Date(2023, time.August, 1, 0, 1, 0, 0, time.UTC), - State: BackupPendingState, + args: args{shard: "shard02", state: BackupRunningState}, + want: nil, + wantPos: -1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + status := &ShardedRedisBackupStatus{ + Backups: tt.fields.Backups, + } + gotBackup, gotPos := status.FindLastBackup(tt.args.shard, tt.args.state) + if !reflect.DeepEqual(gotBackup, tt.want) { + t.Errorf("ShardedRedisBackupStatus.FindBackup() = %v, want %v", gotBackup, tt.want) + } + if gotPos != tt.wantPos { + t.Errorf("ShardedRedisBackupStatus.FindBackup() = %v, want %v", gotPos, tt.wantPos) + } + }) + } +} + +func TestShardedRedisBackupStatus_DeleteBackup(t *testing.T) { + type fields struct { + Backups BackupStatusList + } + type args struct { + pos int + } + tests := []struct { + name string + fields fields + args args + want BackupStatusList + }{ + { + name: "Deletes the backup at the given position from the list", + fields: fields{ + Backups: BackupStatusList{ + { + Shard: "shard02", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 1, 0, 0, time.UTC), + State: BackupPendingState, + }, + { + Shard: "shard01", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 1, 0, 0, time.UTC), + State: BackupPendingState, + }, + { + Shard: "shard02", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 0, 0, 0, time.UTC), + State: BackupRunningState, + }, + { + Shard: "shard01", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 0, 0, 0, time.UTC), + State: BackupRunningState, + }, + }, + }, + args: args{ + pos: 2, + }, + want: BackupStatusList{ + { + Shard: "shard02", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 1, 0, 0, time.UTC), + State: BackupPendingState, + }, + { + Shard: "shard01", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 1, 0, 0, time.UTC), + State: BackupPendingState, + }, + { + Shard: "shard01", + ScheduledFor: metav1.Date(2023, time.August, 1, 0, 0, 0, 0, time.UTC), + State: BackupRunningState, + }, }, }, } @@ -179,8 +256,9 @@ func TestShardedRedisBackupStatus_FindBackup(t *testing.T) { status := &ShardedRedisBackupStatus{ Backups: tt.fields.Backups, } - if got := status.FindLastBackup(tt.args.shard, tt.args.state); !reflect.DeepEqual(got, tt.want) { - t.Errorf("ShardedRedisBackupStatus.FindBackup() = %v, want %v", got, tt.want) + status.DeleteBackup(tt.args.pos) + if !reflect.DeepEqual(status.Backups, tt.want) { + t.Errorf("ShardedRedisBackupStatus.FindBackup() = %v, want %v", status.Backups, tt.want) } }) } diff --git a/controllers/shardedredisbackup_controller.go b/controllers/shardedredisbackup_controller.go index 174ba49e..f47bb8ad 100644 --- a/controllers/shardedredisbackup_controller.go +++ b/controllers/shardedredisbackup_controller.go @@ -127,7 +127,7 @@ func (r *ShardedRedisBackupReconciler) Reconcile(ctx context.Context, req ctrl.R requeue := false runners := make([]threads.RunnableThread, 0, len(cluster.Shards)) for _, shard := range cluster.Shards { - scheduledBackup := instance.Status.FindLastBackup(shard.Name, saasv1alpha1.BackupPendingState) + scheduledBackup, _ := instance.Status.FindLastBackup(shard.Name, saasv1alpha1.BackupPendingState) if scheduledBackup != nil && scheduledBackup.ScheduledFor.Time.Before(now) { // hanlde error when no available RO slaves var roSlaves []*sharded.RedisServer @@ -256,14 +256,17 @@ func (r *ShardedRedisBackupReconciler) reconcileBackupList(ctx context.Context, for _, shard := range shards { // don't schedule if a backup is already running - if runningbackup := instance.Status.FindLastBackup(shard, saasv1alpha1.BackupRunningState); runningbackup != nil { + if runningbackup, _ := instance.Status.FindLastBackup(shard, saasv1alpha1.BackupRunningState); runningbackup != nil { continue } - if lastbackup := instance.Status.FindLastBackup(shard, saasv1alpha1.BackupPendingState); lastbackup != nil { + if lastbackup, pos := instance.Status.FindLastBackup(shard, saasv1alpha1.BackupPendingState); lastbackup != nil { // found a pending backup for this shard if nextRun == lastbackup.ScheduledFor.Time { // already scheduled, do nothing continue + } else { + // already scheduled for a different time, replace with new schedule + instance.Status.DeleteBackup(pos) } } From dc9037358521ff99aa234eb563d71ed787da7eb1 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Wed, 27 Sep 2023 12:01:50 +0200 Subject: [PATCH 38/43] Delete all backup files after the s3 upload --- pkg/redis/backup/s3upload.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/redis/backup/s3upload.go b/pkg/redis/backup/s3upload.go index e02fd391..a3404c1f 100644 --- a/pkg/redis/backup/s3upload.go +++ b/pkg/redis/backup/s3upload.go @@ -60,7 +60,6 @@ func (br *Runner) UploadBackup(ctx context.Context) error { ), // gzip /data/redis-backup---.rdb fmt.Sprintf("gzip %s/%s", path.Dir(br.RedisDBFile), br.BackupFile()), - // TODO: use awscli instead // AWS_ACCESS_KEY_ID=*** AWS_SECRET_ACCESS_KEY=*** s3cmd put /data/redis-backup---.rdb s3:////redis-backup---.rdb fmt.Sprintf("%s=%s %s=%s %s=%s %s s3 cp --only-show-errors %s/%s s3://%s/%s/%s", util.AWSRegionEnvvar, br.AWSRegion, @@ -70,6 +69,7 @@ func (br *Runner) UploadBackup(ctx context.Context) error { path.Dir(br.RedisDBFile), br.BackupFileCompressed(), br.S3Bucket, br.S3Path, br.BackupFileCompressed(), ), + fmt.Sprintf("rm -f %s*", br.BackupFileBaseName()), } for _, command := range commands { From 73e4885ef338835f29891a0e4f71dfdf8d57dcc4 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Wed, 27 Sep 2023 12:02:15 +0200 Subject: [PATCH 39/43] Close transport --- pkg/redis/backup/s3upload.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/redis/backup/s3upload.go b/pkg/redis/backup/s3upload.go index a3404c1f..68f3a699 100644 --- a/pkg/redis/backup/s3upload.go +++ b/pkg/redis/backup/s3upload.go @@ -69,7 +69,7 @@ func (br *Runner) UploadBackup(ctx context.Context) error { path.Dir(br.RedisDBFile), br.BackupFileCompressed(), br.S3Bucket, br.S3Path, br.BackupFileCompressed(), ), - fmt.Sprintf("rm -f %s*", br.BackupFileBaseName()), + fmt.Sprintf("rm -f %s/%s*", path.Dir(br.RedisDBFile), br.BackupFileBaseName()), } for _, command := range commands { @@ -109,6 +109,8 @@ func remoteRun(ctx context.Context, user, addr, port, privateKey, cmd string) (s if err != nil { return "", err } + defer client.Close() + // Create a session. It is one session per command. session, err := client.NewSession() if err != nil { From 736ada72eb8c1c06a7639d9410e94d09cf814fae Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Wed, 27 Sep 2023 13:26:04 +0200 Subject: [PATCH 40/43] Fix test --- controllers/shardedredisbackup_controller_test.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/controllers/shardedredisbackup_controller_test.go b/controllers/shardedredisbackup_controller_test.go index 3dc6b858..494e9a1d 100644 --- a/controllers/shardedredisbackup_controller_test.go +++ b/controllers/shardedredisbackup_controller_test.go @@ -170,18 +170,6 @@ func TestShardedRedisBackupReconciler_reconcileBackupList(t *testing.T) { Message: "backup scheduled", State: saasv1alpha1.BackupPendingState, }, - { - Shard: "shard02", - ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), - Message: "backup scheduled", - State: saasv1alpha1.BackupPendingState, - }, - { - Shard: "shard01", - ScheduledFor: metav1.NewTime(util.MustParseRFC3339("2023-09-01T00:01:00Z")), - Message: "backup scheduled", - State: saasv1alpha1.BackupPendingState, - }, }, }, wantErr: false, From 7b741a15a0d0a8345fed90e70857bd357a361803 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Wed, 27 Sep 2023 16:18:23 +0200 Subject: [PATCH 41/43] Don't use verbose flag to run test in the CI, the output is too verbose and hard to read --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index bcea6f21..8abe2cf2 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.20.0-alpha.10 +VERSION ?= 0.20.0-alpha.18 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") @@ -110,9 +110,9 @@ TEST_PKG = ./api/... ./controllers/... ./pkg/... KUBEBUILDER_ASSETS = "$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" test: manifests generate fmt vet envtest assets ginkgo ## Run tests. - KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) $(GINKGO) -p -v -r $(TEST_PKG) -coverprofile cover.out + KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) $(GINKGO) -p -r $(TEST_PKG) -coverprofile cover.out -test-sequential: manifests generate fmt vet envtest assets ginkgo ## Run tests. +test-debug: manifests generate fmt vet envtest assets ginkgo ## Run tests. KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) $(GINKGO) -v -r $(TEST_PKG) -coverprofile cover.out test-e2e: export KUBECONFIG = $(PWD)/kubeconfig From 794d185ed840a7ca3bea24f36b6ac61826535c9b Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Wed, 27 Sep 2023 16:50:21 +0200 Subject: [PATCH 42/43] Remove commented lines in backup e2e test --- test/e2e/shardedredisbackup_suite_test.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test/e2e/shardedredisbackup_suite_test.go b/test/e2e/shardedredisbackup_suite_test.go index a547509b..9d8025fc 100644 --- a/test/e2e/shardedredisbackup_suite_test.go +++ b/test/e2e/shardedredisbackup_suite_test.go @@ -67,10 +67,6 @@ var _ = Describe("shardedredisbackup e2e suite", func() { }, }, }, - // { - // ObjectMeta: metav1.ObjectMeta{Name: "rs1", Namespace: ns}, - // Spec: saasv1alpha1.RedisShardSpec{MasterIndex: pointer.Int32(2), SlaveCount: pointer.Int32(2)}, - // }, } for i, shard := range shards { @@ -106,11 +102,6 @@ var _ = Describe("shardedredisbackup e2e suite", func() { "redis://" + shards[0].Status.ShardNodes.GetHostPortByPodIndex(1), "redis://" + shards[0].Status.ShardNodes.GetHostPortByPodIndex(2), }, - // shards[1].GetName(): { - // "redis://" + shards[1].Status.ShardNodes.GetHostPortByPodIndex(0), - // "redis://" + shards[1].Status.ShardNodes.GetHostPortByPodIndex(1), - // "redis://" + shards[1].Status.ShardNodes.GetHostPortByPodIndex(2), - // }, }, }, }, From f1bdcf39c1f752500f4e8ff99c68b9c9a785aff3 Mon Sep 17 00:00:00 2001 From: Roi Vazquez Date: Wed, 27 Sep 2023 17:35:36 +0200 Subject: [PATCH 43/43] Fix failure metrics --- pkg/redis/backup/manager.go | 2 ++ pkg/redis/backup/metrics.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/redis/backup/manager.go b/pkg/redis/backup/manager.go index b53456fc..db5308e6 100644 --- a/pkg/redis/backup/manager.go +++ b/pkg/redis/backup/manager.go @@ -119,6 +119,7 @@ func (br *Runner) Start(parentCtx context.Context, l logr.Logger) error { br.status.Finished = true br.status.Error = err br.eventsCh <- event.GenericEvent{Object: br.Instance} + br.publishMetrics() return case err := <-errCh: @@ -126,6 +127,7 @@ func (br *Runner) Start(parentCtx context.Context, l logr.Logger) error { br.status.Finished = true br.status.Error = err br.eventsCh <- event.GenericEvent{Object: br.Instance} + br.publishMetrics() return case <-done: diff --git a/pkg/redis/backup/metrics.go b/pkg/redis/backup/metrics.go index 22cdb17f..99b81a5e 100644 --- a/pkg/redis/backup/metrics.go +++ b/pkg/redis/backup/metrics.go @@ -62,6 +62,6 @@ func (r *Runner) publishMetrics() { } else { backupSize.With(prometheus.Labels{"shard": r.ShardName}).Set(float64(r.status.BackupSize)) backupDuration.With(prometheus.Labels{"shard": r.ShardName}).Set(math.Round(r.status.FinishedAt.Sub(r.Timestamp).Seconds())) - backupSuccessCount.With(prometheus.Labels{"shard": r.ShardName}).Add(1) + backupSuccessCount.With(prometheus.Labels{"shard": r.ShardName}).Inc() } }