Skip to content

Commit

Permalink
Merge pull request #269 from 3scale-ops/redis-backups
Browse files Browse the repository at this point in the history
Redis backups
  • Loading branch information
3scale-robot authored Sep 28, 2023
2 parents c397904 + f1bdcf3 commit e1c07ad
Show file tree
Hide file tree
Showing 82 changed files with 5,188 additions and 2,004 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,6 @@ kubeconfig
.vscode/configurationCache.log
.vscode/dryrun.log
.vscode/targets.log

# helm charts
**/charts
6 changes: 6 additions & 0 deletions .gitleaks.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
[allowlist]
description = "global allow lists"
paths = [
'''test/assets/redis-with-ssh/test-ssh-key''',
'''config/test/redis-backups/kustomization.yaml''',
]
19 changes: 14 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# To re-generate a bundle for another specific version without changing the standard setup, you can:
# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
# - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
VERSION ?= 0.20.0-alpha.10
VERSION ?= 0.20.0-alpha.18

# CHANNELS define the bundle channels used in the bundle.
# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
Expand Down Expand Up @@ -110,13 +110,13 @@ TEST_PKG = ./api/... ./controllers/... ./pkg/...
KUBEBUILDER_ASSETS = "$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)"

test: manifests generate fmt vet envtest assets ginkgo ## Run tests.
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) $(GINKGO) -p -v -r $(TEST_PKG) -coverprofile cover.out
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) $(GINKGO) -p -r $(TEST_PKG) -coverprofile cover.out

test-sequential: manifests generate fmt vet envtest assets ginkgo ## Run tests.
test-debug: manifests generate fmt vet envtest assets ginkgo ## Run tests.
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) $(GINKGO) -v -r $(TEST_PKG) -coverprofile cover.out

test-e2e: export KUBECONFIG = $(PWD)/kubeconfig
test-e2e: manifests ginkgo kind-create kind-deploy ## Runs e2e tests
test-e2e: manifests ginkgo kind-create kind-deploy kind-deploy-backup-assets ## Runs e2e tests
$(GINKGO) -p -r ./test/e2e
$(MAKE) kind-delete

Expand Down Expand Up @@ -277,6 +277,15 @@ kind-undeploy: export KUBECONFIG = $(PWD)/kubeconfig
kind-undeploy: ## Undeploy controller from the Kind K8s cluster
$(KUSTOMIZE) build config/test | kubectl delete -f -

kind-deploy-backup-assets: export KUBECONFIG = $(PWD)/kubeconfig
kind-deploy-backup-assets: kind-load-redis-with-ssh
$(KUSTOMIZE) build config/test/redis-backups --load-restrictor LoadRestrictionsNone --enable-helm | kubectl apply -f -

REDIS_WITH_SSH_IMG = redis-with-ssh:4.0.11-alpine
kind-load-redis-with-ssh:
docker build -t $(REDIS_WITH_SSH_IMG) test/assets/redis-with-ssh
$(KIND) load docker-image $(REDIS_WITH_SSH_IMG)

##@ Build Dependencies

## Location to install dependencies to
Expand All @@ -294,7 +303,7 @@ KIND ?= $(LOCALBIN)/kind
GOBINDATA ?= $(LOCALBIN)/go-bindata

## Tool Versions
KUSTOMIZE_VERSION ?= v3.8.7
KUSTOMIZE_VERSION ?= v5.1.1
CONTROLLER_TOOLS_VERSION ?= v0.11.0
GINKGO_VERSION ?= v2.9.1
CRD_REFDOCS_VERSION ?= v0.0.8
Expand Down
9 changes: 9 additions & 0 deletions PROJECT
Original file line number Diff line number Diff line change
Expand Up @@ -106,4 +106,13 @@ resources:
kind: TwemproxyConfig
path: github.com/3scale/saas-operator/api/v1alpha1
version: v1alpha1
- api:
crdVersion: v1
namespaced: true
controller: true
domain: 3scale.net
group: saas
kind: ShardedRedisBackup
path: github.com/3scale/saas-operator/api/v1alpha1
version: v1alpha1
version: "3"
12 changes: 9 additions & 3 deletions api/v1alpha1/redisshard_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,9 @@ var (
Tag: pointer.String("4.0.11-alpine"),
PullPolicy: (*corev1.PullPolicy)(pointer.String(string(corev1.PullIfNotPresent))),
}
redisShardDefaultMasterIndex int32 = 0
RedisShardDefaultReplicas int32 = 3
redisShardDefaultMasterIndex int32 = 0
redisShardDefaultCommand string = "redis-server /redis/redis.conf"
RedisShardDefaultReplicas int32 = 3
)

// RedisShardSpec defines the desired state of RedisShard
Expand All @@ -51,14 +52,19 @@ type RedisShardSpec struct {
// +operator-sdk:csv:customresourcedefinitions:type=spec
// +optional
SlaveCount *int32 `json:"slaveCount,omitempty"`
// Command overrides the redis container command
// +operator-sdk:csv:customresourcedefinitions:type=spec
// +optional
Command *string `json:"command,omitempty"`
}

// Default implements defaulting for RedisShardSpec
func (spec *RedisShardSpec) Default() {

spec.Image = InitializeImageSpec(spec.Image, redisShardDefaultImage)
spec.MasterIndex = intOrDefault(spec.MasterIndex, &redisShardDefaultMasterIndex)
spec.SlaveCount = intOrDefault(spec.SlaveCount, pointer.Int32(RedisShardDefaultReplicas-1))
spec.SlaveCount = intOrDefault(spec.SlaveCount, util.Pointer(RedisShardDefaultReplicas-1))
spec.Command = stringOrDefault(spec.Command, &redisShardDefaultCommand)
}

type RedisShardNodes struct {
Expand Down
41 changes: 41 additions & 0 deletions api/v1alpha1/sentinel_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,13 @@ limitations under the License.
package v1alpha1

import (
"context"
"sort"
"time"

"github.com/3scale/saas-operator/pkg/redis/client"
redis "github.com/3scale/saas-operator/pkg/redis/server"
"github.com/3scale/saas-operator/pkg/redis/sharded"
"github.com/3scale/saas-operator/pkg/util"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
Expand Down Expand Up @@ -180,6 +184,43 @@ type SentinelStatus struct {
MonitoredShards MonitoredShards `json:"monitoredShards,omitempty"`
}

// ShardedCluster returns a *sharded.Cluster struct from the information reported by the sentinel status instead
// of directly contacting sentinel/redis to gather the state of the cluster. This avoids calls to sentinel/redis
// but is less robust as it depends entirely on the Sentinel controller working properly and without delays.
// As of now, this is used in the SharededRedisBackup controller but not in the TwemproxyConfig controller.
func (ss *SentinelStatus) ShardedCluster(ctx context.Context, pool *redis.ServerPool) (*sharded.Cluster, error) {

// have a list of sentinels but must provide a map
// TODO: at some point change the SentinelStatus.Sentinels to also have a map and avoid this
msentinel := make(map[string]string, len(ss.Sentinels))
for _, s := range ss.Sentinels {
msentinel[s] = "redis://" + s
}

shards := make([]*sharded.Shard, 0, len(ss.MonitoredShards))
// generate slice of shards from status
for _, s := range ss.MonitoredShards {
servers := make([]*sharded.RedisServer, 0, len(s.Servers))
for _, rsd := range s.Servers {
srv, err := pool.GetServer("redis://"+rsd.Address, nil)
if err != nil {
return nil, err
}
servers = append(servers, sharded.NewRedisServerFromParams(srv, rsd.Role, rsd.Config))
}
sort.Slice(servers, func(i, j int) bool {
return servers[i].ID() < servers[j].ID()
})
shards = append(shards, sharded.NewShardFromServers(s.Name, pool, servers...))
}

cluster, err := sharded.NewShardedCluster(ctx, pool, msentinel, shards...)
if err != nil {
return nil, err
}
return cluster, nil
}

type MonitoredShards []MonitoredShard

// MonitoredShards implements sort.Interface based on the Name field.
Expand Down
110 changes: 110 additions & 0 deletions api/v1alpha1/sentinel_types_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
/*
Copyright 2021.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
"context"
"testing"

"github.com/3scale/saas-operator/pkg/redis/client"
redis "github.com/3scale/saas-operator/pkg/redis/server"
"github.com/3scale/saas-operator/pkg/redis/sharded"
"github.com/3scale/saas-operator/pkg/util"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
)

func TestSentinelStatus_ShardedCluster(t *testing.T) {
type fields struct {
Sentinels []string
MonitoredShards MonitoredShards
}
type args struct {
ctx context.Context
pool *redis.ServerPool
}
tests := []struct {
name string
fields fields
args args
want *sharded.Cluster
wantErr bool
}{
{
name: "Generates a sharded.Cluster resource from the sentinel status",
fields: fields{
Sentinels: []string{"127.0.0.1:26379"},
MonitoredShards: []MonitoredShard{
{Name: "shard01",
Servers: map[string]RedisServerDetails{
"srv1": {Role: client.Master, Address: "127.0.0.1:1000", Config: map[string]string{"save": ""}},
"srv2": {Role: client.Slave, Address: "127.0.0.1:2000", Config: map[string]string{"slave-read-only": "yes"}},
}},
{Name: "shard02",
Servers: map[string]RedisServerDetails{
"srv3": {Role: client.Master, Address: "127.0.0.1:3000", Config: map[string]string{}},
"srv4": {Role: client.Slave, Address: "127.0.0.1:4000", Config: map[string]string{}},
}},
},
},
args: args{
ctx: context.TODO(),
pool: redis.NewServerPool(
redis.MustNewServer("redis://127.0.0.1:1000", util.Pointer("srv1")),
redis.MustNewServer("redis://127.0.0.1:2000", util.Pointer("srv2")),
redis.MustNewServer("redis://127.0.0.1:3000", util.Pointer("srv3")),
redis.MustNewServer("redis://127.0.0.1:4000", util.Pointer("srv4")),
redis.MustNewServer("redis://127.0.0.1:26379", util.Pointer("sentinel")),
),
},
want: &sharded.Cluster{
Shards: []*sharded.Shard{
{Name: "shard01",
Servers: []*sharded.RedisServer{
sharded.NewRedisServerFromParams(redis.MustNewServer("redis://127.0.0.1:1000", util.Pointer("srv1")), client.Master, map[string]string{"save": ""}),
sharded.NewRedisServerFromParams(redis.MustNewServer("redis://127.0.0.1:2000", util.Pointer("srv2")), client.Slave, map[string]string{"slave-read-only": "yes"}),
}},
{Name: "shard02",
Servers: []*sharded.RedisServer{
sharded.NewRedisServerFromParams(redis.MustNewServer("redis://127.0.0.1:3000", util.Pointer("srv3")), client.Master, map[string]string{}),
sharded.NewRedisServerFromParams(redis.MustNewServer("redis://127.0.0.1:4000", util.Pointer("srv4")), client.Slave, map[string]string{}),
}},
},
Sentinels: []*sharded.SentinelServer{
sharded.NewSentinelServerFromParams(redis.MustNewServer("redis://127.0.0.1:26379", util.Pointer("sentinel"))),
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ss := &SentinelStatus{
Sentinels: tt.fields.Sentinels,
MonitoredShards: tt.fields.MonitoredShards,
}
got, err := ss.ShardedCluster(tt.args.ctx, tt.args.pool)
if (err != nil) != tt.wantErr {
t.Errorf("SentinelStatus.ShardedCluster() error = %v, wantErr %v", err, tt.wantErr)
return
}
if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(sharded.Cluster{}, sharded.Shard{}, redis.Server{})); len(diff) > 0 {
t.Errorf("SentinelStatus.ShardedCluster() = diff %s", diff)
}
})
}
}
Loading

0 comments on commit e1c07ad

Please sign in to comment.