From 5ba6913d5f5b535aed2b954a0723a4f4b319850f Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Mon, 25 Nov 2024 20:16:37 +0300 Subject: [PATCH 1/2] Add disco mode support --- .golangci.yml | 1 + .version | 2 +- changelog.md | 3 ++ cmd/migrate/build.sh | 2 - cmd/migrate/go.mod | 2 +- cmd/migrate/go.sum | 4 +- conf/config.yml | 9 ++-- deploy/compose/docker-compose.yml | 40 ++++++++++++++ deploy/k8s/service.yaml | 25 +++++++++ deploy/k8s/statefulset-3-node.yaml | 87 ++++++++++++++++++++++++++++++ go.mod | 3 +- go.sum | 6 ++- service/rqlite/main.go | 83 +++++++++++++++++++++++++++- service/startup/service.go | 11 ++++ 14 files changed, 265 insertions(+), 13 deletions(-) delete mode 100755 cmd/migrate/build.sh create mode 100644 deploy/compose/docker-compose.yml create mode 100644 deploy/k8s/service.yaml create mode 100644 deploy/k8s/statefulset-3-node.yaml diff --git a/.golangci.yml b/.golangci.yml index 7f2ac31..bf47957 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -34,6 +34,7 @@ linters: - perfsprint #мнимая производительность в угоду читаемости - gomoddirectives #требует rqlite - tagliatelle + - gomnd linters-settings: funlen: lines: 80 diff --git a/.version b/.version index 94ff29c..ef538c2 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -3.1.1 +3.1.2 diff --git a/changelog.md b/changelog.md index 50ac809..f3ab7d8 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,6 @@ +## v3.1.2 +* возвращена поддержка rqlite.DiscoMode `dns` и `dns-srv` +* добавлена поддержка работы в `kubernetes` c autodiscovery ## v3.1.1 * инструмент миграции перенесен внутрь контейнера ## v3.1.0 diff --git a/cmd/migrate/build.sh b/cmd/migrate/build.sh deleted file mode 100755 index 1936d52..0000000 --- a/cmd/migrate/build.sh +++ /dev/null @@ -1,2 +0,0 @@ -go install github.com/mitchellh/gox@latest -gox -os="linux windows darwin" -arch="amd64" -output="build/{{.OS}}_{{.Arch}}/{{.Dir}}" diff --git a/cmd/migrate/go.mod b/cmd/migrate/go.mod index 1fa14ea..7b4f5e9 100644 --- a/cmd/migrate/go.mod +++ b/cmd/migrate/go.mod @@ -9,7 +9,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.22 github.com/pkg/errors v0.9.1 github.com/pressly/goose/v3 v3.22.1 - github.com/txix-open/isp-kit v1.39.1 + github.com/txix-open/isp-kit v1.39.2 isp-config-service v0.0.0-00010101000000-000000000000 ) diff --git a/cmd/migrate/go.sum b/cmd/migrate/go.sum index 79eb56c..0198f44 100644 --- a/cmd/migrate/go.sum +++ b/cmd/migrate/go.sum @@ -142,8 +142,8 @@ github.com/txix-open/bellows v1.2.0 h1:CXv8nQaZtB/micraeRilYyj/gtfv+bqBgP5aPYQgj github.com/txix-open/bellows v1.2.0/go.mod h1:qbKCy+RTgD30Qpw1fyb3y3jp5Y9mGhLLxgae1l0W92o= github.com/txix-open/etp/v4 v4.0.1 h1:5VSYgGsvnMz0tvUHsjVWFETI0qWBTCF1O3ARCHaCIgA= github.com/txix-open/etp/v4 v4.0.1/go.mod h1:FIu7TUDdwcgmtmF6tFTV1bGA2fRY/iVaPQdsGzvqYc8= -github.com/txix-open/isp-kit v1.39.1 h1:10zEILNQSPY7naiuaQkIKNS2/fnWxuE9bz+dTiAptXw= -github.com/txix-open/isp-kit v1.39.1/go.mod h1:xB6mkoOewZBI1ND8kjU5TR6kvLu3VQI3Rdkh6g6FE4I= +github.com/txix-open/isp-kit v1.39.2 h1:2L2LFwi+sNop9NeLDQ4scIy8LuohzXh4EBejHApAeDM= +github.com/txix-open/isp-kit v1.39.2/go.mod h1:xB6mkoOewZBI1ND8kjU5TR6kvLu3VQI3Rdkh6g6FE4I= github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 h1:Z0Sqf+U+Sc6hfIXtRCIZlbJprkCjwMFbS4ZbEPQ44To= github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99/go.mod h1:pA0NYmhL0uEkkxj55S+S7MZU4e2QIt46x9XSrzrWc3c= go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 h1:4BZHA+B1wXEQoGNHxW8mURaLhcdGwvRnmhGbm+odRbc= diff --git a/conf/config.yml b/conf/config.yml index 6e22c42..aba41c4 100644 --- a/conf/config.yml +++ b/conf/config.yml @@ -1,5 +1,5 @@ configServiceAddress: - ip: 127.0.0.1 + ip: localhost port: 9001 grpcOuterAddress: ip: isp-config-service @@ -21,14 +21,17 @@ credentials: - username: default password: 38kPk0upqI5j perms: ["all"] + - username: monitor + password: monitor + perms: ["ready"] + internalClientCredential: default rqlite: DataPath: "data" NodeID: 1 BootstrapExpect: 1 - RaftAddr: "127.0.0.1:9008" - JoinAddrs: "127.0.0.1:9008" + JoinAddrs: "localhost:4002" FKConstraints: true AutoVacInterval: 12h JoinAs: default diff --git a/deploy/compose/docker-compose.yml b/deploy/compose/docker-compose.yml new file mode 100644 index 0000000..05e8e69 --- /dev/null +++ b/deploy/compose/docker-compose.yml @@ -0,0 +1,40 @@ +services: + isp-config-service-1: + image: "isp-config-service:latest" + container_name: "isp-config-service-1" + hostname: "isp-config-service-1" + restart: unless-stopped + environment: + grpcOuterAddress.ip: isp-config-service-1 + rqlite.NodeID: 1 + rqlite.BootstrapExpect: 3 + rqlite.RaftAddr: "isp-config-service-1:4002" + rqlite.JoinAddrs: "isp-config-service-1:4002,isp-config-service-2:4002,isp-config-service-3:4002" + volumes: + - "./isp-config-service-1/data:/app/data" + isp-config-service-2: + image: "isp-config-service:latest" + container_name: "isp-config-service-2" + hostname: "isp-config-service-2" + restart: unless-stopped + environment: + grpcOuterAddress.ip: isp-config-service-2 + rqlite.NodeID: 2 + rqlite.BootstrapExpect: 3 + rqlite.RaftAddr: "isp-config-service-2:4002" + rqlite.JoinAddrs: "isp-config-service-1:4002,isp-config-service-2:4002,isp-config-service-3:4002" + volumes: + - "./isp-config-service-2/data:/app/data" + isp-config-service-3: + image: "isp-config-service:latest" + container_name: "isp-config-service-3" + hostname: "isp-config-service-3" + restart: unless-stopped + environment: + grpcOuterAddress.ip: isp-config-service-3 + rqlite.NodeID: 3 + rqlite.BootstrapExpect: 3 + rqlite.RaftAddr: "isp-config-service-3:4002" + rqlite.JoinAddrs: "isp-config-service-1:4002,isp-config-service-2:4002,isp-config-service-3:4002" + volumes: + - "./isp-config-service-3/data:/app/data" \ No newline at end of file diff --git a/deploy/k8s/service.yaml b/deploy/k8s/service.yaml new file mode 100644 index 0000000..fc1ca42 --- /dev/null +++ b/deploy/k8s/service.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Service +metadata: + name: isp-config-service-internal +spec: + clusterIP: None + publishNotReadyAddresses: True + selector: + app: isp-config-service + ports: + - protocol: TCP + port: 4001 + targetPort: 4001 +--- +apiVersion: v1 +kind: Service +metadata: + name: isp-config-service +spec: + selector: + app: isp-config-service + ports: + - protocol: TCP + port: 4001 + targetPort: 4001 \ No newline at end of file diff --git a/deploy/k8s/statefulset-3-node.yaml b/deploy/k8s/statefulset-3-node.yaml new file mode 100644 index 0000000..dd21575 --- /dev/null +++ b/deploy/k8s/statefulset-3-node.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: isp-config-service +spec: + selector: + matchLabels: + app: isp-config-service # has to match .spec.template.metadata.labels + serviceName: isp-config-service-internal + replicas: 3 + podManagementPolicy: "Parallel" + template: + metadata: + labels: + app: isp-config-service # has to match .spec.selector.matchLabels + spec: + terminationGracePeriodSeconds: 10 + imagePullSecrets: + - name: regcred + containers: + - name: isp-config-service + image: isp-config-service:latest + imagePullPolicy: Always + env: + - name: rqlite.NodeID + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: rqlite.BootstrapExpect + value: "3" + - name: rqlite.HttpAddr + value: "0.0.0.0:4001" + - name: rqlite.HttpAdv + value: "$HOSTNAME:4001" + - name: rqlite.RaftAddr + value: "0.0.0.0:4002" + - name: rqlite.RaftAdv + value: "$HOSTNAME:4002" + - name: rqlite.JoinAddrs + value: "" + - name: rqlite.DiscoMode + value: "dns" + - name: rqlite.DiscoConfig + value: "{\"name\":\"isp-config-service-internal\"}" + - name: rqlite.JoinInterval + value: "1s" + - name: rqlite.JoinAttempts + value: "120" + - name: rqlite.RaftClusterRemoveOnShutdown + value: "true" + ports: + - containerPort: 4001 + name: rqlite + readinessProbe: + httpGet: + scheme: HTTP + path: /readyz + port: rqlite + httpHeaders: + - name: Authorization + value: Basic bW9uaXRvcjptb25pdG9y + periodSeconds: 5 + timeoutSeconds: 2 + initialDelaySeconds: 2 + livenessProbe: + httpGet: + scheme: HTTP + path: /readyz?noleader + port: rqlite + httpHeaders: + - name: Authorization + value: Basic bW9uaXRvcjptb25pdG9y + initialDelaySeconds: 5 + timeoutSeconds: 2 + failureThreshold: 3 + volumeMounts: + - name: isp-config-service-data + mountPath: /app/data + volumeClaimTemplates: + - metadata: + name: isp-config-service-data + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "local-path" + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/go.mod b/go.mod index 1d1f73f..db6fe3c 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,12 @@ require ( github.com/pressly/goose/v3 v3.22.1 github.com/prometheus/client_golang v1.20.5 github.com/rqlite/gorqlite v0.0.0-20241013203532-4385768ae85d + github.com/rqlite/rqlite-disco-clients v0.0.0-20231230135307-118e35426347 github.com/rqlite/rqlite/v8 v8.32.5 github.com/stretchr/testify v1.9.0 github.com/tidwall/gjson v1.18.0 github.com/txix-open/etp/v4 v4.0.1 - github.com/txix-open/isp-kit v1.39.1 + github.com/txix-open/isp-kit v1.39.2 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/sync v0.8.0 google.golang.org/grpc v1.67.1 diff --git a/go.sum b/go.sum index df76a35..0da77f1 100644 --- a/go.sum +++ b/go.sum @@ -201,6 +201,8 @@ github.com/rqlite/gorqlite v0.0.0-20241013203532-4385768ae85d h1:c88ius/WcN19inn github.com/rqlite/gorqlite v0.0.0-20241013203532-4385768ae85d/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg= github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48 h1:NZ62M+kT0JqhyFUMc8I4SMmfmD4NGJxhb2ePJQXjryc= github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48/go.mod h1:CRnsxgy5G8fAf5J+AM0yrsSdxXHKkIYOaq2sm+Q4DYc= +github.com/rqlite/rqlite-disco-clients v0.0.0-20231230135307-118e35426347 h1:01eOG4AXjYsrsLRITgBFHaDPXys1NfeY2idx25ZJFF8= +github.com/rqlite/rqlite-disco-clients v0.0.0-20231230135307-118e35426347/go.mod h1:pym85nj6JnCI7rM9RxTZ4cubkTQyyg7uLwVydso9B80= github.com/rqlite/rqlite/v8 v8.32.5 h1:wEzDX2HrGWMvFkg2fcCmi+rb1wQ1m5yEefUQfOMvMSY= github.com/rqlite/rqlite/v8 v8.32.5/go.mod h1:a1irqNWEZ+pWuWpgekbScCXp85QtNTJLarN3bVPjn9M= github.com/rqlite/sql v0.0.0-20241029220113-152a320b02f7 h1:Mnz6yd4FWtiD6bbH9WHFFHfrOM2OYTUTmwrsckRc4W8= @@ -237,8 +239,8 @@ github.com/txix-open/etp/v4 v4.0.1 h1:5VSYgGsvnMz0tvUHsjVWFETI0qWBTCF1O3ARCHaCIg github.com/txix-open/etp/v4 v4.0.1/go.mod h1:FIu7TUDdwcgmtmF6tFTV1bGA2fRY/iVaPQdsGzvqYc8= github.com/txix-open/grmq v1.6.0 h1:V4QRb0sI2CMfdMaMsFGvGvUIDGad5vr3khNU7A3LlP8= github.com/txix-open/grmq v1.6.0/go.mod h1:s+NNNv42+32yLpjvl2tpqtyBbMB9e7kKFee3lfa/dyo= -github.com/txix-open/isp-kit v1.39.1 h1:10zEILNQSPY7naiuaQkIKNS2/fnWxuE9bz+dTiAptXw= -github.com/txix-open/isp-kit v1.39.1/go.mod h1:xB6mkoOewZBI1ND8kjU5TR6kvLu3VQI3Rdkh6g6FE4I= +github.com/txix-open/isp-kit v1.39.2 h1:2L2LFwi+sNop9NeLDQ4scIy8LuohzXh4EBejHApAeDM= +github.com/txix-open/isp-kit v1.39.2/go.mod h1:xB6mkoOewZBI1ND8kjU5TR6kvLu3VQI3Rdkh6g6FE4I= github.com/txix-open/jsonschema v1.2.0 h1:B8TrdSsPhDvYv67oi/LVqf7/+PuqFlMsNwUzGeHfl1s= github.com/txix-open/jsonschema v1.2.0/go.mod h1:l8YDZ1nvJrw6uxWowSVOxCV/ebiMJyapffW87ZEqH00= github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 h1:Z0Sqf+U+Sc6hfIXtRCIZlbJprkCjwMFbS4ZbEPQ44To= diff --git a/service/rqlite/main.go b/service/rqlite/main.go index 0e7a686..d9e5b24 100644 --- a/service/rqlite/main.go +++ b/service/rqlite/main.go @@ -5,11 +5,16 @@ package rqlite import ( "context" "crypto/tls" + "encoding/json" "fmt" "github.com/pkg/errors" + "github.com/rqlite/rqlite-disco-clients/dns" + "github.com/rqlite/rqlite-disco-clients/dnssrv" + "github.com/txix-open/isp-kit/config" "log" "net" "os" + "os/exec" "path/filepath" "runtime" "strings" @@ -55,11 +60,17 @@ func main(ctx context.Context, r *Rqlite) error { if err != nil { log.Fatalf("failed to parse command-line flags: %s", err.Error()) } + err = evaluateAdvAddresses(r.cfg) + if err != nil { + return errors.WithMessage(err, "check adv ports") + } localCfg := localConfig{Rqlite: cfg} err = r.cfg.Read(&localCfg) if err != nil { return errors.WithMessage(err, "read local config") } + dumpRqliteConfig, _ := json.MarshalIndent(cfg, "", " ") + log.Printf("rqlite config:\n %s \n", string(dumpRqliteConfig)) err = cfg.Validate() if err != nil { return errors.WithMessage(err, "validate rqlite configuration") @@ -210,6 +221,37 @@ func main(ctx context.Context, r *Rqlite) error { return nil } +func evaluateAdvAddresses(cfg *config.Config) error { + httpAdv := cfg.Optional().String("rqlite.HTTPAdv", "") + shouldEvalHttp := strings.Contains(httpAdv, "$HOSTNAME") + raftAdv := cfg.Optional().String("rqlite.RaftAdv", "") + shouldEvalRaft := strings.Contains(raftAdv, "$HOSTNAME") + if !shouldEvalHttp && !shouldEvalRaft { + return nil + } + + var ( + host string + ) + data, err := exec.Command("hostname", "-f").Output() + if err != nil { + host, err = os.Hostname() + if err != nil { + return errors.WithMessage(err, "rqlite.HTTPAdv is not set, couldn't resolve local hostname") + } + } else { + host = strings.TrimSpace(string(data)) + } + + if shouldEvalHttp { + cfg.Set("rqlite.HTTPAdv", strings.ReplaceAll(httpAdv, "$HOSTNAME", host)) + } + if shouldEvalRaft { + cfg.Set("rqlite.RaftAdv", strings.ReplaceAll(raftAdv, "$HOSTNAME", host)) + } + return nil +} + func createExtensionsStore(cfg *Config) (*extensions.Store, error) { str, err := extensions.NewStore(filepath.Join(cfg.DataPath, "extensions")) if err != nil { @@ -421,7 +463,46 @@ func createCluster(ctx context.Context, cfg *Config, hasPeers bool, client *clus // existing Raft state. return nil } - return nil + + // DNS-based discovery requested. It's OK to proceed with this even if this node + // is already part of a cluster. Re-joining and re-notifying other nodes will be + // ignored when the node is already part of the cluster. + log.Printf("discovery mode: %s", cfg.DiscoMode) + switch cfg.DiscoMode { + case DiscoModeDNS, DiscoModeDNSSRV: + rc := cfg.DiscoConfigReader() + defer func() { + if rc != nil { + rc.Close() + } + }() + + var provider interface { + cluster.AddressProvider + httpd.StatusReporter + } + if cfg.DiscoMode == DiscoModeDNS { + dnsCfg, err := dns.NewConfigFromReader(rc) + if err != nil { + return fmt.Errorf("error reading DNS configuration: %s", err.Error()) + } + provider = dns.NewWithPort(dnsCfg, cfg.RaftPort()) + + } else { + dnssrvCfg, err := dnssrv.NewConfigFromReader(rc) + if err != nil { + return fmt.Errorf("error reading DNS configuration: %s", err.Error()) + } + provider = dnssrv.New(dnssrvCfg) + } + + bs := cluster.NewBootstrapper(provider, client) + bs.SetCredentials(cluster.CredentialsFor(credStr, cfg.JoinAs)) + httpServ.RegisterStatus("disco", provider) + return bs.Boot(ctx, str.ID(), cfg.RaftAdv, clusterSuf, bootDoneFn, cfg.BootstrapExpectTimeout) + default: + return fmt.Errorf("invalid disco mode %s", cfg.DiscoMode) + } } func networkCheckJoinAddrs(joinAddrs []string) error { diff --git a/service/startup/service.go b/service/startup/service.go index d75361b..ba42fd7 100644 --- a/service/startup/service.go +++ b/service/startup/service.go @@ -77,6 +77,17 @@ func New(boot *bootstrap.Bootstrap) (*Service, error) { // nolint:funlen func (s *Service) Run(ctx context.Context) error { + if s.boot.App.Config().Optional().String("KUBERNETES_SERVICE_HOST", "") != "" { + s.logger.Info( + ctx, + "run in kubernetes, sleep extra 5s, reason described here: https://github.com/rqlite/rqlite/blob/master/docker-entrypoint.sh#L96", + ) + select { + case <-ctx.Done(): + case <-time.After(5 * time.Second): //nolint:mnd + } + } + go func() { s.logger.Debug(ctx, "running embedded rqlite...") err := s.rqlite.Run(ctx) From 005510e0a5047924bd7dea1af139f8b6e7165fff Mon Sep 17 00:00:00 2001 From: yankomissarov Date: Tue, 26 Nov 2024 19:04:50 +0300 Subject: [PATCH 2/2] Return default addresses --- conf/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/config.yml b/conf/config.yml index aba41c4..549ccec 100644 --- a/conf/config.yml +++ b/conf/config.yml @@ -31,7 +31,8 @@ rqlite: DataPath: "data" NodeID: 1 BootstrapExpect: 1 - JoinAddrs: "localhost:4002" + RaftAddr: "127.0.0.1:9008" + JoinAddrs: "127.0.0.1:9008" FKConstraints: true AutoVacInterval: 12h JoinAs: default