Merge branch 'v3' into 'master'

V3

See merge request isp/isp-config-service!4

.golangci.yml

@@ -34,6 +34,7 @@ linters:
     - perfsprint #мнимая производительность в угоду читаемости
     - gomoddirectives #требует rqlite
     - tagliatelle
+    - gomnd
 linters-settings:
   funlen:
     lines: 80

.version

@@ -1 +1 @@
-3.1.1
+3.1.2

changelog.md

@@ -1,3 +1,6 @@
+## v3.1.2
+* возвращена поддержка rqlite.DiscoMode `dns` и `dns-srv`
+* добавлена поддержка работы в `kubernetes` c autodiscovery
 ## v3.1.1
 * инструмент миграции перенесен внутрь контейнера
 ## v3.1.0

cmd/migrate/go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/pkg/errors v0.9.1
 	github.com/pressly/goose/v3 v3.22.1
-	github.com/txix-open/isp-kit v1.39.1
+	github.com/txix-open/isp-kit v1.39.2
 	isp-config-service v0.0.0-00010101000000-000000000000
 )
 

cmd/migrate/go.sum

@@ -142,8 +142,8 @@ github.com/txix-open/bellows v1.2.0 h1:CXv8nQaZtB/micraeRilYyj/gtfv+bqBgP5aPYQgj
 github.com/txix-open/bellows v1.2.0/go.mod h1:qbKCy+RTgD30Qpw1fyb3y3jp5Y9mGhLLxgae1l0W92o=
 github.com/txix-open/etp/v4 v4.0.1 h1:5VSYgGsvnMz0tvUHsjVWFETI0qWBTCF1O3ARCHaCIgA=
 github.com/txix-open/etp/v4 v4.0.1/go.mod h1:FIu7TUDdwcgmtmF6tFTV1bGA2fRY/iVaPQdsGzvqYc8=
-github.com/txix-open/isp-kit v1.39.1 h1:10zEILNQSPY7naiuaQkIKNS2/fnWxuE9bz+dTiAptXw=
-github.com/txix-open/isp-kit v1.39.1/go.mod h1:xB6mkoOewZBI1ND8kjU5TR6kvLu3VQI3Rdkh6g6FE4I=
+github.com/txix-open/isp-kit v1.39.2 h1:2L2LFwi+sNop9NeLDQ4scIy8LuohzXh4EBejHApAeDM=
+github.com/txix-open/isp-kit v1.39.2/go.mod h1:xB6mkoOewZBI1ND8kjU5TR6kvLu3VQI3Rdkh6g6FE4I=
 github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 h1:Z0Sqf+U+Sc6hfIXtRCIZlbJprkCjwMFbS4ZbEPQ44To=
 github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99/go.mod h1:pA0NYmhL0uEkkxj55S+S7MZU4e2QIt46x9XSrzrWc3c=
 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 h1:4BZHA+B1wXEQoGNHxW8mURaLhcdGwvRnmhGbm+odRbc=

conf/config.yml

@@ -1,5 +1,5 @@
 configServiceAddress:
-  ip: 127.0.0.1
+  ip: localhost
   port: 9001
 grpcOuterAddress:
   ip: isp-config-service
@@ -21,6 +21,10 @@ credentials:
   - username: default
     password: 38kPk0upqI5j
     perms: ["all"]
+  - username: monitor
+    password: monitor
+    perms: ["ready"]
+
 internalClientCredential: default
 
 rqlite:

deploy/compose/docker-compose.yml

@@ -0,0 +1,40 @@
+services:
+  isp-config-service-1:
+    image: "isp-config-service:latest"
+    container_name: "isp-config-service-1"
+    hostname: "isp-config-service-1"
+    restart: unless-stopped
+    environment:
+      grpcOuterAddress.ip: isp-config-service-1
+      rqlite.NodeID: 1
+      rqlite.BootstrapExpect: 3
+      rqlite.RaftAddr: "isp-config-service-1:4002"
+      rqlite.JoinAddrs: "isp-config-service-1:4002,isp-config-service-2:4002,isp-config-service-3:4002"
+    volumes:
+      - "./isp-config-service-1/data:/app/data"
+  isp-config-service-2:
+    image: "isp-config-service:latest"
+    container_name: "isp-config-service-2"
+    hostname: "isp-config-service-2"
+    restart: unless-stopped
+    environment:
+      grpcOuterAddress.ip: isp-config-service-2
+      rqlite.NodeID: 2
+      rqlite.BootstrapExpect: 3
+      rqlite.RaftAddr: "isp-config-service-2:4002"
+      rqlite.JoinAddrs: "isp-config-service-1:4002,isp-config-service-2:4002,isp-config-service-3:4002"
+    volumes:
+      - "./isp-config-service-2/data:/app/data"
+  isp-config-service-3:
+    image: "isp-config-service:latest"
+    container_name: "isp-config-service-3"
+    hostname: "isp-config-service-3"
+    restart: unless-stopped
+    environment:
+      grpcOuterAddress.ip: isp-config-service-3
+      rqlite.NodeID: 3
+      rqlite.BootstrapExpect: 3
+      rqlite.RaftAddr: "isp-config-service-3:4002"
+      rqlite.JoinAddrs: "isp-config-service-1:4002,isp-config-service-2:4002,isp-config-service-3:4002"
+    volumes:
+      - "./isp-config-service-3/data:/app/data"

deploy/k8s/service.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: isp-config-service-internal
+spec:
+  clusterIP: None
+  publishNotReadyAddresses: True
+  selector:
+    app: isp-config-service
+  ports:
+    - protocol: TCP
+      port: 4001
+      targetPort: 4001
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: isp-config-service
+spec:
+  selector:
+    app: isp-config-service
+  ports:
+    - protocol: TCP
+      port: 4001
+      targetPort: 4001

deploy/k8s/statefulset-3-node.yaml

@@ -0,0 +1,87 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: isp-config-service
+spec:
+  selector:
+    matchLabels:
+      app: isp-config-service # has to match .spec.template.metadata.labels
+  serviceName: isp-config-service-internal
+  replicas: 3
+  podManagementPolicy: "Parallel"
+  template:
+    metadata:
+      labels:
+        app: isp-config-service # has to match .spec.selector.matchLabels
+    spec:
+      terminationGracePeriodSeconds: 10
+      imagePullSecrets:
+        - name: regcred
+      containers:
+        - name: isp-config-service
+          image: isp-config-service:latest
+          imagePullPolicy: Always
+          env:
+            - name: rqlite.NodeID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: rqlite.BootstrapExpect
+              value: "3"
+            - name: rqlite.HttpAddr
+              value: "0.0.0.0:4001"
+            - name: rqlite.HttpAdv
+              value: "$HOSTNAME:4001"
+            - name: rqlite.RaftAddr
+              value: "0.0.0.0:4002"
+            - name: rqlite.RaftAdv
+              value: "$HOSTNAME:4002"
+            - name: rqlite.JoinAddrs
+              value: ""
+            - name: rqlite.DiscoMode
+              value: "dns"
+            - name: rqlite.DiscoConfig
+              value: "{\"name\":\"isp-config-service-internal\"}"
+            - name: rqlite.JoinInterval
+              value: "1s"
+            - name: rqlite.JoinAttempts
+              value: "120"
+            - name: rqlite.RaftClusterRemoveOnShutdown
+              value: "true"
+          ports:
+            - containerPort: 4001
+              name: rqlite
+          readinessProbe:
+            httpGet:
+              scheme: HTTP
+              path: /readyz
+              port: rqlite
+              httpHeaders:
+                - name: Authorization
+                  value: Basic bW9uaXRvcjptb25pdG9y
+            periodSeconds: 5
+            timeoutSeconds: 2
+            initialDelaySeconds: 2
+          livenessProbe:
+              httpGet:
+                scheme: HTTP
+                path: /readyz?noleader
+                port: rqlite
+                httpHeaders:
+                  - name: Authorization
+                    value: Basic bW9uaXRvcjptb25pdG9y
+              initialDelaySeconds: 5
+              timeoutSeconds: 2
+              failureThreshold: 3
+          volumeMounts:
+            - name: isp-config-service-data
+              mountPath: /app/data
+  volumeClaimTemplates:
+    - metadata:
+        name: isp-config-service-data
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        storageClassName: "local-path"
+        resources:
+          requests:
+            storage: 1Gi

go.mod

@@ -9,11 +9,12 @@ require (
 	github.com/pressly/goose/v3 v3.22.1
 	github.com/prometheus/client_golang v1.20.5
 	github.com/rqlite/gorqlite v0.0.0-20241013203532-4385768ae85d
+	github.com/rqlite/rqlite-disco-clients v0.0.0-20231230135307-118e35426347
 	github.com/rqlite/rqlite/v8 v8.32.5
 	github.com/stretchr/testify v1.9.0
 	github.com/tidwall/gjson v1.18.0
 	github.com/txix-open/etp/v4 v4.0.1
-	github.com/txix-open/isp-kit v1.39.1
+	github.com/txix-open/isp-kit v1.39.2
 	github.com/xeipuuv/gojsonschema v1.2.0
 	golang.org/x/sync v0.8.0
 	google.golang.org/grpc v1.67.1

go.sum

@@ -201,6 +201,8 @@ github.com/rqlite/gorqlite v0.0.0-20241013203532-4385768ae85d h1:c88ius/WcN19inn
 github.com/rqlite/gorqlite v0.0.0-20241013203532-4385768ae85d/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg=
 github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48 h1:NZ62M+kT0JqhyFUMc8I4SMmfmD4NGJxhb2ePJQXjryc=
 github.com/rqlite/raft-boltdb/v2 v2.0.0-20230523104317-c08e70f4de48/go.mod h1:CRnsxgy5G8fAf5J+AM0yrsSdxXHKkIYOaq2sm+Q4DYc=
+github.com/rqlite/rqlite-disco-clients v0.0.0-20231230135307-118e35426347 h1:01eOG4AXjYsrsLRITgBFHaDPXys1NfeY2idx25ZJFF8=
+github.com/rqlite/rqlite-disco-clients v0.0.0-20231230135307-118e35426347/go.mod h1:pym85nj6JnCI7rM9RxTZ4cubkTQyyg7uLwVydso9B80=
 github.com/rqlite/rqlite/v8 v8.32.5 h1:wEzDX2HrGWMvFkg2fcCmi+rb1wQ1m5yEefUQfOMvMSY=
 github.com/rqlite/rqlite/v8 v8.32.5/go.mod h1:a1irqNWEZ+pWuWpgekbScCXp85QtNTJLarN3bVPjn9M=
 github.com/rqlite/sql v0.0.0-20241029220113-152a320b02f7 h1:Mnz6yd4FWtiD6bbH9WHFFHfrOM2OYTUTmwrsckRc4W8=
@@ -237,8 +239,8 @@ github.com/txix-open/etp/v4 v4.0.1 h1:5VSYgGsvnMz0tvUHsjVWFETI0qWBTCF1O3ARCHaCIg
 github.com/txix-open/etp/v4 v4.0.1/go.mod h1:FIu7TUDdwcgmtmF6tFTV1bGA2fRY/iVaPQdsGzvqYc8=
 github.com/txix-open/grmq v1.6.0 h1:V4QRb0sI2CMfdMaMsFGvGvUIDGad5vr3khNU7A3LlP8=
 github.com/txix-open/grmq v1.6.0/go.mod h1:s+NNNv42+32yLpjvl2tpqtyBbMB9e7kKFee3lfa/dyo=
-github.com/txix-open/isp-kit v1.39.1 h1:10zEILNQSPY7naiuaQkIKNS2/fnWxuE9bz+dTiAptXw=
-github.com/txix-open/isp-kit v1.39.1/go.mod h1:xB6mkoOewZBI1ND8kjU5TR6kvLu3VQI3Rdkh6g6FE4I=
+github.com/txix-open/isp-kit v1.39.2 h1:2L2LFwi+sNop9NeLDQ4scIy8LuohzXh4EBejHApAeDM=
+github.com/txix-open/isp-kit v1.39.2/go.mod h1:xB6mkoOewZBI1ND8kjU5TR6kvLu3VQI3Rdkh6g6FE4I=
 github.com/txix-open/jsonschema v1.2.0 h1:B8TrdSsPhDvYv67oi/LVqf7/+PuqFlMsNwUzGeHfl1s=
 github.com/txix-open/jsonschema v1.2.0/go.mod h1:l8YDZ1nvJrw6uxWowSVOxCV/ebiMJyapffW87ZEqH00=
 github.com/txix-open/validator/v10 v10.0.0-20240401163703-e83541b40f99 h1:Z0Sqf+U+Sc6hfIXtRCIZlbJprkCjwMFbS4ZbEPQ44To=

service/rqlite/main.go

@@ -5,11 +5,16 @@ package rqlite
 import (
 	"context"
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
 	"github.com/pkg/errors"
+	"github.com/rqlite/rqlite-disco-clients/dns"
+	"github.com/rqlite/rqlite-disco-clients/dnssrv"
+	"github.com/txix-open/isp-kit/config"
 	"log"
 	"net"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -55,11 +60,17 @@ func main(ctx context.Context, r *Rqlite) error {
 	if err != nil {
 		log.Fatalf("failed to parse command-line flags: %s", err.Error())
 	}
+	err = evaluateAdvAddresses(r.cfg)
+	if err != nil {
+		return errors.WithMessage(err, "check adv ports")
+	}
 	localCfg := localConfig{Rqlite: cfg}
 	err = r.cfg.Read(&localCfg)
 	if err != nil {
 		return errors.WithMessage(err, "read local config")
 	}
+	dumpRqliteConfig, _ := json.MarshalIndent(cfg, "", "  ")
+	log.Printf("rqlite config:\n %s \n", string(dumpRqliteConfig))
 	err = cfg.Validate()
 	if err != nil {
 		return errors.WithMessage(err, "validate rqlite configuration")
@@ -210,6 +221,37 @@ func main(ctx context.Context, r *Rqlite) error {
 	return nil
 }
 
+func evaluateAdvAddresses(cfg *config.Config) error {
+	httpAdv := cfg.Optional().String("rqlite.HTTPAdv", "")
+	shouldEvalHttp := strings.Contains(httpAdv, "$HOSTNAME")
+	raftAdv := cfg.Optional().String("rqlite.RaftAdv", "")
+	shouldEvalRaft := strings.Contains(raftAdv, "$HOSTNAME")
+	if !shouldEvalHttp && !shouldEvalRaft {
+		return nil
+	}
+
+	var (
+		host string
+	)
+	data, err := exec.Command("hostname", "-f").Output()
+	if err != nil {
+		host, err = os.Hostname()
+		if err != nil {
+			return errors.WithMessage(err, "rqlite.HTTPAdv is not set, couldn't resolve local hostname")
+		}
+	} else {
+		host = strings.TrimSpace(string(data))
+	}
+
+	if shouldEvalHttp {
+		cfg.Set("rqlite.HTTPAdv", strings.ReplaceAll(httpAdv, "$HOSTNAME", host))
+	}
+	if shouldEvalRaft {
+		cfg.Set("rqlite.RaftAdv", strings.ReplaceAll(raftAdv, "$HOSTNAME", host))
+	}
+	return nil
+}
+
 func createExtensionsStore(cfg *Config) (*extensions.Store, error) {
 	str, err := extensions.NewStore(filepath.Join(cfg.DataPath, "extensions"))
 	if err != nil {
@@ -421,7 +463,46 @@ func createCluster(ctx context.Context, cfg *Config, hasPeers bool, client *clus
 		// existing Raft state.
 		return nil
 	}
-	return nil
+
+	// DNS-based discovery requested. It's OK to proceed with this even if this node
+	// is already part of a cluster. Re-joining and re-notifying other nodes will be
+	// ignored when the node is already part of the cluster.
+	log.Printf("discovery mode: %s", cfg.DiscoMode)
+	switch cfg.DiscoMode {
+	case DiscoModeDNS, DiscoModeDNSSRV:
+		rc := cfg.DiscoConfigReader()
+		defer func() {
+			if rc != nil {
+				rc.Close()
+			}
+		}()
+
+		var provider interface {
+			cluster.AddressProvider
+			httpd.StatusReporter
+		}
+		if cfg.DiscoMode == DiscoModeDNS {
+			dnsCfg, err := dns.NewConfigFromReader(rc)
+			if err != nil {
+				return fmt.Errorf("error reading DNS configuration: %s", err.Error())
+			}
+			provider = dns.NewWithPort(dnsCfg, cfg.RaftPort())
+
+		} else {
+			dnssrvCfg, err := dnssrv.NewConfigFromReader(rc)
+			if err != nil {
+				return fmt.Errorf("error reading DNS configuration: %s", err.Error())
+			}
+			provider = dnssrv.New(dnssrvCfg)
+		}
+
+		bs := cluster.NewBootstrapper(provider, client)
+		bs.SetCredentials(cluster.CredentialsFor(credStr, cfg.JoinAs))
+		httpServ.RegisterStatus("disco", provider)
+		return bs.Boot(ctx, str.ID(), cfg.RaftAdv, clusterSuf, bootDoneFn, cfg.BootstrapExpectTimeout)
+	default:
+		return fmt.Errorf("invalid disco mode %s", cfg.DiscoMode)
+	}
 }
 
 func networkCheckJoinAddrs(joinAddrs []string) error {

service/startup/service.go

@@ -77,6 +77,17 @@ func New(boot *bootstrap.Bootstrap) (*Service, error) {
 
 // nolint:funlen
 func (s *Service) Run(ctx context.Context) error {
+	if s.boot.App.Config().Optional().String("KUBERNETES_SERVICE_HOST", "") != "" {
+		s.logger.Info(
+			ctx,
+			"run in kubernetes, sleep extra 5s, reason described here: https://github.com/rqlite/rqlite/blob/master/docker-entrypoint.sh#L96",
+		)
+		select {
+		case <-ctx.Done():
+		case <-time.After(5 * time.Second): //nolint:mnd
+		}
+	}
+
 	go func() {
 		s.logger.Debug(ctx, "running embedded rqlite...")
 		err := s.rqlite.Run(ctx)