From c8468b2614667045f784927388766888e7e6dc17 Mon Sep 17 00:00:00 2001 From: "ryota.suzuki" Date: Sat, 11 Jan 2025 20:53:09 +0900 Subject: [PATCH 01/12] impl psql persistent plugin --- persistence/postgresql/ddl/journals.sql | 9 ++ persistence/postgresql/ddl/snapshots.sql | 8 ++ persistence/postgresql/provider.go | 26 ++++ persistence/postgresql/provider_state.go | 161 +++++++++++++++++++++++ 4 files changed, 204 insertions(+) create mode 100644 persistence/postgresql/ddl/journals.sql create mode 100644 persistence/postgresql/ddl/snapshots.sql create mode 100644 persistence/postgresql/provider.go create mode 100644 persistence/postgresql/provider_state.go diff --git a/persistence/postgresql/ddl/journals.sql b/persistence/postgresql/ddl/journals.sql new file mode 100644 index 000000000..71e83fe1c --- /dev/null +++ b/persistence/postgresql/ddl/journals.sql @@ -0,0 +1,9 @@ +create table if not exists event_journals +( + actor_name varchar(255) not null, + event_index bigint not null, + event json not null, + message_type varchar(255) not null, + primary key (actor_name, event_index) +); + diff --git a/persistence/postgresql/ddl/snapshots.sql b/persistence/postgresql/ddl/snapshots.sql new file mode 100644 index 000000000..93210eb27 --- /dev/null +++ b/persistence/postgresql/ddl/snapshots.sql @@ -0,0 +1,8 @@ +create table if not exists snapshots +( + id varchar(255) not null, + snapshot_index bigint not null, + snapshot json not null, + message_type varchar(255) not null, + primary key (id) +); diff --git a/persistence/postgresql/provider.go b/persistence/postgresql/provider.go new file mode 100644 index 000000000..36ba2a83f --- /dev/null +++ b/persistence/postgresql/provider.go @@ -0,0 +1,26 @@ +package postgresql + +import ( + context2 "context" + "github.com/asynkron/protoactor-go/actor" + "github.com/asynkron/protoactor-go/persistence" + "github.com/jackc/pgx/v5/pgxpool" + "log/slog" +) + +type Provider struct { + connConfig *pgxpool.Config + actorSystem actor.ActorSystem +} + +func (p *Provider) GetState() persistence.ProviderState { + pool, err := pgxpool.NewWithConfig(context2.Background(), p.connConfig) + if err != nil { + p.actorSystem.Logger().Error("failed to create pgx pool", slog.Any("error", err)) + panic(err) + } + + return NewProviderState(pool, p.actorSystem.Logger()) +} + +var _ persistence.Provider = &Provider{} diff --git a/persistence/postgresql/provider_state.go b/persistence/postgresql/provider_state.go new file mode 100644 index 000000000..a5049d64c --- /dev/null +++ b/persistence/postgresql/provider_state.go @@ -0,0 +1,161 @@ +package postgresql + +import ( + context2 "context" + "encoding/json" + "errors" + "fmt" + "github.com/asynkron/protoactor-go/persistence" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" + "log/slog" + "sync" +) + +type ProviderState struct { + connPool *pgxpool.Pool + logger *slog.Logger + wg sync.WaitGroup +} + +func NewProviderState(conn *pgxpool.Pool, logger *slog.Logger) *ProviderState { + return &ProviderState{connPool: conn, logger: logger} +} + +func (s *ProviderState) GetSnapshot(actorName string) (snapshot interface{}, eventIndex int, ok bool) { + rows, err := s.connPool.Query(context2.Background(), + "SELECT snapshot, message_type, snapshot_index FROM snapshots WHERE actor_name = $1 ORDER BY snapshot_index DESC LIMIT 1") + if err != nil { + s.logger.Error("Error getting snapshot", slog.Any("error", err)) + return nil, 0, false + } + snapshotRow, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[snapshotsTableRow]) + if errors.Is(err, pgx.ErrNoRows) { + return nil, 0, false + } else if err != nil { + s.logger.Error("Error getting snapshot", slog.Any("error", err)) + return nil, 0, false + } + snapshot, err = unmarshalProtoMessage(snapshotRow.Snapshot, snapshotRow.MessageType) + if err != nil { + s.logger.Error("Error unmarshalling snapshot", slog.Any("error", err)) + return nil, 0, false + } + + return snapshot, snapshotRow.SnapshotIndex, true +} + +func (s *ProviderState) PersistSnapshot(actorName string, snapshotIndex int, snapshot proto.Message) { + s.wg.Add(1) + defer s.wg.Done() + bytes, err := json.Marshal(snapshot) + if err != nil { + s.logger.Error("Error marshalling snapshot", slog.Any("error", err)) + return + } + _, err = s.connPool.Exec(context2.Background(), + "INSERT INTO snapshots (id, snapshot, message_type, snapshot_index) VALUES ($1, $2, $3, $4)", + actorName, bytes, proto.MessageName(snapshot), snapshotIndex) + if err != nil { + s.logger.Error("Error persisting snapshot", slog.Any("error", err)) + } +} + +func (s *ProviderState) DeleteSnapshots(actorName string, inclusiveToIndex int) { + _, err := s.connPool.Exec(context2.Background(), + "DELETE FROM snapshots WHERE actor_name = $1 AND snapshot_index <= $2", + actorName, inclusiveToIndex) + if err != nil { + s.logger.Error("Error deleting snapshots", slog.Any("error", err)) + } +} + +func (s *ProviderState) GetEvents(actorName string, eventIndexStart int, eventIndexEnd int, callback func(e interface{})) { + rows, err := s.connPool.Query(context2.Background(), + "SELECT event, message_type, event_index FROM event_journals WHERE actor_name = $1 AND event_index >= $2 AND event_index <= $3", + actorName, eventIndexStart, eventIndexEnd) + if err != nil { + s.logger.Error("Error getting events", slog.Any("error", err)) + return + } + eventRows, err := pgx.CollectRows(rows, pgx.RowToStructByName[journalEventsTableRow]) + events := make([]proto.Message, len(eventRows)) + for _, row := range eventRows { + event, err := unmarshalProtoMessage(row.Event, row.MessageType) + if err != nil { + s.logger.Error("Error unmarshalling event", slog.Any("error", err)) + return + } + events[row.EventIndex] = event + } + + for _, event := range events { + callback(event) + } +} + +func (s *ProviderState) PersistEvent(actorName string, eventIndex int, event proto.Message) { + s.wg.Add(1) + defer s.wg.Done() + bytes, err := json.Marshal(event) + if err != nil { + s.logger.Error("Error marshalling event", slog.Any("error", err)) + return + } + _, err = s.connPool.Exec(context2.Background(), + "INSERT INTO event_journals (actor_name, event, message_type, event_index) VALUES ($1, $2, $3, $4)", + actorName, bytes, proto.MessageName(event), eventIndex) + if err != nil { + s.logger.Error("Error persisting event", slog.Any("error", err)) + } +} + +func (s *ProviderState) DeleteEvents(actorName string, inclusiveToIndex int) { + _, err := s.connPool.Exec(context2.Background(), + "DELETE FROM event_journals WHERE actor_name = $1 AND event_index <= $2", + actorName, inclusiveToIndex) + if err != nil { + s.logger.Error("Error deleting events", slog.Any("error", err)) + } +} + +func (s *ProviderState) Restart() { + s.wg.Wait() +} + +func (s *ProviderState) GetSnapshotInterval() int { + return 1 +} + +var _ persistence.ProviderState = &ProviderState{} + +type snapshotsTableRow struct { + Id string + Snapshot []byte + SnapshotIndex int + MessageType string +} + +type journalEventsTableRow struct { + ActorName string + Event []byte + EventIndex int + MessageType string +} + +func unmarshalProtoMessage(bytes []byte, messageTypeName string) (proto.Message, error) { + mt, err := protoregistry.GlobalTypes.FindMessageByName(protoreflect.FullName(messageTypeName)) + if err != nil { + return nil, fmt.Errorf("error finding message type %s: %w", messageTypeName, err) + } + + pm := mt.New().Interface() + err = json.Unmarshal(bytes, pm) + if err != nil { + return nil, fmt.Errorf("error unmarshalling message %s: %w", messageTypeName, err) + } + return pm, nil +} From 50a0528cc2364a00e5485c95b51c14064dad99d2 Mon Sep 17 00:00:00 2001 From: "ryota.suzuki" Date: Sat, 11 Jan 2025 21:17:10 +0900 Subject: [PATCH 02/12] add example --- examples/persistence-psql/go.mod | 91 ++++++++ examples/persistence-psql/go.sum | 251 +++++++++++++++++++++ examples/persistence-psql/main.go | 271 +++++++++++++++++++++++ persistence/postgresql/ddl/snapshots.sql | 10 +- persistence/postgresql/provider.go | 6 +- persistence/postgresql/provider_state.go | 2 +- 6 files changed, 624 insertions(+), 7 deletions(-) create mode 100644 examples/persistence-psql/go.mod create mode 100644 examples/persistence-psql/go.sum create mode 100644 examples/persistence-psql/main.go diff --git a/examples/persistence-psql/go.mod b/examples/persistence-psql/go.mod new file mode 100644 index 000000000..e18ab334a --- /dev/null +++ b/examples/persistence-psql/go.mod @@ -0,0 +1,91 @@ +module persistence-psql + +go 1.22 + +toolchain go1.23.1 + +replace github.com/asynkron/protoactor-go => ../.. + +require ( + github.com/asynkron/goconsole v0.0.0-20160504192649-bfa12eebf716 + google.golang.org/protobuf v1.33.0 +) + +require ( + github.com/asynkron/protoactor-go v0.0.0-20240116091649-93e384a26d0d + github.com/jackc/pgx/v5 v5.7.2 + github.com/testcontainers/testcontainers-go v0.35.0 +) + +require ( + dario.cat/mergo v1.0.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Workiva/go-datastructures v1.1.3 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/containerd/containerd v1.7.18 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v27.1.1+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.17.4 // indirect + github.com/lithammer/shortuuid/v4 v4.0.0 // indirect + github.com/lmittmann/tint v1.0.3 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/user v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/orcaman/concurrent-map v1.0.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/prometheus/client_golang v1.17.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.1 // indirect + github.com/shirou/gopsutil/v3 v3.23.12 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/stretchr/testify v1.9.0 // indirect + github.com/testcontainers/testcontainers-go/modules/postgres v0.35.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/exporters/prometheus v0.44.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/sdk v1.21.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/examples/persistence-psql/go.sum b/examples/persistence-psql/go.sum new file mode 100644 index 000000000..219e0c1f2 --- /dev/null +++ b/examples/persistence-psql/go.sum @@ -0,0 +1,251 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Workiva/go-datastructures v1.1.3 h1:LRdRrug9tEuKk7TGfz/sct5gjVj44G9pfqDt4qm7ghw= +github.com/Workiva/go-datastructures v1.1.3/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A= +github.com/asynkron/goconsole v0.0.0-20160504192649-bfa12eebf716 h1:SgyG4sXkrlalMoCfp20LiNPNhfJS7ez3opNdtihIxPc= +github.com/asynkron/goconsole v0.0.0-20160504192649-bfa12eebf716/go.mod h1:/zSlF0T2ArAsTG6SVu8d8qlK+19jjudjA3wWsxnGFHg= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= +github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= +github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI= +github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw7k08o4c= +github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= +github.com/lmittmann/tint v1.0.3 h1:W5PHeA2D8bBJVvabNfQD/XW9HPLZK1XoPZH0cq8NouQ= +github.com/lmittmann/tint v1.0.3/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HDbW65HOY= +github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= +github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/testcontainers/testcontainers-go v0.35.0 h1:uADsZpTKFAtp8SLK+hMwSaa+X+JiERHtd4sQAFmXeMo= +github.com/testcontainers/testcontainers-go v0.35.0/go.mod h1:oEVBj5zrfJTrgjwONs1SsRbnBtH9OKl+IGl3UMcr2B4= +github.com/testcontainers/testcontainers-go/modules/postgres v0.35.0 h1:eEGx9kYzZb2cNhRbBrNOCL/YPOM7+RMJiy3bB+ie0/I= +github.com/testcontainers/testcontainers-go/modules/postgres v0.35.0/go.mod h1:hfH71Mia/WWLBgMD2YctYcMlfsbnT0hflweL1dy8Q4s= +github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/exporters/prometheus v0.44.0 h1:08qeJgaPC0YEBu2PQMbqU3rogTlyzpjhCI2b58Yn00w= +go.opentelemetry.io/otel/exporters/prometheus v0.44.0/go.mod h1:ERL2uIeBtg4TxZdojHUwzZfIFlUIjZtxubT5p4h1Gjg= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/sdk/metric v1.21.0 h1:smhI5oD714d6jHE6Tie36fPx4WDFIg+Y6RfAY4ICcR0= +go.opentelemetry.io/otel/sdk/metric v1.21.0/go.mod h1:FJ8RAsoPGv/wYMgBdUJXOm+6pzFY3YdljnXtv1SBE8Q= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0= +google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 h1:W18sezcAYs+3tDZX4F80yctqa12jcP1PUS2gQu1zTPU= +google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= diff --git a/examples/persistence-psql/main.go b/examples/persistence-psql/main.go new file mode 100644 index 000000000..e0895f0d2 --- /dev/null +++ b/examples/persistence-psql/main.go @@ -0,0 +1,271 @@ +package main + +import ( + context2 "context" + "fmt" + "github.com/asynkron/protoactor-go/persistence/postgresql" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/testcontainers/testcontainers-go/modules/postgres" + "log" + "time" + + console "github.com/asynkron/goconsole" + "github.com/asynkron/protoactor-go/actor" + "github.com/asynkron/protoactor-go/persistence" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/reflect/protodesc" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/runtime/protoiface" + "google.golang.org/protobuf/types/descriptorpb" +) + +type protoMsg struct { + state string + set bool + value string +} + +func (p *protoMsg) Reset() {} +func (p *protoMsg) String() string { return p.state } +func (p *protoMsg) ProtoMessage() {} + +type ( + Message struct{ protoMsg } + Snapshot struct{ protoMsg } +) + +func (m *protoMsg) ProtoReflect() protoreflect.Message { return (*message)(m) } + +type message protoMsg + +type messageType struct{} + +func (messageType) New() protoreflect.Message { return &message{} } +func (messageType) Zero() protoreflect.Message { return (*message)(nil) } +func (messageType) Descriptor() protoreflect.MessageDescriptor { return fileDesc.Messages().Get(0) } + +func (m *message) New() protoreflect.Message { return &message{} } +func (m *message) Descriptor() protoreflect.MessageDescriptor { return fileDesc.Messages().Get(0) } +func (m *message) Type() protoreflect.MessageType { return messageType{} } +func (m *message) Interface() protoreflect.ProtoMessage { return (*protoMsg)(m) } +func (m *message) ProtoMethods() *protoiface.Methods { return nil } + +var fieldDescS = fileDesc.Messages().Get(0).Fields().Get(0) + +func (m *message) Range(f func(protoreflect.FieldDescriptor, protoreflect.Value) bool) { + if m.set { + f(fieldDescS, protoreflect.ValueOf(m.value)) + } +} + +func (m *message) Has(fd protoreflect.FieldDescriptor) bool { + if fd == fieldDescS { + return m.set + } + panic("invalid field descriptor") +} + +func (m *message) Clear(fd protoreflect.FieldDescriptor) { + if fd == fieldDescS { + m.value = "" + m.set = false + return + } + panic("invalid field descriptor") +} + +func (m *message) Get(fd protoreflect.FieldDescriptor) protoreflect.Value { + if fd == fieldDescS { + return protoreflect.ValueOf(m.value) + } + panic("invalid field descriptor") +} + +func (m *message) Set(fd protoreflect.FieldDescriptor, v protoreflect.Value) { + if fd == fieldDescS { + m.value = v.String() + m.set = true + return + } + panic("invalid field descriptor") +} + +func (m *message) Mutable(protoreflect.FieldDescriptor) protoreflect.Value { + panic("invalid field descriptor") +} + +func (m *message) NewField(protoreflect.FieldDescriptor) protoreflect.Value { + panic("invalid field descriptor") +} + +func (m *message) WhichOneof(protoreflect.OneofDescriptor) protoreflect.FieldDescriptor { + panic("invalid oneof descriptor") +} + +func (m *message) GetUnknown() protoreflect.RawFields { return nil } + +// func (m *message) SetUnknown(protoreflect.RawFields) { return } +func (m *message) SetUnknown(protoreflect.RawFields) {} + +func (m *message) IsValid() bool { + return m != nil +} + +var fileDesc = func() protoreflect.FileDescriptor { + p := &descriptorpb.FileDescriptorProto{} + if err := prototext.Unmarshal([]byte(descriptorText), p); err != nil { + panic(err) + } + file, err := protodesc.NewFile(p, nil) + if err != nil { + panic(err) + } + return file +}() + +const descriptorText = ` + name: "internal/testprotos/irregular/irregular.proto" + package: "goproto.proto.thirdparty" + message_type { + name: "IrregularMessage" + field { + name: "s" + number: 1 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "s" + } + } + options { + go_package: "google.golang.org/protobuf/internal/testprotos/irregular" + } +` + +type AberrantMessage int + +func (m AberrantMessage) ProtoMessage() {} +func (m AberrantMessage) Reset() {} +func (m AberrantMessage) String() string { return "" } +func (m AberrantMessage) Marshal() ([]byte, error) { return nil, nil } +func (m AberrantMessage) Unmarshal([]byte) error { return nil } + +type Actor struct { + persistence.Mixin + state string +} + +func (a *Actor) Receive(ctx actor.Context) { + switch msg := ctx.Message().(type) { + case *actor.Started: + log.Println("actor started") + case *persistence.RequestSnapshot: + log.Printf("snapshot internal state '%v'", a.state) + a.PersistSnapshot(&Snapshot{protoMsg: protoMsg{state: a.state}}) + case *Snapshot: + a.state = msg.state + log.Printf("recovered from snapshot, internal state changed to '%v'", a.state) + case *persistence.ReplayComplete: + log.Printf("replay completed, internal state changed to '%v'", a.state) + case *Message: + scenario := "received replayed event" + if !a.Recovering() { + a.PersistReceive(msg) + scenario = "received new message" + } + a.state = msg.state + log.Printf("%s, internal state changed to '%v'\n", scenario, a.state) + } +} + +func main() { + ctx := context2.Background() + dbName := "users" + dbUser := "user" + dbPassword := "password" + + postgresContainer, err := postgres.Run(ctx, + "postgres:16-alpine", + postgres.WithDatabase(dbName), + postgres.WithUsername(dbUser), + postgres.WithPassword(dbPassword), + testcontainers.WithWaitStrategy( + wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2). + WithStartupTimeout(5*time.Second), + ), + ) + + defer func() { + if err := testcontainers.TerminateContainer(postgresContainer); err != nil { + log.Printf("failed to terminate container: %s", err) + } + }() + if err != nil { + log.Printf("failed to start container: %s", err) + return + } + + connString := postgresContainer.MustConnectionString(ctx, "sslmode=disable", "application_name=test") + + err = prepareTables(ctx, connString) + if err != nil { + log.Printf("failed to prepare tables: %s", err) + return + } + + config, err := pgxpool.ParseConfig(connString) + if err != nil { + log.Printf("failed to parse connection string: %s", err) + return + } + system := actor.NewActorSystem() + provider := postgresql.NewProvider(config, system) + + rootContext := system.Root + props := actor.PropsFromProducer(func() actor.Actor { return &Actor{} }, + actor.WithReceiverMiddleware(persistence.Using(provider))) + pid, _ := rootContext.SpawnNamed(props, "persistent") + rootContext.Send(pid, &Message{protoMsg: protoMsg{state: "state4"}}) + rootContext.Send(pid, &Message{protoMsg: protoMsg{state: "state5"}}) + + rootContext.PoisonFuture(pid).Wait() + fmt.Printf("*** restart ***\n") + pid, _ = rootContext.SpawnNamed(props, "persistent") + + _, _ = console.ReadLine() +} + +func prepareTables(ctx context2.Context, connString string) error { + pool, err := pgxpool.New(ctx, connString) + if err != nil { + return err + } + + _, err = pool.Exec(context2.Background(), ` +create table if not exists event_journals +( + actor_name varchar(255) not null, + event_index bigint not null, + event json not null, + message_type varchar(255) not null, + primary key (actor_name, event_index) +); + `) + if err != nil { + return err + } + + _, err = pool.Exec(context2.Background(), ` +create table if not exists snapshots +( + actor_name varchar(255) not null, + snapshot_index bigint not null, + snapshot json not null, + message_type varchar(255) not null, + primary key (id) +); + `) + return err +} diff --git a/persistence/postgresql/ddl/snapshots.sql b/persistence/postgresql/ddl/snapshots.sql index 93210eb27..df3ae037f 100644 --- a/persistence/postgresql/ddl/snapshots.sql +++ b/persistence/postgresql/ddl/snapshots.sql @@ -1,8 +1,8 @@ create table if not exists snapshots ( - id varchar(255) not null, - snapshot_index bigint not null, - snapshot json not null, - message_type varchar(255) not null, - primary key (id) + actor_name varchar(255) not null, + snapshot_index bigint not null, + snapshot json not null, + message_type varchar(255) not null, + primary key (actor_name) ); diff --git a/persistence/postgresql/provider.go b/persistence/postgresql/provider.go index 36ba2a83f..ac5ea78e3 100644 --- a/persistence/postgresql/provider.go +++ b/persistence/postgresql/provider.go @@ -10,7 +10,11 @@ import ( type Provider struct { connConfig *pgxpool.Config - actorSystem actor.ActorSystem + actorSystem *actor.ActorSystem +} + +func NewProvider(connConfig *pgxpool.Config, actorSystem *actor.ActorSystem) *Provider { + return &Provider{connConfig: connConfig, actorSystem: actorSystem} } func (p *Provider) GetState() persistence.ProviderState { diff --git a/persistence/postgresql/provider_state.go b/persistence/postgresql/provider_state.go index a5049d64c..e2a006506 100644 --- a/persistence/postgresql/provider_state.go +++ b/persistence/postgresql/provider_state.go @@ -133,7 +133,7 @@ func (s *ProviderState) GetSnapshotInterval() int { var _ persistence.ProviderState = &ProviderState{} type snapshotsTableRow struct { - Id string + ActorName string Snapshot []byte SnapshotIndex int MessageType string From ea2780fb29d9591ae99573435004a432a0966cb6 Mon Sep 17 00:00:00 2001 From: "ryota.suzuki" Date: Sat, 11 Jan 2025 22:43:22 +0900 Subject: [PATCH 03/12] fix example --- examples/persistence-psql/main.go | 248 ++++++++++++++++++----- go.mod | 18 +- go.sum | 32 +-- persistence/postgresql/ddl/snapshots.sql | 2 +- persistence/postgresql/provider_state.go | 27 ++- 5 files changed, 244 insertions(+), 83 deletions(-) diff --git a/examples/persistence-psql/main.go b/examples/persistence-psql/main.go index e0895f0d2..e96b937b5 100644 --- a/examples/persistence-psql/main.go +++ b/examples/persistence-psql/main.go @@ -6,6 +6,7 @@ import ( "github.com/asynkron/protoactor-go/persistence/postgresql" "github.com/jackc/pgx/v5/pgxpool" "github.com/testcontainers/testcontainers-go/modules/postgres" + "google.golang.org/protobuf/reflect/protoregistry" "log" "time" @@ -21,54 +22,106 @@ import ( "google.golang.org/protobuf/types/descriptorpb" ) -type protoMsg struct { +type Message struct { state string set bool value string } -func (p *protoMsg) Reset() {} -func (p *protoMsg) String() string { return p.state } -func (p *protoMsg) ProtoMessage() {} +func (m *Message) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"%s"`, m.state)), nil +} -type ( - Message struct{ protoMsg } - Snapshot struct{ protoMsg } -) +func (m *Message) UnmarshalJSON(bytes []byte) error { + m.state = string(bytes) + m.set = true + return nil +} + +func (m *Message) Zero() protoreflect.Message { + return &Message{} +} + +func (m *Message) Reset() {} +func (m *Message) String() string { return m.state } +func (m *Message) ProtoMessage() {} -func (m *protoMsg) ProtoReflect() protoreflect.Message { return (*message)(m) } +type Snapshot struct { + state string + set bool + value string +} -type message protoMsg +func (s *Snapshot) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"%s"`, s.state)), nil +} + +func (s *Snapshot) UnmarshalJSON(bytes []byte) error { + s.state = string(bytes) + s.set = true + return nil +} + +func (s *Snapshot) Zero() protoreflect.Message { + return &Snapshot{} +} + +func (s *Snapshot) Reset() {} +func (s *Snapshot) String() string { return s.state } +func (s *Snapshot) ProtoMessage() {} + +func (m *Message) ProtoReflect() protoreflect.Message { return m } +func (s *Snapshot) ProtoReflect() protoreflect.Message { return s } type messageType struct{} -func (messageType) New() protoreflect.Message { return &message{} } -func (messageType) Zero() protoreflect.Message { return (*message)(nil) } -func (messageType) Descriptor() protoreflect.MessageDescriptor { return fileDesc.Messages().Get(0) } +func (messageType) New() protoreflect.Message { return &Message{} } +func (messageType) Zero() protoreflect.Message { return (*Message)(nil) } +func (messageType) Descriptor() protoreflect.MessageDescriptor { + return fileDescForMessage.Messages().Get(0) +} -func (m *message) New() protoreflect.Message { return &message{} } -func (m *message) Descriptor() protoreflect.MessageDescriptor { return fileDesc.Messages().Get(0) } -func (m *message) Type() protoreflect.MessageType { return messageType{} } -func (m *message) Interface() protoreflect.ProtoMessage { return (*protoMsg)(m) } -func (m *message) ProtoMethods() *protoiface.Methods { return nil } +type snapshotType struct{} -var fieldDescS = fileDesc.Messages().Get(0).Fields().Get(0) +func (snapshotType) New() protoreflect.Message { return &Snapshot{} } +func (snapshotType) Zero() protoreflect.Message { return (*Snapshot)(nil) } +func (snapshotType) Descriptor() protoreflect.MessageDescriptor { + return fileDescForSnapshot.Messages().Get(0) +} + +func (m *Message) New() protoreflect.Message { return &Message{} } +func (m *Message) Descriptor() protoreflect.MessageDescriptor { + return fileDescForMessage.Messages().Get(0) +} +func (m *Message) Type() protoreflect.MessageType { return messageType{} } +func (m *Message) Interface() protoreflect.ProtoMessage { return m } +func (m *Message) ProtoMethods() *protoiface.Methods { return nil } + +func (s *Snapshot) New() protoreflect.Message { return &Snapshot{} } +func (s *Snapshot) Descriptor() protoreflect.MessageDescriptor { + return fileDescForSnapshot.Messages().Get(0) +} +func (s *Snapshot) Type() protoreflect.MessageType { return snapshotType{} } +func (s *Snapshot) Interface() protoreflect.ProtoMessage { return s } +func (s *Snapshot) ProtoMethods() *protoiface.Methods { return nil } -func (m *message) Range(f func(protoreflect.FieldDescriptor, protoreflect.Value) bool) { +var fieldDescSForMessage = fileDescForMessage.Messages().Get(0).Fields().Get(0) + +func (m *Message) Range(f func(protoreflect.FieldDescriptor, protoreflect.Value) bool) { if m.set { - f(fieldDescS, protoreflect.ValueOf(m.value)) + f(fieldDescSForMessage, protoreflect.ValueOf(m.value)) } } -func (m *message) Has(fd protoreflect.FieldDescriptor) bool { - if fd == fieldDescS { +func (m *Message) Has(fd protoreflect.FieldDescriptor) bool { + if fd == fieldDescSForMessage { return m.set } panic("invalid field descriptor") } -func (m *message) Clear(fd protoreflect.FieldDescriptor) { - if fd == fieldDescS { +func (m *Message) Clear(fd protoreflect.FieldDescriptor) { + if fd == fieldDescSForMessage { m.value = "" m.set = false return @@ -76,15 +129,15 @@ func (m *message) Clear(fd protoreflect.FieldDescriptor) { panic("invalid field descriptor") } -func (m *message) Get(fd protoreflect.FieldDescriptor) protoreflect.Value { - if fd == fieldDescS { +func (m *Message) Get(fd protoreflect.FieldDescriptor) protoreflect.Value { + if fd == fieldDescSForMessage { return protoreflect.ValueOf(m.value) } panic("invalid field descriptor") } -func (m *message) Set(fd protoreflect.FieldDescriptor, v protoreflect.Value) { - if fd == fieldDescS { +func (m *Message) Set(fd protoreflect.FieldDescriptor, v protoreflect.Value) { + if fd == fieldDescSForMessage { m.value = v.String() m.set = true return @@ -92,30 +145,30 @@ func (m *message) Set(fd protoreflect.FieldDescriptor, v protoreflect.Value) { panic("invalid field descriptor") } -func (m *message) Mutable(protoreflect.FieldDescriptor) protoreflect.Value { +func (m *Message) Mutable(protoreflect.FieldDescriptor) protoreflect.Value { panic("invalid field descriptor") } -func (m *message) NewField(protoreflect.FieldDescriptor) protoreflect.Value { +func (m *Message) NewField(protoreflect.FieldDescriptor) protoreflect.Value { panic("invalid field descriptor") } -func (m *message) WhichOneof(protoreflect.OneofDescriptor) protoreflect.FieldDescriptor { +func (m *Message) WhichOneof(protoreflect.OneofDescriptor) protoreflect.FieldDescriptor { panic("invalid oneof descriptor") } -func (m *message) GetUnknown() protoreflect.RawFields { return nil } +func (m *Message) GetUnknown() protoreflect.RawFields { return nil } // func (m *message) SetUnknown(protoreflect.RawFields) { return } -func (m *message) SetUnknown(protoreflect.RawFields) {} +func (m *Message) SetUnknown(protoreflect.RawFields) {} -func (m *message) IsValid() bool { +func (m *Message) IsValid() bool { return m != nil } -var fileDesc = func() protoreflect.FileDescriptor { +var fileDescForMessage = func() protoreflect.FileDescriptor { p := &descriptorpb.FileDescriptorProto{} - if err := prototext.Unmarshal([]byte(descriptorText), p); err != nil { + if err := prototext.Unmarshal([]byte(descriptorTextForMessage), p); err != nil { panic(err) } file, err := protodesc.NewFile(p, nil) @@ -125,11 +178,11 @@ var fileDesc = func() protoreflect.FileDescriptor { return file }() -const descriptorText = ` +const descriptorTextForMessage = ` name: "internal/testprotos/irregular/irregular.proto" package: "goproto.proto.thirdparty" message_type { - name: "IrregularMessage" + name: "Message" field { name: "s" number: 1 @@ -143,13 +196,96 @@ const descriptorText = ` } ` -type AberrantMessage int +var fieldDescSForSnapshot = fileDescForSnapshot.Messages().Get(0).Fields().Get(0) + +func (s *Snapshot) Range(f func(protoreflect.FieldDescriptor, protoreflect.Value) bool) { + if s.set { + f(fieldDescSForSnapshot, protoreflect.ValueOf(s.value)) + } +} + +func (s *Snapshot) Has(fd protoreflect.FieldDescriptor) bool { + if fd == fieldDescSForSnapshot { + return s.set + } + panic("invalid field descriptor") +} + +func (s *Snapshot) Clear(fd protoreflect.FieldDescriptor) { + if fd == fieldDescSForSnapshot { + s.value = "" + s.set = false + return + } + panic("invalid field descriptor") +} + +func (s *Snapshot) Get(fd protoreflect.FieldDescriptor) protoreflect.Value { + if fd == fieldDescSForSnapshot { + return protoreflect.ValueOf(s.value) + } + panic("invalid field descriptor") +} -func (m AberrantMessage) ProtoMessage() {} -func (m AberrantMessage) Reset() {} -func (m AberrantMessage) String() string { return "" } -func (m AberrantMessage) Marshal() ([]byte, error) { return nil, nil } -func (m AberrantMessage) Unmarshal([]byte) error { return nil } +func (s *Snapshot) Set(fd protoreflect.FieldDescriptor, v protoreflect.Value) { + if fd == fieldDescSForSnapshot { + s.value = v.String() + s.set = true + return + } + panic("invalid field descriptor") +} + +func (s *Snapshot) Mutable(protoreflect.FieldDescriptor) protoreflect.Value { + panic("invalid field descriptor") +} + +func (s *Snapshot) NewField(protoreflect.FieldDescriptor) protoreflect.Value { + panic("invalid field descriptor") +} + +func (s *Snapshot) WhichOneof(protoreflect.OneofDescriptor) protoreflect.FieldDescriptor { + panic("invalid oneof descriptor") +} + +func (s *Snapshot) GetUnknown() protoreflect.RawFields { return nil } + +// func (m *message) SetUnknown(protoreflect.RawFields) { return } +func (s *Snapshot) SetUnknown(protoreflect.RawFields) {} + +func (s *Snapshot) IsValid() bool { + return s != nil +} + +var fileDescForSnapshot = func() protoreflect.FileDescriptor { + p := &descriptorpb.FileDescriptorProto{} + if err := prototext.Unmarshal([]byte(descriptorTextForSnapshot), p); err != nil { + panic(err) + } + file, err := protodesc.NewFile(p, nil) + if err != nil { + panic(err) + } + return file +}() + +const descriptorTextForSnapshot = ` + name: "internal/testprotos/irregular/irregular.proto" + package: "goproto.proto.thirdparty" + message_type { + name: "Snapshot" + field { + name: "s" + number: 1 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "s" + } + } + options { + go_package: "google.golang.org/protobuf/internal/testprotos/irregular" + } +` type Actor struct { persistence.Mixin @@ -162,7 +298,7 @@ func (a *Actor) Receive(ctx actor.Context) { log.Println("actor started") case *persistence.RequestSnapshot: log.Printf("snapshot internal state '%v'", a.state) - a.PersistSnapshot(&Snapshot{protoMsg: protoMsg{state: a.state}}) + a.PersistSnapshot(&Snapshot{state: a.state}) case *Snapshot: a.state = msg.state log.Printf("recovered from snapshot, internal state changed to '%v'", a.state) @@ -180,6 +316,12 @@ func (a *Actor) Receive(ctx actor.Context) { } func main() { + if err := protoregistry.GlobalTypes.RegisterMessage(&Message{}); err != nil { + panic(err) + } + if err := protoregistry.GlobalTypes.RegisterMessage(&Snapshot{}); err != nil { + panic(err) + } ctx := context2.Background() dbName := "users" dbUser := "user" @@ -227,8 +369,8 @@ func main() { props := actor.PropsFromProducer(func() actor.Actor { return &Actor{} }, actor.WithReceiverMiddleware(persistence.Using(provider))) pid, _ := rootContext.SpawnNamed(props, "persistent") - rootContext.Send(pid, &Message{protoMsg: protoMsg{state: "state4"}}) - rootContext.Send(pid, &Message{protoMsg: protoMsg{state: "state5"}}) + rootContext.Send(pid, &Message{state: "state4"}) + rootContext.Send(pid, &Message{state: "state5"}) rootContext.PoisonFuture(pid).Wait() fmt.Printf("*** restart ***\n") @@ -260,11 +402,11 @@ create table if not exists event_journals _, err = pool.Exec(context2.Background(), ` create table if not exists snapshots ( - actor_name varchar(255) not null, - snapshot_index bigint not null, - snapshot json not null, - message_type varchar(255) not null, - primary key (id) + actor_name varchar(255) not null, + snapshot_index bigint not null, + snapshot json not null, + message_type varchar(255) not null, + constraint snapshots_pk primary key (actor_name) ); `) return err diff --git a/go.mod b/go.mod index 6259660e2..9d07356fb 100644 --- a/go.mod +++ b/go.mod @@ -32,19 +32,25 @@ require ( github.com/twmb/murmur3 v1.1.8 go.etcd.io/etcd/client/v3 v3.5.10 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa - golang.org/x/sync v0.5.0 + golang.org/x/sync v0.10.0 k8s.io/api v0.28.4 k8s.io/apimachinery v0.28.4 k8s.io/client-go v0.28.4 ) -require github.com/lmittmann/tint v1.0.3 +require ( + github.com/jackc/pgx/v5 v5.7.2 + github.com/lmittmann/tint v1.0.3 +) require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect ) @@ -100,11 +106,11 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.21.0 // indirect - golang.org/x/crypto v0.22.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/term v0.19.0 // indirect - golang.org/x/text v0.14.0 + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect diff --git a/go.sum b/go.sum index cd4625cf7..3de1a5e89 100644 --- a/go.sum +++ b/go.sum @@ -143,6 +143,14 @@ github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI= +github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -323,8 +331,8 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -355,8 +363,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -384,20 +392,20 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -410,8 +418,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8= -golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/persistence/postgresql/ddl/snapshots.sql b/persistence/postgresql/ddl/snapshots.sql index df3ae037f..1866bce0a 100644 --- a/persistence/postgresql/ddl/snapshots.sql +++ b/persistence/postgresql/ddl/snapshots.sql @@ -4,5 +4,5 @@ create table if not exists snapshots snapshot_index bigint not null, snapshot json not null, message_type varchar(255) not null, - primary key (actor_name) + constraint snapshots_pk primary key (actor_name) ); diff --git a/persistence/postgresql/provider_state.go b/persistence/postgresql/provider_state.go index e2a006506..113a49fc2 100644 --- a/persistence/postgresql/provider_state.go +++ b/persistence/postgresql/provider_state.go @@ -27,7 +27,8 @@ func NewProviderState(conn *pgxpool.Pool, logger *slog.Logger) *ProviderState { func (s *ProviderState) GetSnapshot(actorName string) (snapshot interface{}, eventIndex int, ok bool) { rows, err := s.connPool.Query(context2.Background(), - "SELECT snapshot, message_type, snapshot_index FROM snapshots WHERE actor_name = $1 ORDER BY snapshot_index DESC LIMIT 1") + "SELECT actor_name, snapshot, message_type, snapshot_index FROM snapshots WHERE actor_name = $1 ORDER BY snapshot_index DESC LIMIT 1", actorName, + ) if err != nil { s.logger.Error("Error getting snapshot", slog.Any("error", err)) return nil, 0, false @@ -57,7 +58,11 @@ func (s *ProviderState) PersistSnapshot(actorName string, snapshotIndex int, sna return } _, err = s.connPool.Exec(context2.Background(), - "INSERT INTO snapshots (id, snapshot, message_type, snapshot_index) VALUES ($1, $2, $3, $4)", + ` +INSERT INTO snapshots (actor_name, snapshot, message_type, snapshot_index) + VALUES ($1, $2, $3, $4) + ON CONFLICT ON CONSTRAINT snapshots_pk DO UPDATE SET snapshot=$2, message_type=$3, snapshot_index=$4; +`, actorName, bytes, proto.MessageName(snapshot), snapshotIndex) if err != nil { s.logger.Error("Error persisting snapshot", slog.Any("error", err)) @@ -75,7 +80,7 @@ func (s *ProviderState) DeleteSnapshots(actorName string, inclusiveToIndex int) func (s *ProviderState) GetEvents(actorName string, eventIndexStart int, eventIndexEnd int, callback func(e interface{})) { rows, err := s.connPool.Query(context2.Background(), - "SELECT event, message_type, event_index FROM event_journals WHERE actor_name = $1 AND event_index >= $2 AND event_index <= $3", + "SELECT actor_name, event, message_type, event_index FROM event_journals WHERE actor_name = $1 AND event_index >= $2 AND event_index <= $3", actorName, eventIndexStart, eventIndexEnd) if err != nil { s.logger.Error("Error getting events", slog.Any("error", err)) @@ -133,17 +138,17 @@ func (s *ProviderState) GetSnapshotInterval() int { var _ persistence.ProviderState = &ProviderState{} type snapshotsTableRow struct { - ActorName string - Snapshot []byte - SnapshotIndex int - MessageType string + ActorName string `db:"actor_name"` + Snapshot []byte `db:"snapshot"` + SnapshotIndex int `db:"snapshot_index"` + MessageType string `db:"message_type"` } type journalEventsTableRow struct { - ActorName string - Event []byte - EventIndex int - MessageType string + ActorName string `db:"actor_name"` + Event []byte `db:"event"` + EventIndex int `db:"event_index"` + MessageType string `db:"message_type"` } func unmarshalProtoMessage(bytes []byte, messageTypeName string) (proto.Message, error) { From c3fd30a9cde7b6b671ecdf0b6cc814e65a1c4658 Mon Sep 17 00:00:00 2001 From: "ryota.suzuki" Date: Sat, 11 Jan 2025 23:08:28 +0900 Subject: [PATCH 04/12] fix provider input --- examples/persistence-psql/main.go | 3 ++- persistence/postgresql/provider.go | 16 ++++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/examples/persistence-psql/main.go b/examples/persistence-psql/main.go index e96b937b5..076011d0a 100644 --- a/examples/persistence-psql/main.go +++ b/examples/persistence-psql/main.go @@ -362,8 +362,9 @@ func main() { log.Printf("failed to parse connection string: %s", err) return } + pool, err := pgxpool.NewWithConfig(ctx, config) system := actor.NewActorSystem() - provider := postgresql.NewProvider(config, system) + provider := postgresql.NewProvider(pool, system) rootContext := system.Root props := actor.PropsFromProducer(func() actor.Actor { return &Actor{} }, diff --git a/persistence/postgresql/provider.go b/persistence/postgresql/provider.go index ac5ea78e3..9b63591e1 100644 --- a/persistence/postgresql/provider.go +++ b/persistence/postgresql/provider.go @@ -1,30 +1,22 @@ package postgresql import ( - context2 "context" "github.com/asynkron/protoactor-go/actor" "github.com/asynkron/protoactor-go/persistence" "github.com/jackc/pgx/v5/pgxpool" - "log/slog" ) type Provider struct { - connConfig *pgxpool.Config + pool *pgxpool.Pool actorSystem *actor.ActorSystem } -func NewProvider(connConfig *pgxpool.Config, actorSystem *actor.ActorSystem) *Provider { - return &Provider{connConfig: connConfig, actorSystem: actorSystem} +func NewProvider(pool *pgxpool.Pool, actorSystem *actor.ActorSystem) *Provider { + return &Provider{pool: pool, actorSystem: actorSystem} } func (p *Provider) GetState() persistence.ProviderState { - pool, err := pgxpool.NewWithConfig(context2.Background(), p.connConfig) - if err != nil { - p.actorSystem.Logger().Error("failed to create pgx pool", slog.Any("error", err)) - panic(err) - } - - return NewProviderState(pool, p.actorSystem.Logger()) + return NewProviderState(p.pool, p.actorSystem.Logger()) } var _ persistence.Provider = &Provider{} From cdee12650d8f25cb035fd36a84129c25d48d8146 Mon Sep 17 00:00:00 2001 From: "ryota.suzuki" Date: Sun, 12 Jan 2025 16:05:36 +0900 Subject: [PATCH 05/12] impl IdentityStorageLookup --- cluster/identity_storage_lookup.go | 59 +++++++++++++++++---- cluster/identity_storage_placement_actor.go | 1 + cluster/identity_storage_worker.go | 18 ++++++- 3 files changed, 68 insertions(+), 10 deletions(-) create mode 100644 cluster/identity_storage_placement_actor.go diff --git a/cluster/identity_storage_lookup.go b/cluster/identity_storage_lookup.go index 19e429918..bec4be4cc 100644 --- a/cluster/identity_storage_lookup.go +++ b/cluster/identity_storage_lookup.go @@ -8,6 +8,7 @@ import ( const ( placementActorName = "placement-activator" + workerActorName = "identity-storage-worker" pidClusterIdentityStartIndex = len(placementActorName) + 1 ) @@ -18,10 +19,12 @@ type IdentityStorageLookup struct { isClient bool placementActor *actor.PID system *actor.ActorSystem - router *actor.PID + worker *actor.PID memberID string } +var _ IdentityLookup = (*IdentityStorageLookup)(nil) + func newIdentityStorageLookup(storage StorageLookup) *IdentityStorageLookup { this := &IdentityStorageLookup{ Storage: storage, @@ -44,22 +47,60 @@ func RemotePlacementActor(address string) *actor.PID { // // Get returns a PID for a given ClusterIdentity -func (id *IdentityStorageLookup) Get(clusterIdentity *ClusterIdentity) *actor.PID { +func (i *IdentityStorageLookup) Get(clusterIdentity *ClusterIdentity) *actor.PID { msg := newGetPid(clusterIdentity) timeout := 5 * time.Second - res, _ := id.system.Root.RequestFuture(id.router, msg, timeout).Result() + res, _ := i.system.Root.RequestFuture(i.worker, msg, timeout).Result() response := res.(*actor.Future) return response.PID() } -func (id *IdentityStorageLookup) Setup(cluster *Cluster, kinds []string, isClient bool) { - id.cluster = cluster - id.system = cluster.ActorSystem - id.memberID = cluster.ActorSystem.ID +func (i *IdentityStorageLookup) RemovePid(clusterIdentity *ClusterIdentity, pid *actor.PID) { + /// workerにremovePidを送る + if i.system.IsStopped() { + return + } + + i.Storage.RemoveActivation(newSpawnLock(pid.String(), clusterIdentity)) +} + +func (i *IdentityStorageLookup) Shutdown() { + i.system.Root.Stop(i.worker) + if !i.isClient { + i.system.Root.Stop(i.placementActor) + } + + i.RemoveMember(i.memberID) + /// workerもshutdownする +} + +func (i *IdentityStorageLookup) Setup(cluster *Cluster, kinds []string, isClient bool) { + i.cluster = cluster + i.system = cluster.ActorSystem + i.memberID = cluster.ActorSystem.ID - // workerProps := actor.PropsFromProducer(func() actor.Actor { return newIdentityStorageWorker(identity) }) + workerProps := actor.PropsFromProducer(func() actor.Actor { return newIdentityStorageWorker(i) }) + var err error + i.worker, err = i.system.Root.SpawnNamed(workerProps, workerActorName) + if err != nil { + panic(err) + } + i.cluster.ActorSystem.EventStream.SubscribeWithPredicate(func(message interface{}) { + ct := message.(*ClusterTopology) + for _, member := range ct.Left { + i.RemoveMember(member.Id) + } + }, func(m interface{}) bool { + _, ok := m.(*ClusterTopology) + return ok + }) + + if i.isClient { + return + } - // routerProps := identity.system.Root.(workerProps, 50); + //var props = Props.FromProducer(() => new IdentityStoragePlacementActor(Cluster, this)); + //_placementActor = _system.Root.SpawnNamedSystem(props, PlacementActorName); } diff --git a/cluster/identity_storage_placement_actor.go b/cluster/identity_storage_placement_actor.go new file mode 100644 index 000000000..916b1b53b --- /dev/null +++ b/cluster/identity_storage_placement_actor.go @@ -0,0 +1 @@ +package cluster diff --git a/cluster/identity_storage_worker.go b/cluster/identity_storage_worker.go index ddc2fa85a..099c0389a 100644 --- a/cluster/identity_storage_worker.go +++ b/cluster/identity_storage_worker.go @@ -1,6 +1,8 @@ package cluster import ( + "encoding/json" + "log" "github.com/asynkron/protoactor-go/actor" @@ -42,6 +44,20 @@ func (ids *IdentityStorageWorker) Receive(c actor.Context) { c.Respond(newPidResult(existing)) } + activation := ids.storage.TryGetExistingActivation(getPid.ClusterIdentity) + if activation != nil { + pid := &actor.PID{} + go func() { + err := json.Unmarshal([]byte(activation.Pid), pid) + if err != nil { + panic(err) + } + ids.cluster.PidCache.Set(getPid.ClusterIdentity.Identity, getPid.ClusterIdentity.Kind, pid) + }() + + c.Respond(newPidResult(pid)) + return + } + return - // continue } From 80fb535082746c344e18cd3c0de4076847a0184f Mon Sep 17 00:00:00 2001 From: "ryota.suzuki" Date: Sun, 12 Jan 2025 20:35:47 +0900 Subject: [PATCH 06/12] impl identity storage lookup --- cluster/identity_storage_lookup.go | 14 +-- cluster/identity_storage_placement_actor.go | 100 ++++++++++++++++++++ 2 files changed, 108 insertions(+), 6 deletions(-) diff --git a/cluster/identity_storage_lookup.go b/cluster/identity_storage_lookup.go index bec4be4cc..e21af5083 100644 --- a/cluster/identity_storage_lookup.go +++ b/cluster/identity_storage_lookup.go @@ -25,7 +25,7 @@ type IdentityStorageLookup struct { var _ IdentityLookup = (*IdentityStorageLookup)(nil) -func newIdentityStorageLookup(storage StorageLookup) *IdentityStorageLookup { +func NewIdentityStorageLookup(storage StorageLookup) *IdentityStorageLookup { this := &IdentityStorageLookup{ Storage: storage, } @@ -58,7 +58,6 @@ func (i *IdentityStorageLookup) Get(clusterIdentity *ClusterIdentity) *actor.PID } func (i *IdentityStorageLookup) RemovePid(clusterIdentity *ClusterIdentity, pid *actor.PID) { - /// workerにremovePidを送る if i.system.IsStopped() { return } @@ -73,13 +72,14 @@ func (i *IdentityStorageLookup) Shutdown() { } i.RemoveMember(i.memberID) - /// workerもshutdownする + i.system.Root.Stop(i.worker) } -func (i *IdentityStorageLookup) Setup(cluster *Cluster, kinds []string, isClient bool) { +func (i *IdentityStorageLookup) Setup(cluster *Cluster, _ []string, isClient bool) { i.cluster = cluster i.system = cluster.ActorSystem i.memberID = cluster.ActorSystem.ID + i.isClient = isClient workerProps := actor.PropsFromProducer(func() actor.Actor { return newIdentityStorageWorker(i) }) var err error @@ -101,6 +101,8 @@ func (i *IdentityStorageLookup) Setup(cluster *Cluster, kinds []string, isClient return } - //var props = Props.FromProducer(() => new IdentityStoragePlacementActor(Cluster, this)); - //_placementActor = _system.Root.SpawnNamedSystem(props, PlacementActorName); + i.placementActor, err = i.system.Root.SpawnNamed(actor.PropsFromProducer(func() actor.Actor { return newIdentityStoragePlacementActor(cluster, i) }), placementActorName) + if err != nil { + panic(err) + } } diff --git a/cluster/identity_storage_placement_actor.go b/cluster/identity_storage_placement_actor.go index 916b1b53b..d6bcf8f8e 100644 --- a/cluster/identity_storage_placement_actor.go +++ b/cluster/identity_storage_placement_actor.go @@ -1 +1,101 @@ package cluster + +import ( + "github.com/asynkron/protoactor-go/actor" + "github.com/asynkron/protoactor-go/eventstream" + "log/slog" +) + +type GrainMeta struct { + ID *ClusterIdentity + PID *actor.PID +} + +type IdentityStoragePlacementActor struct { + cluster *Cluster + identityStorageLookup *IdentityStorageLookup + subscription *eventstream.Subscription + actors map[string] /* clusterIdentity*/ GrainMeta +} + +func newIdentityStoragePlacementActor(cluster *Cluster, identityStorageLookup *IdentityStorageLookup) *IdentityStoragePlacementActor { + this := &IdentityStoragePlacementActor{ + cluster: cluster, + identityStorageLookup: identityStorageLookup, + } + return this +} + +func (i *IdentityStoragePlacementActor) Receive(context actor.Context) { + switch msg := context.Message().(type) { + case *actor.Started: + i.onStarted(context) + case *actor.Stopping: + i.onStopping(context) + case *ActivationTerminating: + i.onActivationTerminating(context, msg) + case *ActivationRequest: + i.onActivationRequest(context, msg) + } +} + +func (i *IdentityStoragePlacementActor) onStarted(context actor.Context) { + i.cluster.Logger().Info("IdentityStoragePlacementActor started") + i.subscription = i.cluster.ActorSystem.EventStream.Subscribe(func(evt interface{}) { + context.Send(context.Self(), evt) + }) +} + +func (i *IdentityStoragePlacementActor) onStopping(_ actor.Context) { + i.cluster.Logger().Info("IdentityStoragePlacementActor stopping") + i.subscription.Deactivate() + for _, meta := range i.actors { + /// TODO: need to throttle? + i.cluster.ActorSystem.Root.Stop(meta.PID) + } +} + +func (i *IdentityStoragePlacementActor) onActivationTerminating(_ actor.Context, msg *ActivationTerminating) { + if grainMeta, ok := i.actors[msg.ClusterIdentity.AsKey()]; ok { + if grainMeta.PID != msg.Pid { + i.cluster.Logger().Error("PID mismatch", slog.Any("clusterIdentity", msg.ClusterIdentity), slog.Any("expectedPid", grainMeta.PID), slog.Any("actualPid", msg.Pid)) + return + } + delete(i.actors, msg.ClusterIdentity.AsKey()) + i.cluster.PidCache.Remove(msg.ClusterIdentity.Identity, msg.ClusterIdentity.Kind) + i.identityStorageLookup.Storage.RemoveActivation(newSpawnLock(grainMeta.PID.String(), msg.ClusterIdentity)) + } else { + i.cluster.Logger().Error("activation not found", slog.Any("clusterIdentity", msg.ClusterIdentity)) + } +} + +func (i *IdentityStoragePlacementActor) onActivationRequest(context actor.Context, msg *ActivationRequest) { + if grainMeta, ok := i.actors[msg.ClusterIdentity.AsKey()]; ok { + context.Respond(&ActivationResponse{Pid: grainMeta.PID}) + return + } + + clusterKind := i.cluster.GetClusterKind(msg.ClusterIdentity.Kind) + if clusterKind == nil { + context.Logger().Error("Cluster kind not found", slog.Any("clusterIdentity", msg.ClusterIdentity), slog.String("kind", msg.ClusterIdentity.Kind)) + context.Respond(&ActivationResponse{Pid: nil, Failed: true}) + return + } + + i.spawn(msg, context, clusterKind) +} + +func (i *IdentityStoragePlacementActor) spawn(req *ActivationRequest, ctx actor.Context, kind *ActivatedKind) { + props := WithClusterIdentity(kind.Props, req.ClusterIdentity) + pid := ctx.SpawnPrefix(props, req.ClusterIdentity.Identity) + kind.Inc() + + /// TODO: member selectionを考慮 + i.identityStorageLookup.Storage.StoreActivation(i.cluster.ActorSystem.ID, newSpawnLock(pid.String(), req.ClusterIdentity), pid) + i.actors[req.ClusterIdentity.AsKey()] = GrainMeta{ + ID: req.ClusterIdentity, + PID: pid, + } + i.cluster.PidCache.Set(req.ClusterIdentity.Identity, req.ClusterIdentity.Kind, pid) + ctx.Respond(&ActivationResponse{Pid: pid}) +} From 59ac8a2867286531712dee361f9a3e1ee3748197 Mon Sep 17 00:00:00 2001 From: "ryota.suzuki" Date: Sun, 12 Jan 2025 20:40:11 +0900 Subject: [PATCH 07/12] prepare example --- .../Makefile | 11 + .../docker-compose.yml | 48 +++++ .../node1/main.go | 39 ++++ .../node2/main.go | 46 ++++ .../shared/build.sh | 3 + .../shared/lookup.go | 7 + .../shared/protos.pb.go | 197 ++++++++++++++++++ .../shared/protos.proto | 11 + .../shared/protos_grain.pb.go | 7 + 9 files changed, 369 insertions(+) create mode 100644 examples/cluster-identitylookup-postgresql/Makefile create mode 100644 examples/cluster-identitylookup-postgresql/docker-compose.yml create mode 100644 examples/cluster-identitylookup-postgresql/node1/main.go create mode 100644 examples/cluster-identitylookup-postgresql/node2/main.go create mode 100755 examples/cluster-identitylookup-postgresql/shared/build.sh create mode 100644 examples/cluster-identitylookup-postgresql/shared/lookup.go create mode 100644 examples/cluster-identitylookup-postgresql/shared/protos.pb.go create mode 100644 examples/cluster-identitylookup-postgresql/shared/protos.proto create mode 100644 examples/cluster-identitylookup-postgresql/shared/protos_grain.pb.go diff --git a/examples/cluster-identitylookup-postgresql/Makefile b/examples/cluster-identitylookup-postgresql/Makefile new file mode 100644 index 000000000..c4e2ab5f2 --- /dev/null +++ b/examples/cluster-identitylookup-postgresql/Makefile @@ -0,0 +1,11 @@ +start: + tmux new-session -d -s eg + tmux split-window -t "eg:0" -v + tmux send-keys -t "eg:0.0" "go run node1/main.go" Enter + tmux send-keys -t "eg:0.1" "go run node2/main.go" Enter + tmux attach -t eg + tmux kill-session -t eg + + +stop: + tmux kill-session -t eg diff --git a/examples/cluster-identitylookup-postgresql/docker-compose.yml b/examples/cluster-identitylookup-postgresql/docker-compose.yml new file mode 100644 index 000000000..ce1becd1c --- /dev/null +++ b/examples/cluster-identitylookup-postgresql/docker-compose.yml @@ -0,0 +1,48 @@ +version: '3.7' +services: + + consul-agent-1: &consul-agent + image: hashicorp/consul:latest + networks: + - consul + command: "agent -retry-join consul-server-bootstrap -client 0.0.0.0" + + consul-agent-2: + <<: *consul-agent + + consul-agent-3: + <<: *consul-agent + + consul-server-1: &consul-server + <<: *consul-agent + command: "agent -server -retry-join consul-server-bootstrap -client 0.0.0.0" + + consul-server-2: + <<: *consul-server + + consul-server-bootstrap: + <<: *consul-agent + ports: + - "8400:8400" + - "8500:8500" + - "8600:8600" + - "8600:8600/udp" + command: "agent -server -bootstrap-expect 3 -ui -client 0.0.0.0" + + mongodb: + image: mongo:latest + ports: + - 127.0.0.1:27017:27017 + volumes: + - mongodb_data:/data/db + + redis: + image: redis:latest + ports: + - 127.0.0.1:6379:6379 + +networks: + consul: + +volumes: + mongodb_data: \ No newline at end of file diff --git a/examples/cluster-identitylookup-postgresql/node1/main.go b/examples/cluster-identitylookup-postgresql/node1/main.go new file mode 100644 index 000000000..8a507cde2 --- /dev/null +++ b/examples/cluster-identitylookup-postgresql/node1/main.go @@ -0,0 +1,39 @@ +package main + +import ( + "cluster-identitylookup-postgresql/shared" + "fmt" + console "github.com/asynkron/goconsole" + "github.com/asynkron/protoactor-go/actor" + "github.com/asynkron/protoactor-go/cluster" + "github.com/asynkron/protoactor-go/cluster/clusterproviders/consul" + "github.com/asynkron/protoactor-go/remote" +) + +func main() { + c := startNode() + + fmt.Print("\nBoot other nodes and press Enter\n") + console.ReadLine() + pid := c.Get("abc", "hello") + fmt.Printf("Got pid %v", pid) + res, _ := c.Request("abc", "hello", &shared.HelloRequest{Name: "Roger"}) + fmt.Printf("Got response %v", res) + + fmt.Println() + console.ReadLine() + c.Shutdown(true) +} + +func startNode() *cluster.Cluster { + system := actor.NewActorSystem() + + provider, _ := consul.New() + lookup := shared.NewLockUp() + config := remote.Configure("localhost", 0) + clusterConfig := cluster.Configure("my-cluster", provider, lookup, config) + c := cluster.New(system, clusterConfig) + c.StartMember() + + return c +} diff --git a/examples/cluster-identitylookup-postgresql/node2/main.go b/examples/cluster-identitylookup-postgresql/node2/main.go new file mode 100644 index 000000000..a6404eaaa --- /dev/null +++ b/examples/cluster-identitylookup-postgresql/node2/main.go @@ -0,0 +1,46 @@ +package main + +import ( + "fmt" + + "cluster-identitylookup-postgresql/shared" + + console "github.com/asynkron/goconsole" + "github.com/asynkron/protoactor-go/actor" + "github.com/asynkron/protoactor-go/cluster" + "github.com/asynkron/protoactor-go/cluster/clusterproviders/consul" + "github.com/asynkron/protoactor-go/remote" +) + +func main() { + cluster := startNode() + + fmt.Print("\nBoot other nodes and press Enter\n") + console.ReadLine() + + cluster.Shutdown(true) +} + +func startNode() *cluster.Cluster { + system := actor.NewActorSystem() + + provider, _ := consul.New() + lookup := shared.NewLockUp() + config := remote.Configure("localhost", 0) + + props := actor.PropsFromFunc(func(ctx actor.Context) { + switch msg := ctx.Message().(type) { + case *actor.Started: + fmt.Printf("Started %v", msg) + case *shared.HelloRequest: + fmt.Printf("Hello %v\n", msg.Name) + ctx.Respond(&shared.HelloResponse{}) + } + }) + helloKind := cluster.NewKind("hello", props) + clusterConfig := cluster.Configure("my-cluster", provider, lookup, config, cluster.WithKinds(helloKind)) + c := cluster.New(system, clusterConfig) + + c.StartMember() + return c +} diff --git a/examples/cluster-identitylookup-postgresql/shared/build.sh b/examples/cluster-identitylookup-postgresql/shared/build.sh new file mode 100755 index 000000000..6fd6845ec --- /dev/null +++ b/examples/cluster-identitylookup-postgresql/shared/build.sh @@ -0,0 +1,3 @@ +protoc --go_out=. --go_opt=paths=source_relative \ + --plugin=protoc-gen-go-grain=../../../protobuf/protoc-gen-go-grain/protoc-gen-go-grain.sh --go-grain_out=. --go-grain_opt=paths=source_relative \ + -I../../ -I. protos.proto diff --git a/examples/cluster-identitylookup-postgresql/shared/lookup.go b/examples/cluster-identitylookup-postgresql/shared/lookup.go new file mode 100644 index 000000000..f31a0d36b --- /dev/null +++ b/examples/cluster-identitylookup-postgresql/shared/lookup.go @@ -0,0 +1,7 @@ +package shared + +import "github.com/asynkron/protoactor-go/cluster" + +func NewLockUp() cluster.IdentityLookup { + return cluster.NewIdentityStorageLookup(nil) +} diff --git a/examples/cluster-identitylookup-postgresql/shared/protos.pb.go b/examples/cluster-identitylookup-postgresql/shared/protos.pb.go new file mode 100644 index 000000000..c7993c4d2 --- /dev/null +++ b/examples/cluster-identitylookup-postgresql/shared/protos.pb.go @@ -0,0 +1,197 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.27.1 +// protoc v4.25.3 +// source: protos.proto + +package shared + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type HelloRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` +} + +func (x *HelloRequest) Reset() { + *x = HelloRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_protos_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HelloRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HelloRequest) ProtoMessage() {} + +func (x *HelloRequest) ProtoReflect() protoreflect.Message { + mi := &file_protos_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HelloRequest.ProtoReflect.Descriptor instead. +func (*HelloRequest) Descriptor() ([]byte, []int) { + return file_protos_proto_rawDescGZIP(), []int{0} +} + +func (x *HelloRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +type HelloResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *HelloResponse) Reset() { + *x = HelloResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_protos_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HelloResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HelloResponse) ProtoMessage() {} + +func (x *HelloResponse) ProtoReflect() protoreflect.Message { + mi := &file_protos_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HelloResponse.ProtoReflect.Descriptor instead. +func (*HelloResponse) Descriptor() ([]byte, []int) { + return file_protos_proto_rawDescGZIP(), []int{1} +} + +var File_protos_proto protoreflect.FileDescriptor + +var file_protos_proto_rawDesc = []byte{ + 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, + 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x22, 0x22, 0x0a, 0x0c, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x0f, 0x0a, 0x0d, 0x48, 0x65, + 0x6c, 0x6c, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x41, 0x5a, 0x3f, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x73, 0x79, 0x6e, 0x6b, 0x72, + 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x2d, 0x67, 0x6f, + 0x2f, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x2d, 0x62, 0x61, 0x73, 0x69, 0x63, 0x2f, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_protos_proto_rawDescOnce sync.Once + file_protos_proto_rawDescData = file_protos_proto_rawDesc +) + +func file_protos_proto_rawDescGZIP() []byte { + file_protos_proto_rawDescOnce.Do(func() { + file_protos_proto_rawDescData = protoimpl.X.CompressGZIP(file_protos_proto_rawDescData) + }) + return file_protos_proto_rawDescData +} + +var file_protos_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_protos_proto_goTypes = []interface{}{ + (*HelloRequest)(nil), // 0: shared.HelloRequest + (*HelloResponse)(nil), // 1: shared.HelloResponse +} +var file_protos_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_protos_proto_init() } +func file_protos_proto_init() { + if File_protos_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_protos_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HelloRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_protos_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HelloResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_protos_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_protos_proto_goTypes, + DependencyIndexes: file_protos_proto_depIdxs, + MessageInfos: file_protos_proto_msgTypes, + }.Build() + File_protos_proto = out.File + file_protos_proto_rawDesc = nil + file_protos_proto_goTypes = nil + file_protos_proto_depIdxs = nil +} diff --git a/examples/cluster-identitylookup-postgresql/shared/protos.proto b/examples/cluster-identitylookup-postgresql/shared/protos.proto new file mode 100644 index 000000000..437686ef4 --- /dev/null +++ b/examples/cluster-identitylookup-postgresql/shared/protos.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; +package shared; +option go_package = "github.com/asynkron/protoactor-go/examples/cluster-basic/shared"; + +message HelloRequest { + string name = 1; +} + +message HelloResponse { + +} \ No newline at end of file diff --git a/examples/cluster-identitylookup-postgresql/shared/protos_grain.pb.go b/examples/cluster-identitylookup-postgresql/shared/protos_grain.pb.go new file mode 100644 index 000000000..68e231ffa --- /dev/null +++ b/examples/cluster-identitylookup-postgresql/shared/protos_grain.pb.go @@ -0,0 +1,7 @@ +// Code generated by protoc-gen-grain. DO NOT EDIT. +// versions: +// protoc-gen-grain v0.1.0 +// protoc v4.24.3 +// source: protos.proto + +package shared From 6900f82d34443a1d54311349a6b6a905056a725a Mon Sep 17 00:00:00 2001 From: "ryota.suzuki" Date: Sun, 12 Jan 2025 21:14:59 +0900 Subject: [PATCH 08/12] impl identity storage worker --- cluster/identity_storage_worker.go | 74 +++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/cluster/identity_storage_worker.go b/cluster/identity_storage_worker.go index 099c0389a..baa8456eb 100644 --- a/cluster/identity_storage_worker.go +++ b/cluster/identity_storage_worker.go @@ -2,8 +2,9 @@ package cluster import ( "encoding/json" - "log" + "log/slog" + "time" "github.com/asynkron/protoactor-go/actor" ) @@ -47,17 +48,78 @@ func (ids *IdentityStorageWorker) Receive(c actor.Context) { activation := ids.storage.TryGetExistingActivation(getPid.ClusterIdentity) if activation != nil { pid := &actor.PID{} - go func() { + err := json.Unmarshal([]byte(activation.Pid), pid) + if err != nil { + panic(err) + } + if ids.cluster.MemberList.ContainsMemberID(activation.MemberID) { + ids.cluster.PidCache.Set(getPid.ClusterIdentity.Identity, getPid.ClusterIdentity.Kind, pid) + c.Respond(newPidResult(pid)) + return + } else { + /// TODO: 1 memberID につき一度だけ発生させる + c.Logger().Info("MemberID not found in MemberList, removing activation") + ids.storage.RemoveMemberId(activation.MemberID) + } + } + + activator := ids.cluster.MemberList.GetActivatorMember(getPid.ClusterIdentity.Identity, getPid.ClusterIdentity.Kind) + + if activator == "" { + c.Respond(newPidResult(nil)) + return + } + + lock := ids.storage.TryAcquireLock(getPid.ClusterIdentity) + if lock == nil { + activation := ids.storage.WaitForActivation(getPid.ClusterIdentity) + if activation != nil { + pid := &actor.PID{} err := json.Unmarshal([]byte(activation.Pid), pid) if err != nil { panic(err) } - ids.cluster.PidCache.Set(getPid.ClusterIdentity.Identity, getPid.ClusterIdentity.Kind, pid) - }() - - c.Respond(newPidResult(pid)) + if ids.cluster.MemberList.ContainsMemberID(activation.MemberID) { + ids.cluster.PidCache.Set(getPid.ClusterIdentity.Identity, getPid.ClusterIdentity.Kind, pid) + c.Respond(newPidResult(pid)) + return + } else { + /// TODO: 1 memberID につき一度だけ発生させる + c.Logger().Info("MemberID not found in MemberList, removing activation") + ids.storage.RemoveMemberId(activation.MemberID) + } + } + c.Respond(newPidResult(nil)) return } + defer func() { + ids.storage.RemoveLock(*lock) + }() + pid := ids.SpawnActivation(activator, *lock) + c.Respond(newPidResult(pid)) return } + +func (ids *IdentityStorageWorker) SpawnActivation(activatorAddress string, lock SpawnLock) *actor.PID { + ids.cluster.Logger().Info("Spawning activation", slog.Any("spawnTo", activatorAddress), slog.Any("clusterIdentity", lock.ClusterIdentity)) + remotePid := RemotePlacementActor(activatorAddress) + activateRequest := &ActivationRequest{ + ClusterIdentity: lock.ClusterIdentity, + RequestId: lock.LockID, + } + + activationResult, err := ids.cluster.ActorSystem.Root.RequestFuture(remotePid, activateRequest, 5*time.Second).Result() + if err != nil { + ids.cluster.Logger().Error("Failed to activate", slog.Any("error", err)) + return nil + } + + activationResponse := activationResult.(*ActivationResponse) + if activationResponse.Failed { + ids.cluster.Logger().Error("Failed to activate") + return nil + } + ids.cluster.PidCache.Set(lock.ClusterIdentity.Identity, lock.ClusterIdentity.Kind, activationResponse.Pid) + return activationResponse.Pid +} From 1cb597077b1f2041e1a0aec14069334ec083788a Mon Sep 17 00:00:00 2001 From: "ryota.suzuki" Date: Mon, 13 Jan 2025 01:44:04 +0900 Subject: [PATCH 09/12] impl example --- cluster/identity_storage_lookup.go | 15 +- cluster/identity_storage_placement_actor.go | 5 +- cluster/identity_storage_worker.go | 11 +- .../postgresql/activation_waiting_actor.go | 152 +++++++++++++++++ .../postgresql/ddl/activations.sql | 15 ++ .../postgresql/ddl/spawn_locks.sql | 9 + .../postgresql/storage_lookup.go | 156 ++++++++++++++++++ .../node1/main.go | 13 +- .../node2/main.go | 32 +++- .../shared/lookup.go | 101 +++++++++++- 10 files changed, 479 insertions(+), 30 deletions(-) create mode 100644 cluster/identitylookup/postgresql/activation_waiting_actor.go create mode 100644 cluster/identitylookup/postgresql/ddl/activations.sql create mode 100644 cluster/identitylookup/postgresql/ddl/spawn_locks.sql create mode 100644 cluster/identitylookup/postgresql/storage_lookup.go diff --git a/cluster/identity_storage_lookup.go b/cluster/identity_storage_lookup.go index e21af5083..624055e1d 100644 --- a/cluster/identity_storage_lookup.go +++ b/cluster/identity_storage_lookup.go @@ -1,6 +1,8 @@ package cluster import ( + "github.com/lithammer/shortuuid/v4" + "log/slog" "time" "github.com/asynkron/protoactor-go/actor" @@ -51,10 +53,13 @@ func (i *IdentityStorageLookup) Get(clusterIdentity *ClusterIdentity) *actor.PID msg := newGetPid(clusterIdentity) timeout := 5 * time.Second - res, _ := i.system.Root.RequestFuture(i.worker, msg, timeout).Result() - response := res.(*actor.Future) - - return response.PID() + res, err := i.system.Root.RequestFuture(i.worker, msg, timeout).Result() + if err != nil { + i.system.Logger().Error("Failed to get pid", slog.Any("error", err)) + return nil + } + response := res.(*PidResult) + return response.Pid } func (i *IdentityStorageLookup) RemovePid(clusterIdentity *ClusterIdentity, pid *actor.PID) { @@ -62,7 +67,7 @@ func (i *IdentityStorageLookup) RemovePid(clusterIdentity *ClusterIdentity, pid return } - i.Storage.RemoveActivation(newSpawnLock(pid.String(), clusterIdentity)) + i.Storage.RemoveActivation(newSpawnLock(shortuuid.New(), clusterIdentity)) } func (i *IdentityStorageLookup) Shutdown() { diff --git a/cluster/identity_storage_placement_actor.go b/cluster/identity_storage_placement_actor.go index d6bcf8f8e..c4b46a9df 100644 --- a/cluster/identity_storage_placement_actor.go +++ b/cluster/identity_storage_placement_actor.go @@ -22,6 +22,7 @@ func newIdentityStoragePlacementActor(cluster *Cluster, identityStorageLookup *I this := &IdentityStoragePlacementActor{ cluster: cluster, identityStorageLookup: identityStorageLookup, + actors: make(map[string] /* clusterIdentity*/ GrainMeta), } return this } @@ -65,7 +66,7 @@ func (i *IdentityStoragePlacementActor) onActivationTerminating(_ actor.Context, i.cluster.PidCache.Remove(msg.ClusterIdentity.Identity, msg.ClusterIdentity.Kind) i.identityStorageLookup.Storage.RemoveActivation(newSpawnLock(grainMeta.PID.String(), msg.ClusterIdentity)) } else { - i.cluster.Logger().Error("activation not found", slog.Any("clusterIdentity", msg.ClusterIdentity)) + i.cluster.Logger().Error("IdentityStoragePlacementActor#onActivationTerminating activation not found", slog.Any("clusterIdentity", msg.ClusterIdentity)) } } @@ -91,7 +92,7 @@ func (i *IdentityStoragePlacementActor) spawn(req *ActivationRequest, ctx actor. kind.Inc() /// TODO: member selectionを考慮 - i.identityStorageLookup.Storage.StoreActivation(i.cluster.ActorSystem.ID, newSpawnLock(pid.String(), req.ClusterIdentity), pid) + i.identityStorageLookup.Storage.StoreActivation(i.cluster.ActorSystem.ID, newSpawnLock(req.RequestId, req.ClusterIdentity), pid) i.actors[req.ClusterIdentity.AsKey()] = GrainMeta{ ID: req.ClusterIdentity, PID: pid, diff --git a/cluster/identity_storage_worker.go b/cluster/identity_storage_worker.go index baa8456eb..e90a92cbb 100644 --- a/cluster/identity_storage_worker.go +++ b/cluster/identity_storage_worker.go @@ -2,6 +2,7 @@ package cluster import ( "encoding/json" + "fmt" "log" "log/slog" "time" @@ -27,7 +28,7 @@ func newIdentityStorageWorker(storageLookup *IdentityStorageLookup) *IdentitySto // Receive func func (ids *IdentityStorageWorker) Receive(c actor.Context) { m := c.Message() - getPid, ok := m.(GetPid) + getPid, ok := m.(*GetPid) if !ok { return @@ -41,7 +42,7 @@ func (ids *IdentityStorageWorker) Receive(c actor.Context) { existing, _ := ids.cluster.PidCache.Get(getPid.ClusterIdentity.Identity, getPid.ClusterIdentity.Kind) if existing != nil { - log.Printf("Found %s in pidcache", m.(GetPid).ClusterIdentity.ToShortString()) + log.Printf("Found %s in pidcache", m.(*GetPid).ClusterIdentity.ToShortString()) c.Respond(newPidResult(existing)) } @@ -50,7 +51,7 @@ func (ids *IdentityStorageWorker) Receive(c actor.Context) { pid := &actor.PID{} err := json.Unmarshal([]byte(activation.Pid), pid) if err != nil { - panic(err) + panic(fmt.Errorf("IdentityStorageWorker: Failed to unmarshal pid: %v", err)) } if ids.cluster.MemberList.ContainsMemberID(activation.MemberID) { ids.cluster.PidCache.Set(getPid.ClusterIdentity.Identity, getPid.ClusterIdentity.Kind, pid) @@ -63,7 +64,7 @@ func (ids *IdentityStorageWorker) Receive(c actor.Context) { } } - activator := ids.cluster.MemberList.GetActivatorMember(getPid.ClusterIdentity.Identity, getPid.ClusterIdentity.Kind) + activator := ids.cluster.MemberList.GetActivatorMember(getPid.ClusterIdentity.Kind, getPid.ClusterIdentity.Identity) if activator == "" { c.Respond(newPidResult(nil)) @@ -109,7 +110,7 @@ func (ids *IdentityStorageWorker) SpawnActivation(activatorAddress string, lock RequestId: lock.LockID, } - activationResult, err := ids.cluster.ActorSystem.Root.RequestFuture(remotePid, activateRequest, 5*time.Second).Result() + activationResult, err := ids.cluster.ActorSystem.Root.RequestFuture(remotePid, activateRequest, 10*time.Second).Result() if err != nil { ids.cluster.Logger().Error("Failed to activate", slog.Any("error", err)) return nil diff --git a/cluster/identitylookup/postgresql/activation_waiting_actor.go b/cluster/identitylookup/postgresql/activation_waiting_actor.go new file mode 100644 index 000000000..bee5745e7 --- /dev/null +++ b/cluster/identitylookup/postgresql/activation_waiting_actor.go @@ -0,0 +1,152 @@ +package postgresql + +import ( + context2 "context" + "github.com/asynkron/protoactor-go/actor" + "github.com/asynkron/protoactor-go/cluster" + "github.com/asynkron/protoactor-go/scheduler" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/lib/pq" + "iter" + "log/slog" + "slices" + "strings" + "time" +) + +type AddWaiting struct { + clusterIdentity *cluster.ClusterIdentity +} + +type Activated struct { + Pid string + MemberId string +} + +type ActivationCancelled struct { +} + +type ReplyTo struct { + pid *actor.PID +} + +type queryingTimeReached struct { +} + +func ClusterIdentityFromKey(key string) *cluster.ClusterIdentity { + slice := strings.Split(key, "/") + return cluster.NewClusterIdentity(slice[1], slice[0]) +} + +type ActivationWaitingActor struct { + pool *pgxpool.Pool + poolingQueryInterval time.Duration + waitList map[string] /*clusterIdentity#AsKey*/ []*ReplyTo + inflight []*cluster.ClusterIdentity + cancelScheduler scheduler.CancelFunc +} + +func newActivationWaitingActor(pool *pgxpool.Pool, poolingQueryInterval time.Duration) *ActivationWaitingActor { + this := &ActivationWaitingActor{ + pool: pool, + poolingQueryInterval: poolingQueryInterval, + waitList: make(map[string] /*clusterIdentity#AsKey*/ []*ReplyTo), + } + return this +} + +func (a *ActivationWaitingActor) Receive(context actor.Context) { + switch msg := context.Message().(type) { + case *AddWaiting: + a.onAddWaiting(context, msg) + case *actor.Started: + a.onStarted(context) + case *queryingTimeReached: + a.onQueryingTimeReached(context) + case *actor.Stopping: + a.onStopping(context) + } +} + +func (a *ActivationWaitingActor) onAddWaiting(context actor.Context, msg *AddWaiting) { + if _, ok := a.waitList[msg.clusterIdentity.AsKey()]; !ok { + a.waitList[msg.clusterIdentity.AsKey()] = []*ReplyTo{} + } + + a.waitList[msg.clusterIdentity.AsKey()] = append(a.waitList[msg.clusterIdentity.AsKey()], &ReplyTo{ + pid: context.Sender(), + }) +} + +func (a *ActivationWaitingActor) onStarted(context actor.Context) { + /// TODO: clusterの停止に伴ってすべての待ちリストへゴメンを送る + a.cancelScheduler = scheduler.NewTimerScheduler(context).SendRepeatedly(a.poolingQueryInterval, a.poolingQueryInterval, context.Self(), &queryingTimeReached{}) +} + +func (a *ActivationWaitingActor) onQueryingTimeReached(context actor.Context) { + if len(a.waitList) == 0 { + return + } + + if len(a.inflight) > 0 { + return + } + + clear(a.inflight) + for key := range a.waitList { + clusterIdentity := ClusterIdentityFromKey(key) + a.inflight = append(a.inflight, clusterIdentity) + } + + pull, _ := iter.Pull(slices.Chunk(a.inflight, 100)) + for { + next, hasNext := pull() + if !hasNext { + break + } + + var keys []string + for _, identity := range next { + keys = append(keys, identity.AsKey()) + } + + result, err := a.pool.Query(context2.Background(), ` +SELECT * FROM activations WHERE identity_key in ($1) +`, pq.Array(keys)) + if err != nil { + context.Logger().Error("error while querying activations", slog.Any("error", err)) + } + + activations, err := pgx.CollectRows(result, pgx.RowToStructByName[ActivationsTableRow]) + if err != nil { + context.Logger().Error("error while collecting activations", slog.Any("error", err)) + } + + for _, activation := range activations { + for _, replayTo := range a.waitList[activation.IdentityKey] { + + context.Send(replayTo.pid, &Activated{ + Pid: string(activation.Pid), + MemberId: activation.MemberId, + }) + } + delete(a.waitList, activation.IdentityKey) + } + + clear(a.inflight) + } + +} + +func (a *ActivationWaitingActor) onStopping(context actor.Context) { + if a.cancelScheduler != nil { + a.cancelScheduler() + } + + for _, replayTo := range a.waitList { + for _, reply := range replayTo { + context.Send(reply.pid, &ActivationCancelled{}) + } + } +} diff --git a/cluster/identitylookup/postgresql/ddl/activations.sql b/cluster/identitylookup/postgresql/ddl/activations.sql new file mode 100644 index 000000000..e38a45a20 --- /dev/null +++ b/cluster/identitylookup/postgresql/ddl/activations.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS activations +( + identity TEXT NOT NULL, + kind TEXT NOT NULL, + identity_key TEXT NOT NULL, + pid json NOT NULL, + member_id TEXT NOT NULL, + lock_id TEXT NOT NULL, + activated_at TIMESTAMP WITH TIME ZONE DEFAULT now(), + PRIMARY KEY (identity, kind), + FOREIGN KEY (lock_id) references spawn_locks (lock_id) + +); + +CREATE INDEX IF NOT EXISTS activations_member_idx ON activations (member_id); diff --git a/cluster/identitylookup/postgresql/ddl/spawn_locks.sql b/cluster/identitylookup/postgresql/ddl/spawn_locks.sql new file mode 100644 index 000000000..fec24bf3f --- /dev/null +++ b/cluster/identitylookup/postgresql/ddl/spawn_locks.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS spawn_locks +( + lock_id TEXT NOT NULL, + identity TEXT NOT NULL, + kind TEXT NOT NULL, + locked_at TIMESTAMP WITH TIME ZONE DEFAULT now(), + CONSTRAINT spawn_locks_pk PRIMARY KEY (lock_id), + UNIQUE (identity, kind) +); diff --git a/cluster/identitylookup/postgresql/storage_lookup.go b/cluster/identitylookup/postgresql/storage_lookup.go new file mode 100644 index 000000000..24fad886d --- /dev/null +++ b/cluster/identitylookup/postgresql/storage_lookup.go @@ -0,0 +1,156 @@ +package postgresql + +import ( + context2 "context" + "encoding/json" + "errors" + "github.com/asynkron/protoactor-go/actor" + "github.com/asynkron/protoactor-go/cluster" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/lithammer/shortuuid/v4" + "log/slog" + "strings" + "time" +) + +type StorageLookup struct { + pool *pgxpool.Pool + logger *slog.Logger + waiting *actor.PID + rootContext *actor.RootContext + waitingForActivationTimeout time.Duration +} + +var _ cluster.StorageLookup = (*StorageLookup)(nil) + +func NewStorageLookup(pool *pgxpool.Pool, logger *slog.Logger, rootContext *actor.RootContext, waitingForActivationTimeout time.Duration) *StorageLookup { + return &StorageLookup{ + pool: pool, + logger: logger, + rootContext: rootContext, + waitingForActivationTimeout: waitingForActivationTimeout, + } +} + +func (s *StorageLookup) Init() { + var err error + s.waiting, err = s.rootContext.SpawnNamed(actor.PropsFromProducer(func() actor.Actor { + return newActivationWaitingActor(s.pool, 10*time.Second) + }), "activation-waiting-actor") + if err != nil { + panic(err) + } +} + +func (s *StorageLookup) TryGetExistingActivation(clusterIdentity *cluster.ClusterIdentity) *cluster.StoredActivation { + row, err := s.pool.Query(context2.Background(), "SELECT * FROM activations WHERE identity = $1 AND kind = $2", clusterIdentity.Identity, clusterIdentity.Kind) + if err != nil { + s.logger.Error("error while querying for activation", slog.Any("error", err)) + return nil + } + if row, err := pgx.CollectExactlyOneRow(row, pgx.RowToStructByName[ActivationsTableRow]); err == nil { + return &cluster.StoredActivation{ + Pid: string(row.Pid), + MemberID: row.MemberId, + } + } else if errors.Is(err, pgx.ErrNoRows) { + s.logger.Info("activation not found", slog.Any("identity", clusterIdentity.Identity), slog.Any("kind", clusterIdentity.Kind)) + return nil + } else { + s.logger.Error("error while collecting row", slog.Any("error", err)) + } + + return nil +} + +func (s *StorageLookup) TryAcquireLock(clusterIdentity *cluster.ClusterIdentity) *cluster.SpawnLock { + lockId := shortuuid.New() + /// Lockのとりっぱなし防止のための削除を考える + _, err := s.pool.Exec(context2.Background(), "INSERT INTO spawn_locks (lock_id, identity, kind) VALUES ($1, $2, $3)", lockId, clusterIdentity.Identity, clusterIdentity.Kind) + if err != nil { + if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { + s.logger.Info("lock already exists", slog.Any("identity", clusterIdentity.Identity), slog.Any("kind", clusterIdentity.Kind)) + return nil + } + + s.logger.Error("error while inserting lock", slog.Any("error", err)) + return nil + } + + return &cluster.SpawnLock{ + LockID: lockId, + ClusterIdentity: clusterIdentity, + } +} + +func (s *StorageLookup) WaitForActivation(clusterIdentity *cluster.ClusterIdentity) *cluster.StoredActivation { + res, err := s.rootContext.RequestFuture(s.waiting, &AddWaiting{clusterIdentity: clusterIdentity}, s.waitingForActivationTimeout).Result() + if err != nil { + s.logger.Error("error while waiting for activation", slog.Any("error", err)) + return nil + } + + switch res := res.(type) { + case *Activated: + return &cluster.StoredActivation{ + Pid: res.Pid, + MemberID: res.MemberId, + } + case *ActivationCancelled: + return nil + } + + s.logger.Error("unexpected response", slog.Any("response", res)) + return nil +} + +func (s *StorageLookup) RemoveLock(spawnLock cluster.SpawnLock) { + _, err := s.pool.Exec(context2.Background(), "DELETE FROM spawn_locks WHERE lock_id = $1", spawnLock.LockID) + if err != nil { + s.logger.Error("error while deleting lock", slog.Any("error", err)) + } +} + +func (s *StorageLookup) StoreActivation(memberID string, spawnLock *cluster.SpawnLock, pid *actor.PID) { + pidJson, err := json.Marshal(pid) + if err != nil { + s.logger.Error("error while marshalling pid", slog.Any("error", err)) + panic(err) + } + _, err = s.pool.Exec(context2.Background(), "INSERT INTO activations (identity, kind, identity_key, member_id, pid, lock_id) VALUES ($1, $2, $3, $4, $5, $6)", spawnLock.ClusterIdentity.Identity, spawnLock.ClusterIdentity.Kind, spawnLock.ClusterIdentity.AsKey(), memberID, pidJson, spawnLock.LockID) + if err != nil { + s.logger.Error("error while inserting activation", slog.Any("error", err)) + panic(err) + } +} + +func (s *StorageLookup) RemoveActivation(lock *cluster.SpawnLock) { + _, err := s.pool.Exec(context2.Background(), "DELETE FROM activations WHERE lock_id = $1", lock.LockID) + if err != nil { + s.logger.Error("error while deleting activation", slog.Any("error", err)) + } +} + +func (s *StorageLookup) RemoveMemberId(memberID string) { + _, err := s.pool.Exec(context2.Background(), "DELETE FROM activations WHERE member_id = $1", memberID) + if err != nil { + s.logger.Error("error while deleting activations", slog.Any("error", err)) + } +} + +type ActivationsTableRow struct { + Identity string `db:"identity"` + Kind string `db:"kind"` + IdentityKey string `db:"identity_key"` + MemberId string `db:"member_id"` + LockId string `db:"lock_id"` + Pid []byte `db:"pid"` + ActivatedAt time.Time `db:"activated_at"` +} + +type SpanLocksTableRow struct { + LockId string `db:"lock_id"` + Identity string `db:"identity"` + Kind string `db:"kind"` +} diff --git a/examples/cluster-identitylookup-postgresql/node1/main.go b/examples/cluster-identitylookup-postgresql/node1/main.go index 8a507cde2..4f747d344 100644 --- a/examples/cluster-identitylookup-postgresql/node1/main.go +++ b/examples/cluster-identitylookup-postgresql/node1/main.go @@ -6,12 +6,13 @@ import ( console "github.com/asynkron/goconsole" "github.com/asynkron/protoactor-go/actor" "github.com/asynkron/protoactor-go/cluster" - "github.com/asynkron/protoactor-go/cluster/clusterproviders/consul" + "github.com/asynkron/protoactor-go/cluster/clusterproviders/automanaged" "github.com/asynkron/protoactor-go/remote" ) func main() { - c := startNode() + c, clean := startNode() + defer clean() fmt.Print("\nBoot other nodes and press Enter\n") console.ReadLine() @@ -25,15 +26,15 @@ func main() { c.Shutdown(true) } -func startNode() *cluster.Cluster { +func startNode() (*cluster.Cluster, func()) { system := actor.NewActorSystem() - provider, _ := consul.New() - lookup := shared.NewLockUp() + provider := automanaged.New() + lookup, clean := shared.NewLockUp(system) config := remote.Configure("localhost", 0) clusterConfig := cluster.Configure("my-cluster", provider, lookup, config) c := cluster.New(system, clusterConfig) c.StartMember() - return c + return c, clean } diff --git a/examples/cluster-identitylookup-postgresql/node2/main.go b/examples/cluster-identitylookup-postgresql/node2/main.go index a6404eaaa..6df94596c 100644 --- a/examples/cluster-identitylookup-postgresql/node2/main.go +++ b/examples/cluster-identitylookup-postgresql/node2/main.go @@ -2,30 +2,43 @@ package main import ( "fmt" + console "github.com/asynkron/goconsole" + "github.com/asynkron/protoactor-go/cluster/clusterproviders/automanaged" "cluster-identitylookup-postgresql/shared" - console "github.com/asynkron/goconsole" "github.com/asynkron/protoactor-go/actor" "github.com/asynkron/protoactor-go/cluster" - "github.com/asynkron/protoactor-go/cluster/clusterproviders/consul" "github.com/asynkron/protoactor-go/remote" ) func main() { - cluster := startNode() - + cluster, clean := startNode() + defer clean() + pid := cluster.Get("abc", "hello") + fmt.Printf("Got pid %v\n", pid) + res, _ := cluster.Request("abc", "hello", &shared.HelloRequest{Name: "Roger"}) + fmt.Printf("Got response %v\n", res) fmt.Print("\nBoot other nodes and press Enter\n") - console.ReadLine() + for { + st, _ := console.ReadLine() + if st == "exit" { + break + } + pid := cluster.Get("abc", "hello") + fmt.Printf("Got pid %v", pid) + res, _ := cluster.Request("abc", "hello", &shared.HelloRequest{Name: "Roger"}) + fmt.Printf("Got response %v", res) + } cluster.Shutdown(true) } -func startNode() *cluster.Cluster { +func startNode() (*cluster.Cluster, func()) { system := actor.NewActorSystem() - provider, _ := consul.New() - lookup := shared.NewLockUp() + provider := automanaged.New() + lookup, clean := shared.NewLockUp(system) config := remote.Configure("localhost", 0) props := actor.PropsFromFunc(func(ctx actor.Context) { @@ -42,5 +55,6 @@ func startNode() *cluster.Cluster { c := cluster.New(system, clusterConfig) c.StartMember() - return c + + return c, clean } diff --git a/examples/cluster-identitylookup-postgresql/shared/lookup.go b/examples/cluster-identitylookup-postgresql/shared/lookup.go index f31a0d36b..7a20ea97a 100644 --- a/examples/cluster-identitylookup-postgresql/shared/lookup.go +++ b/examples/cluster-identitylookup-postgresql/shared/lookup.go @@ -1,7 +1,102 @@ package shared -import "github.com/asynkron/protoactor-go/cluster" +import ( + "context" + "github.com/asynkron/protoactor-go/actor" + "github.com/asynkron/protoactor-go/cluster" + "github.com/asynkron/protoactor-go/cluster/identitylookup/postgresql" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/testcontainers/testcontainers-go/wait" + "log" + "time" +) -func NewLockUp() cluster.IdentityLookup { - return cluster.NewIdentityStorageLookup(nil) +func NewLockUp(actorSystem *actor.ActorSystem) (cluster.IdentityLookup, func()) { + ctx := context.Background() + dbName := "users" + dbUser := "user" + dbPassword := "password" + + postgresContainer, err := postgres.Run(ctx, + "postgres:16-alpine", + postgres.WithDatabase(dbName), + postgres.WithUsername(dbUser), + postgres.WithPassword(dbPassword), + testcontainers.WithWaitStrategy( + wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2). + WithStartupTimeout(5*time.Second), + ), + ) + + if err != nil { + log.Printf("failed to start container: %s", err) + panic(err) + } + + connString := postgresContainer.MustConnectionString(ctx, "sslmode=disable", "application_name=test") + + pool, err := pgxpool.New(context.Background(), connString) + if err != nil { + panic(err) + } + + if err := prepareTables(ctx, connString); err != nil { + panic(err) + } + return cluster.NewIdentityStorageLookup(postgresql.NewStorageLookup(pool, actorSystem.Logger(), actorSystem.Root, time.Second)), func() { + if err := testcontainers.TerminateContainer(postgresContainer); err != nil { + log.Printf("failed to terminate container: %s", err) + } + } +} + +func prepareTables(ctx context.Context, connString string) error { + pool, err := pgxpool.New(ctx, connString) + if err != nil { + return err + } + + queries := []string{ + ` +CREATE TABLE IF NOT EXISTS spawn_locks +( + lock_id TEXT NOT NULL, + identity TEXT NOT NULL, + kind TEXT NOT NULL, + locked_at TIMESTAMP WITH TIME ZONE DEFAULT now(), + CONSTRAINT spawn_locks_pk PRIMARY KEY (lock_id), + UNIQUE (identity, kind) +); + +`, + ` +CREATE TABLE IF NOT EXISTS activations +( + identity TEXT NOT NULL, + kind TEXT NOT NULL, + identity_key TEXT NOT NULL, + member_id TEXT NOT NULL, + pid json NOT NULL, + lock_id TEXT NOT NULL, + activated_at TIMESTAMP WITH TIME ZONE DEFAULT now(), + PRIMARY KEY (identity, kind) +-- FOREIGN KEY (lock_id) references spawn_locks (lock_id) +); +`, + ` +CREATE INDEX IF NOT EXISTS activations_member_idx ON activations (member_id); +`, + } + + for _, query := range queries { + _, err = pool.Exec(ctx, query) + if err != nil { + return err + } + } + + return nil } From 1e5b920d956ab2ea8128df181a5dd32bf7a12865 Mon Sep 17 00:00:00 2001 From: "ryota.suzuki" Date: Tue, 14 Jan 2025 17:22:07 +0900 Subject: [PATCH 10/12] improve logging --- actor/supervision.go | 10 + cluster/identity_storage_lookup.go | 2 + cluster/identity_storage_placement_actor.go | 18 +- cluster/identity_storage_worker.go | 39 +- .../postgresql/activation_waiting_actor.go | 80 +++- .../postgresql/ddl/activations.sql | 1 + .../postgresql/storage_lookup.go | 54 ++- .../cluster-identitylookup-postgresql/go.mod | 116 +++++ .../cluster-identitylookup-postgresql/go.sum | 451 ++++++++++++++++++ .../node2/main.go | 58 ++- .../shared/lookup.go | 40 +- 11 files changed, 798 insertions(+), 71 deletions(-) create mode 100644 examples/cluster-identitylookup-postgresql/go.mod create mode 100644 examples/cluster-identitylookup-postgresql/go.sum diff --git a/actor/supervision.go b/actor/supervision.go index c7dbf7d53..2cdbe6b7e 100644 --- a/actor/supervision.go +++ b/actor/supervision.go @@ -1,6 +1,8 @@ package actor import ( + "log" + "runtime" "time" ) @@ -22,6 +24,14 @@ type Supervisor interface { } func logFailure(actorSystem *ActorSystem, child *PID, reason interface{}, directive Directive) { + log.Printf("[ERROR] %s\n", reason) + for depth := 0; ; depth++ { + _, file, line, ok := runtime.Caller(depth) + if !ok { + break + } + log.Printf("======> %d: %v:%d", depth, file, line) + } actorSystem.EventStream.Publish(&SupervisorEvent{ Child: child, Reason: reason, diff --git a/cluster/identity_storage_lookup.go b/cluster/identity_storage_lookup.go index 624055e1d..c9ec9891f 100644 --- a/cluster/identity_storage_lookup.go +++ b/cluster/identity_storage_lookup.go @@ -50,6 +50,7 @@ func RemotePlacementActor(address string) *actor.PID { // Get returns a PID for a given ClusterIdentity func (i *IdentityStorageLookup) Get(clusterIdentity *ClusterIdentity) *actor.PID { + i.system.Logger().Info("Get", slog.Any("clusterIdentity", clusterIdentity)) msg := newGetPid(clusterIdentity) timeout := 5 * time.Second @@ -107,6 +108,7 @@ func (i *IdentityStorageLookup) Setup(cluster *Cluster, _ []string, isClient boo } i.placementActor, err = i.system.Root.SpawnNamed(actor.PropsFromProducer(func() actor.Actor { return newIdentityStoragePlacementActor(cluster, i) }), placementActorName) + i.cluster.Logger().Info("placement actor started", slog.Any("actor", i.placementActor)) if err != nil { panic(err) } diff --git a/cluster/identity_storage_placement_actor.go b/cluster/identity_storage_placement_actor.go index c4b46a9df..e66aeaff0 100644 --- a/cluster/identity_storage_placement_actor.go +++ b/cluster/identity_storage_placement_actor.go @@ -16,6 +16,7 @@ type IdentityStoragePlacementActor struct { identityStorageLookup *IdentityStorageLookup subscription *eventstream.Subscription actors map[string] /* clusterIdentity*/ GrainMeta + logger *slog.Logger } func newIdentityStoragePlacementActor(cluster *Cluster, identityStorageLookup *IdentityStorageLookup) *IdentityStoragePlacementActor { @@ -23,6 +24,9 @@ func newIdentityStoragePlacementActor(cluster *Cluster, identityStorageLookup *I cluster: cluster, identityStorageLookup: identityStorageLookup, actors: make(map[string] /* clusterIdentity*/ GrainMeta), + logger: cluster.Logger().With( + slog.String("actorType", "IdentityStoragePlacementActor"), + ), } return this } @@ -41,14 +45,15 @@ func (i *IdentityStoragePlacementActor) Receive(context actor.Context) { } func (i *IdentityStoragePlacementActor) onStarted(context actor.Context) { - i.cluster.Logger().Info("IdentityStoragePlacementActor started") + i.logger.Info("IdentityStoragePlacementActor started") + i.logger = i.logger.With(slog.String("pid", context.Self().Id)) i.subscription = i.cluster.ActorSystem.EventStream.Subscribe(func(evt interface{}) { context.Send(context.Self(), evt) }) } func (i *IdentityStoragePlacementActor) onStopping(_ actor.Context) { - i.cluster.Logger().Info("IdentityStoragePlacementActor stopping") + i.logger.Info("IdentityStoragePlacementActor stopping") i.subscription.Deactivate() for _, meta := range i.actors { /// TODO: need to throttle? @@ -59,14 +64,14 @@ func (i *IdentityStoragePlacementActor) onStopping(_ actor.Context) { func (i *IdentityStoragePlacementActor) onActivationTerminating(_ actor.Context, msg *ActivationTerminating) { if grainMeta, ok := i.actors[msg.ClusterIdentity.AsKey()]; ok { if grainMeta.PID != msg.Pid { - i.cluster.Logger().Error("PID mismatch", slog.Any("clusterIdentity", msg.ClusterIdentity), slog.Any("expectedPid", grainMeta.PID), slog.Any("actualPid", msg.Pid)) + i.logger.Error("PID mismatch", slog.Any("clusterIdentity", msg.ClusterIdentity), slog.Any("expectedPid", grainMeta.PID), slog.Any("actualPid", msg.Pid)) return } delete(i.actors, msg.ClusterIdentity.AsKey()) i.cluster.PidCache.Remove(msg.ClusterIdentity.Identity, msg.ClusterIdentity.Kind) i.identityStorageLookup.Storage.RemoveActivation(newSpawnLock(grainMeta.PID.String(), msg.ClusterIdentity)) } else { - i.cluster.Logger().Error("IdentityStoragePlacementActor#onActivationTerminating activation not found", slog.Any("clusterIdentity", msg.ClusterIdentity)) + i.logger.Error("IdentityStoragePlacementActor#onActivationTerminating activation not found", slog.Any("clusterIdentity", msg.ClusterIdentity)) } } @@ -83,16 +88,17 @@ func (i *IdentityStoragePlacementActor) onActivationRequest(context actor.Contex return } - i.spawn(msg, context, clusterKind) + i.spawnActor(context, msg, clusterKind) } -func (i *IdentityStoragePlacementActor) spawn(req *ActivationRequest, ctx actor.Context, kind *ActivatedKind) { +func (i *IdentityStoragePlacementActor) spawnActor(ctx actor.Context, req *ActivationRequest, kind *ActivatedKind) { props := WithClusterIdentity(kind.Props, req.ClusterIdentity) pid := ctx.SpawnPrefix(props, req.ClusterIdentity.Identity) kind.Inc() /// TODO: member selectionを考慮 i.identityStorageLookup.Storage.StoreActivation(i.cluster.ActorSystem.ID, newSpawnLock(req.RequestId, req.ClusterIdentity), pid) + i.logger.Info("Activation Stored", slog.Any("pid", pid), slog.Any("clusterIdentity", req.ClusterIdentity), slog.String("key", req.ClusterIdentity.AsKey())) i.actors[req.ClusterIdentity.AsKey()] = GrainMeta{ ID: req.ClusterIdentity, PID: pid, diff --git a/cluster/identity_storage_worker.go b/cluster/identity_storage_worker.go index e90a92cbb..756a73e40 100644 --- a/cluster/identity_storage_worker.go +++ b/cluster/identity_storage_worker.go @@ -14,15 +14,16 @@ type IdentityStorageWorker struct { cluster *Cluster lookup *IdentityStorageLookup storage StorageLookup + logger *slog.Logger } func newIdentityStorageWorker(storageLookup *IdentityStorageLookup) *IdentityStorageWorker { - this := &IdentityStorageWorker{ + return &IdentityStorageWorker{ cluster: storageLookup.cluster, lookup: storageLookup, storage: storageLookup.Storage, + logger: storageLookup.cluster.Logger().With(slog.String("actorType", "IdentityStorageWorker")), } - return this } // Receive func @@ -31,6 +32,10 @@ func (ids *IdentityStorageWorker) Receive(c actor.Context) { getPid, ok := m.(*GetPid) if !ok { + if _, ok := m.(*actor.Started); ok { + ids.logger = ids.logger.With(slog.String("pid", c.Self().Id)) + } + return } @@ -42,7 +47,7 @@ func (ids *IdentityStorageWorker) Receive(c actor.Context) { existing, _ := ids.cluster.PidCache.Get(getPid.ClusterIdentity.Identity, getPid.ClusterIdentity.Kind) if existing != nil { - log.Printf("Found %s in pidcache", m.(*GetPid).ClusterIdentity.ToShortString()) + ids.logger.Info("Found pid in pidcache", slog.Any("clusterIdentity", m.(*GetPid).ClusterIdentity.ToShortString())) c.Respond(newPidResult(existing)) } @@ -53,6 +58,7 @@ func (ids *IdentityStorageWorker) Receive(c actor.Context) { if err != nil { panic(fmt.Errorf("IdentityStorageWorker: Failed to unmarshal pid: %v", err)) } + ids.logger.Info("Found activation in storage", slog.Any("pid", pid), slog.Any("clusterIdentity", m.(*GetPid).ClusterIdentity.ToShortString())) if ids.cluster.MemberList.ContainsMemberID(activation.MemberID) { ids.cluster.PidCache.Set(getPid.ClusterIdentity.Identity, getPid.ClusterIdentity.Kind, pid) c.Respond(newPidResult(pid)) @@ -64,6 +70,7 @@ func (ids *IdentityStorageWorker) Receive(c actor.Context) { } } + /// round robinでは普通にNode同士でズレる activator := ids.cluster.MemberList.GetActivatorMember(getPid.ClusterIdentity.Kind, getPid.ClusterIdentity.Identity) if activator == "" { @@ -71,6 +78,19 @@ func (ids *IdentityStorageWorker) Receive(c actor.Context) { return } + // TODO: TryGetExistingActivationからlock取得までに別のNodeでActivateが発生した場合に, + // lockはすでに消されてていて、lockが取得できてしまいこのNodeでActivateが発生してしまう + // lockを消すのを遅延させる？ + + // TryGetExistingActivation -> ない + // TryAcquireLock -> ない + // 両方ないのはまずい + + // Lockには有効期限をもうける? + // 有効期限が切れたら消す。 + // それまでは消さない。 + // LockはNodeを消す時に同時に消す + // Nodeが消えるときにLockが消せなかったら？ lock := ids.storage.TryAcquireLock(getPid.ClusterIdentity) if lock == nil { activation := ids.storage.WaitForActivation(getPid.ClusterIdentity) @@ -78,7 +98,7 @@ func (ids *IdentityStorageWorker) Receive(c actor.Context) { pid := &actor.PID{} err := json.Unmarshal([]byte(activation.Pid), pid) if err != nil { - panic(err) + panic(fmt.Sprintf("IdentityStorageWorker: Failed to unmarshal pid: %v", err)) } if ids.cluster.MemberList.ContainsMemberID(activation.MemberID) { ids.cluster.PidCache.Set(getPid.ClusterIdentity.Identity, getPid.ClusterIdentity.Kind, pid) @@ -96,6 +116,11 @@ func (ids *IdentityStorageWorker) Receive(c actor.Context) { defer func() { ids.storage.RemoveLock(*lock) }() + ids.logger.Info("Lock acquired", + slog.Any("lockId", lock.LockID), + slog.Any("clusterIdentity", getPid.ClusterIdentity.ToShortString()), + slog.Any("activator", activator), + ) pid := ids.SpawnActivation(activator, *lock) c.Respond(newPidResult(pid)) @@ -103,7 +128,7 @@ func (ids *IdentityStorageWorker) Receive(c actor.Context) { } func (ids *IdentityStorageWorker) SpawnActivation(activatorAddress string, lock SpawnLock) *actor.PID { - ids.cluster.Logger().Info("Spawning activation", slog.Any("spawnTo", activatorAddress), slog.Any("clusterIdentity", lock.ClusterIdentity)) + ids.logger.Info("Spawning activation", slog.Any("spawnTo", activatorAddress), slog.Any("clusterIdentity", lock.ClusterIdentity)) remotePid := RemotePlacementActor(activatorAddress) activateRequest := &ActivationRequest{ ClusterIdentity: lock.ClusterIdentity, @@ -112,13 +137,13 @@ func (ids *IdentityStorageWorker) SpawnActivation(activatorAddress string, lock activationResult, err := ids.cluster.ActorSystem.Root.RequestFuture(remotePid, activateRequest, 10*time.Second).Result() if err != nil { - ids.cluster.Logger().Error("Failed to activate", slog.Any("error", err)) + ids.logger.Error("Failed to activate", slog.Any("error", err)) return nil } activationResponse := activationResult.(*ActivationResponse) if activationResponse.Failed { - ids.cluster.Logger().Error("Failed to activate") + ids.logger.Error("Failed to activate") return nil } ids.cluster.PidCache.Set(lock.ClusterIdentity.Identity, lock.ClusterIdentity.Kind, activationResponse.Pid) diff --git a/cluster/identitylookup/postgresql/activation_waiting_actor.go b/cluster/identitylookup/postgresql/activation_waiting_actor.go index bee5745e7..3668b5c16 100644 --- a/cluster/identitylookup/postgresql/activation_waiting_actor.go +++ b/cluster/identitylookup/postgresql/activation_waiting_actor.go @@ -27,8 +27,12 @@ type Activated struct { type ActivationCancelled struct { } -type ReplyTo struct { - pid *actor.PID +type WaitActivationTimeout struct { +} + +type ReplyWaiting struct { + replyTo *actor.PID + countOfRetry int } type queryingTimeReached struct { @@ -40,18 +44,25 @@ func ClusterIdentityFromKey(key string) *cluster.ClusterIdentity { } type ActivationWaitingActor struct { - pool *pgxpool.Pool - poolingQueryInterval time.Duration - waitList map[string] /*clusterIdentity#AsKey*/ []*ReplyTo - inflight []*cluster.ClusterIdentity - cancelScheduler scheduler.CancelFunc + pool *pgxpool.Pool + poolingQueryInterval time.Duration + waitList map[string] /*clusterIdentity#AsKey*/ []*ReplyWaiting + inflight []*cluster.ClusterIdentity + cancelScheduler scheduler.CancelFunc + logger *slog.Logger + actorSystem *actor.ActorSystem + maxQueryRetryPerWaiting int } -func newActivationWaitingActor(pool *pgxpool.Pool, poolingQueryInterval time.Duration) *ActivationWaitingActor { +func newActivationWaitingActor(pool *pgxpool.Pool, poolingQueryInterval time.Duration, logger *slog.Logger) *ActivationWaitingActor { this := &ActivationWaitingActor{ pool: pool, poolingQueryInterval: poolingQueryInterval, - waitList: make(map[string] /*clusterIdentity#AsKey*/ []*ReplyTo), + waitList: make(map[string] /*clusterIdentity#AsKey*/ []*ReplyWaiting), + logger: logger.With( + slog.String("actorType", "ActivationWaitingActor"), + ), + maxQueryRetryPerWaiting: 20, } return this } @@ -71,16 +82,17 @@ func (a *ActivationWaitingActor) Receive(context actor.Context) { func (a *ActivationWaitingActor) onAddWaiting(context actor.Context, msg *AddWaiting) { if _, ok := a.waitList[msg.clusterIdentity.AsKey()]; !ok { - a.waitList[msg.clusterIdentity.AsKey()] = []*ReplyTo{} + a.waitList[msg.clusterIdentity.AsKey()] = []*ReplyWaiting{} } - a.waitList[msg.clusterIdentity.AsKey()] = append(a.waitList[msg.clusterIdentity.AsKey()], &ReplyTo{ - pid: context.Sender(), + a.waitList[msg.clusterIdentity.AsKey()] = append(a.waitList[msg.clusterIdentity.AsKey()], &ReplyWaiting{ + replyTo: context.Sender(), + countOfRetry: 0, }) } func (a *ActivationWaitingActor) onStarted(context actor.Context) { - /// TODO: clusterの停止に伴ってすべての待ちリストへゴメンを送る + a.logger = a.logger.With(slog.String("pid", context.Self().Id)) a.cancelScheduler = scheduler.NewTimerScheduler(context).SendRepeatedly(a.poolingQueryInterval, a.poolingQueryInterval, context.Self(), &queryingTimeReached{}) } @@ -101,19 +113,21 @@ func (a *ActivationWaitingActor) onQueryingTimeReached(context actor.Context) { pull, _ := iter.Pull(slices.Chunk(a.inflight, 100)) for { - next, hasNext := pull() + identities, hasNext := pull() if !hasNext { break } var keys []string - for _, identity := range next { + for _, identity := range identities { keys = append(keys, identity.AsKey()) } + a.logger.Info("Querying activations", slog.Any("keys", keys)) result, err := a.pool.Query(context2.Background(), ` -SELECT * FROM activations WHERE identity_key in ($1) +SELECT * FROM activations WHERE identity_key = ANY($1) `, pq.Array(keys)) + if err != nil { context.Logger().Error("error while querying activations", slog.Any("error", err)) } @@ -125,8 +139,8 @@ SELECT * FROM activations WHERE identity_key in ($1) for _, activation := range activations { for _, replayTo := range a.waitList[activation.IdentityKey] { - - context.Send(replayTo.pid, &Activated{ + a.logger.Info("Activation found", slog.Any("identityKey", activation.IdentityKey), slog.Any("memberId", activation.MemberId)) + context.Send(replayTo.replyTo, &Activated{ Pid: string(activation.Pid), MemberId: activation.MemberId, }) @@ -134,19 +148,41 @@ SELECT * FROM activations WHERE identity_key in ($1) delete(a.waitList, activation.IdentityKey) } - clear(a.inflight) + for _, identity := range identities { + if _, ok := a.waitList[identity.AsKey()]; ok { + remainWaitList := make([]*ReplyWaiting, 0) + for _, replayTo := range a.waitList[identity.AsKey()] { + replayTo.countOfRetry++ + if replayTo.countOfRetry >= a.maxQueryRetryPerWaiting { + a.logger.Info("Activation not found by retry count exceeded", slog.Any("identity", identity)) + context.Send(replayTo.replyTo, &WaitActivationTimeout{}) + } else { + remainWaitList = append(remainWaitList, replayTo) + } + } + if len(remainWaitList) > 0 { + a.waitList[identity.AsKey()] = remainWaitList + } else { + delete(a.waitList, identity.AsKey()) + } + } + } + + a.inflight = make([]*cluster.ClusterIdentity, 0) } } func (a *ActivationWaitingActor) onStopping(context actor.Context) { + /// TODO: clusterの停止に伴ってすべての待ちリストへゴメンを送る + if a.cancelScheduler != nil { a.cancelScheduler() } - for _, replayTo := range a.waitList { - for _, reply := range replayTo { - context.Send(reply.pid, &ActivationCancelled{}) + for _, waiting := range a.waitList { + for _, waiting := range waiting { + context.Send(waiting.replyTo, &ActivationCancelled{}) } } } diff --git a/cluster/identitylookup/postgresql/ddl/activations.sql b/cluster/identitylookup/postgresql/ddl/activations.sql index e38a45a20..9f48d2f05 100644 --- a/cluster/identitylookup/postgresql/ddl/activations.sql +++ b/cluster/identitylookup/postgresql/ddl/activations.sql @@ -8,6 +8,7 @@ CREATE TABLE IF NOT EXISTS activations lock_id TEXT NOT NULL, activated_at TIMESTAMP WITH TIME ZONE DEFAULT now(), PRIMARY KEY (identity, kind), + -- lockの無効化を考慮したい FOREIGN KEY (lock_id) references spawn_locks (lock_id) ); diff --git a/cluster/identitylookup/postgresql/storage_lookup.go b/cluster/identitylookup/postgresql/storage_lookup.go index 24fad886d..f54a00fcf 100644 --- a/cluster/identitylookup/postgresql/storage_lookup.go +++ b/cluster/identitylookup/postgresql/storage_lookup.go @@ -4,13 +4,13 @@ import ( context2 "context" "encoding/json" "errors" + "fmt" "github.com/asynkron/protoactor-go/actor" "github.com/asynkron/protoactor-go/cluster" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/lithammer/shortuuid/v4" "log/slog" - "strings" "time" ) @@ -27,7 +27,7 @@ var _ cluster.StorageLookup = (*StorageLookup)(nil) func NewStorageLookup(pool *pgxpool.Pool, logger *slog.Logger, rootContext *actor.RootContext, waitingForActivationTimeout time.Duration) *StorageLookup { return &StorageLookup{ pool: pool, - logger: logger, + logger: logger.With(slog.String("component", "postgresql.StorageLookup"), slog.String("ActorSystemId", rootContext.ActorSystem().ID)), rootContext: rootContext, waitingForActivationTimeout: waitingForActivationTimeout, } @@ -36,13 +36,19 @@ func NewStorageLookup(pool *pgxpool.Pool, logger *slog.Logger, rootContext *acto func (s *StorageLookup) Init() { var err error s.waiting, err = s.rootContext.SpawnNamed(actor.PropsFromProducer(func() actor.Actor { - return newActivationWaitingActor(s.pool, 10*time.Second) + return newActivationWaitingActor(s.pool, 10*time.Millisecond, s.logger) }), "activation-waiting-actor") if err != nil { - panic(err) + panic(fmt.Sprintf("failed to spawn activation waiting actor: %v", err)) } } +func (s *StorageLookup) Shutdown() *cluster.ClusterIdentity { + s.logger.Info("shutting down") + s.rootContext.Stop(s.waiting) + return nil +} + func (s *StorageLookup) TryGetExistingActivation(clusterIdentity *cluster.ClusterIdentity) *cluster.StoredActivation { row, err := s.pool.Query(context2.Background(), "SELECT * FROM activations WHERE identity = $1 AND kind = $2", clusterIdentity.Identity, clusterIdentity.Kind) if err != nil { @@ -67,17 +73,35 @@ func (s *StorageLookup) TryGetExistingActivation(clusterIdentity *cluster.Cluste func (s *StorageLookup) TryAcquireLock(clusterIdentity *cluster.ClusterIdentity) *cluster.SpawnLock { lockId := shortuuid.New() /// Lockのとりっぱなし防止のための削除を考える - _, err := s.pool.Exec(context2.Background(), "INSERT INTO spawn_locks (lock_id, identity, kind) VALUES ($1, $2, $3)", lockId, clusterIdentity.Identity, clusterIdentity.Kind) - if err != nil { - if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { - s.logger.Info("lock already exists", slog.Any("identity", clusterIdentity.Identity), slog.Any("kind", clusterIdentity.Kind)) - return nil - } + activation := s.TryGetExistingActivation(clusterIdentity) + if activation != nil { + s.logger.Info("activation already exists", slog.Any("identity", clusterIdentity.Identity), slog.Any("kind", clusterIdentity.Kind)) + return nil + } + + /// ロックが解除(Remove)されていてもロックから1minはロックは取れない + /// ロックが解除されていなくてもロックから1hour経過しているならロックは取れる + effect, err := s.pool.Exec(context2.Background(), ` +INSERT INTO spawn_locks (lock_id, identity, kind) +VALUES ($1, $2, $3) +ON CONFLICT (identity, kind) DO UPDATE + SET lock_id = $1, locked_at = NOW(), unlocked_at = NULL +WHERE (spawn_locks.unlocked_at IS NOT NULL AND spawn_locks.locked_at < NOW() - INTERVAL '1 minute') + OR (spawn_locks.unlocked_at IS NULL AND spawn_locks.locked_at < NOW() - INTERVAL '1 hour'); +`, + lockId, clusterIdentity.Identity, clusterIdentity.Kind, + ) + if err != nil { s.logger.Error("error while inserting lock", slog.Any("error", err)) return nil } + if effect.RowsAffected() == 0 { + s.logger.Info("lock already exists", slog.Any("identity", clusterIdentity.Identity), slog.Any("kind", clusterIdentity.Kind)) + return nil + } + return &cluster.SpawnLock{ LockID: lockId, ClusterIdentity: clusterIdentity, @@ -85,6 +109,7 @@ func (s *StorageLookup) TryAcquireLock(clusterIdentity *cluster.ClusterIdentity) } func (s *StorageLookup) WaitForActivation(clusterIdentity *cluster.ClusterIdentity) *cluster.StoredActivation { + s.logger.Info("waiting for activation", slog.Any("identity", clusterIdentity.Identity), slog.Any("kind", clusterIdentity.Kind)) res, err := s.rootContext.RequestFuture(s.waiting, &AddWaiting{clusterIdentity: clusterIdentity}, s.waitingForActivationTimeout).Result() if err != nil { s.logger.Error("error while waiting for activation", slog.Any("error", err)) @@ -99,6 +124,8 @@ func (s *StorageLookup) WaitForActivation(clusterIdentity *cluster.ClusterIdenti } case *ActivationCancelled: return nil + case *WaitActivationTimeout: + return nil } s.logger.Error("unexpected response", slog.Any("response", res)) @@ -106,6 +133,13 @@ func (s *StorageLookup) WaitForActivation(clusterIdentity *cluster.ClusterIdenti } func (s *StorageLookup) RemoveLock(spawnLock cluster.SpawnLock) { + _, err := s.pool.Exec(context2.Background(), "UPDATE spawn_locks SET unlocked_at = now() WHERE lock_id = $1", spawnLock.LockID) + if err != nil { + s.logger.Error("error while deleting lock", slog.Any("error", err)) + } +} + +func (s *StorageLookup) DeleteLock(spawnLock cluster.SpawnLock) { _, err := s.pool.Exec(context2.Background(), "DELETE FROM spawn_locks WHERE lock_id = $1", spawnLock.LockID) if err != nil { s.logger.Error("error while deleting lock", slog.Any("error", err)) diff --git a/examples/cluster-identitylookup-postgresql/go.mod b/examples/cluster-identitylookup-postgresql/go.mod new file mode 100644 index 000000000..5d0c4864d --- /dev/null +++ b/examples/cluster-identitylookup-postgresql/go.mod @@ -0,0 +1,116 @@ +module cluster-identitylookup-postgresql + +go 1.22 + +toolchain go1.23.1 + +replace github.com/asynkron/protoactor-go => ../.. + +require ( + github.com/asynkron/goconsole v0.0.0-20160504192649-bfa12eebf716 + google.golang.org/protobuf v1.33.0 +) + +require ( + github.com/asynkron/protoactor-go v0.0.0-20240822202345-3c0e61ca19c9 + github.com/jackc/pgx/v5 v5.7.2 + github.com/testcontainers/testcontainers-go v0.35.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.35.0 +) + +require ( + dario.cat/mergo v1.0.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/Workiva/go-datastructures v1.1.3 // indirect + github.com/armon/go-metrics v0.4.1 // indirect + github.com/asynkron/gofun v0.0.0-20220329210725-34fed760f4c2 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/containerd/containerd v1.7.18 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v27.1.1+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/fatih/color v1.14.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/consul/api v1.26.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-hclog v1.5.0 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/hashicorp/serf v0.10.1 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.17.4 // indirect + github.com/labstack/echo v3.3.10+incompatible // indirect + github.com/labstack/gommon v0.3.1 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/lithammer/shortuuid/v4 v4.0.0 // indirect + github.com/lmittmann/tint v1.0.3 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect + github.com/moby/sys/user v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/orcaman/concurrent-map v1.0.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/prometheus/client_golang v1.17.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.1 // indirect + github.com/shirou/gopsutil/v3 v3.23.12 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/stretchr/testify v1.9.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasttemplate v1.2.1 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/exporters/prometheus v0.44.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/sdk v1.21.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/grpc v1.64.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/examples/cluster-identitylookup-postgresql/go.sum b/examples/cluster-identitylookup-postgresql/go.sum new file mode 100644 index 000000000..740ddbe89 --- /dev/null +++ b/examples/cluster-identitylookup-postgresql/go.sum @@ -0,0 +1,451 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Workiva/go-datastructures v1.1.3 h1:LRdRrug9tEuKk7TGfz/sct5gjVj44G9pfqDt4qm7ghw= +github.com/Workiva/go-datastructures v1.1.3/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asynkron/goconsole v0.0.0-20160504192649-bfa12eebf716 h1:SgyG4sXkrlalMoCfp20LiNPNhfJS7ez3opNdtihIxPc= +github.com/asynkron/goconsole v0.0.0-20160504192649-bfa12eebf716/go.mod h1:/zSlF0T2ArAsTG6SVu8d8qlK+19jjudjA3wWsxnGFHg= +github.com/asynkron/gofun v0.0.0-20220329210725-34fed760f4c2 h1:jEsFZ9d/ieJGVrx3fSPi8oe/qv21fRmyUL5cS3ZEn5A= +github.com/asynkron/gofun v0.0.0-20220329210725-34fed760f4c2/go.mod h1:5GMOSqaYxNWwuVRWyampTPJEntwz7Mj9J8v1a7gSU2E= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= +github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= +github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/hashicorp/consul/api v1.26.1 h1:5oSXOO5fboPZeW5SN+TdGFP/BILDgBm19OrPZ/pICIM= +github.com/hashicorp/consul/api v1.26.1/go.mod h1:B4sQTeaSO16NtynqrAdwOlahJ7IUDZM9cj2420xYL8A= +github.com/hashicorp/consul/sdk v0.15.0 h1:2qK9nDrr4tiJKRoxPGhm6B7xJjLVIQqkjiab2M4aKjU= +github.com/hashicorp/consul/sdk v0.15.0/go.mod h1:r/OmRRPbHOe0yxNahLw7G9x5WG17E1BIECMtCjcPSNo= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= +github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= +github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= +github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI= +github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg= +github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s= +github.com/labstack/gommon v0.3.1 h1:OomWaJXm7xR6L1HmEtGyQf26TEn7V6X88mktX9kee9o= +github.com/labstack/gommon v0.3.1/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw7k08o4c= +github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= +github.com/lmittmann/tint v1.0.3 h1:W5PHeA2D8bBJVvabNfQD/XW9HPLZK1XoPZH0cq8NouQ= +github.com/lmittmann/tint v1.0.3/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mdelapenya/tlscert v0.1.0 h1:YTpF579PYUX475eOL+6zyEO3ngLTOUWck78NBuJVXaM= +github.com/mdelapenya/tlscert v0.1.0/go.mod h1:wrbyM/DwbFCeCeqdPX/8c6hNOqQgbf0rUDErE1uD+64= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HDbW65HOY= +github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4= +github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/testcontainers/testcontainers-go v0.35.0 h1:uADsZpTKFAtp8SLK+hMwSaa+X+JiERHtd4sQAFmXeMo= +github.com/testcontainers/testcontainers-go v0.35.0/go.mod h1:oEVBj5zrfJTrgjwONs1SsRbnBtH9OKl+IGl3UMcr2B4= +github.com/testcontainers/testcontainers-go/modules/postgres v0.35.0 h1:eEGx9kYzZb2cNhRbBrNOCL/YPOM7+RMJiy3bB+ie0/I= +github.com/testcontainers/testcontainers-go/modules/postgres v0.35.0/go.mod h1:hfH71Mia/WWLBgMD2YctYcMlfsbnT0hflweL1dy8Q4s= +github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/exporters/prometheus v0.44.0 h1:08qeJgaPC0YEBu2PQMbqU3rogTlyzpjhCI2b58Yn00w= +go.opentelemetry.io/otel/exporters/prometheus v0.44.0/go.mod h1:ERL2uIeBtg4TxZdojHUwzZfIFlUIjZtxubT5p4h1Gjg= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/sdk/metric v1.21.0 h1:smhI5oD714d6jHE6Tie36fPx4WDFIg+Y6RfAY4ICcR0= +go.opentelemetry.io/otel/sdk/metric v1.21.0/go.mod h1:FJ8RAsoPGv/wYMgBdUJXOm+6pzFY3YdljnXtv1SBE8Q= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= diff --git a/examples/cluster-identitylookup-postgresql/node2/main.go b/examples/cluster-identitylookup-postgresql/node2/main.go index 6df94596c..0b21f2930 100644 --- a/examples/cluster-identitylookup-postgresql/node2/main.go +++ b/examples/cluster-identitylookup-postgresql/node2/main.go @@ -4,6 +4,7 @@ import ( "fmt" console "github.com/asynkron/goconsole" "github.com/asynkron/protoactor-go/cluster/clusterproviders/automanaged" + "time" "cluster-identitylookup-postgresql/shared" @@ -13,11 +14,23 @@ import ( ) func main() { - cluster, clean := startNode() + c, clean := startNode(autoManagePorts[0]) defer clean() - pid := cluster.Get("abc", "hello") + + c2, clean2 := startNode(autoManagePorts[1]) + defer clean2() + + for { + if c.MemberList.Length() > 1 { + break + } + c.Logger().Info("Waiting for other nodes to join") + time.Sleep(time.Second) + } + + pid := c.Get("abc", "hello") fmt.Printf("Got pid %v\n", pid) - res, _ := cluster.Request("abc", "hello", &shared.HelloRequest{Name: "Roger"}) + res, _ := c.Request("abc", "hello", &shared.HelloRequest{Name: "Roger"}) fmt.Printf("Got response %v\n", res) fmt.Print("\nBoot other nodes and press Enter\n") for { @@ -25,19 +38,44 @@ func main() { if st == "exit" { break } - pid := cluster.Get("abc", "hello") - fmt.Printf("Got pid %v", pid) - res, _ := cluster.Request("abc", "hello", &shared.HelloRequest{Name: "Roger"}) - fmt.Printf("Got response %v", res) + go func() { + messageToCluster(c, st) + }() + go func() { + messageToCluster(c2, st) + }() } - cluster.Shutdown(true) + c.Shutdown(true) +} + +func messageToCluster(c *cluster.Cluster, st string) { + pid := c.Get(st, "hello") + c.Logger().Info("Got pid %v", pid) + res, _ := c.Request(st, "hello", &shared.HelloRequest{Name: "Roger"}) + c.Logger().Info("Got response %v", res) +} + +var autoManagePorts = []int{ + 8081, + 8082, + //8083, +} + +var ( + autoManageHosts []string +) + +func init() { + for _, port := range autoManagePorts { + autoManageHosts = append(autoManageHosts, fmt.Sprintf("localhost:%v", port)) + } } -func startNode() (*cluster.Cluster, func()) { +func startNode(autoManagePort int) (*cluster.Cluster, func()) { system := actor.NewActorSystem() - provider := automanaged.New() + provider := automanaged.NewWithConfig(time.Second*2, autoManagePort, autoManageHosts...) lookup, clean := shared.NewLockUp(system) config := remote.Configure("localhost", 0) diff --git a/examples/cluster-identitylookup-postgresql/shared/lookup.go b/examples/cluster-identitylookup-postgresql/shared/lookup.go index 7a20ea97a..d95d2cb5d 100644 --- a/examples/cluster-identitylookup-postgresql/shared/lookup.go +++ b/examples/cluster-identitylookup-postgresql/shared/lookup.go @@ -10,26 +10,32 @@ import ( "github.com/testcontainers/testcontainers-go/modules/postgres" "github.com/testcontainers/testcontainers-go/wait" "log" + "sync" "time" ) +var containerRunOnce = &sync.Once{} +var postgresContainer *postgres.PostgresContainer + func NewLockUp(actorSystem *actor.ActorSystem) (cluster.IdentityLookup, func()) { ctx := context.Background() dbName := "users" dbUser := "user" dbPassword := "password" - - postgresContainer, err := postgres.Run(ctx, - "postgres:16-alpine", - postgres.WithDatabase(dbName), - postgres.WithUsername(dbUser), - postgres.WithPassword(dbPassword), - testcontainers.WithWaitStrategy( - wait.ForLog("database system is ready to accept connections"). - WithOccurrence(2). - WithStartupTimeout(5*time.Second), - ), - ) + var err error + containerRunOnce.Do(func() { + postgresContainer, err = postgres.Run(ctx, + "postgres:16-alpine", + postgres.WithDatabase(dbName), + postgres.WithUsername(dbUser), + postgres.WithPassword(dbPassword), + testcontainers.WithWaitStrategy( + wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2). + WithStartupTimeout(5*time.Second), + ), + ) + }) if err != nil { log.Printf("failed to start container: %s", err) @@ -37,7 +43,7 @@ func NewLockUp(actorSystem *actor.ActorSystem) (cluster.IdentityLookup, func()) } connString := postgresContainer.MustConnectionString(ctx, "sslmode=disable", "application_name=test") - + println(connString) pool, err := pgxpool.New(context.Background(), connString) if err != nil { panic(err) @@ -46,7 +52,9 @@ func NewLockUp(actorSystem *actor.ActorSystem) (cluster.IdentityLookup, func()) if err := prepareTables(ctx, connString); err != nil { panic(err) } - return cluster.NewIdentityStorageLookup(postgresql.NewStorageLookup(pool, actorSystem.Logger(), actorSystem.Root, time.Second)), func() { + postgresLookup := postgresql.NewStorageLookup(pool, actorSystem.Logger(), actorSystem.Root, time.Second*3) + postgresLookup.Init() + return cluster.NewIdentityStorageLookup(postgresLookup), func() { if err := testcontainers.TerminateContainer(postgresContainer); err != nil { log.Printf("failed to terminate container: %s", err) } @@ -67,8 +75,8 @@ CREATE TABLE IF NOT EXISTS spawn_locks identity TEXT NOT NULL, kind TEXT NOT NULL, locked_at TIMESTAMP WITH TIME ZONE DEFAULT now(), - CONSTRAINT spawn_locks_pk PRIMARY KEY (lock_id), - UNIQUE (identity, kind) + unlocked_at TIMESTAMP WITH TIME ZONE, + PRIMARY KEY (identity, kind) ); `, From cef98cbba9163e88669ed616f780fa3c8d0e6ed6 Mon Sep 17 00:00:00 2001 From: "ryota.suzuki" Date: Wed, 15 Jan 2025 01:28:51 +0900 Subject: [PATCH 11/12] add test --- cluster/identity_storage_lookup.go | 2 +- .../postgresql/activation_waiting_actor.go | 2 +- .../postgresql/ddl/activations.sql | 5 +- .../postgresql/ddl/spawn_locks.sql | 1 + cluster/identitylookup/postgresql/ddl/sql.go | 9 + .../postgresql/storage_lookup.go | 95 +++++--- .../postgresql/storage_lookup_test.go | 226 ++++++++++++++++++ .../{node2 => node}/main.go | 4 + .../node1/main.go | 40 ---- .../shared/lookup.go | 34 +-- 10 files changed, 311 insertions(+), 107 deletions(-) create mode 100644 cluster/identitylookup/postgresql/ddl/sql.go create mode 100644 cluster/identitylookup/postgresql/storage_lookup_test.go rename examples/cluster-identitylookup-postgresql/{node2 => node}/main.go (96%) delete mode 100644 examples/cluster-identitylookup-postgresql/node1/main.go diff --git a/cluster/identity_storage_lookup.go b/cluster/identity_storage_lookup.go index c9ec9891f..727c76baf 100644 --- a/cluster/identity_storage_lookup.go +++ b/cluster/identity_storage_lookup.go @@ -63,7 +63,7 @@ func (i *IdentityStorageLookup) Get(clusterIdentity *ClusterIdentity) *actor.PID return response.Pid } -func (i *IdentityStorageLookup) RemovePid(clusterIdentity *ClusterIdentity, pid *actor.PID) { +func (i *IdentityStorageLookup) RemovePid(clusterIdentity *ClusterIdentity, _ *actor.PID) { if i.system.IsStopped() { return } diff --git a/cluster/identitylookup/postgresql/activation_waiting_actor.go b/cluster/identitylookup/postgresql/activation_waiting_actor.go index 3668b5c16..4585cca53 100644 --- a/cluster/identitylookup/postgresql/activation_waiting_actor.go +++ b/cluster/identitylookup/postgresql/activation_waiting_actor.go @@ -62,7 +62,7 @@ func newActivationWaitingActor(pool *pgxpool.Pool, poolingQueryInterval time.Dur logger: logger.With( slog.String("actorType", "ActivationWaitingActor"), ), - maxQueryRetryPerWaiting: 20, + maxQueryRetryPerWaiting: 3, } return this } diff --git a/cluster/identitylookup/postgresql/ddl/activations.sql b/cluster/identitylookup/postgresql/ddl/activations.sql index 9f48d2f05..7dbc627e2 100644 --- a/cluster/identitylookup/postgresql/ddl/activations.sql +++ b/cluster/identitylookup/postgresql/ddl/activations.sql @@ -7,10 +7,7 @@ CREATE TABLE IF NOT EXISTS activations member_id TEXT NOT NULL, lock_id TEXT NOT NULL, activated_at TIMESTAMP WITH TIME ZONE DEFAULT now(), - PRIMARY KEY (identity, kind), - -- lockの無効化を考慮したい - FOREIGN KEY (lock_id) references spawn_locks (lock_id) - + PRIMARY KEY (identity, kind) ); CREATE INDEX IF NOT EXISTS activations_member_idx ON activations (member_id); diff --git a/cluster/identitylookup/postgresql/ddl/spawn_locks.sql b/cluster/identitylookup/postgresql/ddl/spawn_locks.sql index fec24bf3f..db2742547 100644 --- a/cluster/identitylookup/postgresql/ddl/spawn_locks.sql +++ b/cluster/identitylookup/postgresql/ddl/spawn_locks.sql @@ -3,6 +3,7 @@ CREATE TABLE IF NOT EXISTS spawn_locks lock_id TEXT NOT NULL, identity TEXT NOT NULL, kind TEXT NOT NULL, + member_id TEXT NOT NULL, locked_at TIMESTAMP WITH TIME ZONE DEFAULT now(), CONSTRAINT spawn_locks_pk PRIMARY KEY (lock_id), UNIQUE (identity, kind) diff --git a/cluster/identitylookup/postgresql/ddl/sql.go b/cluster/identitylookup/postgresql/ddl/sql.go new file mode 100644 index 000000000..ca8ee8bb4 --- /dev/null +++ b/cluster/identitylookup/postgresql/ddl/sql.go @@ -0,0 +1,9 @@ +package ddl + +import _ "embed" + +//go:embed spawn_locks.sql +var SpawnLocksSql string + +//go:embed activations.sql +var ActivationsSql string diff --git a/cluster/identitylookup/postgresql/storage_lookup.go b/cluster/identitylookup/postgresql/storage_lookup.go index f54a00fcf..799d93c4e 100644 --- a/cluster/identitylookup/postgresql/storage_lookup.go +++ b/cluster/identitylookup/postgresql/storage_lookup.go @@ -11,6 +11,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/lithammer/shortuuid/v4" "log/slog" + "strings" "time" ) @@ -20,16 +21,18 @@ type StorageLookup struct { waiting *actor.PID rootContext *actor.RootContext waitingForActivationTimeout time.Duration + memberId string } var _ cluster.StorageLookup = (*StorageLookup)(nil) -func NewStorageLookup(pool *pgxpool.Pool, logger *slog.Logger, rootContext *actor.RootContext, waitingForActivationTimeout time.Duration) *StorageLookup { +func NewStorageLookup(pool *pgxpool.Pool, logger *slog.Logger, rootContext *actor.RootContext, waitingForActivationTimeout time.Duration, memberId string) *StorageLookup { return &StorageLookup{ pool: pool, logger: logger.With(slog.String("component", "postgresql.StorageLookup"), slog.String("ActorSystemId", rootContext.ActorSystem().ID)), rootContext: rootContext, waitingForActivationTimeout: waitingForActivationTimeout, + memberId: memberId, } } @@ -50,7 +53,12 @@ func (s *StorageLookup) Shutdown() *cluster.ClusterIdentity { } func (s *StorageLookup) TryGetExistingActivation(clusterIdentity *cluster.ClusterIdentity) *cluster.StoredActivation { - row, err := s.pool.Query(context2.Background(), "SELECT * FROM activations WHERE identity = $1 AND kind = $2", clusterIdentity.Identity, clusterIdentity.Kind) + row, err := s.pool.Query( + context2.Background(), + "SELECT * FROM activations WHERE identity = $1 AND kind = $2", + clusterIdentity.Identity, + clusterIdentity.Kind, + ) if err != nil { s.logger.Error("error while querying for activation", slog.Any("error", err)) return nil @@ -61,7 +69,10 @@ func (s *StorageLookup) TryGetExistingActivation(clusterIdentity *cluster.Cluste MemberID: row.MemberId, } } else if errors.Is(err, pgx.ErrNoRows) { - s.logger.Info("activation not found", slog.Any("identity", clusterIdentity.Identity), slog.Any("kind", clusterIdentity.Kind)) + s.logger.Info("activation not found", + slog.Any("identity", clusterIdentity.Identity), + slog.Any("kind", clusterIdentity.Kind), + ) return nil } else { s.logger.Error("error while collecting row", slog.Any("error", err)) @@ -72,33 +83,34 @@ func (s *StorageLookup) TryGetExistingActivation(clusterIdentity *cluster.Cluste func (s *StorageLookup) TryAcquireLock(clusterIdentity *cluster.ClusterIdentity) *cluster.SpawnLock { lockId := shortuuid.New() - /// Lockのとりっぱなし防止のための削除を考える - activation := s.TryGetExistingActivation(clusterIdentity) - if activation != nil { - s.logger.Info("activation already exists", slog.Any("identity", clusterIdentity.Identity), slog.Any("kind", clusterIdentity.Kind)) - return nil - } - - /// ロックが解除(Remove)されていてもロックから1minはロックは取れない - /// ロックが解除されていなくてもロックから1hour経過しているならロックは取れる effect, err := s.pool.Exec(context2.Background(), ` -INSERT INTO spawn_locks (lock_id, identity, kind) -VALUES ($1, $2, $3) -ON CONFLICT (identity, kind) DO UPDATE - SET lock_id = $1, locked_at = NOW(), unlocked_at = NULL -WHERE (spawn_locks.unlocked_at IS NOT NULL AND spawn_locks.locked_at < NOW() - INTERVAL '1 minute') - OR (spawn_locks.unlocked_at IS NULL AND spawn_locks.locked_at < NOW() - INTERVAL '1 hour'); +INSERT INTO spawn_locks (lock_id, identity, kind, member_id) +SELECT $1, $2, $3, $4 +WHERE NOT EXISTS ( + SELECT 1 + FROM activations + WHERE activations.identity = $2 + AND activations.kind = $3 +); `, - lockId, clusterIdentity.Identity, clusterIdentity.Kind, + lockId, + clusterIdentity.Identity, + clusterIdentity.Kind, + s.memberId, ) + if err != nil { + if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { + s.logger.Info("lock already exists with constraint failure", slog.Any("identity", clusterIdentity.Identity), slog.Any("kind", clusterIdentity.Kind)) + return nil + } s.logger.Error("error while inserting lock", slog.Any("error", err)) return nil } if effect.RowsAffected() == 0 { - s.logger.Info("lock already exists", slog.Any("identity", clusterIdentity.Identity), slog.Any("kind", clusterIdentity.Kind)) + s.logger.Info("lock already exists activation exists yet", slog.Any("identity", clusterIdentity.Identity), slog.Any("kind", clusterIdentity.Kind)) return nil } @@ -109,8 +121,14 @@ WHERE (spawn_locks.unlocked_at IS NOT NULL AND spawn_locks.locked_at < NOW() - I } func (s *StorageLookup) WaitForActivation(clusterIdentity *cluster.ClusterIdentity) *cluster.StoredActivation { - s.logger.Info("waiting for activation", slog.Any("identity", clusterIdentity.Identity), slog.Any("kind", clusterIdentity.Kind)) - res, err := s.rootContext.RequestFuture(s.waiting, &AddWaiting{clusterIdentity: clusterIdentity}, s.waitingForActivationTimeout).Result() + s.logger.Info("waiting for activation", + slog.Any("identity", clusterIdentity.Identity), + slog.Any("kind", clusterIdentity.Kind), + ) + res, err := s.rootContext.RequestFuture( + s.waiting, &AddWaiting{clusterIdentity: clusterIdentity}, + s.waitingForActivationTimeout, + ).Result() if err != nil { s.logger.Error("error while waiting for activation", slog.Any("error", err)) return nil @@ -133,16 +151,11 @@ func (s *StorageLookup) WaitForActivation(clusterIdentity *cluster.ClusterIdenti } func (s *StorageLookup) RemoveLock(spawnLock cluster.SpawnLock) { - _, err := s.pool.Exec(context2.Background(), "UPDATE spawn_locks SET unlocked_at = now() WHERE lock_id = $1", spawnLock.LockID) - if err != nil { - s.logger.Error("error while deleting lock", slog.Any("error", err)) - } -} - -func (s *StorageLookup) DeleteLock(spawnLock cluster.SpawnLock) { _, err := s.pool.Exec(context2.Background(), "DELETE FROM spawn_locks WHERE lock_id = $1", spawnLock.LockID) + s.logger.Info("removing lock", slog.Any("lockId", spawnLock.LockID)) if err != nil { s.logger.Error("error while deleting lock", slog.Any("error", err)) + panic(fmt.Sprintf("error while deleting lock: %v", err)) } } @@ -152,7 +165,16 @@ func (s *StorageLookup) StoreActivation(memberID string, spawnLock *cluster.Spaw s.logger.Error("error while marshalling pid", slog.Any("error", err)) panic(err) } - _, err = s.pool.Exec(context2.Background(), "INSERT INTO activations (identity, kind, identity_key, member_id, pid, lock_id) VALUES ($1, $2, $3, $4, $5, $6)", spawnLock.ClusterIdentity.Identity, spawnLock.ClusterIdentity.Kind, spawnLock.ClusterIdentity.AsKey(), memberID, pidJson, spawnLock.LockID) + _, err = s.pool.Exec(context2.Background(), ` +INSERT INTO activations (identity, kind, identity_key, member_id, pid, lock_id) VALUES ($1, $2, $3, $4, $5, $6) +`, + spawnLock.ClusterIdentity.Identity, + spawnLock.ClusterIdentity.Kind, + spawnLock.ClusterIdentity.AsKey(), + memberID, + pidJson, + spawnLock.LockID, + ) if err != nil { s.logger.Error("error while inserting activation", slog.Any("error", err)) panic(err) @@ -160,10 +182,16 @@ func (s *StorageLookup) StoreActivation(memberID string, spawnLock *cluster.Spaw } func (s *StorageLookup) RemoveActivation(lock *cluster.SpawnLock) { - _, err := s.pool.Exec(context2.Background(), "DELETE FROM activations WHERE lock_id = $1", lock.LockID) + rows, err := s.pool.Query(context2.Background(), `DELETE FROM activations WHERE identity = $1 AND kind = $2 RETURNING *`, lock.ClusterIdentity.Identity, lock.ClusterIdentity.Kind) if err != nil { s.logger.Error("error while deleting activation", slog.Any("error", err)) } + s.logger.Info("removing activation", slog.Any("identity", lock.ClusterIdentity.Identity), slog.Any("kind", lock.ClusterIdentity.Kind)) + deletedRow, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[ActivationsTableRow]) + _, err = s.pool.Exec(context2.Background(), "DELETE FROM spawn_locks WHERE lock_id = $1", deletedRow.LockId) + if err != nil { + s.logger.Error("error while deleting spawn_locks", slog.Any("error", err)) + } } func (s *StorageLookup) RemoveMemberId(memberID string) { @@ -171,6 +199,11 @@ func (s *StorageLookup) RemoveMemberId(memberID string) { if err != nil { s.logger.Error("error while deleting activations", slog.Any("error", err)) } + + _, err = s.pool.Exec(context2.Background(), "DELETE FROM spawn_locks WHERE member_id = $1", memberID) + if err != nil { + s.logger.Error("error while deleting spawn_locks", slog.Any("error", err)) + } } type ActivationsTableRow struct { diff --git a/cluster/identitylookup/postgresql/storage_lookup_test.go b/cluster/identitylookup/postgresql/storage_lookup_test.go new file mode 100644 index 000000000..12def69a8 --- /dev/null +++ b/cluster/identitylookup/postgresql/storage_lookup_test.go @@ -0,0 +1,226 @@ +package postgresql + +import ( + "context" + _ "embed" + "encoding/json" + "github.com/asynkron/protoactor-go/actor" + "github.com/asynkron/protoactor-go/cluster" + "github.com/asynkron/protoactor-go/cluster/identitylookup/postgresql/ddl" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/testcontainers/testcontainers-go/wait" + "log" + "math/rand" + "strconv" + "testing" + "time" +) + +func TestStorageLookup_StoreActivation(t *testing.T) { + actorSystem := actor.NewActorSystem() + lookup, cleanup := PrepareLookup(actorSystem) + defer cleanup() + + clusterIdentity := &cluster.ClusterIdentity{ + Identity: "test", + Kind: "test", + } + + memberID := "test" + + spawnLock := &cluster.SpawnLock{ + LockID: "test", + ClusterIdentity: clusterIdentity, + } + + pid := actor.NewPID("test", "test") + lookup.StoreActivation(memberID, spawnLock, pid) + gotActivation := lookup.TryGetExistingActivation(clusterIdentity) + if gotActivation == nil { + t.Error("activation not found") + return + } + + pidJson, _ := json.Marshal(pid) + if gotActivation.Pid != string(pidJson) { + t.Errorf("expected %s, got %s", string(pidJson), gotActivation.Pid) + } + + if gotActivation.MemberID != memberID { + t.Errorf("expected %s, got %s", memberID, gotActivation.MemberID) + } +} + +func TestStorageLookup_RemoveActivation(t *testing.T) { + actorSystem := actor.NewActorSystem() + lookup, cleanup := PrepareLookup(actorSystem) + defer cleanup() + + clusterIdentity := &cluster.ClusterIdentity{ + Identity: "test", + Kind: "test", + } + + memberID := "test" + + spawnLock := &cluster.SpawnLock{ + LockID: "test", + ClusterIdentity: clusterIdentity, + } + + pid := actor.NewPID("test", "test") + lookup.StoreActivation(memberID, spawnLock, pid) + lookup.RemoveActivation(spawnLock) + gotActivation := lookup.TryGetExistingActivation(clusterIdentity) + if gotActivation != nil { + t.Error("activation not removed") + } +} + +func TestStorageLookup_RemoveMemberId(t *testing.T) { + actorSystem := actor.NewActorSystem() + lookup, cleanup := PrepareLookup(actorSystem) + defer cleanup() + + clusterIdentity := &cluster.ClusterIdentity{ + Identity: "test", + Kind: "test", + } + + memberID := "test" + + spawnLock := &cluster.SpawnLock{ + LockID: "test", + ClusterIdentity: clusterIdentity, + } + + pid := actor.NewPID("test", "test") + lookup.StoreActivation(memberID, spawnLock, pid) + lookup.RemoveMemberId(memberID) + gotActivation := lookup.TryGetExistingActivation(clusterIdentity) + if gotActivation != nil { + t.Error("activation not removed") + } +} + +func TestStorageLookup_TryAcquireLock(t *testing.T) { + + t.Run("lock acquired", func(t *testing.T) { + actorSystem := actor.NewActorSystem() + lookup, cleanup := PrepareLookup(actorSystem) + defer cleanup() + + clusterIdentity := &cluster.ClusterIdentity{ + Identity: strconv.FormatFloat(rand.Float64(), 'f', -1, 64), + Kind: "test", + } + + lock := lookup.TryAcquireLock(clusterIdentity) + if lock == nil { + t.Error("lock not acquired") + return + } + + if lock := lookup.TryAcquireLock(clusterIdentity); lock != nil { + t.Error("lock acquired twice") + } + lookup.RemoveLock(*lock) + if lock := lookup.TryAcquireLock(clusterIdentity); lock == nil { + t.Error("lock not acquired") + } + }) + t.Run("lock cant acquired when activation exists", func(t *testing.T) { + actorSystem := actor.NewActorSystem() + lookup, cleanup := PrepareLookup(actorSystem) + defer cleanup() + + clusterIdentity := &cluster.ClusterIdentity{ + Identity: strconv.FormatFloat(rand.Float64(), 'f', -1, 64), + Kind: "test", + } + + lock := lookup.TryAcquireLock(clusterIdentity) + if lock == nil { + t.Error("lock not acquired") + return + } + + lookup.RemoveLock(*lock) + lookup.StoreActivation("test", + lock, + actor.NewPID("test", "test"), + ) + if lock := lookup.TryAcquireLock(clusterIdentity); lock != nil { + t.Error("lock acquired twice") + } + lookup.RemoveActivation(lock) + if lock := lookup.TryAcquireLock(clusterIdentity); lock == nil { + t.Error("lock not acquired") + } + }) + +} + +func PrepareLookup(actorSystem *actor.ActorSystem) (*StorageLookup, func()) { + ctx := context.Background() + dbName := "users" + dbUser := "user" + dbPassword := "password" + var err error + postgresContainer, err := postgres.Run(ctx, + "postgres:16-alpine", + postgres.WithDatabase(dbName), + postgres.WithUsername(dbUser), + postgres.WithPassword(dbPassword), + testcontainers.WithWaitStrategy( + wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2). + WithStartupTimeout(5*time.Second), + ), + ) + if err != nil { + log.Printf("failed to start container: %s", err) + panic(err) + } + + connString := postgresContainer.MustConnectionString(ctx, "sslmode=disable", "application_name=test") + pool, err := pgxpool.New(context.Background(), connString) + if err != nil { + panic(err) + } + + if err := prepareTables(ctx, connString); err != nil { + panic(err) + } + postgresLookup := NewStorageLookup(pool, actorSystem.Logger(), actorSystem.Root, time.Second*3, actorSystem.ID) + postgresLookup.Init() + return postgresLookup, func() { + postgresLookup.Shutdown() + if err := testcontainers.TerminateContainer(postgresContainer); err != nil { + log.Printf("failed to terminate container: %s", err) + } + } +} + +func prepareTables(ctx context.Context, connString string) error { + pool, err := pgxpool.New(ctx, connString) + if err != nil { + return err + } + + queries := []string{ + ddl.ActivationsSql, + ddl.SpawnLocksSql, + } + + for _, query := range queries { + _, err = pool.Exec(ctx, query) + if err != nil { + return err + } + } + + return nil +} diff --git a/examples/cluster-identitylookup-postgresql/node2/main.go b/examples/cluster-identitylookup-postgresql/node/main.go similarity index 96% rename from examples/cluster-identitylookup-postgresql/node2/main.go rename to examples/cluster-identitylookup-postgresql/node/main.go index 0b21f2930..36cff4198 100644 --- a/examples/cluster-identitylookup-postgresql/node2/main.go +++ b/examples/cluster-identitylookup-postgresql/node/main.go @@ -83,6 +83,10 @@ func startNode(autoManagePort int) (*cluster.Cluster, func()) { switch msg := ctx.Message().(type) { case *actor.Started: fmt.Printf("Started %v", msg) + go func() { + time.Sleep(time.Second * 15) + ctx.Stop(ctx.Self()) + }() case *shared.HelloRequest: fmt.Printf("Hello %v\n", msg.Name) ctx.Respond(&shared.HelloResponse{}) diff --git a/examples/cluster-identitylookup-postgresql/node1/main.go b/examples/cluster-identitylookup-postgresql/node1/main.go deleted file mode 100644 index 4f747d344..000000000 --- a/examples/cluster-identitylookup-postgresql/node1/main.go +++ /dev/null @@ -1,40 +0,0 @@ -package main - -import ( - "cluster-identitylookup-postgresql/shared" - "fmt" - console "github.com/asynkron/goconsole" - "github.com/asynkron/protoactor-go/actor" - "github.com/asynkron/protoactor-go/cluster" - "github.com/asynkron/protoactor-go/cluster/clusterproviders/automanaged" - "github.com/asynkron/protoactor-go/remote" -) - -func main() { - c, clean := startNode() - defer clean() - - fmt.Print("\nBoot other nodes and press Enter\n") - console.ReadLine() - pid := c.Get("abc", "hello") - fmt.Printf("Got pid %v", pid) - res, _ := c.Request("abc", "hello", &shared.HelloRequest{Name: "Roger"}) - fmt.Printf("Got response %v", res) - - fmt.Println() - console.ReadLine() - c.Shutdown(true) -} - -func startNode() (*cluster.Cluster, func()) { - system := actor.NewActorSystem() - - provider := automanaged.New() - lookup, clean := shared.NewLockUp(system) - config := remote.Configure("localhost", 0) - clusterConfig := cluster.Configure("my-cluster", provider, lookup, config) - c := cluster.New(system, clusterConfig) - c.StartMember() - - return c, clean -} diff --git a/examples/cluster-identitylookup-postgresql/shared/lookup.go b/examples/cluster-identitylookup-postgresql/shared/lookup.go index d95d2cb5d..1da462a0f 100644 --- a/examples/cluster-identitylookup-postgresql/shared/lookup.go +++ b/examples/cluster-identitylookup-postgresql/shared/lookup.go @@ -5,6 +5,7 @@ import ( "github.com/asynkron/protoactor-go/actor" "github.com/asynkron/protoactor-go/cluster" "github.com/asynkron/protoactor-go/cluster/identitylookup/postgresql" + "github.com/asynkron/protoactor-go/cluster/identitylookup/postgresql/ddl" "github.com/jackc/pgx/v5/pgxpool" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/postgres" @@ -52,7 +53,7 @@ func NewLockUp(actorSystem *actor.ActorSystem) (cluster.IdentityLookup, func()) if err := prepareTables(ctx, connString); err != nil { panic(err) } - postgresLookup := postgresql.NewStorageLookup(pool, actorSystem.Logger(), actorSystem.Root, time.Second*3) + postgresLookup := postgresql.NewStorageLookup(pool, actorSystem.Logger(), actorSystem.Root, time.Second*3, actorSystem.ID) postgresLookup.Init() return cluster.NewIdentityStorageLookup(postgresLookup), func() { if err := testcontainers.TerminateContainer(postgresContainer); err != nil { @@ -68,35 +69,8 @@ func prepareTables(ctx context.Context, connString string) error { } queries := []string{ - ` -CREATE TABLE IF NOT EXISTS spawn_locks -( - lock_id TEXT NOT NULL, - identity TEXT NOT NULL, - kind TEXT NOT NULL, - locked_at TIMESTAMP WITH TIME ZONE DEFAULT now(), - unlocked_at TIMESTAMP WITH TIME ZONE, - PRIMARY KEY (identity, kind) -); - -`, - ` -CREATE TABLE IF NOT EXISTS activations -( - identity TEXT NOT NULL, - kind TEXT NOT NULL, - identity_key TEXT NOT NULL, - member_id TEXT NOT NULL, - pid json NOT NULL, - lock_id TEXT NOT NULL, - activated_at TIMESTAMP WITH TIME ZONE DEFAULT now(), - PRIMARY KEY (identity, kind) --- FOREIGN KEY (lock_id) references spawn_locks (lock_id) -); -`, - ` -CREATE INDEX IF NOT EXISTS activations_member_idx ON activations (member_id); -`, + ddl.ActivationsSql, + ddl.SpawnLocksSql, } for _, query := range queries { From 166c4d4b48f8b0636e04c5e8cacbf7298d5f90f8 Mon Sep 17 00:00:00 2001 From: "ryota.suzuki" Date: Thu, 16 Jan 2025 11:28:26 +0900 Subject: [PATCH 12/12] rm comment --- cluster/identity_storage_placement_actor.go | 20 ++++++++++++++++++- .../postgresql/activation_waiting_actor.go | 2 -- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/cluster/identity_storage_placement_actor.go b/cluster/identity_storage_placement_actor.go index e66aeaff0..524e0d8c7 100644 --- a/cluster/identity_storage_placement_actor.go +++ b/cluster/identity_storage_placement_actor.go @@ -94,10 +94,28 @@ func (i *IdentityStoragePlacementActor) onActivationRequest(context actor.Contex func (i *IdentityStoragePlacementActor) spawnActor(ctx actor.Context, req *ActivationRequest, kind *ActivatedKind) { props := WithClusterIdentity(kind.Props, req.ClusterIdentity) pid := ctx.SpawnPrefix(props, req.ClusterIdentity.Identity) + + storeActivation := func(pid *actor.PID) (ok bool) { + ok = false + defer func() { + if r := recover(); r != nil { + i.logger.Error("Failed to store activation", slog.Any("error", r)) + ctx.Stop(pid) + ok = false + } + }() + i.identityStorageLookup.Storage.StoreActivation(i.cluster.ActorSystem.ID, newSpawnLock(req.RequestId, req.ClusterIdentity), pid) + return true + } + + ok := storeActivation(pid) + if !ok { + ctx.Respond(&ActivationResponse{Pid: nil, Failed: true}) + return + } kind.Inc() /// TODO: member selectionを考慮 - i.identityStorageLookup.Storage.StoreActivation(i.cluster.ActorSystem.ID, newSpawnLock(req.RequestId, req.ClusterIdentity), pid) i.logger.Info("Activation Stored", slog.Any("pid", pid), slog.Any("clusterIdentity", req.ClusterIdentity), slog.String("key", req.ClusterIdentity.AsKey())) i.actors[req.ClusterIdentity.AsKey()] = GrainMeta{ ID: req.ClusterIdentity, diff --git a/cluster/identitylookup/postgresql/activation_waiting_actor.go b/cluster/identitylookup/postgresql/activation_waiting_actor.go index 4585cca53..c544bee6d 100644 --- a/cluster/identitylookup/postgresql/activation_waiting_actor.go +++ b/cluster/identitylookup/postgresql/activation_waiting_actor.go @@ -174,8 +174,6 @@ SELECT * FROM activations WHERE identity_key = ANY($1) } func (a *ActivationWaitingActor) onStopping(context actor.Context) { - /// TODO: clusterの停止に伴ってすべての待ちリストへゴメンを送る - if a.cancelScheduler != nil { a.cancelScheduler() }