From f48a56026fa3bf4b1cdf1dfd0c5e75bee5505905 Mon Sep 17 00:00:00 2001 From: Johnny Graettinger Date: Mon, 29 Jan 2024 15:52:46 +0000 Subject: [PATCH] gazctl shards prune: add test case for new prune behavior --- cmd/gazctl/gazctlcmd/shards_prune_test.go | 283 ++++++++++++++++++++++ 1 file changed, 283 insertions(+) diff --git a/cmd/gazctl/gazctlcmd/shards_prune_test.go b/cmd/gazctl/gazctlcmd/shards_prune_test.go index 9f453146..28f8abda 100644 --- a/cmd/gazctl/gazctlcmd/shards_prune_test.go +++ b/cmd/gazctl/gazctlcmd/shards_prune_test.go @@ -1,14 +1,297 @@ package gazctlcmd import ( + "encoding/json" + "fmt" "math/rand" "testing" "github.com/stretchr/testify/require" pb "go.gazette.dev/core/broker/protocol" + pc "go.gazette.dev/core/consumer/protocol" "go.gazette.dev/core/consumer/recoverylog" ) +func TestFragmentsAreRetainedWhenAnyHintsRequireThem(t *testing.T) { + // The goal is to allow a shard to recover using any of the persisted + // hints. In order for that to work, all fragments that come after the final + // hinted segment of _any_ hints must be retained. In this case, the primary + // hints are quite old. So we test a fragment that would otherwise be elidgable + // for pruning, and assert that it is retained due to coming after the segment + // in the primary hints. + var fixture = `{ + "header": { + "process_id": { + "zone": "us-central1-c", + "suffix": "flow-reactor-74fcd7f9db-n5c5j" + }, + "route": { + "members": null, + "primary": -1 + }, + "etcd": { + "cluster_id": 9261053468663690000, + "member_id": 14358579373743376000, + "revision": 57121751, + "raft_term": 140 + } + }, + "primary_hints": { + "hints": { + "log": "example/log", + "live_nodes": [ + { + "fnode": 1018, + "segments": [ + { + "author": 3418428788, + "first_seq_no": 1018, + "first_offset": 129100, + "first_checksum": 2628272215, + "last_seq_no": 1024 + } + ] + } + ], + "properties": null + } + }, + "backup_hints": [ + { + "hints": { + "log": "example/log", + "live_nodes": [ + { + "fnode": 1109, + "segments": [ + { + "author": 3418428788, + "first_seq_no": 1109, + "first_offset": 140706, + "first_checksum": 1095500179, + "last_seq_no": 1115, + "last_offset": 141535 + } + ] + }, + { + "fnode": 1289, + "segments": [ + { + "author": 2080950356, + "first_seq_no": 1289, + "first_offset": 152255, + "first_checksum": 777889612, + "last_seq_no": 1290, + "last_offset": 153297 + } + ] + }, + { + "fnode": 1642, + "segments": [ + { + "author": 245017003, + "first_seq_no": 1642, + "first_offset": 245137, + "first_checksum": 3605342194, + "last_seq_no": 1645, + "last_offset": 251592 + } + ] + }, + { + "fnode": 1657, + "segments": [ + { + "author": 3711033845, + "first_seq_no": 1657, + "first_offset": 252114, + "first_checksum": 794580610, + "last_seq_no": 1660, + "last_offset": 252379 + } + ] + }, + { + "fnode": 1661, + "segments": [ + { + "author": 3711033845, + "first_seq_no": 1661, + "first_offset": 252379, + "first_checksum": 157414297, + "last_seq_no": 1665, + "last_offset": 252574 + } + ] + }, + { + "fnode": 1666, + "segments": [ + { + "author": 3711033845, + "first_seq_no": 1666, + "first_offset": 252574, + "first_checksum": 157986885, + "last_seq_no": 1666, + "last_offset": 252610 + } + ] + }, + { + "fnode": 1670, + "segments": [ + { + "author": 3711033845, + "first_seq_no": 1670, + "first_offset": 252737, + "first_checksum": 1747582972, + "last_seq_no": 1673, + "last_offset": 259192 + } + ] + } + ], + "properties": [ + { + "path": "/IDENTITY", + "content": "0a4d4620-a93f-468a-9c12-26de543261f0\n" + } + ] + } + }, + { + "hints": { + "log": "example/log", + "live_nodes": [ + { + "fnode": 1109, + "segments": [ + { + "author": 3418428788, + "first_seq_no": 1109, + "first_offset": 140706, + "first_checksum": 1095500179, + "last_seq_no": 1115, + "last_offset": 141535 + } + ] + }, + { + "fnode": 1289, + "segments": [ + { + "author": 2080950356, + "first_seq_no": 1289, + "first_offset": 152255, + "first_checksum": 777889612, + "last_seq_no": 1290, + "last_offset": 153297 + } + ] + }, + { + "fnode": 1614, + "segments": [ + { + "author": 1701696287, + "first_seq_no": 1614, + "first_offset": 237537, + "first_checksum": 3210869242, + "last_seq_no": 1617, + "last_offset": 243992 + } + ] + }, + { + "fnode": 1629, + "segments": [ + { + "author": 245017003, + "first_seq_no": 1629, + "first_offset": 244514, + "first_checksum": 3153286517, + "last_seq_no": 1632, + "last_offset": 244779 + } + ] + }, + { + "fnode": 1633, + "segments": [ + { + "author": 245017003, + "first_seq_no": 1633, + "first_offset": 244779, + "first_checksum": 3701750198, + "last_seq_no": 1637, + "last_offset": 244974 + } + ] + }, + { + "fnode": 1638, + "segments": [ + { + "author": 245017003, + "first_seq_no": 1638, + "first_offset": 244974, + "first_checksum": 2949463732, + "last_seq_no": 1638, + "last_offset": 245010 + } + ] + }, + { + "fnode": 1642, + "segments": [ + { + "author": 245017003, + "first_seq_no": 1642, + "first_offset": 245137, + "first_checksum": 3605342194, + "last_seq_no": 1645, + "last_offset": 251592 + } + ] + } + ], + "properties": [ + { + "path": "/IDENTITY", + "content": "0a4d4620-a93f-468a-9c12-26de543261f0\n" + } + ] + } + } + ] + }` + + var resp pc.GetHintsResponse + require.NoError(t, json.Unmarshal([]byte(fixture), &resp)) + + var set = make(map[pb.Journal]logUsage) + + for i, hints := range resp.BackupHints { + foldHintsIntoSegments(fmt.Sprintf("backup.%d", i), *hints.Hints, set) + } + // Assert that the fragment would be pruned if we only accounted for the backup hints + require.False(t, set["example/log"].isFragmentNeeded(pb.Fragment{ + Journal: "example/log", + Begin: 243993, + End: 244000, + })) + + // Now add the primary hints and assert that the fragment would be retained + foldHintsIntoSegments("primary", *resp.PrimaryHints.Hints, set) + require.True(t, set["example/log"].isFragmentNeeded(pb.Fragment{ + Journal: "example/log", + Begin: 243993, + End: 244000, + })) +} + func TestSegmentFoldingWithSingleLog(t *testing.T) { var m = make(map[pb.Journal]logUsage)