Skip to content

Commit 09cd7b2

Browse files
vitess-bot[bot]vitess-bot[bot]
authored
[release-18.0] Migrate CreateLookupVindex and ExternalizeVindex to vtctldclient (#14086) (#14183)
Signed-off-by: Matt Lord <mattalord@gmail.com> Co-authored-by: vitess-bot[bot] <108069721+vitess-bot[bot]@users.noreply.github.com>
1 parent 8bb426b commit 09cd7b2

File tree

27 files changed

+8188
-2467
lines changed

27 files changed

+8188
-2467
lines changed

‎go/cmd/vtctldclient/command/root.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828

2929
// These imports ensure init()s within them get called and they register their commands/subcommands.
3030
vreplcommon "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/common"
31+
_ "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/lookupvindex"
3132
_ "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/movetables"
3233
_ "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/reshard"
3334
_ "vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/vdiff"

‎go/cmd/vtctldclient/command/vreplication/common/complete.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,10 @@ var CompleteOptions = struct {
2020

2121
func GetCompleteCommand(opts *SubCommandsOpts) *cobra.Command {
2222
cmd := &cobra.Command{
23-
Use: "complete",
24-
Short: "Complete a MoveTables VReplication workflow.",
25-
Example: `vtctldclient --server localhost:15999 movetables --workflow commerce2customer --target-keyspace customer complete`,
23+
Use: "complete",
24+
Short: fmt.Sprintf("Complete a %s VReplication workflow.", opts.SubCommand),
25+
Example: fmt.Sprintf(`vtctldclient --server localhost:15999 %s --workflow %s --target-keyspace customer complete`,
26+
opts.SubCommand, opts.Workflow),
2627
DisableFlagsInUseLine: true,
2728
Aliases: []string{"Complete"},
2829
Args: cobra.NoArgs,
@@ -68,7 +69,7 @@ func commandComplete(cmd *cobra.Command, args []string) error {
6869
}
6970
output = tout.Bytes()
7071
}
71-
fmt.Printf("%s\n", output)
72+
fmt.Println(string(output))
7273

7374
return nil
7475
}

‎go/cmd/vtctldclient/command/vreplication/lookupvindex/lookupvindex.go

Lines changed: 321 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,321 @@
1+
/*
2+
Copyright 2023 The Vitess Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package lookupvindex
18+
19+
import (
20+
"fmt"
21+
"strings"
22+
23+
"github.com/spf13/cobra"
24+
25+
"vitess.io/vitess/go/cmd/vtctldclient/cli"
26+
"vitess.io/vitess/go/cmd/vtctldclient/command/vreplication/common"
27+
28+
topodatapb "vitess.io/vitess/go/vt/proto/topodata"
29+
vschemapb "vitess.io/vitess/go/vt/proto/vschema"
30+
vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata"
31+
topoprotopb "vitess.io/vitess/go/vt/topo/topoproto"
32+
)
33+
34+
var (
35+
tabletTypesDefault = []topodatapb.TabletType{
36+
topodatapb.TabletType_REPLICA,
37+
topodatapb.TabletType_PRIMARY,
38+
}
39+
40+
baseOptions = struct {
41+
// This is where the lookup table and VReplicaiton workflow
42+
// will be created.
43+
TableKeyspace string
44+
// This will be the name of the Lookup Vindex and the name
45+
// of the VReplication workflow.
46+
Name string
47+
Vschema *vschemapb.Keyspace
48+
}{}
49+
50+
// base is the base command for all actions related to Lookup Vindexes.
51+
base = &cobra.Command{
52+
Use: "LookupVindex --name <name> --table-keyspace <keyspace> [command] [command-flags]",
53+
Short: "Perform commands related to creating, backfilling, and externalizing Lookup Vindexes using VReplication workflows.",
54+
DisableFlagsInUseLine: true,
55+
Aliases: []string{"lookupvindex"},
56+
Args: cobra.NoArgs,
57+
}
58+
59+
createOptions = struct {
60+
Keyspace string
61+
Type string
62+
TableOwner string
63+
TableOwnerColumns []string
64+
TableName string
65+
TableVindexType string
66+
Cells []string
67+
TabletTypes []topodatapb.TabletType
68+
TabletTypesInPreferenceOrder bool
69+
IgnoreNulls bool
70+
ContinueAfterCopyWithOwner bool
71+
}{}
72+
73+
externalizeOptions = struct {
74+
Keyspace string
75+
}{}
76+
77+
parseAndValidateCreate = func(cmd *cobra.Command, args []string) error {
78+
if createOptions.TableName == "" { // Use vindex name
79+
createOptions.TableName = baseOptions.Name
80+
}
81+
if !strings.Contains(createOptions.Type, "lookup") {
82+
return fmt.Errorf("vindex type must be a lookup vindex")
83+
}
84+
baseOptions.Vschema = &vschemapb.Keyspace{
85+
Vindexes: map[string]*vschemapb.Vindex{
86+
baseOptions.Name: {
87+
Type: createOptions.Type,
88+
Params: map[string]string{
89+
"table": baseOptions.TableKeyspace + "." + createOptions.TableName,
90+
"from": strings.Join(createOptions.TableOwnerColumns, ","),
91+
"to": "keyspace_id",
92+
"ignore_nulls": fmt.Sprintf("%t", createOptions.IgnoreNulls),
93+
},
94+
Owner: createOptions.TableOwner,
95+
},
96+
},
97+
Tables: map[string]*vschemapb.Table{
98+
createOptions.TableOwner: {
99+
ColumnVindexes: []*vschemapb.ColumnVindex{
100+
{
101+
Name: baseOptions.Name,
102+
Columns: createOptions.TableOwnerColumns,
103+
},
104+
},
105+
},
106+
createOptions.TableName: {
107+
ColumnVindexes: []*vschemapb.ColumnVindex{
108+
{
109+
// If the vindex name/type is empty then we'll fill this in
110+
// later using the defult for the column types.
111+
Name: createOptions.TableVindexType,
112+
Columns: createOptions.TableOwnerColumns,
113+
},
114+
},
115+
},
116+
},
117+
}
118+
119+
// VReplication specific flags.
120+
ttFlag := cmd.Flags().Lookup("tablet-types")
121+
if ttFlag != nil && ttFlag.Changed {
122+
createOptions.TabletTypes = tabletTypesDefault
123+
}
124+
cFlag := cmd.Flags().Lookup("cells")
125+
if cFlag != nil && cFlag.Changed {
126+
for i, cell := range createOptions.Cells {
127+
createOptions.Cells[i] = strings.TrimSpace(cell)
128+
}
129+
}
130+
return nil
131+
}
132+
133+
// cancel makes a WorkflowDelete call to a vtctld.
134+
cancel = &cobra.Command{
135+
Use: "cancel",
136+
Short: "Cancel the VReplication workflow that backfills the Lookup Vindex.",
137+
Example: `vtctldclient --server localhost:15999 LookupVindex --name corder_lookup_vdx --table-keyspace customer cancel`,
138+
SilenceUsage: true,
139+
DisableFlagsInUseLine: true,
140+
Aliases: []string{"Cancel"},
141+
Args: cobra.NoArgs,
142+
RunE: commandCancel,
143+
}
144+
145+
// create makes a LookupVindexCreate call to a vtctld.
146+
create = &cobra.Command{
147+
Use: "create",
148+
Short: "Create the Lookup Vindex in the specified keyspace and backfill it with a VReplication workflow.",
149+
Example: `vtctldclient --server localhost:15999 LookupVindex --name corder_lookup_vdx --table-keyspace customer create --keyspace customer --type consistent_lookup_unique --table-owner corder --table-owner-columns sku --table-name corder_lookup_tbl --table-vindex-type unicode_loose_xxhash`,
150+
SilenceUsage: true,
151+
DisableFlagsInUseLine: true,
152+
Aliases: []string{"Create"},
153+
Args: cobra.NoArgs,
154+
PreRunE: parseAndValidateCreate,
155+
RunE: commandCreate,
156+
}
157+
158+
// externalize makes a LookupVindexExternalize call to a vtctld.
159+
externalize = &cobra.Command{
160+
Use: "externalize",
161+
Short: "Externalize the Lookup Vindex. If the Vindex has an owner the VReplication workflow will also be deleted.",
162+
Example: `vtctldclient --server localhost:15999 LookupVindex --name corder_lookup_vdx --table-keyspace customer externalize`,
163+
SilenceUsage: true,
164+
DisableFlagsInUseLine: true,
165+
Aliases: []string{"Externalize"},
166+
Args: cobra.NoArgs,
167+
RunE: commandExternalize,
168+
}
169+
170+
// show makes a GetWorkflows call to a vtctld.
171+
show = &cobra.Command{
172+
Use: "show",
173+
Short: "Show the status of the VReplication workflow that backfills the Lookup Vindex.",
174+
Example: `vtctldclient --server localhost:15999 LookupVindex --name corder_lookup_vdx --table-keyspace customer show`,
175+
SilenceUsage: true,
176+
DisableFlagsInUseLine: true,
177+
Aliases: []string{"Show"},
178+
Args: cobra.NoArgs,
179+
RunE: commandShow,
180+
}
181+
)
182+
183+
func commandCancel(cmd *cobra.Command, args []string) error {
184+
cli.FinishedParsing(cmd)
185+
186+
req := &vtctldatapb.WorkflowDeleteRequest{
187+
Keyspace: baseOptions.TableKeyspace,
188+
Workflow: baseOptions.Name,
189+
}
190+
_, err := common.GetClient().WorkflowDelete(common.GetCommandCtx(), req)
191+
if err != nil {
192+
return err
193+
}
194+
195+
output := fmt.Sprintf("LookupVindex %s left in place and the %s VReplication wokflow has been deleted",
196+
baseOptions.Name, baseOptions.Name)
197+
fmt.Println(output)
198+
199+
return nil
200+
}
201+
202+
func commandCreate(cmd *cobra.Command, args []string) error {
203+
tsp := common.GetTabletSelectionPreference(cmd)
204+
cli.FinishedParsing(cmd)
205+
206+
_, err := common.GetClient().LookupVindexCreate(common.GetCommandCtx(), &vtctldatapb.LookupVindexCreateRequest{
207+
Workflow: baseOptions.Name,
208+
Keyspace: createOptions.Keyspace,
209+
Vindex: baseOptions.Vschema,
210+
ContinueAfterCopyWithOwner: createOptions.ContinueAfterCopyWithOwner,
211+
Cells: createOptions.Cells,
212+
TabletTypes: createOptions.TabletTypes,
213+
TabletSelectionPreference: tsp,
214+
})
215+
216+
if err != nil {
217+
return err
218+
}
219+
220+
output := fmt.Sprintf("LookupVindex %s created in the %s keyspace and the %s VReplication wokflow scheduled on the %s shards, use show to view progress",
221+
baseOptions.Name, createOptions.Keyspace, baseOptions.Name, baseOptions.TableKeyspace)
222+
fmt.Println(output)
223+
224+
return nil
225+
}
226+
227+
func commandExternalize(cmd *cobra.Command, args []string) error {
228+
if externalizeOptions.Keyspace == "" {
229+
externalizeOptions.Keyspace = baseOptions.TableKeyspace
230+
}
231+
cli.FinishedParsing(cmd)
232+
233+
resp, err := common.GetClient().LookupVindexExternalize(common.GetCommandCtx(), &vtctldatapb.LookupVindexExternalizeRequest{
234+
Keyspace: externalizeOptions.Keyspace,
235+
// The name of the workflow and lookup vindex.
236+
Name: baseOptions.Name,
237+
// Where the lookup table and VReplication workflow were created.
238+
TableKeyspace: baseOptions.TableKeyspace,
239+
})
240+
241+
if err != nil {
242+
return err
243+
}
244+
245+
output := fmt.Sprintf("LookupVindex %s has been externalized", baseOptions.Name)
246+
if resp.WorkflowDeleted {
247+
output = output + fmt.Sprintf(" and the %s VReplication workflow has been deleted", baseOptions.Name)
248+
}
249+
fmt.Println(output)
250+
251+
return nil
252+
}
253+
254+
func commandShow(cmd *cobra.Command, args []string) error {
255+
cli.FinishedParsing(cmd)
256+
257+
req := &vtctldatapb.GetWorkflowsRequest{
258+
Keyspace: baseOptions.TableKeyspace,
259+
Workflow: baseOptions.Name,
260+
}
261+
resp, err := common.GetClient().GetWorkflows(common.GetCommandCtx(), req)
262+
if err != nil {
263+
return err
264+
}
265+
266+
data, err := cli.MarshalJSONPretty(resp)
267+
if err != nil {
268+
return err
269+
}
270+
271+
fmt.Printf("%s\n", data)
272+
273+
return nil
274+
}
275+
276+
func registerCommands(root *cobra.Command) {
277+
base.PersistentFlags().StringVar(&baseOptions.Name, "name", "", "The name of the Lookup Vindex to create. This will also be the name of the VReplication workflow created to backfill the Lookup Vindex.")
278+
base.MarkPersistentFlagRequired("name")
279+
base.PersistentFlags().StringVar(&baseOptions.TableKeyspace, "table-keyspace", "", "The keyspace to create the lookup table in. This is also where the VReplication workflow is created to backfill the Lookup Vindex.")
280+
base.MarkPersistentFlagRequired("table-keyspace")
281+
root.AddCommand(base)
282+
283+
// This will create the lookup vindex in the specified keyspace
284+
// and setup a VReplication workflow to backfill its lookup table.
285+
create.Flags().StringVar(&createOptions.Keyspace, "keyspace", "", "The keyspace to create the Lookup Vindex in. This is also where the table-owner must exist.")
286+
create.MarkFlagRequired("keyspace")
287+
create.Flags().StringVar(&createOptions.Type, "type", "", "The type of Lookup Vindex to create.")
288+
create.MarkFlagRequired("type")
289+
create.Flags().StringVar(&createOptions.TableOwner, "table-owner", "", "The table holding the data which we should use to backfill the Lookup Vindex. This must exist in the same keyspace as the Lookup Vindex.")
290+
create.MarkFlagRequired("table-owner")
291+
create.Flags().StringSliceVar(&createOptions.TableOwnerColumns, "table-owner-columns", nil, "The columns to read from the owner table. These will be used to build the hash which gets stored as the keyspace_id value in the lookup table.")
292+
create.MarkFlagRequired("table-owner-columns")
293+
create.Flags().StringVar(&createOptions.TableName, "table-name", "", "The name of the lookup table. If not specified, then it will be created using the same name as the Lookup Vindex.")
294+
create.Flags().StringVar(&createOptions.TableVindexType, "table-vindex-type", "", "The primary vindex name/type to use for the lookup table, if the table-keyspace is sharded. This must match the name of a vindex defined in the table-keyspace. If no value is provided then the default type will be used based on the table-owner-columns types.")
295+
create.Flags().BoolVar(&createOptions.IgnoreNulls, "ignore-nulls", false, "Do not add corresponding records in the lookup table if any of the owner table's 'from' fields are NULL.")
296+
create.Flags().BoolVar(&createOptions.ContinueAfterCopyWithOwner, "continue-after-copy-with-owner", true, "Vindex will continue materialization after the backfill completes when an owner is provided.")
297+
// VReplication specific flags.
298+
create.Flags().StringSliceVar(&createOptions.Cells, "cells", nil, "Cells to look in for source tablets to replicate from.")
299+
create.Flags().Var((*topoprotopb.TabletTypeListFlag)(&createOptions.TabletTypes), "tablet-types", "Source tablet types to replicate from.")
300+
create.Flags().BoolVar(&createOptions.TabletTypesInPreferenceOrder, "tablet-types-in-preference-order", true, "When performing source tablet selection, look for candidates in the type order as they are listed in the tablet-types flag.")
301+
base.AddCommand(create)
302+
303+
// This will show the output of GetWorkflows client call
304+
// for the VReplication workflow used.
305+
base.AddCommand(show)
306+
307+
// This will also delete the VReplication workflow if the
308+
// vindex has an owner as the lookup vindex will then be
309+
// managed by VTGate.
310+
externalize.Flags().StringVar(&externalizeOptions.Keyspace, "keyspace", "", "The keyspace containing the Lookup Vindex. If no value is specified then the table-keyspace will be used.")
311+
base.AddCommand(externalize)
312+
313+
// The cancel command deletes the VReplication workflow used
314+
// to backfill the lookup vindex. It ends up making a
315+
// WorkflowDelete VtctldServer call.
316+
base.AddCommand(cancel)
317+
}
318+
319+
func init() {
320+
common.RegisterCommandHandler("LookupVindex", registerCommands)
321+
}

‎go/cmd/vtctldclient/command/vreplication/movetables/movetables.go

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,8 @@ import (
2525
var (
2626
// moveTables is the base command for all actions related to moveTables.
2727
moveTables = &cobra.Command{
28-
Use: "MoveTables --workflow <workflow> --keyspace <keyspace> [command] [command-flags]",
29-
Short: "Perform commands related to moving tables from a source keyspace to a target keyspace.",
30-
Long: `MoveTables commands: Create, Show, Status, SwitchTraffic, ReverseTraffic, Stop, Start, Cancel, and Delete.
31-
See the --help output for each command for more details.`,
28+
Use: "MoveTables --workflow <workflow> --keyspace <keyspace> [command] [command-flags]",
29+
Short: "Perform commands related to moving tables from a source keyspace to a target keyspace.",
3230
DisableFlagsInUseLine: true,
3331
Aliases: []string{"movetables"},
3432
Args: cobra.ExactArgs(1),

‎go/cmd/vtctldclient/command/vreplication/reshard/create.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@ var (
3535
// reshardCreate makes a ReshardCreate gRPC call to a vtctld.
3636
reshardCreate = &cobra.Command{
3737
Use: "create",
38-
Short: "Create and optionally run a reshard VReplication workflow.",
39-
Example: `vtctldclient --server localhost:15999 reshard --workflow customer2customer --target-keyspace customer create --source_shards="0" --target_shards="-80,80-" --cells zone1 --cells zone2 --tablet-types replica`,
38+
Short: "Create and optionally run a Reshard VReplication workflow.",
39+
Example: `vtctldclient --server localhost:15999 reshard --workflow customer2customer --target-keyspace customer create --source-shards="0" --target-shards="-80,80-" --cells zone1 --cells zone2 --tablet-types replica`,
4040
SilenceUsage: true,
4141
DisableFlagsInUseLine: true,
4242
Aliases: []string{"Create"},

‎go/cmd/vtctldclient/command/vreplication/reshard/reshard.go

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,8 @@ import (
2525
var (
2626
// reshard is the base command for all actions related to reshard.
2727
reshard = &cobra.Command{
28-
Use: "Reshard --workflow <workflow> --keyspace <keyspace> [command] [command-flags]",
29-
Short: "Perform commands related to resharding a keyspace.",
30-
Long: `Reshard commands: Create, Show, Status, SwitchTraffic, ReverseTraffic, Stop, Start, Cancel, and Delete.
31-
See the --help output for each command for more details.`,
28+
Use: "Reshard --workflow <workflow> --keyspace <keyspace> [command] [command-flags]",
29+
Short: "Perform commands related to resharding a keyspace.",
3230
DisableFlagsInUseLine: true,
3331
Aliases: []string{"reshard"},
3432
Args: cobra.ExactArgs(1),

0 commit comments

Comments
 (0)