Skip to content

Commit 9432313

Browse files
apstndbapstndb
authored
Migrate to memefish (#121)
* Migrate to memefish * Update testdata * Do go mod tidy * Update to test comments in DDL * Update to reduce useless use of gsqlutils
1 parent 3a330a0 commit 9432313

File tree

10 files changed

+83
-39
lines changed

10 files changed

+83
-39
lines changed

‎go.mod

+8-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
module github.com/cloudspannerecosystem/wrench
22

3-
go 1.21
3+
go 1.23.1
4+
5+
toolchain go1.23.2
46

57
require (
68
cloud.google.com/go/spanner v1.64.0
9+
github.com/apstndb/gsqlutils v0.0.0-20241110070457-b8882df29778
710
github.com/google/uuid v1.6.0
811
github.com/hashicorp/go-multierror v1.1.1
912
github.com/spf13/cobra v1.7.0
@@ -21,6 +24,7 @@ require (
2124
github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 // indirect
2225
github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
2326
github.com/cespare/xxhash/v2 v2.2.0 // indirect
27+
github.com/cloudspannerecosystem/memefish v0.0.0-20241106111047-2b2b4b23a1e7 // indirect
2428
github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50 // indirect
2529
github.com/envoyproxy/go-control-plane v0.12.0 // indirect
2630
github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect
@@ -34,6 +38,7 @@ require (
3438
github.com/googleapis/gax-go/v2 v2.12.5 // indirect
3539
github.com/hashicorp/errwrap v1.1.0 // indirect
3640
github.com/inconshreveable/mousetrap v1.1.0 // indirect
41+
github.com/samber/lo v1.47.0 // indirect
3742
github.com/spf13/pflag v1.0.5 // indirect
3843
go.opencensus.io v0.24.0 // indirect
3944
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
@@ -45,12 +50,13 @@ require (
4550
golang.org/x/net v0.26.0 // indirect
4651
golang.org/x/oauth2 v0.21.0 // indirect
4752
golang.org/x/sync v0.7.0 // indirect
48-
golang.org/x/sys v0.21.0 // indirect
53+
golang.org/x/sys v0.26.0 // indirect
4954
golang.org/x/text v0.16.0 // indirect
5055
golang.org/x/time v0.5.0 // indirect
5156
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
5257
google.golang.org/genproto v0.0.0-20240617180043-68d350f18fd4 // indirect
5358
google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect
5459
google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect
5560
google.golang.org/protobuf v1.34.2 // indirect
61+
spheric.cloud/xiter v0.0.0-20240904151420-c999f37a46b2 // indirect
5662
)

‎go.sum

+17-2
Original file line numberDiff line numberDiff line change
@@ -614,6 +614,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
614614
github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 h1:oVLqHXhnYtUwM89y9T1fXGaK9wTkXHgNp8/ZNMQzUxE=
615615
github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0/go.mod h1:dppbR7CwXD4pgtV9t3wD1812RaLDcBjtblcDF5f1vI0=
616616
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
617+
github.com/MakeNowJust/heredoc/v2 v2.0.1 h1:rlCHh70XXXv7toz95ajQWOWQnN4WNLt0TdpZYIR/J6A=
618+
github.com/MakeNowJust/heredoc/v2 v2.0.1/go.mod h1:6/2Abh5s+hc3g9nbWLe9ObDIOhaRrqsyY9MWy+4JdRM=
617619
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
618620
github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
619621
github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
@@ -624,6 +626,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
624626
github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
625627
github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI=
626628
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
629+
github.com/apstndb/gsqlutils v0.0.0-20241110070457-b8882df29778 h1:eFzpW6G5NzTytQtSnsEIOfH2aAYGnNN5zmbdh2jQFCY=
630+
github.com/apstndb/gsqlutils v0.0.0-20241110070457-b8882df29778/go.mod h1:iy/NytOejQT/jaXcRaYYHthvkTpgWNZxggLVEecd7C4=
627631
github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
628632
github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
629633
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -638,6 +642,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
638642
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
639643
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
640644
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
645+
github.com/cloudspannerecosystem/memefish v0.0.0-20241106111047-2b2b4b23a1e7 h1:vmS9Nvh7ij5EVnMwvkWsL84xzx5cGM3j/SJlM1zfVLE=
646+
github.com/cloudspannerecosystem/memefish v0.0.0-20241106111047-2b2b4b23a1e7/go.mod h1:iYAaNZfVIn4QYfUmXt+3EeHAok/kqpN/fp/8kgDHjx8=
641647
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
642648
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
643649
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -831,6 +837,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
831837
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
832838
github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
833839
github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
840+
github.com/k0kubun/pp v1.3.1-0.20200204103551-99835366d1cc h1:XLjmW07gT7cG/wb6mavIrvAIWBYaTacPo8UOnxGSspA=
841+
github.com/k0kubun/pp v1.3.1-0.20200204103551-99835366d1cc/go.mod h1:qK2ivXw91omfE1uXcpR5kWbAMZRdDOnGbqWlZ7reRFk=
834842
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
835843
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
836844
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
@@ -846,7 +854,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
846854
github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
847855
github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
848856
github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o=
857+
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
858+
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
849859
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
860+
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
850861
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
851862
github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
852863
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
@@ -873,6 +884,8 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f
873884
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
874885
github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w=
875886
github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk=
887+
github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc=
888+
github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU=
876889
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
877890
github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
878891
github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
@@ -1181,8 +1194,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
11811194
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
11821195
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
11831196
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1184-
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
1185-
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
1197+
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
1198+
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
11861199
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
11871200
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
11881201
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
@@ -1618,3 +1631,5 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
16181631
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
16191632
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
16201633
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
1634+
spheric.cloud/xiter v0.0.0-20240904151420-c999f37a46b2 h1:/2/LZep2TpsdSwYhdMsGRh+OVun1UMYlMEip4EEsgg8=
1635+
spheric.cloud/xiter v0.0.0-20240904151420-c999f37a46b2/go.mod h1:i4SlkNfFrn1974zbGZWg8FYXAWLnrS6cYAXtSfmIDhU=

‎pkg/spanner/client_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ func TestLoadDDL(t *testing.T) {
7373
t.Fatalf("failed to load ddl: %v", err)
7474
}
7575

76-
wantDDL, err := os.ReadFile("testdata/schema.sql")
76+
wantDDL, err := os.ReadFile("testdata/schema_loaded.sql")
7777
if err != nil {
7878
t.Fatalf("failed to read ddl file: %v", err)
7979
}

‎pkg/spanner/memefish.go

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package spanner
2+
3+
import (
4+
"github.com/apstndb/gsqlutils"
5+
"github.com/cloudspannerecosystem/memefish"
6+
)
7+
8+
// Directly use of memefish/gsqlutils is permitted only in this file.
9+
func toStatements(filename string, data []byte) ([]string, error) {
10+
rawStmts, err := memefish.SplitRawStatements(filename, string(data))
11+
if err != nil {
12+
return nil, err
13+
}
14+
15+
// need to strip comments because memefish.SplitRawStatements preserve comments, but UpdateDDL doesn't support comments.
16+
var result []string
17+
for _, rawStmt := range rawStmts {
18+
stripped, err := gsqlutils.SimpleStripComments("", rawStmt.Statement)
19+
if err != nil {
20+
return nil, err
21+
}
22+
result = append(result, stripped)
23+
}
24+
return result, nil
25+
}
26+
27+
func isDML(statement string) bool {
28+
token, err := gsqlutils.FirstNonHintToken("", statement)
29+
if err != nil {
30+
return false
31+
}
32+
return token.IsKeywordLike("INSERT")
33+
}
34+
35+
func isPartitionedDML(statement string) bool {
36+
// It is better than regular expression because PDML can be prefixed by statement hints.
37+
token, err := gsqlutils.FirstNonHintToken("", statement)
38+
if err != nil {
39+
return false
40+
}
41+
return token.IsKeywordLike("UPDATE") || token.IsKeywordLike("DELETE")
42+
}

‎pkg/spanner/migration.go

+2-34
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@ import (
2626
"path/filepath"
2727
"regexp"
2828
"strconv"
29-
30-
"cloud.google.com/go/spanner/spansql"
3129
)
3230

3331
var (
@@ -38,9 +36,6 @@ var (
3836
migrationFileRegex = regexp.MustCompile(`^([0-9]+)(?:_([a-zA-Z0-9_\-]+))?(\.up)?\.sql$`)
3937

4038
MigrationNameRegex = regexp.MustCompile(`[a-zA-Z0-9_\-]+`)
41-
42-
dmlRegex = regexp.MustCompile("^(INSERT)[\t\n\f\r ].*")
43-
partitionedDmlRegex = regexp.MustCompile("^(UPDATE|DELETE)[\t\n\f\r ].*")
4439
)
4540

4641
const (
@@ -144,31 +139,11 @@ func LoadMigrations(dir string) (Migrations, error) {
144139
}
145140

146141
func ddlToStatements(filename string, data []byte) ([]string, error) {
147-
ddl, err := spansql.ParseDDL(filename, string(data))
148-
if err != nil {
149-
return nil, err
150-
}
151-
152-
var statements []string
153-
for _, stmt := range ddl.List {
154-
statements = append(statements, stmt.SQL())
155-
}
156-
157-
return statements, nil
142+
return toStatements(filename, data)
158143
}
159144

160145
func dmlToStatements(filename string, data []byte) ([]string, error) {
161-
dml, err := spansql.ParseDML(filename, string(data))
162-
if err != nil {
163-
return nil, err
164-
}
165-
166-
var statements []string
167-
for _, stmt := range dml.List {
168-
statements = append(statements, stmt.SQL())
169-
}
170-
171-
return statements, nil
146+
return toStatements(filename, data)
172147
}
173148

174149
func inspectStatementsKind(statements []string) (statementKind, error) {
@@ -200,10 +175,3 @@ func inspectStatementsKind(statements []string) (statementKind, error) {
200175
}
201176
}
202177

203-
func isDML(statement string) bool {
204-
return dmlRegex.Match([]byte(statement))
205-
}
206-
207-
func isPartitionedDML(statement string) bool {
208-
return partitionedDmlRegex.Match([]byte(statement))
209-
}

‎pkg/spanner/testdata/ddl.sql

+1
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1+
-- Comments must be permitted
12
ALTER TABLE Singers ADD COLUMN Foo STRING(MAX);

‎pkg/spanner/testdata/migrations/000002_test.sql

+1
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1+
-- Comments must be ignored
12
ALTER TABLE Singers ADD COLUMN LastName STRING(MAX);

‎pkg/spanner/testdata/migrations/000003.sql

+1
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1+
-- Comments must be permitted
12
UPDATE Singers SET LastName = "" WHERE LastName IS NULL;

‎pkg/spanner/testdata/schema.sql

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
-- Comments must be permitted
12
CREATE TABLE SchemaMigrations (
23
Version INT64 NOT NULL,
34
Dirty BOOL NOT NULL,

‎pkg/spanner/testdata/schema_loaded.sql

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
CREATE TABLE SchemaMigrations (
2+
Version INT64 NOT NULL,
3+
Dirty BOOL NOT NULL,
4+
) PRIMARY KEY(Version);
5+
6+
CREATE TABLE Singers (
7+
SingerID STRING(36) NOT NULL,
8+
FirstName STRING(1024),
9+
) PRIMARY KEY(SingerID);

0 commit comments

Comments
 (0)