Skip to content

Commit d243fbe

Browse files
committed
make the migration handler more robust
- Allow large ddl file - Support graph syntax
1 parent c3ef66a commit d243fbe

File tree

4 files changed

+29
-163
lines changed

4 files changed

+29
-163
lines changed

database/spanner/spanner.go

Lines changed: 23 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,22 @@ import (
66
"fmt"
77
"io"
88
"log"
9+
"log/slog"
910
nurl "net/url"
1011
"regexp"
11-
"strconv"
1212
"strings"
1313

1414
"cloud.google.com/go/spanner"
1515
sdb "cloud.google.com/go/spanner/admin/database/apiv1"
16-
"cloud.google.com/go/spanner/spansql"
1716

1817
"github.com/golang-migrate/migrate/v4"
1918
"github.com/golang-migrate/migrate/v4/database"
19+
"github.com/golang-migrate/migrate/v4/database/spanner/ddl"
2020

21+
"cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
2122
adminpb "cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
2223
"github.com/hashicorp/go-multierror"
24+
"github.com/samber/lo"
2325
uatomic "go.uber.org/atomic"
2426
"google.golang.org/api/iterator"
2527
)
@@ -51,11 +53,6 @@ var (
5153
type Config struct {
5254
MigrationsTable string
5355
DatabaseName string
54-
// Whether to parse the migration DDL with spansql before
55-
// running them towards Spanner.
56-
// Parsing outputs clean DDL statements such as reformatted
57-
// and void of comments.
58-
CleanStatements bool
5956
}
6057

6158
// Spanner implements database.Driver for Google Cloud Spanner
@@ -127,20 +124,10 @@ func (s *Spanner) Open(url string) (database.Driver, error) {
127124

128125
migrationsTable := purl.Query().Get("x-migrations-table")
129126

130-
cleanQuery := purl.Query().Get("x-clean-statements")
131-
clean := false
132-
if cleanQuery != "" {
133-
clean, err = strconv.ParseBool(cleanQuery)
134-
if err != nil {
135-
return nil, err
136-
}
137-
}
138-
139127
db := &DB{admin: adminClient, data: dataClient}
140128
return WithInstance(db, &Config{
141129
DatabaseName: dbname,
142130
MigrationsTable: migrationsTable,
143-
CleanStatements: clean,
144131
})
145132
}
146133

@@ -174,26 +161,30 @@ func (s *Spanner) Run(migration io.Reader) error {
174161
return err
175162
}
176163

177-
stmts := []string{string(migr)}
178-
if s.config.CleanStatements {
179-
stmts, err = cleanStatements(migr)
180-
if err != nil {
181-
return err
182-
}
164+
stmts, err := ddl.ToMigrationStatements("", string(migr))
165+
if err != nil {
166+
return &database.Error{OrigErr: err, Err: "failed to cleanup statements", Query: migr}
183167
}
184168

185169
ctx := context.Background()
186-
op, err := s.db.admin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{
187-
Database: s.config.DatabaseName,
188-
Statements: stmts,
189-
})
190170

191-
if err != nil {
192-
return &database.Error{OrigErr: err, Err: "migration failed", Query: migr}
193-
}
171+
// Because the statements limit is 10, we have to chunk the statements.
172+
for i, chunk := range lo.Chunk(stmts, 10) {
173+
// The migration is usually very slow, some can take 10 more minutes to run.
174+
// Print some progress so that it doesn't look like the process is stuck.
175+
slog.InfoContext(ctx, "spanner db migration progress", "progress", i*10+len(chunk), "total", len(stmts))
194176

195-
if err := op.Wait(ctx); err != nil {
196-
return &database.Error{OrigErr: err, Err: "migration failed", Query: migr}
177+
op, err := s.db.admin.UpdateDatabaseDdl(ctx, &databasepb.UpdateDatabaseDdlRequest{
178+
Database: s.config.DatabaseName,
179+
Statements: chunk,
180+
})
181+
if err != nil {
182+
return &database.Error{OrigErr: err, Err: "failed to update spanner db ddl", Query: []byte(strings.Join(chunk, "; "))}
183+
}
184+
185+
if err := op.Wait(ctx); err != nil {
186+
return &database.Error{OrigErr: err, Err: "failed to wait for spanner db ddl update", Query: []byte(strings.Join(chunk, "; "))}
187+
}
197188
}
198189

199190
return nil
@@ -339,18 +330,3 @@ func (s *Spanner) ensureVersionTable() (err error) {
339330

340331
return nil
341332
}
342-
343-
func cleanStatements(migration []byte) ([]string, error) {
344-
// The Spanner GCP backend does not yet support comments for the UpdateDatabaseDdl RPC
345-
// (see https://issuetracker.google.com/issues/159730604) we use
346-
// spansql to parse the DDL and output valid stamements without comments
347-
ddl, err := spansql.ParseDDL("", string(migration))
348-
if err != nil {
349-
return nil, err
350-
}
351-
stmts := make([]string, 0, len(ddl.List))
352-
for _, stmt := range ddl.List {
353-
stmts = append(stmts, stmt.SQL())
354-
}
355-
return stmts, nil
356-
}

database/spanner/spanner_test.go

Lines changed: 0 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@ import (
1111
_ "github.com/golang-migrate/migrate/v4/source/file"
1212

1313
"cloud.google.com/go/spanner/spannertest"
14-
"github.com/stretchr/testify/assert"
15-
"github.com/stretchr/testify/require"
1614
)
1715

1816
// withSpannerEmulator is not thread-safe and cannot be used with parallel tests since it sets the emulator
@@ -60,112 +58,3 @@ func TestMigrate(t *testing.T) {
6058
dt.TestMigrate(t, m)
6159
})
6260
}
63-
64-
func TestCleanStatements(t *testing.T) {
65-
testCases := []struct {
66-
name string
67-
multiStatement string
68-
expected []string
69-
}{
70-
{
71-
name: "no statement",
72-
multiStatement: "",
73-
expected: []string{},
74-
},
75-
{
76-
name: "single statement, single line, no semicolon, no comment",
77-
multiStatement: "CREATE TABLE table_name (id STRING(255) NOT NULL) PRIMARY KEY (id)",
78-
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
79-
},
80-
{
81-
name: "single statement, multi line, no semicolon, no comment",
82-
multiStatement: `CREATE TABLE table_name (
83-
id STRING(255) NOT NULL,
84-
) PRIMARY KEY (id)`,
85-
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
86-
},
87-
{
88-
name: "single statement, single line, with semicolon, no comment",
89-
multiStatement: "CREATE TABLE table_name (id STRING(255) NOT NULL) PRIMARY KEY (id);",
90-
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
91-
},
92-
{
93-
name: "single statement, multi line, with semicolon, no comment",
94-
multiStatement: `CREATE TABLE table_name (
95-
id STRING(255) NOT NULL,
96-
) PRIMARY KEY (id);`,
97-
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
98-
},
99-
{
100-
name: "multi statement, with trailing semicolon. no comment",
101-
// From https://github.com/mattes/migrate/pull/281
102-
multiStatement: `CREATE TABLE table_name (
103-
id STRING(255) NOT NULL,
104-
) PRIMARY KEY(id);
105-
106-
CREATE INDEX table_name_id_idx ON table_name (id);`,
107-
expected: []string{`CREATE TABLE table_name (
108-
id STRING(255) NOT NULL,
109-
) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
110-
},
111-
{
112-
name: "multi statement, no trailing semicolon, no comment",
113-
// From https://github.com/mattes/migrate/pull/281
114-
multiStatement: `CREATE TABLE table_name (
115-
id STRING(255) NOT NULL,
116-
) PRIMARY KEY(id);
117-
118-
CREATE INDEX table_name_id_idx ON table_name (id)`,
119-
expected: []string{`CREATE TABLE table_name (
120-
id STRING(255) NOT NULL,
121-
) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
122-
},
123-
{
124-
name: "multi statement, no trailing semicolon, standalone comment",
125-
// From https://github.com/mattes/migrate/pull/281
126-
multiStatement: `CREATE TABLE table_name (
127-
-- standalone comment
128-
id STRING(255) NOT NULL,
129-
) PRIMARY KEY(id);
130-
131-
CREATE INDEX table_name_id_idx ON table_name (id)`,
132-
expected: []string{`CREATE TABLE table_name (
133-
id STRING(255) NOT NULL,
134-
) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
135-
},
136-
{
137-
name: "multi statement, no trailing semicolon, inline comment",
138-
// From https://github.com/mattes/migrate/pull/281
139-
multiStatement: `CREATE TABLE table_name (
140-
id STRING(255) NOT NULL, -- inline comment
141-
) PRIMARY KEY(id);
142-
143-
CREATE INDEX table_name_id_idx ON table_name (id)`,
144-
expected: []string{`CREATE TABLE table_name (
145-
id STRING(255) NOT NULL,
146-
) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
147-
},
148-
{
149-
name: "alter table with SET OPTIONS",
150-
multiStatement: `ALTER TABLE users ALTER COLUMN created
151-
SET OPTIONS (allow_commit_timestamp=true);`,
152-
expected: []string{"ALTER TABLE users ALTER COLUMN created SET OPTIONS (allow_commit_timestamp = true)"},
153-
},
154-
{
155-
name: "column with NUMERIC type",
156-
multiStatement: `CREATE TABLE table_name (
157-
id STRING(255) NOT NULL,
158-
sum NUMERIC,
159-
) PRIMARY KEY (id)`,
160-
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n sum NUMERIC,\n) PRIMARY KEY(id)"},
161-
},
162-
}
163-
164-
for _, tc := range testCases {
165-
t.Run(tc.name, func(t *testing.T) {
166-
stmts, err := cleanStatements([]byte(tc.multiStatement))
167-
require.NoError(t, err, "Error cleaning statements")
168-
assert.Equal(t, tc.expected, stmts)
169-
})
170-
}
171-
}

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,9 @@ require (
3131
github.com/mutecomm/go-sqlcipher/v4 v4.4.0
3232
github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8
3333
github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba
34+
github.com/samber/lo v1.50.0
3435
github.com/snowflakedb/gosnowflake v1.6.19
35-
github.com/stretchr/testify v1.9.0
36+
github.com/stretchr/testify v1.10.0
3637
github.com/xanzy/go-gitlab v0.15.0
3738
github.com/ysmood/got v0.41.0
3839
go.mongodb.org/mongo-driver v1.7.5
@@ -52,7 +53,6 @@ require (
5253
github.com/go-logr/stdr v1.2.2 // indirect
5354
github.com/jackc/puddle/v2 v2.2.1 // indirect
5455
github.com/json-iterator/go v1.1.12 // indirect
55-
github.com/kr/text v0.2.0 // indirect
5656
github.com/moby/docker-image-spec v1.3.1 // indirect
5757
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
5858
github.com/modern-go/reflect2 v1.0.2 // indirect

go.sum

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,6 @@ github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3
149149
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
150150
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
151151
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
152-
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
153152
github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
154153
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
155154
github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369 h1:XNT/Zf5l++1Pyg08/HV04ppB0gKxAqtZQBRYiYrUuYk=
@@ -546,6 +545,8 @@ github.com/rqlite/gorqlite v0.0.0-20230708021416-2acd02b70b79/go.mod h1:xF/KoXmr
546545
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
547546
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
548547
github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
548+
github.com/samber/lo v1.50.0 h1:XrG0xOeHs+4FQ8gJR97zDz5uOFMW7OwFWiFVzqopKgY=
549+
github.com/samber/lo v1.50.0/go.mod h1:RjZyNk6WSnUFRKK6EyOhsRJMqft3G+pg7dCWHQCWvsc=
549550
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
550551
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
551552
github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
@@ -573,8 +574,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
573574
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
574575
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
575576
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
576-
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
577-
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
577+
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
578+
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
578579
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
579580
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
580581
github.com/xanzy/go-gitlab v0.15.0 h1:rWtwKTgEnXyNUGrOArN7yyc3THRkpYcKXIXia9abywQ=

0 commit comments

Comments
 (0)