Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add pgx driver package #517

Merged
merged 14 commits into from
Mar 14, 2021
Merged

Add pgx driver package #517

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
995cc27
replace lib/pq dependency with pgx in postgres driver
dzbee Feb 26, 2021
420fede
fix pgx error handling
dzbee Mar 3, 2021
0abe9fc
update postgres tests to use pgx driver and error reporting
dzbee Mar 3, 2021
7f32fe6
update quoteIdentifier implementation to match lib/pq
dzbee Mar 4, 2021
6c61cf3
move changes for pgx support into new driver package
dzbee Mar 5, 2021
98447ca
add pgx driver to CLI
dzbee Mar 9, 2021
2afbd8d
restore examples used in testing for pgx driver
dzbee Mar 9, 2021
db16491
replace postgres driver name with pgx in DSN in tests and docs
dzbee Mar 9, 2021
611d4d1
consolidate connection string formatting in pgx testing
dzbee Mar 10, 2021
def44ba
lock pgx package at v4
dzbee Mar 10, 2021
d6e4514
update parsed URL scheme instead of using string operation
dzbee Mar 11, 2021
f2fa789
add pgx for builds using make
dzbee Mar 12, 2021
84b6e46
add pgx to top level docs
dzbee Mar 12, 2021
42470f2
remove autogenerated TOC
dzbee Mar 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 18 additions & 15 deletions database/postgres/postgres.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ import (
"github.com/golang-migrate/migrate/v4/database"
"github.com/golang-migrate/migrate/v4/database/multistmt"
multierror "github.com/hashicorp/go-multierror"
"github.com/lib/pq"
"github.com/jackc/pgerrcode"
"github.com/jackc/pgx"
_ "github.com/jackc/pgx/stdlib"
)

func init() {
Expand Down Expand Up @@ -125,7 +127,7 @@ func (p *Postgres) Open(url string) (database.Driver, error) {
return nil, err
}

db, err := sql.Open("postgres", migrate.FilterCustomQuery(purl).String())
db, err := sql.Open("pgx", migrate.FilterCustomQuery(purl).String())
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -254,15 +256,12 @@ func (p *Postgres) runStatement(statement []byte) error {
return nil
}
if _, err := p.conn.ExecContext(ctx, query); err != nil {
if pgErr, ok := err.(*pq.Error); ok {

if pgErr, ok := err.(pgx.PgError); ok {
var line uint
var col uint
var lineColOK bool
if pgErr.Position != "" {
if pos, err := strconv.ParseUint(pgErr.Position, 10, 64); err == nil {
line, col, lineColOK = computeLineFromPos(query, int(pos))
}
}
line, col, lineColOK = computeLineFromPos(query, int(pgErr.Position))
message := fmt.Sprintf("migration failed: %s", pgErr.Message)
if lineColOK {
message = fmt.Sprintf("%s (column %d)", message, col)
Expand Down Expand Up @@ -318,7 +317,7 @@ func (p *Postgres) SetVersion(version int, dirty bool) error {
return &database.Error{OrigErr: err, Err: "transaction start failed"}
}

query := `TRUNCATE ` + pq.QuoteIdentifier(p.config.MigrationsTable)
query := `TRUNCATE ` + quoteIdentifier(p.config.MigrationsTable)
if _, err := tx.Exec(query); err != nil {
if errRollback := tx.Rollback(); errRollback != nil {
err = multierror.Append(err, errRollback)
Expand All @@ -330,7 +329,7 @@ func (p *Postgres) SetVersion(version int, dirty bool) error {
// empty schema version for failed down migration on the first migration
// See: https://github.com/golang-migrate/migrate/issues/330
if version >= 0 || (version == database.NilVersion && dirty) {
query = `INSERT INTO ` + pq.QuoteIdentifier(p.config.MigrationsTable) +
query = `INSERT INTO ` + quoteIdentifier(p.config.MigrationsTable) +
` (version, dirty) VALUES ($1, $2)`
if _, err := tx.Exec(query, version, dirty); err != nil {
if errRollback := tx.Rollback(); errRollback != nil {
Expand All @@ -348,15 +347,15 @@ func (p *Postgres) SetVersion(version int, dirty bool) error {
}

func (p *Postgres) Version() (version int, dirty bool, err error) {
query := `SELECT version, dirty FROM ` + pq.QuoteIdentifier(p.config.MigrationsTable) + ` LIMIT 1`
query := `SELECT version, dirty FROM ` + quoteIdentifier(p.config.MigrationsTable) + ` LIMIT 1`
err = p.conn.QueryRowContext(context.Background(), query).Scan(&version, &dirty)
switch {
case err == sql.ErrNoRows:
return database.NilVersion, false, nil

case err != nil:
if e, ok := err.(*pq.Error); ok {
if e.Code.Name() == "undefined_table" {
if e, ok := err.(pgx.PgError); ok {
if e.SQLState() == pgerrcode.UndefinedTable {
return database.NilVersion, false, nil
}
}
Expand Down Expand Up @@ -398,7 +397,7 @@ func (p *Postgres) Drop() (err error) {
if len(tableNames) > 0 {
// delete one by one ...
for _, t := range tableNames {
query = `DROP TABLE IF EXISTS ` + pq.QuoteIdentifier(t) + ` CASCADE`
query = `DROP TABLE IF EXISTS ` + quoteIdentifier(t) + ` CASCADE`
if _, err := p.conn.ExecContext(context.Background(), query); err != nil {
return &database.Error{OrigErr: err, Query: []byte(query)}
}
Expand Down Expand Up @@ -426,10 +425,14 @@ func (p *Postgres) ensureVersionTable() (err error) {
}
}()

query := `CREATE TABLE IF NOT EXISTS ` + pq.QuoteIdentifier(p.config.MigrationsTable) + ` (version bigint not null primary key, dirty boolean not null)`
query := `CREATE TABLE IF NOT EXISTS ` + quoteIdentifier(p.config.MigrationsTable) + ` (version bigint not null primary key, dirty boolean not null)`
if _, err = p.conn.ExecContext(context.Background(), query); err != nil {
return &database.Error{OrigErr: err, Query: []byte(query)}
}

return nil
}

func quoteIdentifier(s string) string {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Use this implementation and reference it: https://github.com/lib/pq/blob/v1.9.0/conn.go#L1611

return `"` + strings.ReplaceAll(s, `"`, `""`) + `"`
}
11 changes: 6 additions & 5 deletions database/postgres/postgres_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package postgres

// error codes https://github.com/lib/pq/blob/master/error.go
// error codes https://github.com/jackc/pgerrcode/blob/master/errcode.go

import (
"context"
Expand All @@ -9,13 +9,14 @@ import (
"fmt"
"log"

"github.com/golang-migrate/migrate/v4"
"io"
"strconv"
"strings"
"sync"
"testing"

"github.com/golang-migrate/migrate/v4"

"github.com/dhui/dktest"

"github.com/golang-migrate/migrate/v4/database"
Expand Down Expand Up @@ -53,7 +54,7 @@ func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
return false
}

db, err := sql.Open("postgres", pgConnectionString(ip, port))
db, err := sql.Open("pgx", pgConnectionString(ip, port))
if err != nil {
return false
}
Expand Down Expand Up @@ -209,7 +210,7 @@ func TestErrorParsing(t *testing.T) {
}()

wantErr := `migration failed: syntax error at or near "TABLEE" (column 37) in line 1: CREATE TABLE foo ` +
`(foo text); CREATE TABLEE bar (bar text); (details: pq: syntax error at or near "TABLEE")`
`(foo text); CREATE TABLEE bar (bar text); (details: ERROR: syntax error at or near "TABLEE" (SQLSTATE 42601))`
if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLEE bar (bar text);")); err == nil {
t.Fatal("expected err but got nil")
} else if err.Error() != wantErr {
Expand Down Expand Up @@ -436,7 +437,7 @@ func TestWithInstance_Concurrent(t *testing.T) {
// actually a connection pool, and so, each of the below go
// routines will have a high probability of using a separate
// connection, which is something we want to exercise.
db, err := sql.Open("postgres", pgConnectionString(ip, port))
db, err := sql.Open("pgx", pgConnectionString(ip, port))
if err != nil {
t.Fatal(err)
}
Expand Down
6 changes: 5 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,15 @@ require (
github.com/go-sql-driver/mysql v1.5.0
github.com/gobuffalo/here v0.6.0
github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4
github.com/gofrs/uuid v4.0.0+incompatible // indirect
github.com/golang/snappy v0.0.1 // indirect
github.com/google/go-github v17.0.0+incompatible
github.com/gorilla/mux v1.7.4 // indirect
github.com/hashicorp/go-multierror v1.1.0
github.com/jackc/pgconn v1.3.2 // indirect
github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 // indirect
github.com/jackc/pgconn v1.3.2
github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451
github.com/jackc/pgx v3.6.2+incompatible
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like v4.10.1 is the latest release. Any reason why we don't use that?

Copy link
Contributor Author

@dzbee dzbee Mar 10, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That was simply a mistake resulting from the fact that pgx didn't support Go modules until v4. Therefore I was pulling the latest release prior to the introduction of modules into the project. The major version includes some breaking changes for the code I'd written before so I've included some updates with a version bump in def44ba.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Most significantly, in v3 the connection string parsing didn't include any checks on the content scheme name (see https://github.com/jackc/pgx/blob/a413de981978e4100d5736af7aaa2b392511612f/conn.go#L1246 and https://github.com/jackc/pgx/blob/a413de981978e4100d5736af7aaa2b392511612f/conn.go#L979) whereas now the postgres or postresql schemes are explicitly required in the connection string (see https://github.com/jackc/pgconn/blob/5daa019e4eb52df3409ebf17c83116b7c0e827e5/config.go#L199). Therefore I've introduced a string replacement so that after the pgx scheme is used to identify the appropriate driver in migrate, it's replaced with the appropriate postgres scheme for actually opening the connection.

github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
github.com/ktrysmt/go-bitbucket v0.6.4
github.com/lib/pq v1.8.0
Expand Down
9 changes: 9 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
github.com/cockroachdb/cockroach-go v0.0.0-20190925194419-606b3d062051 h1:eApuUG8W2EtBVwxqLlY2wgoqDYOg3WvIHGvW4fUbbow=
github.com/cockroachdb/cockroach-go v0.0.0-20190925194419-606b3d062051/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk=
Expand Down Expand Up @@ -129,6 +130,8 @@ github.com/gobuffalo/here v0.6.0 h1:hYrd0a6gDmWxBM4TnrGw8mQg24iSVoIkHEk7FodQcBI=
github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4 h1:vF83LI8tAakwEwvWZtrIEx7pOySacl2TOxx6eXk4ePo=
github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0=
github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
Expand Down Expand Up @@ -232,11 +235,15 @@ github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9
github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 h1:vr3AYkKovP8uR8AvSGGUK1IDqRa5lAAvEkZG1LKaCRc=
github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ=
github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA=
github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE=
github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s=
github.com/jackc/pgconn v1.3.2 h1:9UIGICxEAW70RQDGilGwsCG63NCcm5amjuBQCFzrmsw=
github.com/jackc/pgconn v1.3.2/go.mod h1:LvCquS3HbBKwgl7KbX9KyqEIumJAbm1UMcTvGaIf3bM=
github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451 h1:WAvSpGf7MsFuzAtK4Vk7R4EVe+liW4x83r4oWu0WHKw=
github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye4717ITLaNwV9mWbJx0dLCpcRzdA=
Expand All @@ -254,6 +261,8 @@ github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX
github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o=
github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I=
github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
Expand Down