From 57f1b81b4205d98eae2ade2eb197f369c85eb064 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Wed, 23 Jan 2019 13:36:49 -0600 Subject: [PATCH 01/59] Fork and get a direct stream to modify --- dump.go | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/dump.go b/dump.go index 4b6b841..87d47af 100644 --- a/dump.go +++ b/dump.go @@ -3,6 +3,7 @@ package mysqldump import ( "database/sql" "errors" + "io" "os" "path" "strings" @@ -67,7 +68,7 @@ UNLOCK TABLES; -- Dump completed on {{ .CompleteTime }} ` -// Creates a MYSQL Dump based on the options supplied through the dumper. +// Dump creates a MySQL dump based on the options supplied through the dumper. func (d *Dumper) Dump() (string, error) { name := time.Now().Format(d.format) p := path.Join(d.dir, name+".sql") @@ -86,28 +87,34 @@ func (d *Dumper) Dump() (string, error) { defer f.Close() + return p, Dump(d.db, f) +} + +// Dump Creates a MYSQL dump from the connection to the stream. +func Dump(db *sql.DB, out io.Writer) error { + var err error data := dump{ DumpVersion: version, Tables: make([]*table, 0), } // Get server version - if data.ServerVersion, err = getServerVersion(d.db); err != nil { - return p, err + if data.ServerVersion, err = getServerVersion(db); err != nil { + return err } // Get tables - tables, err := getTables(d.db) + tables, err := getTables(db) if err != nil { - return p, err + return err } // Get sql for each table for _, name := range tables { - if t, err := createTable(d.db, name); err == nil { + if t, err := createTable(db, name); err == nil { data.Tables = append(data.Tables, t) } else { - return p, err + return err } } @@ -117,13 +124,13 @@ func (d *Dumper) Dump() (string, error) { // Write dump to file t, err := template.New("mysqldump").Parse(tmpl) if err != nil { - return p, err + return err } - if err = t.Execute(f, data); err != nil { - return p, err + if err = t.Execute(out, data); err != nil { + return err } - return p, nil + return nil } func getTables(db *sql.DB) ([]string, error) { @@ -148,11 +155,11 @@ func getTables(db *sql.DB) ([]string, error) { } func getServerVersion(db *sql.DB) (string, error) { - var server_version sql.NullString - if err := db.QueryRow("SELECT version()").Scan(&server_version); err != nil { + var serverVersion sql.NullString + if err := db.QueryRow("SELECT version()").Scan(&serverVersion); err != nil { return "", err } - return server_version.String, nil + return serverVersion.String, nil } func createTable(db *sql.DB, name string) (*table, error) { From c8118b2417f1fd31c2469886edead0cfa2afdedd Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Wed, 23 Jan 2019 13:41:03 -0600 Subject: [PATCH 02/59] Spring cleaning --- dump.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/dump.go b/dump.go index 87d47af..4772e21 100644 --- a/dump.go +++ b/dump.go @@ -179,18 +179,17 @@ func createTable(db *sql.DB, name string) (*table, error) { func createTableSQL(db *sql.DB, name string) (string, error) { // Get table creation SQL - var table_return sql.NullString - var table_sql sql.NullString - err := db.QueryRow("SHOW CREATE TABLE "+name).Scan(&table_return, &table_sql) + var tableReturn, tableSQL sql.NullString + err := db.QueryRow("SHOW CREATE TABLE "+name).Scan(&tableReturn, &tableSQL) if err != nil { return "", err } - if table_return.String != name { + if tableReturn.String != name { return "", errors.New("Returned table is not the same as requested table") } - return table_sql.String, nil + return tableSQL.String, nil } func createTableValues(db *sql.DB, name string) (string, error) { @@ -211,7 +210,7 @@ func createTableValues(db *sql.DB, name string) (string, error) { } // Read data - data_text := make([]string, 0) + dataText := make([]string, 0) for rows.Next() { // Init temp data storage @@ -220,7 +219,7 @@ func createTableValues(db *sql.DB, name string) (string, error) { data := make([]*sql.NullString, len(columns)) ptrs := make([]interface{}, len(columns)) - for i, _ := range data { + for i := range data { ptrs[i] = &data[i] } @@ -239,8 +238,8 @@ func createTableValues(db *sql.DB, name string) (string, error) { } } - data_text = append(data_text, "("+strings.Join(dataStrings, ",")+")") + dataText = append(dataText, "("+strings.Join(dataStrings, ",")+")") } - return strings.Join(data_text, ","), rows.Err() + return strings.Join(dataText, ","), rows.Err() } From cd7bc12931f384ae4f9e9681bb4cbd77a9217bcf Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Wed, 23 Jan 2019 14:13:14 -0600 Subject: [PATCH 03/59] Stream based writer --- dump.go | 108 +++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 76 insertions(+), 32 deletions(-) diff --git a/dump.go b/dump.go index 4772e21..7c55ce4 100644 --- a/dump.go +++ b/dump.go @@ -20,13 +20,17 @@ type table struct { type dump struct { DumpVersion string ServerVersion string - Tables []*table CompleteTime string + HeaderTmpl *template.Template + TableTmpl *template.Template + FooterTmpl *template.Template + Connection *sql.DB + Out io.Writer } -const version = "0.2.2" +const version = "0.3.1" -const tmpl = `-- Go SQL Dump {{ .DumpVersion }} +const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }} -- -- ------------------------------------------------------ -- Server version {{ .ServerVersion }} @@ -41,9 +45,9 @@ const tmpl = `-- Go SQL Dump {{ .DumpVersion }} /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; +` - -{{range .Tables}} +const tableTmpl = ` -- -- Table structure for table {{ .Name }} -- @@ -64,10 +68,72 @@ INSERT INTO {{ .Name }} VALUES {{ .Values }}; {{ end }} /*!40000 ALTER TABLE {{ .Name }} ENABLE KEYS */; UNLOCK TABLES; -{{ end }} +` + +const footerTmpl = ` -- Dump completed on {{ .CompleteTime }} ` +func (data *dump) getTemplates() (err error) { + // Write dump to file + data.HeaderTmpl, err = template.New("mysqldumpHeader").Parse(headerTmpl) + if err != nil { + return + } + + data.TableTmpl, err = template.New("mysqldumpTable").Parse(tableTmpl) + if err != nil { + return + } + + data.FooterTmpl, err = template.New("mysqldumpTable").Parse(footerTmpl) + if err != nil { + return + } + return +} + +func (data *dump) dump() error { + if err := data.HeaderTmpl.Execute(data.Out, data); err != nil { + return err + } + + // Get tables + tables, err := getTables(data.Connection) + if err != nil { + return err + } + + // Get sql for each table + for _, name := range tables { + if err := data.dumpTable(name); err != nil { + return err + } + } + + // Set complete time + data.CompleteTime = time.Now().String() + + if err = data.FooterTmpl.Execute(data.Out, data); err != nil { + return err + } + + return nil +} + +func (data *dump) dumpTable(name string) error { + table, err := createTable(data.Connection, name) + if err != nil { + return err + } + + if err = data.TableTmpl.Execute(data.Out, table); err != nil { + return err + } + + return nil +} + // Dump creates a MySQL dump based on the options supplied through the dumper. func (d *Dumper) Dump() (string, error) { name := time.Now().Format(d.format) @@ -95,7 +161,8 @@ func Dump(db *sql.DB, out io.Writer) error { var err error data := dump{ DumpVersion: version, - Tables: make([]*table, 0), + Connection: db, + Out: out, } // Get server version @@ -103,34 +170,11 @@ func Dump(db *sql.DB, out io.Writer) error { return err } - // Get tables - tables, err := getTables(db) - if err != nil { - return err - } - - // Get sql for each table - for _, name := range tables { - if t, err := createTable(db, name); err == nil { - data.Tables = append(data.Tables, t) - } else { - return err - } - } - - // Set complete time - data.CompleteTime = time.Now().String() - - // Write dump to file - t, err := template.New("mysqldump").Parse(tmpl) - if err != nil { - return err - } - if err = t.Execute(out, data); err != nil { + if err := data.getTemplates(); err != nil { return err } - return nil + return data.dump() } func getTables(db *sql.DB) ([]string, error) { From 846720a32178575d8fd73301f6b5bc25334a1320 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Wed, 23 Jan 2019 14:59:24 -0600 Subject: [PATCH 04/59] Update to MySQL 8 --- dump.go | 11 ++++++----- dump_test.go | 11 ++++------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/dump.go b/dump.go index 7c55ce4..4f3303e 100644 --- a/dump.go +++ b/dump.go @@ -38,7 +38,7 @@ const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }} /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; + SET NAMES utf8mb4; /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; /*!40103 SET TIME_ZONE='+00:00' */; /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; @@ -52,20 +52,21 @@ const tableTmpl = ` -- Table structure for table {{ .Name }} -- -DROP TABLE IF EXISTS {{ .Name }}; +DROP TABLE IF EXISTS ` + "`{{ .Name }}`" + `; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; + SET character_set_client = utf8mb4; {{ .SQL }}; /*!40101 SET character_set_client = @saved_cs_client */; + -- -- Dumping data for table {{ .Name }} -- LOCK TABLES {{ .Name }} WRITE; /*!40000 ALTER TABLE {{ .Name }} DISABLE KEYS */; -{{ if .Values }} +{{- if .Values }} INSERT INTO {{ .Name }} VALUES {{ .Values }}; -{{ end }} +{{- end }} /*!40000 ALTER TABLE {{ .Name }} ENABLE KEYS */; UNLOCK TABLES; ` diff --git a/dump_test.go b/dump_test.go index 24dd7e0..217a7e2 100644 --- a/dump_test.go +++ b/dump_test.go @@ -298,7 +298,7 @@ func TestDumpOk(t *testing.T) { /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; + SET NAMES utf8mb4; /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; /*!40103 SET TIME_ZONE='+00:00' */; /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; @@ -306,26 +306,23 @@ func TestDumpOk(t *testing.T) { /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; - - -- -- Table structure for table Test_Table -- -DROP TABLE IF EXISTS Test_Table; +DROP TABLE IF EXISTS \Test_Table\; /*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; + SET character_set_client = utf8mb4; CREATE TABLE 'Test_Table' (\id\ int(11) NOT NULL AUTO_INCREMENT,\email\ char(60) DEFAULT NULL, \name\ char(60), PRIMARY KEY (\id\))ENGINE=InnoDB DEFAULT CHARSET=latin1; /*!40101 SET character_set_client = @saved_cs_client */; + -- -- Dumping data for table Test_Table -- LOCK TABLES Test_Table WRITE; /*!40000 ALTER TABLE Test_Table DISABLE KEYS */; - INSERT INTO Test_Table VALUES ('1',null,'Test Name 1'),('2','test2@test.de','Test Name 2'); - /*!40000 ALTER TABLE Test_Table ENABLE KEYS */; UNLOCK TABLES; From 61126516402710f83466de63c3272764b8e21dff Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Wed, 23 Jan 2019 15:34:20 -0600 Subject: [PATCH 05/59] readme update --- README.md | 67 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 7e64de5..4eff0c0 100644 --- a/README.md +++ b/README.md @@ -6,47 +6,48 @@ Create MYSQL dumps in Go without the `mysqldump` CLI as a dependancy. package main import ( - "database/sql" - "fmt" + "database/sql" + "fmt" - "github.com/JamesStewy/go-mysqldump" - _ "github.com/go-sql-driver/mysql" + "github.com/JamesStewy/go-mysqldump" + _ "github.com/go-sql-driver/mysql" ) func main() { - // Open connection to database - username := "your-user" - password := "your-pw" - hostname := "your-hostname" - port := "your-port" - dbname := "your-db" + // Open connection to database + config := mysql.NewConfig() + config.User = "your-user" + config.Passwd = "your-pw" + config.DBName = "your-db" + config.Net = "tcp" + config.Addr = "your-hostname:your-port" dumpDir := "dumps" // you should create this directory dumpFilenameFormat := fmt.Sprintf("%s-20060102T150405", dbname) // accepts time layout string and add .sql at the end of file - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%s)/%s", username, password, hostname, port, dbname)) - if err != nil { - fmt.Println("Error opening database: ", err) - return - } - - // Register database with mysqldump - dumper, err := mysqldump.Register(db, dumpDir, dumpFilenameFormat) - if err != nil { - fmt.Println("Error registering databse:", err) - return - } - - // Dump database to file - resultFilename, err := dumper.Dump() - if err != nil { - fmt.Println("Error dumping:", err) - return - } - fmt.Printf("File is saved to %s", resultFilename) - - // Close dumper and connected database - dumper.Close() + db, err := sql.Open("mysql", config.FormatDNS()) + if err != nil { + fmt.Println("Error opening database: ", err) + return + } + + // Register database with mysqldump + dumper, err := mysqldump.Register(db, dumpDir, dumpFilenameFormat) + if err != nil { + fmt.Println("Error registering databse:", err) + return + } + + // Dump database to file + resultFilename, err := dumper.Dump() + if err != nil { + fmt.Println("Error dumping:", err) + return + } + fmt.Printf("File is saved to %s", resultFilename) + + // Close dumper and connected database + dumper.Close() } ``` From 47914c65504fd83b2777b76c44ca7459e506a8d5 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 24 Jan 2019 08:55:06 -0600 Subject: [PATCH 06/59] Go concurrency and backticks --- dump.go | 60 +++++++++++++++++++++++++++++++++------------------- dump_test.go | 26 ++++++++++++++++------- 2 files changed, 56 insertions(+), 30 deletions(-) diff --git a/dump.go b/dump.go index 4f3303e..72e230d 100644 --- a/dump.go +++ b/dump.go @@ -7,6 +7,7 @@ import ( "os" "path" "strings" + "sync" "text/template" "time" ) @@ -21,11 +22,14 @@ type dump struct { DumpVersion string ServerVersion string CompleteTime string - HeaderTmpl *template.Template - TableTmpl *template.Template - FooterTmpl *template.Template - Connection *sql.DB Out io.Writer + Connection *sql.DB + + headerTmpl *template.Template + tableTmpl *template.Template + footerTmpl *template.Template + mux sync.Mutex + wg sync.WaitGroup } const version = "0.3.1" @@ -38,7 +42,7 @@ const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }} /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; - SET NAMES utf8mb4; + SET NAMES utf8mb4 ; /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; /*!40103 SET TIME_ZONE='+00:00' */; /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; @@ -52,9 +56,9 @@ const tableTmpl = ` -- Table structure for table {{ .Name }} -- -DROP TABLE IF EXISTS ` + "`{{ .Name }}`" + `; +DROP TABLE IF EXISTS {{ .Name }}; /*!40101 SET @saved_cs_client = @@character_set_client */; - SET character_set_client = utf8mb4; + SET character_set_client = utf8mb4 ; {{ .SQL }}; /*!40101 SET character_set_client = @saved_cs_client */; @@ -72,22 +76,32 @@ UNLOCK TABLES; ` const footerTmpl = ` +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + -- Dump completed on {{ .CompleteTime }} ` func (data *dump) getTemplates() (err error) { // Write dump to file - data.HeaderTmpl, err = template.New("mysqldumpHeader").Parse(headerTmpl) + data.headerTmpl, err = template.New("mysqldumpHeader").Parse(headerTmpl) if err != nil { return } - data.TableTmpl, err = template.New("mysqldumpTable").Parse(tableTmpl) + data.tableTmpl, err = template.New("mysqldumpTable").Parse(tableTmpl) if err != nil { return } - data.FooterTmpl, err = template.New("mysqldumpTable").Parse(footerTmpl) + data.footerTmpl, err = template.New("mysqldumpTable").Parse(footerTmpl) if err != nil { return } @@ -95,7 +109,7 @@ func (data *dump) getTemplates() (err error) { } func (data *dump) dump() error { - if err := data.HeaderTmpl.Execute(data.Out, data); err != nil { + if err := data.headerTmpl.Execute(data.Out, data); err != nil { return err } @@ -106,20 +120,17 @@ func (data *dump) dump() error { } // Get sql for each table + data.wg.Add(len(tables)) for _, name := range tables { if err := data.dumpTable(name); err != nil { return err } } + data.wg.Wait() // Set complete time data.CompleteTime = time.Now().String() - - if err = data.FooterTmpl.Execute(data.Out, data); err != nil { - return err - } - - return nil + return data.footerTmpl.Execute(data.Out, data) } func (data *dump) dumpTable(name string) error { @@ -128,13 +139,18 @@ func (data *dump) dumpTable(name string) error { return err } - if err = data.TableTmpl.Execute(data.Out, table); err != nil { - return err - } - + go data.writeTable(table) return nil } +func (data *dump) writeTable(table *table) error { + data.mux.Lock() + err := data.tableTmpl.Execute(data.Out, table) + data.mux.Unlock() + data.wg.Done() + return err +} + // Dump creates a MySQL dump based on the options supplied through the dumper. func (d *Dumper) Dump() (string, error) { name := time.Now().Format(d.format) @@ -209,7 +225,7 @@ func getServerVersion(db *sql.DB) (string, error) { func createTable(db *sql.DB, name string) (*table, error) { var err error - t := &table{Name: name} + t := &table{Name: "`" + name + "`"} if t.SQL, err = createTableSQL(db, name); err != nil { return nil, err diff --git a/dump_test.go b/dump_test.go index 217a7e2..b487f6e 100644 --- a/dump_test.go +++ b/dump_test.go @@ -298,7 +298,7 @@ func TestDumpOk(t *testing.T) { /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; - SET NAMES utf8mb4; + SET NAMES utf8mb4 ; /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; /*!40103 SET TIME_ZONE='+00:00' */; /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; @@ -307,25 +307,35 @@ func TestDumpOk(t *testing.T) { /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; -- --- Table structure for table Test_Table +-- Table structure for table \Test_Table\ -- DROP TABLE IF EXISTS \Test_Table\; /*!40101 SET @saved_cs_client = @@character_set_client */; - SET character_set_client = utf8mb4; + SET character_set_client = utf8mb4 ; CREATE TABLE 'Test_Table' (\id\ int(11) NOT NULL AUTO_INCREMENT,\email\ char(60) DEFAULT NULL, \name\ char(60), PRIMARY KEY (\id\))ENGINE=InnoDB DEFAULT CHARSET=latin1; /*!40101 SET character_set_client = @saved_cs_client */; -- --- Dumping data for table Test_Table +-- Dumping data for table \Test_Table\ -- -LOCK TABLES Test_Table WRITE; -/*!40000 ALTER TABLE Test_Table DISABLE KEYS */; -INSERT INTO Test_Table VALUES ('1',null,'Test Name 1'),('2','test2@test.de','Test Name 2'); -/*!40000 ALTER TABLE Test_Table ENABLE KEYS */; +LOCK TABLES \Test_Table\ WRITE; +/*!40000 ALTER TABLE \Test_Table\ DISABLE KEYS */; +INSERT INTO \Test_Table\ VALUES ('1',null,'Test Name 1'),('2','test2@test.de','Test Name 2'); +/*!40000 ALTER TABLE \Test_Table\ ENABLE KEYS */; UNLOCK TABLES; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + ` if !reflect.DeepEqual(result, expected) { From 011beef02d0206d703334895a5a6f0dacb0b3020 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 24 Jan 2019 11:19:54 -0600 Subject: [PATCH 07/59] Breaking things --- dump.go | 57 ++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/dump.go b/dump.go index 72e230d..1c646ec 100644 --- a/dump.go +++ b/dump.go @@ -3,9 +3,11 @@ package mysqldump import ( "database/sql" "errors" + "fmt" "io" "os" "path" + "reflect" "strings" "sync" "text/template" @@ -272,30 +274,55 @@ func createTableValues(db *sql.DB, name string) (string, error) { // Read data dataText := make([]string, 0) - for rows.Next() { - // Init temp data storage - - //ptrs := make([]interface{}, len(columns)) - //var ptrs []interface {} = make([]*sql.NullString, len(columns)) + tt, err := rows.ColumnTypes() + if err != nil { + return "", err + } - data := make([]*sql.NullString, len(columns)) - ptrs := make([]interface{}, len(columns)) - for i := range data { - ptrs[i] = &data[i] + types := make([]reflect.Type, len(tt)) + for i, tp := range tt { + st := tp.ScanType() + if st == nil || st.Kind() == reflect.Slice { + types[i] = reflect.TypeOf(sql.NullString{}) + } else if st.Kind() == reflect.Int || + st.Kind() == reflect.Int8 || + st.Kind() == reflect.Int16 || + st.Kind() == reflect.Int32 || + st.Kind() == reflect.Int64 { + types[i] = reflect.TypeOf(sql.NullInt64{}) + } else { + types[i] = st } - + } + values := make([]interface{}, len(tt)) + for i := range values { + values[i] = reflect.New(types[i]).Interface() + } + for rows.Next() { // Read data - if err := rows.Scan(ptrs...); err != nil { + if err := rows.Scan(values...); err != nil { return "", err } dataStrings := make([]string, len(columns)) - for key, value := range data { - if value != nil && value.Valid { - dataStrings[key] = "'" + value.String + "'" - } else { + for key, value := range values { + if value == nil { dataStrings[key] = "null" + } else if s, ok := value.(*sql.NullString); ok { + if s.Valid { + dataStrings[key] = "'" + strings.Replace(s.String, "\n", "\\n", -1) + "'" + } else { + dataStrings[key] = "NULL" + } + } else if s, ok := value.(*sql.NullInt64); ok { + if s.Valid { + dataStrings[key] = fmt.Sprintf("%d", s.Int64) + } else { + dataStrings[key] = "NULL" + } + } else { + panic(value) } } From 9a9da92a96e0d5bf6ec8d0cbf1d3368d8f011fb1 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 24 Jan 2019 12:54:28 -0600 Subject: [PATCH 08/59] No need to panic --- dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump.go b/dump.go index 1c646ec..076dd3b 100644 --- a/dump.go +++ b/dump.go @@ -322,7 +322,7 @@ func createTableValues(db *sql.DB, name string) (string, error) { dataStrings[key] = "NULL" } } else { - panic(value) + dataStrings[key] = fmt.Sprint("'", value, "'") } } From 3542d95e4cfcde63328b78c374adf184abfe0796 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 24 Jan 2019 12:55:23 -0600 Subject: [PATCH 09/59] 0.3.0 for tag --- dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump.go b/dump.go index 076dd3b..ceb8551 100644 --- a/dump.go +++ b/dump.go @@ -34,7 +34,7 @@ type dump struct { wg sync.WaitGroup } -const version = "0.3.1" +const version = "0.3.0" const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }} -- From 523f634e058d9ef2515429af00f95f6527b913b0 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 24 Jan 2019 12:55:48 -0600 Subject: [PATCH 10/59] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4eff0c0..2c3e3fc 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ import ( "fmt" "github.com/JamesStewy/go-mysqldump" - _ "github.com/go-sql-driver/mysql" + "github.com/go-sql-driver/mysql" ) func main() { From 90fdf91b1fd87c48452f945c1378203885367b85 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 24 Jan 2019 14:51:26 -0600 Subject: [PATCH 11/59] Sanitize and blob values --- dump.go | 15 +++++++++++---- dump_test.go | 1 - sanitize.go | 31 +++++++++++++++++++++++++++++++ sanitize_test.go | 25 +++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 sanitize.go create mode 100644 sanitize_test.go diff --git a/dump.go b/dump.go index ceb8551..005040f 100644 --- a/dump.go +++ b/dump.go @@ -77,8 +77,7 @@ INSERT INTO {{ .Name }} VALUES {{ .Values }}; UNLOCK TABLES; ` -const footerTmpl = ` -/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; +const footerTmpl = `/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; @@ -282,7 +281,9 @@ func createTableValues(db *sql.DB, name string) (string, error) { types := make([]reflect.Type, len(tt)) for i, tp := range tt { st := tp.ScanType() - if st == nil || st.Kind() == reflect.Slice { + if tp.DatabaseTypeName() == "BLOB" { + types[i] = reflect.TypeOf(sql.RawBytes{}) + } else if st == nil || st.Kind() == reflect.Slice { types[i] = reflect.TypeOf(sql.NullString{}) } else if st.Kind() == reflect.Int || st.Kind() == reflect.Int8 || @@ -311,7 +312,7 @@ func createTableValues(db *sql.DB, name string) (string, error) { dataStrings[key] = "null" } else if s, ok := value.(*sql.NullString); ok { if s.Valid { - dataStrings[key] = "'" + strings.Replace(s.String, "\n", "\\n", -1) + "'" + dataStrings[key] = "'" + sanitize(s.String) + "'" } else { dataStrings[key] = "NULL" } @@ -321,6 +322,12 @@ func createTableValues(db *sql.DB, name string) (string, error) { } else { dataStrings[key] = "NULL" } + } else if s, ok := value.(*sql.RawBytes); ok { + if len(*s) == 0 { + dataStrings[key] = "NULL" + } else { + dataStrings[key] = "_binary '" + sanitize(string(*s)) + "'" + } } else { dataStrings[key] = fmt.Sprint("'", value, "'") } diff --git a/dump_test.go b/dump_test.go index b487f6e..21f20ce 100644 --- a/dump_test.go +++ b/dump_test.go @@ -325,7 +325,6 @@ LOCK TABLES \Test_Table\ WRITE; INSERT INTO \Test_Table\ VALUES ('1',null,'Test Name 1'),('2','test2@test.de','Test Name 2'); /*!40000 ALTER TABLE \Test_Table\ ENABLE KEYS */; UNLOCK TABLES; - /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; diff --git a/sanitize.go b/sanitize.go new file mode 100644 index 0000000..adc4676 --- /dev/null +++ b/sanitize.go @@ -0,0 +1,31 @@ +package mysqldump + +import "strings" + +var lazyMySQLReplacer *strings.Replacer + +func mysqlReplacer() *strings.Replacer { + if lazyMySQLReplacer == nil { + lazyMySQLReplacer = strings.NewReplacer( + "\x00", "\\0", + "'", "\\'", + "\"", "\\\"", + "\b", "\\b", + "\n", "\\n", + "\r", "\\r", + // "\t", "\\t", + "\x1A", "\\Z", // ASCII 26 == x1A + "\\", "\\\\", + "%", "\\%", + // "_", "\\_", + ) + } + return lazyMySQLReplacer +} + +// MySQL sanitizes mysql based on +// https://dev.mysql.com/doc/refman/8.0/en/string-literals.html table 9.1 +// needs to be placed in either a single or a double quoted string +func sanitize(input string) string { + return mysqlReplacer().Replace(input) +} diff --git a/sanitize_test.go b/sanitize_test.go new file mode 100644 index 0000000..6303013 --- /dev/null +++ b/sanitize_test.go @@ -0,0 +1,25 @@ +package mysqldump + +import ( + "fmt" + "testing" +) + +func TestForSQLInjection(t *testing.T) { + examples := [][]string{ + /** Query ** Input ** Expected **/ + {"SELECT * WHERE field = '%s';", "test", "SELECT * WHERE field = 'test';"}, + {"'%s'", "'; DROP TABLES `test`;", "'\\'; DROP TABLES `test`;'"}, + {"'%s'", "'+(SELECT name FROM users LIMIT 1)+'", "'\\'+(SELECT name FROM users LIMIT 1)+\\''"}, + {"SELECT '%s'", "\x00x633A5C626F6F742E696E69", "SELECT '\\0x633A5C626F6F742E696E69'"}, + {"WHERE PASSWORD('%s')", "') OR 1=1--", "WHERE PASSWORD('\\') OR 1=1--')"}, + } + var query string + for _, example := range examples { + query = fmt.Sprintf(example[0], sanitize(example[1])) + + if example[2] != query { + t.Fatalf("expected %#v, got %#v", example[2], query) + } + } +} From 46db301fed25f1f61d72b5ce23d0183b96d725c2 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 24 Jan 2019 14:58:20 -0600 Subject: [PATCH 12/59] Version bump for the blob sanitization --- dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump.go b/dump.go index 005040f..d2b5aaf 100644 --- a/dump.go +++ b/dump.go @@ -34,7 +34,7 @@ type dump struct { wg sync.WaitGroup } -const version = "0.3.0" +const version = "0.3.1" const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }} -- From 30b13cdf22f247a2ed083510e997cce6f8f466cc Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 24 Jan 2019 16:36:28 -0600 Subject: [PATCH 13/59] Correct escaping --- dump.go | 2 +- sanitize.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dump.go b/dump.go index d2b5aaf..948b688 100644 --- a/dump.go +++ b/dump.go @@ -34,7 +34,7 @@ type dump struct { wg sync.WaitGroup } -const version = "0.3.1" +const version = "0.3.2" const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }} -- diff --git a/sanitize.go b/sanitize.go index adc4676..5966451 100644 --- a/sanitize.go +++ b/sanitize.go @@ -16,7 +16,7 @@ func mysqlReplacer() *strings.Replacer { // "\t", "\\t", "\x1A", "\\Z", // ASCII 26 == x1A "\\", "\\\\", - "%", "\\%", + // "%", "\\%", // "_", "\\_", ) } From f253cf795efc151a94ab74976f72c29e2744fb4a Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 24 Jan 2019 16:58:01 -0600 Subject: [PATCH 14/59] travis to newer go --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1873832..7294d6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,7 @@ language: go go: - - 1.6 - - 1.7.x + - 1.11.x - master script: From c9037df973d0ad53db44f9a3cf432dd3235fe87b Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Fri, 25 Jan 2019 09:08:23 -0600 Subject: [PATCH 15/59] Failing tests on mock --- .vscode/settings.json | 2 + dump.go | 148 +++++++++++++++----------------- dump_test.go | 192 ++++++++++++++++-------------------------- mysqldump.go | 65 ++++++++------ mysqldump_test.go | 101 ++++++++++++++++++++++ 5 files changed, 286 insertions(+), 222 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 mysqldump_test.go diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..7a73a41 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file diff --git a/dump.go b/dump.go index 948b688..d81e28d 100644 --- a/dump.go +++ b/dump.go @@ -5,8 +5,6 @@ import ( "errors" "fmt" "io" - "os" - "path" "reflect" "strings" "sync" @@ -20,12 +18,11 @@ type table struct { Values string } -type dump struct { - DumpVersion string - ServerVersion string - CompleteTime string - Out io.Writer - Connection *sql.DB +// Data struct to configure dump behavior +type Data struct { + Out io.Writer + Connection *sql.DB + IgnoreTables []string headerTmpl *template.Template tableTmpl *template.Template @@ -34,6 +31,12 @@ type dump struct { wg sync.WaitGroup } +type metaData struct { + DumpVersion string + ServerVersion string + CompleteTime string +} + const version = "0.3.2" const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }} @@ -90,32 +93,27 @@ const footerTmpl = `/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; -- Dump completed on {{ .CompleteTime }} ` -func (data *dump) getTemplates() (err error) { - // Write dump to file - data.headerTmpl, err = template.New("mysqldumpHeader").Parse(headerTmpl) - if err != nil { - return +// Dump data using struct +func (data *Data) Dump() error { + meta := metaData{ + DumpVersion: version, } - data.tableTmpl, err = template.New("mysqldumpTable").Parse(tableTmpl) - if err != nil { - return + // Get server version + if err := meta.updateServerVersion(data.Connection); err != nil { + return err } - data.footerTmpl, err = template.New("mysqldumpTable").Parse(footerTmpl) - if err != nil { - return + if err := data.getTemplates(); err != nil { + return err } - return -} -func (data *dump) dump() error { - if err := data.headerTmpl.Execute(data.Out, data); err != nil { + if err := data.headerTmpl.Execute(data.Out, meta); err != nil { return err } // Get tables - tables, err := getTables(data.Connection) + tables, err := data.getTables() if err != nil { return err } @@ -130,12 +128,16 @@ func (data *dump) dump() error { data.wg.Wait() // Set complete time - data.CompleteTime = time.Now().String() - return data.footerTmpl.Execute(data.Out, data) + meta.CompleteTime = time.Now().String() + return data.footerTmpl.Execute(data.Out, meta) } -func (data *dump) dumpTable(name string) error { - table, err := createTable(data.Connection, name) +// MARK: - Private methods + +// MARK: writter methods + +func (data *Data) dumpTable(name string) error { + table, err := data.createTable(name) if err != nil { return err } @@ -144,7 +146,7 @@ func (data *dump) dumpTable(name string) error { return nil } -func (data *dump) writeTable(table *table) error { +func (data *Data) writeTable(table *table) error { data.mux.Lock() err := data.tableTmpl.Execute(data.Out, table) data.mux.Unlock() @@ -152,54 +154,32 @@ func (data *dump) writeTable(table *table) error { return err } -// Dump creates a MySQL dump based on the options supplied through the dumper. -func (d *Dumper) Dump() (string, error) { - name := time.Now().Format(d.format) - p := path.Join(d.dir, name+".sql") - - // Check dump directory - if e, _ := exists(p); e { - return p, errors.New("Dump '" + name + "' already exists.") - } - - // Create .sql file - f, err := os.Create(p) +// MARK: get methods +func (data *Data) getTemplates() (err error) { + // Write dump to file + data.headerTmpl, err = template.New("mysqldumpHeader").Parse(headerTmpl) if err != nil { - return p, err - } - - defer f.Close() - - return p, Dump(d.db, f) -} - -// Dump Creates a MYSQL dump from the connection to the stream. -func Dump(db *sql.DB, out io.Writer) error { - var err error - data := dump{ - DumpVersion: version, - Connection: db, - Out: out, + return } - // Get server version - if data.ServerVersion, err = getServerVersion(db); err != nil { - return err + data.tableTmpl, err = template.New("mysqldumpTable").Parse(tableTmpl) + if err != nil { + return } - if err := data.getTemplates(); err != nil { - return err + data.footerTmpl, err = template.New("mysqldumpTable").Parse(footerTmpl) + if err != nil { + return } - - return data.dump() + return } -func getTables(db *sql.DB) ([]string, error) { +func (data *Data) getTables() ([]string, error) { tables := make([]string, 0) // Get table list - rows, err := db.Query("SHOW TABLES") + rows, err := data.Connection.Query("SHOW TABLES") if err != nil { return tables, err } @@ -211,38 +191,50 @@ func getTables(db *sql.DB) ([]string, error) { if err := rows.Scan(&table); err != nil { return tables, err } - tables = append(tables, table.String) + if table.Valid && !data.isIgnoredTable(table.String) { + tables = append(tables, table.String) + } } return tables, rows.Err() } -func getServerVersion(db *sql.DB) (string, error) { - var serverVersion sql.NullString - if err := db.QueryRow("SELECT version()").Scan(&serverVersion); err != nil { - return "", err +func (data *Data) isIgnoredTable(name string) bool { + for _, item := range data.IgnoreTables { + if item == name { + return true + } } - return serverVersion.String, nil + return false } -func createTable(db *sql.DB, name string) (*table, error) { +func (data *metaData) updateServerVersion(db *sql.DB) (err error) { + var serverVersion sql.NullString + err = db.QueryRow("SELECT version()").Scan(&serverVersion) + data.ServerVersion = serverVersion.String + return +} + +// MARK: create methods + +func (data *Data) createTable(name string) (*table, error) { var err error t := &table{Name: "`" + name + "`"} - if t.SQL, err = createTableSQL(db, name); err != nil { + if t.SQL, err = data.createTableSQL(name); err != nil { return nil, err } - if t.Values, err = createTableValues(db, name); err != nil { + if t.Values, err = data.createTableValues(name); err != nil { return nil, err } return t, nil } -func createTableSQL(db *sql.DB, name string) (string, error) { +func (data *Data) createTableSQL(name string) (string, error) { // Get table creation SQL var tableReturn, tableSQL sql.NullString - err := db.QueryRow("SHOW CREATE TABLE "+name).Scan(&tableReturn, &tableSQL) + err := data.Connection.QueryRow("SHOW CREATE TABLE "+name).Scan(&tableReturn, &tableSQL) if err != nil { return "", err @@ -254,9 +246,9 @@ func createTableSQL(db *sql.DB, name string) (string, error) { return tableSQL.String, nil } -func createTableValues(db *sql.DB, name string) (string, error) { +func (data *Data) createTableValues(name string) (string, error) { // Get Data - rows, err := db.Query("SELECT * FROM " + name) + rows, err := data.Connection.Query("SELECT * FROM " + name) if err != nil { return "", err } diff --git a/dump_test.go b/dump_test.go index 21f20ce..1b1077e 100644 --- a/dump_test.go +++ b/dump_test.go @@ -1,10 +1,7 @@ package mysqldump import ( - "io/ioutil" - "os" "reflect" - "strings" "testing" sqlmock "github.com/DATA-DOG/go-sqlmock" @@ -24,7 +21,11 @@ func TestGetTablesOk(t *testing.T) { mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(rows) - result, err := getTables(db) + data := Data{ + Connection: db, + } + + result, err := data.getTables() if err != nil { t.Errorf("error was not expected while updating stats: %s", err) } @@ -41,6 +42,42 @@ func TestGetTablesOk(t *testing.T) { } } +func TestIgnoreTablesOk(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) + } + + defer db.Close() + + rows := sqlmock.NewRows([]string{"Tables_in_Testdb"}). + AddRow("Test_Table_1"). + AddRow("Test_Table_2") + + mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(rows) + + data := Data{ + Connection: db, + IgnoreTables: []string{"Test_Table_1"}, + } + + result, err := data.getTables() + if err != nil { + t.Errorf("error was not expected while updating stats: %s", err) + } + + // we make sure that all expectations were met + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled expections: %s", err) + } + + expectedResult := []string{"Test_Table_2"} + + if !reflect.DeepEqual(result, expectedResult) { + t.Fatalf("expected %#v, got %#v", result, expectedResult) + } +} + func TestGetTablesNil(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { @@ -56,7 +93,11 @@ func TestGetTablesNil(t *testing.T) { mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(rows) - result, err := getTables(db) + data := Data{ + Connection: db, + } + + result, err := data.getTables() if err != nil { t.Errorf("error was not expected while updating stats: %s", err) } @@ -66,7 +107,7 @@ func TestGetTablesNil(t *testing.T) { t.Errorf("there were unfulfilled expections: %s", err) } - expectedResult := []string{"Test_Table_1", "", "Test_Table_3"} + expectedResult := []string{"Test_Table_1", "Test_Table_3"} if !reflect.DeepEqual(result, expectedResult) { t.Fatalf("expected %#v, got %#v", expectedResult, result) @@ -86,8 +127,9 @@ func TestGetServerVersionOk(t *testing.T) { mock.ExpectQuery("^SELECT version()").WillReturnRows(rows) - result, err := getServerVersion(db) - if err != nil { + meta := metaData{} + + if err := meta.updateServerVersion(db); err != nil { t.Errorf("error was not expected while updating stats: %s", err) } @@ -98,8 +140,8 @@ func TestGetServerVersionOk(t *testing.T) { expectedResult := "test_version" - if !reflect.DeepEqual(result, expectedResult) { - t.Fatalf("expected %#v, got %#v", expectedResult, result) + if !reflect.DeepEqual(meta.ServerVersion, expectedResult) { + t.Fatalf("expected %#v, got %#v", expectedResult, meta.ServerVersion) } } @@ -116,7 +158,11 @@ func TestCreateTableSQLOk(t *testing.T) { mock.ExpectQuery("^SHOW CREATE TABLE Test_Table$").WillReturnRows(rows) - result, err := createTableSQL(db, "Test_Table") + data := Data{ + Connection: db, + } + + result, err := data.createTableSQL("Test_Table") if err != nil { t.Errorf("error was not expected while updating stats: %s", err) @@ -148,7 +194,11 @@ func TestCreateTableValuesOk(t *testing.T) { mock.ExpectQuery("^SELECT (.+) FROM test$").WillReturnRows(rows) - result, err := createTableValues(db, "test") + data := Data{ + Connection: db, + } + + result, err := data.createTableValues("test") if err != nil { t.Errorf("error was not expected while updating stats: %s", err) } @@ -180,7 +230,11 @@ func TestCreateTableValuesNil(t *testing.T) { mock.ExpectQuery("^SELECT (.+) FROM test$").WillReturnRows(rows) - result, err := createTableValues(db, "test") + data := Data{ + Connection: db, + } + + result, err := data.createTableValues("test") if err != nil { t.Errorf("error was not expected while updating stats: %s", err) } @@ -215,7 +269,11 @@ func TestCreateTableOk(t *testing.T) { mock.ExpectQuery("^SHOW CREATE TABLE Test_Table$").WillReturnRows(createTableRows) mock.ExpectQuery("^SELECT (.+) FROM Test_Table$").WillReturnRows(createTableValueRows) - result, err := createTable(db, "Test_Table") + data := Data{ + Connection: db, + } + + result, err := data.createTable("Test_Table") if err != nil { t.Errorf("error was not expected while updating stats: %s", err) } @@ -235,109 +293,3 @@ func TestCreateTableOk(t *testing.T) { t.Fatalf("expected %#v, got %#v", expectedResult, result) } } - -func TestDumpOk(t *testing.T) { - - tmpFile := "/tmp/test_format.sql" - os.Remove(tmpFile) - - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - - defer db.Close() - - showTablesRows := sqlmock.NewRows([]string{"Tables_in_Testdb"}). - AddRow("Test_Table") - - serverVersionRows := sqlmock.NewRows([]string{"Version()"}). - AddRow("test_version") - - createTableRows := sqlmock.NewRows([]string{"Table", "Create Table"}). - AddRow("Test_Table", "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`email` char(60) DEFAULT NULL, `name` char(60), PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1") - - createTableValueRows := sqlmock.NewRows([]string{"id", "email", "name"}). - AddRow(1, nil, "Test Name 1"). - AddRow(2, "test2@test.de", "Test Name 2") - - mock.ExpectQuery("^SELECT version()").WillReturnRows(serverVersionRows) - mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(showTablesRows) - mock.ExpectQuery("^SHOW CREATE TABLE Test_Table$").WillReturnRows(createTableRows) - mock.ExpectQuery("^SELECT (.+) FROM Test_Table$").WillReturnRows(createTableValueRows) - - dumper := &Dumper{ - db: db, - format: "test_format", - dir: "/tmp/", - } - - path, err := dumper.Dump() - - if path == "" { - t.Errorf("No empty path was expected while dumping the database") - } - - if err != nil { - t.Errorf("error was not expected while dumping the database: %s", err) - } - - f, err := ioutil.ReadFile("/tmp/test_format.sql") - - if err != nil { - t.Errorf("error was not expected while reading the file: %s", err) - } - - result := strings.Replace(strings.Split(string(f), "-- Dump completed")[0], "`", "\\", -1) - - expected := `-- Go SQL Dump ` + version + ` --- --- ------------------------------------------------------ --- Server version test_version - -/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; -/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; -/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; - SET NAMES utf8mb4 ; -/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; -/*!40103 SET TIME_ZONE='+00:00' */; -/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; -/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; -/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; -/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; - --- --- Table structure for table \Test_Table\ --- - -DROP TABLE IF EXISTS \Test_Table\; -/*!40101 SET @saved_cs_client = @@character_set_client */; - SET character_set_client = utf8mb4 ; -CREATE TABLE 'Test_Table' (\id\ int(11) NOT NULL AUTO_INCREMENT,\email\ char(60) DEFAULT NULL, \name\ char(60), PRIMARY KEY (\id\))ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table \Test_Table\ --- - -LOCK TABLES \Test_Table\ WRITE; -/*!40000 ALTER TABLE \Test_Table\ DISABLE KEYS */; -INSERT INTO \Test_Table\ VALUES ('1',null,'Test Name 1'),('2','test2@test.de','Test Name 2'); -/*!40000 ALTER TABLE \Test_Table\ ENABLE KEYS */; -UNLOCK TABLES; -/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; - -/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; -/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; -/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; -/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; -/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; -/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; -/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; - -` - - if !reflect.DeepEqual(result, expected) { - t.Fatalf("expected %#v, got %#v", expected, result) - } -} diff --git a/mysqldump.go b/mysqldump.go index aebe950..7d7ce5e 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -3,44 +3,68 @@ package mysqldump import ( "database/sql" "errors" + "io" "os" + "path" + "time" ) -// Dumper represents a database. -type Dumper struct { - db *sql.DB - format string - dir string -} - /* -Creates a new dumper. +Register a new dumper. db: Database that will be dumped (https://golang.org/pkg/database/sql/#DB). dir: Path to the directory where the dumps will be stored. format: Format to be used to name each dump file. Uses time.Time.Format (https://golang.org/pkg/time/#Time.Format). format appended with '.sql'. */ -func Register(db *sql.DB, dir, format string) (*Dumper, error) { +func Register(db *sql.DB, dir, format string) (*Data, error) { if !isDir(dir) { return nil, errors.New("Invalid directory") } - return &Dumper{ - db: db, - format: format, - dir: dir, + name := time.Now().Format(format) + p := path.Join(dir, name+".sql") + + // Check dump directory + if e, _ := exists(p); e { + return nil, errors.New("Dump '" + name + "' already exists.") + } + + // Create .sql file + f, err := os.Create(p) + + if err != nil { + return nil, err + } + + return &Data{ + Out: f, + Connection: db, }, nil } -// Closes the dumper. +// Dump Creates a MYSQL dump from the connection to the stream. +func Dump(db *sql.DB, out io.Writer) error { + data := Data{ + Connection: db, + Out: out, + } + + return data.Dump() +} + +// Close the dumper. // Will also close the database the dumper is connected to. // // Not required. -func (d *Dumper) Close() error { +func (d *Data) Close() error { defer func() { - d.db = nil + d.Connection = nil + d.Out = nil }() - return d.db.Close() + if file, ok := d.Out.(*os.File); ok { + file.Close() + } + return d.Connection.Close() } func exists(p string) (bool, os.FileInfo) { @@ -56,13 +80,6 @@ func exists(p string) (bool, os.FileInfo) { return true, fi } -func isFile(p string) bool { - if e, fi := exists(p); e { - return fi.Mode().IsRegular() - } - return false -} - func isDir(p string) bool { if e, fi := exists(p); e { return fi.Mode().IsDir() diff --git a/mysqldump_test.go b/mysqldump_test.go new file mode 100644 index 0000000..1ab27fe --- /dev/null +++ b/mysqldump_test.go @@ -0,0 +1,101 @@ +package mysqldump + +import ( + "bytes" + "os" + "reflect" + "strings" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" +) + +func TestDumpOk(t *testing.T) { + + tmpFile := "/tmp/test_format.sql" + os.Remove(tmpFile) + + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) + } + + defer db.Close() + + showTablesRows := sqlmock.NewRows([]string{"Tables_in_Testdb"}). + AddRow("Test_Table") + + serverVersionRows := sqlmock.NewRows([]string{"Version()"}). + AddRow("test_version") + + createTableRows := sqlmock.NewRows([]string{"Table", "Create Table"}). + AddRow("Test_Table", "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`email` char(60) DEFAULT NULL, `name` char(60), PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1") + + createTableValueRows := sqlmock.NewRows([]string{"id", "email", "name"}). + AddRow(1, nil, "Test Name 1"). + AddRow(2, "test2@test.de", "Test Name 2") + + mock.ExpectQuery("^SELECT version()").WillReturnRows(serverVersionRows) + mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(showTablesRows) + mock.ExpectQuery("^SHOW CREATE TABLE Test_Table$").WillReturnRows(createTableRows) + mock.ExpectQuery("^SELECT (.+) FROM Test_Table$").WillReturnRows(createTableValueRows) + + buf := new(bytes.Buffer) + err = Dump(db, buf) + if err != nil { + t.Fatalf("an error '%s' was not expected when dumping a stub database connection", err) + } + + result := strings.Replace(strings.Split(buf.String(), "-- Dump completed")[0], "`", "\\", -1) + + expected := `-- Go SQL Dump ` + version + ` +-- +-- ------------------------------------------------------ +-- Server version test_version + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; + SET NAMES utf8mb4 ; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table \Test_Table\ +-- + +DROP TABLE IF EXISTS \Test_Table\; +/*!40101 SET @saved_cs_client = @@character_set_client */; + SET character_set_client = utf8mb4 ; +CREATE TABLE 'Test_Table' (\id\ int(11) NOT NULL AUTO_INCREMENT,\email\ char(60) DEFAULT NULL, \name\ char(60), PRIMARY KEY (\id\))ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table \Test_Table\ +-- + +LOCK TABLES \Test_Table\ WRITE; +/*!40000 ALTER TABLE \Test_Table\ DISABLE KEYS */; +INSERT INTO \Test_Table\ VALUES (1,null,'Test Name 1'),(2,'test2@test.de','Test Name 2'); +/*!40000 ALTER TABLE \Test_Table\ ENABLE KEYS */; +UNLOCK TABLES; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +` + + if !reflect.DeepEqual(result, expected) { + t.Fatalf("expected %#v, got %#v", expected, result) + } +} From fd3ebcf3dbdf2de621b83cc075b50ea4aa570088 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Fri, 25 Jan 2019 09:34:27 -0600 Subject: [PATCH 16/59] Ability to ignore tables --- dump.go | 8 +++----- dump_test.go | 6 +++--- mysqldump_test.go | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/dump.go b/dump.go index d81e28d..8239434 100644 --- a/dump.go +++ b/dump.go @@ -275,16 +275,14 @@ func (data *Data) createTableValues(name string) (string, error) { st := tp.ScanType() if tp.DatabaseTypeName() == "BLOB" { types[i] = reflect.TypeOf(sql.RawBytes{}) - } else if st == nil || st.Kind() == reflect.Slice { - types[i] = reflect.TypeOf(sql.NullString{}) - } else if st.Kind() == reflect.Int || + } else if st != nil && (st.Kind() == reflect.Int || st.Kind() == reflect.Int8 || st.Kind() == reflect.Int16 || st.Kind() == reflect.Int32 || - st.Kind() == reflect.Int64 { + st.Kind() == reflect.Int64) { types[i] = reflect.TypeOf(sql.NullInt64{}) } else { - types[i] = st + types[i] = reflect.TypeOf(sql.NullString{}) } } values := make([]interface{}, len(tt)) diff --git a/dump_test.go b/dump_test.go index 1b1077e..7efdd7d 100644 --- a/dump_test.go +++ b/dump_test.go @@ -244,7 +244,7 @@ func TestCreateTableValuesNil(t *testing.T) { t.Errorf("there were unfulfilled expections: %s", err) } - expectedResult := "('1',null,'Test Name 1'),('2','test2@test.de','Test Name 2'),('3','','Test Name 3')" + expectedResult := "('1',NULL,'Test Name 1'),('2','test2@test.de','Test Name 2'),('3','','Test Name 3')" if !reflect.DeepEqual(result, expectedResult) { t.Fatalf("expected %#v, got %#v", expectedResult, result) @@ -284,9 +284,9 @@ func TestCreateTableOk(t *testing.T) { } expectedResult := &table{ - Name: "Test_Table", + Name: "`Test_Table`", SQL: "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`s` char(60) DEFAULT NULL, PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1", - Values: "('1',null,'Test Name 1'),('2','test2@test.de','Test Name 2')", + Values: "('1',NULL,'Test Name 1'),('2','test2@test.de','Test Name 2')", } if !reflect.DeepEqual(result, expectedResult) { diff --git a/mysqldump_test.go b/mysqldump_test.go index 1ab27fe..389c9ae 100644 --- a/mysqldump_test.go +++ b/mysqldump_test.go @@ -80,7 +80,7 @@ CREATE TABLE 'Test_Table' (\id\ int(11) NOT NULL AUTO_INCREMENT,\email\ char(60) LOCK TABLES \Test_Table\ WRITE; /*!40000 ALTER TABLE \Test_Table\ DISABLE KEYS */; -INSERT INTO \Test_Table\ VALUES (1,null,'Test Name 1'),(2,'test2@test.de','Test Name 2'); +INSERT INTO \Test_Table\ VALUES ('1',NULL,'Test Name 1'),('2','test2@test.de','Test Name 2'); /*!40000 ALTER TABLE \Test_Table\ ENABLE KEYS */; UNLOCK TABLES; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; From 79fd69db1ebb9c77dd65cc53dfbd5af49892f286 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Fri, 25 Jan 2019 09:57:12 -0600 Subject: [PATCH 17/59] Update docs --- README.md | 7 +++---- doc.go | 44 ++++++++++++++++++++++++++------------------ dump.go | 19 +++++++++++++------ mysqldump.go | 2 +- 4 files changed, 43 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 2c3e3fc..ff6518a 100644 --- a/README.md +++ b/README.md @@ -46,11 +46,10 @@ func main() { } fmt.Printf("File is saved to %s", resultFilename) - // Close dumper and connected database + // Close dumper, connected database and file stream. dumper.Close() } - ``` -[![GoDoc](https://godoc.org/github.com/JamesStewy/go-mysqldump?status.svg)](https://godoc.org/github.com/JamesStewy/go-mysqldump) -[![Build Status](https://travis-ci.org/JamesStewy/go-mysqldump.svg?branch=master)](https://travis-ci.org/JamesStewy/go-mysqldump) +[![GoDoc](https://godoc.org/github.com/jamf/go-mysqldump?status.svg)](https://godoc.org/github.com/jamf/go-mysqldump) +[![Build Status](https://travis-ci.org/jamf/go-mysqldump.svg?branch=master)](https://travis-ci.org/jamf/go-mysqldump) diff --git a/doc.go b/doc.go index 0851a97..2bff1e7 100644 --- a/doc.go +++ b/doc.go @@ -8,39 +8,47 @@ This example uses the mymysql driver (example 7 https://github.com/ziutek/mymysq package main import ( - "database/sql" - "fmt" - "github.com/JamesStewy/go-mysqldump" - "github.com/ziutek/mymysql/godrv" - "time" + "database/sql" + "fmt" + + "github.com/JamesStewy/go-mysqldump" + "github.com/go-sql-driver/mysql" ) func main() { - // Register the mymysql driver - godrv.Register("SET NAMES utf8") - // Open connection to database - db, err := sql.Open("mymysql", "tcp:host:port*database/user/password") - if err != nil { - fmt.Println("Error opening databse:", err) + config := mysql.NewConfig() + config.User = "your-user" + config.Passwd = "your-pw" + config.DBName = "your-db" + config.Net = "tcp" + config.Addr = "your-hostname:your-port" + + dumpDir := "dumps" // you should create this directory + dumpFilenameFormat := fmt.Sprintf("%s-20060102T150405", dbname) // accepts time layout string and add .sql at the end of file + + db, err := sql.Open("mysql", config.FormatDNS()) + if err != nil { + fmt.Println("Error opening database: ", err) return } // Register database with mysqldump - dumper, err := mysqldump.Register(db, "dumps", time.ANSIC) + dumper, err := mysqldump.Register(db, dumpDir, dumpFilenameFormat) if err != nil { - fmt.Println("Error registering databse:", err) - return + fmt.Println("Error registering databse:", err) + return } // Dump database to file - err = dumper.Dump() + resultFilename, err := dumper.Dump() if err != nil { - fmt.Println("Error dumping:", err) - return + fmt.Println("Error dumping:", err) + return } + fmt.Printf("File is saved to %s", resultFilename) - // Close dumper and connected database + // Close dumper, connected database and file stream. dumper.Close() } */ diff --git a/dump.go b/dump.go index 8239434..5584d53 100644 --- a/dump.go +++ b/dump.go @@ -12,13 +12,14 @@ import ( "time" ) -type table struct { - Name string - SQL string - Values string -} +/*Data struct to configure dump behavior + + * Out: Stream to wite to + + * Connection: Database connection to dump -// Data struct to configure dump behavior + * IgnoreTables: Mark sensitive tables to ignore + */ type Data struct { Out io.Writer Connection *sql.DB @@ -31,6 +32,12 @@ type Data struct { wg sync.WaitGroup } +type table struct { + Name string + SQL string + Values string +} + type metaData struct { DumpVersion string ServerVersion string diff --git a/mysqldump.go b/mysqldump.go index 7d7ce5e..23ae7cb 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -53,7 +53,7 @@ func Dump(db *sql.DB, out io.Writer) error { } // Close the dumper. -// Will also close the database the dumper is connected to. +// Will also close the database the dumper is connected to as well as the out stream if it is a *os.File. // // Not required. func (d *Data) Close() error { From 36aed1dc7d33e27e72fa8ee9e939f9ca1b524768 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Fri, 25 Jan 2019 09:59:28 -0600 Subject: [PATCH 18/59] Remove vscode artifacts --- .vscode/settings.json | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 7a73a41..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,2 +0,0 @@ -{ -} \ No newline at end of file From 219da405821cc4792766f66f7c144e49d64369f8 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Fri, 25 Jan 2019 10:02:34 -0600 Subject: [PATCH 19/59] 0.3.3 release for ignoring tables --- dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump.go b/dump.go index 5584d53..b19ebbe 100644 --- a/dump.go +++ b/dump.go @@ -44,7 +44,7 @@ type metaData struct { CompleteTime string } -const version = "0.3.2" +const version = "0.3.3" const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }} -- From 53d026e9cc4e10ae4adb102b80056dc36ebd6593 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Fri, 25 Jan 2019 10:11:13 -0600 Subject: [PATCH 20/59] Update docs --- dump.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/dump.go b/dump.go index b19ebbe..69b4462 100644 --- a/dump.go +++ b/dump.go @@ -12,14 +12,13 @@ import ( "time" ) -/*Data struct to configure dump behavior +/* +Data struct to configure dump behavior - * Out: Stream to wite to - - * Connection: Database connection to dump - - * IgnoreTables: Mark sensitive tables to ignore - */ + Out: Stream to wite to + Connection: Database connection to dump + IgnoreTables: Mark sensitive tables to ignore +*/ type Data struct { Out io.Writer Connection *sql.DB From 5671c8039c161e88bc020ed77e7624e1e66257d5 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Fri, 25 Jan 2019 10:15:00 -0600 Subject: [PATCH 21/59] Update urls --- README.md | 2 +- doc.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ff6518a..d80fe30 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ import ( "database/sql" "fmt" - "github.com/JamesStewy/go-mysqldump" + "github.com/jamf/go-mysqldump" "github.com/go-sql-driver/mysql" ) diff --git a/doc.go b/doc.go index 2bff1e7..f19caf2 100644 --- a/doc.go +++ b/doc.go @@ -3,7 +3,7 @@ Create MYSQL dumps in Go without the 'mysqldump' CLI as a dependancy. Example -This example uses the mymysql driver (example 7 https://github.com/ziutek/mymysql) to connect to a mysql instance. +This example uses the mysql driver (https://github.com/go-sql-driver/mysql) to connect to a mysql instance. package main @@ -11,7 +11,7 @@ This example uses the mymysql driver (example 7 https://github.com/ziutek/mymysq "database/sql" "fmt" - "github.com/JamesStewy/go-mysqldump" + "github.com/jamf/go-mysqldump" "github.com/go-sql-driver/mysql" ) From ac1de70a9a399f1bad49956bd96504e473ba2589 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Fri, 25 Jan 2019 12:49:08 -0600 Subject: [PATCH 22/59] Missed an upcase --- dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump.go b/dump.go index 69b4462..394b68b 100644 --- a/dump.go +++ b/dump.go @@ -305,7 +305,7 @@ func (data *Data) createTableValues(name string) (string, error) { for key, value := range values { if value == nil { - dataStrings[key] = "null" + dataStrings[key] = "NULL" } else if s, ok := value.(*sql.NullString); ok { if s.Valid { dataStrings[key] = "'" + sanitize(s.String) + "'" From 530393caf485f208137b31d1bcdf2cba96714af1 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Mon, 28 Jan 2019 08:44:29 -0600 Subject: [PATCH 23/59] Collect error from goroutine --- dump.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dump.go b/dump.go index 69b4462..09beca6 100644 --- a/dump.go +++ b/dump.go @@ -29,6 +29,7 @@ type Data struct { footerTmpl *template.Template mux sync.Mutex wg sync.WaitGroup + err error } type table struct { @@ -132,6 +133,9 @@ func (data *Data) Dump() error { } } data.wg.Wait() + if data.err != nil { + return data.err + } // Set complete time meta.CompleteTime = time.Now().String() @@ -152,12 +156,14 @@ func (data *Data) dumpTable(name string) error { return nil } -func (data *Data) writeTable(table *table) error { +func (data *Data) writeTable(table *table) { data.mux.Lock() - err := data.tableTmpl.Execute(data.Out, table) + if err := data.tableTmpl.Execute(data.Out, table); err != nil && data.err == nil { + data.err = err + } data.mux.Unlock() data.wg.Done() - return err + return } // MARK: get methods From 06c8b9cea55cbc5b4ef2347f6bf6a15a80b1894d Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Mon, 28 Jan 2019 09:48:58 -0600 Subject: [PATCH 24/59] Clean up comments --- dump.go | 15 ++------------- sanitize.go | 16 ++++++---------- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/dump.go b/dump.go index 09beca6..d54468c 100644 --- a/dump.go +++ b/dump.go @@ -106,7 +106,6 @@ func (data *Data) Dump() error { DumpVersion: version, } - // Get server version if err := meta.updateServerVersion(data.Connection); err != nil { return err } @@ -119,13 +118,11 @@ func (data *Data) Dump() error { return err } - // Get tables tables, err := data.getTables() if err != nil { return err } - // Get sql for each table data.wg.Add(len(tables)) for _, name := range tables { if err := data.dumpTable(name); err != nil { @@ -137,7 +134,6 @@ func (data *Data) Dump() error { return data.err } - // Set complete time meta.CompleteTime = time.Now().String() return data.footerTmpl.Execute(data.Out, meta) } @@ -168,8 +164,8 @@ func (data *Data) writeTable(table *table) { // MARK: get methods +// getTemplates initilaizes the templates on data from the constants in this file func (data *Data) getTemplates() (err error) { - // Write dump to file data.headerTmpl, err = template.New("mysqldumpHeader").Parse(headerTmpl) if err != nil { return @@ -190,14 +186,12 @@ func (data *Data) getTemplates() (err error) { func (data *Data) getTables() ([]string, error) { tables := make([]string, 0) - // Get table list rows, err := data.Connection.Query("SHOW TABLES") if err != nil { return tables, err } defer rows.Close() - // Read result for rows.Next() { var table sql.NullString if err := rows.Scan(&table); err != nil { @@ -244,7 +238,6 @@ func (data *Data) createTable(name string) (*table, error) { } func (data *Data) createTableSQL(name string) (string, error) { - // Get table creation SQL var tableReturn, tableSQL sql.NullString err := data.Connection.QueryRow("SHOW CREATE TABLE "+name).Scan(&tableReturn, &tableSQL) @@ -259,14 +252,12 @@ func (data *Data) createTableSQL(name string) (string, error) { } func (data *Data) createTableValues(name string) (string, error) { - // Get Data rows, err := data.Connection.Query("SELECT * FROM " + name) if err != nil { return "", err } defer rows.Close() - // Get columns columns, err := rows.Columns() if err != nil { return "", err @@ -275,7 +266,6 @@ func (data *Data) createTableValues(name string) (string, error) { return "", errors.New("No columns in table " + name + ".") } - // Read data dataText := make([]string, 0) tt, err := rows.ColumnTypes() if err != nil { @@ -302,7 +292,6 @@ func (data *Data) createTableValues(name string) (string, error) { values[i] = reflect.New(types[i]).Interface() } for rows.Next() { - // Read data if err := rows.Scan(values...); err != nil { return "", err } @@ -311,7 +300,7 @@ func (data *Data) createTableValues(name string) (string, error) { for key, value := range values { if value == nil { - dataStrings[key] = "null" + dataStrings[key] = "NULL" } else if s, ok := value.(*sql.NullString); ok { if s.Valid { dataStrings[key] = "'" + sanitize(s.String) + "'" diff --git a/sanitize.go b/sanitize.go index 5966451..1f8fca3 100644 --- a/sanitize.go +++ b/sanitize.go @@ -4,7 +4,10 @@ import "strings" var lazyMySQLReplacer *strings.Replacer -func mysqlReplacer() *strings.Replacer { +// sanitize MySQL based on +// https://dev.mysql.com/doc/refman/8.0/en/string-literals.html table 9.1 +// needs to be placed in either a single or a double quoted string +func sanitize(input string) string { if lazyMySQLReplacer == nil { lazyMySQLReplacer = strings.NewReplacer( "\x00", "\\0", @@ -13,19 +16,12 @@ func mysqlReplacer() *strings.Replacer { "\b", "\\b", "\n", "\\n", "\r", "\\r", - // "\t", "\\t", + // "\t", "\\t", Tab literals are acceptable in reads "\x1A", "\\Z", // ASCII 26 == x1A "\\", "\\\\", // "%", "\\%", // "_", "\\_", ) } - return lazyMySQLReplacer -} - -// MySQL sanitizes mysql based on -// https://dev.mysql.com/doc/refman/8.0/en/string-literals.html table 9.1 -// needs to be placed in either a single or a double quoted string -func sanitize(input string) string { - return mysqlReplacer().Replace(input) + return lazyMySQLReplacer.Replace(input) } From c87ced309a02b65e327ea2349db70e1475104dbd Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Mon, 28 Jan 2019 11:26:33 -0600 Subject: [PATCH 25/59] io.Closer will close any closable connection --- dump.go | 2 +- mysqldump.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dump.go b/dump.go index d54468c..cefd833 100644 --- a/dump.go +++ b/dump.go @@ -44,7 +44,7 @@ type metaData struct { CompleteTime string } -const version = "0.3.3" +const version = "0.3.4" const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }} -- diff --git a/mysqldump.go b/mysqldump.go index 23ae7cb..b77d1be 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -53,7 +53,7 @@ func Dump(db *sql.DB, out io.Writer) error { } // Close the dumper. -// Will also close the database the dumper is connected to as well as the out stream if it is a *os.File. +// Will also close the database the dumper is connected to as well as the out stream if it has a Close method. // // Not required. func (d *Data) Close() error { @@ -61,8 +61,8 @@ func (d *Data) Close() error { d.Connection = nil d.Out = nil }() - if file, ok := d.Out.(*os.File); ok { - file.Close() + if out, ok := d.Out.(io.Closer); ok { + out.Close() } return d.Connection.Close() } From 87c70ac43c9786a3178ca6b185b9e8a719c9683d Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Tue, 29 Jan 2019 08:52:56 -0600 Subject: [PATCH 26/59] Data error to return out of execute --- dump.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dump.go b/dump.go index cefd833..af22e27 100644 --- a/dump.go +++ b/dump.go @@ -143,6 +143,9 @@ func (data *Data) Dump() error { // MARK: writter methods func (data *Data) dumpTable(name string) error { + if data.err != nil { + return data.err + } table, err := data.createTable(name) if err != nil { return err @@ -154,7 +157,7 @@ func (data *Data) dumpTable(name string) error { func (data *Data) writeTable(table *table) { data.mux.Lock() - if err := data.tableTmpl.Execute(data.Out, table); err != nil && data.err == nil { + if err := data.tableTmpl.Execute(data.Out, table); err != nil { data.err = err } data.mux.Unlock() From 03b8ebdc514a62907f227b7d152ee7dfe419c322 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Wed, 30 Jan 2019 08:28:45 -0600 Subject: [PATCH 27/59] Lock on err --- dump.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dump.go b/dump.go index af22e27..83fbdef 100644 --- a/dump.go +++ b/dump.go @@ -156,12 +156,18 @@ func (data *Data) dumpTable(name string) error { } func (data *Data) writeTable(table *table) { + // Keep a counter of how many tables have been written + defer data.wg.Done() + + // Force this method into serial data.mux.Lock() - if err := data.tableTmpl.Execute(data.Out, table); err != nil { + defer data.mux.Unlock() + + if data.err != nil { + return + } else if err := data.tableTmpl.Execute(data.Out, table); err != nil { data.err = err } - data.mux.Unlock() - data.wg.Done() return } From d5d8630d36cc7461251fb9ffc8585e5293f78821 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 31 Jan 2019 14:41:08 -0600 Subject: [PATCH 28/59] Update readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d80fe30..764c285 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ import ( "database/sql" "fmt" - "github.com/jamf/go-mysqldump" + "github.com/JamesStewy/go-mysqldump" "github.com/go-sql-driver/mysql" ) @@ -51,5 +51,5 @@ func main() { } ``` -[![GoDoc](https://godoc.org/github.com/jamf/go-mysqldump?status.svg)](https://godoc.org/github.com/jamf/go-mysqldump) -[![Build Status](https://travis-ci.org/jamf/go-mysqldump.svg?branch=master)](https://travis-ci.org/jamf/go-mysqldump) +[![GoDoc](https://godoc.org/github.com/JamesStewy/go-mysqldump?status.svg)](https://godoc.org/github.com/JamesStewy/go-mysqldump) +[![Build Status](https://travis-ci.org/JamesStewy/go-mysqldump.svg?branch=master)](https://travis-ci.org/JamesStewy/go-mysqldump) From 123b8ce985459aae9f2d50e79b48234f429aaf2e Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Fri, 1 Feb 2019 14:35:48 -0600 Subject: [PATCH 29/59] Sanitize query from createTable --- dump.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dump.go b/dump.go index 83fbdef..1fc9343 100644 --- a/dump.go +++ b/dump.go @@ -233,13 +233,13 @@ func (data *metaData) updateServerVersion(db *sql.DB) (err error) { func (data *Data) createTable(name string) (*table, error) { var err error - t := &table{Name: "`" + name + "`"} + t := &table{Name: fmt.Sprintf("`%s`", name)} - if t.SQL, err = data.createTableSQL(name); err != nil { + if t.SQL, err = data.createTableSQL(t.Name); err != nil { return nil, err } - if t.Values, err = data.createTableValues(name); err != nil { + if t.Values, err = data.createTableValues(t.Name); err != nil { return nil, err } From d22e0682dbd82389897249d38f19af71582e706e Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Fri, 1 Feb 2019 15:00:53 -0600 Subject: [PATCH 30/59] Test pass --- dump.go | 10 +++++----- dump_test.go | 10 +++++----- mysqldump_test.go | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dump.go b/dump.go index 1fc9343..4b8469e 100644 --- a/dump.go +++ b/dump.go @@ -233,13 +233,13 @@ func (data *metaData) updateServerVersion(db *sql.DB) (err error) { func (data *Data) createTable(name string) (*table, error) { var err error - t := &table{Name: fmt.Sprintf("`%s`", name)} + t := &table{Name: "`" + name + "`"} - if t.SQL, err = data.createTableSQL(t.Name); err != nil { + if t.SQL, err = data.createTableSQL(name); err != nil { return nil, err } - if t.Values, err = data.createTableValues(t.Name); err != nil { + if t.Values, err = data.createTableValues(name); err != nil { return nil, err } @@ -248,7 +248,7 @@ func (data *Data) createTable(name string) (*table, error) { func (data *Data) createTableSQL(name string) (string, error) { var tableReturn, tableSQL sql.NullString - err := data.Connection.QueryRow("SHOW CREATE TABLE "+name).Scan(&tableReturn, &tableSQL) + err := data.Connection.QueryRow("SHOW CREATE TABLE `"+name+"`").Scan(&tableReturn, &tableSQL) if err != nil { return "", err @@ -261,7 +261,7 @@ func (data *Data) createTableSQL(name string) (string, error) { } func (data *Data) createTableValues(name string) (string, error) { - rows, err := data.Connection.Query("SELECT * FROM " + name) + rows, err := data.Connection.Query("SELECT * FROM `" + name + "`") if err != nil { return "", err } diff --git a/dump_test.go b/dump_test.go index 7efdd7d..91ff4f6 100644 --- a/dump_test.go +++ b/dump_test.go @@ -156,7 +156,7 @@ func TestCreateTableSQLOk(t *testing.T) { rows := sqlmock.NewRows([]string{"Table", "Create Table"}). AddRow("Test_Table", "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`s` char(60) DEFAULT NULL, PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1") - mock.ExpectQuery("^SHOW CREATE TABLE Test_Table$").WillReturnRows(rows) + mock.ExpectQuery("^SHOW CREATE TABLE `Test_Table`$").WillReturnRows(rows) data := Data{ Connection: db, @@ -192,7 +192,7 @@ func TestCreateTableValuesOk(t *testing.T) { AddRow(1, "test@test.de", "Test Name 1"). AddRow(2, "test2@test.de", "Test Name 2") - mock.ExpectQuery("^SELECT (.+) FROM test$").WillReturnRows(rows) + mock.ExpectQuery("^SELECT (.+) FROM `test`$").WillReturnRows(rows) data := Data{ Connection: db, @@ -228,7 +228,7 @@ func TestCreateTableValuesNil(t *testing.T) { AddRow(2, "test2@test.de", "Test Name 2"). AddRow(3, "", "Test Name 3") - mock.ExpectQuery("^SELECT (.+) FROM test$").WillReturnRows(rows) + mock.ExpectQuery("^SELECT (.+) FROM `test`$").WillReturnRows(rows) data := Data{ Connection: db, @@ -266,8 +266,8 @@ func TestCreateTableOk(t *testing.T) { AddRow(1, nil, "Test Name 1"). AddRow(2, "test2@test.de", "Test Name 2") - mock.ExpectQuery("^SHOW CREATE TABLE Test_Table$").WillReturnRows(createTableRows) - mock.ExpectQuery("^SELECT (.+) FROM Test_Table$").WillReturnRows(createTableValueRows) + mock.ExpectQuery("^SHOW CREATE TABLE `Test_Table`$").WillReturnRows(createTableRows) + mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows) data := Data{ Connection: db, diff --git a/mysqldump_test.go b/mysqldump_test.go index 389c9ae..0270255 100644 --- a/mysqldump_test.go +++ b/mysqldump_test.go @@ -37,8 +37,8 @@ func TestDumpOk(t *testing.T) { mock.ExpectQuery("^SELECT version()").WillReturnRows(serverVersionRows) mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(showTablesRows) - mock.ExpectQuery("^SHOW CREATE TABLE Test_Table$").WillReturnRows(createTableRows) - mock.ExpectQuery("^SELECT (.+) FROM Test_Table$").WillReturnRows(createTableValueRows) + mock.ExpectQuery("^SHOW CREATE TABLE `Test_Table`$").WillReturnRows(createTableRows) + mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows) buf := new(bytes.Buffer) err = Dump(db, buf) From c41f2ea23f1cff1c920ca8b5631a6a5443c57e4e Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Sat, 9 Feb 2019 16:45:08 -0600 Subject: [PATCH 31/59] File replace only works with the file --- doc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc.go b/doc.go index f19caf2..187caa1 100644 --- a/doc.go +++ b/doc.go @@ -11,7 +11,7 @@ This example uses the mysql driver (https://github.com/go-sql-driver/mysql) to c "database/sql" "fmt" - "github.com/jamf/go-mysqldump" + "github.com/JamesStewy/go-mysqldump" "github.com/go-sql-driver/mysql" ) From 0937960ca350989d3537493a6d2a9ac6ebb0ca79 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Mon, 18 Mar 2019 08:42:35 -0500 Subject: [PATCH 32/59] Switch statment when reading --- dump.go | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/dump.go b/dump.go index 4b8469e..fbfab1b 100644 --- a/dump.go +++ b/dump.go @@ -100,6 +100,8 @@ const footerTmpl = `/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; -- Dump completed on {{ .CompleteTime }} ` +const nullType = "NULL" + // Dump data using struct func (data *Data) Dump() error { meta := metaData{ @@ -309,27 +311,30 @@ func (data *Data) createTableValues(name string) (string, error) { for key, value := range values { if value == nil { - dataStrings[key] = "NULL" - } else if s, ok := value.(*sql.NullString); ok { - if s.Valid { - dataStrings[key] = "'" + sanitize(s.String) + "'" - } else { - dataStrings[key] = "NULL" - } - } else if s, ok := value.(*sql.NullInt64); ok { - if s.Valid { - dataStrings[key] = fmt.Sprintf("%d", s.Int64) - } else { - dataStrings[key] = "NULL" - } - } else if s, ok := value.(*sql.RawBytes); ok { - if len(*s) == 0 { - dataStrings[key] = "NULL" - } else { - dataStrings[key] = "_binary '" + sanitize(string(*s)) + "'" - } + dataStrings[key] = nullType } else { - dataStrings[key] = fmt.Sprint("'", value, "'") + switch s := value.(type) { + case *sql.NullString: + if s.Valid { + dataStrings[key] = "'" + sanitize(s.String) + "'" + } else { + dataStrings[key] = nullType + } + case *sql.NullInt64: + if s.Valid { + dataStrings[key] = fmt.Sprintf("%d", s.Int64) + } else { + dataStrings[key] = nullType + } + case *sql.RawBytes: + if len(*s) == 0 { + dataStrings[key] = nullType + } else { + dataStrings[key] = "_binary '" + sanitize(string(*s)) + "'" + } + default: + dataStrings[key] = fmt.Sprint("'", value, "'") + } } } From 9823f56968b29d8d2dcebdfb079ae5075538a5da Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Mon, 18 Mar 2019 09:21:02 -0500 Subject: [PATCH 33/59] All in the templates --- dump.go | 21 ++++++++++++--------- dump_test.go | 2 +- mysqldump_test.go | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/dump.go b/dump.go index fbfab1b..35a10d8 100644 --- a/dump.go +++ b/dump.go @@ -35,7 +35,7 @@ type Data struct { type table struct { Name string SQL string - Values string + Values []string } type metaData struct { @@ -81,7 +81,10 @@ DROP TABLE IF EXISTS {{ .Name }}; LOCK TABLES {{ .Name }} WRITE; /*!40000 ALTER TABLE {{ .Name }} DISABLE KEYS */; {{- if .Values }} -INSERT INTO {{ .Name }} VALUES {{ .Values }}; +INSERT INTO {{ .Name }} VALUES +{{- range $index, $element := .Values -}} +{{- if $index }},{{ else }} {{ end -}}{{ $element }} +{{- end -}}; {{- end }} /*!40000 ALTER TABLE {{ .Name }} ENABLE KEYS */; UNLOCK TABLES; @@ -262,25 +265,25 @@ func (data *Data) createTableSQL(name string) (string, error) { return tableSQL.String, nil } -func (data *Data) createTableValues(name string) (string, error) { +func (data *Data) createTableValues(name string) ([]string, error) { rows, err := data.Connection.Query("SELECT * FROM `" + name + "`") if err != nil { - return "", err + return nil, err } defer rows.Close() columns, err := rows.Columns() if err != nil { - return "", err + return nil, err } if len(columns) == 0 { - return "", errors.New("No columns in table " + name + ".") + return nil, errors.New("No columns in table " + name + ".") } dataText := make([]string, 0) tt, err := rows.ColumnTypes() if err != nil { - return "", err + return nil, err } types := make([]reflect.Type, len(tt)) @@ -304,7 +307,7 @@ func (data *Data) createTableValues(name string) (string, error) { } for rows.Next() { if err := rows.Scan(values...); err != nil { - return "", err + return dataText, err } dataStrings := make([]string, len(columns)) @@ -341,5 +344,5 @@ func (data *Data) createTableValues(name string) (string, error) { dataText = append(dataText, "("+strings.Join(dataStrings, ",")+")") } - return strings.Join(dataText, ","), rows.Err() + return dataText, rows.Err() } diff --git a/dump_test.go b/dump_test.go index 91ff4f6..1b9b09d 100644 --- a/dump_test.go +++ b/dump_test.go @@ -286,7 +286,7 @@ func TestCreateTableOk(t *testing.T) { expectedResult := &table{ Name: "`Test_Table`", SQL: "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`s` char(60) DEFAULT NULL, PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1", - Values: "('1',NULL,'Test Name 1'),('2','test2@test.de','Test Name 2')", + Values: []string{"('1',NULL,'Test Name 1')", "('2','test2@test.de','Test Name 2')"}, } if !reflect.DeepEqual(result, expectedResult) { diff --git a/mysqldump_test.go b/mysqldump_test.go index 0270255..eed34f3 100644 --- a/mysqldump_test.go +++ b/mysqldump_test.go @@ -96,6 +96,6 @@ UNLOCK TABLES; ` if !reflect.DeepEqual(result, expected) { - t.Fatalf("expected %#v, got %#v", expected, result) + t.Fatalf("expected \n%#v, got \n%#v", expected, result) } } From 63aca4f8a34f9b8c1b43d4bf1c1a0a4e3b3b1d76 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Mon, 18 Mar 2019 09:29:34 -0500 Subject: [PATCH 34/59] Fix test type --- dump_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dump_test.go b/dump_test.go index 1b9b09d..b4441ac 100644 --- a/dump_test.go +++ b/dump_test.go @@ -208,7 +208,7 @@ func TestCreateTableValuesOk(t *testing.T) { t.Errorf("there were unfulfilled expections: %s", err) } - expectedResult := "('1','test@test.de','Test Name 1'),('2','test2@test.de','Test Name 2')" + expectedResult := []string{"('1','test@test.de','Test Name 1')", "('2','test2@test.de','Test Name 2')"} if !reflect.DeepEqual(result, expectedResult) { t.Fatalf("expected %#v, got %#v", expectedResult, result) @@ -244,7 +244,7 @@ func TestCreateTableValuesNil(t *testing.T) { t.Errorf("there were unfulfilled expections: %s", err) } - expectedResult := "('1',NULL,'Test Name 1'),('2','test2@test.de','Test Name 2'),('3','','Test Name 3')" + expectedResult := []string{"('1',NULL,'Test Name 1')", "('2','test2@test.de','Test Name 2')", "('3','','Test Name 3')"} if !reflect.DeepEqual(result, expectedResult) { t.Fatalf("expected %#v, got %#v", expectedResult, result) From 6cb5948e03725b434e8704f61538509af16f3a0d Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Mon, 18 Mar 2019 16:03:10 -0500 Subject: [PATCH 35/59] Version bump --- dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump.go b/dump.go index 35a10d8..44ab13e 100644 --- a/dump.go +++ b/dump.go @@ -44,7 +44,7 @@ type metaData struct { CompleteTime string } -const version = "0.3.4" +const version = "0.3.5" const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }} -- From 34666eb1f2f351c2cc4be4be0e91affbd95d89c3 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 28 Mar 2019 17:13:49 -0500 Subject: [PATCH 36/59] WIP 1 --- dump.go | 160 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 90 insertions(+), 70 deletions(-) diff --git a/dump.go b/dump.go index 44ab13e..80186ce 100644 --- a/dump.go +++ b/dump.go @@ -33,9 +33,13 @@ type Data struct { } type table struct { - Name string - SQL string - Values []string + Name string + Err error + + data *Data + rows *sql.Rows + types []reflect.Type + values []interface{} } type metaData struct { @@ -68,22 +72,22 @@ const tableTmpl = ` -- Table structure for table {{ .Name }} -- -DROP TABLE IF EXISTS {{ .Name }}; +DROP TABLE IF EXISTS {{ .NameEsc }}; /*!40101 SET @saved_cs_client = @@character_set_client */; SET character_set_client = utf8mb4 ; -{{ .SQL }}; +{{ .CreateSQL }}; /*!40101 SET character_set_client = @saved_cs_client */; -- -- Dumping data for table {{ .Name }} -- -LOCK TABLES {{ .Name }} WRITE; +LOCK TABLES {{ .NameEsc }} WRITE; /*!40000 ALTER TABLE {{ .Name }} DISABLE KEYS */; {{- if .Values }} INSERT INTO {{ .Name }} VALUES -{{- range $index, $element := .Values -}} -{{- if $index }},{{ else }} {{ end -}}{{ $element }} +{{- range .Next -}} +, {{ end -}}{{ .RowValues }} {{- end -}}; {{- end }} /*!40000 ALTER TABLE {{ .Name }} ENABLE KEYS */; @@ -237,112 +241,128 @@ func (data *metaData) updateServerVersion(db *sql.DB) (err error) { // MARK: create methods func (data *Data) createTable(name string) (*table, error) { - var err error - t := &table{Name: "`" + name + "`"} - - if t.SQL, err = data.createTableSQL(name); err != nil { - return nil, err - } - - if t.Values, err = data.createTableValues(name); err != nil { - return nil, err + t := &table{ + Name: name, + data: data, } return t, nil } -func (data *Data) createTableSQL(name string) (string, error) { +func (table *table) NameEsc() string { + return "`" + table.Name + "`" +} + +func (table *table) CreateSQL() (string, error) { var tableReturn, tableSQL sql.NullString - err := data.Connection.QueryRow("SHOW CREATE TABLE `"+name+"`").Scan(&tableReturn, &tableSQL) + err := table.data.Connection.QueryRow("SHOW CREATE TABLE "+table.NameEsc()).Scan(&tableReturn, &tableSQL) if err != nil { return "", err } - if tableReturn.String != name { + if tableReturn.String != table.Name { return "", errors.New("Returned table is not the same as requested table") } return tableSQL.String, nil } -func (data *Data) createTableValues(name string) ([]string, error) { - rows, err := data.Connection.Query("SELECT * FROM `" + name + "`") +// defer rows.Close() +func (table *table) Init() (err error) { + if len(table.types) != 0 { + return errors.New("can't init twice") + } + + table.rows, err = table.data.Connection.Query("SELECT * FROM " + table.NameEsc()) if err != nil { - return nil, err + return err } - defer rows.Close() - columns, err := rows.Columns() + columns, err := table.rows.Columns() if err != nil { - return nil, err + return err } if len(columns) == 0 { - return nil, errors.New("No columns in table " + name + ".") + return errors.New("No columns in table " + table.Name + ".") } - dataText := make([]string, 0) - tt, err := rows.ColumnTypes() + tt, err := table.rows.ColumnTypes() if err != nil { - return nil, err + return err } - types := make([]reflect.Type, len(tt)) + table.types = make([]reflect.Type, len(tt)) for i, tp := range tt { st := tp.ScanType() if tp.DatabaseTypeName() == "BLOB" { - types[i] = reflect.TypeOf(sql.RawBytes{}) + table.types[i] = reflect.TypeOf(sql.RawBytes{}) } else if st != nil && (st.Kind() == reflect.Int || st.Kind() == reflect.Int8 || st.Kind() == reflect.Int16 || st.Kind() == reflect.Int32 || st.Kind() == reflect.Int64) { - types[i] = reflect.TypeOf(sql.NullInt64{}) + table.types[i] = reflect.TypeOf(sql.NullInt64{}) } else { - types[i] = reflect.TypeOf(sql.NullString{}) + table.types[i] = reflect.TypeOf(sql.NullString{}) } } - values := make([]interface{}, len(tt)) - for i := range values { - values[i] = reflect.New(types[i]).Interface() + table.values = make([]interface{}, len(tt)) + for i := range table.values { + table.values[i] = reflect.New(table.types[i]).Interface() } - for rows.Next() { - if err := rows.Scan(values...); err != nil { - return dataText, err + return nil +} + +func (table *table) Next() bool { + if table.rows == nil { + if err := table.Init(); err != nil { + table.Err = err + return false + } + } else if table.rows.Next() { + if err := table.rows.Scan(table.values...); err != nil { + table.Err = err + return false } + } else { + table.rows.Close() + table.rows = nil + return false + } + return true +} + +func (table *table) RowValues() (string, error) { + dataStrings := make([]string, len(table.values)) - dataStrings := make([]string, len(columns)) - - for key, value := range values { - if value == nil { - dataStrings[key] = nullType - } else { - switch s := value.(type) { - case *sql.NullString: - if s.Valid { - dataStrings[key] = "'" + sanitize(s.String) + "'" - } else { - dataStrings[key] = nullType - } - case *sql.NullInt64: - if s.Valid { - dataStrings[key] = fmt.Sprintf("%d", s.Int64) - } else { - dataStrings[key] = nullType - } - case *sql.RawBytes: - if len(*s) == 0 { - dataStrings[key] = nullType - } else { - dataStrings[key] = "_binary '" + sanitize(string(*s)) + "'" - } - default: - dataStrings[key] = fmt.Sprint("'", value, "'") + for key, value := range table.values { + if value == nil { + dataStrings[key] = nullType + } else { + switch s := value.(type) { + case *sql.NullString: + if s.Valid { + dataStrings[key] = "'" + sanitize(s.String) + "'" + } else { + dataStrings[key] = nullType } + case *sql.NullInt64: + if s.Valid { + dataStrings[key] = fmt.Sprintf("%d", s.Int64) + } else { + dataStrings[key] = nullType + } + case *sql.RawBytes: + if len(*s) == 0 { + dataStrings[key] = nullType + } else { + dataStrings[key] = "_binary '" + sanitize(string(*s)) + "'" + } + default: + dataStrings[key] = fmt.Sprint("'", value, "'") } } - - dataText = append(dataText, "("+strings.Join(dataStrings, ",")+")") } - return dataText, rows.Err() + return "(" + strings.Join(dataStrings, ",") + ")", table.rows.Err() } From c7cef007aaf2fd0d11817ebdc1583f9405a0bd11 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Fri, 29 Mar 2019 08:55:26 -0500 Subject: [PATCH 37/59] Major refactor --- dump.go | 10 ++-- dump_test.go | 154 +++++++++++++++++---------------------------------- 2 files changed, 56 insertions(+), 108 deletions(-) diff --git a/dump.go b/dump.go index 80186ce..5004f0a 100644 --- a/dump.go +++ b/dump.go @@ -84,10 +84,10 @@ DROP TABLE IF EXISTS {{ .NameEsc }}; LOCK TABLES {{ .NameEsc }} WRITE; /*!40000 ALTER TABLE {{ .Name }} DISABLE KEYS */; -{{- if .Values }} -INSERT INTO {{ .Name }} VALUES +{{- if .Next }} +INSERT INTO {{ .Name }} VALUES {{ .RowValues }} {{- range .Next -}} -, {{ end -}}{{ .RowValues }} +, {{ .RowValues }} {{- end -}}; {{- end }} /*!40000 ALTER TABLE {{ .Name }} ENABLE KEYS */; @@ -319,7 +319,9 @@ func (table *table) Next() bool { table.Err = err return false } - } else if table.rows.Next() { + } + // Fallthrough + if table.rows.Next() { if err := table.rows.Scan(table.values...); err != nil { table.Err = err return false diff --git a/dump_test.go b/dump_test.go index b4441ac..af0cfc3 100644 --- a/dump_test.go +++ b/dump_test.go @@ -5,14 +5,12 @@ import ( "testing" sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/assert" ) func TestGetTablesOk(t *testing.T) { db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - + assert.NoError(t, err, "an error was not expected when opening a stub database connection") defer db.Close() rows := sqlmock.NewRows([]string{"Tables_in_Testdb"}). @@ -26,28 +24,17 @@ func TestGetTablesOk(t *testing.T) { } result, err := data.getTables() - if err != nil { - t.Errorf("error was not expected while updating stats: %s", err) - } + assert.NoError(t, err) // we make sure that all expectations were met - if err := mock.ExpectationsWereMet(); err != nil { - t.Errorf("there were unfulfilled expections: %s", err) - } - - expectedResult := []string{"Test_Table_1", "Test_Table_2"} + assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections") - if !reflect.DeepEqual(result, expectedResult) { - t.Fatalf("expected %#v, got %#v", result, expectedResult) - } + assert.EqualValues(t, []string{"Test_Table_1", "Test_Table_2"}, result) } func TestIgnoreTablesOk(t *testing.T) { db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - + assert.NoError(t, err, "an error was not expected when opening a stub database connection") defer db.Close() rows := sqlmock.NewRows([]string{"Tables_in_Testdb"}). @@ -62,27 +49,17 @@ func TestIgnoreTablesOk(t *testing.T) { } result, err := data.getTables() - if err != nil { - t.Errorf("error was not expected while updating stats: %s", err) - } + assert.NoError(t, err) // we make sure that all expectations were met - if err := mock.ExpectationsWereMet(); err != nil { - t.Errorf("there were unfulfilled expections: %s", err) - } - - expectedResult := []string{"Test_Table_2"} + assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections") - if !reflect.DeepEqual(result, expectedResult) { - t.Fatalf("expected %#v, got %#v", result, expectedResult) - } + assert.EqualValues(t, []string{"Test_Table_2"}, result) } func TestGetTablesNil(t *testing.T) { db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } + assert.NoError(t, err, "an error was not expected when opening a stub database connection") defer db.Close() @@ -98,27 +75,17 @@ func TestGetTablesNil(t *testing.T) { } result, err := data.getTables() - if err != nil { - t.Errorf("error was not expected while updating stats: %s", err) - } + assert.NoError(t, err) // we make sure that all expectations were met - if err := mock.ExpectationsWereMet(); err != nil { - t.Errorf("there were unfulfilled expections: %s", err) - } - - expectedResult := []string{"Test_Table_1", "Test_Table_3"} + assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections") - if !reflect.DeepEqual(result, expectedResult) { - t.Fatalf("expected %#v, got %#v", expectedResult, result) - } + assert.EqualValues(t, []string{"Test_Table_1", "Test_Table_3"}, result) } func TestGetServerVersionOk(t *testing.T) { db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } + assert.NoError(t, err, "an error was not expected when opening a stub database connection") defer db.Close() @@ -129,28 +96,17 @@ func TestGetServerVersionOk(t *testing.T) { meta := metaData{} - if err := meta.updateServerVersion(db); err != nil { - t.Errorf("error was not expected while updating stats: %s", err) - } + assert.NoError(t, meta.updateServerVersion(db), "error was not expected while updating stats") // we make sure that all expectations were met - if err := mock.ExpectationsWereMet(); err != nil { - t.Errorf("there were unfulfilled expections: %s", err) - } + assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections") - expectedResult := "test_version" - - if !reflect.DeepEqual(meta.ServerVersion, expectedResult) { - t.Fatalf("expected %#v, got %#v", expectedResult, meta.ServerVersion) - } + assert.Equal(t, "test_version", meta.ServerVersion) } func TestCreateTableSQLOk(t *testing.T) { db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - + assert.NoError(t, err, "an error was not expected when opening a stub database connection") defer db.Close() rows := sqlmock.NewRows([]string{"Table", "Create Table"}). @@ -162,16 +118,14 @@ func TestCreateTableSQLOk(t *testing.T) { Connection: db, } - result, err := data.createTableSQL("Test_Table") + table, err := data.createTable("Test_Table") + assert.NoError(t, err) - if err != nil { - t.Errorf("error was not expected while updating stats: %s", err) - } + result, err := table.CreateSQL() + assert.NoError(t, err) // we make sure that all expectations were met - if err := mock.ExpectationsWereMet(); err != nil { - t.Errorf("there were unfulfilled expections: %s", err) - } + assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections") expectedResult := "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`s` char(60) DEFAULT NULL, PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1" @@ -180,12 +134,9 @@ func TestCreateTableSQLOk(t *testing.T) { } } -func TestCreateTableValuesOk(t *testing.T) { +func TestCreateTableRowValues(t *testing.T) { db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - + assert.NoError(t, err, "an error was not expected when opening a stub database connection") defer db.Close() rows := sqlmock.NewRows([]string{"id", "email", "name"}). @@ -198,29 +149,23 @@ func TestCreateTableValuesOk(t *testing.T) { Connection: db, } - result, err := data.createTableValues("test") - if err != nil { - t.Errorf("error was not expected while updating stats: %s", err) - } + table, err := data.createTable("test") + assert.NoError(t, err) - // we make sure that all expectations were met - if err := mock.ExpectationsWereMet(); err != nil { - t.Errorf("there were unfulfilled expections: %s", err) - } + assert.True(t, table.Next()) + // TODO + result, err := table.RowValues() + assert.NoError(t, err) - expectedResult := []string{"('1','test@test.de','Test Name 1')", "('2','test2@test.de','Test Name 2')"} + // we make sure that all expectations were met + assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections") - if !reflect.DeepEqual(result, expectedResult) { - t.Fatalf("expected %#v, got %#v", expectedResult, result) - } + assert.EqualValues(t, "('1','test@test.de','Test Name 1')", result) } -func TestCreateTableValuesNil(t *testing.T) { +func TestCreateTableAllValuesWithNil(t *testing.T) { db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - + assert.NoError(t, err, "an error was not expected when opening a stub database connection") defer db.Close() rows := sqlmock.NewRows([]string{"id", "email", "name"}). @@ -234,21 +179,22 @@ func TestCreateTableValuesNil(t *testing.T) { Connection: db, } - result, err := data.createTableValues("test") - if err != nil { - t.Errorf("error was not expected while updating stats: %s", err) + table, err := data.createTable("test") + assert.NoError(t, err) + + results := make([]string, 0) + for table.Next() { + row, err := table.RowValues() + assert.NoError(t, err) + results = append(results, row) } // we make sure that all expectations were met - if err := mock.ExpectationsWereMet(); err != nil { - t.Errorf("there were unfulfilled expections: %s", err) - } + assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections") - expectedResult := []string{"('1',NULL,'Test Name 1')", "('2','test2@test.de','Test Name 2')", "('3','','Test Name 3')"} + expectedResults := []string{"('1',NULL,'Test Name 1')", "('2','test2@test.de','Test Name 2')", "('3','','Test Name 3')"} - if !reflect.DeepEqual(result, expectedResult) { - t.Fatalf("expected %#v, got %#v", expectedResult, result) - } + assert.EqualValues(t, expectedResults, results) } func TestCreateTableOk(t *testing.T) { @@ -284,9 +230,9 @@ func TestCreateTableOk(t *testing.T) { } expectedResult := &table{ - Name: "`Test_Table`", - SQL: "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`s` char(60) DEFAULT NULL, PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1", - Values: []string{"('1',NULL,'Test Name 1')", "('2','test2@test.de','Test Name 2')"}, + Name: "`Test_Table`", + // SQL: "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`s` char(60) DEFAULT NULL, PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1", + // Values: []string{"('1',NULL,'Test Name 1')", "('2','test2@test.de','Test Name 2')"}, } if !reflect.DeepEqual(result, expectedResult) { From bb5c4f926eee933aa8c72a1c66e1bdf4514846b2 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Fri, 29 Mar 2019 13:33:18 -0500 Subject: [PATCH 38/59] Data streams --- dump.go | 106 ++++++++++++++++++++++------------------------ dump_test.go | 64 +++++++++++++++++----------- mysqldump_test.go | 35 ++++++--------- 3 files changed, 103 insertions(+), 102 deletions(-) diff --git a/dump.go b/dump.go index 5004f0a..b9fcc85 100644 --- a/dump.go +++ b/dump.go @@ -7,7 +7,6 @@ import ( "io" "reflect" "strings" - "sync" "text/template" "time" ) @@ -27,8 +26,6 @@ type Data struct { headerTmpl *template.Template tableTmpl *template.Template footerTmpl *template.Template - mux sync.Mutex - wg sync.WaitGroup err error } @@ -69,7 +66,7 @@ const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }} const tableTmpl = ` -- --- Table structure for table {{ .Name }} +-- Table structure for table {{ .NameEsc }} -- DROP TABLE IF EXISTS {{ .NameEsc }}; @@ -79,18 +76,16 @@ DROP TABLE IF EXISTS {{ .NameEsc }}; /*!40101 SET character_set_client = @saved_cs_client */; -- --- Dumping data for table {{ .Name }} +-- Dumping data for table {{ .NameEsc }} -- LOCK TABLES {{ .NameEsc }} WRITE; -/*!40000 ALTER TABLE {{ .Name }} DISABLE KEYS */; +/*!40000 ALTER TABLE {{ .NameEsc }} DISABLE KEYS */; {{- if .Next }} -INSERT INTO {{ .Name }} VALUES {{ .RowValues }} -{{- range .Next -}} -, {{ .RowValues }} -{{- end -}}; +INSERT INTO {{ .NameEsc }} VALUES {{ .RowValues }} +{{- range $value := .Stream }},{{ $value }}{{ end -}}; {{- end }} -/*!40000 ALTER TABLE {{ .Name }} ENABLE KEYS */; +/*!40000 ALTER TABLE {{ .NameEsc }} ENABLE KEYS */; UNLOCK TABLES; ` @@ -132,13 +127,11 @@ func (data *Data) Dump() error { return err } - data.wg.Add(len(tables)) for _, name := range tables { if err := data.dumpTable(name); err != nil { return err } } - data.wg.Wait() if data.err != nil { return data.err } @@ -160,24 +153,14 @@ func (data *Data) dumpTable(name string) error { return err } - go data.writeTable(table) - return nil + return data.writeTable(table) } -func (data *Data) writeTable(table *table) { - // Keep a counter of how many tables have been written - defer data.wg.Done() - - // Force this method into serial - data.mux.Lock() - defer data.mux.Unlock() - - if data.err != nil { - return - } else if err := data.tableTmpl.Execute(data.Out, table); err != nil { - data.err = err +func (data *Data) writeTable(table *table) error { + if err := data.tableTmpl.Execute(data.Out, table); err != nil { + return err } - return + return table.Err } // MARK: get methods @@ -255,11 +238,10 @@ func (table *table) NameEsc() string { func (table *table) CreateSQL() (string, error) { var tableReturn, tableSQL sql.NullString - err := table.data.Connection.QueryRow("SHOW CREATE TABLE "+table.NameEsc()).Scan(&tableReturn, &tableSQL) - - if err != nil { + if err := table.data.Connection.QueryRow("SHOW CREATE TABLE "+table.NameEsc()).Scan(&tableReturn, &tableSQL); err != nil { return "", err } + if tableReturn.String != table.Name { return "", errors.New("Returned table is not the same as requested table") } @@ -325,6 +307,9 @@ func (table *table) Next() bool { if err := table.rows.Scan(table.values...); err != nil { table.Err = err return false + } else if err := table.rows.Err(); err != nil { + table.Err = err + return false } } else { table.rows.Close() @@ -334,37 +319,46 @@ func (table *table) Next() bool { return true } -func (table *table) RowValues() (string, error) { +func (table *table) RowValues() string { dataStrings := make([]string, len(table.values)) for key, value := range table.values { - if value == nil { + switch s := value.(type) { + case nil: dataStrings[key] = nullType - } else { - switch s := value.(type) { - case *sql.NullString: - if s.Valid { - dataStrings[key] = "'" + sanitize(s.String) + "'" - } else { - dataStrings[key] = nullType - } - case *sql.NullInt64: - if s.Valid { - dataStrings[key] = fmt.Sprintf("%d", s.Int64) - } else { - dataStrings[key] = nullType - } - case *sql.RawBytes: - if len(*s) == 0 { - dataStrings[key] = nullType - } else { - dataStrings[key] = "_binary '" + sanitize(string(*s)) + "'" - } - default: - dataStrings[key] = fmt.Sprint("'", value, "'") + case *sql.NullString: + if s.Valid { + dataStrings[key] = "'" + sanitize(s.String) + "'" + } else { + dataStrings[key] = nullType + } + case *sql.NullInt64: + if s.Valid { + dataStrings[key] = fmt.Sprintf("%d", s.Int64) + } else { + dataStrings[key] = nullType } + case *sql.RawBytes: + if len(*s) == 0 { + dataStrings[key] = nullType + } else { + dataStrings[key] = "_binary '" + sanitize(string(*s)) + "'" + } + default: + dataStrings[key] = fmt.Sprint("'", value, "'") } } - return "(" + strings.Join(dataStrings, ",") + ")", table.rows.Err() + return "(" + strings.Join(dataStrings, ",") + ")" +} + +func (table *table) Stream() <-chan string { + valueOut := make(chan string, 1) + go func(out chan string) { + defer close(out) + for table.Next() { + out <- table.RowValues() + } + }(valueOut) + return valueOut } diff --git a/dump_test.go b/dump_test.go index af0cfc3..e40879c 100644 --- a/dump_test.go +++ b/dump_test.go @@ -1,7 +1,9 @@ package mysqldump import ( + "bytes" "reflect" + "strings" "testing" sqlmock "github.com/DATA-DOG/go-sqlmock" @@ -153,9 +155,9 @@ func TestCreateTableRowValues(t *testing.T) { assert.NoError(t, err) assert.True(t, table.Next()) - // TODO - result, err := table.RowValues() - assert.NoError(t, err) + + result := table.RowValues() + assert.NoError(t, table.Err) // we make sure that all expectations were met assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections") @@ -184,8 +186,8 @@ func TestCreateTableAllValuesWithNil(t *testing.T) { results := make([]string, 0) for table.Next() { - row, err := table.RowValues() - assert.NoError(t, err) + row := table.RowValues() + assert.NoError(t, table.Err) results = append(results, row) } @@ -199,9 +201,7 @@ func TestCreateTableAllValuesWithNil(t *testing.T) { func TestCreateTableOk(t *testing.T) { db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } + assert.NoError(t, err, "an error was not expected when opening a stub database connection") defer db.Close() @@ -215,27 +215,43 @@ func TestCreateTableOk(t *testing.T) { mock.ExpectQuery("^SHOW CREATE TABLE `Test_Table`$").WillReturnRows(createTableRows) mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows) + var buf bytes.Buffer data := Data{ Connection: db, + Out: &buf, } - result, err := data.createTable("Test_Table") - if err != nil { - t.Errorf("error was not expected while updating stats: %s", err) - } + assert.NoError(t, data.getTemplates()) - // we make sure that all expectations were met - if err := mock.ExpectationsWereMet(); err != nil { - t.Errorf("there were unfulfilled expections: %s", err) - } + table, err := data.createTable("Test_Table") + assert.NoError(t, err) - expectedResult := &table{ - Name: "`Test_Table`", - // SQL: "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`s` char(60) DEFAULT NULL, PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1", - // Values: []string{"('1',NULL,'Test Name 1')", "('2','test2@test.de','Test Name 2')"}, - } + data.writeTable(table) - if !reflect.DeepEqual(result, expectedResult) { - t.Fatalf("expected %#v, got %#v", expectedResult, result) - } + // we make sure that all expectations were met + assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections") + + expectedResult := ` +-- +-- Table structure for table ~Test_Table~ +-- + +DROP TABLE IF EXISTS ~Test_Table~; +/*!40101 SET @saved_cs_client = @@character_set_client */; + SET character_set_client = utf8mb4 ; +CREATE TABLE 'Test_Table' (~id~ int(11) NOT NULL AUTO_INCREMENT,~s~ char(60) DEFAULT NULL, PRIMARY KEY (~id~))ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table ~Test_Table~ +-- + +LOCK TABLES ~Test_Table~ WRITE; +/*!40000 ALTER TABLE ~Test_Table~ DISABLE KEYS */; +INSERT INTO ~Test_Table~ VALUES ('1',NULL,'Test Name 1'),('2','test2@test.de','Test Name 2'); +/*!40000 ALTER TABLE ~Test_Table~ ENABLE KEYS */; +UNLOCK TABLES; +` + result := strings.Replace(buf.String(), "`", "~", -1) + assert.Equal(t, expectedResult, result) } diff --git a/mysqldump_test.go b/mysqldump_test.go index eed34f3..1af8833 100644 --- a/mysqldump_test.go +++ b/mysqldump_test.go @@ -3,11 +3,11 @@ package mysqldump import ( "bytes" "os" - "reflect" "strings" "testing" sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/assert" ) func TestDumpOk(t *testing.T) { @@ -16,10 +16,7 @@ func TestDumpOk(t *testing.T) { os.Remove(tmpFile) db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - + assert.NoError(t, err, "an error was not expected when opening a stub database connection") defer db.Close() showTablesRows := sqlmock.NewRows([]string{"Tables_in_Testdb"}). @@ -41,12 +38,9 @@ func TestDumpOk(t *testing.T) { mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows) buf := new(bytes.Buffer) - err = Dump(db, buf) - if err != nil { - t.Fatalf("an error '%s' was not expected when dumping a stub database connection", err) - } + assert.NoError(t, Dump(db, buf), "an error was not expected when dumping a stub database connection") - result := strings.Replace(strings.Split(buf.String(), "-- Dump completed")[0], "`", "\\", -1) + result := strings.Replace(strings.Split(buf.String(), "-- Dump completed")[0], "`", "~", -1) expected := `-- Go SQL Dump ` + version + ` -- @@ -65,23 +59,23 @@ func TestDumpOk(t *testing.T) { /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; -- --- Table structure for table \Test_Table\ +-- Table structure for table ~Test_Table~ -- -DROP TABLE IF EXISTS \Test_Table\; +DROP TABLE IF EXISTS ~Test_Table~; /*!40101 SET @saved_cs_client = @@character_set_client */; SET character_set_client = utf8mb4 ; -CREATE TABLE 'Test_Table' (\id\ int(11) NOT NULL AUTO_INCREMENT,\email\ char(60) DEFAULT NULL, \name\ char(60), PRIMARY KEY (\id\))ENGINE=InnoDB DEFAULT CHARSET=latin1; +CREATE TABLE 'Test_Table' (~id~ int(11) NOT NULL AUTO_INCREMENT,~email~ char(60) DEFAULT NULL, ~name~ char(60), PRIMARY KEY (~id~))ENGINE=InnoDB DEFAULT CHARSET=latin1; /*!40101 SET character_set_client = @saved_cs_client */; -- --- Dumping data for table \Test_Table\ +-- Dumping data for table ~Test_Table~ -- -LOCK TABLES \Test_Table\ WRITE; -/*!40000 ALTER TABLE \Test_Table\ DISABLE KEYS */; -INSERT INTO \Test_Table\ VALUES ('1',NULL,'Test Name 1'),('2','test2@test.de','Test Name 2'); -/*!40000 ALTER TABLE \Test_Table\ ENABLE KEYS */; +LOCK TABLES ~Test_Table~ WRITE; +/*!40000 ALTER TABLE ~Test_Table~ DISABLE KEYS */; +INSERT INTO ~Test_Table~ VALUES ('1',NULL,'Test Name 1'),('2','test2@test.de','Test Name 2'); +/*!40000 ALTER TABLE ~Test_Table~ ENABLE KEYS */; UNLOCK TABLES; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; @@ -94,8 +88,5 @@ UNLOCK TABLES; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; ` - - if !reflect.DeepEqual(result, expected) { - t.Fatalf("expected \n%#v, got \n%#v", expected, result) - } + assert.Equal(t, expected, result) } From dabcd5769cebc35a0416c9990a5fd5e3896df530 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Fri, 29 Mar 2019 13:55:59 -0500 Subject: [PATCH 39/59] With data streams --- dump.go | 29 ++++++++++-------- mysqldump_test.go | 76 ++++++++++++++++++++++++++--------------------- 2 files changed, 58 insertions(+), 47 deletions(-) diff --git a/dump.go b/dump.go index b9fcc85..ff48712 100644 --- a/dump.go +++ b/dump.go @@ -47,6 +47,7 @@ type metaData struct { const version = "0.3.5" +// takes a *metaData const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }} -- -- ------------------------------------------------------ @@ -64,6 +65,21 @@ const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }} /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; ` +// takes a *metaData +const footerTmpl = `/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on {{ .CompleteTime }} +` + +// Takes a *table const tableTmpl = ` -- -- Table structure for table {{ .NameEsc }} @@ -89,19 +105,6 @@ INSERT INTO {{ .NameEsc }} VALUES {{ .RowValues }} UNLOCK TABLES; ` -const footerTmpl = `/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; - -/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; -/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; -/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; -/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; -/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; -/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; -/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; - --- Dump completed on {{ .CompleteTime }} -` - const nullType = "NULL" // Dump data using struct diff --git a/mysqldump_test.go b/mysqldump_test.go index 1af8833..40c55d3 100644 --- a/mysqldump_test.go +++ b/mysqldump_test.go @@ -2,7 +2,6 @@ package mysqldump import ( "bytes" - "os" "strings" "testing" @@ -10,39 +9,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestDumpOk(t *testing.T) { - - tmpFile := "/tmp/test_format.sql" - os.Remove(tmpFile) - - db, mock, err := sqlmock.New() - assert.NoError(t, err, "an error was not expected when opening a stub database connection") - defer db.Close() - - showTablesRows := sqlmock.NewRows([]string{"Tables_in_Testdb"}). - AddRow("Test_Table") - - serverVersionRows := sqlmock.NewRows([]string{"Version()"}). - AddRow("test_version") - - createTableRows := sqlmock.NewRows([]string{"Table", "Create Table"}). - AddRow("Test_Table", "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`email` char(60) DEFAULT NULL, `name` char(60), PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1") - - createTableValueRows := sqlmock.NewRows([]string{"id", "email", "name"}). - AddRow(1, nil, "Test Name 1"). - AddRow(2, "test2@test.de", "Test Name 2") - - mock.ExpectQuery("^SELECT version()").WillReturnRows(serverVersionRows) - mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(showTablesRows) - mock.ExpectQuery("^SHOW CREATE TABLE `Test_Table`$").WillReturnRows(createTableRows) - mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows) - - buf := new(bytes.Buffer) - assert.NoError(t, Dump(db, buf), "an error was not expected when dumping a stub database connection") - - result := strings.Replace(strings.Split(buf.String(), "-- Dump completed")[0], "`", "~", -1) - - expected := `-- Go SQL Dump ` + version + ` +const expected = `-- Go SQL Dump ` + version + ` -- -- ------------------------------------------------------ -- Server version test_version @@ -88,5 +55,46 @@ UNLOCK TABLES; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; ` + +func RunDump(t testing.TB) string { + db, mock, err := sqlmock.New() + assert.NoError(t, err, "an error was not expected when opening a stub database connection") + defer db.Close() + + showTablesRows := sqlmock.NewRows([]string{"Tables_in_Testdb"}). + AddRow("Test_Table") + + serverVersionRows := sqlmock.NewRows([]string{"Version()"}). + AddRow("test_version") + + createTableRows := sqlmock.NewRows([]string{"Table", "Create Table"}). + AddRow("Test_Table", "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`email` char(60) DEFAULT NULL, `name` char(60), PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1") + + createTableValueRows := sqlmock.NewRows([]string{"id", "email", "name"}). + AddRow(1, nil, "Test Name 1"). + AddRow(2, "test2@test.de", "Test Name 2") + + mock.ExpectQuery("^SELECT version()").WillReturnRows(serverVersionRows) + mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(showTablesRows) + mock.ExpectQuery("^SHOW CREATE TABLE `Test_Table`$").WillReturnRows(createTableRows) + mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows) + + var buf bytes.Buffer + assert.NoError(t, Dump(db, &buf), "an error was not expected when dumping a stub database connection") + + return buf.String() +} + +func TestDumpOk(t *testing.T) { + out := RunDump(t) + + result := strings.Replace(strings.Split(out, "-- Dump completed")[0], "`", "~", -1) + assert.Equal(t, expected, result) } + +func BenchmarkDump(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = RunDump(b) + } +} From db8611786a7156f181fc26b3e8404280e2d58985 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Fri, 29 Mar 2019 15:19:52 -0500 Subject: [PATCH 40/59] Bump the version --- dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump.go b/dump.go index ff48712..af7e303 100644 --- a/dump.go +++ b/dump.go @@ -45,7 +45,7 @@ type metaData struct { CompleteTime string } -const version = "0.3.5" +const version = "0.4.0" // takes a *metaData const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }} From 70aca041a5e80289c21e57c96929c8a4d2e3afd7 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 4 Apr 2019 11:55:42 -0500 Subject: [PATCH 41/59] inset spliting --- dump.go | 92 +++++++++++++++++++++++++++++----------- dump_test.go | 117 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 183 insertions(+), 26 deletions(-) diff --git a/dump.go b/dump.go index af7e303..fd4ee89 100644 --- a/dump.go +++ b/dump.go @@ -1,12 +1,12 @@ package mysqldump import ( + "bytes" "database/sql" "errors" "fmt" "io" "reflect" - "strings" "text/template" "time" ) @@ -19,9 +19,10 @@ Data struct to configure dump behavior IgnoreTables: Mark sensitive tables to ignore */ type Data struct { - Out io.Writer - Connection *sql.DB - IgnoreTables []string + Out io.Writer + Connection *sql.DB + IgnoreTables []string + MaxAllowedPacket int headerTmpl *template.Template tableTmpl *template.Template @@ -45,7 +46,10 @@ type metaData struct { CompleteTime string } -const version = "0.4.0" +const ( + version = "0.4.1" + defaultMaxAllowedPacket = 4194304 +) // takes a *metaData const headerTmpl = `-- Go SQL Dump {{ .DumpVersion }} @@ -97,10 +101,9 @@ DROP TABLE IF EXISTS {{ .NameEsc }}; LOCK TABLES {{ .NameEsc }} WRITE; /*!40000 ALTER TABLE {{ .NameEsc }} DISABLE KEYS */; -{{- if .Next }} -INSERT INTO {{ .NameEsc }} VALUES {{ .RowValues }} -{{- range $value := .Stream }},{{ $value }}{{ end -}}; -{{- end }} +{{ range $value := .Stream }} +{{- $value }} +{{ end -}} /*!40000 ALTER TABLE {{ .NameEsc }} ENABLE KEYS */; UNLOCK TABLES; ` @@ -112,6 +115,7 @@ func (data *Data) Dump() error { meta := metaData{ DumpVersion: version, } + data.initMaxPacketSize() if err := meta.updateServerVersion(data.Connection); err != nil { return err @@ -187,6 +191,17 @@ func (data *Data) getTemplates() (err error) { return } +func (data *Data) initMaxPacketSize() { + if data.MaxAllowedPacket <= 0 { + data.MaxAllowedPacket = defaultMaxAllowedPacket + + var maxSize int + if err := data.Connection.QueryRow("SELECT @@global.max_allowed_packet;").Scan(&maxSize); err == nil { + data.MaxAllowedPacket = maxSize + } + } +} + func (data *Data) getTables() ([]string, error) { tables := make([]string, 0) @@ -219,7 +234,7 @@ func (data *Data) isIgnoredTable(name string) bool { func (data *metaData) updateServerVersion(db *sql.DB) (err error) { var serverVersion sql.NullString - err = db.QueryRow("SELECT version()").Scan(&serverVersion) + err = db.QueryRow("SELECT version();").Scan(&serverVersion) data.ServerVersion = serverVersion.String return } @@ -323,45 +338,74 @@ func (table *table) Next() bool { } func (table *table) RowValues() string { - dataStrings := make([]string, len(table.values)) + return table.RowBuffer().String() +} + +func (table *table) RowBuffer() *bytes.Buffer { + var b bytes.Buffer + b.WriteString("(") for key, value := range table.values { + if key != 0 { + b.WriteString(",") + } switch s := value.(type) { case nil: - dataStrings[key] = nullType + b.WriteString(nullType) case *sql.NullString: if s.Valid { - dataStrings[key] = "'" + sanitize(s.String) + "'" + fmt.Fprintf(&b, "'%s'", sanitize(s.String)) } else { - dataStrings[key] = nullType + b.WriteString(nullType) } case *sql.NullInt64: if s.Valid { - dataStrings[key] = fmt.Sprintf("%d", s.Int64) + fmt.Fprintf(&b, "%d", s.Int64) } else { - dataStrings[key] = nullType + b.WriteString(nullType) } case *sql.RawBytes: if len(*s) == 0 { - dataStrings[key] = nullType + b.WriteString(nullType) } else { - dataStrings[key] = "_binary '" + sanitize(string(*s)) + "'" + fmt.Fprintf(&b, "_binary '%s'", sanitize(string(*s))) } default: - dataStrings[key] = fmt.Sprint("'", value, "'") + fmt.Fprintf(&b, "'%s'", value) } } + b.WriteString(")") - return "(" + strings.Join(dataStrings, ",") + ")" + return &b } func (table *table) Stream() <-chan string { valueOut := make(chan string, 1) - go func(out chan string) { - defer close(out) + go func() { + defer close(valueOut) + var insert bytes.Buffer + for table.Next() { - out <- table.RowValues() + b := table.RowBuffer() + // Truncate our insert if it won't fit + if insert.Len() != 0 && insert.Len()+b.Len() > table.data.MaxAllowedPacket-1 { + insert.WriteString(";") + valueOut <- insert.String() + insert.Reset() + } + + if insert.Len() == 0 { + fmt.Fprintf(&insert, "INSERT INTO %s VALUES ", table.NameEsc()) + } else { + insert.WriteString(",") + } + b.WriteTo(&insert) + b.Reset() + } + if insert.Len() != 0 { + insert.WriteString(";") + valueOut <- insert.String() } - }(valueOut) + }() return valueOut } diff --git a/dump_test.go b/dump_test.go index e40879c..a79a31c 100644 --- a/dump_test.go +++ b/dump_test.go @@ -165,6 +165,59 @@ func TestCreateTableRowValues(t *testing.T) { assert.EqualValues(t, "('1','test@test.de','Test Name 1')", result) } +func TestCreateTableValuesSteam(t *testing.T) { + db, mock, err := sqlmock.New() + assert.NoError(t, err, "an error was not expected when opening a stub database connection") + defer db.Close() + + rows := sqlmock.NewRows([]string{"id", "email", "name"}). + AddRow(1, "test@test.de", "Test Name 1"). + AddRow(2, "test2@test.de", "Test Name 2") + + mock.ExpectQuery("^SELECT (.+) FROM `test`$").WillReturnRows(rows) + + data := Data{ + Connection: db, + MaxAllowedPacket: 4096, + } + + table, err := data.createTable("test") + assert.NoError(t, err) + + s := table.Stream() + assert.EqualValues(t, "INSERT INTO `test` VALUES ('1','test@test.de','Test Name 1'),('2','test2@test.de','Test Name 2');", <-s) + + // we make sure that all expectations were met + assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections") +} + +func TestCreateTableValuesSteamSmallPackets(t *testing.T) { + db, mock, err := sqlmock.New() + assert.NoError(t, err, "an error was not expected when opening a stub database connection") + defer db.Close() + + rows := sqlmock.NewRows([]string{"id", "email", "name"}). + AddRow(1, "test@test.de", "Test Name 1"). + AddRow(2, "test2@test.de", "Test Name 2") + + mock.ExpectQuery("^SELECT (.+) FROM `test`$").WillReturnRows(rows) + + data := Data{ + Connection: db, + MaxAllowedPacket: 64, + } + + table, err := data.createTable("test") + assert.NoError(t, err) + + s := table.Stream() + assert.EqualValues(t, "INSERT INTO `test` VALUES ('1','test@test.de','Test Name 1');", <-s) + assert.EqualValues(t, "INSERT INTO `test` VALUES ('2','test2@test.de','Test Name 2');", <-s) + + // we make sure that all expectations were met + assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections") +} + func TestCreateTableAllValuesWithNil(t *testing.T) { db, mock, err := sqlmock.New() assert.NoError(t, err, "an error was not expected when opening a stub database connection") @@ -217,8 +270,9 @@ func TestCreateTableOk(t *testing.T) { var buf bytes.Buffer data := Data{ - Connection: db, - Out: &buf, + Connection: db, + Out: &buf, + MaxAllowedPacket: 4096, } assert.NoError(t, data.getTemplates()) @@ -255,3 +309,62 @@ UNLOCK TABLES; result := strings.Replace(buf.String(), "`", "~", -1) assert.Equal(t, expectedResult, result) } + +func TestCreateTableOkSmallPackets(t *testing.T) { + db, mock, err := sqlmock.New() + assert.NoError(t, err, "an error was not expected when opening a stub database connection") + + defer db.Close() + + createTableRows := sqlmock.NewRows([]string{"Table", "Create Table"}). + AddRow("Test_Table", "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`s` char(60) DEFAULT NULL, PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1") + + createTableValueRows := sqlmock.NewRows([]string{"id", "email", "name"}). + AddRow(1, nil, "Test Name 1"). + AddRow(2, "test2@test.de", "Test Name 2") + + mock.ExpectQuery("^SHOW CREATE TABLE `Test_Table`$").WillReturnRows(createTableRows) + mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows) + + var buf bytes.Buffer + data := Data{ + Connection: db, + Out: &buf, + MaxAllowedPacket: 64, + } + + assert.NoError(t, data.getTemplates()) + + table, err := data.createTable("Test_Table") + assert.NoError(t, err) + + data.writeTable(table) + + // we make sure that all expectations were met + assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections") + + expectedResult := ` +-- +-- Table structure for table ~Test_Table~ +-- + +DROP TABLE IF EXISTS ~Test_Table~; +/*!40101 SET @saved_cs_client = @@character_set_client */; + SET character_set_client = utf8mb4 ; +CREATE TABLE 'Test_Table' (~id~ int(11) NOT NULL AUTO_INCREMENT,~s~ char(60) DEFAULT NULL, PRIMARY KEY (~id~))ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table ~Test_Table~ +-- + +LOCK TABLES ~Test_Table~ WRITE; +/*!40000 ALTER TABLE ~Test_Table~ DISABLE KEYS */; +INSERT INTO ~Test_Table~ VALUES ('1',NULL,'Test Name 1'); +INSERT INTO ~Test_Table~ VALUES ('2','test2@test.de','Test Name 2'); +/*!40000 ALTER TABLE ~Test_Table~ ENABLE KEYS */; +UNLOCK TABLES; +` + result := strings.Replace(buf.String(), "`", "~", -1) + assert.Equal(t, expectedResult, result) +} From 1cff840d9f29a8e6b84a321a0abf794c37b6fd09 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 4 Apr 2019 15:37:15 -0500 Subject: [PATCH 42/59] The MySQL way --- dump.go | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/dump.go b/dump.go index fd4ee89..22d3360 100644 --- a/dump.go +++ b/dump.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "reflect" + "runtime" "text/template" "time" ) @@ -47,7 +48,9 @@ type metaData struct { } const ( - version = "0.4.1" + // Version is exported for other builds + Version = "0.4.1" + defaultMaxAllowedPacket = 4194304 ) @@ -113,9 +116,12 @@ const nullType = "NULL" // Dump data using struct func (data *Data) Dump() error { meta := metaData{ - DumpVersion: version, + DumpVersion: Version, + } + + if data.MaxAllowedPacket == 0 { + data.MaxAllowedPacket = defaultMaxAllowedPacket } - data.initMaxPacketSize() if err := meta.updateServerVersion(data.Connection); err != nil { return err @@ -191,17 +197,6 @@ func (data *Data) getTemplates() (err error) { return } -func (data *Data) initMaxPacketSize() { - if data.MaxAllowedPacket <= 0 { - data.MaxAllowedPacket = defaultMaxAllowedPacket - - var maxSize int - if err := data.Connection.QueryRow("SELECT @@global.max_allowed_packet;").Scan(&maxSize); err == nil { - data.MaxAllowedPacket = maxSize - } - } -} - func (data *Data) getTables() ([]string, error) { tables := make([]string, 0) @@ -400,7 +395,17 @@ func (table *table) Stream() <-chan string { insert.WriteString(",") } b.WriteTo(&insert) - b.Reset() + + // debug + var m runtime.MemStats + runtime.ReadMemStats(&m) + // For info on each, see: https://golang.org/pkg/runtime/#MemStats + fmt.Print("\tBuffer = ", bToMiB(uint64(insert.Len())), " MiB") + fmt.Print("\tCapacity = ", bToMiB(uint64(insert.Cap())), " MiB") + fmt.Print("\tAlloc = ", bToMiB(m.Alloc), " MiB") + fmt.Print("\tTotalAlloc = ", bToMiB(m.TotalAlloc), " MiB") + fmt.Print("\tSys = ", bToMiB(m.Sys), " MiB") + fmt.Print("\tNumGC = ", m.NumGC, "\n") } if insert.Len() != 0 { insert.WriteString(";") @@ -409,3 +414,7 @@ func (table *table) Stream() <-chan string { }() return valueOut } + +func bToMiB(b uint64) uint64 { + return b / 1024 / 1024 +} From 4a93eb24d0e081a243732c8179a14063522ad1c9 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 4 Apr 2019 15:38:46 -0500 Subject: [PATCH 43/59] Remove debug --- dump.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/dump.go b/dump.go index 22d3360..595b3ff 100644 --- a/dump.go +++ b/dump.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "reflect" - "runtime" "text/template" "time" ) @@ -395,17 +394,6 @@ func (table *table) Stream() <-chan string { insert.WriteString(",") } b.WriteTo(&insert) - - // debug - var m runtime.MemStats - runtime.ReadMemStats(&m) - // For info on each, see: https://golang.org/pkg/runtime/#MemStats - fmt.Print("\tBuffer = ", bToMiB(uint64(insert.Len())), " MiB") - fmt.Print("\tCapacity = ", bToMiB(uint64(insert.Cap())), " MiB") - fmt.Print("\tAlloc = ", bToMiB(m.Alloc), " MiB") - fmt.Print("\tTotalAlloc = ", bToMiB(m.TotalAlloc), " MiB") - fmt.Print("\tSys = ", bToMiB(m.Sys), " MiB") - fmt.Print("\tNumGC = ", m.NumGC, "\n") } if insert.Len() != 0 { insert.WriteString(";") @@ -414,7 +402,3 @@ func (table *table) Stream() <-chan string { }() return valueOut } - -func bToMiB(b uint64) uint64 { - return b / 1024 / 1024 -} From de45383ea5d7b82e971edb14f3e84ab3da41eb52 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 4 Apr 2019 15:47:09 -0500 Subject: [PATCH 44/59] Fix the test --- mysqldump_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysqldump_test.go b/mysqldump_test.go index 40c55d3..38fd6c7 100644 --- a/mysqldump_test.go +++ b/mysqldump_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" ) -const expected = `-- Go SQL Dump ` + version + ` +const expected = `-- Go SQL Dump ` + Version + ` -- -- ------------------------------------------------------ -- Server version test_version From e58e36d1ce1191a9a4417bde978db66029a24691 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 4 Apr 2019 16:20:04 -0500 Subject: [PATCH 45/59] Update export --- dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump.go b/dump.go index 595b3ff..28c19b2 100644 --- a/dump.go +++ b/dump.go @@ -47,7 +47,7 @@ type metaData struct { } const ( - // Version is exported for other builds + // Version of this plugin for easy reference Version = "0.4.1" defaultMaxAllowedPacket = 4194304 From c2fe4afc935babfc52170b3eca80ce00ed33c0d1 Mon Sep 17 00:00:00 2001 From: Grzegorz K Date: Wed, 26 Jun 2019 23:07:07 +0200 Subject: [PATCH 46/59] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d80fe30..d702edc 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,9 @@ func main() { config.Addr = "your-hostname:your-port" dumpDir := "dumps" // you should create this directory - dumpFilenameFormat := fmt.Sprintf("%s-20060102T150405", dbname) // accepts time layout string and add .sql at the end of file + dumpFilenameFormat := fmt.Sprintf("%s-20060102T150405", config.DBName) // accepts time layout string and add .sql at the end of file - db, err := sql.Open("mysql", config.FormatDNS()) + db, err := sql.Open("mysql", config.FormatDSN()) if err != nil { fmt.Println("Error opening database: ", err) return @@ -39,12 +39,12 @@ func main() { } // Dump database to file - resultFilename, err := dumper.Dump() + err := dumper.Dump() if err != nil { fmt.Println("Error dumping:", err) return } - fmt.Printf("File is saved to %s", resultFilename) + fmt.Printf("File is saved to %s", dumpFilenameFormat) // Close dumper, connected database and file stream. dumper.Close() From 53765f2cdf3944be067d5d97005c988f904f1183 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Tue, 24 Sep 2019 11:57:10 -0500 Subject: [PATCH 47/59] Add the ability to lock all tables --- dump.go | 33 +++++++++++++++++-------- dump_test.go | 21 ++++++---------- mysqldump.go | 8 +++--- mysqldump_test.go | 63 +++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 89 insertions(+), 36 deletions(-) diff --git a/dump.go b/dump.go index 28c19b2..4b0ade2 100644 --- a/dump.go +++ b/dump.go @@ -23,6 +23,7 @@ type Data struct { Connection *sql.DB IgnoreTables []string MaxAllowedPacket int + LockTables bool headerTmpl *template.Template tableTmpl *template.Template @@ -139,6 +140,24 @@ func (data *Data) Dump() error { return err } + // Lock all tables before dumping if present + if data.LockTables && len(tables) > 0 { + var b bytes.Buffer + b.WriteString("LOCK TABLES ") + for index, name := range tables { + if index != 0 { + b.WriteString(",") + } + b.WriteString("`" + name + "` READ /*!32311 LOCAL */") + } + + if _, err := data.Connection.Exec(b.String()); err != nil { + return err + } + + defer data.Connection.Exec("UNLOCK TABLES") + } + for _, name := range tables { if err := data.dumpTable(name); err != nil { return err @@ -160,11 +179,7 @@ func (data *Data) dumpTable(name string) error { if data.err != nil { return data.err } - table, err := data.createTable(name) - if err != nil { - return err - } - + table := data.createTable(name) return data.writeTable(table) } @@ -228,20 +243,18 @@ func (data *Data) isIgnoredTable(name string) bool { func (data *metaData) updateServerVersion(db *sql.DB) (err error) { var serverVersion sql.NullString - err = db.QueryRow("SELECT version();").Scan(&serverVersion) + err = db.QueryRow("SELECT version()").Scan(&serverVersion) data.ServerVersion = serverVersion.String return } // MARK: create methods -func (data *Data) createTable(name string) (*table, error) { - t := &table{ +func (data *Data) createTable(name string) *table { + return &table{ Name: name, data: data, } - - return t, nil } func (table *table) NameEsc() string { diff --git a/dump_test.go b/dump_test.go index a79a31c..41fd1cf 100644 --- a/dump_test.go +++ b/dump_test.go @@ -120,8 +120,7 @@ func TestCreateTableSQLOk(t *testing.T) { Connection: db, } - table, err := data.createTable("Test_Table") - assert.NoError(t, err) + table := data.createTable("Test_Table") result, err := table.CreateSQL() assert.NoError(t, err) @@ -151,8 +150,7 @@ func TestCreateTableRowValues(t *testing.T) { Connection: db, } - table, err := data.createTable("test") - assert.NoError(t, err) + table := data.createTable("test") assert.True(t, table.Next()) @@ -181,8 +179,7 @@ func TestCreateTableValuesSteam(t *testing.T) { MaxAllowedPacket: 4096, } - table, err := data.createTable("test") - assert.NoError(t, err) + table := data.createTable("test") s := table.Stream() assert.EqualValues(t, "INSERT INTO `test` VALUES ('1','test@test.de','Test Name 1'),('2','test2@test.de','Test Name 2');", <-s) @@ -207,8 +204,7 @@ func TestCreateTableValuesSteamSmallPackets(t *testing.T) { MaxAllowedPacket: 64, } - table, err := data.createTable("test") - assert.NoError(t, err) + table := data.createTable("test") s := table.Stream() assert.EqualValues(t, "INSERT INTO `test` VALUES ('1','test@test.de','Test Name 1');", <-s) @@ -234,8 +230,7 @@ func TestCreateTableAllValuesWithNil(t *testing.T) { Connection: db, } - table, err := data.createTable("test") - assert.NoError(t, err) + table := data.createTable("test") results := make([]string, 0) for table.Next() { @@ -277,8 +272,7 @@ func TestCreateTableOk(t *testing.T) { assert.NoError(t, data.getTemplates()) - table, err := data.createTable("Test_Table") - assert.NoError(t, err) + table := data.createTable("Test_Table") data.writeTable(table) @@ -335,8 +329,7 @@ func TestCreateTableOkSmallPackets(t *testing.T) { assert.NoError(t, data.getTemplates()) - table, err := data.createTable("Test_Table") - assert.NoError(t, err) + table := data.createTable("Test_Table") data.writeTable(table) diff --git a/mysqldump.go b/mysqldump.go index b77d1be..3345d44 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -39,17 +39,17 @@ func Register(db *sql.DB, dir, format string) (*Data, error) { return &Data{ Out: f, Connection: db, + LockTables: true, }, nil } // Dump Creates a MYSQL dump from the connection to the stream. func Dump(db *sql.DB, out io.Writer) error { - data := Data{ + return (&Data{ Connection: db, Out: out, - } - - return data.Dump() + LockTables: true, + }).Dump() } // Close the dumper. diff --git a/mysqldump_test.go b/mysqldump_test.go index 38fd6c7..aefff7b 100644 --- a/mysqldump_test.go +++ b/mysqldump_test.go @@ -2,6 +2,7 @@ package mysqldump import ( "bytes" + "io/ioutil" "strings" "testing" @@ -56,11 +57,12 @@ UNLOCK TABLES; ` -func RunDump(t testing.TB) string { +func RunDump(t testing.TB, data *Data) { db, mock, err := sqlmock.New() assert.NoError(t, err, "an error was not expected when opening a stub database connection") defer db.Close() + data.Connection = db showTablesRows := sqlmock.NewRows([]string{"Tables_in_Testdb"}). AddRow("Test_Table") @@ -74,27 +76,72 @@ func RunDump(t testing.TB) string { AddRow(1, nil, "Test Name 1"). AddRow(2, "test2@test.de", "Test Name 2") - mock.ExpectQuery("^SELECT version()").WillReturnRows(serverVersionRows) + mock.ExpectQuery(`^SELECT version\(\)$`).WillReturnRows(serverVersionRows) mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(showTablesRows) + mock.ExpectExec("^LOCK TABLES `Test_Table` READ /\\*!32311 LOCAL \\*/$").WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectQuery("^SHOW CREATE TABLE `Test_Table`$").WillReturnRows(createTableRows) mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows) + assert.NoError(t, data.Dump(), "an error was not expected when dumping a stub database connection") +} + +func TestDumpOk(t *testing.T) { var buf bytes.Buffer - assert.NoError(t, Dump(db, &buf), "an error was not expected when dumping a stub database connection") - return buf.String() + RunDump(t, &Data{ + Out: &buf, + LockTables: true, + }) + + result := strings.Replace(strings.Split(buf.String(), "-- Dump completed")[0], "`", "~", -1) + + assert.Equal(t, expected, result) } -func TestDumpOk(t *testing.T) { - out := RunDump(t) +func TestNoLockOk(t *testing.T) { + var buf bytes.Buffer - result := strings.Replace(strings.Split(out, "-- Dump completed")[0], "`", "~", -1) + data := &Data{ + Out: &buf, + LockTables: false, + } + + db, mock, err := sqlmock.New() + assert.NoError(t, err, "an error was not expected when opening a stub database connection") + defer db.Close() + + data.Connection = db + showTablesRows := sqlmock.NewRows([]string{"Tables_in_Testdb"}). + AddRow("Test_Table") + + serverVersionRows := sqlmock.NewRows([]string{"Version()"}). + AddRow("test_version") + + createTableRows := sqlmock.NewRows([]string{"Table", "Create Table"}). + AddRow("Test_Table", "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`email` char(60) DEFAULT NULL, `name` char(60), PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1") + + createTableValueRows := sqlmock.NewRows([]string{"id", "email", "name"}). + AddRow(1, nil, "Test Name 1"). + AddRow(2, "test2@test.de", "Test Name 2") + + mock.ExpectQuery(`^SELECT version\(\)$`).WillReturnRows(serverVersionRows) + mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(showTablesRows) + mock.ExpectQuery("^SHOW CREATE TABLE `Test_Table`$").WillReturnRows(createTableRows) + mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows) + + assert.NoError(t, data.Dump(), "an error was not expected when dumping a stub database connection") + + result := strings.Replace(strings.Split(buf.String(), "-- Dump completed")[0], "`", "~", -1) assert.Equal(t, expected, result) } func BenchmarkDump(b *testing.B) { + data := &Data{ + Out: ioutil.Discard, + LockTables: true, + } for i := 0; i < b.N; i++ { - _ = RunDump(b) + RunDump(b, data) } } From 89292d2818caa992cb3b3bc48c5f301217ba4c53 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Tue, 24 Sep 2019 14:07:32 -0500 Subject: [PATCH 48/59] Bump the version number --- dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump.go b/dump.go index 4b0ade2..aba45f4 100644 --- a/dump.go +++ b/dump.go @@ -49,7 +49,7 @@ type metaData struct { const ( // Version of this plugin for easy reference - Version = "0.4.1" + Version = "0.5.0" defaultMaxAllowedPacket = 4194304 ) From f00cfb1736a770db9b8aac198608c5b7a529dbda Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Wed, 25 Sep 2019 11:11:28 -0500 Subject: [PATCH 49/59] Starting to transaction wrap --- dump.go | 59 ++++++++++++++++++++++++++++++++--------------- dump_test.go | 28 ++++++++++++++++------ mysqldump_test.go | 8 +++++-- 3 files changed, 67 insertions(+), 28 deletions(-) diff --git a/dump.go b/dump.go index aba45f4..dec2855 100644 --- a/dump.go +++ b/dump.go @@ -14,17 +14,21 @@ import ( /* Data struct to configure dump behavior - Out: Stream to wite to - Connection: Database connection to dump - IgnoreTables: Mark sensitive tables to ignore + Out: Stream to wite to + Connection: Database connection to dump + IgnoreTables: Mark sensitive tables to ignore + LockTables: Lock all tables for the duration of the dump + SingleTransaction: Do the entire dump in one transaction */ type Data struct { - Out io.Writer - Connection *sql.DB - IgnoreTables []string - MaxAllowedPacket int - LockTables bool - + Out io.Writer + Connection *sql.DB + IgnoreTables []string + MaxAllowedPacket int + LockTables bool + SingleTransaction bool + + tx *sql.Tx headerTmpl *template.Template tableTmpl *template.Template footerTmpl *template.Template @@ -49,7 +53,7 @@ type metaData struct { const ( // Version of this plugin for easy reference - Version = "0.5.0" + Version = "0.6.0" defaultMaxAllowedPacket = 4194304 ) @@ -123,15 +127,20 @@ func (data *Data) Dump() error { data.MaxAllowedPacket = defaultMaxAllowedPacket } - if err := meta.updateServerVersion(data.Connection); err != nil { + if err := data.getTemplates(); err != nil { return err } - if err := data.getTemplates(); err != nil { + if err := data.headerTmpl.Execute(data.Out, meta); err != nil { return err } - if err := data.headerTmpl.Execute(data.Out, meta); err != nil { + if err := data.begin(); err != nil { + return err + } + defer data.rollback() + + if err := meta.updateServerVersion(data); err != nil { return err } @@ -173,6 +182,18 @@ func (data *Data) Dump() error { // MARK: - Private methods +func (data *Data) begin() (err error) { + data.tx, err = data.Connection.BeginTx(nil, &sql.TxOptions{ + Isolation: sql.LevelRepeatableRead, + ReadOnly: true, + }) + return +} + +func (data *Data) rollback() error { + return data.tx.Rollback() +} + // MARK: writter methods func (data *Data) dumpTable(name string) error { @@ -214,7 +235,7 @@ func (data *Data) getTemplates() (err error) { func (data *Data) getTables() ([]string, error) { tables := make([]string, 0) - rows, err := data.Connection.Query("SHOW TABLES") + rows, err := data.tx.Query("SHOW TABLES") if err != nil { return tables, err } @@ -241,10 +262,10 @@ func (data *Data) isIgnoredTable(name string) bool { return false } -func (data *metaData) updateServerVersion(db *sql.DB) (err error) { +func (meta *metaData) updateServerVersion(data *Data) (err error) { var serverVersion sql.NullString - err = db.QueryRow("SELECT version()").Scan(&serverVersion) - data.ServerVersion = serverVersion.String + err = data.tx.QueryRow("SELECT version()").Scan(&serverVersion) + meta.ServerVersion = serverVersion.String return } @@ -263,7 +284,7 @@ func (table *table) NameEsc() string { func (table *table) CreateSQL() (string, error) { var tableReturn, tableSQL sql.NullString - if err := table.data.Connection.QueryRow("SHOW CREATE TABLE "+table.NameEsc()).Scan(&tableReturn, &tableSQL); err != nil { + if err := table.data.tx.QueryRow("SHOW CREATE TABLE "+table.NameEsc()).Scan(&tableReturn, &tableSQL); err != nil { return "", err } @@ -280,7 +301,7 @@ func (table *table) Init() (err error) { return errors.New("can't init twice") } - table.rows, err = table.data.Connection.Query("SELECT * FROM " + table.NameEsc()) + table.rows, err = table.data.tx.Query("SELECT * FROM " + table.NameEsc()) if err != nil { return err } diff --git a/dump_test.go b/dump_test.go index 41fd1cf..11f1c66 100644 --- a/dump_test.go +++ b/dump_test.go @@ -10,10 +10,24 @@ import ( "github.com/stretchr/testify/assert" ) -func TestGetTablesOk(t *testing.T) { +func GetMockData() (data *Data, mock sqlmock.Sqlmock, err error) { db, mock, err := sqlmock.New() + if err != nil { + return + } + mock.ExpectBegin() + + data = &Data{ + Connection: db, + } + err = data.begin() + return +} + +func TestGetTablesOk(t *testing.T) { + data, mock, err := GetMockData() assert.NoError(t, err, "an error was not expected when opening a stub database connection") - defer db.Close() + defer data.Close() rows := sqlmock.NewRows([]string{"Tables_in_Testdb"}). AddRow("Test_Table_1"). @@ -21,10 +35,6 @@ func TestGetTablesOk(t *testing.T) { mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(rows) - data := Data{ - Connection: db, - } - result, err := data.getTables() assert.NoError(t, err) @@ -94,11 +104,15 @@ func TestGetServerVersionOk(t *testing.T) { rows := sqlmock.NewRows([]string{"Version()"}). AddRow("test_version") + mock.ExpectBegin() mock.ExpectQuery("^SELECT version()").WillReturnRows(rows) meta := metaData{} - assert.NoError(t, meta.updateServerVersion(db), "error was not expected while updating stats") + tx, _ := db.Begin() + assert.NoError(t, meta.updateServerVersion(&Data{ + tx: tx, + }), "error was not expected while updating stats") // we make sure that all expectations were met assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections") diff --git a/mysqldump_test.go b/mysqldump_test.go index aefff7b..26f5114 100644 --- a/mysqldump_test.go +++ b/mysqldump_test.go @@ -76,11 +76,13 @@ func RunDump(t testing.TB, data *Data) { AddRow(1, nil, "Test Name 1"). AddRow(2, "test2@test.de", "Test Name 2") + mock.ExpectBegin() mock.ExpectQuery(`^SELECT version\(\)$`).WillReturnRows(serverVersionRows) - mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(showTablesRows) + mock.ExpectQuery(`^SHOW TABLES$`).WillReturnRows(showTablesRows) mock.ExpectExec("^LOCK TABLES `Test_Table` READ /\\*!32311 LOCAL \\*/$").WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectQuery("^SHOW CREATE TABLE `Test_Table`$").WillReturnRows(createTableRows) mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows) + mock.ExpectRollback() assert.NoError(t, data.Dump(), "an error was not expected when dumping a stub database connection") } @@ -124,10 +126,12 @@ func TestNoLockOk(t *testing.T) { AddRow(1, nil, "Test Name 1"). AddRow(2, "test2@test.de", "Test Name 2") + mock.ExpectBegin() mock.ExpectQuery(`^SELECT version\(\)$`).WillReturnRows(serverVersionRows) - mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(showTablesRows) + mock.ExpectQuery(`^SHOW TABLES$`).WillReturnRows(showTablesRows) mock.ExpectQuery("^SHOW CREATE TABLE `Test_Table`$").WillReturnRows(createTableRows) mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows) + mock.ExpectRollback() assert.NoError(t, data.Dump(), "an error was not expected when dumping a stub database connection") From fdb7b50d5ad8a187056cd470ce34b850f12fd0a8 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Wed, 25 Sep 2019 15:16:31 -0500 Subject: [PATCH 50/59] Have all statments run on the same transaction --- dump.go | 31 ++++++++-------- dump_test.go | 103 +++++++++++++++++---------------------------------- 2 files changed, 48 insertions(+), 86 deletions(-) diff --git a/dump.go b/dump.go index dec2855..f8f6aa2 100644 --- a/dump.go +++ b/dump.go @@ -2,6 +2,7 @@ package mysqldump import ( "bytes" + "context" "database/sql" "errors" "fmt" @@ -14,19 +15,17 @@ import ( /* Data struct to configure dump behavior - Out: Stream to wite to - Connection: Database connection to dump - IgnoreTables: Mark sensitive tables to ignore - LockTables: Lock all tables for the duration of the dump - SingleTransaction: Do the entire dump in one transaction + Out: Stream to wite to + Connection: Database connection to dump + IgnoreTables: Mark sensitive tables to ignore + LockTables: Lock all tables for the duration of the dump */ type Data struct { - Out io.Writer - Connection *sql.DB - IgnoreTables []string - MaxAllowedPacket int - LockTables bool - SingleTransaction bool + Out io.Writer + Connection *sql.DB + IgnoreTables []string + MaxAllowedPacket int + LockTables bool tx *sql.Tx headerTmpl *template.Template @@ -131,10 +130,6 @@ func (data *Data) Dump() error { return err } - if err := data.headerTmpl.Execute(data.Out, meta); err != nil { - return err - } - if err := data.begin(); err != nil { return err } @@ -144,6 +139,10 @@ func (data *Data) Dump() error { return err } + if err := data.headerTmpl.Execute(data.Out, meta); err != nil { + return err + } + tables, err := data.getTables() if err != nil { return err @@ -183,7 +182,7 @@ func (data *Data) Dump() error { // MARK: - Private methods func (data *Data) begin() (err error) { - data.tx, err = data.Connection.BeginTx(nil, &sql.TxOptions{ + data.tx, err = data.Connection.BeginTx(context.Background(), &sql.TxOptions{ Isolation: sql.LevelRepeatableRead, ReadOnly: true, }) diff --git a/dump_test.go b/dump_test.go index 11f1c66..ec0ca12 100644 --- a/dump_test.go +++ b/dump_test.go @@ -2,6 +2,7 @@ package mysqldump import ( "bytes" + "database/sql" "reflect" "strings" "testing" @@ -10,8 +11,9 @@ import ( "github.com/stretchr/testify/assert" ) -func GetMockData() (data *Data, mock sqlmock.Sqlmock, err error) { - db, mock, err := sqlmock.New() +func getMockData() (data *Data, mock sqlmock.Sqlmock, err error) { + var db *sql.DB + db, mock, err = sqlmock.New() if err != nil { return } @@ -25,7 +27,7 @@ func GetMockData() (data *Data, mock sqlmock.Sqlmock, err error) { } func TestGetTablesOk(t *testing.T) { - data, mock, err := GetMockData() + data, mock, err := getMockData() assert.NoError(t, err, "an error was not expected when opening a stub database connection") defer data.Close() @@ -45,9 +47,9 @@ func TestGetTablesOk(t *testing.T) { } func TestIgnoreTablesOk(t *testing.T) { - db, mock, err := sqlmock.New() + data, mock, err := getMockData() assert.NoError(t, err, "an error was not expected when opening a stub database connection") - defer db.Close() + defer data.Close() rows := sqlmock.NewRows([]string{"Tables_in_Testdb"}). AddRow("Test_Table_1"). @@ -55,10 +57,7 @@ func TestIgnoreTablesOk(t *testing.T) { mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(rows) - data := Data{ - Connection: db, - IgnoreTables: []string{"Test_Table_1"}, - } + data.IgnoreTables = []string{"Test_Table_1"} result, err := data.getTables() assert.NoError(t, err) @@ -70,10 +69,9 @@ func TestIgnoreTablesOk(t *testing.T) { } func TestGetTablesNil(t *testing.T) { - db, mock, err := sqlmock.New() + data, mock, err := getMockData() assert.NoError(t, err, "an error was not expected when opening a stub database connection") - - defer db.Close() + defer data.Close() rows := sqlmock.NewRows([]string{"Tables_in_Testdb"}). AddRow("Test_Table_1"). @@ -82,10 +80,6 @@ func TestGetTablesNil(t *testing.T) { mock.ExpectQuery("^SHOW TABLES$").WillReturnRows(rows) - data := Data{ - Connection: db, - } - result, err := data.getTables() assert.NoError(t, err) @@ -96,23 +90,18 @@ func TestGetTablesNil(t *testing.T) { } func TestGetServerVersionOk(t *testing.T) { - db, mock, err := sqlmock.New() + data, mock, err := getMockData() assert.NoError(t, err, "an error was not expected when opening a stub database connection") - - defer db.Close() + defer data.Close() rows := sqlmock.NewRows([]string{"Version()"}). AddRow("test_version") - mock.ExpectBegin() mock.ExpectQuery("^SELECT version()").WillReturnRows(rows) meta := metaData{} - tx, _ := db.Begin() - assert.NoError(t, meta.updateServerVersion(&Data{ - tx: tx, - }), "error was not expected while updating stats") + assert.NoError(t, meta.updateServerVersion(data), "error was not expected while updating stats") // we make sure that all expectations were met assert.NoError(t, mock.ExpectationsWereMet(), "there were unfulfilled expections") @@ -121,19 +110,15 @@ func TestGetServerVersionOk(t *testing.T) { } func TestCreateTableSQLOk(t *testing.T) { - db, mock, err := sqlmock.New() + data, mock, err := getMockData() assert.NoError(t, err, "an error was not expected when opening a stub database connection") - defer db.Close() + defer data.Close() rows := sqlmock.NewRows([]string{"Table", "Create Table"}). AddRow("Test_Table", "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`s` char(60) DEFAULT NULL, PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1") mock.ExpectQuery("^SHOW CREATE TABLE `Test_Table`$").WillReturnRows(rows) - data := Data{ - Connection: db, - } - table := data.createTable("Test_Table") result, err := table.CreateSQL() @@ -150,9 +135,9 @@ func TestCreateTableSQLOk(t *testing.T) { } func TestCreateTableRowValues(t *testing.T) { - db, mock, err := sqlmock.New() + data, mock, err := getMockData() assert.NoError(t, err, "an error was not expected when opening a stub database connection") - defer db.Close() + defer data.Close() rows := sqlmock.NewRows([]string{"id", "email", "name"}). AddRow(1, "test@test.de", "Test Name 1"). @@ -160,10 +145,6 @@ func TestCreateTableRowValues(t *testing.T) { mock.ExpectQuery("^SELECT (.+) FROM `test`$").WillReturnRows(rows) - data := Data{ - Connection: db, - } - table := data.createTable("test") assert.True(t, table.Next()) @@ -178,9 +159,9 @@ func TestCreateTableRowValues(t *testing.T) { } func TestCreateTableValuesSteam(t *testing.T) { - db, mock, err := sqlmock.New() + data, mock, err := getMockData() assert.NoError(t, err, "an error was not expected when opening a stub database connection") - defer db.Close() + defer data.Close() rows := sqlmock.NewRows([]string{"id", "email", "name"}). AddRow(1, "test@test.de", "Test Name 1"). @@ -188,10 +169,7 @@ func TestCreateTableValuesSteam(t *testing.T) { mock.ExpectQuery("^SELECT (.+) FROM `test`$").WillReturnRows(rows) - data := Data{ - Connection: db, - MaxAllowedPacket: 4096, - } + data.MaxAllowedPacket = 4096 table := data.createTable("test") @@ -203,9 +181,9 @@ func TestCreateTableValuesSteam(t *testing.T) { } func TestCreateTableValuesSteamSmallPackets(t *testing.T) { - db, mock, err := sqlmock.New() + data, mock, err := getMockData() assert.NoError(t, err, "an error was not expected when opening a stub database connection") - defer db.Close() + defer data.Close() rows := sqlmock.NewRows([]string{"id", "email", "name"}). AddRow(1, "test@test.de", "Test Name 1"). @@ -213,10 +191,7 @@ func TestCreateTableValuesSteamSmallPackets(t *testing.T) { mock.ExpectQuery("^SELECT (.+) FROM `test`$").WillReturnRows(rows) - data := Data{ - Connection: db, - MaxAllowedPacket: 64, - } + data.MaxAllowedPacket = 64 table := data.createTable("test") @@ -229,9 +204,9 @@ func TestCreateTableValuesSteamSmallPackets(t *testing.T) { } func TestCreateTableAllValuesWithNil(t *testing.T) { - db, mock, err := sqlmock.New() + data, mock, err := getMockData() assert.NoError(t, err, "an error was not expected when opening a stub database connection") - defer db.Close() + defer data.Close() rows := sqlmock.NewRows([]string{"id", "email", "name"}). AddRow(1, nil, "Test Name 1"). @@ -240,10 +215,6 @@ func TestCreateTableAllValuesWithNil(t *testing.T) { mock.ExpectQuery("^SELECT (.+) FROM `test`$").WillReturnRows(rows) - data := Data{ - Connection: db, - } - table := data.createTable("test") results := make([]string, 0) @@ -262,10 +233,9 @@ func TestCreateTableAllValuesWithNil(t *testing.T) { } func TestCreateTableOk(t *testing.T) { - db, mock, err := sqlmock.New() + data, mock, err := getMockData() assert.NoError(t, err, "an error was not expected when opening a stub database connection") - - defer db.Close() + defer data.Close() createTableRows := sqlmock.NewRows([]string{"Table", "Create Table"}). AddRow("Test_Table", "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`s` char(60) DEFAULT NULL, PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1") @@ -278,11 +248,8 @@ func TestCreateTableOk(t *testing.T) { mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows) var buf bytes.Buffer - data := Data{ - Connection: db, - Out: &buf, - MaxAllowedPacket: 4096, - } + data.Out = &buf + data.MaxAllowedPacket = 4096 assert.NoError(t, data.getTemplates()) @@ -319,10 +286,9 @@ UNLOCK TABLES; } func TestCreateTableOkSmallPackets(t *testing.T) { - db, mock, err := sqlmock.New() + data, mock, err := getMockData() assert.NoError(t, err, "an error was not expected when opening a stub database connection") - - defer db.Close() + defer data.Close() createTableRows := sqlmock.NewRows([]string{"Table", "Create Table"}). AddRow("Test_Table", "CREATE TABLE 'Test_Table' (`id` int(11) NOT NULL AUTO_INCREMENT,`s` char(60) DEFAULT NULL, PRIMARY KEY (`id`))ENGINE=InnoDB DEFAULT CHARSET=latin1") @@ -335,11 +301,8 @@ func TestCreateTableOkSmallPackets(t *testing.T) { mock.ExpectQuery("^SELECT (.+) FROM `Test_Table`$").WillReturnRows(createTableValueRows) var buf bytes.Buffer - data := Data{ - Connection: db, - Out: &buf, - MaxAllowedPacket: 64, - } + data.Out = &buf + data.MaxAllowedPacket = 64 assert.NoError(t, data.getTemplates()) From 4517c0a452fd5e7ef87a7fd09a264917e43e8108 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Wed, 25 Sep 2019 15:42:16 -0500 Subject: [PATCH 51/59] There is no locking by default just a read transaction --- mysqldump.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/mysqldump.go b/mysqldump.go index 3345d44..258da8c 100644 --- a/mysqldump.go +++ b/mysqldump.go @@ -39,7 +39,6 @@ func Register(db *sql.DB, dir, format string) (*Data, error) { return &Data{ Out: f, Connection: db, - LockTables: true, }, nil } @@ -48,7 +47,6 @@ func Dump(db *sql.DB, out io.Writer) error { return (&Data{ Connection: db, Out: out, - LockTables: true, }).Dump() } From adf372a145bb1471bb2504aa423a6aa690ee1bf6 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Wed, 25 Sep 2019 15:51:31 -0500 Subject: [PATCH 52/59] Change the version down --- dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump.go b/dump.go index f8f6aa2..8ff8020 100644 --- a/dump.go +++ b/dump.go @@ -52,7 +52,7 @@ type metaData struct { const ( // Version of this plugin for easy reference - Version = "0.6.0" + Version = "0.5.0" defaultMaxAllowedPacket = 4194304 ) From ca1be9d7b5c41d008f8ebfc8cc0b494837257e65 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 26 Sep 2019 09:21:27 -0500 Subject: [PATCH 53/59] Convert to go mod --- go.mod | 6 ++++++ go.sum | 13 +++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 go.mod create mode 100644 go.sum diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2961500 --- /dev/null +++ b/go.mod @@ -0,0 +1,6 @@ +module github.com/BrandonRoehl/go-mysqldump + +require ( + github.com/DATA-DOG/go-sqlmock v1.3.0 + github.com/stretchr/testify v1.4.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9165637 --- /dev/null +++ b/go.sum @@ -0,0 +1,13 @@ +github.com/DATA-DOG/go-sqlmock v1.3.0 h1:ljjRxlddjfChBJdFKJs5LuCwCWPLaC1UZLwAo3PBBMk= +github.com/DATA-DOG/go-sqlmock v1.3.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 55278b1d13d8ea12637dc236271eb1961eac50af Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Thu, 26 Sep 2019 10:26:33 -0500 Subject: [PATCH 54/59] Update comments and pointers --- .travis.yml | 2 +- dump.go | 16 +++++++++++----- go.mod | 4 +++- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7294d6b..7d468ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: go go: - - 1.11.x + - 1.13.x - master script: diff --git a/dump.go b/dump.go index 8ff8020..a9dd0bd 100644 --- a/dump.go +++ b/dump.go @@ -15,10 +15,11 @@ import ( /* Data struct to configure dump behavior - Out: Stream to wite to - Connection: Database connection to dump - IgnoreTables: Mark sensitive tables to ignore - LockTables: Lock all tables for the duration of the dump + Out: Stream to wite to + Connection: Database connection to dump + IgnoreTables: Mark sensitive tables to ignore + MaxAllowedPacket: Sets the largest packet size to use in backups + LockTables: Lock all tables for the duration of the dump */ type Data struct { Out io.Writer @@ -130,6 +131,9 @@ func (data *Data) Dump() error { return err } + // Start the read only transaction and defer the rollback until the end + // This way the database will have the exact state it did at the begining of + // the backup and nothing can be accidentally committed if err := data.begin(); err != nil { return err } @@ -181,6 +185,8 @@ func (data *Data) Dump() error { // MARK: - Private methods +// begin starts a read only transaction that will be whatever the database was +// when it was called func (data *Data) begin() (err error) { data.tx, err = data.Connection.BeginTx(context.Background(), &sql.TxOptions{ Isolation: sql.LevelRepeatableRead, @@ -189,6 +195,7 @@ func (data *Data) begin() (err error) { return } +// rollback cancels the transaction func (data *Data) rollback() error { return data.tx.Rollback() } @@ -294,7 +301,6 @@ func (table *table) CreateSQL() (string, error) { return tableSQL.String, nil } -// defer rows.Close() func (table *table) Init() (err error) { if len(table.types) != 0 { return errors.New("can't init twice") diff --git a/go.mod b/go.mod index 2961500..13a96e9 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ -module github.com/BrandonRoehl/go-mysqldump +module github.com/jamf/go-mysqldump require ( github.com/DATA-DOG/go-sqlmock v1.3.0 github.com/stretchr/testify v1.4.0 ) + +go 1.13 From cfdd6976eda5b7be1b767ba835c36298f8f2caee Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Mon, 30 Sep 2019 12:02:29 -0500 Subject: [PATCH 55/59] Fix the comment --- dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump.go b/dump.go index a9dd0bd..a0d3db4 100644 --- a/dump.go +++ b/dump.go @@ -18,7 +18,7 @@ Data struct to configure dump behavior Out: Stream to wite to Connection: Database connection to dump IgnoreTables: Mark sensitive tables to ignore - MaxAllowedPacket: Sets the largest packet size to use in backups + MaxAllowedPacket: Sets the largest packet size to use in backups LockTables: Lock all tables for the duration of the dump */ type Data struct { From d72543438a53876afe1f084ac040889c698918dc Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Tue, 1 Oct 2019 11:13:08 -0500 Subject: [PATCH 56/59] Consolidate the for loops --- dump.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dump.go b/dump.go index a0d3db4..a5f09b4 100644 --- a/dump.go +++ b/dump.go @@ -325,6 +325,7 @@ func (table *table) Init() (err error) { } table.types = make([]reflect.Type, len(tt)) + table.values = make([]interface{}, len(tt)) for i, tp := range tt { st := tp.ScanType() if tp.DatabaseTypeName() == "BLOB" { @@ -338,9 +339,6 @@ func (table *table) Init() (err error) { } else { table.types[i] = reflect.TypeOf(sql.NullString{}) } - } - table.values = make([]interface{}, len(tt)) - for i := range table.values { table.values[i] = reflect.New(table.types[i]).Interface() } return nil From b6b4ed75ffb576006aae290b135001f1b9b03314 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Tue, 8 Oct 2019 13:48:12 -0500 Subject: [PATCH 57/59] Don't need to store the reflection type --- dump.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/dump.go b/dump.go index a5f09b4..1bcc4e8 100644 --- a/dump.go +++ b/dump.go @@ -41,7 +41,6 @@ type table struct { data *Data rows *sql.Rows - types []reflect.Type values []interface{} } @@ -302,7 +301,7 @@ func (table *table) CreateSQL() (string, error) { } func (table *table) Init() (err error) { - if len(table.types) != 0 { + if len(table.values) != 0 { return errors.New("can't init twice") } @@ -324,22 +323,22 @@ func (table *table) Init() (err error) { return err } - table.types = make([]reflect.Type, len(tt)) + var t reflect.Type table.values = make([]interface{}, len(tt)) for i, tp := range tt { st := tp.ScanType() if tp.DatabaseTypeName() == "BLOB" { - table.types[i] = reflect.TypeOf(sql.RawBytes{}) + t = reflect.TypeOf(sql.RawBytes{}) } else if st != nil && (st.Kind() == reflect.Int || st.Kind() == reflect.Int8 || st.Kind() == reflect.Int16 || st.Kind() == reflect.Int32 || st.Kind() == reflect.Int64) { - table.types[i] = reflect.TypeOf(sql.NullInt64{}) + t = reflect.TypeOf(sql.NullInt64{}) } else { - table.types[i] = reflect.TypeOf(sql.NullString{}) + t = reflect.TypeOf(sql.NullString{}) } - table.values[i] = reflect.New(table.types[i]).Interface() + table.values[i] = reflect.New(t).Interface() } return nil } From f5bcf36cdbf2d81fdfc33469f15fbfdc9d3adf95 Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Tue, 8 Oct 2019 13:48:33 -0500 Subject: [PATCH 58/59] Don't need to store the reflection type --- dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump.go b/dump.go index 1bcc4e8..b3f5f4c 100644 --- a/dump.go +++ b/dump.go @@ -52,7 +52,7 @@ type metaData struct { const ( // Version of this plugin for easy reference - Version = "0.5.0" + Version = "0.5.1" defaultMaxAllowedPacket = 4194304 ) From 4f247b0f635ae5b5ba72423270b04a4cd12e8bbf Mon Sep 17 00:00:00 2001 From: Brandon Roehl Date: Wed, 9 Oct 2019 11:12:30 -0500 Subject: [PATCH 59/59] Don't need DEP --- Gopkg.lock | 15 --------------- Gopkg.toml | 26 -------------------------- 2 files changed, 41 deletions(-) delete mode 100644 Gopkg.lock delete mode 100644 Gopkg.toml diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index 276890d..0000000 --- a/Gopkg.lock +++ /dev/null @@ -1,15 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - name = "github.com/DATA-DOG/go-sqlmock" - packages = ["."] - revision = "d76b18b42f285b792bf985118980ce9eacea9d10" - version = "v1.3.0" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - inputs-digest = "988d3551024f0c7c7eb0c26b50b95e2f2d9ac8d7e71f35f5e3b226afc317987f" - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index 8b3f987..0000000 --- a/Gopkg.toml +++ /dev/null @@ -1,26 +0,0 @@ - -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" - - -[[constraint]] - name = "github.com/DATA-DOG/go-sqlmock" - version = "~1.3.0"