Skip to content

Commit b285144

Browse files
Chinoholo0807Chinoholo0807
authored
ci: support MySQL data sync (casdoor#2443)
* feat: support tool for mysql master-slave sync * feat: support mysql master-master sync * feat: improve log * feat: improve code * fix: fix bug when len(res) ==0 * fix: fix bug when len(res) ==0 * feat: support master-slave sync * feat: add deleteSlaveUser for TestStopMasterSlaveSync * feat: add deleteSlaveUser for TestStopMasterSlaveSync
1 parent 49c6ce2 commit b285144

File tree

5 files changed

+425
-0
lines changed

5 files changed

+425
-0
lines changed

‎sync_v2/cmd_test.go

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
// Copyright 2023 The Casdoor Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
//go:build !skipCi
16+
// +build !skipCi
17+
18+
package sync_v2
19+
20+
import (
21+
"testing"
22+
23+
_ "github.com/go-sql-driver/mysql"
24+
)
25+
26+
/*
27+
The following config should be added to my.cnf:
28+
29+
gtid_mode=on
30+
enforce_gtid_consistency=on
31+
binlog-format=ROW
32+
server-id = 1 # this should be different for each mysql instance (1,2)
33+
auto_increment_offset = 1 # this is same as server-id
34+
auto_increment_increment = 2 # this is same as the number of mysql instances (2)
35+
log-bin = mysql-bin
36+
replicate-do-db = casdoor # this is the database name
37+
binlog-do-db = casdoor # this is the database name
38+
*/
39+
40+
var Configs = []Database{
41+
{
42+
host: "test-db.v2tl.com",
43+
port: 3306,
44+
username: "root",
45+
password: "password",
46+
database: "casdoor",
47+
// the following two fields are used to create replication user, you don't need to change them
48+
slaveUser: "repl_user",
49+
slavePassword: "repl_user",
50+
},
51+
{
52+
host: "localhost",
53+
port: 3306,
54+
username: "root",
55+
password: "password",
56+
database: "casdoor",
57+
// the following two fields are used to create replication user, you don't need to change them
58+
slaveUser: "repl_user",
59+
slavePassword: "repl_user",
60+
},
61+
}
62+
63+
func TestStartMasterSlaveSync(t *testing.T) {
64+
// for example, this is aliyun rds
65+
db0 := newDatabase(&Configs[0])
66+
// for example, this is local mysql instance
67+
db1 := newDatabase(&Configs[1])
68+
69+
createSlaveUser(db0)
70+
// db0 is master, db1 is slave
71+
startSlave(db0, db1)
72+
}
73+
74+
func TestStopMasterSlaveSync(t *testing.T) {
75+
// for example, this is aliyun rds
76+
db0 := newDatabase(&Configs[0])
77+
// for example, this is local mysql instance
78+
db1 := newDatabase(&Configs[1])
79+
80+
stopSlave(db1)
81+
deleteSlaveUser(db0)
82+
}
83+
84+
func TestStartMasterMasterSync(t *testing.T) {
85+
db0 := newDatabase(&Configs[0])
86+
db1 := newDatabase(&Configs[1])
87+
createSlaveUser(db0)
88+
createSlaveUser(db1)
89+
// db0 is master, db1 is slave
90+
startSlave(db0, db1)
91+
// db1 is master, db0 is slave
92+
startSlave(db1, db0)
93+
}
94+
95+
func TestStopMasterMasterSync(t *testing.T) {
96+
db0 := newDatabase(&Configs[0])
97+
db1 := newDatabase(&Configs[1])
98+
stopSlave(db0)
99+
stopSlave(db1)
100+
deleteSlaveUser(db0)
101+
deleteSlaveUser(db1)
102+
}
103+
104+
func TestShowSlaveStatus(t *testing.T) {
105+
db0 := newDatabase(&Configs[0])
106+
db1 := newDatabase(&Configs[1])
107+
slaveStatus(db0)
108+
slaveStatus(db1)
109+
}
110+
111+
func TestShowMasterStatus(t *testing.T) {
112+
db0 := newDatabase(&Configs[0])
113+
db1 := newDatabase(&Configs[1])
114+
masterStatus(db0)
115+
masterStatus(db1)
116+
}

‎sync_v2/db.go

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// Copyright 2023 The Casdoor Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package sync_v2
16+
17+
import (
18+
"fmt"
19+
"log"
20+
21+
"github.com/xorm-io/xorm"
22+
)
23+
24+
type Database struct {
25+
host string
26+
port int
27+
database string
28+
username string
29+
password string
30+
slaveUser string
31+
slavePassword string
32+
engine *xorm.Engine
33+
}
34+
35+
func (db *Database) exec(format string, args ...interface{}) []map[string]string {
36+
sql := fmt.Sprintf(format, args...)
37+
res, err := db.engine.QueryString(sql)
38+
if err != nil {
39+
panic(err)
40+
}
41+
return res
42+
}
43+
44+
func createEngine(dataSourceName string) (*xorm.Engine, error) {
45+
engine, err := xorm.NewEngine("mysql", dataSourceName)
46+
if err != nil {
47+
return nil, err
48+
}
49+
50+
// ping mysql
51+
err = engine.Ping()
52+
if err != nil {
53+
return nil, err
54+
}
55+
56+
engine.ShowSQL(true)
57+
log.Println("mysql connection success")
58+
return engine, nil
59+
}
60+
61+
func newDatabase(db *Database) *Database {
62+
dataSourceName := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", db.username, db.password, db.host, db.port, db.database)
63+
engine, err := createEngine(dataSourceName)
64+
if err != nil {
65+
panic(err)
66+
}
67+
68+
db.engine = engine
69+
return db
70+
}

‎sync_v2/master.go

+89
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
// Copyright 2023 The Casdoor Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package sync_v2
16+
17+
import (
18+
"fmt"
19+
"log"
20+
)
21+
22+
func deleteSlaveUser(masterdb *Database) {
23+
defer func() {
24+
if err := recover(); err != nil {
25+
log.Fatalln(err)
26+
}
27+
}()
28+
masterdb.exec("delete from mysql.user where user = '%v'", masterdb.slaveUser)
29+
masterdb.exec("flush privileges")
30+
}
31+
32+
func createSlaveUser(masterdb *Database) {
33+
res := make([]map[string]string, 0)
34+
defer func() {
35+
if err := recover(); err != nil {
36+
log.Fatalln(err)
37+
}
38+
}()
39+
res = masterdb.exec("show databases")
40+
dbNames := make([]string, 0, len(res))
41+
for _, dbInfo := range res {
42+
dbName := dbInfo["Database"]
43+
dbNames = append(dbNames, dbName)
44+
}
45+
log.Println("dbs in mysql: ", dbNames)
46+
res = masterdb.exec("show tables")
47+
tableNames := make([]string, 0, len(res))
48+
for _, table := range res {
49+
tableName := table[fmt.Sprintf("Tables_in_%v", masterdb.database)]
50+
tableNames = append(tableNames, tableName)
51+
}
52+
log.Printf("tables in %v: %v", masterdb.database, tableNames)
53+
54+
// delete user to prevent user already exists
55+
res = masterdb.exec("delete from mysql.user where user = '%v'", masterdb.slaveUser)
56+
res = masterdb.exec("flush privileges")
57+
58+
// create replication user
59+
res = masterdb.exec("create user '%s'@'%s' identified by '%s'", masterdb.slaveUser, "%", masterdb.slavePassword)
60+
res = masterdb.exec("select host, user from mysql.user where user = '%v'", masterdb.slaveUser)
61+
log.Println("user: ", res[0])
62+
res = masterdb.exec("grant replication slave on *.* to '%s'@'%s'", masterdb.slaveUser, "%")
63+
res = masterdb.exec("flush privileges")
64+
res = masterdb.exec("show grants for '%s'@'%s'", masterdb.slaveUser, "%")
65+
log.Println("grants: ", res[0])
66+
67+
// check env
68+
res = masterdb.exec("show variables like 'server_id'")
69+
log.Println("server_id: ", res[0]["Value"])
70+
res = masterdb.exec("show variables like 'log_bin'")
71+
log.Println("log_bin: ", res[0]["Value"])
72+
res = masterdb.exec("show variables like 'binlog_format'")
73+
log.Println("binlog_format: ", res[0]["Value"])
74+
res = masterdb.exec("show variables like 'binlog_row_image'")
75+
}
76+
77+
func masterStatus(masterdb *Database) {
78+
res := masterdb.exec("show master status")
79+
if len(res) == 0 {
80+
log.Printf("no master status for master [%v:%v]\n", masterdb.host, masterdb.port)
81+
return
82+
}
83+
pos := res[0]["Position"]
84+
file := res[0]["File"]
85+
log.Println("*****check master status*****")
86+
log.Println("master:", masterdb.host, ":", masterdb.port)
87+
log.Println("file:", file, ", position:", pos, ", master status:", res)
88+
log.Println("*****************************")
89+
}

‎sync_v2/slave.go

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// Copyright 2023 The Casdoor Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package sync_v2
16+
17+
import "log"
18+
19+
// slaveStatus shows slave status
20+
func slaveStatus(slavedb *Database) {
21+
res := slavedb.exec("show slave status")
22+
if len(res) == 0 {
23+
log.Printf("no slave status for slave [%v:%v]\n", slavedb.host, slavedb.port)
24+
return
25+
}
26+
log.Println("*****check slave status*****")
27+
log.Println("slave:", slavedb.host, ":", slavedb.port)
28+
masterServerId := res[0]["Master_Server_Id"]
29+
log.Println("master server id:", masterServerId)
30+
lastError := res[0]["Last_Error"]
31+
log.Println("last error:", lastError) // this should be empty
32+
lastIoError := res[0]["Last_IO_Error"]
33+
log.Println("last io error:", lastIoError) // this should be empty
34+
slaveIoState := res[0]["Slave_IO_State"]
35+
log.Println("slave io state:", slaveIoState)
36+
slaveIoRunning := res[0]["Slave_IO_Running"]
37+
log.Println("slave io running:", slaveIoRunning) // this should be Yes
38+
slaveSqlRunning := res[0]["Slave_SQL_Running"]
39+
log.Println("slave sql running:", slaveSqlRunning) // this should be Yes
40+
slaveSqlRunningState := res[0]["Slave_SQL_Running_State"]
41+
log.Println("slave sql running state:", slaveSqlRunningState)
42+
slaveSecondsBehindMaster := res[0]["Seconds_Behind_Master"]
43+
log.Println("seconds behind master:", slaveSecondsBehindMaster) // this should be 0, if not, it means the slave is behind the master
44+
log.Println("slave status:", res)
45+
log.Println("****************************")
46+
}
47+
48+
// stopSlave stops slave
49+
func stopSlave(slavedb *Database) {
50+
defer func() {
51+
if err := recover(); err != nil {
52+
log.Fatalln(err)
53+
}
54+
}()
55+
slavedb.exec("stop slave")
56+
slaveStatus(slavedb)
57+
}
58+
59+
// startSlave starts slave
60+
func startSlave(masterdb *Database, slavedb *Database) {
61+
res := make([]map[string]string, 0)
62+
defer func() {
63+
if err := recover(); err != nil {
64+
log.Fatalln(err)
65+
}
66+
}()
67+
stopSlave(slavedb)
68+
// get the info about master
69+
res = masterdb.exec("show master status")
70+
if len(res) == 0 {
71+
log.Println("no master status")
72+
return
73+
}
74+
pos := res[0]["Position"]
75+
file := res[0]["File"]
76+
log.Println("file:", file, ", position:", pos, ", master status:", res)
77+
res = slavedb.exec("stop slave")
78+
res = slavedb.exec(
79+
"change master to master_host='%v', master_port=%v, master_user='%v', master_password='%v', master_log_file='%v', master_log_pos=%v;",
80+
masterdb.host, masterdb.port, masterdb.slaveUser, masterdb.slavePassword, file, pos,
81+
)
82+
res = slavedb.exec("start slave")
83+
slaveStatus(slavedb)
84+
}

0 commit comments

Comments
 (0)