diff --git a/.github/workflows/cluster_endtoend_mysql80.yml b/.github/workflows/cluster_endtoend_mysql80.yml index 39e2e9f6fc6..368043df402 100644 --- a/.github/workflows/cluster_endtoend_mysql80.yml +++ b/.github/workflows/cluster_endtoend_mysql80.yml @@ -29,33 +29,13 @@ jobs: - name: Get dependencies run: | - - sudo DEBIAN_FRONTEND="noninteractive" apt-get update - - # Uninstall any previously installed MySQL first - sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - - # Install mysql80 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client - - # Install everything else we need, and configure sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" - + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb diff --git a/.github/workflows/cluster_endtoend_upgrade.yml b/.github/workflows/cluster_endtoend_upgrade.yml index 67c9a57c6a5..a95a620b0cb 100644 --- a/.github/workflows/cluster_endtoend_upgrade.yml +++ b/.github/workflows/cluster_endtoend_upgrade.yml @@ -45,6 +45,7 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* diff --git a/.github/workflows/e2e_race.yml b/.github/workflows/e2e_race.yml index 9e33b36bd67..ae963e294e7 100644 --- a/.github/workflows/e2e_race.yml +++ b/.github/workflows/e2e_race.yml @@ -34,6 +34,7 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* diff --git a/.github/workflows/legacy_local_example.yml b/.github/workflows/legacy_local_example.yml index bd3c52c7e2d..cb8c15932ce 100644 --- a/.github/workflows/legacy_local_example.yml +++ b/.github/workflows/legacy_local_example.yml @@ -40,6 +40,7 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* diff --git a/.github/workflows/local_example.yml b/.github/workflows/local_example.yml index e2674a5f64c..be3ba1c5977 100644 --- a/.github/workflows/local_example.yml +++ b/.github/workflows/local_example.yml @@ -40,6 +40,7 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* diff --git a/.github/workflows/region_example.yml b/.github/workflows/region_example.yml index 76f84e15b9f..c88e3592f54 100644 --- a/.github/workflows/region_example.yml +++ b/.github/workflows/region_example.yml @@ -40,6 +40,7 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* diff --git a/.github/workflows/unit_test_mysql80.yml b/.github/workflows/unit_test_mysql80.yml index 80f224e77c2..8670785a3ce 100644 --- a/.github/workflows/unit_test_mysql80.yml +++ b/.github/workflows/unit_test_mysql80.yml @@ -43,6 +43,7 @@ jobs: sudo rm -rf /etc/mysql # mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* diff --git a/go/vt/sqlparser/ast.go b/go/vt/sqlparser/ast.go index 2c77fe8e124..4b7a7f235e2 100644 --- a/go/vt/sqlparser/ast.go +++ b/go/vt/sqlparser/ast.go @@ -64,6 +64,7 @@ type ( AffectedTables() TableNames SetTable(qualifier string, name string) SetFromTables(tables TableNames) + SetFullyParsed(fullyParsed bool) Statement } @@ -72,6 +73,7 @@ type ( iDBDDLStatement() IsFullyParsed() bool GetDatabaseName() string + SetFullyParsed(bool) Statement } @@ -658,41 +660,69 @@ func (*TruncateTable) IsFullyParsed() bool { return true } +// SetFullyParsed implements the DDLStatement interface +func (*TruncateTable) SetFullyParsed(bool) {} + // IsFullyParsed implements the DDLStatement interface func (*RenameTable) IsFullyParsed() bool { return true } +// SetFullyParsed implements the DDLStatement interface +func (node *RenameTable) SetFullyParsed(fullyParsed bool) {} + // IsFullyParsed implements the DDLStatement interface func (node *CreateTable) IsFullyParsed() bool { return node.FullyParsed } +// SetFullyParsed implements the DDLStatement interface +func (node *CreateTable) SetFullyParsed(fullyParsed bool) { + node.FullyParsed = fullyParsed +} + // IsFullyParsed implements the DDLStatement interface func (node *AlterTable) IsFullyParsed() bool { return node.FullyParsed } +// SetFullyParsed implements the DDLStatement interface +func (node *AlterTable) SetFullyParsed(fullyParsed bool) { + node.FullyParsed = fullyParsed +} + // IsFullyParsed implements the DDLStatement interface func (node *CreateView) IsFullyParsed() bool { return true } +// SetFullyParsed implements the DDLStatement interface +func (node *CreateView) SetFullyParsed(fullyParsed bool) {} + // IsFullyParsed implements the DDLStatement interface func (node *DropView) IsFullyParsed() bool { return true } +// SetFullyParsed implements the DDLStatement interface +func (node *DropView) SetFullyParsed(fullyParsed bool) {} + // IsFullyParsed implements the DDLStatement interface func (node *DropTable) IsFullyParsed() bool { return true } +// SetFullyParsed implements the DDLStatement interface +func (node *DropTable) SetFullyParsed(fullyParsed bool) {} + // IsFullyParsed implements the DDLStatement interface func (node *AlterView) IsFullyParsed() bool { return true } +// SetFullyParsed implements the DDLStatement interface +func (node *AlterView) SetFullyParsed(fullyParsed bool) {} + // IsTemporary implements the DDLStatement interface func (*TruncateTable) IsTemporary() bool { return false @@ -1212,16 +1242,29 @@ func (node *DropDatabase) IsFullyParsed() bool { return true } +// SetFullyParsed implements the DBDDLStatement interface +func (node *DropDatabase) SetFullyParsed(fullyParsed bool) {} + // IsFullyParsed implements the DBDDLStatement interface func (node *CreateDatabase) IsFullyParsed() bool { return node.FullyParsed } +// SetFullyParsed implements the DBDDLStatement interface +func (node *CreateDatabase) SetFullyParsed(fullyParsed bool) { + node.FullyParsed = fullyParsed +} + // IsFullyParsed implements the DBDDLStatement interface func (node *AlterDatabase) IsFullyParsed() bool { return node.FullyParsed } +// SetFullyParsed implements the DBDDLStatement interface +func (node *AlterDatabase) SetFullyParsed(fullyParsed bool) { + node.FullyParsed = fullyParsed +} + // GetDatabaseName implements the DBDDLStatement interface func (node *DropDatabase) GetDatabaseName() string { return node.DBName.String() diff --git a/go/vt/sqlparser/parse_test.go b/go/vt/sqlparser/parse_test.go index 053281e05ab..289588ebaf5 100644 --- a/go/vt/sqlparser/parse_test.go +++ b/go/vt/sqlparser/parse_test.go @@ -1016,8 +1016,13 @@ var ( }, { input: "alter table a upgrade partitioning", }, { - input: "alter table a partition by range (id) (partition p0 values less than (10), partition p1 values less than (maxvalue))", - output: "alter table a", + input: "alter table a partition by range (id) (partition p0 values less than (10), partition p1 values less than (maxvalue))", + output: "alter table a", + partialDDL: true, + }, { + input: "create database a garbage values", + output: "create database a", + partialDDL: true, }, { input: "alter table `Post With Space` drop foreign key `Post With Space_ibfk_1`", }, { @@ -1063,15 +1068,17 @@ var ( }, { input: "alter table a add check (ch_1) not enforced", }, { - input: "alter table a drop check ch_1", - output: "alter table a", + input: "alter table a drop check ch_1", + output: "alter table a", + partialDDL: true, }, { input: "alter table a drop foreign key kx", }, { input: "alter table a drop primary key", }, { - input: "alter table a drop constraint", - output: "alter table a", + input: "alter table a drop constraint", + output: "alter table a", + partialDDL: true, }, { input: "alter table a drop id", output: "alter table a drop column id", @@ -1107,13 +1114,16 @@ var ( input: "alter schema d collate = 'utf8_bin' character set = geostd8 character set = geostd8", output: "alter database d collate 'utf8_bin' character set geostd8 character set geostd8", }, { - input: "create table a", + input: "create table a", + partialDDL: true, }, { - input: "CREATE TABLE a", - output: "create table a", + input: "CREATE TABLE a", + output: "create table a", + partialDDL: true, }, { - input: "create table `a`", - output: "create table a", + input: "create table `a`", + output: "create table a", + partialDDL: true, }, { input: "create table function_default (x varchar(25) default (trim(' check ')))", output: "create table function_default (\n\tx varchar(25) default (trim(' check '))\n)", @@ -1146,11 +1156,13 @@ var ( input: "create table if not exists a (\n\t`a` int\n)", output: "create table if not exists a (\n\ta int\n)", }, { - input: "create table a ignore me this is garbage", - output: "create table a", + input: "create table a ignore me this is garbage", + output: "create table a", + partialDDL: true, }, { - input: "create table a (a int, b char, c garbage)", - output: "create table a", + input: "create table a (a int, b char, c garbage)", + output: "create table a", + partialDDL: true, }, { input: "create table a (b1 bool not null primary key, b2 boolean not null)", output: "create table a (\n\tb1 bool not null primary key,\n\tb2 boolean not null\n)", @@ -1818,6 +1830,10 @@ var ( input: "create database test_db default collate 'utf8mb4_general_ci' collate utf8mb4_general_ci", }, { input: "create database test_db character set geostd8", + }, { + input: "alter table corder zzzz zzzz zzzz", + output: "alter table corder", + partialDDL: true, }, { input: "create database test_db character set * unparsable", output: "create database test_db", @@ -1956,9 +1972,9 @@ func TestValid(t *testing.T) { // For mysql 8.0 syntax, the query is not entirely parsed. // Add more structs as we go on adding full parsing support for DDL constructs for 5.7 syntax. switch x := tree.(type) { - case *CreateDatabase: + case DBDDLStatement: assert.Equal(t, !tcase.partialDDL, x.IsFullyParsed()) - case *AlterDatabase: + case DDLStatement: assert.Equal(t, !tcase.partialDDL, x.IsFullyParsed()) } // This test just exercises the tree walking functionality. diff --git a/go/vt/sqlparser/parser.go b/go/vt/sqlparser/parser.go index c06cb662e6c..034cd31f74b 100644 --- a/go/vt/sqlparser/parser.go +++ b/go/vt/sqlparser/parser.go @@ -81,6 +81,12 @@ func Parse2(sql string) (Statement, BindVars, error) { return nil, nil, fmt.Errorf("extra characters encountered after end of DDL: '%s'", string(val)) } log.Warningf("ignoring error parsing DDL '%s': %v", sql, tokenizer.LastError) + switch x := tokenizer.partialDDL.(type) { + case DBDDLStatement: + x.SetFullyParsed(false) + case DDLStatement: + x.SetFullyParsed(false) + } tokenizer.ParseTree = tokenizer.partialDDL return tokenizer.ParseTree, tokenizer.BindVars, nil } diff --git a/go/vt/sqlparser/sql.go b/go/vt/sqlparser/sql.go index c6b79430f96..f4cd773bca1 100644 --- a/go/vt/sqlparser/sql.go +++ b/go/vt/sqlparser/sql.go @@ -8780,9 +8780,9 @@ yydefault: { // Change this to an alter statement if yyDollar[3].colIdent.Lowered() == "primary" { - yyLOCAL = &AlterTable{Table: yyDollar[5].tableName, AlterOptions: append([]AlterOption{&DropKey{Type: PrimaryKeyType}}, yyDollar[6].alterOptionsUnion()...)} + yyLOCAL = &AlterTable{FullyParsed: true, Table: yyDollar[5].tableName, AlterOptions: append([]AlterOption{&DropKey{Type: PrimaryKeyType}}, yyDollar[6].alterOptionsUnion()...)} } else { - yyLOCAL = &AlterTable{Table: yyDollar[5].tableName, AlterOptions: append([]AlterOption{&DropKey{Type: NormalKeyType, Name: yyDollar[3].colIdent}}, yyDollar[6].alterOptionsUnion()...)} + yyLOCAL = &AlterTable{FullyParsed: true, Table: yyDollar[5].tableName, AlterOptions: append([]AlterOption{&DropKey{Type: NormalKeyType, Name: yyDollar[3].colIdent}}, yyDollar[6].alterOptionsUnion()...)} } } yyVAL.union = yyLOCAL diff --git a/go/vt/sqlparser/sql.y b/go/vt/sqlparser/sql.y index 38fd54530f5..cda663a7152 100644 --- a/go/vt/sqlparser/sql.y +++ b/go/vt/sqlparser/sql.y @@ -2331,9 +2331,9 @@ drop_statement: { // Change this to an alter statement if $3.Lowered() == "primary" { - $$ = &AlterTable{Table: $5,AlterOptions: append([]AlterOption{&DropKey{Type:PrimaryKeyType}},$6...)} + $$ = &AlterTable{FullyParsed: true, Table: $5,AlterOptions: append([]AlterOption{&DropKey{Type:PrimaryKeyType}},$6...)} } else { - $$ = &AlterTable{Table: $5,AlterOptions: append([]AlterOption{&DropKey{Type:NormalKeyType, Name:$3}},$6...)} + $$ = &AlterTable{FullyParsed: true, Table: $5,AlterOptions: append([]AlterOption{&DropKey{Type:NormalKeyType, Name:$3}},$6...)} } } | DROP VIEW exists_opt view_name_list restrict_or_cascade_opt diff --git a/test/templates/cluster_endtoend_test.tpl b/test/templates/cluster_endtoend_test.tpl index 8b5dccaa004..2597dd0e452 100644 --- a/test/templates/cluster_endtoend_test.tpl +++ b/test/templates/cluster_endtoend_test.tpl @@ -27,8 +27,15 @@ jobs: - name: Get dependencies run: | + {{- if .Ubuntu20}} + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + {{- end}} sudo apt-get update + {{- if .Ubuntu20}} + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + {{- else }} sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + {{- end}} sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/test/templates/unit_test.tpl b/test/templates/unit_test.tpl index 37a20971d54..6c3095d6b33 100644 --- a/test/templates/unit_test.tpl +++ b/test/templates/unit_test.tpl @@ -62,6 +62,7 @@ jobs: {{if (eq .Platform "mysql80")}} # mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*