diff --git a/.github/workflows/cluster_endtoend_mysql80.yml b/.github/workflows/cluster_endtoend_mysql80.yml index 5579996f715..58969cbf14d 100644 --- a/.github/workflows/cluster_endtoend_mysql80.yml +++ b/.github/workflows/cluster_endtoend_mysql80.yml @@ -34,33 +34,18 @@ jobs: - name: Get dependencies run: | - - sudo DEBIAN_FRONTEND="noninteractive" apt-get update - - # Uninstall any previously installed MySQL first - sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - - # Install mysql80 + # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" - + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download # install JUnit report formatter diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml index a7c17b80b4b..7ac95d82252 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml @@ -29,12 +29,6 @@ jobs: run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range - # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file - run: | - echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - # DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED! - - name: Check out code uses: actions/checkout@v2 diff --git a/.github/workflows/cluster_endtoend_upgrade.yml b/.github/workflows/cluster_endtoend_upgrade.yml index de00f7ede18..52c015c9528 100644 --- a/.github/workflows/cluster_endtoend_upgrade.yml +++ b/.github/workflows/cluster_endtoend_upgrade.yml @@ -63,32 +63,18 @@ jobs: run: | # This prepares general purpose binary dependencies # as well as latest version specific go modules - - sudo DEBIAN_FRONTEND="noninteractive" apt-get update - - # Uninstall any previously installed MySQL first - sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - - # Install mysql80 + # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb diff --git a/.github/workflows/e2e_race.yml b/.github/workflows/e2e_race.yml index 6ac28a859d3..8aaa066f651 100644 --- a/.github/workflows/e2e_race.yml +++ b/.github/workflows/e2e_race.yml @@ -21,33 +21,18 @@ jobs: - name: Get dependencies run: | - - sudo DEBIAN_FRONTEND="noninteractive" apt-get update - - # Uninstall any previously installed MySQL first - sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - - # Install mysql80 + # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" - + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download - name: Run make minimaltools diff --git a/.github/workflows/legacy_local_example.yml b/.github/workflows/legacy_local_example.yml index 9d9c1e04729..3edecc4566a 100644 --- a/.github/workflows/legacy_local_example.yml +++ b/.github/workflows/legacy_local_example.yml @@ -27,32 +27,18 @@ jobs: - name: Get dependencies run: | if [ ${{matrix.os}} = "ubuntu-latest" ]; then - - sudo DEBIAN_FRONTEND="noninteractive" apt-get update - - # Uninstall any previously installed MySQL first - sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - - # Install mysql80 + # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld elif [ ${{matrix.os}} = "macos-latest" ]; then brew install mysql@5.7 make unzip etcd curl git wget fi diff --git a/.github/workflows/local_example.yml b/.github/workflows/local_example.yml index b11ae596270..7c2cfa1630e 100644 --- a/.github/workflows/local_example.yml +++ b/.github/workflows/local_example.yml @@ -27,32 +27,18 @@ jobs: - name: Get dependencies run: | if [ ${{matrix.os}} = "ubuntu-latest" ]; then - - sudo DEBIAN_FRONTEND="noninteractive" apt-get update - - # Uninstall any previously installed MySQL first - sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - - # Install mysql80 + # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld elif [ ${{matrix.os}} = "macos-latest" ]; then brew install mysql@5.7 make unzip etcd curl git wget fi diff --git a/.github/workflows/region_example.yml b/.github/workflows/region_example.yml index 7f2c7eec1c3..e2c097c33b3 100644 --- a/.github/workflows/region_example.yml +++ b/.github/workflows/region_example.yml @@ -27,32 +27,18 @@ jobs: - name: Get dependencies run: | if [ ${{matrix.os}} = "ubuntu-latest" ]; then - - sudo DEBIAN_FRONTEND="noninteractive" apt-get update - - # Uninstall any previously installed MySQL first - sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - - # Install mysql80 + # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld elif [ ${{matrix.os}} = "macos-latest" ]; then brew install mysql@5.7 make unzip etcd curl git wget fi diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go index 243f3c402b6..77a5c331442 100644 --- a/test/ci_workflow_gen.go +++ b/test/ci_workflow_gen.go @@ -32,7 +32,9 @@ const ( unitTestTemplate = "templates/unit_test.tpl" unitTestDatabases = "mysql57, mariadb103, mysql80, mariadb102" - clusterTestTemplate = "templates/cluster_endtoend_test.tpl" + // An empty string will cause the default non platform specific template + // to be used. + clusterTestTemplateFormatStr = "templates/cluster_endtoend_test%s.tpl" unitTestSelfHostedTemplate = "templates/unit_test_self_hosted.tpl" unitTestSelfHostedDatabases = "" @@ -125,7 +127,7 @@ type unitTest struct { } type clusterTest struct { - Name, Shard string + Name, Shard, Platform string MakeTools, InstallXtraBackup bool Ubuntu20 bool } @@ -255,6 +257,7 @@ func generateSelfHostedClusterWorkflows() error { if err != nil { return err } + filePath := fmt.Sprintf("%s/cluster_endtoend_%s.yml", workflowConfigDir, cluster) err = writeFileFromTemplate(clusterTestSelfHostedTemplate, filePath, test) if err != nil { @@ -289,12 +292,18 @@ func generateClusterWorkflows() { for _, ubuntu20Cluster := range ubuntu20Clusters { if ubuntu20Cluster == cluster { test.Ubuntu20 = true + test.Platform = "mysql80" break } } path := fmt.Sprintf("%s/cluster_endtoend_%s.yml", workflowConfigDir, cluster) - err := writeFileFromTemplate(clusterTestTemplate, path, test) + var tplPlatform string + if test.Platform != "" { + tplPlatform = "_" + test.Platform + } + template := fmt.Sprintf(clusterTestTemplateFormatStr, tplPlatform) + err := writeFileFromTemplate(template, path, test) if err != nil { log.Print(err) } diff --git a/test/templates/cluster_endtoend_test_mysql80.tpl b/test/templates/cluster_endtoend_test_mysql80.tpl new file mode 100644 index 00000000000..181ccf18d6b --- /dev/null +++ b/test/templates/cluster_endtoend_test_mysql80.tpl @@ -0,0 +1,99 @@ +name: {{.Name}} +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{"{{"}} github.ref {{"}}"}}, '{{.Name}}') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{`{{ github.event.pull_request.head.sha }}`}}" + +jobs: + build: + name: Run endtoend tests on {{.Name}} + {{if .Ubuntu20}}runs-on: ubuntu-20.04{{else}}runs-on: ubuntu-18.04{{end}} + + steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17 + + - name: Set up python + uses: actions/setup-python@v2 + + - name: Tune the OS + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + - name: Check out code + uses: actions/checkout@v2 + + - name: Get dependencies + run: | + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go get -u github.com/vitessio/go-junit-report@HEAD + + {{if .InstallXtraBackup}} + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get update + sudo apt-get install percona-xtrabackup-24 + + {{end}} + + {{if .MakeTools}} + + - name: Installing zookeeper and consul + run: | + make tools + + {{end}} + + - name: Setup launchable dependencies + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + timeout-minutes: 30 + run: | + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard {{.Shard}} | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt + if: always() +