diff --git a/.github/actions/setup-mysql/action.yml b/.github/actions/setup-mysql/action.yml new file mode 100644 index 00000000000..444c2133a82 --- /dev/null +++ b/.github/actions/setup-mysql/action.yml @@ -0,0 +1,64 @@ +name: 'Setup MySQL' +description: 'Set up MySQL server for use in GitHub Actions workflows' +inputs: + flavor: + description: 'The flavor of MySQL to use (e.g. mysql-5.7, mysql-8.0, mysql-8.4)' + required: true +runs: + using: "composite" + steps: + - name: Setup MySQL + shell: bash + run: | + export DEBIAN_FRONTEND="noninteractive" + sudo apt-get update + + # Uninstall any previously installed MySQL first + # sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + # sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + + # We have to install this old version of libaio1. See also: + # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 + wget http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb && \ + sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb && \ + rm libaio1_0.3.112-13build1_amd64.deb + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb + + if [[ "${{ inputs.flavor }}" == "mysql-5.7" ]]; then + # Bionic packages are still compatible for Jammy since there's no MySQL 5.7 + # packages for Jammy. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + # libtinfo5 is also needed for older MySQL 5.7 builds. + curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6 + elif [[ "${{ inputs.flavor }}" == "mysql-8.0" ]]; then + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + elif [[ "${{ inputs.flavor }}" == "mysql-8.4" ]]; then + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.4-lts | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + else + echo "Unsupported MySQL flavor: ${{ inputs.flavor }}" + exit 1 + fi + + sudo service mysql stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" diff --git a/.github/actions/tune-os/action.yml b/.github/actions/tune-os/action.yml new file mode 100644 index 00000000000..618e3bfedd2 --- /dev/null +++ b/.github/actions/tune-os/action.yml @@ -0,0 +1,22 @@ +name: "Tune OS" +description: "Tune the OS for better performance during CI runs" +runs: + using: "composite" + steps: + - name: Tune the OS + shell: bash + run: | + # Install eatmydata and ensure it's used for every operation + sudo apt-get update && sudo apt-get install -y eatmydata + echo "/usr/\${LIB}/libeatmydata.so" | sudo tee -a /etc/ld.so.preload + + # Increase the range of ephemeral ports. + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + # Don't waste a bunch of time processing man-db triggers + echo "set man-db/auto-update false" | sudo debconf-communicate + sudo dpkg-reconfigure man-db diff --git a/.github/workflows/check_make_vtadmin_authz_testgen.yml b/.github/workflows/check_make_vtadmin_authz_testgen.yml index 2d67332f872..171fbe89775 100644 --- a/.github/workflows/check_make_vtadmin_authz_testgen.yml +++ b/.github/workflows/check_make_vtadmin_authz_testgen.yml @@ -48,20 +48,18 @@ jobs: - 'go/vt/vtadmin/**' - '.github/workflows/check_make_vtadmin_authz_testgen.yml' - - name: Set up Go - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.vtadmin_changes == 'true' - with: - go-version-file: go.mod - - name: Setup GitHub access token if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.vtadmin_changes == 'true' run: git config --global url.https://${{ secrets.GH_ACCESS_TOKEN }}@github.com/.insteadOf https://github.com/ - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.vtadmin_changes == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.vtadmin_changes == 'true' + uses: ./.github/actions/tune-os + + - name: Set up Go + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + if: steps.changes.outputs.vtadmin_changes == 'true' + uses: ./.github/actions/tune-os - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.vtadmin_changes == 'true' diff --git a/.github/workflows/check_make_vtadmin_web_proto.yml b/.github/workflows/check_make_vtadmin_web_proto.yml index 3bee038531c..8610b5bc2f7 100644 --- a/.github/workflows/check_make_vtadmin_web_proto.yml +++ b/.github/workflows/check_make_vtadmin_web_proto.yml @@ -31,6 +31,10 @@ jobs: with: persist-credentials: 'false' + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: ./.github/actions/tune-os + - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 diff --git a/.github/workflows/cluster_endtoend_12.yml b/.github/workflows/cluster_endtoend_12.yml index 99f598c68ca..5a2654c0855 100644 --- a/.github/workflows/cluster_endtoend_12.yml +++ b/.github/workflows/cluster_endtoend_12.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard 12 | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard 12 | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_13.yml b/.github/workflows/cluster_endtoend_13.yml index 8697bc25e9a..ad7934357e2 100644 --- a/.github/workflows/cluster_endtoend_13.yml +++ b/.github/workflows/cluster_endtoend_13.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard 13 | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard 13 | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_15.yml b/.github/workflows/cluster_endtoend_15.yml index 846905e011d..50884c56807 100644 --- a/.github/workflows/cluster_endtoend_15.yml +++ b/.github/workflows/cluster_endtoend_15.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard 15 | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard 15 | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_18.yml b/.github/workflows/cluster_endtoend_18.yml index a91abcb3e15..1ea85299d7c 100644 --- a/.github/workflows/cluster_endtoend_18.yml +++ b/.github/workflows/cluster_endtoend_18.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,43 +82,27 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter @@ -139,7 +114,7 @@ jobs: make tools - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -164,16 +139,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard 18 | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard 18 | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_21.yml b/.github/workflows/cluster_endtoend_21.yml index 96d3befc04a..e30a9382e7b 100644 --- a/.github/workflows/cluster_endtoend_21.yml +++ b/.github/workflows/cluster_endtoend_21.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,43 +82,27 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter @@ -141,7 +116,7 @@ jobs: mv minio /usr/local/bin - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -166,16 +141,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard 21 | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard 21 | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_backup_pitr.yml b/.github/workflows/cluster_endtoend_backup_pitr.yml index 336bc01a518..021d882c149 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard backup_pitr | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard backup_pitr | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml b/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml index 3a46f5ce9e0..eb1c3871e43 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard backup_pitr_mysqlshell | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard backup_pitr_mysqlshell | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml b/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml index 38b57f26277..3d36ad015e5 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,14 +82,8 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -107,28 +92,33 @@ jobs: # Setup Percona Server for MySQL 8.0 sudo apt-get -qq update - sudo apt-get -qq install -y lsb-release gnupg2 curl + sudo apt-get -qq install -y lsb-release gnupg2 wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo percona-release setup ps80 sudo apt-get -qq update - # Install everything else we need, and configure - sudo apt-get -qq install -y percona-server-server percona-server-client make unzip g++ etcd-client etcd-server git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y percona-server-server percona-server-client sudo service mysql stop - sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo apt-get -qq install -y percona-xtrabackup-80 lz4 + + # Install everything else we need, and configure + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 + + sudo service etcd stop + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - sudo apt-get -qq install -y percona-xtrabackup-80 lz4 - - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -153,16 +143,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard backup_pitr_xtrabackup | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard backup_pitr_xtrabackup | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml b/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml index a05929b6e55..54f1ce5f029 100644 --- a/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml +++ b/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -177,16 +152,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard ers_prs_newfeatures_heavy | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard ers_prs_newfeatures_heavy | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_mysql80.yml b/.github/workflows/cluster_endtoend_mysql80.yml index 5519e40f7b1..7b1aad39731 100644 --- a/.github/workflows/cluster_endtoend_mysql80.yml +++ b/.github/workflows/cluster_endtoend_mysql80.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard mysql80 | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard mysql80 | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_mysql_server_vault.yml b/.github/workflows/cluster_endtoend_mysql_server_vault.yml index 21d42af0b8e..6ed11b567a6 100644 --- a/.github/workflows/cluster_endtoend_mysql_server_vault.yml +++ b/.github/workflows/cluster_endtoend_mysql_server_vault.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,43 +82,27 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter @@ -139,7 +114,7 @@ jobs: make tools - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -164,16 +139,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard mysql_server_vault | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard mysql_server_vault | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_onlineddl_revert.yml b/.github/workflows/cluster_endtoend_onlineddl_revert.yml index 479d76a4e38..5c9df5c7d2d 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revert.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revert.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -92,50 +83,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -160,16 +135,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard onlineddl_revert | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard onlineddl_revert | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml index 663610dcfd3..142f317b167 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -92,50 +83,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -160,16 +135,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard onlineddl_scheduler | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard onlineddl_scheduler | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml index 5e60d807c90..d07f9168452 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -92,50 +83,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -168,16 +143,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard onlineddl_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml index fd83b59797e..5c5fc4a421e 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -92,50 +83,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -168,16 +143,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard onlineddl_vrepl_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml index 18a7d7518a0..6b1e7dfd5fd 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -92,50 +83,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -168,16 +143,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_stress_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard onlineddl_vrepl_stress_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml index 3c6379f85f5..ef2e9b90b5d 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -92,50 +83,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -168,16 +143,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard onlineddl_vrepl_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml index 2b66c08fda6..23940ce86e6 100644 --- a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml +++ b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -92,50 +83,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -168,16 +143,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard schemadiff_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard schemadiff_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_tabletmanager_consul.yml b/.github/workflows/cluster_endtoend_tabletmanager_consul.yml index b5616bd8213..cf2baf559f9 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_consul.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_consul.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,43 +82,27 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter @@ -139,7 +114,7 @@ jobs: make tools - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -164,16 +139,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard tabletmanager_consul | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard tabletmanager_consul | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml index 9c9732f58ef..58a66735fd1 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard tabletmanager_tablegc | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard tabletmanager_tablegc | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml b/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml index 6e6a1a9d131..a7e641a41b5 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard tabletmanager_throttler_topo | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard tabletmanager_throttler_topo | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_topo_connection_cache.yml b/.github/workflows/cluster_endtoend_topo_connection_cache.yml index 09642741649..a1e8a87cac4 100644 --- a/.github/workflows/cluster_endtoend_topo_connection_cache.yml +++ b/.github/workflows/cluster_endtoend_topo_connection_cache.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard topo_connection_cache | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard topo_connection_cache | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml index 9726c8707fa..43d8e2e6a8a 100644 --- a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml +++ b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -185,16 +160,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vreplication_across_db_versions | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vreplication_across_db_versions | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vreplication_basic.yml b/.github/workflows/cluster_endtoend_vreplication_basic.yml index de16ec7bb18..3b96b922ac8 100644 --- a/.github/workflows/cluster_endtoend_vreplication_basic.yml +++ b/.github/workflows/cluster_endtoend_vreplication_basic.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -185,16 +160,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vreplication_basic | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vreplication_basic | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml index eb529fc3615..3c9dd3c49f7 100644 --- a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml +++ b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -185,16 +160,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vreplication_cellalias | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vreplication_cellalias | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml b/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml index e9d2792fd13..c6cb067ba69 100644 --- a/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml +++ b/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -185,16 +160,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vreplication_copy_parallel | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vreplication_copy_parallel | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml b/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml index 36052ef082f..851b970e923 100644 --- a/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml +++ b/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -185,16 +160,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vreplication_foreign_key_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vreplication_foreign_key_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml b/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml index 3ae6bc6e69e..55589fba198 100644 --- a/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml +++ b/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -185,16 +160,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vreplication_mariadb_to_mysql | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vreplication_mariadb_to_mysql | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vreplication_migrate.yml b/.github/workflows/cluster_endtoend_vreplication_migrate.yml index a69fa0527c3..bd6de8a4bbe 100644 --- a/.github/workflows/cluster_endtoend_vreplication_migrate.yml +++ b/.github/workflows/cluster_endtoend_vreplication_migrate.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -185,16 +160,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vreplication_migrate | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vreplication_migrate | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml b/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml index df2e8ad9408..d01ac331ad4 100644 --- a/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml +++ b/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -185,16 +160,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vreplication_multi_tenant | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vreplication_multi_tenant | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml b/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml index 1721108e461..e81a72ca8f0 100644 --- a/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml +++ b/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -185,16 +160,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vreplication_partial_movetables_and_materialize | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vreplication_partial_movetables_and_materialize | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vreplication_v2.yml b/.github/workflows/cluster_endtoend_vreplication_v2.yml index 320dc471a51..264689a7812 100644 --- a/.github/workflows/cluster_endtoend_vreplication_v2.yml +++ b/.github/workflows/cluster_endtoend_vreplication_v2.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -185,16 +160,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vreplication_v2 | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vreplication_v2 | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vreplication_vdiff2.yml b/.github/workflows/cluster_endtoend_vreplication_vdiff2.yml index 940bdb43fb9..e2e3fee00c5 100644 --- a/.github/workflows/cluster_endtoend_vreplication_vdiff2.yml +++ b/.github/workflows/cluster_endtoend_vreplication_vdiff2.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -185,16 +160,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vreplication_vdiff2 | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vreplication_vdiff2 | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vreplication_vtctldclient_movetables_tz.yml b/.github/workflows/cluster_endtoend_vreplication_vtctldclient_movetables_tz.yml index 3bf5a2d1888..df42d9d0dd4 100644 --- a/.github/workflows/cluster_endtoend_vreplication_vtctldclient_movetables_tz.yml +++ b/.github/workflows/cluster_endtoend_vreplication_vtctldclient_movetables_tz.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -185,16 +160,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vreplication_vtctldclient_movetables_tz | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vreplication_vtctldclient_movetables_tz | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vstream.yml b/.github/workflows/cluster_endtoend_vstream.yml index 542a4de16b2..623da7ae94b 100644 --- a/.github/workflows/cluster_endtoend_vstream.yml +++ b/.github/workflows/cluster_endtoend_vstream.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vstream | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vstream | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtbackup.yml b/.github/workflows/cluster_endtoend_vtbackup.yml index 3f39954f9b9..cea50d17866 100644 --- a/.github/workflows/cluster_endtoend_vtbackup.yml +++ b/.github/workflows/cluster_endtoend_vtbackup.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtbackup | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtbackup | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml index 7aba0db5f98..ccedb32382d 100644 --- a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -177,16 +152,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtctlbackup_sharded_clustertest_heavy | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtctlbackup_sharded_clustertest_heavy | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml index d9ad265dc68..74b5d823668 100644 --- a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml +++ b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_concurrentdml | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_concurrentdml | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml b/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml index 4cc6696e3d5..a80512b19a2 100644 --- a/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml +++ b/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_foreignkey_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_foreignkey_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_gen4.yml b/.github/workflows/cluster_endtoend_vtgate_gen4.yml index fe65a391c73..883cfb439b8 100644 --- a/.github/workflows/cluster_endtoend_vtgate_gen4.yml +++ b/.github/workflows/cluster_endtoend_vtgate_gen4.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_gen4 | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_gen4 | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml b/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml index 09aaad182c2..29220ef229a 100644 --- a/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -177,16 +152,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_general_heavy | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_general_heavy | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_godriver.yml b/.github/workflows/cluster_endtoend_vtgate_godriver.yml index b92f757cacf..69c4e6b6245 100644 --- a/.github/workflows/cluster_endtoend_vtgate_godriver.yml +++ b/.github/workflows/cluster_endtoend_vtgate_godriver.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_godriver | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_godriver | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml b/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml index b841d9ca87e..483988b8640 100644 --- a/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml +++ b/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_partial_keyspace -partial-keyspace=true | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_partial_keyspace -partial-keyspace=true | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_plantests.yml b/.github/workflows/cluster_endtoend_vtgate_plantests.yml index 1fa1957459b..ac16f6a9bc1 100644 --- a/.github/workflows/cluster_endtoend_vtgate_plantests.yml +++ b/.github/workflows/cluster_endtoend_vtgate_plantests.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_plantests | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_plantests | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_queries.yml b/.github/workflows/cluster_endtoend_vtgate_queries.yml index 4808170c46a..609318f10e9 100644 --- a/.github/workflows/cluster_endtoend_vtgate_queries.yml +++ b/.github/workflows/cluster_endtoend_vtgate_queries.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_queries | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_queries | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml index 317445a3f50..177a8d013e3 100644 --- a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml +++ b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_readafterwrite | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_readafterwrite | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml index e8e7d12ba86..f3d9b0e0b08 100644 --- a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml +++ b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_reservedconn | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_reservedconn | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_schema.yml b/.github/workflows/cluster_endtoend_vtgate_schema.yml index 2104805c593..126087295e9 100644 --- a/.github/workflows/cluster_endtoend_vtgate_schema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_schema.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_schema | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_schema | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml b/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml index 9f5f9533be2..b3fa390bce7 100644 --- a/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml +++ b/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_schema_tracker | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_schema_tracker | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml b/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml index f223633c661..058f3804c55 100644 --- a/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml +++ b/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_tablet_healthcheck_cache | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_tablet_healthcheck_cache | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_topo.yml b/.github/workflows/cluster_endtoend_vtgate_topo.yml index 40bf05f4b71..947a30ec43d 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_topo | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_topo | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml b/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml index cc88c2468a3..7fb82421386 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,43 +82,27 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter @@ -139,7 +114,7 @@ jobs: make tools - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -164,16 +139,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_topo_consul | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_topo_consul | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml b/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml index 1255b0fb384..0de211c4fd4 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_topo_etcd | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_topo_etcd | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_transaction.yml b/.github/workflows/cluster_endtoend_vtgate_transaction.yml index 93cd6987769..afc338dcc16 100644 --- a/.github/workflows/cluster_endtoend_vtgate_transaction.yml +++ b/.github/workflows/cluster_endtoend_vtgate_transaction.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_transaction -build-tag=debug2PC | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_transaction -build-tag=debug2PC | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml index 931ab9ee6eb..c9fee3e908c 100644 --- a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml +++ b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_unsharded | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_unsharded | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml b/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml index 86d3337ada9..9ebc1a117a9 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -177,16 +152,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_vindex_heavy | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_vindex_heavy | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_vschema.yml b/.github/workflows/cluster_endtoend_vtgate_vschema.yml index b6ab90ec570..cc14a0ea46f 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vschema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vschema.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtgate_vschema | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtgate_vschema | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtorc.yml b/.github/workflows/cluster_endtoend_vtorc.yml index 2f62b54306f..b0191f83659 100644 --- a/.github/workflows/cluster_endtoend_vtorc.yml +++ b/.github/workflows/cluster_endtoend_vtorc.yml @@ -38,20 +38,11 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check Memory run: | totalMem=$(free -g | awk 'NR==2 {print $2}') echo "total memory $totalMem GB" - if [[ "$totalMem" -lt 15 ]]; then + if [[ "$totalMem" -lt 15 ]]; then echo "Less memory than required" exit 1 fi @@ -100,50 +91,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -168,16 +143,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtorc | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vtorc | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml b/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml index 3219b1c805e..26e0727b955 100644 --- a/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml +++ b/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,50 +82,34 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' timeout-minutes: 10 run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update - - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y mysql-shell # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -159,16 +134,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vttablet_prscomplex | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard vttablet_prscomplex | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_xb_backup.yml b/.github/workflows/cluster_endtoend_xb_backup.yml index aaedc20e399..29567f35d0f 100644 --- a/.github/workflows/cluster_endtoend_xb_backup.yml +++ b/.github/workflows/cluster_endtoend_xb_backup.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,14 +82,8 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -107,28 +92,33 @@ jobs: # Setup Percona Server for MySQL 8.0 sudo apt-get -qq update - sudo apt-get -qq install -y lsb-release gnupg2 curl + sudo apt-get -qq install -y lsb-release gnupg2 wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo percona-release setup ps80 sudo apt-get -qq update - # Install everything else we need, and configure - sudo apt-get -qq install -y percona-server-server percona-server-client make unzip g++ etcd-client etcd-server git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y percona-server-server percona-server-client sudo service mysql stop - sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo apt-get -qq install -y percona-xtrabackup-80 lz4 + + # Install everything else we need, and configure + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 + + sudo service etcd stop + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - sudo apt-get -qq install -y percona-xtrabackup-80 lz4 - - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -153,16 +143,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard xb_backup | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard xb_backup | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/cluster_endtoend_xb_recovery.yml b/.github/workflows/cluster_endtoend_xb_recovery.yml index 9cd509e0482..d9ad677f026 100644 --- a/.github/workflows/cluster_endtoend_xb_recovery.yml +++ b/.github/workflows/cluster_endtoend_xb_recovery.yml @@ -38,15 +38,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -91,14 +82,8 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -107,28 +92,33 @@ jobs: # Setup Percona Server for MySQL 8.0 sudo apt-get -qq update - sudo apt-get -qq install -y lsb-release gnupg2 curl + sudo apt-get -qq install -y lsb-release gnupg2 wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo percona-release setup ps80 sudo apt-get -qq update - # Install everything else we need, and configure - sudo apt-get -qq install -y percona-server-server percona-server-client make unzip g++ etcd-client etcd-server git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y percona-server-server percona-server-client sudo service mysql stop - sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo apt-get -qq install -y percona-xtrabackup-80 lz4 + + # Install everything else we need, and configure + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 + + sudo service etcd stop + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - sudo apt-get -qq install -y percona-xtrabackup-80 lz4 - - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -153,16 +143,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard xb_recovery | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker=false -follow -shard xb_recovery | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml new file mode 100644 index 00000000000..cea1ce13174 --- /dev/null +++ b/.github/workflows/codecov.yml @@ -0,0 +1,111 @@ +name: Code Coverage +on: + push: + branches: + - "main" + - "release-[0-9]+.[0-9]" + tags: '**' + pull_request: + branches: '**' +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Code Coverage') + cancel-in-progress: true + +permissions: read-all + +jobs: + test: + name: Code Coverage + runs-on: ubuntu-24.04 + + steps: + - name: Check out code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - name: Check for changes in files relevant to code coverage + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + changed_files: + - .github/workflows/codecov.yml + - 'go/**' + - go.mod + - go.sum + - Makefile + + - name: Setup GitHub access token + if: steps.changes.outputs.changed_files == 'true' + run: git config --global url.https://${{ secrets.GH_ACCESS_TOKEN }}@github.com/.insteadOf https://github.com/ + + - name: Set up Go + if: steps.changes.outputs.changed_files == 'true' + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.changes.outputs.changed_files == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.changes.outputs.changed_files == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.changed_files == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + + - name: Get dependencies + if: steps.changes.outputs.changed_files == 'true' + run: | + export DEBIAN_FRONTEND="noninteractive" + sudo apt-get update + + sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk + + mkdir -p dist bin + curl -s -L https://github.com/coreos/etcd/releases/download/v3.5.17/etcd-v3.5.17-linux-amd64.tar.gz | tar -zxC dist + mv dist/etcd-v3.5.17-linux-amd64/{etcd,etcdctl} bin/ + + go mod download + go install golang.org/x/tools/cmd/goimports@latest + + - name: Run make tools + if: steps.changes.outputs.changed_files == 'true' + run: | + make tools + + - name: Run unit tests and generate code coverage reports + if: steps.changes.outputs.changed_files == 'true' + timeout-minutes: 45 + run: | + set -exo pipefail + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + + export NOVTADMINBUILD=1 + + # Exclude endtoend tests from the coverage report. + # TODO: figure out how best to include our endtoend tests in the coverage report. + rm -rf go/test/endtoend go/*/endtoend go/vt/*/endtoend go/cmd/vttestserver + + make unit_test_cover + + # Restore the files we deleted as codecov tries to fix their paths. + git reset --hard HEAD + + - name: Upload coverage reports to codecov.io + if: steps.changes.outputs.changed_files == 'true' + uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # https://github.com/codecov/codecov-action/releases/tag/v5.0.7 + with: + fail_ci_if_error: true + verbose: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/codeql_analysis.yml b/.github/workflows/codeql_analysis.yml new file mode 100644 index 00000000000..c98780ca527 --- /dev/null +++ b/.github/workflows/codeql_analysis.yml @@ -0,0 +1,92 @@ +name: "CodeQL" + +on: + push: + branches: + - main + - release-**.0 + schedule: + - cron: '0 0 * * 1' + workflow_dispatch: + +permissions: read-all + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-24.04 + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go', 'javascript', 'python' ] + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - name: Set up Go + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version-file: go.mod + + - name: Tune the OS + uses: ./.github/actions/tune-os + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify cu stom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + - name: Setup MySQL + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + + - name: Get base dependencies + timeout-minutes: 10 + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget + + sudo service etcd stop + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Building binaries + timeout-minutes: 30 + run: | + source build.env + make build + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + + - name: Slack Workflow Notification + if: ${{ failure() }} + uses: Gamesight/slack-workflow-status@master + with: + repo_token: ${{secrets.GITHUB_TOKEN}} + slack_webhook_url: ${{secrets.SLACK_WEBHOOK_URL}} + channel: '#codeql' + name: 'CodeQL Workflows' + + - name: Fail if needed + if: ${{ failure() }} + run: | + exit 1 diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml new file mode 100644 index 00000000000..55fb4e3f82d --- /dev/null +++ b/.github/workflows/create_release.yml @@ -0,0 +1,70 @@ +# This creates a {tar.gz,deb,rpm} file and uploads it to a release. +# To trigger this, create a new release.. but make sure that you publish +# it immediately and do not save it as a DRAFT. + +name: Release +on: + release: + types: [created] + +permissions: + contents: write + actions: read + +jobs: + build: + name: Create Release + runs-on: ubuntu-24.04 + steps: + - name: Check out code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: 'false' + + - name: Set up Go + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 + with: + go-version-file: go.mod + cache: 'false' + + - name: Tune the OS + uses: ./.github/actions/tune-os + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y make ruby ruby-dev + go mod download + + # We use fpm to package our artifacts, we want to pin the version we use and + # ensure the checksum of the package matches the one published on the package's webpage. + # https://rubygems.org/gems/fpm/versions + - name: Get fpm + run: | + FPM_VERSION=1.16.0 + gem fetch fpm -v $FPM_VERSION + + # Reviewers: The expected checksum MUST ALWAYS match the one published on this website: + # https://rubygems.org/gems/fpm/versions + EXPECTED_CHECKSUM="d9eafe613cfbdf9d3b8ef2e321e194cd0a2d300ce37f716c0be1b3a42b7db5df" + + GOT_CHECKSUM=$(sha256sum fpm-$FPM_VERSION.gem | awk '{print $1}') + + if [[ "$GOT_CHECKSUM" != "$EXPECTED_CHECKSUM" ]]; then + echo "Checksum validation failed" + echo "Expected: $EXPECTED_CHECKSUM" + echo "Got: $GOT_CHECKSUM" + exit 1 + fi + + sudo gem install fpm-$FPM_VERSION.gem + + - name: Make Packages + run: | + ./tools/make-release-packages.sh + + - name: Upload Files + uses: csexton/release-asset-action@3567794e918fa3068116688122a76cdeb57b5f09 # v3.0.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + pattern: "releases/*.{tar.gz,rpm,deb}" diff --git a/.github/workflows/docker_test_cluster.yml b/.github/workflows/docker_test_cluster.yml index 6d94d537c90..b65200910b8 100644 --- a/.github/workflows/docker_test_cluster.yml +++ b/.github/workflows/docker_test_cluster.yml @@ -66,9 +66,8 @@ jobs: run: git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/ - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os - name: Run tests which require docker - 2 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/e2e_race.yml b/.github/workflows/e2e_race.yml index 9b598f57471..bd9e7ed31f6 100644 --- a/.github/workflows/e2e_race.yml +++ b/.github/workflows/e2e_race.yml @@ -61,28 +61,25 @@ jobs: run: git config --global url.https://${{ secrets.GH_ACCESS_TOKEN }}@github.com/.insteadOf https://github.com/ - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download - name: Run make minimaltools diff --git a/.github/workflows/endtoend.yml b/.github/workflows/endtoend.yml index b853847a0c1..c9cc39adef2 100644 --- a/.github/workflows/endtoend.yml +++ b/.github/workflows/endtoend.yml @@ -61,20 +61,23 @@ jobs: run: git config --global url.https://${{ secrets.GH_ACCESS_TOKEN }}@github.com/.insteadOf https://github.com/ - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download - name: Run make minimaltools @@ -95,4 +98,4 @@ jobs: # which musn't be more than 107 characters long. export VTDATAROOT="/tmp/" - eatmydata -- tools/e2e_test_runner.sh + tools/e2e_test_runner.sh diff --git a/.github/workflows/local_example.yml b/.github/workflows/local_example.yml index 47fe89845b3..b864cdb7ebe 100644 --- a/.github/workflows/local_example.yml +++ b/.github/workflows/local_example.yml @@ -70,9 +70,8 @@ jobs: run: git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/ - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.examples == 'true' + uses: ./.github/actions/tune-os - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' @@ -97,4 +96,4 @@ jobs: if [ "${{matrix.os}}" = "macos-latest" ]; then export PATH="/usr/local/opt/mysql@5.7/bin:$PATH" fi - eatmydata -- go run test.go -print-log -follow -retry=1 local_example + go run test.go -print-log -follow -retry=1 local_example diff --git a/.github/workflows/region_example.yml b/.github/workflows/region_example.yml index a2fa96de5fd..51c4d3fe21c 100644 --- a/.github/workflows/region_example.yml +++ b/.github/workflows/region_example.yml @@ -70,9 +70,8 @@ jobs: run: git config --global url.https://$GH_ACCESS_TOKEN@github.com/.insteadOf https://github.com/ - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.examples == 'true' + uses: ./.github/actions/tune-os - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' @@ -97,4 +96,4 @@ jobs: if [ "${{matrix.os}}" = "macos-latest" ]; then export PATH="/usr/local/opt/mysql@5.7/bin:$PATH" fi - eatmydata -- go run test.go -print-log -follow -retry=1 region_example + go run test.go -print-log -follow -retry=1 region_example diff --git a/.github/workflows/static_checks_etc.yml b/.github/workflows/static_checks_etc.yml index 8a02ef719c8..b0313dc721f 100644 --- a/.github/workflows/static_checks_etc.yml +++ b/.github/workflows/static_checks_etc.yml @@ -134,9 +134,8 @@ jobs: run: git config --global url.https://${{ secrets.GH_ACCESS_TOKEN }}@github.com/.insteadOf https://github.com/ - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.go_files == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.go_files == 'true' + uses: ./.github/actions/tune-os - name: Run go fmt if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.go_files == 'true' diff --git a/.github/workflows/unit_race.yml b/.github/workflows/unit_race.yml index ebd46c44c87..d5cd118d76f 100644 --- a/.github/workflows/unit_race.yml +++ b/.github/workflows/unit_race.yml @@ -34,15 +34,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl \ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -85,9 +76,14 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.unit_tests == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.unit_tests == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' @@ -95,16 +91,8 @@ jobs: export DEBIAN_FRONTEND="noninteractive" sudo apt-get -qq update - # mysql80 - sudo apt-get -qq install -y mysql-server mysql-client - - sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata + sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk - sudo service mysql stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" - mkdir -p dist bin curl -s -L https://github.com/coreos/etcd/releases/download/v3.5.17/etcd-v3.5.17-linux-amd64.tar.gz | tar -zxC dist mv dist/etcd-v3.5.17-linux-amd64/{etcd,etcdctl} bin/ @@ -121,7 +109,7 @@ jobs: make tools - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -142,16 +130,17 @@ jobs: export NOVTADMINBUILD=1 export VTEVALENGINETEST="0" - eatmydata -- make unit_test_race | tee -a output.txt | go-junit-report -set-exit-code > report.xml + make unit_test_race | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/unit_race_evalengine.yml b/.github/workflows/unit_race_evalengine.yml index caed5f5f613..4e17d7b65f6 100644 --- a/.github/workflows/unit_race_evalengine.yml +++ b/.github/workflows/unit_race_evalengine.yml @@ -34,15 +34,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl \ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -85,9 +76,14 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.unit_tests == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.unit_tests == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' @@ -95,16 +91,8 @@ jobs: export DEBIAN_FRONTEND="noninteractive" sudo apt-get -qq update - # mysql80 - sudo apt-get -qq install -y mysql-server mysql-client - - sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata + sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk - sudo service mysql stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" - mkdir -p dist bin curl -s -L https://github.com/coreos/etcd/releases/download/v3.5.17/etcd-v3.5.17-linux-amd64.tar.gz | tar -zxC dist mv dist/etcd-v3.5.17-linux-amd64/{etcd,etcdctl} bin/ @@ -121,7 +109,7 @@ jobs: make tools - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -142,16 +130,17 @@ jobs: export NOVTADMINBUILD=1 export VTEVALENGINETEST="1" - eatmydata -- make unit_test_race | tee -a output.txt | go-junit-report -set-exit-code > report.xml + make unit_test_race | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/unit_test_evalengine_mysql80.yml b/.github/workflows/unit_test_evalengine_mysql80.yml index 09e451499b9..0c5ab6f0acb 100644 --- a/.github/workflows/unit_test_evalengine_mysql80.yml +++ b/.github/workflows/unit_test_evalengine_mysql80.yml @@ -37,15 +37,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl \ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -88,41 +79,20 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.unit_tests == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.unit_tests == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' run: | export DEBIAN_FRONTEND="noninteractive" - sudo apt-get update - - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client - - sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata - - sudo service mysql stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk mkdir -p dist bin curl -s -L https://github.com/coreos/etcd/releases/download/v3.5.17/etcd-v3.5.17-linux-amd64.tar.gz | tar -zxC dist @@ -130,7 +100,7 @@ jobs: go mod download go install golang.org/x/tools/cmd/goimports@latest - + # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -140,7 +110,7 @@ jobs: make tools - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -165,17 +135,18 @@ jobs: # We sometimes need to alter the behavior based on the platform we're # testing, e.g. MySQL 5.7 vs 8.0. export CI_DB_PLATFORM="mysql80" - - eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/unit_test_evalengine_mysql84.yml b/.github/workflows/unit_test_evalengine_mysql84.yml index 3e16a791771..368d876af62 100644 --- a/.github/workflows/unit_test_evalengine_mysql84.yml +++ b/.github/workflows/unit_test_evalengine_mysql84.yml @@ -37,15 +37,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl \ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -88,43 +79,20 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.unit_tests == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.unit_tests == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.4 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' run: | export DEBIAN_FRONTEND="noninteractive" - sudo apt-get update - - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.4-lts | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client - - sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata - - sudo service mysql stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk mkdir -p dist bin curl -s -L https://github.com/coreos/etcd/releases/download/v3.5.17/etcd-v3.5.17-linux-amd64.tar.gz | tar -zxC dist @@ -132,7 +100,7 @@ jobs: go mod download go install golang.org/x/tools/cmd/goimports@latest - + # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -142,7 +110,7 @@ jobs: make tools - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -167,17 +135,18 @@ jobs: # We sometimes need to alter the behavior based on the platform we're # testing, e.g. MySQL 5.7 vs 8.0. export CI_DB_PLATFORM="mysql84" - - eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/unit_test_mysql80.yml b/.github/workflows/unit_test_mysql80.yml index 4eb2d4c7eb0..e1c8523db04 100644 --- a/.github/workflows/unit_test_mysql80.yml +++ b/.github/workflows/unit_test_mysql80.yml @@ -37,15 +37,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl \ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -88,41 +79,20 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.unit_tests == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.unit_tests == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' run: | export DEBIAN_FRONTEND="noninteractive" - sudo apt-get update - - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client - - sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata - - sudo service mysql stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk mkdir -p dist bin curl -s -L https://github.com/coreos/etcd/releases/download/v3.5.17/etcd-v3.5.17-linux-amd64.tar.gz | tar -zxC dist @@ -130,7 +100,7 @@ jobs: go mod download go install golang.org/x/tools/cmd/goimports@latest - + # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -140,7 +110,7 @@ jobs: make tools - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -165,17 +135,18 @@ jobs: # We sometimes need to alter the behavior based on the platform we're # testing, e.g. MySQL 5.7 vs 8.0. export CI_DB_PLATFORM="mysql80" - - eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/unit_test_mysql84.yml b/.github/workflows/unit_test_mysql84.yml index e2c73d3f207..ef351f78b13 100644 --- a/.github/workflows/unit_test_mysql84.yml +++ b/.github/workflows/unit_test_mysql84.yml @@ -37,15 +37,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl \ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -88,43 +79,20 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.unit_tests == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.changes.outputs.unit_tests == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.4 + - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' run: | export DEBIAN_FRONTEND="noninteractive" - sudo apt-get update - - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.4-lts | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client - - sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata - - sudo service mysql stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk mkdir -p dist bin curl -s -L https://github.com/coreos/etcd/releases/download/v3.5.17/etcd-v3.5.17-linux-amd64.tar.gz | tar -zxC dist @@ -132,7 +100,7 @@ jobs: go mod download go install golang.org/x/tools/cmd/goimports@latest - + # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -142,7 +110,7 @@ jobs: make tools - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -167,17 +135,18 @@ jobs: # We sometimes need to alter the behavior based on the platform we're # testing, e.g. MySQL 5.7 vs 8.0. export CI_DB_PLATFORM="mysql84" - - eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml index 474c8d91fc6..b2d2d4cc3d3 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml @@ -88,21 +88,25 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get base dependencies timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | - sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata - - sudo service mysql stop + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep + sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download # install JUnit report formatter @@ -189,7 +193,7 @@ jobs: mkdir -p /tmp/vtdataroot set -x source build.env - eatmydata -- go run test.go -skip-build -docker=false -print-log -follow -tag upgrade_downgrade_backups + go run test.go -skip-build -docker=false -print-log -follow -tag upgrade_downgrade_backups # Swap binaries again, use current version's VTTablet, and last release's VTBackup - name: Use current version VTTablet, and other version VTBackup @@ -213,4 +217,4 @@ jobs: mkdir -p /tmp/vtdataroot set -x source build.env - eatmydata -- go run test.go -skip-build -keep-data -docker=false -print-log -follow -tag upgrade_downgrade_backups + go run test.go -skip-build -keep-data -docker=false -print-log -follow -tag upgrade_downgrade_backups diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml index 76e028e3ed2..5756e543e44 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml @@ -87,21 +87,25 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get base dependencies timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | - sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata - - sudo service mysql stop + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep + sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download # install JUnit report formatter @@ -178,7 +182,7 @@ jobs: mkdir -p /tmp/vtdataroot set -x source build.env - eatmydata -- go run test.go -skip-build -docker=false -print-log -follow -tag upgrade_downgrade_backups + go run test.go -skip-build -docker=false -print-log -follow -tag upgrade_downgrade_backups # Swap binaries again, use current version's VTTablet, and next release's VTBackup - name: Use current version VTTablet, and other version VTBackup @@ -202,4 +206,4 @@ jobs: mkdir -p /tmp/vtdataroot set -x source build.env - eatmydata -- go run test.go -skip-build -keep-data -docker=false -print-log -follow -tag upgrade_downgrade_backups + go run test.go -skip-build -keep-data -docker=false -print-log -follow -tag upgrade_downgrade_backups diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual.yml b/.github/workflows/upgrade_downgrade_test_backups_manual.yml index 0e19a8328f9..4e3b1b83ea6 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_manual.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_manual.yml @@ -92,31 +92,25 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get base dependencies timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update - # Install MySQL 8.0 - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata grep + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep - sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml index c4729be726a..ef8a592378b 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml @@ -90,9 +90,14 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get base dependencies timeout-minutes: 10 @@ -100,22 +105,10 @@ jobs: run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update - # Install MySQL 8.0 - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client - # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata grep + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep - sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD diff --git a/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml b/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml index 0cd4a582762..498f5abb644 100644 --- a/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml +++ b/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml @@ -100,38 +100,24 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get base dependencies timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - # Install mysql80 - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget - sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -242,7 +228,7 @@ jobs: source build.env export PRIMARY_TABLET_BINARY_SUFFIX="" export REPLICA_TABLET_BINARY_SUFFIX="-last" - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow # Running a test with primary tablet at version n-1 and replica vttablet at version n (current SHA) - name: Run Online DDL tests (primary=N-1, replica=N) @@ -254,7 +240,7 @@ jobs: source build.env export PRIMARY_TABLET_BINARY_SUFFIX="-last" export REPLICA_TABLET_BINARY_SUFFIX="" - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow # Running a test with primary tablet at version n+1 and replica vttablet at version n (current SHA) - name: Run Online DDL tests (primary=N+1, replica=N) @@ -266,7 +252,7 @@ jobs: source build.env export PRIMARY_TABLET_BINARY_SUFFIX="-next" export REPLICA_TABLET_BINARY_SUFFIX="" - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow # Running a test with primary tablet at version n (current SHA) and replica vttablet at version n+1 - name: Run Online DDL tests (primary=N, replica=N+1) @@ -278,4 +264,4 @@ jobs: source build.env export PRIMARY_TABLET_BINARY_SUFFIX="" export REPLICA_TABLET_BINARY_SUFFIX="-next" - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml index 6691ffe1d33..6eb0b9a5f3f 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml @@ -89,36 +89,24 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get base dependencies timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - # Install mysql80 - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget - sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -193,7 +181,7 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries # Swap the binaries again. This time, vtgate will be at version n, and vttablet/vtctld will be at version n-1 - name: Use current version VTGate, and other version VTTablet/VTctld @@ -223,4 +211,4 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries_2.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries_2.yml index d17b381f235..fc850bf16fa 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_queries_2.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries_2.yml @@ -89,36 +89,24 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get base dependencies timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - # Install mysql80 - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget - sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -193,7 +181,7 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 # Swap the binaries again. This time, vtgate will be at version n, and vttablet/vtctld will be at version n-1 - name: Use current version VTGate, and other version VTTablet/VTctld @@ -223,4 +211,4 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries_2_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries_2_next_release.yml index b30cd739a1e..2360a14cc60 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_queries_2_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries_2_next_release.yml @@ -90,38 +90,24 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get base dependencies timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - # Install mysql80 - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget - sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -193,7 +179,7 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 # Swap the binaries again. This time, vtgate will be at version n, and vttablet will be at version n+1 - name: Use current version VTGate, and other version VTTablet @@ -217,4 +203,4 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries_2 diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml index 4b804b6610a..3844fcf8b6f 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml @@ -90,38 +90,24 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get base dependencies timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - # Install mysql80 - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget - sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -193,7 +179,7 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries # Swap the binaries again. This time, vtgate will be at version n, and vttablet will be at version n+1 - name: Use current version VTGate, and other version VTTablet @@ -217,4 +203,4 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml b/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml index 49321221d81..e3d72fa1f38 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml @@ -89,38 +89,24 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get base dependencies timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - # Install mysql80 - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget - sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -199,7 +185,7 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_schema + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_schema # Swap the binaries again. This time, vtgate will be at version n, and vttablet will be at version n-1 - name: Use current version VTGate, and other version VTTablet @@ -223,4 +209,4 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_schema + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_schema diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml index c2b43a6d6c1..f3156a392fc 100644 --- a/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml @@ -90,38 +90,24 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + - name: Setup MySQL + if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + - name: Get base dependencies timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - # Install mysql80 - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget - sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -194,7 +180,7 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_schema + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_schema # Swap the binaries again. This time, vtgate will be at version n, and vttablet will be at version n+1 - name: Use current version VTGate, and other version VTTablet @@ -218,4 +204,4 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_schema + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_schema diff --git a/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml b/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml index 3ac10fb911c..cdfb3c69860 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml @@ -90,38 +90,24 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get base dependencies timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - # Install mysql80 - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget - sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -192,4 +178,4 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_reparent + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_reparent diff --git a/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml b/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml index df5817b8c2d..3539b208796 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml @@ -90,38 +90,24 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get base dependencies timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - # Install mysql80 - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget - sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -198,4 +184,4 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_reparent + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_reparent diff --git a/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml b/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml index c8afb3ab001..a7a5332c548 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml @@ -89,36 +89,24 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get base dependencies timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - # Install mysql80 - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget - sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -195,4 +183,4 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_reparent + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_reparent diff --git a/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml b/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml index d1ab2cae897..0755f5cb1c8 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml @@ -89,38 +89,24 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get base dependencies timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - # Install mysql80 - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget - sudo service mysql stop sudo service etcd stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -196,4 +182,4 @@ jobs: mkdir -p /tmp/vtdataroot source build.env - eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_reparent + go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_reparent diff --git a/.github/workflows/upgrade_downgrade_test_semi_sync.yml b/.github/workflows/upgrade_downgrade_test_semi_sync.yml index daba61db555..1326f3ec7b3 100644 --- a/.github/workflows/upgrade_downgrade_test_semi_sync.yml +++ b/.github/workflows/upgrade_downgrade_test_semi_sync.yml @@ -85,21 +85,25 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get base dependencies timeout-minutes: 10 if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | - sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata - - sudo service mysql stop + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep + sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download # install JUnit report formatter diff --git a/.github/workflows/vitess_tester_vtgate.yml b/.github/workflows/vitess_tester_vtgate.yml index eb687f9770b..7a735b42a48 100644 --- a/.github/workflows/vitess_tester_vtgate.yml +++ b/.github/workflows/vitess_tester_vtgate.yml @@ -37,15 +37,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -90,32 +81,25 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter @@ -125,7 +109,7 @@ jobs: go install github.com/vitessio/vt/go/vt@e43009309f599378504905d4b804460f47822ac5 - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -163,14 +147,15 @@ jobs: i=$((i+1)) done - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat report*.xml diff --git a/.github/workflows/vtadmin_web_build.yml b/.github/workflows/vtadmin_web_build.yml index ca31e5b6349..ab50c17d2bc 100644 --- a/.github/workflows/vtadmin_web_build.yml +++ b/.github/workflows/vtadmin_web_build.yml @@ -40,6 +40,9 @@ jobs: with: persist-credentials: 'false' + - name: Tune the OS + uses: ./.github/actions/tune-os + - uses: actions/setup-node@v4 if: steps.skip-workflow.outputs.skip-workflow == 'false' with: diff --git a/.github/workflows/vtadmin_web_lint.yml b/.github/workflows/vtadmin_web_lint.yml index 0c6410012a6..138264c2880 100644 --- a/.github/workflows/vtadmin_web_lint.yml +++ b/.github/workflows/vtadmin_web_lint.yml @@ -40,6 +40,9 @@ jobs: with: persist-credentials: 'false' + - name: Tune the OS + uses: ./.github/actions/tune-os + - uses: actions/setup-node@v4 if: steps.skip-workflow.outputs.skip-workflow == 'false' with: diff --git a/.github/workflows/vtadmin_web_unit_tests.yml b/.github/workflows/vtadmin_web_unit_tests.yml index 751bca4899c..59bf3400bc8 100644 --- a/.github/workflows/vtadmin_web_unit_tests.yml +++ b/.github/workflows/vtadmin_web_unit_tests.yml @@ -40,6 +40,9 @@ jobs: with: persist-credentials: 'false' + - name: Tune the OS + uses: ./.github/actions/tune-os + - uses: actions/setup-node@v4 if: steps.skip-workflow.outputs.skip-workflow == 'false' with: diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml index 77d6cfa5d3d..c25bf968f2d 100644 --- a/.github/workflows/vtop_example.yml +++ b/.github/workflows/vtop_example.yml @@ -29,15 +29,6 @@ jobs: fi echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - - if [[ "${{github.event.pull_request}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -78,15 +69,14 @@ jobs: run: git config --global url.https://${{ secrets.GH_ACCESS_TOKEN }}@github.com/.insteadOf https://github.com/ - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | # Install everything we need, and configure - sudo apt-get install -y eatmydata make + sudo apt-get install -y make go mod download # needed for vtctldclient @@ -105,4 +95,4 @@ jobs: timeout-minutes: 60 run: | source build.env - eatmydata -- go run test.go -docker=false -skip-build -print-log -follow -retry=1 -timeout=60m vtop_example + go run test.go -docker=false -skip-build -print-log -follow -retry=1 -timeout=60m vtop_example diff --git a/test/templates/cluster_endtoend_test.tpl b/test/templates/cluster_endtoend_test.tpl index a717c26485e..1d2580177a9 100644 --- a/test/templates/cluster_endtoend_test.tpl +++ b/test/templates/cluster_endtoend_test.tpl @@ -36,22 +36,13 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "{{"${{github.event.pull_request}}"}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "{{"Authorization: token ${{ secrets.GITHUB_TOKEN }}"}}" \ - -H "Accept: application/vnd.github.v3+json" \ - "{{"https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}"}}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - {{if .MemoryCheck}} - name: Check Memory run: | totalMem=$(free -g | awk 'NR==2 {print $2}') echo "total memory $totalMem GB" - if [[ "$totalMem" -lt 15 ]]; then + if [[ "$totalMem" -lt 15 ]]; then echo "Less memory than required" exit 1 fi @@ -107,14 +98,16 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + {{if not .InstallXtraBackup}} + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 + {{ end }} - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -124,53 +117,37 @@ jobs: # Setup Percona Server for MySQL 8.0 sudo apt-get -qq update - sudo apt-get -qq install -y lsb-release gnupg2 curl + sudo apt-get -qq install -y lsb-release gnupg2 wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo percona-release setup ps80 sudo apt-get -qq update - # Install everything else we need, and configure - sudo apt-get -qq install -y percona-server-server percona-server-client make unzip g++ etcd-client etcd-server git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y percona-server-server percona-server-client - {{else}} + sudo service mysql stop - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get -qq update + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - # We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb + sudo apt-get -qq install -y percona-xtrabackup-80 lz4 - # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + {{else}} + + sudo apt-get -qq install -y mysql-shell {{end}} - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 + sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - {{if .InstallXtraBackup}} - - sudo apt-get -qq install -y percona-xtrabackup-80 lz4 - - {{end}} - {{if .NeedsMinio }} - name: Install Minio if: steps.skip-workflow.outputs.skip-workflow == 'false' @@ -190,7 +167,7 @@ jobs: {{end}} - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -247,16 +224,17 @@ jobs: # Some of these tests require specific locales to be installed. # See https://github.com/cncf/automation/commit/49f2ad7a791a62ff7d038002bbb2b1f074eed5d5 # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker={{if .Docker}}true -flavor={{.Platform}}{{else}}false{{end}} -follow -shard {{.Shard}}{{if .PartialKeyspace}} -partial-keyspace=true {{end}}{{if .BuildTag}} -build-tag={{.BuildTag}} {{end}} | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker={{if .Docker}}true -flavor={{.Platform}}{{else}}false{{end}} -follow -shard {{.Shard}}{{if .PartialKeyspace}} -partial-keyspace=true {{end}}{{if .BuildTag}} -build-tag={{.BuildTag}} {{end}} | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "{{"${{steps.skip-workflow.outputs.is_draft}}"}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/test/templates/cluster_endtoend_test_mysql57.tpl b/test/templates/cluster_endtoend_test_mysql57.tpl index d5bbfa5053c..138f6fd1376 100644 --- a/test/templates/cluster_endtoend_test_mysql57.tpl +++ b/test/templates/cluster_endtoend_test_mysql57.tpl @@ -40,20 +40,13 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - PR_DATA=$(curl \ - -H "{{"Authorization: token ${{ secrets.GITHUB_TOKEN }}"}}" \ - -H "Accept: application/vnd.github.v3+json" \ - "{{"https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}"}}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - {{if .MemoryCheck}} - name: Check Memory run: | totalMem=$(free -g | awk 'NR==2 {print $2}') echo "total memory $totalMem GB" - if [[ "$totalMem" -lt 15 ]]; then + if [[ "$totalMem" -lt 15 ]]; then echo "Less memory than required" exit 1 fi @@ -109,48 +102,22 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-5.7 - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | sudo apt-get update - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - # Bionic packages are still compatible for Jammy since there's no MySQL 5.7 - # packages for Jammy. - echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections - echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - # We have to install this old version of libaio1. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6 + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata - - sudo service mysql stop sudo service etcd stop # install JUnit report formatter @@ -183,7 +150,7 @@ jobs: {{end}} - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -227,16 +194,17 @@ jobs: {{end}} # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker={{if .Docker}}true -flavor={{.Platform}}{{else}}false{{end}} -follow -shard {{.Shard}}{{if .PartialKeyspace}} -partial-keyspace=true {{end}} | tee -a output.txt | go-junit-report -set-exit-code > report.xml + go run test.go -docker={{if .Docker}}true -flavor={{.Platform}}{{else}}false{{end}} -follow -shard {{.Shard}}{{if .PartialKeyspace}} -partial-keyspace=true {{end}} | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "{{"${{steps.skip-workflow.outputs.is_draft}}"}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt diff --git a/test/templates/cluster_vitess_tester.tpl b/test/templates/cluster_vitess_tester.tpl index a743b19b99f..363691d9c75 100644 --- a/test/templates/cluster_vitess_tester.tpl +++ b/test/templates/cluster_vitess_tester.tpl @@ -35,15 +35,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "{{"${{github.event.pull_request}}"}}" != "" ]]; then - PR_DATA=$(curl -s\ - -H "{{"Authorization: token ${{ secrets.GITHUB_TOKEN }}"}}" \ - -H "Accept: application/vnd.github.v3+json" \ - "{{"https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}"}}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -90,32 +81,25 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Limit local port range to not use ports that overlap with server side - # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.end_to_end == 'true' + uses: ./.github/actions/setup-mysql + with: + flavor: mysql-8.0 - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update + # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6 + sudo apt-get -qq install -y make unzip g++ etcd-client etcd-server curl git wget xz-utils libncurses6 - sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download # install JUnit report formatter @@ -125,7 +109,7 @@ jobs: go install github.com/vitessio/vt/go/vt@e43009309f599378504905d4b804460f47822ac5 - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -163,14 +147,15 @@ jobs: i=$((i+1)) done - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' && always() run: | - if [[ "{{"${{steps.skip-workflow.outputs.is_draft}}"}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat report*.xml diff --git a/test/templates/unit_test.tpl b/test/templates/unit_test.tpl index 135c9652d37..41f2a157aff 100644 --- a/test/templates/unit_test.tpl +++ b/test/templates/unit_test.tpl @@ -35,15 +35,6 @@ jobs: echo Skip ${skip} echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT - if [[ "{{"${{github.event.pull_request}}"}}" != "" ]]; then - PR_DATA=$(curl \ - -H "{{"Authorization: token ${{ secrets.GITHUB_TOKEN }}"}}" \ - -H "Accept: application/vnd.github.v3+json" \ - "{{"https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}"}}") - draft=$(echo "$PR_DATA" | jq .draft -r) - echo "is_draft=${draft}" >> $GITHUB_OUTPUT - fi - - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -88,67 +79,28 @@ jobs: uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' - run: | - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" - # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio - echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p /etc/sysctl.conf + if: steps.changes.outputs.unit_tests == 'true' + uses: ./.github/actions/tune-os + + - name: Setup MySQL + if: steps.changes.outputs.unit_tests == 'true' + uses: ./.github/actions/setup-mysql + with: + {{ if (eq .Platform "mysql57") -}} + flavor: mysql-5.7 + {{ end }} + {{- if (eq .Platform "mysql80") -}} + flavor: mysql-8.0 + {{ end }} + {{- if (eq .Platform "mysql84") -}} + flavor: mysql-8.4 + {{ end }} - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' run: | export DEBIAN_FRONTEND="noninteractive" - sudo apt-get update - - # Uninstall any previously installed MySQL first - # sudo systemctl stop apparmor - sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common - sudo apt-get -y autoremove - sudo apt-get -y autoclean - # sudo deluser mysql - sudo rm -rf /var/lib/mysql - sudo rm -rf /etc/mysql - - # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.35-1_all.deb - - {{if (eq .Platform "mysql57")}} - # Bionic packages are still compatible for Jammy since there's no MySQL 5.7 - # packages for Jammy. - echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections - echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - # We have to install this old version of libaio1. See also: - # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb - sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb - # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6 - {{end}} - - {{if (eq .Platform "mysql80")}} - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client - {{end}} - - {{if (eq .Platform "mysql84")}} - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.4-lts | sudo debconf-set-selections - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client - {{end}} - - sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata - - sudo service mysql stop - sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk mkdir -p dist bin curl -s -L https://github.com/coreos/etcd/releases/download/v3.5.17/etcd-v3.5.17-linux-amd64.tar.gz | tar -zxC dist @@ -156,7 +108,7 @@ jobs: go mod download go install golang.org/x/tools/cmd/goimports@latest - + # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD @@ -166,7 +118,7 @@ jobs: make tools - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' run: | # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up pip3 install --user launchable~=1.0 > /dev/null @@ -191,17 +143,18 @@ jobs: # We sometimes need to alter the behavior based on the platform we're # testing, e.g. MySQL 5.7 vs 8.0. export CI_DB_PLATFORM="{{.Platform}}" - - eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - name: Print test output and Record test result in launchable if PR is not a draft - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Record test results in launchable if PR is not a draft + if: github.event_name == 'pull_request' && github.event.pull_request.draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' && always() run: | - if [[ "{{"${{steps.skip-workflow.outputs.is_draft}}"}}" == "false" ]]; then - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - fi + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + - name: Print test output + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | # print test output cat output.txt