[release-16.0] Examples: Add lib functions to wait for shard states (#12239)

.github/workflows/region_example.yml

@@ -105,5 +105,4 @@ jobs:
         if [ ${{matrix.os}} = "macos-latest" ]; then
           export PATH="/usr/local/opt/mysql@5.7/bin:$PATH"
         fi
-        sed -i 's/user\/my-vitess/runner\/work\/vitess\/vitess/g' examples/region_sharding/main_vschema_sharded.json #set correct path to countries.json
         eatmydata -- go run test.go -print-log -follow -retry=1 region_example

.github/workflows/upgrade_downgrade_test_backups_manual.yml

@@ -191,18 +191,18 @@ jobs:
     # We also insert a few rows in our three tables.
     - name: Create the example Vitess cluster with all components using version N
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/start_cluster.sh
+        source build.env ; cd examples/backups
+        ./start_cluster.sh
 
     # Taking a backup
     - name: Take a backup of all the shards
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 2
+      timeout-minutes: 5
       run: |
-        source build.env ; cd examples/local
-        ./backups/take_backups.sh
+        source build.env ; cd examples/backups
+        ./take_backups.sh
 
     # We insert more data in every table after the backup.
     # When we restore the backup made in the previous step, we do not want to see the rows we are about to insert now.
@@ -214,7 +214,7 @@ jobs:
     - name: Insert more data after the backup
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples; source ./common/env.sh
 
         echo "insert into customer(email) values('new_user_1@domain.com');" | mysql
         echo "insert into product(sku, description, price) values('SKU-1009', 'description', 89);" | mysql
@@ -223,10 +223,10 @@ jobs:
     # Stop all the tablets and remove their data
     - name: Stop tablets
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/stop_tablets.sh
+        source build.env ; cd examples/backups
+        ./stop_tablets.sh
 
     # We downgrade: we use the version N-1 of vttablet
     - name: Downgrade - Swap binaries, use VTTablet N-1
@@ -241,18 +241,18 @@ jobs:
     # Starting the tablets again, they will automatically start restoring the last backup.
     - name: Start new tablets and restore
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/restart_tablets.sh
+        source build.env ; cd examples/backups
+        ./restart_tablets.sh
         # give enough time to the tablets to restore the backup
-        sleep 60
+        sleep 90
 
     # Count the number of rows in each table to make sure the restoration is successful.
     - name: Assert the number of rows in every table
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples ; source ./common/env.sh
 
         echo "select count(sku) from product;" | mysql 2>&1| grep 2
         echo "select count(email) from customer;" | mysql 2>&1| grep 5
@@ -262,7 +262,7 @@ jobs:
     - name: Insert more rows in the tables
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples ; source ./common/env.sh
 
         echo "insert into customer(email) values('new_user_2@domain.com');" | mysql
         echo "insert into product(sku, description, price) values('SKU-1011', 'description', 111);" | mysql
@@ -271,18 +271,18 @@ jobs:
     # Taking a second backup of the cluster.
     - name: Take a second backup of all the shards
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 2
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/take_backups.sh
+        source build.env ; cd examples/backups
+        ./take_backups.sh
 
     # Stopping the tablets so we can perform the upgrade.
     - name: Stop tablets
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/stop_tablets.sh
+        source build.env ; cd examples/backups
+        ./stop_tablets.sh
 
     # We upgrade: we swap binaries and use the version N of the tablet.
     - name: Upgrade - Swap binaries, use VTTablet N
@@ -297,18 +297,18 @@ jobs:
     # Starting the tablets again and restoring the previous backup.
     - name: Start new tablets and restore
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/restart_tablets.sh
+        source build.env ; cd examples/backups
+        ./restart_tablets.sh
         # give enough time to the tablets to restore the backup
-        sleep 60
+        sleep 90
 
     # We count the number of rows in every table to check that the restore step was successful.
     - name: Assert the number of rows in every table
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples ; source ./common/env.sh
 
         echo "select count(sku) from product;" | mysql 2>&1| grep 3
         echo "select count(email) from customer;" | mysql 2>&1| grep 6

.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml

@@ -194,18 +194,18 @@ jobs:
     # We also insert a few rows in our three tables.
     - name: Create the example Vitess cluster with all components using version N
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/start_cluster.sh
+        source build.env ; cd examples/backups
+        ./start_cluster.sh
 
     # Taking a backup
     - name: Take a backup of all the shards
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 2
+      timeout-minutes: 5
       run: |
-        source build.env ; cd examples/local
-        ./backups/take_backups.sh
+        source build.env ; cd examples/backups
+        ./take_backups.sh
 
     # We insert more data in every table after the backup.
     # When we restore the backup made in the next step, we do not want to see the rows we are about to insert now.
@@ -217,7 +217,7 @@ jobs:
     - name: Insert more data after the backup
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples ; source ./common/env.sh
 
         echo "insert into customer(email) values('new_user_1@domain.com');" | mysql
         echo "insert into product(sku, description, price) values('SKU-1009', 'description', 89);" | mysql
@@ -226,10 +226,10 @@ jobs:
     # Stop all the tablets and remove their data
     - name: Stop tablets
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/stop_tablets.sh
+        source build.env ; cd examples/backups
+        ./stop_tablets.sh
 
     # We downgrade: we use the version N+1 of vttablet
     - name: Downgrade - Swap binaries, use VTTablet N+1
@@ -244,18 +244,18 @@ jobs:
     # Starting the tablets again, they will automatically start restoring the last backup.
     - name: Start new tablets and restore
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/restart_tablets.sh
+        source build.env ; cd examples/backups
+        ./restart_tablets.sh
         # give enough time to the tablets to restore the backup
         sleep 60
 
     # Count the number of rows in each table to make sure the restoration is successful.
     - name: Assert the number of rows in every table
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples ; source ./common/env.sh
 
         echo "select count(sku) from product;" | mysql 2>&1| grep 2
         echo "select count(email) from customer;" | mysql 2>&1| grep 5
@@ -265,7 +265,7 @@ jobs:
     - name: Insert more rows in the tables
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples ; source ./common/env.sh
 
         echo "insert into customer(email) values('new_user_2@domain.com');" | mysql
         echo "insert into product(sku, description, price) values('SKU-1011', 'description', 111);" | mysql
@@ -274,18 +274,18 @@ jobs:
     # Taking a second backup of the cluster.
     - name: Take a second backup of all the shards
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 2
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/take_backups.sh
+        source build.env ; cd examples/backups
+        ./take_backups.sh
 
     # Stopping the tablets so we can perform the upgrade.
     - name: Stop tablets
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/stop_tablets.sh
+        source build.env ; cd examples/backups
+        ./stop_tablets.sh
 
     # We upgrade: we swap binaries and use the version N of the tablet.
     - name: Upgrade - Swap binaries, use VTTablet N
@@ -300,18 +300,18 @@ jobs:
     # Starting the tablets again and restoring the next backup.
     - name: Start new tablets and restore
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/restart_tablets.sh
+        source build.env ; cd examples/backups
+        ./restart_tablets.sh
         # give enough time to the tablets to restore the backup
-        sleep 60
+        sleep 90
 
     # We count the number of rows in every table to check that the restore step was successful.
     - name: Assert the number of rows in every table
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples ; source ./common/env.sh
 
         echo "select count(sku) from product;" | mysql 2>&1| grep 3
         echo "select count(email) from customer;" | mysql 2>&1| grep 6

docker/local/Dockerfile

@@ -12,7 +12,7 @@ RUN /vt/dist/install_dependencies.sh mysql57
 
 COPY docker/local/install_local_dependencies.sh /vt/dist/install_local_dependencies.sh
 RUN /vt/dist/install_local_dependencies.sh
-RUN echo "source /vt/local/env.sh" >> /etc/bash.bashrc
+RUN echo "source /vt/common/env.sh" >> /etc/bash.bashrc
 
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0

docker/mini/Dockerfile

@@ -35,8 +35,9 @@ COPY docker/mini/orchestrator-vitess-mini.conf.json /etc/orchestrator.conf.json
 RUN chown vitess:vitess /etc/orchestrator.conf.json
 
 COPY docker/mini/docker-entry /vt/dist/docker/mini/docker-entry
-COPY examples/local/scripts /vt/dist/scripts
-COPY examples/local/env.sh /vt/dist/scripts/env.sh
+COPY examples/common/scripts /vt/dist/scripts
+COPY examples/common/env.sh /vt/dist/scripts/env.sh
+COPY examples/common/lib/utils.sh /vt/dist/scripts/lib/utils.sh
 COPY docker/mini/vtctld-mini-up.sh /vt/dist/scripts/vtctld-mini-up.sh
 COPY docker/mini/vttablet-mini-up.sh /vt/dist/scripts/vttablet-mini-up.sh
 COPY docker/mini/orchestrator-up.sh /vt/dist/scripts/orchestrator-up.sh

examples/local/backups/restart_tablets.sh → examples/backups/restart_tablets.sh

@@ -17,21 +17,21 @@
 # this script brings up new tablets for the two new shards that we will
 # be creating in the customer keyspace and copies the schema
 
-source ./env.sh
+source ../common/env.sh
 
 for i in 100 101 102; do
-  CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh
-  CELL=zone1 KEYSPACE=commerce TABLET_UID=$i ./scripts/vttablet-up.sh
+  CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-up.sh
+  CELL=zone1 KEYSPACE=commerce TABLET_UID=$i ../common/scripts/vttablet-up.sh
 done
 
 for i in 200 201 202; do
-  CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh
-  SHARD=-80 CELL=zone1 KEYSPACE=customer TABLET_UID=$i ./scripts/vttablet-up.sh
+  CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-up.sh
+  SHARD=-80 CELL=zone1 KEYSPACE=customer TABLET_UID=$i ../common/scripts/vttablet-up.sh
 done
 
 for i in 300 301 302; do
-  CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh
-  SHARD=80- CELL=zone1 KEYSPACE=customer TABLET_UID=$i ./scripts/vttablet-up.sh
+  CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-up.sh
+  SHARD=80- CELL=zone1 KEYSPACE=customer TABLET_UID=$i ../common/scripts/vttablet-up.sh
 done
 sleep 5
 
@@ -40,7 +40,7 @@ sleep 5
 # complete before we start InitShardPrimary, otherwise we end up reading the
 # tablet type to RESTORE and do not set semi-sync, which leads to the primary
 # hanging on writes.
-totalTime=300
+totalTime=600
 for i in 101 201 301; do
   while [ $totalTime -gt 0 ]; do
     status=$(curl "http://$hostname:15$i/debug/status_details")