Examples: Add lib functions to wait for shard states

.github/workflows/region_example.yml

@@ -105,5 +105,4 @@ jobs:
         if [ ${{matrix.os}} = "macos-latest" ]; then
           export PATH="/usr/local/opt/mysql@5.7/bin:$PATH"
         fi
-        sed -i 's/user\/my-vitess/runner\/work\/vitess\/vitess/g' examples/region_sharding/main_vschema_sharded.json #set correct path to countries.json
         eatmydata -- go run test.go -print-log -follow -retry=1 region_example

.github/workflows/upgrade_downgrade_test_backups_manual.yml

@@ -191,18 +191,18 @@ jobs:
     # We also insert a few rows in our three tables.
     - name: Create the example Vitess cluster with all components using version N
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/start_cluster.sh
+        source build.env ; cd examples/backups
+        ./start_cluster.sh
 
     # Taking a backup
     - name: Take a backup of all the shards
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 2
+      timeout-minutes: 5
       run: |
-        source build.env ; cd examples/local
-        ./backups/take_backups.sh
+        source build.env ; cd examples/backups
+        ./take_backups.sh
 
     # We insert more data in every table after the backup.
     # When we restore the backup made in the previous step, we do not want to see the rows we are about to insert now.
@@ -214,7 +214,7 @@ jobs:
     - name: Insert more data after the backup
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples; source ./common/env.sh
 
         echo "insert into customer(email) values('new_user_1@domain.com');" | mysql
         echo "insert into product(sku, description, price) values('SKU-1009', 'description', 89);" | mysql
@@ -223,10 +223,10 @@ jobs:
     # Stop all the tablets and remove their data
     - name: Stop tablets
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/stop_tablets.sh
+        source build.env ; cd examples/backups
+        ./stop_tablets.sh
 
     # We downgrade: we use the version N-1 of vttablet
     - name: Downgrade - Swap binaries, use VTTablet N-1
@@ -241,18 +241,18 @@ jobs:
     # Starting the tablets again, they will automatically start restoring the last backup.
     - name: Start new tablets and restore
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/restart_tablets.sh
+        source build.env ; cd examples/backups
+        ./restart_tablets.sh
         # give enough time to the tablets to restore the backup
-        sleep 60
+        sleep 90
 
     # Count the number of rows in each table to make sure the restoration is successful.
     - name: Assert the number of rows in every table
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples ; source ./common/env.sh
 
         echo "select count(sku) from product;" | mysql 2>&1| grep 2
         echo "select count(email) from customer;" | mysql 2>&1| grep 5
@@ -262,7 +262,7 @@ jobs:
     - name: Insert more rows in the tables
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples ; source ./common/env.sh
 
         echo "insert into customer(email) values('new_user_2@domain.com');" | mysql
         echo "insert into product(sku, description, price) values('SKU-1011', 'description', 111);" | mysql
@@ -271,18 +271,18 @@ jobs:
     # Taking a second backup of the cluster.
     - name: Take a second backup of all the shards
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 2
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/take_backups.sh
+        source build.env ; cd examples/backups
+        ./take_backups.sh
 
     # Stopping the tablets so we can perform the upgrade.
     - name: Stop tablets
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/stop_tablets.sh
+        source build.env ; cd examples/backups
+        ./stop_tablets.sh
 
     # We upgrade: we swap binaries and use the version N of the tablet.
     - name: Upgrade - Swap binaries, use VTTablet N
@@ -297,18 +297,18 @@ jobs:
     # Starting the tablets again and restoring the previous backup.
     - name: Start new tablets and restore
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/restart_tablets.sh
+        source build.env ; cd examples/backups
+        ./restart_tablets.sh
         # give enough time to the tablets to restore the backup
-        sleep 60
+        sleep 90
 
     # We count the number of rows in every table to check that the restore step was successful.
     - name: Assert the number of rows in every table
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples ; source ./common/env.sh
 
         echo "select count(sku) from product;" | mysql 2>&1| grep 3
         echo "select count(email) from customer;" | mysql 2>&1| grep 6

.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml

@@ -194,18 +194,18 @@ jobs:
     # We also insert a few rows in our three tables.
     - name: Create the example Vitess cluster with all components using version N
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/start_cluster.sh
+        source build.env ; cd examples/backups
+        ./start_cluster.sh
 
     # Taking a backup
     - name: Take a backup of all the shards
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 2
+      timeout-minutes: 5
       run: |
-        source build.env ; cd examples/local
-        ./backups/take_backups.sh
+        source build.env ; cd examples/backups
+        ./take_backups.sh
 
     # We insert more data in every table after the backup.
     # When we restore the backup made in the next step, we do not want to see the rows we are about to insert now.
@@ -217,7 +217,7 @@ jobs:
     - name: Insert more data after the backup
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples ; source ./common/env.sh
 
         echo "insert into customer(email) values('new_user_1@domain.com');" | mysql
         echo "insert into product(sku, description, price) values('SKU-1009', 'description', 89);" | mysql
@@ -226,10 +226,10 @@ jobs:
     # Stop all the tablets and remove their data
     - name: Stop tablets
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/stop_tablets.sh
+        source build.env ; cd examples/backups
+        ./stop_tablets.sh
 
     # We downgrade: we use the version N+1 of vttablet
     - name: Downgrade - Swap binaries, use VTTablet N+1
@@ -244,18 +244,18 @@ jobs:
     # Starting the tablets again, they will automatically start restoring the last backup.
     - name: Start new tablets and restore
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/restart_tablets.sh
+        source build.env ; cd examples/backups
+        ./restart_tablets.sh
         # give enough time to the tablets to restore the backup
         sleep 60
 
     # Count the number of rows in each table to make sure the restoration is successful.
     - name: Assert the number of rows in every table
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples ; source ./common/env.sh
 
         echo "select count(sku) from product;" | mysql 2>&1| grep 2
         echo "select count(email) from customer;" | mysql 2>&1| grep 5
@@ -265,7 +265,7 @@ jobs:
     - name: Insert more rows in the tables
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples ; source ./common/env.sh
 
         echo "insert into customer(email) values('new_user_2@domain.com');" | mysql
         echo "insert into product(sku, description, price) values('SKU-1011', 'description', 111);" | mysql
@@ -274,18 +274,18 @@ jobs:
     # Taking a second backup of the cluster.
     - name: Take a second backup of all the shards
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 2
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/take_backups.sh
+        source build.env ; cd examples/backups
+        ./take_backups.sh
 
     # Stopping the tablets so we can perform the upgrade.
     - name: Stop tablets
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/stop_tablets.sh
+        source build.env ; cd examples/backups
+        ./stop_tablets.sh
 
     # We upgrade: we swap binaries and use the version N of the tablet.
     - name: Upgrade - Swap binaries, use VTTablet N
@@ -300,18 +300,18 @@ jobs:
     # Starting the tablets again and restoring the next backup.
     - name: Start new tablets and restore
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
-      timeout-minutes: 5
+      timeout-minutes: 10
       run: |
-        source build.env ; cd examples/local
-        ./backups/restart_tablets.sh
+        source build.env ; cd examples/backups
+        ./restart_tablets.sh
         # give enough time to the tablets to restore the backup
-        sleep 60
+        sleep 90
 
     # We count the number of rows in every table to check that the restore step was successful.
     - name: Assert the number of rows in every table
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       run: |
-        source build.env ; cd examples/local ; source ./env.sh
+        source build.env ; cd examples ; source ./common/env.sh
 
         echo "select count(sku) from product;" | mysql 2>&1| grep 3
         echo "select count(email) from customer;" | mysql 2>&1| grep 6

examples/local/backups/restart_tablets.sh → examples/backups/restart_tablets.sh

@@ -17,21 +17,21 @@
 # this script brings up new tablets for the two new shards that we will
 # be creating in the customer keyspace and copies the schema
 
-source ./env.sh
+source ../common/env.sh
 
 for i in 100 101 102; do
-  CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh
-  CELL=zone1 KEYSPACE=commerce TABLET_UID=$i ./scripts/vttablet-up.sh
+  CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-up.sh
+  CELL=zone1 KEYSPACE=commerce TABLET_UID=$i ../common/scripts/vttablet-up.sh
 done
 
 for i in 200 201 202; do
-  CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh
-  SHARD=-80 CELL=zone1 KEYSPACE=customer TABLET_UID=$i ./scripts/vttablet-up.sh
+  CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-up.sh
+  SHARD=-80 CELL=zone1 KEYSPACE=customer TABLET_UID=$i ../common/scripts/vttablet-up.sh
 done
 
 for i in 300 301 302; do
-  CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh
-  SHARD=80- CELL=zone1 KEYSPACE=customer TABLET_UID=$i ./scripts/vttablet-up.sh
+  CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-up.sh
+  SHARD=80- CELL=zone1 KEYSPACE=customer TABLET_UID=$i ../common/scripts/vttablet-up.sh
 done
 sleep 5
 
@@ -40,7 +40,7 @@ sleep 5
 # complete before we start InitShardPrimary, otherwise we end up reading the
 # tablet type to RESTORE and do not set semi-sync, which leads to the primary
 # hanging on writes.
-totalTime=300
+totalTime=600
 for i in 101 201 301; do
   while [ $totalTime -gt 0 ]; do
     status=$(curl "http://$hostname:15$i/debug/status_details")

examples/local/backups/start_cluster.sh → examples/backups/start_cluster.sh

@@ -17,56 +17,60 @@
 # this script brings up new tablets for the two new shards that we will
 # be creating in the customer keyspace and copies the schema
 
-source ./env.sh
+source ../common/env.sh
 
 # start topo server
 if [ "${TOPO}" = "zk2" ]; then
-	CELL=zone1 ./scripts/zk-up.sh
+	CELL=zone1 ../common/scripts/zk-up.sh
 elif [ "${TOPO}" = "k8s" ]; then
-	CELL=zone1 ./scripts/k3s-up.sh
+	CELL=zone1 ../common/scripts/k3s-up.sh
 else
-	CELL=zone1 ./scripts/etcd-up.sh
+	CELL=zone1 ../common/scripts/etcd-up.sh
 fi
 
 # start vtctld
-CELL=zone1 ./scripts/vtctld-up.sh
+CELL=zone1 ../common/scripts/vtctld-up.sh
 
 
 # start vttablets for keyspace commerce
 for i in 100 101 102; do
-	CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh
-	CELL=zone1 KEYSPACE=commerce TABLET_UID=$i ./scripts/vttablet-up.sh
+	CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-up.sh
+	CELL=zone1 KEYSPACE=commerce TABLET_UID=$i ../common/scripts/vttablet-up.sh
 done
 
 # set one of the replicas to primary
 vtctldclient InitShardPrimary --force commerce/0 zone1-100
 
 # create the schema for commerce
-vtctlclient ApplySchema -- --sql-file backups/create_commerce_schema.sql commerce
-vtctlclient ApplyVSchema -- --vschema_file ./vschema_commerce_seq.json commerce
+vtctlclient ApplySchema -- --sql-file ./create_commerce_schema.sql commerce || fail "Could not apply schema for the commerce keyspace"
+vtctlclient ApplyVSchema -- --vschema_file ../local/vschema_commerce_seq.json commerce || fail "Could not apply vschema for the commerce keyspace"
 
 
 # start vttablets for keyspace customer
 for i in 200 201 202; do
-	CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh
-	SHARD=-80 CELL=zone1 KEYSPACE=customer TABLET_UID=$i ./scripts/vttablet-up.sh
+	CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-up.sh
+	SHARD=-80 CELL=zone1 KEYSPACE=customer TABLET_UID=$i ../common/scripts/vttablet-up.sh
 done
 for i in 300 301 302; do
-	CELL=zone1 TABLET_UID=$i ./scripts/mysqlctl-up.sh
-	SHARD=80- CELL=zone1 KEYSPACE=customer TABLET_UID=$i ./scripts/vttablet-up.sh
+	CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-up.sh
+	SHARD=80- CELL=zone1 KEYSPACE=customer TABLET_UID=$i ../common/scripts/vttablet-up.sh
 done
 
 # set one of the replicas to primary
 vtctldclient InitShardPrimary --force customer/-80 zone1-200
 vtctldclient InitShardPrimary --force customer/80- zone1-300
 
+for shard in "-80" "80-"; do
+	wait_for_healthy_shard customer "${shard}" || exit 1
+done
+
 # create the schema for customer
-vtctlclient ApplySchema -- --sql-file backups/create_customer_schema.sql customer
-vtctlclient ApplyVSchema -- --vschema_file ./vschema_customer_sharded.json customer
+vtctlclient ApplySchema -- --sql-file ./create_customer_schema.sql customer || fail "Could not apply schema for the customer keyspace"
+vtctlclient ApplyVSchema -- --vschema_file ../local/vschema_customer_sharded.json customer || fail "Could not apply vschema for the customer keyspace"
 
 
 # start vtgate
-CELL=zone1 ./scripts/vtgate-up.sh
+CELL=zone1 ../common/scripts/vtgate-up.sh
 
 sleep 5