fix: shadowfork latest bug

README.md

@@ -579,7 +579,7 @@ network_params:
   # The block height of the shadowfork
   # This is used to sync the network from a snapshot at a specific block height
   # Defaults to "latest"
-  # Example: shadowfork_block_height: 240000
+  # Example: shadowfork_block_height: 340000 for hoodi
   shadowfork_block_height: "latest"
 
   # The number of data column sidecar subnets used in the gossipsub protocol

src/network_launcher/public_network.star

@@ -10,6 +10,34 @@ def launch(
     plan, participants, network_params, global_tolerations, global_node_selectors
 ):
     if network_params.force_snapshot_sync:
+        # Fetch block data and determine block height
+        if network_params.shadowfork_block_height == "latest":
+            latest_block = plan.run_sh(
+                name="fetch-latest-block-data-public",
+                description="Fetching the latest block data for public network",
+                run="mkdir -p /blocks && \
+                BASE_URL='"
+                + network_params.network_sync_base_url
+                + network_params.network
+                + '\' && \
+                LATEST_BLOCK=$(curl -s "${BASE_URL}/geth/latest") && \
+                echo "Latest block number: $LATEST_BLOCK" && \
+                echo $LATEST_BLOCK > /blocks/block_height.txt',
+                store=[StoreSpec(src="/blocks", name="latest")],
+            )
+        else:
+            latest_block = plan.run_sh(
+                name="fetch-specific-block-data-public",
+                description="Fetching block data for specific block",
+                run="mkdir -p /blocks && \
+                echo Specific block number: "
+                + str(network_params.shadowfork_block_height)
+                + " && echo "
+                + str(network_params.shadowfork_block_height)
+                + " > /blocks/block_height.txt",
+                store=[StoreSpec(src="/blocks", name="latest")],
+            )
+
         for index, participant in enumerate(participants):
             tolerations = input_parser.get_client_tolerations(
                 participant.el_tolerations,
@@ -35,13 +63,15 @@ def launch(
                 config=ServiceConfig(
                     image="alpine:3.19.1",
                     cmd=[
-                        "apk add --no-cache curl tar zstd && curl -s -L "
+                        "apk add --no-cache curl tar zstd && "
+                        + "BLOCK_HEIGHT=$(cat /shared/block_height.txt) && "
+                        + 'echo "Using block height: $BLOCK_HEIGHT" && '
+                        + "curl -s -L "
                         + network_params.network_sync_base_url
                         + network_params.network
                         + "/"
                         + el_type
-                        + "/latest"
-                        + "/snapshot.tar.zst"
+                        + "/$BLOCK_HEIGHT/snapshot.tar.zst"
                         + " | tar -I zstd -xvf - -C /data/"
                         + el_type
                         + "/execution-data"
@@ -58,6 +88,7 @@ def launch(
                                 el_type + "_volume_size"
                             ],
                         ),
+                        "/shared": "latest",
                     },
                     tolerations=tolerations,
                     node_selectors=node_selectors,

src/network_launcher/shadowfork.star

@@ -24,19 +24,45 @@ def shadowfork_prep(
         network_id = constants.NETWORK_ID[
             base_network
         ]  # overload the network id to match the network name
-    latest_block = plan.run_sh(
-        name="fetch-latest-block",
-        description="Fetching the latest block",
-        run="mkdir -p /shadowfork && \
-            curl -s -o /shadowfork/latest_block.json "
-        + network_params.network_sync_base_url
-        + base_network
-        + "/geth/"
-        + str(network_params.shadowfork_block_height)
-        + "/_snapshot_eth_getBlockByNumber.json && \
-           cat /shadowfork/latest_block.json",
-        store=[StoreSpec(src="/shadowfork", name="latest_blocks")],
-    )
+
+    # Fetch block data and determine block height
+    if network_params.shadowfork_block_height == "latest":
+        latest_block = plan.run_sh(
+            name="fetch-latest-block-data-sf",
+            description="Fetching the latest block data",
+            run="mkdir -p /shadowfork && \
+            BASE_URL='"
+            + network_params.network_sync_base_url
+            + base_network
+            + '\' && \
+            LATEST_BLOCK=$(curl -s "${BASE_URL}/geth/latest") && \
+            echo "Latest block number: $LATEST_BLOCK" && \
+            echo $LATEST_BLOCK > /shadowfork/block_height.txt && \
+            URL="${BASE_URL}/geth/$LATEST_BLOCK/_snapshot_eth_getBlockByNumber.json" && \
+            echo "Fetching from URL: $URL" && \
+            curl -s -f -o /shadowfork/latest_block.json "$URL" || { echo "Curl failed with exit code $?"; exit 1; } && \
+            cat /shadowfork/latest_block.json',
+            store=[StoreSpec(src="/shadowfork", name="latest_blocks")],
+        )
+    else:
+        latest_block = plan.run_sh(
+            name="fetch-block-data-sf",
+            description="Fetching block data for specific block",
+            run="mkdir -p /shadowfork && \
+            BLOCK_HEIGHT='"
+            + str(network_params.shadowfork_block_height)
+            + "' && \
+            echo $BLOCK_HEIGHT > /shadowfork/block_height.txt && \
+            BASE_URL='"
+            + network_params.network_sync_base_url
+            + base_network
+            + '\' && \
+            URL="${BASE_URL}/geth/$BLOCK_HEIGHT/_snapshot_eth_getBlockByNumber.json" && \
+            echo "Fetching from URL: $URL" && \
+            curl -s -f -o /shadowfork/latest_block.json "$URL" || { echo "Curl failed with exit code $?"; exit 1; } && \
+            cat /shadowfork/latest_block.json',
+            store=[StoreSpec(src="/shadowfork", name="latest_blocks")],
+        )
 
     for index, participant in enumerate(participants):
         tolerations = input_parser.get_client_tolerations(
@@ -56,19 +82,20 @@ def shadowfork_prep(
         index_str = shared_utils.zfill_custom(index + 1, len(str(len(participants))))
 
         el_service_name = "el-{0}-{1}-{2}".format(index_str, el_type, cl_type)
-        shadowfork_data = plan.add_service(
+        plan.add_service(
             name="shadowfork-{0}".format(el_service_name),
             config=ServiceConfig(
                 image="alpine:3.19.1",
                 cmd=[
-                    "apk add --no-cache curl tar zstd && curl -s -L "
+                    "apk add --no-cache curl tar zstd && "
+                    + "BLOCK_HEIGHT=$(cat /shared/block_height.txt) && "
+                    + 'echo "Using block height: $BLOCK_HEIGHT" && '
+                    + "curl -s -L "
                     + network_params.network_sync_base_url
                     + base_network
                     + "/"
                     + el_type
-                    + "/"
-                    + str(network_params.shadowfork_block_height)
-                    + "/snapshot.tar.zst"
+                    + "/$BLOCK_HEIGHT/snapshot.tar.zst"
                     + " | tar -I zstd -xvf - -C /data/"
                     + el_type
                     + "/execution-data"
@@ -85,6 +112,7 @@ def shadowfork_prep(
                             el_type + "_volume_size"
                         ],
                     ),
+                    "/shared": "latest_blocks",
                 },
                 tolerations=tolerations,
                 node_selectors=node_selectors,