Clean up some dataproc references from aws quickstart

Author: chewy-zlai

distribution/run_zipline_quickstart_aws.py

@@ -10,7 +10,7 @@ def main():
         quickstart_sh = os.path.join(os.path.dirname(os.path.realpath(__file__))
                                      , "run_zipline_quickstart_aws.sh")
 
-        # run the bash script run_zipline_quickstart_gcp.sh subprocess command
+        # run the bash script run_zipline_quickstart_aws.sh subprocess command
         # with the temporary directory as the argument
         subprocess.run([f"bash {quickstart_sh} {temp_dir}"], shell=True)
 

distribution/run_zipline_quickstart_aws.sh

@@ -29,15 +29,15 @@ python3 -m venv tmp_chronon
 source tmp_chronon/bin/activate
 
 # Download the wheel
-aws s3 cp gs://zipline-artifacts-canary/jars/$WHEEL_FILE .
+aws s3 cp s3://zipline-artifacts-canary/jars/$WHEEL_FILE .
 
 # Install the wheel (force)
 pip uninstall zipline-ai
 pip install --force-reinstall $WHEEL_FILE
 
 export PYTHONPATH="${PYTHONPATH}:$(pwd)"
 
-# function to check dataproc job id state
+# function to check emr job id state
 function check_emr_job_state() {
   STEP_ID=$1
   if [ -z "$STEP_ID" ]; then
@@ -68,20 +68,20 @@ check_emr_job_state $BACKFILL_JOB_ID
 echo -e "${GREEN}<<<<<.....................................GROUP-BY-UPLOAD.....................................>>>>>\033[0m"
 touch tmp_gbu.out
 zipline run --mode upload --conf production/group_bys/quickstart/purchases.v1_test --ds  2023-12-01 --emr 2>&1 | tee /dev/tty tmp_gbu.out
-GBU_JOB_ID=$(cat tmp_gbu.out | grep "$DATAPROC_SUBMITTER_ID_STR" | cut -d " " -f5)
+GBU_JOB_ID=$(cat tmp_gbu.out | grep "$EMR_SUBMITTER_ID_STR" | cut -d " " -f5)
 check_emr_job_state $GBU_JOB_ID
 
 # Need to wait for upload to finish
 echo -e "${GREEN}<<<<<.....................................UPLOAD-TO-KV.....................................>>>>>\033[0m"
 touch tmp_upload_to_kv.out
 zipline run --mode upload-to-kv --conf production/group_bys/quickstart/purchases.v1_test --partition-string=2023-12-01 --emr 2>&1 | tee /dev/tty tmp_upload_to_kv.out
-UPLOAD_TO_KV_JOB_ID=$(cat tmp_upload_to_kv.out | grep "$DATAPROC_SUBMITTER_ID_STR" | cut -d " " -f5)
+UPLOAD_TO_KV_JOB_ID=$(cat tmp_upload_to_kv.out | grep "$EMR_SUBMITTER_ID_STR" | cut -d " " -f5)
 check_emr_job_state $UPLOAD_TO_KV_JOB_ID
 
 echo -e "${GREEN}<<<<< .....................................METADATA-UPLOAD.....................................>>>>>\033[0m"
 touch tmp_metadata_upload.out
 zipline run --mode metadata-upload --conf production/group_bys/quickstart/purchases.v1_test --emr 2>&1 | tee /dev/tty tmp_metadata_upload.out
-METADATA_UPLOAD_JOB_ID=$(cat tmp_metadata_upload.out | grep "$DATAPROC_SUBMITTER_ID_STR" | cut -d " " -f5)
+METADATA_UPLOAD_JOB_ID=$(cat tmp_metadata_upload.out | grep "$EMR_SUBMITTER_ID_STR" | cut -d " " -f5)
 check_emr_job_state $METADATA_UPLOAD_JOB_ID
 
 # Need to wait for upload-to-kv to finish