diff --git a/dataflow/conftest.py b/dataflow/conftest.py index a1f81eac6f6..2bb6c6b3ecf 100644 --- a/dataflow/conftest.py +++ b/dataflow/conftest.py @@ -85,7 +85,7 @@ def bucket_name(test_name: str, location: str, unique_id: str) -> Iterator[str]: # Try to remove all files before deleting the bucket. # Deleting a bucket with too many files results in an error. try: - run_cmd("gsutil", "-m", "rm", "-rf", f"gs://{bucket_name}/*") + run_cmd("gcloud", "storage", "rm", "--recursive", "--continue-on-error", f"gs://{bucket_name}/*") except RuntimeError: # If no files were found and it fails, ignore the error. pass diff --git a/dataflow/encryption-keys/README.md b/dataflow/encryption-keys/README.md index 0545d63d321..036540108d8 100644 --- a/dataflow/encryption-keys/README.md +++ b/dataflow/encryption-keys/README.md @@ -25,7 +25,7 @@ Additionally, for this sample you need the following: ```sh export BUCKET=your-gcs-bucket - gsutil mb gs://$BUCKET + gcloud storage buckets create gs://$BUCKET ``` 1. [Create a symmetric key ring](https://cloud.google.com/kms/docs/creating-keys). @@ -174,10 +174,10 @@ To avoid incurring charges to your GCP account for the resources used: ```sh # Remove only the files created by this sample. -gsutil -m rm -rf "gs://$BUCKET/samples/dataflow/kms" +gcloud storage rm --recursive --continue-on-error "gs://$BUCKET/samples/dataflow/kms" # [optional] Remove the Cloud Storage bucket. -gsutil rb gs://$BUCKET +gcloud storage buckets delete gs://$BUCKET # Remove the BigQuery table. bq rm -f -t $PROJECT:$DATASET.$TABLE diff --git a/dataflow/flex-templates/getting_started/README.md b/dataflow/flex-templates/getting_started/README.md index d7ee5d38bd3..383fdca052e 100644 --- a/dataflow/flex-templates/getting_started/README.md +++ b/dataflow/flex-templates/getting_started/README.md @@ -9,7 +9,7 @@ Make sure you have followed the ```sh export BUCKET="your--bucket" -gsutil mb gs://$BUCKET +gcloud storage buckets create gs://$BUCKET ``` ## create an Artifact Registry repository @@ -51,4 +51,3 @@ gcloud dataflow flex-template run "flex-`date +%Y%m%d-%H%M%S`" \ For more information about building and running flex templates, see 📝 [Use Flex Templates](https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates). - diff --git a/dataflow/flex-templates/pipeline_with_dependencies/README.md b/dataflow/flex-templates/pipeline_with_dependencies/README.md index 99385639297..79bbabf9761 100644 --- a/dataflow/flex-templates/pipeline_with_dependencies/README.md +++ b/dataflow/flex-templates/pipeline_with_dependencies/README.md @@ -73,7 +73,7 @@ rules. It is optional. export PROJECT="project-id" export BUCKET="your-bucket" export REGION="us-central1" -gsutil mb -p $PROJECT gs://$BUCKET +gcloud storage buckets create gs://$BUCKET --project=$PROJECT ``` ## Create an Artifact Registry repository @@ -165,7 +165,7 @@ gcloud dataflow flex-template run "flex-`date +%Y%m%d-%H%M%S`" \ After the pipeline finishes, use the following command to inspect the output: ```bash -gsutil cat gs://$BUCKET/output* +gcloud storage cat gs://$BUCKET/output* ``` ## Optional: Update the dependencies in the requirements file and rebuild the Docker images diff --git a/dataflow/flex-templates/streaming_beam/README.md b/dataflow/flex-templates/streaming_beam/README.md index 66d891ce526..2a21e44220b 100644 --- a/dataflow/flex-templates/streaming_beam/README.md +++ b/dataflow/flex-templates/streaming_beam/README.md @@ -25,7 +25,7 @@ Additionally, for this sample you need the following: ```sh export BUCKET="your-gcs-bucket" - gsutil mb gs://$BUCKET + gcloud storage buckets create gs://$BUCKET ``` 1. Create a @@ -231,7 +231,7 @@ The following sections describe how to delete or turn off these resources. 1. Delete the template spec file from Cloud Storage. ```sh - gsutil rm $TEMPLATE_PATH + gcloud storage rm $TEMPLATE_PATH ``` 1. Delete the Flex Template container image from Container Registry. @@ -277,7 +277,7 @@ The following sections describe how to delete or turn off these resources. > These objects cannot be recovered. > > ```sh - > gsutil rm -r gs://$BUCKET + > gcloud storage rm --recursive gs://$BUCKET > ``` ## Limitations diff --git a/dataflow/gemma-flex-template/README.md b/dataflow/gemma-flex-template/README.md index 8ade42a8c46..0e082cc2bb7 100644 --- a/dataflow/gemma-flex-template/README.md +++ b/dataflow/gemma-flex-template/README.md @@ -43,7 +43,7 @@ Click [here to create a GCS bucket](https://console.cloud.google.com/storage/cre ```sh export GCS_BUCKET="your--bucket" -gsutil mb gs://$GCS_BUCKET +gcloud storage buckets create gs://$GCS_BUCKET ``` Make sure your GCS bucket name does __not__ include the `gs://` prefix diff --git a/dataflow/gemma-flex-template/e2e_test.py b/dataflow/gemma-flex-template/e2e_test.py index f95f78ec089..be281a9c984 100644 --- a/dataflow/gemma-flex-template/e2e_test.py +++ b/dataflow/gemma-flex-template/e2e_test.py @@ -92,7 +92,7 @@ def responses_subscription( @pytest.fixture(scope="session") def flex_template_image(utils: Utils) -> str: - conftest.run_cmd("gsutil", "cp", "-r", GEMMA_GCS, ".") + conftest.run_cmd("gcloud", "storage", "cp", "--recursive", GEMMA_GCS, ".") yield from utils.cloud_build_submit(NAME) diff --git a/dataflow/gemma/e2e_test.py b/dataflow/gemma/e2e_test.py index e2510716f4b..6f65fb15959 100644 --- a/dataflow/gemma/e2e_test.py +++ b/dataflow/gemma/e2e_test.py @@ -60,7 +60,7 @@ def test_name() -> str: @pytest.fixture(scope="session") def container_image(utils: Utils) -> str: # Copy Gemma onto the local environment - conftest.run_cmd("gsutil", "cp", "-r", GEMMA_GCS, ".") + conftest.run_cmd("gcloud", "storage", "cp", "--recursive", GEMMA_GCS, ".") yield from utils.cloud_build_submit(NAME) diff --git a/dataflow/run-inference/tests/e2e_test.py b/dataflow/run-inference/tests/e2e_test.py index 0428af3dd28..70be7d6878d 100644 --- a/dataflow/run-inference/tests/e2e_test.py +++ b/dataflow/run-inference/tests/e2e_test.py @@ -95,7 +95,7 @@ def dataflow_job( ) -> Iterator[str]: # Upload the state dict to Cloud Storage. state_dict_gcs = f"gs://{bucket_name}/temp/state_dict.pt" - conftest.run_cmd("gsutil", "cp", "-n", state_dict_path, state_dict_gcs) + conftest.run_cmd("gcloud", "storage", "cp", "--no-clobber", state_dict_path, state_dict_gcs) # Launch the streaming Dataflow pipeline. conftest.run_cmd( diff --git a/dataflow/run_template/README.md b/dataflow/run_template/README.md index c73fb76c504..dda9f4570aa 100644 --- a/dataflow/run_template/README.md +++ b/dataflow/run_template/README.md @@ -29,7 +29,7 @@ Additionally, for this sample you need the following: ```sh export BUCKET=your-gcs-bucket - gsutil mb gs://$BUCKET + gcloud storage buckets create gs://$BUCKET ``` 1. Clone the `python-docs-samples` repository. diff --git a/dataproc/snippets/README.md b/dataproc/snippets/README.md index 98622be7dc1..442b8fa55bb 100644 --- a/dataproc/snippets/README.md +++ b/dataproc/snippets/README.md @@ -64,7 +64,7 @@ To run list_clusters.py: To run submit_job_to_cluster.py, first create a GCS bucket (used by Cloud Dataproc to stage files) from the Cloud Console or with gsutil: - gsutil mb gs:// + gcloud storage buckets create gs:// Next, set the following environment variables: diff --git a/dataproc/snippets/python-api-walkthrough.md b/dataproc/snippets/python-api-walkthrough.md index c5eb884a8f0..2ca94cca3ca 100644 --- a/dataproc/snippets/python-api-walkthrough.md +++ b/dataproc/snippets/python-api-walkthrough.md @@ -65,7 +65,7 @@ an explanation of how the code works. * To create a new bucket, run the following command. Your bucket name must be unique. - gsutil mb -p {{project-id}} gs://your-bucket-name + gcloud storage buckets create --project={{project-id}} gs://your-bucket-name 2. Set environment variables. @@ -145,12 +145,12 @@ Cluster cluster-name successfully deleted. If you created a Cloud Storage bucket to use for this walkthrough, you can run the following command to delete the bucket (the bucket must be empty). - gsutil rb gs://$BUCKET + gcloud storage buckets delete gs://$BUCKET * You can run the following command to **delete the bucket and all objects within it. Note: the deleted objects cannot be recovered.** - gsutil rm -r gs://$BUCKET + gcloud storage rm --recursive gs://$BUCKET * **For more information.** See the [Dataproc documentation](https://cloud.google.com/dataproc/docs/)