From 6ed79732110ed6b100e0ae43a5030213e75d5263 Mon Sep 17 00:00:00 2001
From: gurusai-voleti <gvoleti@google.com>
Date: Wed, 24 Dec 2025 13:14:03 +0000
Subject: [PATCH] chore: Migrate gsutil usage to gcloud storage

---
 ...ation Pipeline with Dataflow and BigQuery.ipynb |  8 ++++----
 .../text-similarity-analysis/README.md             |  6 +++---
 .../text-similarity-analysis/etl/run_pipeline.sh   |  2 +-
 ...imator Train, Evaluate, Export, Explained.ipynb |  4 +---
 ...abyweight preprocessing with tf.Transform.ipynb |  8 ++++----
 ...byweight Estimation with Transformed Data.ipynb |  2 +-
 .../tfx/01_tf_estimator_deepdive.ipynb             | 14 +++++++-------
 00_Miscellaneous/tfx/03_eda_with_tfdv.ipynb        |  9 +++------
 .../mnist-keras-2/Keras_MNIST_CMLE.ipynb           |  2 +-
 .../mnist-keras-2/Keras_MNIST_local.ipynb          |  2 +-
 10 files changed, 26 insertions(+), 31 deletions(-)

diff --git a/00_Miscellaneous/model_evaluation_pipeline/Tutorial - TF Model Evaluation Pipeline with Dataflow and BigQuery.ipynb b/00_Miscellaneous/model_evaluation_pipeline/Tutorial - TF Model Evaluation Pipeline with Dataflow and BigQuery.ipynb
index 7c6fe0c..8d0ea28 100644
--- a/00_Miscellaneous/model_evaluation_pipeline/Tutorial - TF Model Evaluation Pipeline with Dataflow and BigQuery.ipynb	
+++ b/00_Miscellaneous/model_evaluation_pipeline/Tutorial - TF Model Evaluation Pipeline with Dataflow and BigQuery.ipynb	
@@ -219,10 +219,10 @@
    "source": [
     "%%bash\n",
     "\n",
-    "if ! gsutil ls | grep -q gs://${BUCKET}/; then\n",
-    "  gsutil mb -l ${REGION} gs://${BUCKET}\n",
+    "if ! gcloud storage ls | grep -q gs://${BUCKET}/; then\n",
+    "  gcloud storage buckets create --location ${REGION} gs://${BUCKET}\n",
     "  # copy canonical set of preprocessed files if you didn't do previous notebook\n",
-    "  gsutil -m cp -R gs://cloud-training-demos/babyweight gs://${BUCKET}\n",
+    "  gcloud storage cp --recursive gs://cloud-training-demos/babyweight gs://${BUCKET}\n",
     "fi"
    ]
   },
@@ -242,7 +242,7 @@
     "JOBNAME=babyweight_$(date -u +%y%m%d_%H%M%S)\n",
     "echo $OUTDIR $REGION $JOBNAME\n",
     "\n",
-    "gsutil -m rm -rf $OUTDIR\n",
+    "gcloud storage rm --recursive --continue-on-error $OUTDIR\n",
     "\n",
     "gcloud ml-engine jobs submit training $JOBNAME \\\n",
     "  --region=$REGION \\\n",
diff --git a/00_Miscellaneous/text-similarity-analysis/README.md b/00_Miscellaneous/text-similarity-analysis/README.md
index 08cf7d0..7157af2 100644
--- a/00_Miscellaneous/text-similarity-analysis/README.md
+++ b/00_Miscellaneous/text-similarity-analysis/README.md
@@ -34,8 +34,8 @@ export BUCKET=gs://[your-bucket-name]
 mkdir temp reuters
 tar -zxvf reuters21578.tar.gz -C temp/
 mv temp/*.sgm reuters/ && rm -rf temp
-gsutil mb $BUCKET
-gsutil -m cp -R reuters $BUCKET
+gcloud storage buckets create $BUCKET
+gcloud storage cp --recursive reuters $BUCKET
 ```
 
 ## Setup python environment and sample code
@@ -108,7 +108,7 @@ Before running the pipeline, you can remove previous working directory
 with below command if you want.
 
 ```bash
-gsutil rm -r $ROOT
+gcloud storage rm --recursive $ROOT
 ```
 
 Finally, you can run the pipeline with this command.
diff --git a/00_Miscellaneous/text-similarity-analysis/etl/run_pipeline.sh b/00_Miscellaneous/text-similarity-analysis/etl/run_pipeline.sh
index 4453e68..0fb0144 100644
--- a/00_Miscellaneous/text-similarity-analysis/etl/run_pipeline.sh
+++ b/00_Miscellaneous/text-similarity-analysis/etl/run_pipeline.sh
@@ -43,7 +43,7 @@ JOB_NAME=text-analysis
 MACHINE_TYPE=n1-highmem-2
 
 # Remove Root directory before running dataflow job.
-gsutil rm -r $ROOT_DIR
+gcloud storage rm --recursive $ROOT_DIR
 
 # Command to invoke dataflow job.
 python run_pipeline.py \
diff --git a/00_Miscellaneous/tf_train_eval_export/Tutorial - TensorFlow Estimator Train, Evaluate, Export, Explained.ipynb b/00_Miscellaneous/tf_train_eval_export/Tutorial - TensorFlow Estimator Train, Evaluate, Export, Explained.ipynb
index cefc1bd..2a9a607 100644
--- a/00_Miscellaneous/tf_train_eval_export/Tutorial - TensorFlow Estimator Train, Evaluate, Export, Explained.ipynb	
+++ b/00_Miscellaneous/tf_train_eval_export/Tutorial - TensorFlow Estimator Train, Evaluate, Export, Explained.ipynb	
@@ -76,9 +76,7 @@
    "source": [
     "DATA_DIR='data'\n",
     "!mkdir $DATA_DIR\n",
-    "!gsutil cp gs://cloud-samples-data/ml-engine/census/data/adult.data.csv $DATA_DIR\n",
-    "!gsutil cp gs://cloud-samples-data/ml-engine/census/data/adult.test.csv $DATA_DIR"
-   ]
+    "!gcloud storage cp gs://cloud-samples-data/ml-engine/census/data/adult.data.csv $DATA_DIR\n",    "!gcloud storage cp gs://cloud-samples-data/ml-engine/census/data/adult.test.csv $DATA_DIR"   ]
   },
   {
    "cell_type": "code",
diff --git a/00_Miscellaneous/tf_transform/tft-01 - Babyweight preprocessing with tf.Transform.ipynb b/00_Miscellaneous/tf_transform/tft-01 - Babyweight preprocessing with tf.Transform.ipynb
index 55d215d..68f4be0 100644
--- a/00_Miscellaneous/tf_transform/tft-01 - Babyweight preprocessing with tf.Transform.ipynb	
+++ b/00_Miscellaneous/tf_transform/tft-01 - Babyweight preprocessing with tf.Transform.ipynb	
@@ -589,28 +589,28 @@
     "echo 'transformed data:' \n",
     "if [ \"${RUN_LOCAL}\" = \"True \" ] \n",
     "then ls ${ROOT_DIR}/transformed \n",
-    "else gsutil ls gs://${BUCKET}/${ROOT_DIR}/transformed \n",
+    "else gcloud storage ls gs://${BUCKET}/${ROOT_DIR}/transformed \n",
     "fi\n",
     "echo ''\n",
     "\n",
     "echo 'transformed metadata:'  \n",
     "if [ \"${RUN_LOCAL}\" = \"True \" ] \n",
     "then ls ${ROOT_DIR}/transform/transformed_metadata\n",
-    "else gsutil ls gs://${BUCKET}/${ROOT_DIR}/transform/transformed_metadata \n",
+    "else gcloud storage ls gs://${BUCKET}/${ROOT_DIR}/transform/transformed_metadata \n",
     "fi\n",
     "echo ''\n",
     "\n",
     "echo 'transform artefact:'   \n",
     "if [ \"${RUN_LOCAL}\" = \"True \" ] \n",
     "then ls ${ROOT_DIR}/transform/transform_fn\n",
-    "else gsutil ls gs://${BUCKET}/${ROOT_DIR}/transform/transform_fn \n",
+    "else gcloud storage ls gs://${BUCKET}/${ROOT_DIR}/transform/transform_fn \n",
     "fi\n",
     "echo ''\n",
     "\n",
     "echo 'transform assets:'\n",
     "if [ \"${RUN_LOCAL}\" = \"True \" ] \n",
     "then ls ${ROOT_DIR}/transform/transform_fn/assets\n",
-    "else gsutil ls gs://${BUCKET}/${ROOT_DIR}/transform/transform_fn/assets \n",
+    "else gcloud storage ls gs://${BUCKET}/${ROOT_DIR}/transform/transform_fn/assets \n",
     "fi\n",
     "echo ''"
    ]
diff --git a/00_Miscellaneous/tf_transform/tft-02 - Babyweight Estimation with Transformed Data.ipynb b/00_Miscellaneous/tf_transform/tft-02 - Babyweight Estimation with Transformed Data.ipynb
index 1d89f2b..80252b6 100644
--- a/00_Miscellaneous/tf_transform/tft-02 - Babyweight Estimation with Transformed Data.ipynb	
+++ b/00_Miscellaneous/tf_transform/tft-02 - Babyweight Estimation with Transformed Data.ipynb	
@@ -446,7 +446,7 @@
     "%%bash\n",
     "if [ ${RUN_LOCAL} ]\n",
     "then \n",
-    "saved_model_dir=$(gsutil ls ${export_dir} | tail -n 1)\n",
+    "saved_model_dir=$(gcloud storage ls ${export_dir} | tail -n 1)\n",
     "else\n",
     "saved_model_dir=${export_dir}/$(ls ${export_dir} | tail -n 1)\n",
     "fi\n",
diff --git a/00_Miscellaneous/tfx/01_tf_estimator_deepdive.ipynb b/00_Miscellaneous/tfx/01_tf_estimator_deepdive.ipynb
index 5846418..d5f1952 100644
--- a/00_Miscellaneous/tfx/01_tf_estimator_deepdive.ipynb
+++ b/00_Miscellaneous/tfx/01_tf_estimator_deepdive.ipynb
@@ -107,8 +107,8 @@
    "source": [
     "DATA_DIR='data'\n",
     "!mkdir $DATA_DIR\n",
-    "!gsutil cp gs://cloud-samples-data/ml-engine/census/data/adult.data.csv $DATA_DIR\n",
-    "!gsutil cp gs://cloud-samples-data/ml-engine/census/data/adult.test.csv $DATA_DIR"
+    "!gcloud storage cp gs://cloud-samples-data/ml-engine/census/data/adult.data.csv $DATA_DIR\n",
+    "!gcloud storage cp gs://cloud-samples-data/ml-engine/census/data/adult.test.csv $DATA_DIR"
    ]
   },
   {
@@ -694,7 +694,7 @@
    "outputs": [],
    "source": [
     "print model_dir\n",
-    "!gsutil ls {model_dir}"
+    "!gcloud storage ls {model_dir}"
    ]
   },
   {
@@ -784,7 +784,7 @@
    },
    "outputs": [],
    "source": [
-    "!gsutil ls gs://ksalama-gcs-cloudml/others/models/census/dnn_classifier/export/1552582374"
+    "!gcloud storage ls gs://ksalama-gcs-cloudml/others/models/census/dnn_classifier/export/1552582374"
    ]
   },
   {
@@ -804,7 +804,7 @@
     "%%bash\n",
     "\n",
     "saved_models_base=${MODEL_DIR}/export/\n",
-    "saved_model_dir=$(gsutil ls ${saved_models_base} | tail -n 1)\n",
+    "saved_model_dir=$(gcloud storage ls ${saved_models_base} | tail -n 1)\n",
     "saved_model_cli show --dir=${saved_model_dir} --all"
    ]
   },
@@ -937,7 +937,7 @@
    },
    "outputs": [],
    "source": [
-    "!gsutil ls {model_dir}/export/estimate"
+    "!gcloud storage ls {model_dir}/export/estimate"
    ]
   },
   {
@@ -1172,7 +1172,7 @@
     "%%bash\n",
     "\n",
     "saved_models_base=${MODEL_DIR}/export/\n",
-    "saved_model_dir=$(gsutil ls ${saved_models_base} | tail -n 1)\n",
+    "saved_model_dir=$(gcloud storage ls ${saved_models_base} | tail -n 1)\n",
     "saved_model_cli show --dir=${saved_model_dir} --all"
    ]
   },
diff --git a/00_Miscellaneous/tfx/03_eda_with_tfdv.ipynb b/00_Miscellaneous/tfx/03_eda_with_tfdv.ipynb
index 5d50497..6c5afde 100644
--- a/00_Miscellaneous/tfx/03_eda_with_tfdv.ipynb
+++ b/00_Miscellaneous/tfx/03_eda_with_tfdv.ipynb
@@ -559,8 +559,7 @@
     }
    ],
    "source": [
-    "#!gsutil ls {DATA_DIR}/*\n",
-    "!ls {DATA_DIR}/*"
+    "#!gcloud storage ls {DATA_DIR}/*\n",    "!ls {DATA_DIR}/*"
    ]
   },
   {
@@ -631,8 +630,7 @@
    ],
    "source": [
     "!ls {stats_location}\n",
-    "#!gsutil ls {stats_location}"
-   ]
+    "#!gcloud storage ls {stats_location}"   ]
   },
   {
    "cell_type": "markdown",
@@ -1333,8 +1331,7 @@
     }
    ],
    "source": [
-    "#!gsutil ls {DATA_DIR}/*\n",
-    "!ls {DATA_DIR}/*"
+    "#!gcloud storage ls {DATA_DIR}/*\n",    "!ls {DATA_DIR}/*"
    ]
   },
   {
diff --git a/Experimental/distribution/multi-node/mnist-keras-2/Keras_MNIST_CMLE.ipynb b/Experimental/distribution/multi-node/mnist-keras-2/Keras_MNIST_CMLE.ipynb
index bd1fa0a..51b5aed 100644
--- a/Experimental/distribution/multi-node/mnist-keras-2/Keras_MNIST_CMLE.ipynb
+++ b/Experimental/distribution/multi-node/mnist-keras-2/Keras_MNIST_CMLE.ipynb
@@ -81,7 +81,7 @@
    "source": [
     "%%bash --out MODEL_LOCATION\n",
     "\n",
-    "MODEL_LOCATION=$(gsutil ls gs://$BUCKET/$MODEL_DIR/export | tail -1)\n",
+    "MODEL_LOCATION=$(gcloud storage ls gs://$BUCKET/$MODEL_DIR/export | tail -1)\n",
     "echo $MODEL_LOCATION"
    ]
   },
diff --git a/Experimental/distribution/multi-node/mnist-keras-2/Keras_MNIST_local.ipynb b/Experimental/distribution/multi-node/mnist-keras-2/Keras_MNIST_local.ipynb
index 1310056..a1bfb2a 100644
--- a/Experimental/distribution/multi-node/mnist-keras-2/Keras_MNIST_local.ipynb
+++ b/Experimental/distribution/multi-node/mnist-keras-2/Keras_MNIST_local.ipynb
@@ -258,7 +258,7 @@
    "source": [
     "%%bash\n",
     "\n",
-    "MODEL_LOCATION=$(gsutil ls gs://${BUCKET_NAME}/${MODEL_NAME}/export/ | tail -1)\n",
+    "MODEL_LOCATION=$(gcloud storage ls gs://${BUCKET_NAME}/${MODEL_NAME}/export/ | tail -1)\n",
     "echo \"export model location:\" ${MODEL_LOCATION}\n",
     "\n",
     "saved_model_cli show --dir ${MODEL_LOCATION} --tag_set serve --signature_def serving_default"