Skip to content

chore: Migrate gsutil usage to gcloud storage #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gurusai-voleti wants to merge 1 commit into
base: master
Choose a base branch
from
Open

chore: Migrate gsutil usage to gcloud storage #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions ...luation_pipeline/Tutorial - TF Model Evaluation Pipeline with Dataflow and BigQuery.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,10 +219,10 @@
"source": [
"%%bash\n",
"\n",
"if ! gsutil ls | grep -q gs://${BUCKET}/; then\n",
" gsutil mb -l ${REGION} gs://${BUCKET}\n",
"if ! gcloud storage ls | grep -q gs://${BUCKET}/; then\n",
" gcloud storage buckets create --location ${REGION} gs://${BUCKET}\n",
" # copy canonical set of preprocessed files if you didn't do previous notebook\n",
" gsutil -m cp -R gs://cloud-training-demos/babyweight gs://${BUCKET}\n",
" gcloud storage cp --recursive gs://cloud-training-demos/babyweight gs://${BUCKET}\n",
"fi"
]
},
Expand All @@ -242,7 +242,7 @@
"JOBNAME=babyweight_$(date -u +%y%m%d_%H%M%S)\n",
"echo $OUTDIR $REGION $JOBNAME\n",
"\n",
"gsutil -m rm -rf $OUTDIR\n",
"gcloud storage rm --recursive --continue-on-error $OUTDIR\n",
"\n",
"gcloud ml-engine jobs submit training $JOBNAME \\\n",
" --region=$REGION \\\n",
Expand Down
6 changes: 3 additions & 3 deletions 00_Miscellaneous/text-similarity-analysis/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ export BUCKET=gs://[your-bucket-name]
mkdir temp reuters
tar -zxvf reuters21578.tar.gz -C temp/
mv temp/*.sgm reuters/ && rm -rf temp
gsutil mb $BUCKET
gsutil -m cp -R reuters $BUCKET
gcloud storage buckets create $BUCKET
gcloud storage cp --recursive reuters $BUCKET
```

## Setup python environment and sample code
Expand Down Expand Up @@ -108,7 +108,7 @@ Before running the pipeline, you can remove previous working directory
with below command if you want.

```bash
gsutil rm -r $ROOT
gcloud storage rm --recursive $ROOT
```

Finally, you can run the pipeline with this command.
Expand Down
2 changes: 1 addition & 1 deletion 00_Miscellaneous/text-similarity-analysis/etl/run_pipeline.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ JOB_NAME=text-analysis
MACHINE_TYPE=n1-highmem-2

# Remove Root directory before running dataflow job.
gsutil rm -r $ROOT_DIR
gcloud storage rm --recursive $ROOT_DIR

# Command to invoke dataflow job.
python run_pipeline.py \
Expand Down
4 changes: 1 addition & 3 deletions ...rain_eval_export/Tutorial - TensorFlow Estimator Train, Evaluate, Export, Explained.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,7 @@
"source": [
"DATA_DIR='data'\n",
"!mkdir $DATA_DIR\n",
"!gsutil cp gs://cloud-samples-data/ml-engine/census/data/adult.data.csv $DATA_DIR\n",
"!gsutil cp gs://cloud-samples-data/ml-engine/census/data/adult.test.csv $DATA_DIR"
]
"!gcloud storage cp gs://cloud-samples-data/ml-engine/census/data/adult.data.csv $DATA_DIR\n", "!gcloud storage cp gs://cloud-samples-data/ml-engine/census/data/adult.test.csv $DATA_DIR" ]
},
{
"cell_type": "code",
Expand Down
8 changes: 4 additions & 4 deletions 00_Miscellaneous/tf_transform/tft-01 - Babyweight preprocessing with tf.Transform.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -589,28 +589,28 @@
"echo 'transformed data:' \n",
"if [ \"${RUN_LOCAL}\" = \"True \" ] \n",
"then ls ${ROOT_DIR}/transformed \n",
"else gsutil ls gs://${BUCKET}/${ROOT_DIR}/transformed \n",
"else gcloud storage ls gs://${BUCKET}/${ROOT_DIR}/transformed \n",
"fi\n",
"echo ''\n",
"\n",
"echo 'transformed metadata:' \n",
"if [ \"${RUN_LOCAL}\" = \"True \" ] \n",
"then ls ${ROOT_DIR}/transform/transformed_metadata\n",
"else gsutil ls gs://${BUCKET}/${ROOT_DIR}/transform/transformed_metadata \n",
"else gcloud storage ls gs://${BUCKET}/${ROOT_DIR}/transform/transformed_metadata \n",
"fi\n",
"echo ''\n",
"\n",
"echo 'transform artefact:' \n",
"if [ \"${RUN_LOCAL}\" = \"True \" ] \n",
"then ls ${ROOT_DIR}/transform/transform_fn\n",
"else gsutil ls gs://${BUCKET}/${ROOT_DIR}/transform/transform_fn \n",
"else gcloud storage ls gs://${BUCKET}/${ROOT_DIR}/transform/transform_fn \n",
"fi\n",
"echo ''\n",
"\n",
"echo 'transform assets:'\n",
"if [ \"${RUN_LOCAL}\" = \"True \" ] \n",
"then ls ${ROOT_DIR}/transform/transform_fn/assets\n",
"else gsutil ls gs://${BUCKET}/${ROOT_DIR}/transform/transform_fn/assets \n",
"else gcloud storage ls gs://${BUCKET}/${ROOT_DIR}/transform/transform_fn/assets \n",
"fi\n",
"echo ''"
]
Expand Down
2 changes: 1 addition & 1 deletion 00_Miscellaneous/tf_transform/tft-02 - Babyweight Estimation with Transformed Data.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -446,7 +446,7 @@
"%%bash\n",
"if [ ${RUN_LOCAL} ]\n",
"then \n",
"saved_model_dir=$(gsutil ls ${export_dir} | tail -n 1)\n",
"saved_model_dir=$(gcloud storage ls ${export_dir} | tail -n 1)\n",
"else\n",
"saved_model_dir=${export_dir}/$(ls ${export_dir} | tail -n 1)\n",
"fi\n",
Expand Down
14 changes: 7 additions & 7 deletions 00_Miscellaneous/tfx/01_tf_estimator_deepdive.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,8 @@
"source": [
"DATA_DIR='data'\n",
"!mkdir $DATA_DIR\n",
"!gsutil cp gs://cloud-samples-data/ml-engine/census/data/adult.data.csv $DATA_DIR\n",
"!gsutil cp gs://cloud-samples-data/ml-engine/census/data/adult.test.csv $DATA_DIR"
"!gcloud storage cp gs://cloud-samples-data/ml-engine/census/data/adult.data.csv $DATA_DIR\n",
"!gcloud storage cp gs://cloud-samples-data/ml-engine/census/data/adult.test.csv $DATA_DIR"
]
},
{
Expand Down Expand Up @@ -694,7 +694,7 @@
"outputs": [],
"source": [
"print model_dir\n",
"!gsutil ls {model_dir}"
"!gcloud storage ls {model_dir}"
]
},
{
Expand Down Expand Up @@ -784,7 +784,7 @@
},
"outputs": [],
"source": [
"!gsutil ls gs://ksalama-gcs-cloudml/others/models/census/dnn_classifier/export/1552582374"
"!gcloud storage ls gs://ksalama-gcs-cloudml/others/models/census/dnn_classifier/export/1552582374"
]
},
{
Expand All @@ -804,7 +804,7 @@
"%%bash\n",
"\n",
"saved_models_base=${MODEL_DIR}/export/\n",
"saved_model_dir=$(gsutil ls ${saved_models_base} | tail -n 1)\n",
"saved_model_dir=$(gcloud storage ls ${saved_models_base} | tail -n 1)\n",
"saved_model_cli show --dir=${saved_model_dir} --all"
]
},
Expand Down Expand Up @@ -937,7 +937,7 @@
},
"outputs": [],
"source": [
"!gsutil ls {model_dir}/export/estimate"
"!gcloud storage ls {model_dir}/export/estimate"
]
},
{
Expand Down Expand Up @@ -1172,7 +1172,7 @@
"%%bash\n",
"\n",
"saved_models_base=${MODEL_DIR}/export/\n",
"saved_model_dir=$(gsutil ls ${saved_models_base} | tail -n 1)\n",
"saved_model_dir=$(gcloud storage ls ${saved_models_base} | tail -n 1)\n",
"saved_model_cli show --dir=${saved_model_dir} --all"
]
},
Expand Down
9 changes: 3 additions & 6 deletions 00_Miscellaneous/tfx/03_eda_with_tfdv.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -559,8 +559,7 @@
}
],
"source": [
"#!gsutil ls {DATA_DIR}/*\n",
"!ls {DATA_DIR}/*"
"#!gcloud storage ls {DATA_DIR}/*\n", "!ls {DATA_DIR}/*"
]
},
{
Expand Down Expand Up @@ -631,8 +630,7 @@
],
"source": [
"!ls {stats_location}\n",
"#!gsutil ls {stats_location}"
]
"#!gcloud storage ls {stats_location}" ]
},
{
"cell_type": "markdown",
Expand Down Expand Up @@ -1333,8 +1331,7 @@
}
],
"source": [
"#!gsutil ls {DATA_DIR}/*\n",
"!ls {DATA_DIR}/*"
"#!gcloud storage ls {DATA_DIR}/*\n", "!ls {DATA_DIR}/*"
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion Experimental/distribution/multi-node/mnist-keras-2/Keras_MNIST_CMLE.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@
"source": [
"%%bash --out MODEL_LOCATION\n",
"\n",
"MODEL_LOCATION=$(gsutil ls gs://$BUCKET/$MODEL_DIR/export | tail -1)\n",
"MODEL_LOCATION=$(gcloud storage ls gs://$BUCKET/$MODEL_DIR/export | tail -1)\n",
"echo $MODEL_LOCATION"
]
},
Expand Down
2 changes: 1 addition & 1 deletion Experimental/distribution/multi-node/mnist-keras-2/Keras_MNIST_local.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@
"source": [
"%%bash\n",
"\n",
"MODEL_LOCATION=$(gsutil ls gs://${BUCKET_NAME}/${MODEL_NAME}/export/ | tail -1)\n",
"MODEL_LOCATION=$(gcloud storage ls gs://${BUCKET_NAME}/${MODEL_NAME}/export/ | tail -1)\n",
"echo \"export model location:\" ${MODEL_LOCATION}\n",
"\n",
"saved_model_cli show --dir ${MODEL_LOCATION} --tag_set serve --signature_def serving_default"
Expand Down