chore: Migrate gsutil usage to gcloud storage

bazel/deps-cache.py

@@ -160,7 +160,7 @@ def verify(tmpdir, name, source, cache, upload, **kwargs):
     same = cmp(source_file, cache_file)
     if not same and args.populate:
         print(f"[ Uploading {upload} ]")
-        subprocess.run(["gsutil", "-q", "cp", source_file, upload], check=False)
+        subprocess.run(["gcloud", "storage", "cp", source_file, upload], check=False)
         print(f"[ Reverifying {cache} ]")
         urlretrieve(cache, cache_file)
         same = cmp(source_file, cache_file)

ci/cloudbuild/builds/rotate-keys.sh

@@ -69,12 +69,12 @@ for key_base in "${active_key_bases[@]}"; do
   for filetype in "json" "p12"; do
     bucket_path="${bucket}/${key_base}.${filetype}"
     io::log "Checking for active key at ${bucket_path}"
-    if ! gsutil -q stat "${bucket_path}"; then
+    if ! gcloud storage objects list --stat --fetch-encrypted-object-hashes "${bucket_path}"; then
       io::log "Not found. Creating ${bucket_path}"
       gcloud iam service-accounts keys create - \
         --iam-account="${account}" \
         --key-file-type="${filetype}" |
-        gsutil cp - "${bucket_path}"
+        gcloud storage cp - "${bucket_path}"
     fi
   done
 done
@@ -84,9 +84,9 @@ stale_key_base="key-$(date +"%Y-%m" --date="now - 45 days")"
 for filetype in "json" "p12"; do
   bucket_path="${bucket}/${stale_key_base}.${filetype}"
   io::log "Checking for stale key at ${bucket_path}"
-  if gsutil -q stat "${bucket_path}"; then
+  if gcloud storage objects list --stat --fetch-encrypted-object-hashes "${bucket_path}"; then
     io::log "Removing ${bucket_path}"
-    gsutil rm "${bucket_path}"
+    gcloud storage rm "${bucket_path}"
   fi
 done
 

ci/cloudbuild/notifiers/bigquery/deploy.sh

@@ -30,7 +30,7 @@ cd "${PROGRAM_DIR}"
 
 readonly BUCKET="gs://cloud-cpp-testing-resources-configs/google-cloud-cpp"
 io::log_h1 "Copying bigquery.yaml to ${BUCKET}"
-gsutil cp bigquery.yaml gs://cloud-cpp-testing-resources-configs/google-cloud-cpp
+gcloud storage cp bigquery.yaml gs://cloud-cpp-testing-resources-configs/google-cloud-cpp
 
 readonly SERVICE="google-cloud-cpp-gcb-bigquery-notifier"
 io::log_h1 "Deploying Cloud Run service ${SERVICE}"

ci/cloudbuild/notifiers/logs/README.md

@@ -45,9 +45,9 @@ PROJECT_NUMBER=$(gcloud projects list \
     --format="value(project_number)" \
     --limit=1)
 
-gsutil iam ch \
-    "serviceAccount:${PROJECT_NUMBER}-compute@developer.gserviceaccount.com:objectAdmin" \
-    "gs://${BUCKET_NAME}"
+gcloud storage buckets add-iam-policy-binding "gs://${BUCKET_NAME}" \
+    --member="serviceAccount:${PROJECT_NUMBER}-compute@developer.gserviceaccount.com" \
+    --role="objectAdmin"
 ```
 
 ## Create the Docker image

ci/kokoro/lib/vcpkg.sh

@@ -48,7 +48,7 @@ install_vcpkg() {
   create_gcloud_config
   activate_service_account_keyfile "${CACHE_KEYFILE}"
   export CLOUDSDK_ACTIVE_CONFIG_NAME="${GCLOUD_CONFIG}"
-  io::run gsutil ls "gs://${CACHE_BUCKET}/"
+  io::run gcloud storage ls "gs://${CACHE_BUCKET}/"
   # Eventually we can remove this, but the current caches contain both `ccache`
   # files (which we want), and `vcpkg` files (which we don't want).
   io::run rm -fr "${HOME}/.cache/vcpkg"

external/googleapis/renovate.sh

@@ -38,7 +38,7 @@ fi
 
 DOWNLOAD="$(mktemp)"
 curl -fsSL "https://github.com/${REPO}/archive/${COMMIT}.tar.gz" -o "${DOWNLOAD}"
-gsutil -q cp "${DOWNLOAD}" "gs://cloud-cpp-community-archive/com_google_googleapis/${COMMIT}.tar.gz"
+gcloud storage cp "${DOWNLOAD}" "gs://cloud-cpp-community-archive/com_google_googleapis/${COMMIT}.tar.gz"
 SHA256=$(sha256sum "${DOWNLOAD}" | sed "s/ .*//")
 SHA256_BASE64=$(openssl dgst -sha256 -binary <"${DOWNLOAD}" | openssl base64 -A)
 PIPERORIGIN_REVID=

google/cloud/storage/doc/setup-test-resources.md

@@ -69,7 +69,7 @@ list of allowed locations. In this example, we'll use `us-central1`.
 
 ```console
 BUCKET=... # e.g. cloud-cpp-testing-bucket
-gsutil -l us-central1 gs://${BUCKET}
+gcloud storage buckets create gs://${BUCKET} --location=us-central1
 ```
 
 ## Setting up a Destination Bucket
@@ -79,7 +79,7 @@ initial bucket.
 
 ```console
 DESTINATION_BUCKET=... # e.g. cloud-cpp-testing-destination-bucket
-gsutil -l us-east1 gs://${DESTINATION_BUCKET}
+gcloud storage buckets create gs://${DESTINATION_BUCKET} --location=us-east1
 ```
 
 ## Setting up a Pub/Sub Topic
@@ -106,13 +106,13 @@ Authorize our service account to use the new key:
 
 ```
 KEYNAME=projects/${PROJECT_NAME}/locations/global/keyRings/${KR}/cryptoKeys/${KEYID}
-gsutil kms authorize -p ${PROJECT_NAME} -k ${KEYNAME}
+gcloud storage service-agent --project ${PROJECT_NAME} --authorize-cmek=${KEYNAME}
 ```
 
 Configure the bucket to use this new key as the default:
 
 ```
-gsutil kms encryption -k ${KEYNAME} gs://${BUCKET}
+gcloud storage buckets update gs://${BUCKET} --default-encryption-key=${KEYNAME}
 ```
 
 ## Setting up a Service Account for HMAC Keys

google/cloud/storage/hashing_options_test.cc

@@ -41,7 +41,7 @@ TEST(ComputeMD5HashTest, Simple) {
 TEST(ComputeCrc32cChecksumTest, Empty) {
   std::string actual = ComputeCrc32cChecksum("");
   // Use this command to get the expected value:
-  // echo -n '' > foo.txt && gsutil hash foo.txt
+  // echo -n '' > foo.txt && gcloud storage hash foo.txt
   EXPECT_EQ("AAAAAA==", actual);
 }
 
@@ -50,7 +50,7 @@ TEST(ComputeCrc32cChecksumTest, Simple) {
       ComputeCrc32cChecksum("The quick brown fox jumps over the lazy dog");
   // I used this command to get the expected value:
   // /bin/echo -n "The quick brown fox jumps over the lazy dog" > foo.txt &&
-  // gsutil hash foo.txt
+  // gcloud storage hash foo.txt
   EXPECT_EQ("ImIEBA==", actual);
 }
 

google/cloud/storage/internal/grpc/object_metadata_parser_test.cc

@@ -55,7 +55,7 @@ TEST(GrpcClientFromProto, ObjectSimple) {
       seconds: 1565194924
       nanos: 234567890
     }
-    # These magic numbers can be obtained using `gsutil hash` and then
+    # These magic numbers can be obtained using `gcloud storage hash` and then
     # transforming the output from base64 to binary using tools like xxd(1).
     checksums {
       crc32c: 576848900

google/cloud/storage/testing/upload_hash_cases.cc

@@ -21,12 +21,12 @@ namespace storage {
 namespace testing {
 namespace {
 // These values were obtained using:
-// /bin/echo -n '' > foo.txt && gsutil hash foo.txt
+// /bin/echo -n '' > foo.txt && gcloud storage hash foo.txt
 auto constexpr kEmptyStringCrc32cChecksum = "AAAAAA==";
 auto constexpr kEmptyStringMD5Hash = "1B2M2Y8AsgTpgAmY7PhCfg==";
 
 // /bin/echo -n 'The quick brown fox jumps over the lazy dog' > foo.txt
-// gsutil hash foo.txt
+// gcloud storage hash foo.txt
 auto constexpr kQuickFoxCrc32cChecksum = "ImIEBA==";
 auto constexpr kQuickFoxMD5Hash = "nhB9nTcrtoJr2B01QqQZ1g==";