From f57ed5a5765ae2a36a883dce10e678c5b7dce9d8 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 27 Jan 2026 11:49:09 +0000 Subject: [PATCH 1/3] chore: Migrate gsutil usage to gcloud storage --- bazel/deps-cache.py | 2 +- ci/cloudbuild/builds/rotate-keys.sh | 8 ++++---- ci/cloudbuild/notifiers/bigquery/deploy.sh | 2 +- ci/cloudbuild/notifiers/logs/README.md | 6 +++--- ci/kokoro/lib/vcpkg.sh | 2 +- external/googleapis/renovate.sh | 2 +- google/cloud/storage/doc/setup-test-resources.md | 6 ++++-- google/cloud/storage/hashing_options_test.cc | 4 ++-- .../storage/internal/grpc/object_metadata_parser_test.cc | 2 +- google/cloud/storage/testing/upload_hash_cases.cc | 4 ++-- 10 files changed, 20 insertions(+), 18 deletions(-) diff --git a/bazel/deps-cache.py b/bazel/deps-cache.py index d351455ace2a7..e983706d4545f 100755 --- a/bazel/deps-cache.py +++ b/bazel/deps-cache.py @@ -160,7 +160,7 @@ def verify(tmpdir, name, source, cache, upload, **kwargs): same = cmp(source_file, cache_file) if not same and args.populate: print(f"[ Uploading {upload} ]") - subprocess.run(["gsutil", "-q", "cp", source_file, upload], check=False) + subprocess.run(["gcloud", "storage", "cp", source_file, upload], check=False) print(f"[ Reverifying {cache} ]") urlretrieve(cache, cache_file) same = cmp(source_file, cache_file) diff --git a/ci/cloudbuild/builds/rotate-keys.sh b/ci/cloudbuild/builds/rotate-keys.sh index cb3e28c944b91..6d42c1cf03011 100755 --- a/ci/cloudbuild/builds/rotate-keys.sh +++ b/ci/cloudbuild/builds/rotate-keys.sh @@ -69,12 +69,12 @@ for key_base in "${active_key_bases[@]}"; do for filetype in "json" "p12"; do bucket_path="${bucket}/${key_base}.${filetype}" io::log "Checking for active key at ${bucket_path}" - if ! gsutil -q stat "${bucket_path}"; then + if ! gcloud storage objects list --stat --fetch-encrypted-object-hashes "${bucket_path}"; then io::log "Not found. Creating ${bucket_path}" gcloud iam service-accounts keys create - \ --iam-account="${account}" \ --key-file-type="${filetype}" | - gsutil cp - "${bucket_path}" + gcloud storage cp - "${bucket_path}" fi done done @@ -84,9 +84,9 @@ stale_key_base="key-$(date +"%Y-%m" --date="now - 45 days")" for filetype in "json" "p12"; do bucket_path="${bucket}/${stale_key_base}.${filetype}" io::log "Checking for stale key at ${bucket_path}" - if gsutil -q stat "${bucket_path}"; then + if gcloud storage objects list --stat --fetch-encrypted-object-hashes "${bucket_path}"; then io::log "Removing ${bucket_path}" - gsutil rm "${bucket_path}" + gcloud storage rm "${bucket_path}" fi done diff --git a/ci/cloudbuild/notifiers/bigquery/deploy.sh b/ci/cloudbuild/notifiers/bigquery/deploy.sh index d620eea95ffb9..055a420c5fd4e 100755 --- a/ci/cloudbuild/notifiers/bigquery/deploy.sh +++ b/ci/cloudbuild/notifiers/bigquery/deploy.sh @@ -30,7 +30,7 @@ cd "${PROGRAM_DIR}" readonly BUCKET="gs://cloud-cpp-testing-resources-configs/google-cloud-cpp" io::log_h1 "Copying bigquery.yaml to ${BUCKET}" -gsutil cp bigquery.yaml gs://cloud-cpp-testing-resources-configs/google-cloud-cpp +gcloud storage cp bigquery.yaml gs://cloud-cpp-testing-resources-configs/google-cloud-cpp readonly SERVICE="google-cloud-cpp-gcb-bigquery-notifier" io::log_h1 "Deploying Cloud Run service ${SERVICE}" diff --git a/ci/cloudbuild/notifiers/logs/README.md b/ci/cloudbuild/notifiers/logs/README.md index 77ed00733051d..a65bb762ac1ba 100644 --- a/ci/cloudbuild/notifiers/logs/README.md +++ b/ci/cloudbuild/notifiers/logs/README.md @@ -45,9 +45,9 @@ PROJECT_NUMBER=$(gcloud projects list \ --format="value(project_number)" \ --limit=1) -gsutil iam ch \ - "serviceAccount:${PROJECT_NUMBER}-compute@developer.gserviceaccount.com:objectAdmin" \ - "gs://${BUCKET_NAME}" +gcloud storage buckets add-iam-policy-binding "gs://${BUCKET_NAME}" \ + --member="serviceAccount:${PROJECT_NUMBER}-compute@developer.gserviceaccount.com" \ + --role="objectAdmin" ``` ## Create the Docker image diff --git a/ci/kokoro/lib/vcpkg.sh b/ci/kokoro/lib/vcpkg.sh index 17e6e21bfc8d2..4a7975358a7ed 100644 --- a/ci/kokoro/lib/vcpkg.sh +++ b/ci/kokoro/lib/vcpkg.sh @@ -48,7 +48,7 @@ install_vcpkg() { create_gcloud_config activate_service_account_keyfile "${CACHE_KEYFILE}" export CLOUDSDK_ACTIVE_CONFIG_NAME="${GCLOUD_CONFIG}" - io::run gsutil ls "gs://${CACHE_BUCKET}/" + io::run gcloud storage ls "gs://${CACHE_BUCKET}/" # Eventually we can remove this, but the current caches contain both `ccache` # files (which we want), and `vcpkg` files (which we don't want). io::run rm -fr "${HOME}/.cache/vcpkg" diff --git a/external/googleapis/renovate.sh b/external/googleapis/renovate.sh index 1bb5af0057a85..c4d44f1ef7367 100755 --- a/external/googleapis/renovate.sh +++ b/external/googleapis/renovate.sh @@ -38,7 +38,7 @@ fi DOWNLOAD="$(mktemp)" curl -fsSL "https://github.com/${REPO}/archive/${COMMIT}.tar.gz" -o "${DOWNLOAD}" -gsutil -q cp "${DOWNLOAD}" "gs://cloud-cpp-community-archive/com_google_googleapis/${COMMIT}.tar.gz" +gcloud storage cp "${DOWNLOAD}" "gs://cloud-cpp-community-archive/com_google_googleapis/${COMMIT}.tar.gz" SHA256=$(sha256sum "${DOWNLOAD}" | sed "s/ .*//") SHA256_BASE64=$(openssl dgst -sha256 -binary <"${DOWNLOAD}" | openssl base64 -A) PIPERORIGIN_REVID= diff --git a/google/cloud/storage/doc/setup-test-resources.md b/google/cloud/storage/doc/setup-test-resources.md index bed8089f9c31c..683da109f8723 100644 --- a/google/cloud/storage/doc/setup-test-resources.md +++ b/google/cloud/storage/doc/setup-test-resources.md @@ -69,6 +69,7 @@ list of allowed locations. In this example, we'll use `us-central1`. ```console BUCKET=... # e.g. cloud-cpp-testing-bucket +# Gsutil command could not be translated. Reason: The top-level flag "-l" is not a valid gsutil flag and is not found in the migration guide. gsutil -l us-central1 gs://${BUCKET} ``` @@ -79,6 +80,7 @@ initial bucket. ```console DESTINATION_BUCKET=... # e.g. cloud-cpp-testing-destination-bucket +# Gsutil command could not be translated. Reason: The top-level flag "-l" is not a valid gsutil flag and is not found in the migration guide. gsutil -l us-east1 gs://${DESTINATION_BUCKET} ``` @@ -106,13 +108,13 @@ Authorize our service account to use the new key: ``` KEYNAME=projects/${PROJECT_NAME}/locations/global/keyRings/${KR}/cryptoKeys/${KEYID} -gsutil kms authorize -p ${PROJECT_NAME} -k ${KEYNAME} +gcloud storage service-agent --project ${PROJECT_NAME} --authorize-cmek=${KEYNAME} ``` Configure the bucket to use this new key as the default: ``` -gsutil kms encryption -k ${KEYNAME} gs://${BUCKET} +gcloud storage buckets update gs://${BUCKET} --default-encryption-key=${KEYNAME} ``` ## Setting up a Service Account for HMAC Keys diff --git a/google/cloud/storage/hashing_options_test.cc b/google/cloud/storage/hashing_options_test.cc index 186fe6298668f..047731d655c6b 100644 --- a/google/cloud/storage/hashing_options_test.cc +++ b/google/cloud/storage/hashing_options_test.cc @@ -41,7 +41,7 @@ TEST(ComputeMD5HashTest, Simple) { TEST(ComputeCrc32cChecksumTest, Empty) { std::string actual = ComputeCrc32cChecksum(""); // Use this command to get the expected value: - // echo -n '' > foo.txt && gsutil hash foo.txt + // echo -n '' > foo.txt && gcloud storage hash foo.txt EXPECT_EQ("AAAAAA==", actual); } @@ -50,7 +50,7 @@ TEST(ComputeCrc32cChecksumTest, Simple) { ComputeCrc32cChecksum("The quick brown fox jumps over the lazy dog"); // I used this command to get the expected value: // /bin/echo -n "The quick brown fox jumps over the lazy dog" > foo.txt && - // gsutil hash foo.txt + // gcloud storage hash foo.txt EXPECT_EQ("ImIEBA==", actual); } diff --git a/google/cloud/storage/internal/grpc/object_metadata_parser_test.cc b/google/cloud/storage/internal/grpc/object_metadata_parser_test.cc index 16974958f2b71..a497a5b0ef7cc 100644 --- a/google/cloud/storage/internal/grpc/object_metadata_parser_test.cc +++ b/google/cloud/storage/internal/grpc/object_metadata_parser_test.cc @@ -55,7 +55,7 @@ TEST(GrpcClientFromProto, ObjectSimple) { seconds: 1565194924 nanos: 234567890 } - # These magic numbers can be obtained using `gsutil hash` and then + # These magic numbers can be obtained using `gcloud storage hash` and then # transforming the output from base64 to binary using tools like xxd(1). checksums { crc32c: 576848900 diff --git a/google/cloud/storage/testing/upload_hash_cases.cc b/google/cloud/storage/testing/upload_hash_cases.cc index 21586fb4a00d9..35a7dcb23b11d 100644 --- a/google/cloud/storage/testing/upload_hash_cases.cc +++ b/google/cloud/storage/testing/upload_hash_cases.cc @@ -21,12 +21,12 @@ namespace storage { namespace testing { namespace { // These values were obtained using: -// /bin/echo -n '' > foo.txt && gsutil hash foo.txt +// /bin/echo -n '' > foo.txt && gcloud storage hash foo.txt auto constexpr kEmptyStringCrc32cChecksum = "AAAAAA=="; auto constexpr kEmptyStringMD5Hash = "1B2M2Y8AsgTpgAmY7PhCfg=="; // /bin/echo -n 'The quick brown fox jumps over the lazy dog' > foo.txt -// gsutil hash foo.txt +// gcloud storage hash foo.txt auto constexpr kQuickFoxCrc32cChecksum = "ImIEBA=="; auto constexpr kQuickFoxMD5Hash = "nhB9nTcrtoJr2B01QqQZ1g=="; From 8564fba7d8e8c8d39e02c38834b38747d9be74da Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Wed, 28 Jan 2026 11:39:33 +0530 Subject: [PATCH 2/3] Apply suggestion from @gemini-code-assist[bot] Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- google/cloud/storage/doc/setup-test-resources.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/google/cloud/storage/doc/setup-test-resources.md b/google/cloud/storage/doc/setup-test-resources.md index 683da109f8723..2ca63f6a7e54c 100644 --- a/google/cloud/storage/doc/setup-test-resources.md +++ b/google/cloud/storage/doc/setup-test-resources.md @@ -80,8 +80,7 @@ initial bucket. ```console DESTINATION_BUCKET=... # e.g. cloud-cpp-testing-destination-bucket -# Gsutil command could not be translated. Reason: The top-level flag "-l" is not a valid gsutil flag and is not found in the migration guide. -gsutil -l us-east1 gs://${DESTINATION_BUCKET} +gcloud storage buckets create gs://${DESTINATION_BUCKET} --location=us-east1 ``` ## Setting up a Pub/Sub Topic From 2ba366db562c6c15c6a2bf802b83d79df040d754 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Wed, 28 Jan 2026 11:41:25 +0530 Subject: [PATCH 3/3] Update setup-test-resources.md --- google/cloud/storage/doc/setup-test-resources.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/google/cloud/storage/doc/setup-test-resources.md b/google/cloud/storage/doc/setup-test-resources.md index 2ca63f6a7e54c..c82124c9818b4 100644 --- a/google/cloud/storage/doc/setup-test-resources.md +++ b/google/cloud/storage/doc/setup-test-resources.md @@ -69,8 +69,7 @@ list of allowed locations. In this example, we'll use `us-central1`. ```console BUCKET=... # e.g. cloud-cpp-testing-bucket -# Gsutil command could not be translated. Reason: The top-level flag "-l" is not a valid gsutil flag and is not found in the migration guide. -gsutil -l us-central1 gs://${BUCKET} +gcloud storage buckets create gs://${BUCKET} --location=us-central1 ``` ## Setting up a Destination Bucket