From e6322a7330de64e9106260f288baea327e266e25 Mon Sep 17 00:00:00 2001 From: Rick Getz Date: Tue, 18 Nov 2025 12:25:34 -0500 Subject: [PATCH 1/5] feat: allow concurrent artifact uploads --- samcli/commands/deploy/command.py | 12 +++ samcli/commands/deploy/core/options.py | 1 + samcli/commands/deploy/deploy_context.py | 3 + samcli/commands/deploy/guided_context.py | 13 +++ samcli/commands/deploy/utils.py | 3 + samcli/commands/package/package_context.py | 3 + samcli/lib/package/artifact_exporter.py | 71 +++++++++++++- samcli/lib/package/ecr_uploader.py | 8 +- schema/samcli.json | 7 +- tests/unit/commands/deploy/test_command.py | 36 +++++-- .../lib/package/test_artifact_exporter.py | 93 +++++++++++++++++++ 11 files changed, 237 insertions(+), 13 deletions(-) diff --git a/samcli/commands/deploy/command.py b/samcli/commands/deploy/command.py index dd750ce5e5..ff0d7e2943 100644 --- a/samcli/commands/deploy/command.py +++ b/samcli/commands/deploy/command.py @@ -146,6 +146,12 @@ @image_repository_option @image_repositories_option @force_upload_option +@click.option( + "--parallel-upload", + is_flag=True, + default=False, + help="Enable parallel upload of artifacts to S3/ECR during packaging before deployment.", +) @s3_prefix_option @kms_key_id_option @role_arn_option @@ -177,6 +183,7 @@ def cli( image_repository, image_repositories, force_upload, + parallel_upload, no_progressbar, s3_prefix, kms_key_id, @@ -212,6 +219,7 @@ def cli( image_repository, image_repositories, force_upload, + parallel_upload, no_progressbar, s3_prefix, kms_key_id, @@ -246,6 +254,7 @@ def do_cli( image_repository, image_repositories, force_upload, + parallel_upload, no_progressbar, s3_prefix, kms_key_id, @@ -300,6 +309,7 @@ def do_cli( config_env=config_env, config_file=config_file, disable_rollback=disable_rollback, + parallel_upload=parallel_upload, ) guided_context.run() else: @@ -331,6 +341,7 @@ def do_cli( kms_key_id=kms_key_id, use_json=use_json, force_upload=force_upload, + parallel_upload=guided_context.guided_parallel_upload if guided else parallel_upload, no_progressbar=no_progressbar, metadata=metadata, on_deploy=True, @@ -357,6 +368,7 @@ def do_cli( image_repository=guided_context.guided_image_repository if guided else image_repository, image_repositories=guided_context.guided_image_repositories if guided else image_repositories, force_upload=force_upload, + parallel_upload=guided_context.guided_parallel_upload if guided else parallel_upload, no_progressbar=no_progressbar, s3_prefix=guided_context.guided_s3_prefix if guided else s3_prefix, kms_key_id=kms_key_id, diff --git a/samcli/commands/deploy/core/options.py b/samcli/commands/deploy/core/options.py index 44503368af..1af6dffdcb 100644 --- a/samcli/commands/deploy/core/options.py +++ b/samcli/commands/deploy/core/options.py @@ -37,6 +37,7 @@ "disable_rollback", "on_failure", "force_upload", + "parallel_upload", "max_wait_duration", ] diff --git a/samcli/commands/deploy/deploy_context.py b/samcli/commands/deploy/deploy_context.py index 33ac171156..7036910037 100644 --- a/samcli/commands/deploy/deploy_context.py +++ b/samcli/commands/deploy/deploy_context.py @@ -75,6 +75,7 @@ def __init__( poll_delay, on_failure, max_wait_duration, + parallel_upload=False, ): self.template_file = template_file self.stack_name = stack_name @@ -82,6 +83,7 @@ def __init__( self.image_repository = image_repository self.image_repositories = image_repositories self.force_upload = force_upload + self.parallel_upload = parallel_upload self.no_progressbar = no_progressbar self.s3_prefix = s3_prefix self.kms_key_id = kms_key_id @@ -164,6 +166,7 @@ def run(self): self.signing_profiles, self.use_changeset, self.disable_rollback, + self.parallel_upload, ) return self.deploy( self.stack_name, diff --git a/samcli/commands/deploy/guided_context.py b/samcli/commands/deploy/guided_context.py index 8657debd57..a2030647af 100644 --- a/samcli/commands/deploy/guided_context.py +++ b/samcli/commands/deploy/guided_context.py @@ -62,6 +62,7 @@ def __init__( config_env=None, config_file=None, disable_rollback=None, + parallel_upload=False, ): self.template_file = template_file self.stack_name = stack_name @@ -95,6 +96,8 @@ def __init__( self.color = Colored() self.function_provider = None self.disable_rollback = disable_rollback + self.parallel_upload = parallel_upload + self.guided_parallel_upload = None @property def guided_capabilities(self): @@ -162,6 +165,14 @@ def guided_prompts(self, parameter_override_keys): click.secho("\t#Preserves the state of previously provisioned resources when an operation fails") disable_rollback = confirm(f"\t{self.start_bold}Disable rollback{self.end_bold}", default=self.disable_rollback) + if self.parallel_upload: + parallel_upload = True + else: + click.secho("\t#Speed up artifact uploads by running them in parallel") + parallel_upload = confirm( + f"\t{self.start_bold}Enable parallel uploads{self.end_bold}", default=False + ) + self.prompt_authorization(stacks) self.prompt_code_signing_settings(stacks) @@ -204,6 +215,7 @@ def guided_prompts(self, parameter_override_keys): self.guided_s3_prefix = stack_name self.guided_region = region self.guided_profile = self.profile + self.guided_parallel_upload = parallel_upload self._capabilities = input_capabilities if input_capabilities else default_capabilities self._parameter_overrides = ( input_parameter_overrides if input_parameter_overrides else self.parameter_overrides_from_cmdline @@ -587,6 +599,7 @@ def run(self): capabilities=self._capabilities, signing_profiles=self.signing_profiles, disable_rollback=self.disable_rollback, + parallel_upload=self.guided_parallel_upload, ) @staticmethod diff --git a/samcli/commands/deploy/utils.py b/samcli/commands/deploy/utils.py index 421a16c75d..56270e5ae9 100644 --- a/samcli/commands/deploy/utils.py +++ b/samcli/commands/deploy/utils.py @@ -20,6 +20,7 @@ def print_deploy_args( signing_profiles, use_changeset, disable_rollback, + parallel_upload, ): """ Print a table of the values that are used during a sam deploy. @@ -48,6 +49,7 @@ def print_deploy_args( :param signing_profiles: Signing profile details which will be used to sign functions/layers :param use_changeset: Flag to use or skip the usage of changesets :param disable_rollback: Preserve the state of previously provisioned resources when an operation fails. + :param parallel_upload: Whether artifact uploads run in parallel prior to deployment. """ _parameters = parameter_overrides.copy() @@ -70,6 +72,7 @@ def print_deploy_args( if use_changeset: click.echo(f"\tConfirm changeset : {confirm_changeset}") click.echo(f"\tDisable rollback : {disable_rollback}") + click.echo(f"\tParallel uploads : {parallel_upload}") if image_repository: msg = "Deployment image repository : " # NOTE(sriram-mv): tab length is 8 spaces. diff --git a/samcli/commands/package/package_context.py b/samcli/commands/package/package_context.py index 4a64981745..1d813fef39 100644 --- a/samcli/commands/package/package_context.py +++ b/samcli/commands/package/package_context.py @@ -71,6 +71,7 @@ def __init__( parameter_overrides=None, on_deploy=False, signing_profiles=None, + parallel_upload=False, ): self.template_file = template_file self.s3_bucket = s3_bucket @@ -81,6 +82,7 @@ def __init__( self.output_template_file = output_template_file self.use_json = use_json self.force_upload = force_upload + self.parallel_upload = parallel_upload self.no_progressbar = no_progressbar self.metadata = metadata self.region = region @@ -161,6 +163,7 @@ def _export(self, template_path, use_json): self.code_signer, normalize_template=True, normalize_parameters=True, + parallel_upload=self.parallel_upload, ) exported_template = template.export() diff --git a/samcli/lib/package/artifact_exporter.py b/samcli/lib/package/artifact_exporter.py index bc38ee4a5e..dd8a661838 100644 --- a/samcli/lib/package/artifact_exporter.py +++ b/samcli/lib/package/artifact_exporter.py @@ -15,7 +15,9 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. import os -from typing import Dict, List, Optional +import threading +from concurrent.futures import FIRST_EXCEPTION, ThreadPoolExecutor, wait +from typing import Callable, Dict, List, Optional from botocore.utils import set_value_from_jmespath @@ -52,6 +54,28 @@ # NOTE: sriram-mv, A cyclic dependency on `Template` needs to be broken. +DEFAULT_PARALLEL_UPLOAD_WORKERS = max(4, min(32, (os.cpu_count() or 1) * 2)) + + +class _ThreadSafeUploadCache: + """Simple thread-safe mapping used to deduplicate uploads across threads.""" + + def __init__(self, initial: Optional[Dict[str, str]] = None): + self._cache = initial or {} + self._lock = threading.Lock() + + def __contains__(self, key: str) -> bool: # pragma: no cover - small helper + with self._lock: + return key in self._cache + + def __getitem__(self, key: str) -> str: # pragma: no cover - small helper + with self._lock: + return self._cache[key] + + def __setitem__(self, key: str, value: str) -> None: # pragma: no cover - small helper + with self._lock: + self._cache[key] = value + class CloudFormationStackResource(ResourceZip): """ @@ -89,6 +113,7 @@ def do_export(self, resource_id, resource_dict, parent_dir): normalize_template=True, normalize_parameters=True, parent_stack_id=resource_id, + parallel_upload=getattr(self, "parallel_upload", False), ).export() exported_template_str = yaml_dump(exported_template_dict) @@ -179,6 +204,7 @@ def __init__( normalize_template: bool = False, normalize_parameters: bool = False, parent_stack_id: str = "", + parallel_upload: bool = False, ): """ Reads the template and makes it ready for export @@ -202,6 +228,7 @@ def __init__( self.metadata_to_export = metadata_to_export self.uploaders = uploaders self.parent_stack_id = parent_stack_id + self.parallel_upload = parallel_upload def _export_global_artifacts(self, template_dict: Dict) -> Dict: """ @@ -279,7 +306,10 @@ def export(self) -> Dict: cache: Optional[Dict] = None if is_experimental_enabled(ExperimentalFlag.PackagePerformance): cache = {} + if cache is not None and self.parallel_upload: + cache = _ThreadSafeUploadCache(cache) + export_jobs: List[Callable[[], None]] = [] for resource_logical_id, resource in self.template_dict["Resources"].items(): resource_type = resource.get("Type", None) resource_dict = resource.get("Properties", {}) @@ -291,12 +321,45 @@ def export(self) -> Dict: continue if resource_dict.get("PackageType", ZIP) != exporter_class.ARTIFACT_TYPE: continue - # Export code resources - exporter = exporter_class(self.uploaders, self.code_signer, cache) - exporter.export(full_path, resource_dict, self.template_dir) + + export_jobs.append( + self._build_export_job(exporter_class, full_path, resource_dict, cache) + ) + + if self.parallel_upload and export_jobs: + self._execute_jobs_in_parallel(export_jobs) + else: + for job in export_jobs: + job() return self.template_dict + def _build_export_job( + self, + exporter_class, + resource_full_path: str, + resource_dict: Dict, + cache: Optional[Dict], + ) -> Callable[[], None]: + def _job() -> None: + exporter = exporter_class(self.uploaders, self.code_signer, cache) + setattr(exporter, "parallel_upload", self.parallel_upload) + exporter.export(resource_full_path, resource_dict, self.template_dir) + + return _job + + def _execute_jobs_in_parallel(self, jobs: List[Callable[[], None]]) -> None: + max_workers = min(len(jobs), DEFAULT_PARALLEL_UPLOAD_WORKERS) + if max_workers <= 1: + jobs[0]() + return + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [executor.submit(job) for job in jobs] + wait(futures, return_when=FIRST_EXCEPTION) + for future in futures: + future.result() + def delete(self, retain_resources: List): """ Deletes all the artifacts referenced by the given Cloudformation template diff --git a/samcli/lib/package/ecr_uploader.py b/samcli/lib/package/ecr_uploader.py index ae0337cfdb..af93528999 100644 --- a/samcli/lib/package/ecr_uploader.py +++ b/samcli/lib/package/ecr_uploader.py @@ -4,6 +4,7 @@ import base64 import logging +import threading from io import StringIO from pathlib import Path from typing import Dict @@ -50,6 +51,7 @@ def __init__( self.stream = StreamWriter(stream=stream, auto_flush=True) self.log_streamer = LogStreamer(stream=self.stream) self.login_session_active = False + self._login_lock = threading.Lock() @property def docker_client(self): @@ -88,8 +90,10 @@ def upload(self, image, resource_name): :return: remote ECR image path that has been uploaded. """ if not self.login_session_active: - self.login() - self.login_session_active = True + with self._login_lock: + if not self.login_session_active: + self.login() + self.login_session_active = True # Sometimes the `resource_name` is used as the `image` parameter to `tag_translation`. # This is because these two cases (directly from an archive or by ID) are effectively diff --git a/schema/samcli.json b/schema/samcli.json index 9c14395d64..17510947c1 100644 --- a/schema/samcli.json +++ b/schema/samcli.json @@ -1227,6 +1227,11 @@ "type": "boolean", "description": "Indicates whether to override existing files in the S3 bucket. Specify this flag to upload artifacts even if they match existing artifacts in the S3 bucket." }, + "parallel_upload": { + "title": "parallel_upload", + "type": "boolean", + "description": "Enable parallel upload of artifacts to S3/ECR before deployment." + }, "s3_prefix": { "title": "s3_prefix", "type": "string", @@ -2346,4 +2351,4 @@ } } } -} \ No newline at end of file +} diff --git a/tests/unit/commands/deploy/test_command.py b/tests/unit/commands/deploy/test_command.py index 51b3b89bc8..ab6c64b7f5 100644 --- a/tests/unit/commands/deploy/test_command.py +++ b/tests/unit/commands/deploy/test_command.py @@ -35,6 +35,7 @@ def setUp(self): self.fail_on_empty_changset = True self.role_arn = "role_arn" self.force_upload = False + self.parallel_upload = False self.no_progressbar = False self.metadata = {"abc": "def"} self.region = None @@ -81,6 +82,7 @@ def test_all_args(self, mock_deploy_context, mock_deploy_click, mock_package_con image_repository=self.image_repository, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -114,6 +116,7 @@ def test_all_args(self, mock_deploy_context, mock_deploy_click, mock_package_con image_repository=self.image_repository, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -171,7 +174,7 @@ def test_all_args_guided_no_to_authorization_confirmation_prompt( context_mock = Mock() mockauth_per_resource.return_value = [("HelloWorldResource1", False), ("HelloWorldResource2", False)] mock_deploy_context.return_value.__enter__.return_value = context_mock - mock_confirm.side_effect = [True, True, False, True, False] + mock_confirm.side_effect = [True, True, False, False, True, False] mock_prompt.side_effect = [ "sam-app", "us-east-1", @@ -199,6 +202,7 @@ def test_all_args_guided_no_to_authorization_confirmation_prompt( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -268,7 +272,7 @@ def test_all_args_guided_use_defaults( mock_sam_function_provider.return_value.get_all.return_value = [function_mock] mockauth_per_resource.return_value = [("HelloWorldResource", False)] mock_deploy_context.return_value.__enter__.return_value = context_mock - mock_confirm.side_effect = [True, False, True, True, True, True, True] + mock_confirm.side_effect = [True, False, True, False, True, True, True, True] mock_prompt.side_effect = [ "sam-app", "us-east-1", @@ -301,6 +305,7 @@ def test_all_args_guided_use_defaults( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -334,6 +339,7 @@ def test_all_args_guided_use_defaults( image_repository=None, image_repositories={"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/managed-ecr"}, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix="sam-app", kms_key_id=self.kms_key_id, @@ -374,6 +380,7 @@ def test_all_args_guided_use_defaults( s3_prefix="sam-app", signing_profiles=self.signing_profiles, disable_rollback=True, + parallel_upload=self.parallel_upload, ) mock_managed_stack.assert_called_with(profile=self.profile, region="us-east-1") self.assertEqual(context_mock.run.call_count, 1) @@ -419,7 +426,7 @@ def test_all_args_guided( mock_sam_function_provider.return_value.get_all.return_value = [function_mock] mockauth_per_resource.return_value = [("HelloWorldResource", False)] mock_deploy_context.return_value.__enter__.return_value = context_mock - mock_confirm.side_effect = [True, False, True, True, True, True, True] + mock_confirm.side_effect = [True, False, True, False, True, True, True, True] mock_prompt.side_effect = [ "sam-app", "us-east-1", @@ -447,6 +454,7 @@ def test_all_args_guided( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -480,6 +488,7 @@ def test_all_args_guided( image_repository=None, image_repositories={"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1"}, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix="sam-app", kms_key_id=self.kms_key_id, @@ -520,6 +529,7 @@ def test_all_args_guided( s3_prefix="sam-app", signing_profiles=self.signing_profiles, disable_rollback=True, + parallel_upload=self.parallel_upload, ) mock_managed_stack.assert_called_with(profile=self.profile, region="us-east-1") self.assertEqual(context_mock.run.call_count, 1) @@ -584,7 +594,7 @@ def test_all_args_guided_no_save_echo_param_to_config( "testconfig.toml", "test-env", ] - mock_confirm.side_effect = [True, False, True, True, True, True, True] + mock_confirm.side_effect = [True, False, True, False, True, True, True, True] mock_managed_stack.return_value = "managed-s3-bucket" mock_signer_config_per_function.return_value = ({}, {}) @@ -596,6 +606,7 @@ def test_all_args_guided_no_save_echo_param_to_config( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -629,6 +640,7 @@ def test_all_args_guided_no_save_echo_param_to_config( image_repository=None, image_repositories={"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1"}, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix="sam-app", kms_key_id=self.kms_key_id, @@ -745,7 +757,7 @@ def test_all_args_guided_no_params_save_config( "testconfig.toml", "test-env", ] - mock_confirm.side_effect = [True, False, True, True, True, True, True] + mock_confirm.side_effect = [True, False, True, False, True, True, True, True] mock_get_cmd_names.return_value = ["deploy"] mock_managed_stack.return_value = "managed-s3-bucket" mock_signer_config_per_function.return_value = ({}, {}) @@ -757,6 +769,7 @@ def test_all_args_guided_no_params_save_config( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -790,6 +803,7 @@ def test_all_args_guided_no_params_save_config( image_repository=None, image_repositories={"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1"}, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix="sam-app", kms_key_id=self.kms_key_id, @@ -885,7 +899,7 @@ def test_all_args_guided_no_params_no_save_config( "us-east-1", ("CAPABILITY_IAM",), ] - mock_confirm.side_effect = [True, True, False, True, False, True, True] + mock_confirm.side_effect = [True, True, False, False, True, False, True, True] mock_managed_stack.return_value = "managed-s3-bucket" mock_signer_config_per_function.return_value = ({}, {}) @@ -898,6 +912,7 @@ def test_all_args_guided_no_params_no_save_config( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -931,6 +946,7 @@ def test_all_args_guided_no_params_no_save_config( image_repository=None, image_repositories={"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1"}, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix="sam-app", kms_key_id=self.kms_key_id, @@ -976,6 +992,7 @@ def test_all_args_resolve_s3( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -1007,6 +1024,7 @@ def test_all_args_resolve_s3( stack_name=self.stack_name, s3_bucket="managed-s3-bucket", force_upload=self.force_upload, + parallel_upload=self.parallel_upload, image_repository=None, image_repositories=None, no_progressbar=self.no_progressbar, @@ -1042,6 +1060,7 @@ def test_resolve_s3_and_s3_bucket_both_set(self): image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -1094,6 +1113,7 @@ def test_all_args_resolve_image_repos( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -1125,6 +1145,7 @@ def test_all_args_resolve_image_repos( stack_name=self.stack_name, s3_bucket=self.s3_bucket, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, image_repository=None, image_repositories={"HelloWorldFunction1": self.image_repository}, no_progressbar=self.no_progressbar, @@ -1169,6 +1190,7 @@ def test_passing_parameter_overrides_to_context( image_repository=self.image_repository, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -1202,6 +1224,7 @@ def test_passing_parameter_overrides_to_context( image_repository=self.image_repository, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -1233,6 +1256,7 @@ def test_passing_parameter_overrides_to_context( kms_key_id=self.kms_key_id, use_json=self.use_json, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, metadata=self.metadata, on_deploy=True, diff --git a/tests/unit/lib/package/test_artifact_exporter.py b/tests/unit/lib/package/test_artifact_exporter.py index c4d9e81fe9..301aa0609b 100644 --- a/tests/unit/lib/package/test_artifact_exporter.py +++ b/tests/unit/lib/package/test_artifact_exporter.py @@ -22,6 +22,7 @@ CloudFormationStackResource, CloudFormationStackSetResource, ServerlessApplicationResource, + _ThreadSafeUploadCache, ) from samcli.lib.package.packageable_resources import ( GraphQLApiCodeResource, @@ -1201,6 +1202,7 @@ def test_export_cloudformation_stack(self, TemplateMock): normalize_parameters=True, normalize_template=True, parent_stack_id="id", + parallel_upload=False, ) template_instance_mock.export.assert_called_once_with() self.s3_uploader_mock.upload.assert_called_once_with(mock.ANY, mock.ANY) @@ -1377,6 +1379,7 @@ def test_export_serverless_application(self, TemplateMock): normalize_parameters=True, normalize_template=True, parent_stack_id="id", + parallel_upload=False, ) template_instance_mock.export.assert_called_once_with() self.s3_uploader_mock.upload.assert_called_once_with(mock.ANY, mock.ANY) @@ -1543,6 +1546,96 @@ def test_template_export(self, yaml_parse_mock): resource_type2_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock, None) resource_type2_instance.export.assert_called_once_with("Resource2", mock.ANY, template_dir) + @patch.object(Template, "_execute_jobs_in_parallel") + @patch("samcli.lib.package.artifact_exporter.yaml_parse") + def test_template_export_parallel_invokes_executor(self, yaml_parse_mock, executor_mock): + parent_dir = os.path.sep + template_dir = os.path.join(parent_dir, "foo", "bar") + template_path = os.path.join(template_dir, "path") + template_str = self.example_yaml_template() + + resource_type1_class = Mock() + resource_type1_class.RESOURCE_TYPE = "resource_type1" + resource_type1_class.ARTIFACT_TYPE = ZIP + resource_type1_class.EXPORT_DESTINATION = Destination.S3 + resource_type1_class.return_value = Mock() + + resource_type2_class = Mock() + resource_type2_class.RESOURCE_TYPE = "resource_type2" + resource_type2_class.ARTIFACT_TYPE = ZIP + resource_type2_class.EXPORT_DESTINATION = Destination.S3 + resource_type2_class.return_value = Mock() + + resources_to_export = [resource_type1_class, resource_type2_class] + properties = {"foo": "bar"} + template_dict = { + "Resources": { + "Resource1": {"Type": "resource_type1", "Properties": properties}, + "Resource2": {"Type": "resource_type2", "Properties": properties}, + } + } + + yaml_parse_mock.return_value = template_dict + with patch("samcli.lib.package.artifact_exporter.open", mock.mock_open(read_data=template_str)): + template_exporter = Template( + template_path, + parent_dir, + self.uploaders_mock, + self.code_signer_mock, + resources_to_export, + parallel_upload=True, + ) + template_exporter.export() + + executor_mock.assert_called_once() + jobs = executor_mock.call_args[0][0] + self.assertEqual(len(jobs), 2) + + @patch("samcli.lib.package.artifact_exporter.is_experimental_enabled") + @patch("samcli.lib.package.artifact_exporter.yaml_parse") + def test_template_export_parallel_wraps_cache(self, yaml_parse_mock, is_experimental_enabled_mock): + is_experimental_enabled_mock.side_effect = lambda *args: { + (ExperimentalFlag.PackagePerformance,): True, + }.get(args, False) + + parent_dir = os.path.sep + template_dir = os.path.join(parent_dir, "foo", "bar") + template_path = os.path.join(template_dir, "path") + template_str = self.example_yaml_template() + + resource_type_class = Mock() + resource_type_class.RESOURCE_TYPE = "resource_type1" + resource_type_class.ARTIFACT_TYPE = ZIP + resource_type_class.EXPORT_DESTINATION = Destination.S3 + resource_type_instance = Mock() + resource_type_class.return_value = resource_type_instance + + captured_cache = {} + + def capture_cache(uploaders, code_signer, cache): + nonlocal captured_cache + captured_cache = cache + return resource_type_instance + + resource_type_class.side_effect = capture_cache + + properties = {"foo": "bar"} + template_dict = {"Resources": {"Resource1": {"Type": "resource_type1", "Properties": properties}}} + + yaml_parse_mock.return_value = template_dict + with patch("samcli.lib.package.artifact_exporter.open", mock.mock_open(read_data=template_str)): + template_exporter = Template( + template_path, + parent_dir, + self.uploaders_mock, + self.code_signer_mock, + [resource_type_class], + parallel_upload=True, + ) + template_exporter.export() + + self.assertIsInstance(captured_cache, _ThreadSafeUploadCache) + @patch("samcli.lib.package.artifact_exporter.is_experimental_enabled") @patch("samcli.lib.package.artifact_exporter.yaml_parse") def test_template_export_with_experimental_flag(self, yaml_parse_mock, is_experimental_enabled_mock): From dae5be7980a4b1d5e75229432ecd909cec3d3525 Mon Sep 17 00:00:00 2001 From: Rick Getz Date: Tue, 18 Nov 2025 17:48:25 -0500 Subject: [PATCH 2/5] docs: update docs to inlcude parallel uploads flag --- docs/sam-config-docs.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/sam-config-docs.md b/docs/sam-config-docs.md index 944a78c288..48bdac676c 100644 --- a/docs/sam-config-docs.md +++ b/docs/sam-config-docs.md @@ -23,6 +23,16 @@ output_template_file="packaged.yaml" [default.deploy.parameters] stack_name="using_config_file" + +### Specifying a Boolean deployment option + +``` +[default.deploy.parameters] +parallel_upload=true +``` + +Setting `parallel_upload` to `true` is equivalent to passing `--parallel-upload` on +`sam deploy`, enabling concurrent S3/ECR uploads during the packaging phase. capabilities="CAPABILITY_IAM" region="us-east-1" profile="srirammv" @@ -94,4 +104,3 @@ stack_name="using_config_file" [default.build.parameters] debug=true ``` - From 2f424b65efe68f4146c1b146509ab577fdca3d5b Mon Sep 17 00:00:00 2001 From: Rick Getz Date: Tue, 18 Nov 2025 18:00:21 -0500 Subject: [PATCH 3/5] chore: run black formatter --- samcli/commands/deploy/guided_context.py | 4 +--- samcli/lib/package/artifact_exporter.py | 4 +--- schema/samcli.json | 6 +++--- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/samcli/commands/deploy/guided_context.py b/samcli/commands/deploy/guided_context.py index a2030647af..8c9edea9e4 100644 --- a/samcli/commands/deploy/guided_context.py +++ b/samcli/commands/deploy/guided_context.py @@ -169,9 +169,7 @@ def guided_prompts(self, parameter_override_keys): parallel_upload = True else: click.secho("\t#Speed up artifact uploads by running them in parallel") - parallel_upload = confirm( - f"\t{self.start_bold}Enable parallel uploads{self.end_bold}", default=False - ) + parallel_upload = confirm(f"\t{self.start_bold}Enable parallel uploads{self.end_bold}", default=False) self.prompt_authorization(stacks) self.prompt_code_signing_settings(stacks) diff --git a/samcli/lib/package/artifact_exporter.py b/samcli/lib/package/artifact_exporter.py index dd8a661838..bf9a6f776b 100644 --- a/samcli/lib/package/artifact_exporter.py +++ b/samcli/lib/package/artifact_exporter.py @@ -322,9 +322,7 @@ def export(self) -> Dict: if resource_dict.get("PackageType", ZIP) != exporter_class.ARTIFACT_TYPE: continue - export_jobs.append( - self._build_export_job(exporter_class, full_path, resource_dict, cache) - ) + export_jobs.append(self._build_export_job(exporter_class, full_path, resource_dict, cache)) if self.parallel_upload and export_jobs: self._execute_jobs_in_parallel(export_jobs) diff --git a/schema/samcli.json b/schema/samcli.json index 17510947c1..115562fc5d 100644 --- a/schema/samcli.json +++ b/schema/samcli.json @@ -1147,7 +1147,7 @@ "properties": { "parameters": { "title": "Parameters for the deploy command", - "description": "Available parameters for the deploy command:\n* guided:\nSpecify this flag to allow SAM CLI to guide you through the deployment using guided prompts.\n* template_file:\nAWS SAM template which references built artifacts for resources in the template. (if applicable)\n* no_execute_changeset:\nIndicates whether to execute the change set. Specify this flag to view stack changes before executing the change set.\n* fail_on_empty_changeset:\nSpecify whether AWS SAM CLI should return a non-zero exit code if there are no changes to be made to the stack. Defaults to a non-zero exit code.\n* confirm_changeset:\nPrompt to confirm if the computed changeset is to be deployed by SAM CLI.\n* disable_rollback:\nPreserves the state of previously provisioned resources when an operation fails.\n* on_failure:\nProvide an action to determine what will happen when a stack fails to create. Three actions are available:\n\n- ROLLBACK: This will rollback a stack to a previous known good state.\n\n- DELETE: The stack will rollback to a previous state if one exists, otherwise the stack will be deleted.\n\n- DO_NOTHING: The stack will not rollback or delete, this is the same as disabling rollback.\n\nDefault behaviour is ROLLBACK.\n\n\n\nThis option is mutually exclusive with --disable-rollback/--no-disable-rollback. You can provide\n--on-failure or --disable-rollback/--no-disable-rollback but not both at the same time.\n* max_wait_duration:\nMaximum duration in minutes to wait for the deployment to complete.\n* stack_name:\nName of the AWS CloudFormation stack.\n* s3_bucket:\nAWS S3 bucket where artifacts referenced in the template are uploaded.\n* image_repository:\nAWS ECR repository URI where artifacts referenced in the template are uploaded.\n* image_repositories:\nMapping of Function Logical ID to AWS ECR Repository URI.\n\nExample: Function_Logical_ID=ECR_Repo_Uri\nThis option can be specified multiple times.\n* force_upload:\nIndicates whether to override existing files in the S3 bucket. Specify this flag to upload artifacts even if they match existing artifacts in the S3 bucket.\n* s3_prefix:\nPrefix name that is added to the artifact's name when it is uploaded to the AWS S3 bucket.\n* kms_key_id:\nThe ID of an AWS KMS key that is used to encrypt artifacts that are at rest in the AWS S3 bucket.\n* role_arn:\nARN of an IAM role that AWS Cloudformation assumes when executing a deployment change set.\n* use_json:\nIndicates whether to use JSON as the format for the output AWS CloudFormation template. YAML is used by default.\n* resolve_s3:\nAutomatically resolve AWS S3 bucket for non-guided deployments. Enabling this option will also create a managed default AWS S3 bucket for you. If one does not provide a --s3-bucket value, the managed bucket will be used. Do not use --guided with this option.\n* resolve_image_repos:\nAutomatically create and delete ECR repositories for image-based functions in non-guided deployments. A companion stack containing ECR repos for each function will be deployed along with the template stack. Automatically created image repositories will be deleted if the corresponding functions are removed.\n* metadata:\nMap of metadata to attach to ALL the artifacts that are referenced in the template.\n* notification_arns:\nARNs of SNS topics that AWS Cloudformation associates with the stack.\n* tags:\nList of tags to associate with the stack.\n* parameter_overrides:\nString that contains AWS CloudFormation parameter overrides encoded as key=value pairs.\n* signing_profiles:\nA string that contains Code Sign configuration parameters as FunctionOrLayerNameToSign=SigningProfileName:SigningProfileOwner Since signing profile owner is optional, it could also be written as FunctionOrLayerNameToSign=SigningProfileName\n* no_progressbar:\nDoes not showcase a progress bar when uploading artifacts to S3 and pushing docker images to ECR\n* capabilities:\nList of capabilities that one must specify before AWS Cloudformation can create certain stacks.\n\nAccepted Values: CAPABILITY_IAM, CAPABILITY_NAMED_IAM, CAPABILITY_RESOURCE_POLICY, CAPABILITY_AUTO_EXPAND.\n\nLearn more at: https://docs.aws.amazon.com/serverlessrepo/latest/devguide/acknowledging-application-capabilities.html\n* profile:\nSelect a specific profile from your credential file to get AWS credentials.\n* region:\nSet the AWS Region of the service. (e.g. us-east-1)\n* beta_features:\nEnable/Disable beta features.\n* debug:\nTurn on debug logging to print debug message generated by AWS SAM CLI and display timestamps.\n* save_params:\nSave the parameters provided via the command line to the configuration file.", + "description": "Available parameters for the deploy command:\n* guided:\nSpecify this flag to allow SAM CLI to guide you through the deployment using guided prompts.\n* template_file:\nAWS SAM template which references built artifacts for resources in the template. (if applicable)\n* no_execute_changeset:\nIndicates whether to execute the change set. Specify this flag to view stack changes before executing the change set.\n* fail_on_empty_changeset:\nSpecify whether AWS SAM CLI should return a non-zero exit code if there are no changes to be made to the stack. Defaults to a non-zero exit code.\n* confirm_changeset:\nPrompt to confirm if the computed changeset is to be deployed by SAM CLI.\n* disable_rollback:\nPreserves the state of previously provisioned resources when an operation fails.\n* on_failure:\nProvide an action to determine what will happen when a stack fails to create. Three actions are available:\n\n- ROLLBACK: This will rollback a stack to a previous known good state.\n\n- DELETE: The stack will rollback to a previous state if one exists, otherwise the stack will be deleted.\n\n- DO_NOTHING: The stack will not rollback or delete, this is the same as disabling rollback.\n\nDefault behaviour is ROLLBACK.\n\n\n\nThis option is mutually exclusive with --disable-rollback/--no-disable-rollback. You can provide\n--on-failure or --disable-rollback/--no-disable-rollback but not both at the same time.\n* max_wait_duration:\nMaximum duration in minutes to wait for the deployment to complete.\n* stack_name:\nName of the AWS CloudFormation stack.\n* s3_bucket:\nAWS S3 bucket where artifacts referenced in the template are uploaded.\n* image_repository:\nAWS ECR repository URI where artifacts referenced in the template are uploaded.\n* image_repositories:\nMapping of Function Logical ID to AWS ECR Repository URI.\n\nExample: Function_Logical_ID=ECR_Repo_Uri\nThis option can be specified multiple times.\n* force_upload:\nIndicates whether to override existing files in the S3 bucket. Specify this flag to upload artifacts even if they match existing artifacts in the S3 bucket.\n* parallel_upload:\nEnable parallel upload of artifacts to S3/ECR during packaging before deployment.\n* s3_prefix:\nPrefix name that is added to the artifact's name when it is uploaded to the AWS S3 bucket.\n* kms_key_id:\nThe ID of an AWS KMS key that is used to encrypt artifacts that are at rest in the AWS S3 bucket.\n* role_arn:\nARN of an IAM role that AWS Cloudformation assumes when executing a deployment change set.\n* use_json:\nIndicates whether to use JSON as the format for the output AWS CloudFormation template. YAML is used by default.\n* resolve_s3:\nAutomatically resolve AWS S3 bucket for non-guided deployments. Enabling this option will also create a managed default AWS S3 bucket for you. If one does not provide a --s3-bucket value, the managed bucket will be used. Do not use --guided with this option.\n* resolve_image_repos:\nAutomatically create and delete ECR repositories for image-based functions in non-guided deployments. A companion stack containing ECR repos for each function will be deployed along with the template stack. Automatically created image repositories will be deleted if the corresponding functions are removed.\n* metadata:\nMap of metadata to attach to ALL the artifacts that are referenced in the template.\n* notification_arns:\nARNs of SNS topics that AWS Cloudformation associates with the stack.\n* tags:\nList of tags to associate with the stack.\n* parameter_overrides:\nString that contains AWS CloudFormation parameter overrides encoded as key=value pairs.\n* signing_profiles:\nA string that contains Code Sign configuration parameters as FunctionOrLayerNameToSign=SigningProfileName:SigningProfileOwner Since signing profile owner is optional, it could also be written as FunctionOrLayerNameToSign=SigningProfileName\n* no_progressbar:\nDoes not showcase a progress bar when uploading artifacts to S3 and pushing docker images to ECR\n* capabilities:\nList of capabilities that one must specify before AWS Cloudformation can create certain stacks.\n\nAccepted Values: CAPABILITY_IAM, CAPABILITY_NAMED_IAM, CAPABILITY_RESOURCE_POLICY, CAPABILITY_AUTO_EXPAND.\n\nLearn more at: https://docs.aws.amazon.com/serverlessrepo/latest/devguide/acknowledging-application-capabilities.html\n* profile:\nSelect a specific profile from your credential file to get AWS credentials.\n* region:\nSet the AWS Region of the service. (e.g. us-east-1)\n* beta_features:\nEnable/Disable beta features.\n* debug:\nTurn on debug logging to print debug message generated by AWS SAM CLI and display timestamps.\n* save_params:\nSave the parameters provided via the command line to the configuration file.", "type": "object", "properties": { "guided": { @@ -1230,7 +1230,7 @@ "parallel_upload": { "title": "parallel_upload", "type": "boolean", - "description": "Enable parallel upload of artifacts to S3/ECR before deployment." + "description": "Enable parallel upload of artifacts to S3/ECR during packaging before deployment." }, "s3_prefix": { "title": "s3_prefix", @@ -2351,4 +2351,4 @@ } } } -} +} \ No newline at end of file From 5153c9e59acd703738f4363c494b736575642a9c Mon Sep 17 00:00:00 2001 From: Rick Getz Date: Tue, 18 Nov 2025 18:14:37 -0500 Subject: [PATCH 4/5] test: ensure parallel-upload is recognized in tests --- samcli/lib/package/artifact_exporter.py | 28 +++++++---- .../commands/deploy/test_guided_context.py | 46 +++++++++++++------ .../unit/commands/samconfig/test_samconfig.py | 2 + 3 files changed, 52 insertions(+), 24 deletions(-) diff --git a/samcli/lib/package/artifact_exporter.py b/samcli/lib/package/artifact_exporter.py index bf9a6f776b..32cf22d1df 100644 --- a/samcli/lib/package/artifact_exporter.py +++ b/samcli/lib/package/artifact_exporter.py @@ -16,6 +16,7 @@ # language governing permissions and limitations under the License. import os import threading +from collections.abc import MutableMapping from concurrent.futures import FIRST_EXCEPTION, ThreadPoolExecutor, wait from typing import Callable, Dict, List, Optional @@ -57,17 +58,14 @@ DEFAULT_PARALLEL_UPLOAD_WORKERS = max(4, min(32, (os.cpu_count() or 1) * 2)) -class _ThreadSafeUploadCache: +class _ThreadSafeUploadCache(MutableMapping[str, str]): """Simple thread-safe mapping used to deduplicate uploads across threads.""" - def __init__(self, initial: Optional[Dict[str, str]] = None): - self._cache = initial or {} + def __init__(self, initial: Optional[MutableMapping[str, str]] = None): + # Copy into a regular dict so we can safely snapshot under a lock + self._cache: Dict[str, str] = dict(initial or {}) self._lock = threading.Lock() - def __contains__(self, key: str) -> bool: # pragma: no cover - small helper - with self._lock: - return key in self._cache - def __getitem__(self, key: str) -> str: # pragma: no cover - small helper with self._lock: return self._cache[key] @@ -76,6 +74,18 @@ def __setitem__(self, key: str, value: str) -> None: # pragma: no cover - small with self._lock: self._cache[key] = value + def __delitem__(self, key: str) -> None: # pragma: no cover - small helper + with self._lock: + del self._cache[key] + + def __iter__(self): # pragma: no cover - small helper + with self._lock: + return iter(dict(self._cache)) + + def __len__(self) -> int: # pragma: no cover - small helper + with self._lock: + return len(self._cache) + class CloudFormationStackResource(ResourceZip): """ @@ -303,7 +313,7 @@ def export(self) -> Dict: self._apply_global_values() self.template_dict = self._export_global_artifacts(self.template_dict) - cache: Optional[Dict] = None + cache: Optional[MutableMapping[str, str]] = None if is_experimental_enabled(ExperimentalFlag.PackagePerformance): cache = {} if cache is not None and self.parallel_upload: @@ -337,7 +347,7 @@ def _build_export_job( exporter_class, resource_full_path: str, resource_dict: Dict, - cache: Optional[Dict], + cache: Optional[MutableMapping[str, str]], ) -> Callable[[], None]: def _job() -> None: exporter = exporter_class(self.uploaders, self.code_signer, cache) diff --git a/tests/unit/commands/deploy/test_guided_context.py b/tests/unit/commands/deploy/test_guided_context.py index b52c8fbac5..cce1e1f51e 100644 --- a/tests/unit/commands/deploy/test_guided_context.py +++ b/tests/unit/commands/deploy/test_guided_context.py @@ -73,7 +73,7 @@ def test_guided_prompts_check_defaults_non_public_resources_zips( patched_auth_per_resource.return_value = [ ("HelloWorldFunction", True), ] - patched_confirm.side_effect = [True, False, False, "", True, True, True] + patched_confirm.side_effect = [True, False, False, False, "", True, True, True] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) self.gc.guided_prompts(parameter_override_keys=None) @@ -82,6 +82,7 @@ def test_guided_prompts_check_defaults_non_public_resources_zips( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call(f"\t{self.gc.start_bold}Save arguments to configuration file{self.gc.end_bold}", default=True), call( f"\t {self.gc.start_bold}Delete the unreferenced repositories listed above when deploying?{self.gc.end_bold}", @@ -125,7 +126,7 @@ def test_guided_prompts_check_defaults_public_resources_zips( patched_get_buildable_stacks.return_value = (Mock(), []) # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] - patched_confirm.side_effect = [True, False, False, True, False, True, True] + patched_confirm.side_effect = [True, False, False, False, True, False, True, True] patched_manage_stack.return_value = "managed_s3_stack" self.gc.guided_prompts(parameter_override_keys=None) # Now to check for all the defaults on confirmations. @@ -133,6 +134,7 @@ def test_guided_prompts_check_defaults_public_resources_zips( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -192,7 +194,7 @@ def test_guided_prompts_check_defaults_public_resources_images( # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] get_resource_full_path_by_id_mock.return_value = None - patched_confirm.side_effect = [True, False, False, True, False, True, True] + patched_confirm.side_effect = [True, False, False, False, True, False, True, True] patched_manage_stack.return_value = "managed_s3_stack" self.gc.guided_prompts(parameter_override_keys=None) # Now to check for all the defaults on confirmations. @@ -200,6 +202,7 @@ def test_guided_prompts_check_defaults_public_resources_images( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -230,6 +233,7 @@ def test_guided_prompts_check_defaults_public_resources_images( call("\t#Shows you resources changes to be deployed and require a 'Y' to initiate deploy"), call("\t#SAM needs permission to be able to create roles to connect to the resources in your template"), call("\t#Preserves the state of previously provisioned resources when an operation fails"), + call("\t#Speed up artifact uploads by running them in parallel"), call("\n\tManaged S3 bucket: managed_s3_stack", bold=True), ] self.assertEqual(expected_click_secho_calls, patched_click_secho.call_args_list) @@ -270,7 +274,7 @@ def test_guided_prompts_check_defaults_public_resources_images_ecr_url( # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] get_resource_full_path_by_id_mock.return_value = "RandomFunction" - patched_confirm.side_effect = [True, False, False, True, False, True, True] + patched_confirm.side_effect = [True, False, False, False, True, False, True, True] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) self.gc.guided_prompts(parameter_override_keys=None) @@ -279,6 +283,7 @@ def test_guided_prompts_check_defaults_public_resources_images_ecr_url( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -307,6 +312,7 @@ def test_guided_prompts_check_defaults_public_resources_images_ecr_url( call("\t#Shows you resources changes to be deployed and require a 'Y' to initiate deploy"), call("\t#SAM needs permission to be able to create roles to connect to the resources in your template"), call("\t#Preserves the state of previously provisioned resources when an operation fails"), + call("\t#Speed up artifact uploads by running them in parallel"), call("\n\tManaged S3 bucket: managed_s3_stack", bold=True), ] self.assertEqual(expected_click_secho_calls, patched_click_secho.call_args_list) @@ -347,7 +353,7 @@ def test_guided_prompts_images_illegal_image_uri( # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] get_resource_full_path_by_id_mock.return_value = "RandomFunction" - patched_confirm.side_effect = [True, False, False, True, False, False, True] + patched_confirm.side_effect = [True, False, False, False, True, False, False, True] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) with self.assertRaises(GuidedDeployFailedError): @@ -393,7 +399,7 @@ def test_guided_prompts_images_missing_repo( ] # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] - patched_confirm.side_effect = [True, False, False, True, False, True, True] + patched_confirm.side_effect = [True, False, False, False, True, False, True, True] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) @@ -403,6 +409,7 @@ def test_guided_prompts_images_missing_repo( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -431,6 +438,7 @@ def test_guided_prompts_images_missing_repo( call("\t#Shows you resources changes to be deployed and require a 'Y' to initiate deploy"), call("\t#SAM needs permission to be able to create roles to connect to the resources in your template"), call("\t#Preserves the state of previously provisioned resources when an operation fails"), + call("\t#Speed up artifact uploads by running them in parallel"), call("\n\tManaged S3 bucket: managed_s3_stack", bold=True), ] self.assertEqual(expected_click_secho_calls, patched_click_secho.call_args_list) @@ -472,7 +480,7 @@ def test_guided_prompts_images_no_repo( # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] patched_get_resource_full_path_by_id.return_value = "RandomFunction" - patched_confirm.side_effect = [True, False, False, True, False, False, True] + patched_confirm.side_effect = [True, False, False, False, True, False, False, True] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) @@ -482,6 +490,7 @@ def test_guided_prompts_images_no_repo( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -514,6 +523,7 @@ def test_guided_prompts_images_no_repo( call("\t#Shows you resources changes to be deployed and require a 'Y' to initiate deploy"), call("\t#SAM needs permission to be able to create roles to connect to the resources in your template"), call("\t#Preserves the state of previously provisioned resources when an operation fails"), + call("\t#Speed up artifact uploads by running them in parallel"), call("\n\tManaged S3 bucket: managed_s3_stack", bold=True), ] self.assertEqual(expected_click_secho_calls, patched_click_secho.call_args_list) @@ -554,7 +564,7 @@ def test_guided_prompts_images_deny_deletion( ] # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] - patched_confirm.side_effect = [True, False, False, True, False, True, False] + patched_confirm.side_effect = [True, False, False, False, True, False, True, False] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) with self.assertRaises(GuidedDeployFailedError): @@ -597,7 +607,7 @@ def test_guided_prompts_images_blank_image_repository( ] # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] - patched_confirm.side_effect = [True, False, False, True, False, False, True] + patched_confirm.side_effect = [True, False, False, False, True, False, False, True] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) with self.assertRaises(GuidedDeployFailedError): @@ -642,13 +652,14 @@ def test_guided_prompts_with_given_capabilities( patched_get_buildable_stacks.return_value = (Mock(), []) self.gc.capabilities = given_capabilities # Series of inputs to confirmations so that full range of questions are asked. - patched_confirm.side_effect = [True, False, False, "", True, True, True] + patched_confirm.side_effect = [True, False, False, False, "", True, True, True] self.gc.guided_prompts(parameter_override_keys=None) # Now to check for all the defaults on confirmations. expected_confirmation_calls = [ call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call(f"\t{self.gc.start_bold}Save arguments to configuration file{self.gc.end_bold}", default=True), call( f"\t {self.gc.start_bold}Delete the unreferenced repositories listed above when deploying?{self.gc.end_bold}", @@ -690,7 +701,7 @@ def test_guided_prompts_check_configuration_file_prompt_calls( patched_signer_config_per_function.return_value = ({}, {}) # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] - patched_confirm.side_effect = [True, False, False, True, True, True, True] + patched_confirm.side_effect = [True, False, False, False, True, True, True, True] patched_manage_stack.return_value = "managed_s3_stack" patched_get_resource_full_path_by_id.return_value = "RandomFunction" self.gc.guided_prompts(parameter_override_keys=None) @@ -699,6 +710,7 @@ def test_guided_prompts_check_configuration_file_prompt_calls( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -752,7 +764,7 @@ def test_guided_prompts_check_parameter_from_template( # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] patched_get_resource_full_path_by_id.return_value = "RandomFunction" - patched_confirm.side_effect = [True, False, False, True, False, True, True] + patched_confirm.side_effect = [True, False, False, False, True, False, True, True] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) parameter_override_from_template = {"MyTestKey": {"Default": "MyTemplateDefaultVal"}} @@ -763,6 +775,7 @@ def test_guided_prompts_check_parameter_from_template( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -811,7 +824,7 @@ def test_guided_prompts_check_parameter_from_cmd_or_config( # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] patched_get_resource_full_path_by_id.return_value = "RandomFunction" - patched_confirm.side_effect = [True, False, False, True, False, True, True] + patched_confirm.side_effect = [True, False, False, False, True, False, True, True] patched_signer_config_per_function.return_value = ({}, {}) patched_manage_stack.return_value = "managed_s3_stack" parameter_override_from_template = {"MyTestKey": {"Default": "MyTemplateDefaultVal"}} @@ -822,6 +835,7 @@ def test_guided_prompts_check_parameter_from_cmd_or_config( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -884,7 +898,7 @@ def test_guided_prompts_with_code_signing( patched_signer_config_per_function.return_value = given_code_signing_configs patched_get_buildable_stacks.return_value = (Mock(), []) # Series of inputs to confirmations so that full range of questions are asked. - patched_confirm.side_effect = [True, False, False, given_sign_packages_flag, "", True, True, True] + patched_confirm.side_effect = [True, False, False, False, given_sign_packages_flag, "", True, True, True] patched_get_resource_full_path_by_id.return_value = "RandomFunction" self.gc.guided_prompts(parameter_override_keys=None) # Now to check for all the defaults on confirmations. @@ -892,6 +906,7 @@ def test_guided_prompts_with_code_signing( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}Do you want to sign your code?{self.gc.end_bold}", default=True, @@ -955,7 +970,7 @@ def test_guided_prompts_check_default_config_region( # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] patched_get_resource_full_path_by_id.return_value = "RandomFunction" - patched_confirm.side_effect = [True, False, False, True, True, True, True] + patched_confirm.side_effect = [True, False, False, False, True, True, True, True] patched_signer_config_per_function.return_value = ({}, {}) patched_manage_stack.return_value = "managed_s3_stack" patched_get_default_aws_region.return_value = "default_config_region" @@ -967,6 +982,7 @@ def test_guided_prompts_check_default_config_region( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, diff --git a/tests/unit/commands/samconfig/test_samconfig.py b/tests/unit/commands/samconfig/test_samconfig.py index 1a7b929c5d..53380c3f7a 100644 --- a/tests/unit/commands/samconfig/test_samconfig.py +++ b/tests/unit/commands/samconfig/test_samconfig.py @@ -934,6 +934,7 @@ def test_deploy(self, do_cli_mock, template_artifacts_mock1, template_artifacts_ None, True, False, + False, "myprefix", "mykms", {"Key": "Value"}, @@ -1049,6 +1050,7 @@ def test_deploy_different_parameter_override_format( None, True, False, + False, "myprefix", "mykms", {"Key1": "Value1", "Key2": "Multiple spaces in the value"}, From f4a5f0b2c2f250968813700acc1f8638a2f1df66 Mon Sep 17 00:00:00 2001 From: Rick Getz Date: Tue, 18 Nov 2025 21:41:31 -0500 Subject: [PATCH 5/5] chore: increase docker timeout --- samcli/local/docker/container_client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/samcli/local/docker/container_client.py b/samcli/local/docker/container_client.py index 5a1bebc468..815274ca55 100644 --- a/samcli/local/docker/container_client.py +++ b/samcli/local/docker/container_client.py @@ -96,6 +96,9 @@ def __init__(self, client_version, base_url=None): # Specify minimum version client_params["version"] = client_version + # Increase client timeout to tolerate longer pushes/pulls + client_params["timeout"] = int(os.environ.get("SAM_CLI_DOCKER_TIMEOUT", "600")) + # Initialize DockerClient with processed parameters LOG.debug(f"Creating container client with parameters: {client_params}") super().__init__(**client_params)