diff --git a/docs/sam-config-docs.md b/docs/sam-config-docs.md index 944a78c288..48bdac676c 100644 --- a/docs/sam-config-docs.md +++ b/docs/sam-config-docs.md @@ -23,6 +23,16 @@ output_template_file="packaged.yaml" [default.deploy.parameters] stack_name="using_config_file" + +### Specifying a Boolean deployment option + +``` +[default.deploy.parameters] +parallel_upload=true +``` + +Setting `parallel_upload` to `true` is equivalent to passing `--parallel-upload` on +`sam deploy`, enabling concurrent S3/ECR uploads during the packaging phase. capabilities="CAPABILITY_IAM" region="us-east-1" profile="srirammv" @@ -94,4 +104,3 @@ stack_name="using_config_file" [default.build.parameters] debug=true ``` - diff --git a/samcli/commands/deploy/command.py b/samcli/commands/deploy/command.py index dd750ce5e5..ff0d7e2943 100644 --- a/samcli/commands/deploy/command.py +++ b/samcli/commands/deploy/command.py @@ -146,6 +146,12 @@ @image_repository_option @image_repositories_option @force_upload_option +@click.option( + "--parallel-upload", + is_flag=True, + default=False, + help="Enable parallel upload of artifacts to S3/ECR during packaging before deployment.", +) @s3_prefix_option @kms_key_id_option @role_arn_option @@ -177,6 +183,7 @@ def cli( image_repository, image_repositories, force_upload, + parallel_upload, no_progressbar, s3_prefix, kms_key_id, @@ -212,6 +219,7 @@ def cli( image_repository, image_repositories, force_upload, + parallel_upload, no_progressbar, s3_prefix, kms_key_id, @@ -246,6 +254,7 @@ def do_cli( image_repository, image_repositories, force_upload, + parallel_upload, no_progressbar, s3_prefix, kms_key_id, @@ -300,6 +309,7 @@ def do_cli( config_env=config_env, config_file=config_file, disable_rollback=disable_rollback, + parallel_upload=parallel_upload, ) guided_context.run() else: @@ -331,6 +341,7 @@ def do_cli( kms_key_id=kms_key_id, use_json=use_json, force_upload=force_upload, + parallel_upload=guided_context.guided_parallel_upload if guided else parallel_upload, no_progressbar=no_progressbar, metadata=metadata, on_deploy=True, @@ -357,6 +368,7 @@ def do_cli( image_repository=guided_context.guided_image_repository if guided else image_repository, image_repositories=guided_context.guided_image_repositories if guided else image_repositories, force_upload=force_upload, + parallel_upload=guided_context.guided_parallel_upload if guided else parallel_upload, no_progressbar=no_progressbar, s3_prefix=guided_context.guided_s3_prefix if guided else s3_prefix, kms_key_id=kms_key_id, diff --git a/samcli/commands/deploy/core/options.py b/samcli/commands/deploy/core/options.py index 44503368af..1af6dffdcb 100644 --- a/samcli/commands/deploy/core/options.py +++ b/samcli/commands/deploy/core/options.py @@ -37,6 +37,7 @@ "disable_rollback", "on_failure", "force_upload", + "parallel_upload", "max_wait_duration", ] diff --git a/samcli/commands/deploy/deploy_context.py b/samcli/commands/deploy/deploy_context.py index 33ac171156..7036910037 100644 --- a/samcli/commands/deploy/deploy_context.py +++ b/samcli/commands/deploy/deploy_context.py @@ -75,6 +75,7 @@ def __init__( poll_delay, on_failure, max_wait_duration, + parallel_upload=False, ): self.template_file = template_file self.stack_name = stack_name @@ -82,6 +83,7 @@ def __init__( self.image_repository = image_repository self.image_repositories = image_repositories self.force_upload = force_upload + self.parallel_upload = parallel_upload self.no_progressbar = no_progressbar self.s3_prefix = s3_prefix self.kms_key_id = kms_key_id @@ -164,6 +166,7 @@ def run(self): self.signing_profiles, self.use_changeset, self.disable_rollback, + self.parallel_upload, ) return self.deploy( self.stack_name, diff --git a/samcli/commands/deploy/guided_context.py b/samcli/commands/deploy/guided_context.py index 8657debd57..8c9edea9e4 100644 --- a/samcli/commands/deploy/guided_context.py +++ b/samcli/commands/deploy/guided_context.py @@ -62,6 +62,7 @@ def __init__( config_env=None, config_file=None, disable_rollback=None, + parallel_upload=False, ): self.template_file = template_file self.stack_name = stack_name @@ -95,6 +96,8 @@ def __init__( self.color = Colored() self.function_provider = None self.disable_rollback = disable_rollback + self.parallel_upload = parallel_upload + self.guided_parallel_upload = None @property def guided_capabilities(self): @@ -162,6 +165,12 @@ def guided_prompts(self, parameter_override_keys): click.secho("\t#Preserves the state of previously provisioned resources when an operation fails") disable_rollback = confirm(f"\t{self.start_bold}Disable rollback{self.end_bold}", default=self.disable_rollback) + if self.parallel_upload: + parallel_upload = True + else: + click.secho("\t#Speed up artifact uploads by running them in parallel") + parallel_upload = confirm(f"\t{self.start_bold}Enable parallel uploads{self.end_bold}", default=False) + self.prompt_authorization(stacks) self.prompt_code_signing_settings(stacks) @@ -204,6 +213,7 @@ def guided_prompts(self, parameter_override_keys): self.guided_s3_prefix = stack_name self.guided_region = region self.guided_profile = self.profile + self.guided_parallel_upload = parallel_upload self._capabilities = input_capabilities if input_capabilities else default_capabilities self._parameter_overrides = ( input_parameter_overrides if input_parameter_overrides else self.parameter_overrides_from_cmdline @@ -587,6 +597,7 @@ def run(self): capabilities=self._capabilities, signing_profiles=self.signing_profiles, disable_rollback=self.disable_rollback, + parallel_upload=self.guided_parallel_upload, ) @staticmethod diff --git a/samcli/commands/deploy/utils.py b/samcli/commands/deploy/utils.py index 421a16c75d..56270e5ae9 100644 --- a/samcli/commands/deploy/utils.py +++ b/samcli/commands/deploy/utils.py @@ -20,6 +20,7 @@ def print_deploy_args( signing_profiles, use_changeset, disable_rollback, + parallel_upload, ): """ Print a table of the values that are used during a sam deploy. @@ -48,6 +49,7 @@ def print_deploy_args( :param signing_profiles: Signing profile details which will be used to sign functions/layers :param use_changeset: Flag to use or skip the usage of changesets :param disable_rollback: Preserve the state of previously provisioned resources when an operation fails. + :param parallel_upload: Whether artifact uploads run in parallel prior to deployment. """ _parameters = parameter_overrides.copy() @@ -70,6 +72,7 @@ def print_deploy_args( if use_changeset: click.echo(f"\tConfirm changeset : {confirm_changeset}") click.echo(f"\tDisable rollback : {disable_rollback}") + click.echo(f"\tParallel uploads : {parallel_upload}") if image_repository: msg = "Deployment image repository : " # NOTE(sriram-mv): tab length is 8 spaces. diff --git a/samcli/commands/package/package_context.py b/samcli/commands/package/package_context.py index 4a64981745..1d813fef39 100644 --- a/samcli/commands/package/package_context.py +++ b/samcli/commands/package/package_context.py @@ -71,6 +71,7 @@ def __init__( parameter_overrides=None, on_deploy=False, signing_profiles=None, + parallel_upload=False, ): self.template_file = template_file self.s3_bucket = s3_bucket @@ -81,6 +82,7 @@ def __init__( self.output_template_file = output_template_file self.use_json = use_json self.force_upload = force_upload + self.parallel_upload = parallel_upload self.no_progressbar = no_progressbar self.metadata = metadata self.region = region @@ -161,6 +163,7 @@ def _export(self, template_path, use_json): self.code_signer, normalize_template=True, normalize_parameters=True, + parallel_upload=self.parallel_upload, ) exported_template = template.export() diff --git a/samcli/lib/package/artifact_exporter.py b/samcli/lib/package/artifact_exporter.py index bc38ee4a5e..32cf22d1df 100644 --- a/samcli/lib/package/artifact_exporter.py +++ b/samcli/lib/package/artifact_exporter.py @@ -15,7 +15,10 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. import os -from typing import Dict, List, Optional +import threading +from collections.abc import MutableMapping +from concurrent.futures import FIRST_EXCEPTION, ThreadPoolExecutor, wait +from typing import Callable, Dict, List, Optional from botocore.utils import set_value_from_jmespath @@ -52,6 +55,37 @@ # NOTE: sriram-mv, A cyclic dependency on `Template` needs to be broken. +DEFAULT_PARALLEL_UPLOAD_WORKERS = max(4, min(32, (os.cpu_count() or 1) * 2)) + + +class _ThreadSafeUploadCache(MutableMapping[str, str]): + """Simple thread-safe mapping used to deduplicate uploads across threads.""" + + def __init__(self, initial: Optional[MutableMapping[str, str]] = None): + # Copy into a regular dict so we can safely snapshot under a lock + self._cache: Dict[str, str] = dict(initial or {}) + self._lock = threading.Lock() + + def __getitem__(self, key: str) -> str: # pragma: no cover - small helper + with self._lock: + return self._cache[key] + + def __setitem__(self, key: str, value: str) -> None: # pragma: no cover - small helper + with self._lock: + self._cache[key] = value + + def __delitem__(self, key: str) -> None: # pragma: no cover - small helper + with self._lock: + del self._cache[key] + + def __iter__(self): # pragma: no cover - small helper + with self._lock: + return iter(dict(self._cache)) + + def __len__(self) -> int: # pragma: no cover - small helper + with self._lock: + return len(self._cache) + class CloudFormationStackResource(ResourceZip): """ @@ -89,6 +123,7 @@ def do_export(self, resource_id, resource_dict, parent_dir): normalize_template=True, normalize_parameters=True, parent_stack_id=resource_id, + parallel_upload=getattr(self, "parallel_upload", False), ).export() exported_template_str = yaml_dump(exported_template_dict) @@ -179,6 +214,7 @@ def __init__( normalize_template: bool = False, normalize_parameters: bool = False, parent_stack_id: str = "", + parallel_upload: bool = False, ): """ Reads the template and makes it ready for export @@ -202,6 +238,7 @@ def __init__( self.metadata_to_export = metadata_to_export self.uploaders = uploaders self.parent_stack_id = parent_stack_id + self.parallel_upload = parallel_upload def _export_global_artifacts(self, template_dict: Dict) -> Dict: """ @@ -276,10 +313,13 @@ def export(self) -> Dict: self._apply_global_values() self.template_dict = self._export_global_artifacts(self.template_dict) - cache: Optional[Dict] = None + cache: Optional[MutableMapping[str, str]] = None if is_experimental_enabled(ExperimentalFlag.PackagePerformance): cache = {} + if cache is not None and self.parallel_upload: + cache = _ThreadSafeUploadCache(cache) + export_jobs: List[Callable[[], None]] = [] for resource_logical_id, resource in self.template_dict["Resources"].items(): resource_type = resource.get("Type", None) resource_dict = resource.get("Properties", {}) @@ -291,12 +331,43 @@ def export(self) -> Dict: continue if resource_dict.get("PackageType", ZIP) != exporter_class.ARTIFACT_TYPE: continue - # Export code resources - exporter = exporter_class(self.uploaders, self.code_signer, cache) - exporter.export(full_path, resource_dict, self.template_dir) + + export_jobs.append(self._build_export_job(exporter_class, full_path, resource_dict, cache)) + + if self.parallel_upload and export_jobs: + self._execute_jobs_in_parallel(export_jobs) + else: + for job in export_jobs: + job() return self.template_dict + def _build_export_job( + self, + exporter_class, + resource_full_path: str, + resource_dict: Dict, + cache: Optional[MutableMapping[str, str]], + ) -> Callable[[], None]: + def _job() -> None: + exporter = exporter_class(self.uploaders, self.code_signer, cache) + setattr(exporter, "parallel_upload", self.parallel_upload) + exporter.export(resource_full_path, resource_dict, self.template_dir) + + return _job + + def _execute_jobs_in_parallel(self, jobs: List[Callable[[], None]]) -> None: + max_workers = min(len(jobs), DEFAULT_PARALLEL_UPLOAD_WORKERS) + if max_workers <= 1: + jobs[0]() + return + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [executor.submit(job) for job in jobs] + wait(futures, return_when=FIRST_EXCEPTION) + for future in futures: + future.result() + def delete(self, retain_resources: List): """ Deletes all the artifacts referenced by the given Cloudformation template diff --git a/samcli/lib/package/ecr_uploader.py b/samcli/lib/package/ecr_uploader.py index ae0337cfdb..af93528999 100644 --- a/samcli/lib/package/ecr_uploader.py +++ b/samcli/lib/package/ecr_uploader.py @@ -4,6 +4,7 @@ import base64 import logging +import threading from io import StringIO from pathlib import Path from typing import Dict @@ -50,6 +51,7 @@ def __init__( self.stream = StreamWriter(stream=stream, auto_flush=True) self.log_streamer = LogStreamer(stream=self.stream) self.login_session_active = False + self._login_lock = threading.Lock() @property def docker_client(self): @@ -88,8 +90,10 @@ def upload(self, image, resource_name): :return: remote ECR image path that has been uploaded. """ if not self.login_session_active: - self.login() - self.login_session_active = True + with self._login_lock: + if not self.login_session_active: + self.login() + self.login_session_active = True # Sometimes the `resource_name` is used as the `image` parameter to `tag_translation`. # This is because these two cases (directly from an archive or by ID) are effectively diff --git a/samcli/local/docker/container_client.py b/samcli/local/docker/container_client.py index 6b127802d7..bcb6d78202 100644 --- a/samcli/local/docker/container_client.py +++ b/samcli/local/docker/container_client.py @@ -97,6 +97,9 @@ def __init__(self, base_url=None): # Specify minimum version self.client_params["version"] = os.environ.get(GlobalConfig.DOCKER_API_ENV_VAR, DOCKER_MIN_API_VERSION) + # Increase client timeout to tolerate longer pushes/pulls + client_params["timeout"] = int(os.environ.get("SAM_CLI_DOCKER_TIMEOUT", "600")) + # Initialize DockerClient with processed parameters LOG.debug(f"Creating container client with parameters: {self.client_params}") super().__init__(**self.client_params) diff --git a/schema/samcli.json b/schema/samcli.json index 84ae577f12..fcc3726b85 100644 --- a/schema/samcli.json +++ b/schema/samcli.json @@ -1228,7 +1228,7 @@ "properties": { "parameters": { "title": "Parameters for the deploy command", - "description": "Available parameters for the deploy command:\n* guided:\nSpecify this flag to allow SAM CLI to guide you through the deployment using guided prompts.\n* template_file:\nAWS SAM template which references built artifacts for resources in the template. (if applicable)\n* no_execute_changeset:\nIndicates whether to execute the change set. Specify this flag to view stack changes before executing the change set.\n* fail_on_empty_changeset:\nSpecify whether AWS SAM CLI should return a non-zero exit code if there are no changes to be made to the stack. Defaults to a non-zero exit code.\n* confirm_changeset:\nPrompt to confirm if the computed changeset is to be deployed by SAM CLI.\n* disable_rollback:\nPreserves the state of previously provisioned resources when an operation fails.\n* on_failure:\nProvide an action to determine what will happen when a stack fails to create. Three actions are available:\n\n- ROLLBACK: This will rollback a stack to a previous known good state.\n\n- DELETE: The stack will rollback to a previous state if one exists, otherwise the stack will be deleted.\n\n- DO_NOTHING: The stack will not rollback or delete, this is the same as disabling rollback.\n\nDefault behaviour is ROLLBACK.\n\n\n\nThis option is mutually exclusive with --disable-rollback/--no-disable-rollback. You can provide\n--on-failure or --disable-rollback/--no-disable-rollback but not both at the same time.\n* max_wait_duration:\nMaximum duration in minutes to wait for the deployment to complete.\n* stack_name:\nName of the AWS CloudFormation stack.\n* s3_bucket:\nAWS S3 bucket where artifacts referenced in the template are uploaded.\n* image_repository:\nAWS ECR repository URI where artifacts referenced in the template are uploaded.\n* image_repositories:\nMapping of Function Logical ID to AWS ECR Repository URI.\n\nExample: Function_Logical_ID=ECR_Repo_Uri\nThis option can be specified multiple times.\n* force_upload:\nIndicates whether to override existing files in the S3 bucket. Specify this flag to upload artifacts even if they match existing artifacts in the S3 bucket.\n* s3_prefix:\nPrefix name that is added to the artifact's name when it is uploaded to the AWS S3 bucket.\n* kms_key_id:\nThe ID of an AWS KMS key that is used to encrypt artifacts that are at rest in the AWS S3 bucket.\n* role_arn:\nARN of an IAM role that AWS Cloudformation assumes when executing a deployment change set.\n* use_json:\nIndicates whether to use JSON as the format for the output AWS CloudFormation template. YAML is used by default.\n* resolve_s3:\nAutomatically resolve AWS S3 bucket for non-guided deployments. Enabling this option will also create a managed default AWS S3 bucket for you. If one does not provide a --s3-bucket value, the managed bucket will be used. Do not use --guided with this option.\n* resolve_image_repos:\nAutomatically create and delete ECR repositories for image-based functions in non-guided deployments. A companion stack containing ECR repos for each function will be deployed along with the template stack. Automatically created image repositories will be deleted if the corresponding functions are removed.\n* metadata:\nMap of metadata to attach to ALL the artifacts that are referenced in the template.\n* notification_arns:\nARNs of SNS topics that AWS Cloudformation associates with the stack.\n* tags:\nList of tags to associate with the stack.\n* parameter_overrides:\nString that contains AWS CloudFormation parameter overrides encoded as key=value pairs.\n* signing_profiles:\nA string that contains Code Sign configuration parameters as FunctionOrLayerNameToSign=SigningProfileName:SigningProfileOwner Since signing profile owner is optional, it could also be written as FunctionOrLayerNameToSign=SigningProfileName\n* no_progressbar:\nDoes not showcase a progress bar when uploading artifacts to S3 and pushing docker images to ECR\n* capabilities:\nList of capabilities that one must specify before AWS Cloudformation can create certain stacks.\n\nAccepted Values: CAPABILITY_IAM, CAPABILITY_NAMED_IAM, CAPABILITY_RESOURCE_POLICY, CAPABILITY_AUTO_EXPAND.\n\nLearn more at: https://docs.aws.amazon.com/serverlessrepo/latest/devguide/acknowledging-application-capabilities.html\n* profile:\nSelect a specific profile from your credential file to get AWS credentials.\n* region:\nSet the AWS Region of the service. (e.g. us-east-1)\n* beta_features:\nEnable/Disable beta features.\n* debug:\nTurn on debug logging to print debug message generated by AWS SAM CLI and display timestamps.\n* save_params:\nSave the parameters provided via the command line to the configuration file.", + "description": "Available parameters for the deploy command:\n* guided:\nSpecify this flag to allow SAM CLI to guide you through the deployment using guided prompts.\n* template_file:\nAWS SAM template which references built artifacts for resources in the template. (if applicable)\n* no_execute_changeset:\nIndicates whether to execute the change set. Specify this flag to view stack changes before executing the change set.\n* fail_on_empty_changeset:\nSpecify whether AWS SAM CLI should return a non-zero exit code if there are no changes to be made to the stack. Defaults to a non-zero exit code.\n* confirm_changeset:\nPrompt to confirm if the computed changeset is to be deployed by SAM CLI.\n* disable_rollback:\nPreserves the state of previously provisioned resources when an operation fails.\n* on_failure:\nProvide an action to determine what will happen when a stack fails to create. Three actions are available:\n\n- ROLLBACK: This will rollback a stack to a previous known good state.\n\n- DELETE: The stack will rollback to a previous state if one exists, otherwise the stack will be deleted.\n\n- DO_NOTHING: The stack will not rollback or delete, this is the same as disabling rollback.\n\nDefault behaviour is ROLLBACK.\n\n\n\nThis option is mutually exclusive with --disable-rollback/--no-disable-rollback. You can provide\n--on-failure or --disable-rollback/--no-disable-rollback but not both at the same time.\n* max_wait_duration:\nMaximum duration in minutes to wait for the deployment to complete.\n* stack_name:\nName of the AWS CloudFormation stack.\n* s3_bucket:\nAWS S3 bucket where artifacts referenced in the template are uploaded.\n* image_repository:\nAWS ECR repository URI where artifacts referenced in the template are uploaded.\n* image_repositories:\nMapping of Function Logical ID to AWS ECR Repository URI.\n\nExample: Function_Logical_ID=ECR_Repo_Uri\nThis option can be specified multiple times.\n* force_upload:\nIndicates whether to override existing files in the S3 bucket. Specify this flag to upload artifacts even if they match existing artifacts in the S3 bucket.\n* parallel_upload:\nEnable parallel upload of artifacts to S3/ECR during packaging before deployment.\n* s3_prefix:\nPrefix name that is added to the artifact's name when it is uploaded to the AWS S3 bucket.\n* kms_key_id:\nThe ID of an AWS KMS key that is used to encrypt artifacts that are at rest in the AWS S3 bucket.\n* role_arn:\nARN of an IAM role that AWS Cloudformation assumes when executing a deployment change set.\n* use_json:\nIndicates whether to use JSON as the format for the output AWS CloudFormation template. YAML is used by default.\n* resolve_s3:\nAutomatically resolve AWS S3 bucket for non-guided deployments. Enabling this option will also create a managed default AWS S3 bucket for you. If one does not provide a --s3-bucket value, the managed bucket will be used. Do not use --guided with this option.\n* resolve_image_repos:\nAutomatically create and delete ECR repositories for image-based functions in non-guided deployments. A companion stack containing ECR repos for each function will be deployed along with the template stack. Automatically created image repositories will be deleted if the corresponding functions are removed.\n* metadata:\nMap of metadata to attach to ALL the artifacts that are referenced in the template.\n* notification_arns:\nARNs of SNS topics that AWS Cloudformation associates with the stack.\n* tags:\nList of tags to associate with the stack.\n* parameter_overrides:\nString that contains AWS CloudFormation parameter overrides encoded as key=value pairs.\n* signing_profiles:\nA string that contains Code Sign configuration parameters as FunctionOrLayerNameToSign=SigningProfileName:SigningProfileOwner Since signing profile owner is optional, it could also be written as FunctionOrLayerNameToSign=SigningProfileName\n* no_progressbar:\nDoes not showcase a progress bar when uploading artifacts to S3 and pushing docker images to ECR\n* capabilities:\nList of capabilities that one must specify before AWS Cloudformation can create certain stacks.\n\nAccepted Values: CAPABILITY_IAM, CAPABILITY_NAMED_IAM, CAPABILITY_RESOURCE_POLICY, CAPABILITY_AUTO_EXPAND.\n\nLearn more at: https://docs.aws.amazon.com/serverlessrepo/latest/devguide/acknowledging-application-capabilities.html\n* profile:\nSelect a specific profile from your credential file to get AWS credentials.\n* region:\nSet the AWS Region of the service. (e.g. us-east-1)\n* beta_features:\nEnable/Disable beta features.\n* debug:\nTurn on debug logging to print debug message generated by AWS SAM CLI and display timestamps.\n* save_params:\nSave the parameters provided via the command line to the configuration file.", "type": "object", "properties": { "guided": { @@ -1308,6 +1308,11 @@ "type": "boolean", "description": "Indicates whether to override existing files in the S3 bucket. Specify this flag to upload artifacts even if they match existing artifacts in the S3 bucket." }, + "parallel_upload": { + "title": "parallel_upload", + "type": "boolean", + "description": "Enable parallel upload of artifacts to S3/ECR during packaging before deployment." + }, "s3_prefix": { "title": "s3_prefix", "type": "string", diff --git a/tests/unit/commands/deploy/test_command.py b/tests/unit/commands/deploy/test_command.py index 51b3b89bc8..ab6c64b7f5 100644 --- a/tests/unit/commands/deploy/test_command.py +++ b/tests/unit/commands/deploy/test_command.py @@ -35,6 +35,7 @@ def setUp(self): self.fail_on_empty_changset = True self.role_arn = "role_arn" self.force_upload = False + self.parallel_upload = False self.no_progressbar = False self.metadata = {"abc": "def"} self.region = None @@ -81,6 +82,7 @@ def test_all_args(self, mock_deploy_context, mock_deploy_click, mock_package_con image_repository=self.image_repository, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -114,6 +116,7 @@ def test_all_args(self, mock_deploy_context, mock_deploy_click, mock_package_con image_repository=self.image_repository, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -171,7 +174,7 @@ def test_all_args_guided_no_to_authorization_confirmation_prompt( context_mock = Mock() mockauth_per_resource.return_value = [("HelloWorldResource1", False), ("HelloWorldResource2", False)] mock_deploy_context.return_value.__enter__.return_value = context_mock - mock_confirm.side_effect = [True, True, False, True, False] + mock_confirm.side_effect = [True, True, False, False, True, False] mock_prompt.side_effect = [ "sam-app", "us-east-1", @@ -199,6 +202,7 @@ def test_all_args_guided_no_to_authorization_confirmation_prompt( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -268,7 +272,7 @@ def test_all_args_guided_use_defaults( mock_sam_function_provider.return_value.get_all.return_value = [function_mock] mockauth_per_resource.return_value = [("HelloWorldResource", False)] mock_deploy_context.return_value.__enter__.return_value = context_mock - mock_confirm.side_effect = [True, False, True, True, True, True, True] + mock_confirm.side_effect = [True, False, True, False, True, True, True, True] mock_prompt.side_effect = [ "sam-app", "us-east-1", @@ -301,6 +305,7 @@ def test_all_args_guided_use_defaults( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -334,6 +339,7 @@ def test_all_args_guided_use_defaults( image_repository=None, image_repositories={"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/managed-ecr"}, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix="sam-app", kms_key_id=self.kms_key_id, @@ -374,6 +380,7 @@ def test_all_args_guided_use_defaults( s3_prefix="sam-app", signing_profiles=self.signing_profiles, disable_rollback=True, + parallel_upload=self.parallel_upload, ) mock_managed_stack.assert_called_with(profile=self.profile, region="us-east-1") self.assertEqual(context_mock.run.call_count, 1) @@ -419,7 +426,7 @@ def test_all_args_guided( mock_sam_function_provider.return_value.get_all.return_value = [function_mock] mockauth_per_resource.return_value = [("HelloWorldResource", False)] mock_deploy_context.return_value.__enter__.return_value = context_mock - mock_confirm.side_effect = [True, False, True, True, True, True, True] + mock_confirm.side_effect = [True, False, True, False, True, True, True, True] mock_prompt.side_effect = [ "sam-app", "us-east-1", @@ -447,6 +454,7 @@ def test_all_args_guided( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -480,6 +488,7 @@ def test_all_args_guided( image_repository=None, image_repositories={"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1"}, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix="sam-app", kms_key_id=self.kms_key_id, @@ -520,6 +529,7 @@ def test_all_args_guided( s3_prefix="sam-app", signing_profiles=self.signing_profiles, disable_rollback=True, + parallel_upload=self.parallel_upload, ) mock_managed_stack.assert_called_with(profile=self.profile, region="us-east-1") self.assertEqual(context_mock.run.call_count, 1) @@ -584,7 +594,7 @@ def test_all_args_guided_no_save_echo_param_to_config( "testconfig.toml", "test-env", ] - mock_confirm.side_effect = [True, False, True, True, True, True, True] + mock_confirm.side_effect = [True, False, True, False, True, True, True, True] mock_managed_stack.return_value = "managed-s3-bucket" mock_signer_config_per_function.return_value = ({}, {}) @@ -596,6 +606,7 @@ def test_all_args_guided_no_save_echo_param_to_config( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -629,6 +640,7 @@ def test_all_args_guided_no_save_echo_param_to_config( image_repository=None, image_repositories={"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1"}, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix="sam-app", kms_key_id=self.kms_key_id, @@ -745,7 +757,7 @@ def test_all_args_guided_no_params_save_config( "testconfig.toml", "test-env", ] - mock_confirm.side_effect = [True, False, True, True, True, True, True] + mock_confirm.side_effect = [True, False, True, False, True, True, True, True] mock_get_cmd_names.return_value = ["deploy"] mock_managed_stack.return_value = "managed-s3-bucket" mock_signer_config_per_function.return_value = ({}, {}) @@ -757,6 +769,7 @@ def test_all_args_guided_no_params_save_config( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -790,6 +803,7 @@ def test_all_args_guided_no_params_save_config( image_repository=None, image_repositories={"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1"}, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix="sam-app", kms_key_id=self.kms_key_id, @@ -885,7 +899,7 @@ def test_all_args_guided_no_params_no_save_config( "us-east-1", ("CAPABILITY_IAM",), ] - mock_confirm.side_effect = [True, True, False, True, False, True, True] + mock_confirm.side_effect = [True, True, False, False, True, False, True, True] mock_managed_stack.return_value = "managed-s3-bucket" mock_signer_config_per_function.return_value = ({}, {}) @@ -898,6 +912,7 @@ def test_all_args_guided_no_params_no_save_config( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -931,6 +946,7 @@ def test_all_args_guided_no_params_no_save_config( image_repository=None, image_repositories={"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1"}, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix="sam-app", kms_key_id=self.kms_key_id, @@ -976,6 +992,7 @@ def test_all_args_resolve_s3( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -1007,6 +1024,7 @@ def test_all_args_resolve_s3( stack_name=self.stack_name, s3_bucket="managed-s3-bucket", force_upload=self.force_upload, + parallel_upload=self.parallel_upload, image_repository=None, image_repositories=None, no_progressbar=self.no_progressbar, @@ -1042,6 +1060,7 @@ def test_resolve_s3_and_s3_bucket_both_set(self): image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -1094,6 +1113,7 @@ def test_all_args_resolve_image_repos( image_repository=None, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -1125,6 +1145,7 @@ def test_all_args_resolve_image_repos( stack_name=self.stack_name, s3_bucket=self.s3_bucket, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, image_repository=None, image_repositories={"HelloWorldFunction1": self.image_repository}, no_progressbar=self.no_progressbar, @@ -1169,6 +1190,7 @@ def test_passing_parameter_overrides_to_context( image_repository=self.image_repository, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -1202,6 +1224,7 @@ def test_passing_parameter_overrides_to_context( image_repository=self.image_repository, image_repositories=None, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, s3_prefix=self.s3_prefix, kms_key_id=self.kms_key_id, @@ -1233,6 +1256,7 @@ def test_passing_parameter_overrides_to_context( kms_key_id=self.kms_key_id, use_json=self.use_json, force_upload=self.force_upload, + parallel_upload=self.parallel_upload, no_progressbar=self.no_progressbar, metadata=self.metadata, on_deploy=True, diff --git a/tests/unit/commands/deploy/test_guided_context.py b/tests/unit/commands/deploy/test_guided_context.py index b52c8fbac5..cce1e1f51e 100644 --- a/tests/unit/commands/deploy/test_guided_context.py +++ b/tests/unit/commands/deploy/test_guided_context.py @@ -73,7 +73,7 @@ def test_guided_prompts_check_defaults_non_public_resources_zips( patched_auth_per_resource.return_value = [ ("HelloWorldFunction", True), ] - patched_confirm.side_effect = [True, False, False, "", True, True, True] + patched_confirm.side_effect = [True, False, False, False, "", True, True, True] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) self.gc.guided_prompts(parameter_override_keys=None) @@ -82,6 +82,7 @@ def test_guided_prompts_check_defaults_non_public_resources_zips( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call(f"\t{self.gc.start_bold}Save arguments to configuration file{self.gc.end_bold}", default=True), call( f"\t {self.gc.start_bold}Delete the unreferenced repositories listed above when deploying?{self.gc.end_bold}", @@ -125,7 +126,7 @@ def test_guided_prompts_check_defaults_public_resources_zips( patched_get_buildable_stacks.return_value = (Mock(), []) # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] - patched_confirm.side_effect = [True, False, False, True, False, True, True] + patched_confirm.side_effect = [True, False, False, False, True, False, True, True] patched_manage_stack.return_value = "managed_s3_stack" self.gc.guided_prompts(parameter_override_keys=None) # Now to check for all the defaults on confirmations. @@ -133,6 +134,7 @@ def test_guided_prompts_check_defaults_public_resources_zips( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -192,7 +194,7 @@ def test_guided_prompts_check_defaults_public_resources_images( # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] get_resource_full_path_by_id_mock.return_value = None - patched_confirm.side_effect = [True, False, False, True, False, True, True] + patched_confirm.side_effect = [True, False, False, False, True, False, True, True] patched_manage_stack.return_value = "managed_s3_stack" self.gc.guided_prompts(parameter_override_keys=None) # Now to check for all the defaults on confirmations. @@ -200,6 +202,7 @@ def test_guided_prompts_check_defaults_public_resources_images( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -230,6 +233,7 @@ def test_guided_prompts_check_defaults_public_resources_images( call("\t#Shows you resources changes to be deployed and require a 'Y' to initiate deploy"), call("\t#SAM needs permission to be able to create roles to connect to the resources in your template"), call("\t#Preserves the state of previously provisioned resources when an operation fails"), + call("\t#Speed up artifact uploads by running them in parallel"), call("\n\tManaged S3 bucket: managed_s3_stack", bold=True), ] self.assertEqual(expected_click_secho_calls, patched_click_secho.call_args_list) @@ -270,7 +274,7 @@ def test_guided_prompts_check_defaults_public_resources_images_ecr_url( # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] get_resource_full_path_by_id_mock.return_value = "RandomFunction" - patched_confirm.side_effect = [True, False, False, True, False, True, True] + patched_confirm.side_effect = [True, False, False, False, True, False, True, True] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) self.gc.guided_prompts(parameter_override_keys=None) @@ -279,6 +283,7 @@ def test_guided_prompts_check_defaults_public_resources_images_ecr_url( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -307,6 +312,7 @@ def test_guided_prompts_check_defaults_public_resources_images_ecr_url( call("\t#Shows you resources changes to be deployed and require a 'Y' to initiate deploy"), call("\t#SAM needs permission to be able to create roles to connect to the resources in your template"), call("\t#Preserves the state of previously provisioned resources when an operation fails"), + call("\t#Speed up artifact uploads by running them in parallel"), call("\n\tManaged S3 bucket: managed_s3_stack", bold=True), ] self.assertEqual(expected_click_secho_calls, patched_click_secho.call_args_list) @@ -347,7 +353,7 @@ def test_guided_prompts_images_illegal_image_uri( # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] get_resource_full_path_by_id_mock.return_value = "RandomFunction" - patched_confirm.side_effect = [True, False, False, True, False, False, True] + patched_confirm.side_effect = [True, False, False, False, True, False, False, True] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) with self.assertRaises(GuidedDeployFailedError): @@ -393,7 +399,7 @@ def test_guided_prompts_images_missing_repo( ] # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] - patched_confirm.side_effect = [True, False, False, True, False, True, True] + patched_confirm.side_effect = [True, False, False, False, True, False, True, True] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) @@ -403,6 +409,7 @@ def test_guided_prompts_images_missing_repo( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -431,6 +438,7 @@ def test_guided_prompts_images_missing_repo( call("\t#Shows you resources changes to be deployed and require a 'Y' to initiate deploy"), call("\t#SAM needs permission to be able to create roles to connect to the resources in your template"), call("\t#Preserves the state of previously provisioned resources when an operation fails"), + call("\t#Speed up artifact uploads by running them in parallel"), call("\n\tManaged S3 bucket: managed_s3_stack", bold=True), ] self.assertEqual(expected_click_secho_calls, patched_click_secho.call_args_list) @@ -472,7 +480,7 @@ def test_guided_prompts_images_no_repo( # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] patched_get_resource_full_path_by_id.return_value = "RandomFunction" - patched_confirm.side_effect = [True, False, False, True, False, False, True] + patched_confirm.side_effect = [True, False, False, False, True, False, False, True] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) @@ -482,6 +490,7 @@ def test_guided_prompts_images_no_repo( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -514,6 +523,7 @@ def test_guided_prompts_images_no_repo( call("\t#Shows you resources changes to be deployed and require a 'Y' to initiate deploy"), call("\t#SAM needs permission to be able to create roles to connect to the resources in your template"), call("\t#Preserves the state of previously provisioned resources when an operation fails"), + call("\t#Speed up artifact uploads by running them in parallel"), call("\n\tManaged S3 bucket: managed_s3_stack", bold=True), ] self.assertEqual(expected_click_secho_calls, patched_click_secho.call_args_list) @@ -554,7 +564,7 @@ def test_guided_prompts_images_deny_deletion( ] # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] - patched_confirm.side_effect = [True, False, False, True, False, True, False] + patched_confirm.side_effect = [True, False, False, False, True, False, True, False] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) with self.assertRaises(GuidedDeployFailedError): @@ -597,7 +607,7 @@ def test_guided_prompts_images_blank_image_repository( ] # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] - patched_confirm.side_effect = [True, False, False, True, False, False, True] + patched_confirm.side_effect = [True, False, False, False, True, False, False, True] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) with self.assertRaises(GuidedDeployFailedError): @@ -642,13 +652,14 @@ def test_guided_prompts_with_given_capabilities( patched_get_buildable_stacks.return_value = (Mock(), []) self.gc.capabilities = given_capabilities # Series of inputs to confirmations so that full range of questions are asked. - patched_confirm.side_effect = [True, False, False, "", True, True, True] + patched_confirm.side_effect = [True, False, False, False, "", True, True, True] self.gc.guided_prompts(parameter_override_keys=None) # Now to check for all the defaults on confirmations. expected_confirmation_calls = [ call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call(f"\t{self.gc.start_bold}Save arguments to configuration file{self.gc.end_bold}", default=True), call( f"\t {self.gc.start_bold}Delete the unreferenced repositories listed above when deploying?{self.gc.end_bold}", @@ -690,7 +701,7 @@ def test_guided_prompts_check_configuration_file_prompt_calls( patched_signer_config_per_function.return_value = ({}, {}) # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] - patched_confirm.side_effect = [True, False, False, True, True, True, True] + patched_confirm.side_effect = [True, False, False, False, True, True, True, True] patched_manage_stack.return_value = "managed_s3_stack" patched_get_resource_full_path_by_id.return_value = "RandomFunction" self.gc.guided_prompts(parameter_override_keys=None) @@ -699,6 +710,7 @@ def test_guided_prompts_check_configuration_file_prompt_calls( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -752,7 +764,7 @@ def test_guided_prompts_check_parameter_from_template( # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] patched_get_resource_full_path_by_id.return_value = "RandomFunction" - patched_confirm.side_effect = [True, False, False, True, False, True, True] + patched_confirm.side_effect = [True, False, False, False, True, False, True, True] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) parameter_override_from_template = {"MyTestKey": {"Default": "MyTemplateDefaultVal"}} @@ -763,6 +775,7 @@ def test_guided_prompts_check_parameter_from_template( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -811,7 +824,7 @@ def test_guided_prompts_check_parameter_from_cmd_or_config( # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] patched_get_resource_full_path_by_id.return_value = "RandomFunction" - patched_confirm.side_effect = [True, False, False, True, False, True, True] + patched_confirm.side_effect = [True, False, False, False, True, False, True, True] patched_signer_config_per_function.return_value = ({}, {}) patched_manage_stack.return_value = "managed_s3_stack" parameter_override_from_template = {"MyTestKey": {"Default": "MyTemplateDefaultVal"}} @@ -822,6 +835,7 @@ def test_guided_prompts_check_parameter_from_cmd_or_config( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, @@ -884,7 +898,7 @@ def test_guided_prompts_with_code_signing( patched_signer_config_per_function.return_value = given_code_signing_configs patched_get_buildable_stacks.return_value = (Mock(), []) # Series of inputs to confirmations so that full range of questions are asked. - patched_confirm.side_effect = [True, False, False, given_sign_packages_flag, "", True, True, True] + patched_confirm.side_effect = [True, False, False, False, given_sign_packages_flag, "", True, True, True] patched_get_resource_full_path_by_id.return_value = "RandomFunction" self.gc.guided_prompts(parameter_override_keys=None) # Now to check for all the defaults on confirmations. @@ -892,6 +906,7 @@ def test_guided_prompts_with_code_signing( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}Do you want to sign your code?{self.gc.end_bold}", default=True, @@ -955,7 +970,7 @@ def test_guided_prompts_check_default_config_region( # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] patched_get_resource_full_path_by_id.return_value = "RandomFunction" - patched_confirm.side_effect = [True, False, False, True, True, True, True] + patched_confirm.side_effect = [True, False, False, False, True, True, True, True] patched_signer_config_per_function.return_value = ({}, {}) patched_manage_stack.return_value = "managed_s3_stack" patched_get_default_aws_region.return_value = "default_config_region" @@ -967,6 +982,7 @@ def test_guided_prompts_check_default_config_region( call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Allow SAM CLI IAM role creation{self.gc.end_bold}", default=True), call(f"\t{self.gc.start_bold}Disable rollback{self.gc.end_bold}", default=False), + call(f"\t{self.gc.start_bold}Enable parallel uploads{self.gc.end_bold}", default=False), call( f"\t{self.gc.start_bold}HelloWorldFunction has no authentication. Is this okay?{self.gc.end_bold}", default=False, diff --git a/tests/unit/commands/samconfig/test_samconfig.py b/tests/unit/commands/samconfig/test_samconfig.py index 07457bb14f..08e79f0b82 100644 --- a/tests/unit/commands/samconfig/test_samconfig.py +++ b/tests/unit/commands/samconfig/test_samconfig.py @@ -940,6 +940,7 @@ def test_deploy(self, do_cli_mock, template_artifacts_mock1, template_artifacts_ None, True, False, + False, "myprefix", "mykms", {"Key": "Value"}, @@ -1055,6 +1056,7 @@ def test_deploy_different_parameter_override_format( None, True, False, + False, "myprefix", "mykms", {"Key1": "Value1", "Key2": "Multiple spaces in the value"}, diff --git a/tests/unit/lib/package/test_artifact_exporter.py b/tests/unit/lib/package/test_artifact_exporter.py index c4d9e81fe9..301aa0609b 100644 --- a/tests/unit/lib/package/test_artifact_exporter.py +++ b/tests/unit/lib/package/test_artifact_exporter.py @@ -22,6 +22,7 @@ CloudFormationStackResource, CloudFormationStackSetResource, ServerlessApplicationResource, + _ThreadSafeUploadCache, ) from samcli.lib.package.packageable_resources import ( GraphQLApiCodeResource, @@ -1201,6 +1202,7 @@ def test_export_cloudformation_stack(self, TemplateMock): normalize_parameters=True, normalize_template=True, parent_stack_id="id", + parallel_upload=False, ) template_instance_mock.export.assert_called_once_with() self.s3_uploader_mock.upload.assert_called_once_with(mock.ANY, mock.ANY) @@ -1377,6 +1379,7 @@ def test_export_serverless_application(self, TemplateMock): normalize_parameters=True, normalize_template=True, parent_stack_id="id", + parallel_upload=False, ) template_instance_mock.export.assert_called_once_with() self.s3_uploader_mock.upload.assert_called_once_with(mock.ANY, mock.ANY) @@ -1543,6 +1546,96 @@ def test_template_export(self, yaml_parse_mock): resource_type2_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock, None) resource_type2_instance.export.assert_called_once_with("Resource2", mock.ANY, template_dir) + @patch.object(Template, "_execute_jobs_in_parallel") + @patch("samcli.lib.package.artifact_exporter.yaml_parse") + def test_template_export_parallel_invokes_executor(self, yaml_parse_mock, executor_mock): + parent_dir = os.path.sep + template_dir = os.path.join(parent_dir, "foo", "bar") + template_path = os.path.join(template_dir, "path") + template_str = self.example_yaml_template() + + resource_type1_class = Mock() + resource_type1_class.RESOURCE_TYPE = "resource_type1" + resource_type1_class.ARTIFACT_TYPE = ZIP + resource_type1_class.EXPORT_DESTINATION = Destination.S3 + resource_type1_class.return_value = Mock() + + resource_type2_class = Mock() + resource_type2_class.RESOURCE_TYPE = "resource_type2" + resource_type2_class.ARTIFACT_TYPE = ZIP + resource_type2_class.EXPORT_DESTINATION = Destination.S3 + resource_type2_class.return_value = Mock() + + resources_to_export = [resource_type1_class, resource_type2_class] + properties = {"foo": "bar"} + template_dict = { + "Resources": { + "Resource1": {"Type": "resource_type1", "Properties": properties}, + "Resource2": {"Type": "resource_type2", "Properties": properties}, + } + } + + yaml_parse_mock.return_value = template_dict + with patch("samcli.lib.package.artifact_exporter.open", mock.mock_open(read_data=template_str)): + template_exporter = Template( + template_path, + parent_dir, + self.uploaders_mock, + self.code_signer_mock, + resources_to_export, + parallel_upload=True, + ) + template_exporter.export() + + executor_mock.assert_called_once() + jobs = executor_mock.call_args[0][0] + self.assertEqual(len(jobs), 2) + + @patch("samcli.lib.package.artifact_exporter.is_experimental_enabled") + @patch("samcli.lib.package.artifact_exporter.yaml_parse") + def test_template_export_parallel_wraps_cache(self, yaml_parse_mock, is_experimental_enabled_mock): + is_experimental_enabled_mock.side_effect = lambda *args: { + (ExperimentalFlag.PackagePerformance,): True, + }.get(args, False) + + parent_dir = os.path.sep + template_dir = os.path.join(parent_dir, "foo", "bar") + template_path = os.path.join(template_dir, "path") + template_str = self.example_yaml_template() + + resource_type_class = Mock() + resource_type_class.RESOURCE_TYPE = "resource_type1" + resource_type_class.ARTIFACT_TYPE = ZIP + resource_type_class.EXPORT_DESTINATION = Destination.S3 + resource_type_instance = Mock() + resource_type_class.return_value = resource_type_instance + + captured_cache = {} + + def capture_cache(uploaders, code_signer, cache): + nonlocal captured_cache + captured_cache = cache + return resource_type_instance + + resource_type_class.side_effect = capture_cache + + properties = {"foo": "bar"} + template_dict = {"Resources": {"Resource1": {"Type": "resource_type1", "Properties": properties}}} + + yaml_parse_mock.return_value = template_dict + with patch("samcli.lib.package.artifact_exporter.open", mock.mock_open(read_data=template_str)): + template_exporter = Template( + template_path, + parent_dir, + self.uploaders_mock, + self.code_signer_mock, + [resource_type_class], + parallel_upload=True, + ) + template_exporter.export() + + self.assertIsInstance(captured_cache, _ThreadSafeUploadCache) + @patch("samcli.lib.package.artifact_exporter.is_experimental_enabled") @patch("samcli.lib.package.artifact_exporter.yaml_parse") def test_template_export_with_experimental_flag(self, yaml_parse_mock, is_experimental_enabled_mock):