diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7de804c..7420897 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,9 +50,9 @@ jobs: with: submodules: 'true' - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: - go-version: '1.20.x' + go-version: '1.24.x' cache: true cache-dependency-path: terraform-provider-aws/go.sum @@ -148,9 +148,9 @@ jobs: options="-s -v --ls-start --gather-metrics" fi - if [[ ${{ matrix.service_partition.service }} == "lambda" ]] + if [[ ${{ matrix.service_partition.service }} == "organizations" ]] then - make prepare-lambda + make prepare-organizations fi if [[ ${{ matrix.service_partition.partition }} == "All" ]] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 271255b..6db0db8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,16 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: - - repo: https://github.com/psf/black - rev: 22.3.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.12.2 hooks: - - id: black + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format - repo: https://github.com/pycqa/isort - rev: 5.9.1 + rev: 6.0.1 hooks: - id: isort name: isort (python) diff --git a/Makefile b/Makefile index 69d35ad..042fb63 100644 --- a/Makefile +++ b/Makefile @@ -31,11 +31,11 @@ init_precommit: ## Install the pre-commit hook into your local git repository ($(VENV_RUN); pre-commit install) lint: ## Run linting - @echo "Running black... " - $(VENV_RUN); black --check . + @echo "Running ruff... " + $(VENV_RUN); python -m ruff check . format: ## Run formatting - $(VENV_RUN); python -m isort .; python -m black . + $(VENV_RUN); python -m isort .; python -m ruff format . reset-submodules: ## Reset the submodules to the specified commit git submodule foreach git reset --hard @@ -43,9 +43,9 @@ reset-submodules: ## Reset the submodules to the specified commit get-submodules: ## Get the submodules git submodule update --init --recursive -prepare-lambda: ## Prepare the lambda function for deployment - @test -d terraform-provider-aws || echo "Please run 'git submodule update --init --recursive' to get the terraform-provider-aws submodule" - @cp -r terraform-provider-aws/internal/service/lambda/test-fixtures ./test-bin/ && echo "Copied test-fixtures to test-bin" +prepare-organizations: install + $(VENV_RUN); awslocal organizations create-organization --feature-set ALL -.PHONY: usage venv install init_precommit lint format reset-submodules \ No newline at end of file + +.PHONY: usage venv install init_precommit lint format reset-submodules diff --git a/README.md b/README.md index 4996348..0c7b0fa 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ AWS_ALTERNATE_REGION='us-west-2' python -m pytest terraform-provider-aws/interna | `AWS_THIRD_SECRET_ACCESS_KEY` | test | | `AWS_THIRD_ACCESS_KEY_ID` | test | | `AWS_THIRD_REGION` | eu-west-1 | +| `AWS_ENDPOINT_URL` | http://localhost:4566 | --- diff --git a/conftest.py b/conftest.py index beaaec7..2f7b144 100644 --- a/conftest.py +++ b/conftest.py @@ -2,7 +2,6 @@ import json import os import re -from os.path import dirname, realpath, relpath from pathlib import Path import pytest @@ -55,8 +54,8 @@ def runtest(self): """Run the test case""" cwd = os.getcwd() - service_path = dirname(Path(*relpath(self.path).split(os.sep)[1:])) - service = service_path.split(os.sep)[-1] + service_dir = self.path.parent + service = service_dir.name env = dict(os.environ) env.update( @@ -71,18 +70,20 @@ def runtest(self): "AWS_THIRD_SECRET_ACCESS_KEY": "test", "AWS_THIRD_ACCESS_KEY_ID": "test", "AWS_THIRD_REGION": "eu-west-1", + "AWS_ENDPOINT_URL": "http://localhost:4566", } ) + test_binary = os.path.join(cwd, "test-bin", f"{service}.test") cmd = [ - f"./{service}.test", + test_binary, "-test.v", "-test.parallel=1", "-test.count=1", "-test.timeout=60m", f"-test.run={self.name}", ] - return_code, stdout = execute_command(cmd, env, f"{cwd}/test-bin") + return_code, stdout = execute_command(cmd, env, str(service_dir)) if return_code != 0: raise GoException(returncode=return_code, stderr=stdout) elif IS_GATHER_METRICS: @@ -216,7 +217,7 @@ def pytest_sessionfinish(session, exitstatus): def _startup_localstack(): try: _localstack_health_check() - except: + except Exception: os.system( "DEBUG=1 FAIL_FAST=1 DNS_ADDRESS=127.0.0.1 EXTENSION_DEV_MODE=1 DISABLE_EVENTS=1 LOCALSTACK_AUTH_TOKEN=$LOCALSTACK_AUTH_TOKEN localstack start -d" ) diff --git a/etc/0001-Force-S3-Use-Path-Style.patch b/etc/0001-Force-S3-Use-Path-Style.patch new file mode 100644 index 0000000..fb483f3 --- /dev/null +++ b/etc/0001-Force-S3-Use-Path-Style.patch @@ -0,0 +1,24 @@ +From 83f8df495c5fc187d925a7dd61f93d1fdc4f405b Mon Sep 17 00:00:00 2001 +From: Bert Blommers +Date: Sun, 13 Aug 2023 21:16:30 +0000 +Subject: [PATCH] Force S3 path-style addressing + +--- + internal/provider/sdkv2/provider.go | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/internal/provider/sdkv2/provider.go b/internal/provider/sdkv2/provider.go +index 603f4e6b1f..d2f1c3b7c2 100644 +--- a/internal/provider/sdkv2/provider.go ++++ b/internal/provider/sdkv2/provider.go +@@ -352,7 +352,7 @@ func configure(ctx context.Context, provider *schema.Provider, d *schema.Resourc + MaxRetries: 25, // Set default here, not in schema (muxing with v6 provider). + Profile: d.Get("profile").(string), + Region: d.Get("region").(string), +- S3UsePathStyle: d.Get("s3_use_path_style").(bool), ++ S3UsePathStyle: true, + SecretKey: d.Get("secret_key").(string), + SkipCredsValidation: d.Get("skip_credentials_validation").(bool), + SkipRegionValidation: d.Get("skip_region_validation").(bool), +-- +2.25.1 diff --git a/etc/0001-Patch-Hardcode-endpoints-to-local-server.patch b/etc/0001-Patch-Hardcode-endpoints-to-local-server.patch deleted file mode 100644 index c6182f2..0000000 --- a/etc/0001-Patch-Hardcode-endpoints-to-local-server.patch +++ /dev/null @@ -1,61 +0,0 @@ -From 83f8df495c5fc187d925a7dd61f93d1fdc4f405b Mon Sep 17 00:00:00 2001 -From: Bert Blommers -Date: Sun, 13 Aug 2023 21:16:30 +0000 -Subject: [PATCH] Patch endpoints to localhost:4566 - ---- - internal/conns/config.go | 14 ++++++++++++++ - internal/provider/provider.go | 4 ++-- - 2 files changed, 16 insertions(+), 2 deletions(-) - -diff --git a/internal/conns/config.go b/internal/conns/config.go -index 72c9cabde0..1f2e0d00e9 100644 ---- a/internal/conns/config.go -+++ b/internal/conns/config.go -@@ -55,10 +55,24 @@ type Config struct { - UseFIPSEndpoint bool - } - -+// XXX: added by bblommers -+func GetLocalEndpoints() map[string]string { -+ const localEndpoint = "http://localhost:4566" -+ var localEndpoints = map[string]string{} -+ for _, name := range names.Aliases() { -+ localEndpoints[name] = localEndpoint -+ } -+ return localEndpoints -+} -+ - // ConfigureProvider configures the provided provider Meta (instance data). - func (c *Config) ConfigureProvider(ctx context.Context, client *AWSClient) (*AWSClient, diag.Diagnostics) { - var diags diag.Diagnostics - -+ // XXX: added by bblommers -+ // insert custom endpoints -+ c.Endpoints = GetLocalEndpoints() -+ - awsbaseConfig := awsbase.Config{ - AccessKey: c.AccessKey, - APNInfo: StdUserAgentProducts(c.TerraformVersion), -diff --git a/internal/provider/provider.go b/internal/provider/provider.go -index 88c6ea9538..cfe78c5549 100644 ---- a/internal/provider/provider.go -+++ b/internal/provider/provider.go -@@ -452,13 +452,13 @@ func configure(ctx context.Context, provider *schema.Provider, d *schema.Resourc - CustomCABundle: d.Get("custom_ca_bundle").(string), - EC2MetadataServiceEndpoint: d.Get("ec2_metadata_service_endpoint").(string), - EC2MetadataServiceEndpointMode: d.Get("ec2_metadata_service_endpoint_mode").(string), -- Endpoints: make(map[string]string), -+ Endpoints: conns.GetLocalEndpoints(), // XXX: added by bblommers - HTTPProxy: d.Get("http_proxy").(string), - Insecure: d.Get("insecure").(bool), - MaxRetries: 25, // Set default here, not in schema (muxing with v6 provider). - Profile: d.Get("profile").(string), - Region: d.Get("region").(string), -- S3UsePathStyle: d.Get("s3_use_path_style").(bool), -+ S3UsePathStyle: true, - SecretKey: d.Get("secret_key").(string), - SkipCredsValidation: d.Get("skip_credentials_validation").(bool), - SkipRegionValidation: d.Get("skip_region_validation").(bool), --- -2.25.1 diff --git a/requirements.txt b/requirements.txt index 096b099..0651dc8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,11 @@ click>=8.1.3 -pytest>=7.2.0 -docker>=6.0.1 -requests>=2.28.2 -black>=22.1 -isort>=5.10 +pytest>=8.4.1 +docker>=7.1.0 +requests>=2.32.4 +ruff>=0.12.1 +isort>=6.0.1 pytest-xdist>=3.1.0 -pre-commit>=2.21.0 -localstack>=1.4.0.dev -PyYAML~=6.0 \ No newline at end of file +pre-commit>=4.2.0 +localstack>=4.6.0 +PyYAML~=6.0 +awscli-local diff --git a/terraform-provider-aws b/terraform-provider-aws index 96ac19e..819851a 160000 --- a/terraform-provider-aws +++ b/terraform-provider-aws @@ -1 +1 @@ -Subproject commit 96ac19e4c1feb5edee5f30aba29be233109e8717 +Subproject commit 819851a1477d8567b2e4c68dae85566c8c1efd85 diff --git a/terraform_pytest/constants.py b/terraform_pytest/constants.py index 3978123..3119ee5 100644 --- a/terraform_pytest/constants.py +++ b/terraform_pytest/constants.py @@ -10,7 +10,7 @@ TF_REPO_SERVICE_PATH = os.path.join(TF_REPO_PATH, TF_REPO_SERVICE_FOLDER) # list of patch files to apply to the terraform repo -TF_REPO_PATCH_FILES = ["etc/0001-Patch-Hardcode-endpoints-to-local-server.patch"] +TF_REPO_PATCH_FILES = ["etc/0001-Force-S3-Use-Path-Style.patch"] # list of services that are supported by the localstack community edition LS_COMMUNITY_SERVICES = [ diff --git a/terraform_pytest/get_tf_partitions.py b/terraform_pytest/get_tf_partitions.py index cbd6c63..6f6be58 100644 --- a/terraform_pytest/get_tf_partitions.py +++ b/terraform_pytest/get_tf_partitions.py @@ -1,5 +1,4 @@ # Prints a JSON dict mapping the different partitions in the terraform-tests.yaml to their service -import json import sys import yaml diff --git a/terraform_pytest/main.py b/terraform_pytest/main.py index fab37a6..0dc13b6 100644 --- a/terraform_pytest/main.py +++ b/terraform_pytest/main.py @@ -65,7 +65,7 @@ def clean_command(): if __name__ == "__main__": - cli.add_command(build_command) cli.add_command(patch_command) + cli.add_command(build_command) cli.add_command(clean_command) cli() diff --git a/terraform_pytest/utils.py b/terraform_pytest/utils.py index a25c3a5..3342d13 100644 --- a/terraform_pytest/utils.py +++ b/terraform_pytest/utils.py @@ -8,7 +8,6 @@ FAILING_SERVICES, LS_COMMUNITY_SERVICES, LS_PRO_SERVICES, - TF_REPO_NAME, TF_REPO_PATCH_FILES, TF_REPO_PATH, TF_REPO_SERVICE_PATH, @@ -143,15 +142,27 @@ def patch_repository(): return: None """ - logging.info(f"Initiating patching process for repository: {TF_REPO_NAME}...") + logging.info("Initiating patching process for repository: %s...", TF_REPO_PATH) for patch_file in TF_REPO_PATCH_FILES: patch_file_path = os.path.realpath(patch_file) + logging.info("Applying patch: %s", patch_file) + + # Check if patch can be applied + cmd = ["git", "apply", "--check", patch_file_path] + return_code, stdout = execute_command(cmd=cmd, cwd=TF_REPO_PATH) + + if return_code != 0: + logging.error("Patch check failed for %s", patch_file) + logging.error(stdout) + raise Exception(f"Failed to apply patch {patch_file}") + + # Apply the patch cmd = ["git", "apply", patch_file_path] return_code, stdout = execute_command(cmd=cmd, cwd=TF_REPO_PATH) if return_code != 0: logging.error("Failure encountered during repository patching.") logging.error(stdout) + raise Exception(f"Failed to apply patch {patch_file}") else: - if stdout: - logging.info(f"{patch_file} has been patched successfully.") + logging.info("%s has been applied successfully.", patch_file)