Update upstream testsuite

.github/workflows/main.yml

@@ -50,9 +50,9 @@ jobs:
       with:
         submodules: 'true'
 
-    - uses: actions/setup-go@v3
+    - uses: actions/setup-go@v5
       with:
-        go-version: '1.20.x'
+        go-version: '1.24.x'
         cache: true
         cache-dependency-path: terraform-provider-aws/go.sum
 
@@ -148,9 +148,9 @@ jobs:
           options="-s -v --ls-start --gather-metrics"
         fi
 
-        if [[ ${{ matrix.service_partition.service }} == "lambda" ]]
+        if [[ ${{ matrix.service_partition.service }} == "organizations" ]]
         then
-          make prepare-lambda
+          make prepare-organizations
         fi
 
         if [[ ${{ matrix.service_partition.partition }} == "All" ]] 

.pre-commit-config.yaml

@@ -1,13 +1,16 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
-  - repo: https://github.com/psf/black
-    rev: 22.3.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.12.2
     hooks:
-      - id: black
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format
 
   - repo: https://github.com/pycqa/isort
-    rev: 5.9.1
+    rev: 6.0.1
     hooks:
       - id: isort
         name: isort (python)

Makefile

@@ -31,21 +31,21 @@ init_precommit:				## Install the pre-commit hook into your local git repository
 	($(VENV_RUN); pre-commit install)
 
 lint:						## Run linting
-	@echo "Running black... "
-	$(VENV_RUN); black --check .
+	@echo "Running ruff... "
+	$(VENV_RUN); python -m ruff check .
 
 format:						## Run formatting
-	$(VENV_RUN); python -m isort .; python -m black .
+	$(VENV_RUN); python -m isort .; python -m ruff format .
 
 reset-submodules:			## Reset the submodules to the specified commit
 	git submodule foreach git reset --hard
 
 get-submodules:				## Get the submodules
 	git submodule update --init --recursive
 
-prepare-lambda:				## Prepare the lambda function for deployment
-	@test -d terraform-provider-aws || echo "Please run 'git submodule update --init --recursive' to get the terraform-provider-aws submodule"
-	@cp -r terraform-provider-aws/internal/service/lambda/test-fixtures ./test-bin/ && echo "Copied test-fixtures to test-bin"
 
+prepare-organizations: install
+	$(VENV_RUN); awslocal organizations create-organization --feature-set ALL
 
-.PHONY: usage venv install init_precommit lint format reset-submodules
+
+.PHONY: usage venv install init_precommit lint format reset-submodules

README.md

@@ -91,6 +91,7 @@ AWS_ALTERNATE_REGION='us-west-2' python -m pytest terraform-provider-aws/interna
 | `AWS_THIRD_SECRET_ACCESS_KEY`        | test          |
 | `AWS_THIRD_ACCESS_KEY_ID`            | test          |
 | `AWS_THIRD_REGION`                   | eu-west-1     |
+| `AWS_ENDPOINT_URL`                   | http://localhost:4566 |
 
 ---
 

conftest.py

@@ -2,7 +2,6 @@
 import json
 import os
 import re
-from os.path import dirname, realpath, relpath
 from pathlib import Path
 
 import pytest
@@ -55,8 +54,8 @@ def runtest(self):
         """Run the test case"""
 
         cwd = os.getcwd()
-        service_path = dirname(Path(*relpath(self.path).split(os.sep)[1:]))
-        service = service_path.split(os.sep)[-1]
+        service_dir = self.path.parent
+        service = service_dir.name
 
         env = dict(os.environ)
         env.update(
@@ -71,18 +70,20 @@ def runtest(self):
                 "AWS_THIRD_SECRET_ACCESS_KEY": "test",
                 "AWS_THIRD_ACCESS_KEY_ID": "test",
                 "AWS_THIRD_REGION": "eu-west-1",
+                "AWS_ENDPOINT_URL": "http://localhost:4566",
             }
         )
 
+        test_binary = os.path.join(cwd, "test-bin", f"{service}.test")
         cmd = [
-            f"./{service}.test",
+            test_binary,
             "-test.v",
             "-test.parallel=1",
             "-test.count=1",
             "-test.timeout=60m",
             f"-test.run={self.name}",
         ]
-        return_code, stdout = execute_command(cmd, env, f"{cwd}/test-bin")
+        return_code, stdout = execute_command(cmd, env, str(service_dir))
         if return_code != 0:
             raise GoException(returncode=return_code, stderr=stdout)
         elif IS_GATHER_METRICS:
@@ -216,7 +217,7 @@ def pytest_sessionfinish(session, exitstatus):
 def _startup_localstack():
     try:
         _localstack_health_check()
-    except:
+    except Exception:
         os.system(
             "DEBUG=1 FAIL_FAST=1 DNS_ADDRESS=127.0.0.1 EXTENSION_DEV_MODE=1 DISABLE_EVENTS=1 LOCALSTACK_AUTH_TOKEN=$LOCALSTACK_AUTH_TOKEN localstack start -d"
         )

etc/0001-Force-S3-Use-Path-Style.patch

@@ -0,0 +1,24 @@
+From 83f8df495c5fc187d925a7dd61f93d1fdc4f405b Mon Sep 17 00:00:00 2001
+From: Bert Blommers <info@bertblommers.nl>
+Date: Sun, 13 Aug 2023 21:16:30 +0000
+Subject: [PATCH] Force S3 path-style addressing
+
+---
+ internal/provider/sdkv2/provider.go | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/internal/provider/sdkv2/provider.go b/internal/provider/sdkv2/provider.go
+index 603f4e6b1f..d2f1c3b7c2 100644
+--- a/internal/provider/sdkv2/provider.go
++++ b/internal/provider/sdkv2/provider.go
+@@ -352,7 +352,7 @@ func configure(ctx context.Context, provider *schema.Provider, d *schema.Resourc
+ 		MaxRetries:                     25, // Set default here, not in schema (muxing with v6 provider).
+ 		Profile:                        d.Get("profile").(string),
+ 		Region:                         d.Get("region").(string),
+-		S3UsePathStyle:                 d.Get("s3_use_path_style").(bool),
++		S3UsePathStyle:                 true,
+ 		SecretKey:                      d.Get("secret_key").(string),
+ 		SkipCredsValidation:            d.Get("skip_credentials_validation").(bool),
+ 		SkipRegionValidation:           d.Get("skip_region_validation").(bool),
+--
+2.25.1

requirements.txt

@@ -1,10 +1,11 @@
 click>=8.1.3
-pytest>=7.2.0
-docker>=6.0.1
-requests>=2.28.2
-black>=22.1
-isort>=5.10
+pytest>=8.4.1
+docker>=7.1.0
+requests>=2.32.4
+ruff>=0.12.1
+isort>=6.0.1
 pytest-xdist>=3.1.0
-pre-commit>=2.21.0
-localstack>=1.4.0.dev
-PyYAML~=6.0
+pre-commit>=4.2.0
+localstack>=4.6.0
+PyYAML~=6.0
+awscli-local

terraform_pytest/constants.py

@@ -10,7 +10,7 @@
 TF_REPO_SERVICE_PATH = os.path.join(TF_REPO_PATH, TF_REPO_SERVICE_FOLDER)
 
 # list of patch files to apply to the terraform repo
-TF_REPO_PATCH_FILES = ["etc/0001-Patch-Hardcode-endpoints-to-local-server.patch"]
+TF_REPO_PATCH_FILES = ["etc/0001-Force-S3-Use-Path-Style.patch"]
 
 # list of services that are supported by the localstack community edition
 LS_COMMUNITY_SERVICES = [

terraform_pytest/get_tf_partitions.py

@@ -1,5 +1,4 @@
 # Prints a JSON dict mapping the different partitions in the terraform-tests.yaml to their service
-import json
 import sys
 
 import yaml

terraform_pytest/main.py

@@ -65,7 +65,7 @@ def clean_command():
 
 
 if __name__ == "__main__":
-    cli.add_command(build_command)
     cli.add_command(patch_command)
+    cli.add_command(build_command)
     cli.add_command(clean_command)
     cli()

terraform_pytest/utils.py

@@ -8,7 +8,6 @@
     FAILING_SERVICES,
     LS_COMMUNITY_SERVICES,
     LS_PRO_SERVICES,
-    TF_REPO_NAME,
     TF_REPO_PATCH_FILES,
     TF_REPO_PATH,
     TF_REPO_SERVICE_PATH,
@@ -143,15 +142,27 @@ def patch_repository():
 
     return: None
     """
-    logging.info(f"Initiating patching process for repository: {TF_REPO_NAME}...")
+    logging.info("Initiating patching process for repository: %s...", TF_REPO_PATH)
     for patch_file in TF_REPO_PATCH_FILES:
         patch_file_path = os.path.realpath(patch_file)
+        logging.info("Applying patch: %s", patch_file)
+
+        # Check if patch can be applied
+        cmd = ["git", "apply", "--check", patch_file_path]
+        return_code, stdout = execute_command(cmd=cmd, cwd=TF_REPO_PATH)
+
+        if return_code != 0:
+            logging.error("Patch check failed for %s", patch_file)
+            logging.error(stdout)
+            raise Exception(f"Failed to apply patch {patch_file}")
+
+        # Apply the patch
         cmd = ["git", "apply", patch_file_path]
         return_code, stdout = execute_command(cmd=cmd, cwd=TF_REPO_PATH)
 
         if return_code != 0:
             logging.error("Failure encountered during repository patching.")
             logging.error(stdout)
+            raise Exception(f"Failed to apply patch {patch_file}")
         else:
-            if stdout:
-                logging.info(f"{patch_file} has been patched successfully.")
+            logging.info("%s has been applied successfully.", patch_file)