From 07f21fed99b1e91530d5a7cee28668395e6db058 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 11 Feb 2026 13:05:54 +0100 Subject: [PATCH 1/7] refactor: split integration tests into integration and e2e test suites Move Actor-based tests (which build and deploy Actors on the Apify platform) from tests/integration/actor/ to tests/e2e/, and flatten tests/integration/apify_api/ into tests/integration/. This gives a clearer separation: integration tests make real API calls but run locally, while e2e tests require building/deploying Actors on the platform. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/_tests.yaml | 12 ++ pyproject.toml | 5 + tests/e2e/README.md | 110 ++++++++++++ tests/{integration/actor => e2e}/__init__.py | 0 tests/e2e/_utils.py | 17 ++ .../actor_source_base/Dockerfile | 0 .../actor_source_base/requirements.txt | 0 .../actor => e2e}/actor_source_base/server.py | 0 .../actor_source_base/src/__init__.py | 0 .../actor_source_base/src/__main__.py | 0 .../actor_source_base/src/main.py | 0 tests/{integration/actor => e2e}/conftest.py | 84 ++++++++- .../actor => e2e}/test_actor_api_helpers.py | 2 +- .../actor => e2e}/test_actor_call_timeouts.py | 0 .../actor => e2e}/test_actor_charge.py | 0 .../test_actor_create_proxy_configuration.py | 0 .../actor => e2e}/test_actor_dataset.py | 2 +- .../actor => e2e}/test_actor_events.py | 0 .../test_actor_key_value_store.py | 2 +- .../actor => e2e}/test_actor_lifecycle.py | 0 .../actor => e2e}/test_actor_log.py | 0 .../actor => e2e}/test_actor_request_queue.py | 2 +- .../actor => e2e}/test_actor_scrapy.py | 0 .../actor => e2e}/test_apify_storages.py | 0 .../test_crawlers_with_storages.py | 0 .../actor => e2e}/test_fixtures.py | 0 tests/integration/README.md | 163 +----------------- tests/integration/apify_api/__init__.py | 0 .../{apify_api => }/test_apify_storages.py | 0 .../{apify_api => }/test_request_queue.py | 2 +- 30 files changed, 237 insertions(+), 164 deletions(-) create mode 100644 tests/e2e/README.md rename tests/{integration/actor => e2e}/__init__.py (100%) create mode 100644 tests/e2e/_utils.py rename tests/{integration/actor => e2e}/actor_source_base/Dockerfile (100%) rename tests/{integration/actor => e2e}/actor_source_base/requirements.txt (100%) rename tests/{integration/actor => e2e}/actor_source_base/server.py (100%) rename tests/{integration/actor => e2e}/actor_source_base/src/__init__.py (100%) rename tests/{integration/actor => e2e}/actor_source_base/src/__main__.py (100%) rename tests/{integration/actor => e2e}/actor_source_base/src/main.py (100%) rename tests/{integration/actor => e2e}/conftest.py (80%) rename tests/{integration/actor => e2e}/test_actor_api_helpers.py (99%) rename tests/{integration/actor => e2e}/test_actor_call_timeouts.py (100%) rename tests/{integration/actor => e2e}/test_actor_charge.py (100%) rename tests/{integration/actor => e2e}/test_actor_create_proxy_configuration.py (100%) rename tests/{integration/actor => e2e}/test_actor_dataset.py (99%) rename tests/{integration/actor => e2e}/test_actor_events.py (100%) rename tests/{integration/actor => e2e}/test_actor_key_value_store.py (99%) rename tests/{integration/actor => e2e}/test_actor_lifecycle.py (100%) rename tests/{integration/actor => e2e}/test_actor_log.py (100%) rename tests/{integration/actor => e2e}/test_actor_request_queue.py (99%) rename tests/{integration/actor => e2e}/test_actor_scrapy.py (100%) rename tests/{integration/actor => e2e}/test_apify_storages.py (100%) rename tests/{integration/actor => e2e}/test_crawlers_with_storages.py (100%) rename tests/{integration/actor => e2e}/test_fixtures.py (100%) delete mode 100644 tests/integration/apify_api/__init__.py rename tests/integration/{apify_api => }/test_apify_storages.py (100%) rename tests/integration/{apify_api => }/test_request_queue.py (99%) diff --git a/.github/workflows/_tests.yaml b/.github/workflows/_tests.yaml index 1bf30fc9..d6050142 100644 --- a/.github/workflows/_tests.yaml +++ b/.github/workflows/_tests.yaml @@ -29,3 +29,15 @@ jobs: python_version_for_codecov: "3.14" operating_system_for_codecov: ubuntu-latest tests_concurrency: "16" + + e2e_tests: + name: E2E tests + uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main + secrets: inherit + with: + python_versions: '["3.10", "3.14"]' + operating_systems: '["ubuntu-latest"]' + python_version_for_codecov: "3.14" + operating_system_for_codecov: ubuntu-latest + run_tests_command: "uv run poe e2e-tests-cov" + tests_concurrency: "16" diff --git a/pyproject.toml b/pyproject.toml index a02b8627..80f9a55c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -155,6 +155,9 @@ indent-style = "space" "**/{tests}/{integration}/*" = [ "PLC0415", # `import` should be at the top-level of a file ] +"**/{tests}/e2e/*" = [ + "PLC0415", # `import` should be at the top-level of a file +] "**/{docs,website}/**" = [ "D", # Everything from the pydocstyle "INP001", # File {filename} is part of an implicit namespace package, add an __init__.py @@ -234,6 +237,8 @@ unit-tests = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/unit unit-tests-cov = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/apify --cov-report=xml:coverage-unit.xml tests/unit" integration-tests = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/integration" integration-tests-cov = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/apify --cov-report=xml:coverage-integration.xml tests/integration" +e2e-tests = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} tests/e2e" +e2e-tests-cov = "uv run pytest --numprocesses=${TESTS_CONCURRENCY:-auto} --cov=src/apify --cov-report=xml:coverage-e2e.xml tests/e2e" check-code = ["lint", "type-check", "unit-tests"] [tool.poe.tasks.install-dev] diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 00000000..30a692e7 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,110 @@ +# E2E tests + +These tests build and run Actors using the Python SDK on the Apify platform. They are slower than integration tests (see `tests/integration/`) because they need to build and deploy Actors. Preferably try to write integration tests first, and only write E2E tests when you need to test something that can only be tested on the platform. + +## Running + +```bash +# Set the API token +export APIFY_TEST_USER_API_TOKEN= + +# Run the tests +uv run poe e2e-tests +``` + +If you want to run the tests on a different environment than the main Apify platform, set the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL. + +## How to write tests + +There are two fixtures which you can use to write tests: + +### `apify_client_async` + +This fixture just gives you an instance of `ApifyClientAsync` configured with the right token and API URL, so you don't have to do that yourself. + +```python +async def test_something(apify_client_async: ApifyClientAsync) -> None: + assert await apify_client_async.user('me').get() is not None +``` + +### `make_actor` + +This fixture returns a factory function for creating Actors on the Apify platform. + +For the Actor source, the fixture takes the files from `tests/e2e/actor_source_base`, builds the Apify SDK wheel from the current codebase, and adds the Actor source you passed to the fixture as an argument. You have to pass exactly one of the `main_func`, `main_py` and `source_files` arguments. + +The created Actor will be uploaded to the platform, built there, and after the test finishes, it will be automatically deleted. If the Actor build fails, it will not be deleted, so that you can check why the build failed. + +### Creating test Actor straight from a Python function + +You can create Actors straight from a Python function. This is great because you can have the test Actor source code checked with the linter. + +```python +async def test_something( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: + async def main() -> None: + async with Actor: + print('Hello!') + + actor = await make_actor(label='something', main_func=main) + run_result = await run_actor(actor) + + assert run_result.status == 'SUCCEEDED' +``` + +These Actors will have the `src/main.py` file set to the `main` function definition, prepended with `import asyncio` and `from apify import Actor`, for your convenience. + +### Creating Actor from source files + +You can also pass the source files directly if you need something more complex (e.g. pass some fixed value to the Actor source code or use multiple source files). + +To pass the source code of the `src/main.py` file directly, use the `main_py` argument to `make_actor`: + +```python +async def test_something( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: + expected_output = f'ACTOR_OUTPUT_{crypto_random_object_id(5)}' + main_py_source = f""" + import asyncio + from datetime import datetime + from apify import Actor + async def main(): + async with Actor: + print('Hello! It is ' + datetime.now().time()) + await Actor.set_value('OUTPUT', '{expected_output}') + """ + + actor = await make_actor(label='something', main_py=main_py_source) + await run_actor(actor) + + output_record = await actor.last_run().key_value_store().get_record('OUTPUT') + assert output_record is not None + assert output_record['value'] == expected_output +``` + +### Asserts + +Since test Actors are not executed as standard pytest tests, we don't get introspection of assertion expressions. In case of failure, only a bare `AssertionError` is shown, without the left and right values. This means, we must include explicit assertion messages to aid potential debugging. + +```python +async def test_add_and_fetch_requests( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: + """Test basic functionality of adding and fetching requests.""" + + async def main() -> None: + async with Actor: + rq = await Actor.open_request_queue() + await rq.add_request(f'https://apify.com/') + assert is_finished is False, f'is_finished={is_finished}' + + actor = await make_actor(label='rq-test', main_func=main) + run_result = await run_actor(actor) + + assert run_result.status == 'SUCCEEDED' +``` diff --git a/tests/integration/actor/__init__.py b/tests/e2e/__init__.py similarity index 100% rename from tests/integration/actor/__init__.py rename to tests/e2e/__init__.py diff --git a/tests/e2e/_utils.py b/tests/e2e/_utils.py new file mode 100644 index 00000000..b5323272 --- /dev/null +++ b/tests/e2e/_utils.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from crawlee._utils.crypto import crypto_random_object_id + + +def generate_unique_resource_name(label: str) -> str: + """Generates a unique resource name, which will contain the given label.""" + name_template = 'python-sdk-tests-{}-generated-{}' + template_length = len(name_template.format('', '')) + api_name_limit = 63 + generated_random_id_length = 8 + label_length_limit = api_name_limit - template_length - generated_random_id_length + + label = label.replace('_', '-') + assert len(label) <= label_length_limit, f'Max label length is {label_length_limit}, but got {len(label)}' + + return name_template.format(label, crypto_random_object_id(generated_random_id_length)) diff --git a/tests/integration/actor/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile similarity index 100% rename from tests/integration/actor/actor_source_base/Dockerfile rename to tests/e2e/actor_source_base/Dockerfile diff --git a/tests/integration/actor/actor_source_base/requirements.txt b/tests/e2e/actor_source_base/requirements.txt similarity index 100% rename from tests/integration/actor/actor_source_base/requirements.txt rename to tests/e2e/actor_source_base/requirements.txt diff --git a/tests/integration/actor/actor_source_base/server.py b/tests/e2e/actor_source_base/server.py similarity index 100% rename from tests/integration/actor/actor_source_base/server.py rename to tests/e2e/actor_source_base/server.py diff --git a/tests/integration/actor/actor_source_base/src/__init__.py b/tests/e2e/actor_source_base/src/__init__.py similarity index 100% rename from tests/integration/actor/actor_source_base/src/__init__.py rename to tests/e2e/actor_source_base/src/__init__.py diff --git a/tests/integration/actor/actor_source_base/src/__main__.py b/tests/e2e/actor_source_base/src/__main__.py similarity index 100% rename from tests/integration/actor/actor_source_base/src/__main__.py rename to tests/e2e/actor_source_base/src/__main__.py diff --git a/tests/integration/actor/actor_source_base/src/main.py b/tests/e2e/actor_source_base/src/main.py similarity index 100% rename from tests/integration/actor/actor_source_base/src/main.py rename to tests/e2e/actor_source_base/src/main.py diff --git a/tests/integration/actor/conftest.py b/tests/e2e/conftest.py similarity index 80% rename from tests/integration/actor/conftest.py rename to tests/e2e/conftest.py index cba40178..cb894087 100644 --- a/tests/integration/actor/conftest.py +++ b/tests/e2e/conftest.py @@ -13,10 +13,13 @@ from filelock import FileLock from apify_client import ApifyClient, ApifyClientAsync -from apify_shared.consts import ActorJobStatus, ActorPermissionLevel, ActorSourceType +from apify_shared.consts import ActorJobStatus, ActorPermissionLevel, ActorSourceType, ApifyEnvVars +from crawlee import service_locator -from .._utils import generate_unique_resource_name +import apify._actor +from ._utils import generate_unique_resource_name from apify._models import ActorRun +from apify.storage_clients._apify._alias_resolving import AliasResolver if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping @@ -26,7 +29,78 @@ _TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN' _API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL' -_SDK_ROOT_PATH = Path(__file__).parent.parent.parent.parent.resolve() +_SDK_ROOT_PATH = Path(__file__).parent.parent.parent.resolve() + + +@pytest.fixture(scope='session') +def apify_token() -> str: + api_token = os.getenv(_TOKEN_ENV_VAR) + + if not api_token: + raise RuntimeError(f'{_TOKEN_ENV_VAR} environment variable is missing, cannot run tests!') + + return api_token + + +@pytest.fixture(scope='session') +def apify_client_async(apify_token: str) -> ApifyClientAsync: + """Create an instance of the ApifyClientAsync.""" + api_url = os.getenv(_API_URL_ENV_VAR) + + return ApifyClientAsync(apify_token, api_url=api_url) + + +@pytest.fixture +def prepare_test_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Callable[[], None]: + """Prepare the testing environment by resetting the global state before each test. + + This fixture ensures that the global state of the package is reset to a known baseline before each test runs. + It also configures a temporary storage directory for test isolation. + + Args: + monkeypatch: Test utility provided by pytest for patching. + tmp_path: A unique temporary directory path provided by pytest for test isolation. + + Returns: + A callable that prepares the test environment. + """ + + def _prepare_test_env() -> None: + if hasattr(apify._actor.Actor, '__wrapped__'): + delattr(apify._actor.Actor, '__wrapped__') + + apify._actor.Actor._is_initialized = False + + # Set the environment variable for the local storage directory to the temporary path. + monkeypatch.setenv(ApifyEnvVars.LOCAL_STORAGE_DIR, str(tmp_path)) + + # Reset the services in the service locator. + service_locator._configuration = None + service_locator._event_manager = None + service_locator._storage_client = None + service_locator.storage_instance_manager.clear_cache() + + # Reset the AliasResolver class state. + AliasResolver._alias_map = {} + AliasResolver._alias_init_lock = None + + # Verify that the test environment was set up correctly. + assert os.environ.get(ApifyEnvVars.LOCAL_STORAGE_DIR) == str(tmp_path) + + return _prepare_test_env + + +@pytest.fixture(autouse=True) +def _isolate_test_environment(prepare_test_env: Callable[[], None]) -> None: + """Isolate the testing environment by resetting global state before each test. + + This fixture ensures that each test starts with a clean slate and that any modifications during the test + do not affect subsequent tests. It runs automatically for all tests. + + Args: + prepare_test_env: Fixture to prepare the environment before each test. + """ + prepare_test_env() @pytest.fixture(scope='session') @@ -70,13 +144,13 @@ def sdk_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) - def actor_base_source_files(sdk_wheel_path: Path) -> dict[str, str | bytes]: """Create a dictionary of the base source files for a testing Actor. - It takes the files from `tests/integration/actor_source_base`, builds the Apify SDK wheel from + It takes the files from `tests/e2e/actor_source_base`, builds the Apify SDK wheel from the current codebase, and adds them all together in a dictionary. """ source_files: dict[str, str | bytes] = {} # First read the actor_source_base files - actor_source_base_path = _SDK_ROOT_PATH / 'tests/integration/actor/actor_source_base' + actor_source_base_path = _SDK_ROOT_PATH / 'tests/e2e/actor_source_base' for path in actor_source_base_path.glob('**/*'): if not path.is_file(): diff --git a/tests/integration/actor/test_actor_api_helpers.py b/tests/e2e/test_actor_api_helpers.py similarity index 99% rename from tests/integration/actor/test_actor_api_helpers.py rename to tests/e2e/test_actor_api_helpers.py index 1a18adb7..3747dd3b 100644 --- a/tests/integration/actor/test_actor_api_helpers.py +++ b/tests/e2e/test_actor_api_helpers.py @@ -7,7 +7,7 @@ from apify_shared.consts import ActorPermissionLevel from crawlee._utils.crypto import crypto_random_object_id -from .._utils import generate_unique_resource_name +from ._utils import generate_unique_resource_name from apify import Actor from apify._models import ActorRun diff --git a/tests/integration/actor/test_actor_call_timeouts.py b/tests/e2e/test_actor_call_timeouts.py similarity index 100% rename from tests/integration/actor/test_actor_call_timeouts.py rename to tests/e2e/test_actor_call_timeouts.py diff --git a/tests/integration/actor/test_actor_charge.py b/tests/e2e/test_actor_charge.py similarity index 100% rename from tests/integration/actor/test_actor_charge.py rename to tests/e2e/test_actor_charge.py diff --git a/tests/integration/actor/test_actor_create_proxy_configuration.py b/tests/e2e/test_actor_create_proxy_configuration.py similarity index 100% rename from tests/integration/actor/test_actor_create_proxy_configuration.py rename to tests/e2e/test_actor_create_proxy_configuration.py diff --git a/tests/integration/actor/test_actor_dataset.py b/tests/e2e/test_actor_dataset.py similarity index 99% rename from tests/integration/actor/test_actor_dataset.py rename to tests/e2e/test_actor_dataset.py index 409df584..c80bb342 100644 --- a/tests/integration/actor/test_actor_dataset.py +++ b/tests/e2e/test_actor_dataset.py @@ -4,7 +4,7 @@ from apify_shared.consts import ApifyEnvVars -from .._utils import generate_unique_resource_name +from ._utils import generate_unique_resource_name from apify import Actor if TYPE_CHECKING: diff --git a/tests/integration/actor/test_actor_events.py b/tests/e2e/test_actor_events.py similarity index 100% rename from tests/integration/actor/test_actor_events.py rename to tests/e2e/test_actor_events.py diff --git a/tests/integration/actor/test_actor_key_value_store.py b/tests/e2e/test_actor_key_value_store.py similarity index 99% rename from tests/integration/actor/test_actor_key_value_store.py rename to tests/e2e/test_actor_key_value_store.py index 2ed9af29..19d63b0f 100644 --- a/tests/integration/actor/test_actor_key_value_store.py +++ b/tests/e2e/test_actor_key_value_store.py @@ -4,7 +4,7 @@ from apify_shared.consts import ApifyEnvVars -from .._utils import generate_unique_resource_name +from ._utils import generate_unique_resource_name from apify import Actor if TYPE_CHECKING: diff --git a/tests/integration/actor/test_actor_lifecycle.py b/tests/e2e/test_actor_lifecycle.py similarity index 100% rename from tests/integration/actor/test_actor_lifecycle.py rename to tests/e2e/test_actor_lifecycle.py diff --git a/tests/integration/actor/test_actor_log.py b/tests/e2e/test_actor_log.py similarity index 100% rename from tests/integration/actor/test_actor_log.py rename to tests/e2e/test_actor_log.py diff --git a/tests/integration/actor/test_actor_request_queue.py b/tests/e2e/test_actor_request_queue.py similarity index 99% rename from tests/integration/actor/test_actor_request_queue.py rename to tests/e2e/test_actor_request_queue.py index 1cc4c543..f9071c7e 100644 --- a/tests/integration/actor/test_actor_request_queue.py +++ b/tests/e2e/test_actor_request_queue.py @@ -3,7 +3,7 @@ import asyncio from typing import TYPE_CHECKING -from .._utils import generate_unique_resource_name +from ._utils import generate_unique_resource_name from apify import Actor from apify._models import ActorRun diff --git a/tests/integration/actor/test_actor_scrapy.py b/tests/e2e/test_actor_scrapy.py similarity index 100% rename from tests/integration/actor/test_actor_scrapy.py rename to tests/e2e/test_actor_scrapy.py diff --git a/tests/integration/actor/test_apify_storages.py b/tests/e2e/test_apify_storages.py similarity index 100% rename from tests/integration/actor/test_apify_storages.py rename to tests/e2e/test_apify_storages.py diff --git a/tests/integration/actor/test_crawlers_with_storages.py b/tests/e2e/test_crawlers_with_storages.py similarity index 100% rename from tests/integration/actor/test_crawlers_with_storages.py rename to tests/e2e/test_crawlers_with_storages.py diff --git a/tests/integration/actor/test_fixtures.py b/tests/e2e/test_fixtures.py similarity index 100% rename from tests/integration/actor/test_fixtures.py rename to tests/e2e/test_fixtures.py diff --git a/tests/integration/README.md b/tests/integration/README.md index 46e3b433..1993e85f 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,162 +1,17 @@ # Integration tests -There are two different groups of integration tests in this repository: -- Apify API integration tests. These test that the Apify SDK is correctly communicating with Apify API through Apify client. -- Actor integration tests. These test that the Apify SDK can be used in Actors deployed to Apify platform. These are very high level tests, and they test communication with the API and correct interaction with the Apify platform. +These tests make real requests to the Apify API (as opposed to unit tests that mock API calls), but they do not build or deploy Actors on the platform. They can be fully debugged locally and are faster than E2E tests. -To run these tests, you need to set the `APIFY_TEST_USER_API_TOKEN` environment variable to the API token of the Apify user you want to use for the tests, and then start them with `uv run poe integration-tests`. +Preferably try to write integration tests on this level if possible. Only write E2E tests (see `tests/e2e/`) when you need to test something that can only be tested by building and running an Actor on the platform. -## Apify API integration tests -The tests are making real requests to the Apify API as opposed to the unit tests that are mocking such API calls. On the other hand they are faster than `Actor integration tests` as they do not require building and deploying the Actor. These test can be also fully debugged locally. Preferably try to write integration tests on this level if possible. +## Running +```bash +# Set the API token +export APIFY_TEST_USER_API_TOKEN= -## Actor integration tests -We have integration tests which build and run Actors using the Python SDK on the Apify platform. These integration tests are slower than `Apify API integration tests` as they need to build and deploy Actors on the platform. Preferably try to write `Apify API integration tests` first, and only write `Actor integration tests` when you need to test something that can only be tested on the platform. - -If you want to run the integration tests on a different environment than the main Apify platform, you need to set the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL to the Apify API you want to use. - -### How to write tests - -There are two fixtures which you can use to write tests: - -#### `apify_client_async` - -This fixture just gives you an instance of `ApifyClientAsync` configured with the right token and API URL, so you don't have to do that yourself. - -```python -async def test_something(apify_client_async: ApifyClientAsync) -> None: - assert await apify_client_async.user('me').get() is not None +# Run the tests +uv run poe integration-tests ``` -#### `make_actor` - -This fixture returns a factory function for creating Actors on the Apify platform. - -For the Actor source, the fixture takes the files from `tests/integration/actor_source_base`, builds the Apify SDK wheel from the current codebase, and adds the Actor source you passed to the fixture as an argument. You have to pass exactly one of the `main_func`, `main_py` and `source_files` arguments. - -The created Actor will be uploaded to the platform, built there, and after the test finishes, it will be automatically deleted. If the Actor build fails, it will not be deleted, so that you can check why the build failed. - -#### Creating test Actor straight from a Python function - -You can create Actors straight from a Python function. This is great because you can have the test Actor source code checked with the linter. - -```python -async def test_something( - make_actor: MakeActorFunction, - run_actor: RunActorFunction, -) -> None: - async def main() -> None: - async with Actor: - print('Hello!') - - actor = await make_actor(label='something', main_func=main) - run_result = await run_actor(actor) - - assert run_result.status == 'SUCCEEDED' -``` - -These Actors will have the `src/main.py` file set to the `main` function definition, prepended with `import asyncio` and `from apify import Actor`, for your convenience. - -You can also pass extra imports directly to the main function: - -```python -async def test_something( - make_actor: MakeActorFunction, - run_actor: RunActorFunction, -) -> None: - async def main(): - import os - from apify_shared.consts import ActorEventTypes, ActorEnvVars - async with Actor: - print('The Actor is running with ' + os.getenv(ActorEnvVars.MEMORY_MBYTES) + 'MB of memory') - await Actor.on(ActorEventTypes.SYSTEM_INFO, lambda event_data: print(event_data)) - - actor = await make_actor(label='something', main_func=main) - run_result = await run_actor(actor) - - assert run_result.status == 'SUCCEEDED' -``` - -#### Creating Actor from source files - -You can also pass the source files directly if you need something more complex (e.g. pass some fixed value to the Actor source code or use multiple source files). - -To pass the source code of the `src/main.py` file directly, use the `main_py` argument to `make_actor`: - -```python -async def test_something( - make_actor: MakeActorFunction, - run_actor: RunActorFunction, -) -> None: - expected_output = f'ACTOR_OUTPUT_{crypto_random_object_id(5)}' - main_py_source = f""" - import asyncio - from datetime import datetime - from apify import Actor - async def main(): - async with Actor: - print('Hello! It is ' + datetime.now().time()) - await Actor.set_value('OUTPUT', '{expected_output}') - """ - - actor = await make_actor(label='something', main_py=main_py_source) - await run_actor(actor) - - output_record = await actor.last_run().key_value_store().get_record('OUTPUT') - assert output_record is not None - assert output_record['value'] == expected_output -``` - -Or you can pass multiple source files with the `source_files` argument, if you need something really complex: - -```python -async def test_something( - make_actor: MakeActorFunction, - run_actor: RunActorFunction, -) -> None: - actor_source_files = { - 'src/utils.py': """ - from datetime import datetime, timezone - - def get_current_datetime(): - return datetime.now(timezone.utc) - """, - 'src/main.py': """ - import asyncio - from apify import Actor - from .utils import get_current_datetime - - async def main(): - async with Actor: - current_datetime = get_current_datetime() - print('Hello! It is ' + current_datetime.time()) - """, - } - actor = await make_actor(label='something', source_files=actor_source_files) - actor_run = await run_actor(actor) - - assert actor_run.status == 'SUCCEEDED' -``` - -#### Asserts - -Since test Actors are not executed as standard pytest tests, we don't get introspection of assertion expressions. In case of failure, only a bare `AssertionError` is shown, without the left and right values. This means, we must include explicit assertion messages to aid potential debugging. - -```python -async def test_add_and_fetch_requests( - make_actor: MakeActorFunction, - run_actor: RunActorFunction, -) -> None: - """Test basic functionality of adding and fetching requests.""" - - async def main() -> None: - async with Actor: - rq = await Actor.open_request_queue() - await rq.add_request(f'https://apify.com/') - assert is_finished is False, f'is_finished={is_finished}' - - actor = await make_actor(label='rq-test', main_func=main) - run_result = await run_actor(actor) - - assert run_result.status == 'SUCCEEDED' -``` +If you want to run the tests against a different environment than the main Apify platform, set the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL. diff --git a/tests/integration/apify_api/__init__.py b/tests/integration/apify_api/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/integration/apify_api/test_apify_storages.py b/tests/integration/test_apify_storages.py similarity index 100% rename from tests/integration/apify_api/test_apify_storages.py rename to tests/integration/test_apify_storages.py diff --git a/tests/integration/apify_api/test_request_queue.py b/tests/integration/test_request_queue.py similarity index 99% rename from tests/integration/apify_api/test_request_queue.py rename to tests/integration/test_request_queue.py index e90c1600..a551e80c 100644 --- a/tests/integration/apify_api/test_request_queue.py +++ b/tests/integration/test_request_queue.py @@ -12,7 +12,7 @@ from crawlee import service_locator from crawlee.crawlers import BasicCrawler -from .._utils import generate_unique_resource_name +from ._utils import generate_unique_resource_name from apify import Actor, Request from apify.storage_clients import ApifyStorageClient from apify.storage_clients._apify._utils import unique_key_to_request_id From e63343ce67c3d19842f5a4e54230b967aad5dc03 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 11 Feb 2026 13:08:00 +0100 Subject: [PATCH 2/7] docs: add unit tests README and make all test READMEs consistent All three test directories (unit, integration, e2e) now have READMEs with a consistent structure: description, running instructions, and key fixtures. Co-Authored-By: Claude Opus 4.6 --- tests/e2e/README.md | 84 ++++++++++++++++--------------------- tests/integration/README.md | 14 ++++--- tests/unit/README.md | 15 +++++++ 3 files changed, 60 insertions(+), 53 deletions(-) create mode 100644 tests/unit/README.md diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 30a692e7..8f9d5ff1 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -1,43 +1,30 @@ # E2E tests -These tests build and run Actors using the Python SDK on the Apify platform. They are slower than integration tests (see `tests/integration/`) because they need to build and deploy Actors. Preferably try to write integration tests first, and only write E2E tests when you need to test something that can only be tested on the platform. +These tests build and run Actors using the Python SDK on the Apify platform. They are slower than integration tests (see [`tests/integration/`](../integration/)) because they need to build and deploy Actors. + +When writing new tests, prefer integration tests if possible. Only write E2E tests when you need to test something that requires building and running an Actor on the platform. ## Running ```bash -# Set the API token export APIFY_TEST_USER_API_TOKEN= - -# Run the tests uv run poe e2e-tests ``` -If you want to run the tests on a different environment than the main Apify platform, set the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL. - -## How to write tests - -There are two fixtures which you can use to write tests: - -### `apify_client_async` - -This fixture just gives you an instance of `ApifyClientAsync` configured with the right token and API URL, so you don't have to do that yourself. - -```python -async def test_something(apify_client_async: ApifyClientAsync) -> None: - assert await apify_client_async.user('me').get() is not None -``` - -### `make_actor` +To run against a different environment, also set `APIFY_INTEGRATION_TESTS_API_URL`. -This fixture returns a factory function for creating Actors on the Apify platform. +## Key fixtures -For the Actor source, the fixture takes the files from `tests/e2e/actor_source_base`, builds the Apify SDK wheel from the current codebase, and adds the Actor source you passed to the fixture as an argument. You have to pass exactly one of the `main_func`, `main_py` and `source_files` arguments. +- **`apify_client_async`** — A session-scoped `ApifyClientAsync` instance configured with the test token and API URL. +- **`prepare_test_env`** / **`_isolate_test_environment`** (autouse) — Resets global state and sets `APIFY_LOCAL_STORAGE_DIR` to a temporary directory before each test. +- **`make_actor`** — Factory for creating temporary Actors on the Apify platform (built, then auto-deleted after the test). +- **`run_actor`** — Starts an Actor run and waits for completion (10 min timeout). -The created Actor will be uploaded to the platform, built there, and after the test finishes, it will be automatically deleted. If the Actor build fails, it will not be deleted, so that you can check why the build failed. +## How to write tests -### Creating test Actor straight from a Python function +### Creating an Actor from a Python function -You can create Actors straight from a Python function. This is great because you can have the test Actor source code checked with the linter. +You can create Actors straight from a Python function. This is great because the test Actor source code gets checked by the linter. ```python async def test_something( @@ -54,13 +41,11 @@ async def test_something( assert run_result.status == 'SUCCEEDED' ``` -These Actors will have the `src/main.py` file set to the `main` function definition, prepended with `import asyncio` and `from apify import Actor`, for your convenience. - -### Creating Actor from source files +The `src/main.py` file will be set to the function definition, prepended with `import asyncio` and `from apify import Actor`. You can add extra imports directly inside the function body. -You can also pass the source files directly if you need something more complex (e.g. pass some fixed value to the Actor source code or use multiple source files). +### Creating an Actor from source files -To pass the source code of the `src/main.py` file directly, use the `main_py` argument to `make_actor`: +Pass the `main_py` argument for a single-file Actor: ```python async def test_something( @@ -74,7 +59,6 @@ async def test_something( from apify import Actor async def main(): async with Actor: - print('Hello! It is ' + datetime.now().time()) await Actor.set_value('OUTPUT', '{expected_output}') """ @@ -86,25 +70,31 @@ async def test_something( assert output_record['value'] == expected_output ``` -### Asserts - -Since test Actors are not executed as standard pytest tests, we don't get introspection of assertion expressions. In case of failure, only a bare `AssertionError` is shown, without the left and right values. This means, we must include explicit assertion messages to aid potential debugging. +Or pass `source_files` for multi-file Actors: ```python -async def test_add_and_fetch_requests( - make_actor: MakeActorFunction, - run_actor: RunActorFunction, -) -> None: - """Test basic functionality of adding and fetching requests.""" +actor_source_files = { + 'src/utils.py': """ + from datetime import datetime, timezone + def get_current_datetime(): + return datetime.now(timezone.utc) + """, + 'src/main.py': """ + import asyncio + from apify import Actor + from .utils import get_current_datetime + async def main(): + async with Actor: + print('Hello! It is ' + str(get_current_datetime())) + """, +} +actor = await make_actor(label='something', source_files=actor_source_files) +``` - async def main() -> None: - async with Actor: - rq = await Actor.open_request_queue() - await rq.add_request(f'https://apify.com/') - assert is_finished is False, f'is_finished={is_finished}' +### Assertions inside Actors - actor = await make_actor(label='rq-test', main_func=main) - run_result = await run_actor(actor) +Since test Actors are not executed as standard pytest tests, we don't get introspection of assertion expressions. In case of failure, only a bare `AssertionError` is shown. Always include explicit assertion messages: - assert run_result.status == 'SUCCEEDED' +```python +assert is_finished is False, f'is_finished={is_finished}' ``` diff --git a/tests/integration/README.md b/tests/integration/README.md index 1993e85f..d84875b8 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,17 +1,19 @@ # Integration tests -These tests make real requests to the Apify API (as opposed to unit tests that mock API calls), but they do not build or deploy Actors on the platform. They can be fully debugged locally and are faster than E2E tests. +These tests make real requests to the Apify API, but do not build or deploy Actors on the platform. They are faster than E2E tests and can be fully debugged locally. -Preferably try to write integration tests on this level if possible. Only write E2E tests (see `tests/e2e/`) when you need to test something that can only be tested by building and running an Actor on the platform. +When writing new tests, prefer this level if possible. Only write E2E tests (see [`tests/e2e/`](../e2e/)) when you need to test something that requires building and running an Actor on the platform. ## Running ```bash -# Set the API token export APIFY_TEST_USER_API_TOKEN= - -# Run the tests uv run poe integration-tests ``` -If you want to run the tests against a different environment than the main Apify platform, set the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL. +To run against a different environment, also set `APIFY_INTEGRATION_TESTS_API_URL`. + +## Key fixtures + +- **`apify_client_async`** — A session-scoped `ApifyClientAsync` instance configured with the test token and API URL. +- **`prepare_test_env`** / **`_isolate_test_environment`** (autouse) — Resets global state and sets `APIFY_LOCAL_STORAGE_DIR` to a temporary directory before each test. diff --git a/tests/unit/README.md b/tests/unit/README.md new file mode 100644 index 00000000..24b8461d --- /dev/null +++ b/tests/unit/README.md @@ -0,0 +1,15 @@ +# Unit tests + +These tests verify the SDK's internal logic in isolation, without making any real API calls. All external dependencies (Apify API, platform services) are mocked. They are fast and can be run without any environment variables or credentials. + +## Running + +```bash +uv run poe unit-tests +``` + +## Key fixtures + +- **`prepare_test_env`** / **`_isolate_test_environment`** (autouse) — Resets global state (Actor initialization, service locator, storage) and sets `APIFY_LOCAL_STORAGE_DIR` to a temporary directory before each test. +- **`apify_client_async_patcher`** — Helper for patching `ApifyClientAsync` methods to return fixed values or replacement functions, with automatic call tracking. +- **`httpserver`** — Local HTTP server (via `pytest-httpserver`) for testing HTTP interactions without real network calls. From 8a7eb709bc02bd9ea6ac4250120f50022bd868dd Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 11 Feb 2026 13:13:12 +0100 Subject: [PATCH 3/7] ci: remove concurrency limits from integration and e2e test jobs The reusable workflow `python_integration_tests.yaml` enforces `concurrency: integration_tests` and `max-parallel: 1`, which meant only one test matrix job could run at a time across both integration and e2e suites. Inline the job definitions to remove both limitations. All matrix combinations now run in parallel. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/_tests.yaml | 129 +++++++++++++++++++++++++++++----- 1 file changed, 110 insertions(+), 19 deletions(-) diff --git a/.github/workflows/_tests.yaml b/.github/workflows/_tests.yaml index d6050142..b201ae8c 100644 --- a/.github/workflows/_tests.yaml +++ b/.github/workflows/_tests.yaml @@ -20,24 +20,115 @@ jobs: tests_concurrency: "1" integration_tests: - name: Integration tests - uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main - secrets: inherit - with: - python_versions: '["3.10", "3.14"]' - operating_systems: '["ubuntu-latest"]' - python_version_for_codecov: "3.14" - operating_system_for_codecov: ubuntu-latest - tests_concurrency: "16" + name: Integration tests (${{ matrix.python-version }}, ${{ matrix.os }}) + + if: >- + ${{ + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.owner.login == 'apify') || + (github.event_name == 'push' && github.ref == 'refs/heads/master') + }} + + strategy: + matrix: + os: ["ubuntu-latest"] + python-version: ["3.10", "3.14"] + + runs-on: ${{ matrix.os }} + + env: + TESTS_CONCURRENCY: "16" + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + + - name: Set up uv package manager + uses: astral-sh/setup-uv@v7 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Python dependencies + run: uv run poe install-dev + + - name: Run integration tests + run: uv run poe integration-tests-cov + env: + APIFY_TEST_USER_API_TOKEN: ${{ secrets.APIFY_TEST_USER_PYTHON_SDK_API_TOKEN }} + APIFY_TEST_USER_2_API_TOKEN: ${{ secrets.APIFY_TEST_USER_2_API_TOKEN }} + + - name: Upload integration test coverage + if: >- + ${{ + matrix.os == 'ubuntu-latest' && + matrix.python-version == '3.14' && + env.CODECOV_TOKEN != '' + }} + uses: codecov/codecov-action@v5 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + token: ${{ env.CODECOV_TOKEN }} + files: coverage-integration.xml + flags: integration e2e_tests: - name: E2E tests - uses: apify/workflows/.github/workflows/python_integration_tests.yaml@main - secrets: inherit - with: - python_versions: '["3.10", "3.14"]' - operating_systems: '["ubuntu-latest"]' - python_version_for_codecov: "3.14" - operating_system_for_codecov: ubuntu-latest - run_tests_command: "uv run poe e2e-tests-cov" - tests_concurrency: "16" + name: E2E tests (${{ matrix.python-version }}, ${{ matrix.os }}) + + if: >- + ${{ + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.owner.login == 'apify') || + (github.event_name == 'push' && github.ref == 'refs/heads/master') + }} + + strategy: + matrix: + os: ["ubuntu-latest"] + python-version: ["3.10", "3.14"] + + runs-on: ${{ matrix.os }} + + env: + TESTS_CONCURRENCY: "16" + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + + - name: Set up uv package manager + uses: astral-sh/setup-uv@v7 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Python dependencies + run: uv run poe install-dev + + - name: Run E2E tests + run: uv run poe e2e-tests-cov + env: + APIFY_TEST_USER_API_TOKEN: ${{ secrets.APIFY_TEST_USER_PYTHON_SDK_API_TOKEN }} + APIFY_TEST_USER_2_API_TOKEN: ${{ secrets.APIFY_TEST_USER_2_API_TOKEN }} + + - name: Upload E2E test coverage + if: >- + ${{ + matrix.os == 'ubuntu-latest' && + matrix.python-version == '3.14' && + env.CODECOV_TOKEN != '' + }} + uses: codecov/codecov-action@v5 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + token: ${{ env.CODECOV_TOKEN }} + files: coverage-e2e.xml + flags: e2e From 38f5c0424e35d4dce5d4d01a36624547e51058f0 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 11 Feb 2026 13:23:08 +0100 Subject: [PATCH 4/7] ci: add max-parallel: 1 to e2e tests to avoid platform memory limits E2E tests build and run Actors on the Apify platform, which can exceed the account memory limit when multiple matrix jobs run concurrently. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/_tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/_tests.yaml b/.github/workflows/_tests.yaml index b201ae8c..e8ea609b 100644 --- a/.github/workflows/_tests.yaml +++ b/.github/workflows/_tests.yaml @@ -86,6 +86,7 @@ jobs: }} strategy: + max-parallel: 1 # E2E tests build and run Actors on the platform, limit parallelism to avoid exceeding memory limits. matrix: os: ["ubuntu-latest"] python-version: ["3.10", "3.14"] From b41cafbe8808c0a706a6afcb1801327e8f44b363 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 12 Feb 2026 09:13:36 +0100 Subject: [PATCH 5/7] ignore PLC0415 for all tests --- pyproject.toml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 80f9a55c..1656e9ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,18 +145,13 @@ indent-style = "space" "**/{tests}/*" = [ "D", # Everything from the pydocstyle "INP001", # File {filename} is part of an implicit namespace package, add an __init__.py + "PLC0415", # `import` should be at the top-level of a file "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable "S101", # Use of assert detected "SLF001", # Private member accessed: `{name}` "T20", # flake8-print - "TRY301", # Abstract `raise` to an inner function "TID252", # Prefer absolute imports over relative imports from parent modules -] -"**/{tests}/{integration}/*" = [ - "PLC0415", # `import` should be at the top-level of a file -] -"**/{tests}/e2e/*" = [ - "PLC0415", # `import` should be at the top-level of a file + "TRY301", # Abstract `raise` to an inner function ] "**/{docs,website}/**" = [ "D", # Everything from the pydocstyle From 4a0bd80cfd44a09a9ece74e169e3fea2b5fd8730 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 12 Feb 2026 09:20:36 +0100 Subject: [PATCH 6/7] Try to update the E2E parallelization --- .github/workflows/_tests.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_tests.yaml b/.github/workflows/_tests.yaml index e8ea609b..66d504a6 100644 --- a/.github/workflows/_tests.yaml +++ b/.github/workflows/_tests.yaml @@ -86,7 +86,11 @@ jobs: }} strategy: - max-parallel: 1 # E2E tests build and run Actors on the platform, limit parallelism to avoid exceeding memory limits. + # E2E tests build and run Actors on the platform. Limit parallel workflows to 2 to allow running both + # the oldest and newest Python versions simultaneously, while not exceeding the platform's limits. Two + # parallel workflows with 8 pytest workers each is a compromise between good test parallelization and + # not being rate-limited by the platform. + max-parallel: 2 matrix: os: ["ubuntu-latest"] python-version: ["3.10", "3.14"] @@ -94,7 +98,7 @@ jobs: runs-on: ${{ matrix.os }} env: - TESTS_CONCURRENCY: "16" + TESTS_CONCURRENCY: "8" steps: - name: Checkout repository From 994e97059eb997bf7ff1b39faf6aa2f1183f3534 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 12 Feb 2026 10:20:50 +0100 Subject: [PATCH 7/7] revert max-parallel --- .github/workflows/_tests.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_tests.yaml b/.github/workflows/_tests.yaml index 66d504a6..9151c61f 100644 --- a/.github/workflows/_tests.yaml +++ b/.github/workflows/_tests.yaml @@ -86,11 +86,10 @@ jobs: }} strategy: - # E2E tests build and run Actors on the platform. Limit parallel workflows to 2 to allow running both - # the oldest and newest Python versions simultaneously, while not exceeding the platform's limits. Two - # parallel workflows with 8 pytest workers each is a compromise between good test parallelization and - # not being rate-limited by the platform. - max-parallel: 2 + # E2E tests build and run Actors on the platform. Limit parallel workflows to 1 to avoid exceeding + # the platform's memory limits. A single workflow with 16 pytest workers provides good test + # parallelization while staying within platform constraints. + max-parallel: 1 matrix: os: ["ubuntu-latest"] python-version: ["3.10", "3.14"] @@ -98,7 +97,7 @@ jobs: runs-on: ${{ matrix.os }} env: - TESTS_CONCURRENCY: "8" + TESTS_CONCURRENCY: "16" steps: - name: Checkout repository