Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ async def iterate_keys(
raise TypeError('The items data was received in an incorrect format.')

# Get all keys, sorted alphabetically
keys = sorted(items_data.keys()) # ty: ignore[invalid-argument-type]
keys = sorted(items_data.keys())

# Apply exclusive_start_key filter if provided
if exclusive_start_key is not None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ class TestInput:
expected_pw_count=0,
expected_static_count=2,
# Lack of ty support, see https://github.com/astral-sh/ty/issues/2348.
rendering_types=cycle(['static']), # ty: ignore[invalid-argument-type]
rendering_types=cycle(['static']),
detection_probability_recommendation=cycle([0]),
),
id='Static only',
Expand All @@ -125,7 +125,7 @@ class TestInput:
TestInput(
expected_pw_count=2,
expected_static_count=0,
rendering_types=cycle(['client only']), # ty: ignore[invalid-argument-type]
rendering_types=cycle(['client only']),
detection_probability_recommendation=cycle([0]),
),
id='Client only',
Expand All @@ -134,7 +134,7 @@ class TestInput:
TestInput(
expected_pw_count=1,
expected_static_count=1,
rendering_types=cycle(['static', 'client only']), # ty: ignore[invalid-argument-type]
rendering_types=cycle(['static', 'client only']),
detection_probability_recommendation=cycle([0]),
),
id='Mixed',
Expand All @@ -143,7 +143,7 @@ class TestInput:
TestInput(
expected_pw_count=2,
expected_static_count=2,
rendering_types=cycle(['static', 'client only']), # ty: ignore[invalid-argument-type]
rendering_types=cycle(['static', 'client only']),
detection_probability_recommendation=cycle([1]),
),
id='Enforced rendering type detection',
Expand Down Expand Up @@ -207,7 +207,7 @@ async def pre_nav_hook(context: AdaptivePlaywrightPreNavCrawlingContext) -> None
async def test_adaptive_crawling_parsel(test_urls: list[str]) -> None:
"""Top level test for parsel. Only one argument combination. (The rest of code is tested with bs variant.)"""
predictor = _SimpleRenderingTypePredictor(
rendering_types=cycle(['static', 'client only']), # ty: ignore[invalid-argument-type]
rendering_types=cycle(['static', 'client only']),
detection_probability_recommendation=cycle([0]),
)

Expand Down Expand Up @@ -693,7 +693,7 @@ async def test_adaptive_context_helpers_on_changed_selector(test_urls: list[str]
dynamically changed text instead of the original static text.
"""
browser_only_predictor_no_detection = _SimpleRenderingTypePredictor(
rendering_types=cycle(['client only']), # ty: ignore[invalid-argument-type]
rendering_types=cycle(['client only']),
detection_probability_recommendation=cycle([0]),
)
expected_h3_tag = f'<h3>{_H3_CHANGED_TEXT}</h3>'
Expand All @@ -719,7 +719,7 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
async def test_adaptive_context_query_non_existing_element(test_urls: list[str]) -> None:
"""Test that querying non-existing selector returns `None`"""
browser_only_predictor_no_detection = _SimpleRenderingTypePredictor(
rendering_types=cycle(['client only']), # ty: ignore[invalid-argument-type]
rendering_types=cycle(['client only']),
detection_probability_recommendation=cycle([0]),
)

Expand All @@ -746,8 +746,7 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
TestInput(
expected_pw_count=0,
expected_static_count=2,
# Lack of ty support, see https://github.com/astral-sh/ty/issues/2348.
rendering_types=cycle(['static']), # ty: ignore[invalid-argument-type]
rendering_types=cycle(['static']),
detection_probability_recommendation=cycle([0]),
),
id='Static only',
Expand All @@ -756,7 +755,7 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
TestInput(
expected_pw_count=2,
expected_static_count=0,
rendering_types=cycle(['client only']), # ty: ignore[invalid-argument-type]
rendering_types=cycle(['client only']),
detection_probability_recommendation=cycle([0]),
),
id='Client only',
Expand All @@ -765,7 +764,7 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
TestInput(
expected_pw_count=2,
expected_static_count=2,
rendering_types=cycle(['static', 'client only']), # ty: ignore[invalid-argument-type]
rendering_types=cycle(['static', 'client only']),
detection_probability_recommendation=cycle([1]),
),
id='Enforced rendering type detection',
Expand Down
24 changes: 13 additions & 11 deletions tests/unit/crawlers/_basic/test_basic_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1093,17 +1093,19 @@ async def handler(context: BasicCrawlingContext) -> None:
else:
assert final_statistics.msg == 'Final request statistics:'

# ignore[attr-defined] since `extra` parameters are not defined for `LogRecord`
assert final_statistics.requests_finished == 4
assert final_statistics.requests_failed == 33
assert final_statistics.retry_histogram == [1, 4, 8]
assert final_statistics.request_avg_failed_duration == 99.0
assert final_statistics.request_avg_finished_duration == 0.483
assert final_statistics.requests_finished_per_minute == 0.33
assert final_statistics.requests_failed_per_minute == 0.1
assert final_statistics.request_total_duration == 720.0
assert final_statistics.requests_total == 37
assert final_statistics.crawler_runtime == 300.0
# `extra` parameters are not defined on `LogRecord`, so we cast to `Any` to access them.
record = cast('Any', final_statistics)

assert record.requests_finished == 4
assert record.requests_failed == 33
assert record.retry_histogram == [1, 4, 8]
assert record.request_avg_failed_duration == 99.0
assert record.request_avg_finished_duration == 0.483
assert record.requests_finished_per_minute == 0.33
assert record.requests_failed_per_minute == 0.1
assert record.request_total_duration == 720.0
assert record.requests_total == 37
assert record.crawler_runtime == 300.0


async def test_crawler_manual_stop() -> None:
Expand Down
40 changes: 20 additions & 20 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading