Stabilize test suite: fix flaky timing tests, add rerun support, and handle Windows resource leaks

CHANGES/11992.contrib.rst

@@ -0,0 +1,4 @@
+Fixed flaky import time test for Python 3.12+ -- by :user:`rodrigobnogueira`.
+
+Refactored to use version comparison instead of explicit version list,
+making the test future-proof for new Python releases.

CONTRIBUTORS.txt

@@ -318,6 +318,7 @@ Raúl Cumplido
 Required Field
 Robert Lu
 Robert Nikolich
+Rodrigo Nogueira
 Roman Markeloff
 Roman Podoliaka
 Roman Postnov

aiohttp/pytest_plugin.py

@@ -3,7 +3,7 @@
 import inspect
 import warnings
 from collections.abc import Awaitable, Callable, Iterator
-from typing import Any, Protocol, TypeVar, overload
+from typing import Any, NamedTuple, Protocol, TypeVar, overload
 
 import pytest
 
@@ -64,6 +64,37 @@ def __call__(
     ) -> Awaitable[RawTestServer]: ...
 
 
+class RerunThresholdParams(NamedTuple):
+    """Parameters for dynamic threshold calculation in flaky tests."""
+
+    base: float
+    increment_per_rerun: float
+
+
+@pytest.fixture
+def rerun_adjusted_threshold(request: pytest.FixtureRequest) -> float:
+    """Calculate dynamic threshold based on rerun count (via indirect parametrization).
+
+    Returns ``base + (rerun_count * increment_per_rerun)``.
+    The ``rerun_count`` is determined from ``pytest-rerunfailures`` (0 for initial run,
+    1 for first rerun, etc.).
+
+    Usage::
+
+        @pytest.mark.flaky(reruns=3)
+        @pytest.mark.parametrize(
+            "rerun_adjusted_threshold",
+            [RerunThresholdParams(base=0.02, increment_per_rerun=0.02)],
+            indirect=True,
+        )
+        def test_timing(rerun_adjusted_threshold: float) -> None: ...
+    """
+    param: RerunThresholdParams = request.param
+    execution_count: int = getattr(request.node, "execution_count", 0)
+    rerun_count = max(0, execution_count - 1)
+    return param.base + (rerun_count * param.increment_per_rerun)
+
+
 def pytest_addoption(parser):  # type: ignore[no-untyped-def]
     parser.addoption(
         "--aiohttp-fast",

requirements/test-common.in

@@ -8,6 +8,7 @@ proxy.py >= 2.4.4rc5
 pytest
 pytest-cov
 pytest-mock
+pytest-rerunfailures
 pytest-xdist
 pytest_codspeed
 python-on-whales

requirements/test-common.txt

@@ -72,13 +72,16 @@ pytest==9.0.2
     #   pytest-codspeed
     #   pytest-cov
     #   pytest-mock
+    #   pytest-rerunfailures
     #   pytest-xdist
 pytest-codspeed==4.2.0
     # via -r requirements/test-common.in
 pytest-cov==7.0.0
     # via -r requirements/test-common.in
 pytest-mock==3.15.1
     # via -r requirements/test-common.in
+pytest-rerunfailures==16.1
+    # via -r requirements/test-common.in
 pytest-xdist==3.8.0
     # via -r requirements/test-common.in
 python-dateutil==2.9.0.post0

requirements/test.txt

@@ -105,13 +105,16 @@ pytest==9.0.2
     #   pytest-codspeed
     #   pytest-cov
     #   pytest-mock
+    #   pytest-rerunfailures
     #   pytest-xdist
 pytest-codspeed==4.2.0
     # via -r requirements/test-common.in
 pytest-cov==7.0.0
     # via -r requirements/test-common.in
 pytest-mock==3.15.1
     # via -r requirements/test-common.in
+pytest-rerunfailures==16.1
+    # via -r requirements/test-common.in
 pytest-xdist==3.8.0
     # via -r requirements/test-common.in
 python-dateutil==2.9.0.post0

tests/conftest.py

@@ -49,6 +49,14 @@
     TRUSTME = False
 
 
+def pytest_configure(config: pytest.Config) -> None:
+    if os.name == "nt" and sys.version_info[:2] in ((3, 10), (3, 11)):
+        config.addinivalue_line(
+            "filterwarnings",
+            "ignore:Exception ignored in.*socket.*:pytest.PytestUnraisableExceptionWarning",
+        )
+
+
 try:
     if sys.platform == "win32":
         import winloop as uvloop
@@ -431,13 +439,14 @@ async def make_client_request(
     loop: asyncio.AbstractEventLoop,
 ) -> AsyncIterator[Callable[[str, URL, Unpack[ClientRequestArgs]], ClientRequest]]:
     """Fixture to help creating test ClientRequest objects with defaults."""
-    request = session = None
+    requests: list[ClientRequest] = []
+    sessions: list[ClientSession] = []
 
     def maker(
         method: str, url: URL, **kwargs: Unpack[ClientRequestArgs]
     ) -> ClientRequest:
-        nonlocal request, session
         session = ClientSession()
+        sessions.append(session)
         default_args: ClientRequestArgs = {
             "loop": loop,
             "params": {},
@@ -462,14 +471,15 @@ def maker(
             "server_hostname": None,
         }
         request = ClientRequest(method, url, **(default_args | kwargs))
+        requests.append(request)
         return request
 
     yield maker
 
-    if request is not None:
-        await request._close()
-        assert session is not None
-        await session.close()
+    await asyncio.gather(
+        *[request._close() for request in requests],
+        *[session.close() for session in sessions],
+    )
 
 
 @pytest.fixture

tests/test_client_middleware_digest_auth.py

@@ -24,7 +24,7 @@
 )
 from aiohttp.client_reqrep import ClientResponse
 from aiohttp.payload import BytesIOPayload
-from aiohttp.pytest_plugin import AiohttpServer
+from aiohttp.pytest_plugin import AiohttpServer, RerunThresholdParams
 from aiohttp.web import Application, Request, Response
 
 
@@ -1331,13 +1331,28 @@ async def handler(request: Request) -> Response:
     assert auth_algorithms[0] == "MD5-sess"  # Not "MD5-SESS"
 
 
-def test_regex_performance() -> None:
+_REGEX_TIME_THRESHOLD = RerunThresholdParams(base=0.02, increment_per_rerun=0.02)
+
+
+@pytest.mark.flaky(reruns=3)
+@pytest.mark.parametrize(
+    "rerun_adjusted_threshold", [_REGEX_TIME_THRESHOLD], indirect=True
+)
+def test_regex_performance(rerun_adjusted_threshold: float) -> None:
+    """Test that the regex pattern doesn't suffer from ReDoS issues.
+
+    Threshold starts at 20ms and increases on each rerun for CI variability.
+    """
     value = "0" * 54773 + "\\0=a"
+
     start = time.perf_counter()
     matches = _HEADER_PAIRS_PATTERN.findall(value)
-    end = time.perf_counter()
+    elapsed = time.perf_counter() - start
+
+    # Relaxed for CI/platform variability (e.g., macOS runners ~40-50ms observed)
+    assert elapsed < rerun_adjusted_threshold, (
+        f"Regex took {elapsed * 1000:.1f}ms, "
+        f"expected <{rerun_adjusted_threshold * 1000:.0f}ms - potential ReDoS issue"
+    )
 
-    # If this is taking more than 10ms, there's probably a performance/ReDoS issue.
-    assert (end - start) < 0.01
-    # This example probably shouldn't produce a match either.
     assert not matches

tests/test_cookie_helpers.py

@@ -19,6 +19,7 @@
     parse_set_cookie_headers,
     preserve_morsel_with_coded_value,
 )
+from aiohttp.pytest_plugin import RerunThresholdParams
 
 
 def test_known_attrs_is_superset_of_morsel_reserved() -> None:
@@ -637,15 +638,30 @@ def test_cookie_pattern_matches_partitioned_attribute(test_string: str) -> None:
     assert match.group("key").lower() == "partitioned"
 
 
-def test_cookie_pattern_performance() -> None:
+_COOKIE_PATTERN_TIME_THRESHOLD = RerunThresholdParams(
+    base=0.02, increment_per_rerun=0.02
+)
+
+
+@pytest.mark.flaky(reruns=3)
+@pytest.mark.parametrize(
+    "rerun_adjusted_threshold", [_COOKIE_PATTERN_TIME_THRESHOLD], indirect=True
+)
+def test_cookie_pattern_performance(rerun_adjusted_threshold: float) -> None:
+    """Test that the cookie pattern doesn't suffer from ReDoS issues.
+
+    This test is marked as flaky because timing can vary on loaded CI machines.
+    CI failure observed: ~20ms on Windows.
+    """
     value = "a" + "=" * 21651 + "\x00"
     start = time.perf_counter()
     match = helpers._COOKIE_PATTERN.match(value)
-    end = time.perf_counter()
+    elapsed = time.perf_counter() - start
 
-    # If this is taking more than 10ms, there's probably a performance/ReDoS issue.
-    assert (end - start) < 0.01
-    # This example shouldn't produce a match either.
+    assert elapsed < rerun_adjusted_threshold, (
+        f"Pattern took {elapsed * 1000:.1f}ms, "
+        f"expected <{rerun_adjusted_threshold * 1000:.0f}ms - potential ReDoS issue"
+    )
     assert match is None
 
 

tests/test_imports.py

@@ -5,6 +5,8 @@
 
 import pytest
 
+from aiohttp.pytest_plugin import RerunThresholdParams
+
 
 def test___all__(pytester: pytest.Pytester) -> None:
     """See https://github.com/aio-libs/aiohttp/issues/6197"""
@@ -28,56 +30,54 @@ def test_web___all__(pytester: pytest.Pytester) -> None:
     result.assert_outcomes(passed=0, errors=0)
 
 
-_IS_CI_ENV = os.getenv("CI") == "true"
-_XDIST_WORKER_COUNT = int(os.getenv("PYTEST_XDIST_WORKER_COUNT", 0))
-_IS_XDIST_RUN = _XDIST_WORKER_COUNT > 1
-
-_TARGET_TIMINGS_BY_PYTHON_VERSION = {
-    "3.12": (
-        # 3.12+ is expected to be a bit slower due to performance trade-offs,
-        # and even slower under pytest-xdist, especially in CI
-        _XDIST_WORKER_COUNT * 100 * (1 if _IS_CI_ENV else 1.53)
-        if _IS_XDIST_RUN
-        else 295
+_IMPORT_TIME_THRESHOLD_PY312 = 300
+_IMPORT_TIME_THRESHOLD_DEFAULT = 200
+_IMPORT_TIME_INCREMENT_PER_RERUN = 50
+_IMPORT_TIME_THRESHOLD = RerunThresholdParams(
+    base=(
+        _IMPORT_TIME_THRESHOLD_PY312
+        if sys.version_info >= (3, 12)
+        else _IMPORT_TIME_THRESHOLD_DEFAULT
     ),
-}
-_TARGET_TIMINGS_BY_PYTHON_VERSION["3.13"] = _TARGET_TIMINGS_BY_PYTHON_VERSION["3.12"]
+    increment_per_rerun=_IMPORT_TIME_INCREMENT_PER_RERUN,
+)
 
 
 @pytest.mark.internal
 @pytest.mark.dev_mode
+@pytest.mark.flaky(reruns=3)
+@pytest.mark.parametrize(
+    "rerun_adjusted_threshold", [_IMPORT_TIME_THRESHOLD], indirect=True
+)
 @pytest.mark.skipif(
     not sys.platform.startswith("linux") or platform.python_implementation() == "PyPy",
     reason="Timing is more reliable on Linux",
 )
-def test_import_time(pytester: pytest.Pytester) -> None:
+def test_import_time(
+    rerun_adjusted_threshold: float, pytester: pytest.Pytester
+) -> None:
     """Check that importing aiohttp doesn't take too long.
 
     Obviously, the time may vary on different machines and may need to be adjusted
     from time to time, but this should provide an early warning if something is
     added that significantly increases import time.
+
+    Threshold increases by _IMPORT_TIME_INCREMENT_PER_RERUN ms on each rerun
+    to account for CI variability.
     """
     root = Path(__file__).parent.parent
     old_path = os.environ.get("PYTHONPATH")
     os.environ["PYTHONPATH"] = os.pathsep.join([str(root)] + sys.path)
 
-    best_time_ms = 1000
     cmd = "import timeit; print(int(timeit.timeit('import aiohttp', number=1) * 1000))"
     try:
-        for _ in range(3):
-            r = pytester.run(sys.executable, "-We", "-c", cmd)
-
-            assert not r.stderr.str()
-            runtime_ms = int(r.stdout.str())
-            if runtime_ms < best_time_ms:
-                best_time_ms = runtime_ms
+        r = pytester.run(sys.executable, "-We", "-c", cmd)
+        assert not r.stderr.str(), r.stderr.str()
+        runtime_ms = int(r.stdout.str())
     finally:
         if old_path is None:
             os.environ.pop("PYTHONPATH")
         else:
             os.environ["PYTHONPATH"] = old_path
 
-    expected_time = _TARGET_TIMINGS_BY_PYTHON_VERSION.get(
-        f"{sys.version_info.major}.{sys.version_info.minor}", 200
-    )
-    assert best_time_ms < expected_time
+    assert runtime_ms < rerun_adjusted_threshold

tests/test_web_request.py

@@ -17,7 +17,7 @@
 from aiohttp import ETag, HttpVersion, web
 from aiohttp.base_protocol import BaseProtocol
 from aiohttp.http_parser import RawRequestMessage
-from aiohttp.pytest_plugin import AiohttpClient
+from aiohttp.pytest_plugin import AiohttpClient, RerunThresholdParams
 from aiohttp.streams import StreamReader
 from aiohttp.test_utils import make_mocked_request
 from aiohttp.web_request import _FORWARDED_PAIR_RE
@@ -600,15 +600,23 @@ def test_single_forwarded_header() -> None:
     assert req.forwarded[0]["proto"] == "identifier"
 
 
-def test_forwarded_re_performance() -> None:
+_FORWARDED_RE_TIME_THRESHOLD = RerunThresholdParams(base=0.02, increment_per_rerun=0.02)
+
+
+@pytest.mark.flaky(reruns=3)
+@pytest.mark.parametrize(
+    "rerun_adjusted_threshold", [_FORWARDED_RE_TIME_THRESHOLD], indirect=True
+)
+def test_forwarded_re_performance(rerun_adjusted_threshold: float) -> None:
     value = "{" + "f" * 54773 + "z\x00a=v"
     start = time.perf_counter()
     match = _FORWARDED_PAIR_RE.match(value)
-    end = time.perf_counter()
+    elapsed = time.perf_counter() - start
 
-    # If this is taking more than 10ms, there's probably a performance/ReDoS issue.
-    assert (end - start) < 0.01
-    # This example shouldn't produce a match either.
+    assert elapsed < rerun_adjusted_threshold, (
+        f"Regex took {elapsed * 1000:.1f}ms, "
+        f"expected <{rerun_adjusted_threshold * 1000:.0f}ms - potential ReDoS issue"
+    )
     assert match is None