From 32a3659aed0d17ea5cd82cc33a8aa3dae97fd555 Mon Sep 17 00:00:00 2001 From: Colin-b Date: Thu, 20 Oct 2022 23:21:08 +0200 Subject: [PATCH 1/2] Handle async callbacks --- CHANGELOG.md | 2 + README.md | 34 +++++- pytest_httpx/_httpx_mock.py | 61 +++++++++-- tests/test_httpx_async.py | 200 ++++++++++++++++++++++++++++++++++-- 4 files changed, 277 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5da1660..2c991d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Fixed +- `httpx_mock.add_callback` now handles async callbacks. ## [0.21.0] - 2022-05-24 ### Changed diff --git a/README.md b/README.md index da48a69..6921a5d 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Build status Coverage Code style: black -Number of tests +Number of tests Number of downloads

@@ -444,6 +444,38 @@ def test_dynamic_response(httpx_mock: HTTPXMock): ``` +Alternatively, callbacks can also be asynchronous. + +As in the following sample simulating network latency on some responses only. + +```python +import asyncio +import httpx +import pytest +from pytest_httpx import HTTPXMock + + +@pytest.mark.asyncio +async def test_dynamic_async_response(httpx_mock: HTTPXMock): + async def simulate_network_latency(request: httpx.Request): + await asyncio.sleep(1) + return httpx.Response( + status_code=200, json={"url": str(request.url)}, + ) + + httpx_mock.add_callback(simulate_network_latency) + httpx_mock.add_response() + + async with httpx.AsyncClient() as client: + responses = await asyncio.gather( + # Response will be received after one second + client.get("https://test_url"), + # Response will instantly be received (1 second before the first request) + client.get("https://test_url") + ) + +``` + ### Raising exceptions You can simulate HTTPX exception throwing by raising an exception in your callback or use `httpx_mock.add_exception` with the exception instance. diff --git a/pytest_httpx/_httpx_mock.py b/pytest_httpx/_httpx_mock.py index 66f026b..7f3c5c7 100644 --- a/pytest_httpx/_httpx_mock.py +++ b/pytest_httpx/_httpx_mock.py @@ -1,5 +1,6 @@ +import inspect import re -from typing import List, Union, Optional, Callable, Tuple, Pattern, Any, Dict +from typing import List, Union, Optional, Callable, Tuple, Pattern, Any, Dict, Awaitable from urllib.parse import parse_qs import httpx @@ -89,7 +90,12 @@ def __init__(self) -> None: self._callbacks: List[ Tuple[ _RequestMatcher, - Callable[[httpx.Request], Optional[httpx.Response]], + Callable[ + [httpx.Request], + Union[ + Optional[httpx.Response], Awaitable[Optional[httpx.Response]] + ], + ], ] ] = [] @@ -135,7 +141,12 @@ def add_response( self.add_callback(lambda request: response, **matchers) def add_callback( - self, callback: Callable[[httpx.Request], Optional[httpx.Response]], **matchers + self, + callback: Callable[ + [httpx.Request], + Union[Optional[httpx.Response], Awaitable[Optional[httpx.Response]]], + ], + **matchers, ) -> None: """ Mock the action that will take place if a request match. @@ -180,12 +191,26 @@ def _handle_request( response = callback(request) if response: - # Allow to read the response on client side - response.is_stream_consumed = False - response.is_closed = False - if hasattr(response, "_content"): - del response._content - return response + return _unread(response) + + raise httpx.TimeoutException( + self._explain_that_no_response_was_found(request), request=request + ) + + async def _handle_async_request( + self, + request: httpx.Request, + ) -> httpx.Response: + self._requests.append(request) + + callback = self._get_callback(request) + if callback: + response = callback(request) + + if response: + if inspect.isawaitable(response): + response = await response + return _unread(response) raise httpx.TimeoutException( self._explain_that_no_response_was_found(request), request=request @@ -221,7 +246,12 @@ def _explain_that_no_response_was_found(self, request: httpx.Request) -> str: def _get_callback( self, request: httpx.Request - ) -> Optional[Callable[[httpx.Request], Optional[httpx.Response]]]: + ) -> Optional[ + Callable[ + [httpx.Request], + Union[Optional[httpx.Response], Awaitable[Optional[httpx.Response]]], + ] + ]: callbacks = [ (matcher, callback) for matcher, callback in self._callbacks @@ -304,4 +334,13 @@ def __init__(self, mock: HTTPXMock): self.mock = mock async def handle_async_request(self, *args, **kwargs) -> httpx.Response: - return self.mock._handle_request(*args, **kwargs) + return await self.mock._handle_async_request(*args, **kwargs) + + +def _unread(response: httpx.Response) -> httpx.Response: + # Allow to read the response on client side + response.is_stream_consumed = False + response.is_closed = False + if hasattr(response, "_content"): + del response._content + return response diff --git a/tests/test_httpx_async.py b/tests/test_httpx_async.py index ba02a74..ef5d0cb 100644 --- a/tests/test_httpx_async.py +++ b/tests/test_httpx_async.py @@ -1,4 +1,7 @@ +import asyncio +import math import re +import time import httpx import pytest @@ -166,7 +169,7 @@ async def test_stream_response_streaming(httpx_mock: HTTPXMock) -> None: # Assert that stream still behaves the proper way (can only be consumed once per request) with pytest.raises(httpx.StreamConsumed): async for part in response.aiter_raw(): - pass + pass # pragma: no cover async with client.stream(method="GET", url="https://test_url") as response: assert [part async for part in response.aiter_raw()] == [ @@ -176,7 +179,7 @@ async def test_stream_response_streaming(httpx_mock: HTTPXMock) -> None: # Assert that stream still behaves the proper way (can only be consumed once per request) with pytest.raises(httpx.StreamConsumed): async for part in response.aiter_raw(): - pass + pass # pragma: no cover @pytest.mark.asyncio @@ -194,7 +197,7 @@ async def test_content_response_streaming(httpx_mock: HTTPXMock) -> None: # Assert that stream still behaves the proper way (can only be consumed once per request) with pytest.raises(httpx.StreamConsumed): async for part in response.aiter_raw(): - pass + pass # pragma: no cover async with client.stream(method="GET", url="https://test_url") as response: assert [part async for part in response.aiter_raw()] == [ @@ -203,7 +206,7 @@ async def test_content_response_streaming(httpx_mock: HTTPXMock) -> None: # Assert that stream still behaves the proper way (can only be consumed once per request) with pytest.raises(httpx.StreamConsumed): async for part in response.aiter_raw(): - pass + pass # pragma: no cover @pytest.mark.asyncio @@ -221,7 +224,7 @@ async def test_text_response_streaming(httpx_mock: HTTPXMock) -> None: # Assert that stream still behaves the proper way (can only be consumed once per request) with pytest.raises(httpx.StreamConsumed): async for part in response.aiter_raw(): - pass + pass # pragma: no cover async with client.stream(method="GET", url="https://test_url") as response: assert [part async for part in response.aiter_raw()] == [ @@ -230,7 +233,7 @@ async def test_text_response_streaming(httpx_mock: HTTPXMock) -> None: # Assert that stream still behaves the proper way (can only be consumed once per request) with pytest.raises(httpx.StreamConsumed): async for part in response.aiter_raw(): - pass + pass # pragma: no cover @pytest.mark.asyncio @@ -243,14 +246,14 @@ async def test_default_response_streaming(httpx_mock: HTTPXMock) -> None: # Assert that stream still behaves the proper way (can only be consumed once per request) with pytest.raises(httpx.StreamConsumed): async for part in response.aiter_raw(): - pass + pass # pragma: no cover async with client.stream(method="GET", url="https://test_url") as response: assert [part async for part in response.aiter_raw()] == [] # Assert that stream still behaves the proper way (can only be consumed once per request) with pytest.raises(httpx.StreamConsumed): async for part in response.aiter_raw(): - pass + pass # pragma: no cover @pytest.mark.asyncio @@ -487,6 +490,66 @@ def custom_response2(request: httpx.Request) -> httpx.Response: assert response.http_version == "HTTP/1.1" +@pytest.mark.asyncio +async def test_async_callback_with_await_statement(httpx_mock: HTTPXMock) -> None: + async def simulate_network_latency(request: httpx.Request): + await asyncio.sleep(1) + return httpx.Response( + status_code=200, + json={"url": str(request.url), "time": time.time()}, + ) + + def instant_response(request: httpx.Request) -> httpx.Response: + return httpx.Response( + status_code=200, json={"url": str(request.url), "time": time.time()} + ) + + httpx_mock.add_callback(simulate_network_latency) + httpx_mock.add_callback(instant_response) + httpx_mock.add_response(json={"url": "not a callback"}) + + async with httpx.AsyncClient() as client: + responses = await asyncio.gather( + client.get("https://slow"), + client.get("https://fast_with_callback"), + client.get("https://fast_with_response"), + ) + slow_response = responses[0].json() + assert slow_response["url"] == "https://slow" + + fast_callback_response = responses[1].json() + assert fast_callback_response["url"] == "https://fast_with_callback" + + fast_response = responses[2].json() + assert fast_response["url"] == "not a callback" + + # Ensure slow request was properly awaited (did not block subsequent async queries) + assert math.isclose(slow_response["time"], fast_callback_response["time"] + 1) + + +@pytest.mark.asyncio +async def test_async_callback_with_pattern_in_url(httpx_mock: HTTPXMock) -> None: + async def custom_response(request: httpx.Request) -> httpx.Response: + return httpx.Response(status_code=200, json={"url": str(request.url)}) + + async def custom_response2(request: httpx.Request) -> httpx.Response: + return httpx.Response( + status_code=200, + extensions={"http_version": b"HTTP/2.0"}, + json={"url": str(request.url)}, + ) + + httpx_mock.add_callback(custom_response, url=re.compile(".*test.*")) + httpx_mock.add_callback(custom_response2, url="https://unmatched") + + async with httpx.AsyncClient() as client: + response = await client.get("https://unmatched") + assert response.http_version == "HTTP/2.0" + + response = await client.get("https://test_url") + assert response.http_version == "HTTP/1.1" + + @pytest.mark.asyncio async def test_with_many_responses_urls_instances(httpx_mock: HTTPXMock) -> None: httpx_mock.add_response( @@ -762,6 +825,22 @@ def raise_timeout(request: httpx.Request) -> httpx.Response: assert str(exception_info.value) == "Unable to read within 5.0" +@pytest.mark.asyncio +async def test_async_callback_raising_exception(httpx_mock: HTTPXMock) -> None: + async def raise_timeout(request: httpx.Request) -> httpx.Response: + raise httpx.ReadTimeout( + f"Unable to read within {request.extensions['timeout']['read']}", + request=request, + ) + + httpx_mock.add_callback(raise_timeout, url="https://test_url") + + async with httpx.AsyncClient() as client: + with pytest.raises(httpx.ReadTimeout) as exception_info: + await client.get("https://test_url") + assert str(exception_info.value) == "Unable to read within 5.0" + + @pytest.mark.asyncio async def test_request_exception_raising(httpx_mock: HTTPXMock) -> None: httpx_mock.add_exception( @@ -800,6 +879,19 @@ def custom_response(request: httpx.Request) -> httpx.Response: assert response.headers["content-type"] == "application/json" +@pytest.mark.asyncio +async def test_async_callback_returning_response(httpx_mock: HTTPXMock) -> None: + async def custom_response(request: httpx.Request) -> httpx.Response: + return httpx.Response(status_code=200, json={"url": str(request.url)}) + + httpx_mock.add_callback(custom_response, url="https://test_url") + + async with httpx.AsyncClient() as client: + response = await client.get("https://test_url") + assert response.json() == {"url": "https://test_url"} + assert response.headers["content-type"] == "application/json" + + @pytest.mark.asyncio async def test_callback_executed_twice(httpx_mock: HTTPXMock) -> None: def custom_response(request: httpx.Request) -> httpx.Response: @@ -817,6 +909,23 @@ def custom_response(request: httpx.Request) -> httpx.Response: assert response.headers["content-type"] == "application/json" +@pytest.mark.asyncio +async def test_async_callback_executed_twice(httpx_mock: HTTPXMock) -> None: + async def custom_response(request: httpx.Request) -> httpx.Response: + return httpx.Response(status_code=200, json=["content"]) + + httpx_mock.add_callback(custom_response) + + async with httpx.AsyncClient() as client: + response = await client.get("https://test_url") + assert response.json() == ["content"] + assert response.headers["content-type"] == "application/json" + + response = await client.post("https://test_url") + assert response.json() == ["content"] + assert response.headers["content-type"] == "application/json" + + @pytest.mark.asyncio async def test_callback_registered_after_response(httpx_mock: HTTPXMock) -> None: def custom_response(request: httpx.Request) -> httpx.Response: @@ -840,6 +949,29 @@ def custom_response(request: httpx.Request) -> httpx.Response: assert response.headers["content-type"] == "application/json" +@pytest.mark.asyncio +async def test_async_callback_registered_after_response(httpx_mock: HTTPXMock) -> None: + async def custom_response(request: httpx.Request) -> httpx.Response: + return httpx.Response(status_code=200, json=["content2"]) + + httpx_mock.add_response(json=["content1"]) + httpx_mock.add_callback(custom_response) + + async with httpx.AsyncClient() as client: + response = await client.get("https://test_url") + assert response.json() == ["content1"] + assert response.headers["content-type"] == "application/json" + + response = await client.post("https://test_url") + assert response.json() == ["content2"] + assert response.headers["content-type"] == "application/json" + + # Assert that the last registered callback is sent again even if there is a response + response = await client.post("https://test_url") + assert response.json() == ["content2"] + assert response.headers["content-type"] == "application/json" + + @pytest.mark.asyncio async def test_response_registered_after_callback(httpx_mock: HTTPXMock) -> None: def custom_response(request: httpx.Request) -> httpx.Response: @@ -863,6 +995,29 @@ def custom_response(request: httpx.Request) -> httpx.Response: assert response.headers["content-type"] == "application/json" +@pytest.mark.asyncio +async def test_response_registered_after_async_callback(httpx_mock: HTTPXMock) -> None: + async def custom_response(request: httpx.Request) -> httpx.Response: + return httpx.Response(status_code=200, json=["content1"]) + + httpx_mock.add_callback(custom_response) + httpx_mock.add_response(json=["content2"]) + + async with httpx.AsyncClient() as client: + response = await client.get("https://test_url") + assert response.json() == ["content1"] + assert response.headers["content-type"] == "application/json" + + response = await client.post("https://test_url") + assert response.json() == ["content2"] + assert response.headers["content-type"] == "application/json" + + # Assert that the last registered response is sent again even if there is a callback + response = await client.post("https://test_url") + assert response.json() == ["content2"] + assert response.headers["content-type"] == "application/json" + + @pytest.mark.asyncio async def test_callback_matching_method(httpx_mock: HTTPXMock) -> None: def custom_response(request: httpx.Request) -> httpx.Response: @@ -880,6 +1035,23 @@ def custom_response(request: httpx.Request) -> httpx.Response: assert response.headers["content-type"] == "application/json" +@pytest.mark.asyncio +async def test_async_callback_matching_method(httpx_mock: HTTPXMock) -> None: + async def custom_response(request: httpx.Request) -> httpx.Response: + return httpx.Response(status_code=200, json=["content"]) + + httpx_mock.add_callback(custom_response, method="GET") + + async with httpx.AsyncClient() as client: + response = await client.get("https://test_url") + assert response.json() == ["content"] + assert response.headers["content-type"] == "application/json" + + response = await client.get("https://test_url2") + assert response.json() == ["content"] + assert response.headers["content-type"] == "application/json" + + def test_request_retrieval_with_more_than_one(testdir: Testdir) -> None: """ Single request cannot be returned if there is more than one matching. @@ -1485,3 +1657,15 @@ async def test_elapsed_when_add_callback(httpx_mock: HTTPXMock) -> None: async with httpx.AsyncClient() as client: response = await client.get("https://test_url") assert response.elapsed is not None + + +@pytest.mark.asyncio +async def test_elapsed_when_add_async_callback(httpx_mock: HTTPXMock) -> None: + async def custom_response(request: httpx.Request) -> httpx.Response: + return httpx.Response(status_code=200, json={"foo": "bar"}) + + httpx_mock.add_callback(custom_response) + + async with httpx.AsyncClient() as client: + response = await client.get("https://test_url") + assert response.elapsed is not None From 9fe393516cd7d2d1fed2b12b68985634d460b9ec Mon Sep 17 00:00:00 2001 From: Colin-b Date: Thu, 20 Oct 2022 23:23:30 +0200 Subject: [PATCH 2/2] Release version 0.21.1 today --- CHANGELOG.md | 5 ++++- pytest_httpx/version.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c991d9..1cd98be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## [0.21.1] - 2022-10-20 ### Fixed - `httpx_mock.add_callback` now handles async callbacks. @@ -225,7 +227,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - First release, should be considered as unstable for now as design might change. -[Unreleased]: https://github.com/Colin-b/pytest_httpx/compare/v0.21.0...HEAD +[Unreleased]: https://github.com/Colin-b/pytest_httpx/compare/v0.21.1...HEAD +[0.21.1]: https://github.com/Colin-b/pytest_httpx/compare/v0.21.0...v0.21.1 [0.21.0]: https://github.com/Colin-b/pytest_httpx/compare/v0.20.0...v0.21.0 [0.20.0]: https://github.com/Colin-b/pytest_httpx/compare/v0.19.0...v0.20.0 [0.19.0]: https://github.com/Colin-b/pytest_httpx/compare/v0.18.0...v0.19.0 diff --git a/pytest_httpx/version.py b/pytest_httpx/version.py index 3ee084a..5e3dfb3 100644 --- a/pytest_httpx/version.py +++ b/pytest_httpx/version.py @@ -3,4 +3,4 @@ # Major should be incremented in case there is a breaking change. (eg: 2.5.8 -> 3.0.0) # Minor should be incremented in case there is an enhancement. (eg: 2.5.8 -> 2.6.0) # Patch should be incremented in case there is a bug fix. (eg: 2.5.8 -> 2.5.9) -__version__ = "0.21.0" +__version__ = "0.21.1"