From 72bf1a234c2fb028a31311c4ee251a72d4192db9 Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Fri, 11 Aug 2023 01:08:52 +0200 Subject: [PATCH 1/6] Update pyproject.toml Update fake-useragent to latest version (1.2.1). Disclaimer: I'm the maintainer. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d65c6413..dee545cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ classifiers = [ ] dependencies = [ "click~=8.1.3", - "fake-useragent~=1.1.3", + "fake-useragent~=1.2.1", "pandas>=2,<3", "pydantic~=1.10.13", "python-dotenv~=1.0.0", From c259818322ce1817d7c6ba56aadb093d64177eaf Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Fri, 11 Aug 2023 01:13:19 +0200 Subject: [PATCH 2/6] Update base_provider.py Don't need to use external parameter anymore. Also just use random on the list of chrome browsers UA strings. --- camply/providers/base_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/camply/providers/base_provider.py b/camply/providers/base_provider.py index 4cf7749c..613234c2 100644 --- a/camply/providers/base_provider.py +++ b/camply/providers/base_provider.py @@ -82,7 +82,7 @@ def __init__(self): """ Initialize with a session """ - _user_agent = UserAgent(use_external_data=False, browsers=["chrome"]).chrome + _user_agent = UserAgent(browsers=["chrome"]).random self.session = requests.Session() self.headers = {"User-Agent": _user_agent} self.session.headers = self.headers From 42ac97fe524a0fe83eda058b914b9674903bad25 Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Fri, 11 Aug 2023 01:13:55 +0200 Subject: [PATCH 3/6] Update going_to_camp_provider.py Don't need to use external parameter anymore. Also just use random on the list of chrome browsers UA strings. --- camply/providers/going_to_camp/going_to_camp_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/camply/providers/going_to_camp/going_to_camp_provider.py b/camply/providers/going_to_camp/going_to_camp_provider.py index b908f51c..5a93e03c 100644 --- a/camply/providers/going_to_camp/going_to_camp_provider.py +++ b/camply/providers/going_to_camp/going_to_camp_provider.py @@ -336,7 +336,7 @@ def _api_request( if endpoint: url = endpoint.format(hostname) user_agent = { - "User-Agent": UserAgent(use_external_data=False, browsers=["chrome"]).chrome + "User-Agent": UserAgent(browsers=["chrome"]).random } response = requests.get(url=url, headers=user_agent, params=params, timeout=30) if response.ok is False: From 34be8dbc22104dcd4bc6d3cad9c3d58986872449 Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Fri, 11 Aug 2023 01:14:24 +0200 Subject: [PATCH 4/6] Update yellowstone_lodging.py Don't need to use external parameter anymore. Also just use random on the list of chrome browsers UA strings. --- camply/providers/xanterra/yellowstone_lodging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/camply/providers/xanterra/yellowstone_lodging.py b/camply/providers/xanterra/yellowstone_lodging.py index 46bda0e0..f7ea7c24 100755 --- a/camply/providers/xanterra/yellowstone_lodging.py +++ b/camply/providers/xanterra/yellowstone_lodging.py @@ -89,7 +89,7 @@ def _try_retry_get_data(endpoint: str, params: Optional[dict] = None) -> dict: """ yellowstone_headers = {} user_agent = { - "User-Agent": UserAgent(use_external_data=False, browsers=["chrome"]).chrome + "User-Agent": UserAgent(browsers=["chrome"]).random } yellowstone_headers.update(user_agent) yellowstone_headers.update(STANDARD_HEADERS) From d4a35baf4eaafdfd5138820ee8db9e502b35d487 Mon Sep 17 00:00:00 2001 From: Melroy van den Berg Date: Fri, 11 Aug 2023 01:14:59 +0200 Subject: [PATCH 5/6] Update recdotgov_provider.py Don't need to use external parameter anymore. Also just use random on the list of chrome browsers UA strings. --- camply/providers/recreation_dot_gov/recdotgov_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/camply/providers/recreation_dot_gov/recdotgov_provider.py b/camply/providers/recreation_dot_gov/recdotgov_provider.py index 942988bc..4ffd4ca7 100644 --- a/camply/providers/recreation_dot_gov/recdotgov_provider.py +++ b/camply/providers/recreation_dot_gov/recdotgov_provider.py @@ -58,7 +58,7 @@ def __init__(self, api_key: Optional[str] = None): "accept": "application/json", "apikey": _api_key, } - _user_agent = UserAgent(use_external_data=False, browsers=["chrome"]).chrome + _user_agent = UserAgent(browsers=["chrome"]).random self._user_agent = {"User-Agent": _user_agent} @property From c1094abc2120ed440b861043396a782980948248 Mon Sep 17 00:00:00 2001 From: Justin Flannery Date: Tue, 6 Feb 2024 00:24:44 -0700 Subject: [PATCH 6/6] chore: lock files and more user-agents --- camply/providers/going_to_camp/going_to_camp_provider.py | 4 +--- camply/providers/recreation_dot_gov/recdotgov_provider.py | 4 +--- camply/providers/usedirect/usedirect.py | 3 +++ camply/providers/xanterra/yellowstone_lodging.py | 4 +--- pyproject.toml | 2 +- requirements.txt | 4 ++-- requirements/requirements-all.py3.10.txt | 4 ++-- requirements/requirements-all.py3.11.txt | 4 ++-- requirements/requirements-all.py3.12.txt | 4 ++-- requirements/requirements-all.py3.8.txt | 4 ++-- requirements/requirements-all.py3.9.txt | 8 +++----- requirements/requirements-docs.txt | 6 +++--- requirements/requirements-test.txt | 6 +++--- 13 files changed, 26 insertions(+), 31 deletions(-) diff --git a/camply/providers/going_to_camp/going_to_camp_provider.py b/camply/providers/going_to_camp/going_to_camp_provider.py index 5a93e03c..c395595e 100644 --- a/camply/providers/going_to_camp/going_to_camp_provider.py +++ b/camply/providers/going_to_camp/going_to_camp_provider.py @@ -335,9 +335,7 @@ def _api_request( url = None if endpoint: url = endpoint.format(hostname) - user_agent = { - "User-Agent": UserAgent(browsers=["chrome"]).random - } + user_agent = {"User-Agent": UserAgent(browsers=["chrome"]).random} response = requests.get(url=url, headers=user_agent, params=params, timeout=30) if response.ok is False: error_message = f"Receiving bad data from GoingToCamp API: status_code: {response.status_code}: {response.text}" diff --git a/camply/providers/recreation_dot_gov/recdotgov_provider.py b/camply/providers/recreation_dot_gov/recdotgov_provider.py index 4ffd4ca7..0fbee5df 100644 --- a/camply/providers/recreation_dot_gov/recdotgov_provider.py +++ b/camply/providers/recreation_dot_gov/recdotgov_provider.py @@ -579,9 +579,7 @@ def make_recdotgov_request( requests.Response """ # BUILD THE HEADERS EXPECTED FROM THE API - user_agent = { - "User-Agent": UserAgent(use_external_data=False, browsers=["chrome"]).chrome - } + user_agent = {"User-Agent": UserAgent(browsers=["chrome"]).random} headers = STANDARD_HEADERS.copy() headers.update(user_agent) headers.update(RecreationBookingConfig.API_REFERRERS) diff --git a/camply/providers/usedirect/usedirect.py b/camply/providers/usedirect/usedirect.py index 212718e4..29008a46 100644 --- a/camply/providers/usedirect/usedirect.py +++ b/camply/providers/usedirect/usedirect.py @@ -11,6 +11,7 @@ from typing import Any, Dict, List, Optional, Union import ratelimit +from fake_useragent import UserAgent from pydantic import ValidationError from camply.config import FileConfig @@ -322,6 +323,8 @@ def get_campsites_response( key: value for key, value in data.items() if value not in [None, [], ""] } url = f"{self.base_url}/{self.rdr_path}/{UseDirectConfig.AVAILABILITY_ENDPOINT}" + random_ua = UserAgent(browsers=["chrome"]).random + self.json_headers["User-Agent"] = random_ua response = self.make_http_request_retry( url=url, method="POST", diff --git a/camply/providers/xanterra/yellowstone_lodging.py b/camply/providers/xanterra/yellowstone_lodging.py index f7ea7c24..b09022d1 100755 --- a/camply/providers/xanterra/yellowstone_lodging.py +++ b/camply/providers/xanterra/yellowstone_lodging.py @@ -88,9 +88,7 @@ def _try_retry_get_data(endpoint: str, params: Optional[dict] = None) -> dict: dict """ yellowstone_headers = {} - user_agent = { - "User-Agent": UserAgent(browsers=["chrome"]).random - } + user_agent = {"User-Agent": UserAgent(browsers=["chrome"]).random} yellowstone_headers.update(user_agent) yellowstone_headers.update(STANDARD_HEADERS) yellowstone_headers.update(YellowstoneConfig.API_REFERRERS) diff --git a/pyproject.toml b/pyproject.toml index dee545cc..fa9a2755 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ classifiers = [ ] dependencies = [ "click~=8.1.3", - "fake-useragent~=1.2.1", + "fake-useragent~=1.4.0", "pandas>=2,<3", "pydantic~=1.10.13", "python-dotenv~=1.0.0", diff --git a/requirements.txt b/requirements.txt index 310218fc..b69a7818 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by hatch-pip-compile with Python 3.11 # # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -32,7 +32,7 @@ click==8.1.7 # apprise # rich-click # trogon -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via hatch.envs.default idna==3.6 # via requests diff --git a/requirements/requirements-all.py3.10.txt b/requirements/requirements-all.py3.10.txt index ca0b5426..0170fcee 100644 --- a/requirements/requirements-all.py3.10.txt +++ b/requirements/requirements-all.py3.10.txt @@ -8,7 +8,7 @@ # - pytest-xdist~=3.2.1 # - freezegun~=1.2.1 # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -46,7 +46,7 @@ exceptiongroup==1.2.0 # via pytest execnet==2.0.2 # via pytest-xdist -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via hatch.envs.all.py3.10 freezegun==1.2.2 # via hatch.envs.all.py3.10 diff --git a/requirements/requirements-all.py3.11.txt b/requirements/requirements-all.py3.11.txt index e877dde5..70706116 100644 --- a/requirements/requirements-all.py3.11.txt +++ b/requirements/requirements-all.py3.11.txt @@ -8,7 +8,7 @@ # - pytest-xdist~=3.2.1 # - freezegun~=1.2.1 # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -44,7 +44,7 @@ coverage==7.4.0 # pytest-cov execnet==2.0.2 # via pytest-xdist -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via hatch.envs.all.py3.11 freezegun==1.2.2 # via hatch.envs.all.py3.11 diff --git a/requirements/requirements-all.py3.12.txt b/requirements/requirements-all.py3.12.txt index b0445e01..50018f65 100644 --- a/requirements/requirements-all.py3.12.txt +++ b/requirements/requirements-all.py3.12.txt @@ -8,7 +8,7 @@ # - pytest-xdist~=3.2.1 # - freezegun~=1.2.1 # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -44,7 +44,7 @@ coverage==7.4.0 # pytest-cov execnet==2.0.2 # via pytest-xdist -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via hatch.envs.all.py3.12 freezegun==1.2.2 # via hatch.envs.all.py3.12 diff --git a/requirements/requirements-all.py3.8.txt b/requirements/requirements-all.py3.8.txt index c4d6ea8a..fd2c7d09 100644 --- a/requirements/requirements-all.py3.8.txt +++ b/requirements/requirements-all.py3.8.txt @@ -8,7 +8,7 @@ # - pytest-xdist~=3.2.1 # - freezegun~=1.2.1 # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -46,7 +46,7 @@ exceptiongroup==1.2.0 # via pytest execnet==2.0.2 # via pytest-xdist -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via hatch.envs.all.py3.8 freezegun==1.2.2 # via hatch.envs.all.py3.8 diff --git a/requirements/requirements-all.py3.9.txt b/requirements/requirements-all.py3.9.txt index 45661f98..931a1097 100644 --- a/requirements/requirements-all.py3.9.txt +++ b/requirements/requirements-all.py3.9.txt @@ -8,7 +8,7 @@ # - pytest-xdist~=3.2.1 # - freezegun~=1.2.1 # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -39,14 +39,12 @@ click==8.1.7 # rich-click # trogon coverage==7.4.0 - # via - # coverage - # pytest-cov + # via pytest-cov exceptiongroup==1.2.0 # via pytest execnet==2.0.2 # via pytest-xdist -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via hatch.envs.all.py3.9 freezegun==1.2.2 # via hatch.envs.all.py3.9 diff --git a/requirements/requirements-docs.txt b/requirements/requirements-docs.txt index 408f0b1a..9b429926 100644 --- a/requirements/requirements-docs.txt +++ b/requirements/requirements-docs.txt @@ -1,7 +1,7 @@ # # This file is autogenerated by hatch-pip-compile with Python 3.11 # -# [constraints] requirements.txt (SHA256: 7d2ce7d5ed6d5060713188a3a42ce8fdfb3aa0da2ebcb6a000c552ab9fb0cf33) +# [constraints] requirements.txt (SHA256: 2a92e1c6e109ef4c9ce314b9a60b303169d2ddf6cb3b4c4c9df4288cec4ac2ef) # # - markdown-callouts # - markdown-exec @@ -16,7 +16,7 @@ # - mkdocstrings-python # - pymdown-extensions # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -51,7 +51,7 @@ colorama==0.4.6 # via # griffe # mkdocs-material -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via # -c requirements.txt # hatch.envs.docs diff --git a/requirements/requirements-test.txt b/requirements/requirements-test.txt index 6e5788a8..bff0d2c0 100644 --- a/requirements/requirements-test.txt +++ b/requirements/requirements-test.txt @@ -1,7 +1,7 @@ # # This file is autogenerated by hatch-pip-compile with Python 3.11 # -# [constraints] requirements.txt (SHA256: 7d2ce7d5ed6d5060713188a3a42ce8fdfb3aa0da2ebcb6a000c552ab9fb0cf33) +# [constraints] requirements.txt (SHA256: 2a92e1c6e109ef4c9ce314b9a60b303169d2ddf6cb3b4c4c9df4288cec4ac2ef) # # - pytest~=7.3.1 # - pytest-cov~=4.0.0 @@ -10,7 +10,7 @@ # - pytest-xdist~=3.2.1 # - freezegun~=1.2.1 # - click~=8.1.3 -# - fake-useragent~=1.1.3 +# - fake-useragent~=1.4.0 # - pandas<3,>=2 # - pydantic~=1.10.13 # - python-dotenv~=1.0.0 @@ -52,7 +52,7 @@ coverage==7.2.7 # pytest-cov execnet==1.9.0 # via pytest-xdist -fake-useragent==1.1.3 +fake-useragent==1.4.0 # via # -c requirements.txt # hatch.envs.test