Remove unused setting and improve settings.md docs

q-m · Jan 10, 2025 · f86183d · f86183d
1 parent 1a7ce0e
commit f86183d
Show file tree

Hide file tree

Showing 4 changed files with 1 addition and 12 deletions.
diff --git a/docs/settings.md b/docs/settings.md
@@ -57,12 +57,4 @@ This setting defines the location of the WACZ file that should be used as a sour
 SW_WACZ_CRAWL = True
 ```
 
-Setting to ignore original `start_requests`, just yield all responses found.
-
-### `SW_WACZ_TIMEOUT`
-
-```python
-SW_WACZ_TIMEOUT = 60
-```
-
-Transport parameter for retrieving the `SW_WACZ_SOURCE_URI` from the defined location.
+Setting to control the scraping behavior. If set to `False`, the scraper will bypass the WACZ middleware/downloadermiddleware during the crawling process.
diff --git a/scrapy_webarchive/spidermiddlewares.py b/scrapy_webarchive/spidermiddlewares.py
@@ -30,7 +30,6 @@ def __init__(self, settings: Settings, stats: StatsCollector) -> None:
 
         self.wacz_uris = re.split(r"\s*,\s*", wacz_uri)
         self.crawl = settings.get("SW_WACZ_CRAWL", False)
-        self.timeout = settings.getfloat("SW_WACZ_TIMEOUT", 60.0)
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:

diff --git a/tests/test_downloadermiddlewares.py b/tests/test_downloadermiddlewares.py
@@ -24,7 +24,6 @@ def _get_settings(self, **new_settings):
         settings = {
             "SW_WACZ_SOURCE_URI": self._get_wacz_source_url(),
             "SW_WACZ_CRAWL": True,
-            "SW_WACZ_TIMEOUT": 60,
         }
         settings.update(new_settings)
         return Settings(settings)

diff --git a/tests/test_middleware.py b/tests/test_middleware.py
@@ -17,7 +17,6 @@ def setup_method(self):
     def _get_settings(self, **new_settings):
         settings = {
             "SW_WACZ_SOURCE_URI": get_test_data_path("warc_1_1", "quotes.wacz").as_uri(),
-            "SW_WACZ_TIMEOUT": 60,
         }
         settings.update(new_settings)
         return Settings(settings)