Add retry to requests

ingestify/infra/fetch/http.py

@@ -6,11 +6,27 @@
 from typing import Optional, Callable, Tuple
 
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3 import Retry
 
 from ingestify.domain.models import DraftFile, File
 from ingestify.utils import utcnow
 
 
+retry_strategy = Retry(
+    total=4,  # Maximum number of retries
+    backoff_factor=2,  # Exponential backoff factor (e.g., 2 means 1, 2, 4, 8 seconds, ...)
+    status_forcelist=[429, 500, 502, 503, 504],  # HTTP status codes to retry on
+)
+
+adapter = HTTPAdapter(max_retries=retry_strategy)
+
+# Create a new session object
+session = requests.Session()
+session.mount('http://', adapter)
+session.mount('https://', adapter)
+
+
 def retrieve_http(
     url,
     current_file: Optional[File] = None,
@@ -41,7 +57,7 @@ def retrieve_http(
         else:
             raise Exception(f"Don't know how to use {key}")
 
-    response = requests.get(url, headers=headers, **http_kwargs)
+    response = session.get(url, headers=headers, **http_kwargs)
     response.raise_for_status()
     if response.status_code == 304:
         # Not modified