when checking if url is input root url normalize the url without the …

…trailing /

src/helpers.py

@@ -17,6 +17,13 @@
 logger = logging.getLogger('apify')
 
 
+def normalize_url(url: str) -> str:
+    """Normalizes the URL by removing trailing slash."""
+    parsed_url = urlparse(url)
+    normalized = parsed_url._replace(path=parsed_url.path.rstrip('/'))
+    return normalized.geturl()
+
+
 def get_hostname_path_string_from_url(url: str) -> str:
     """Extracts the hostname and path from the URL."""
     parsed_url = urlparse(url)

src/main.py

@@ -6,7 +6,7 @@
 
 from apify import Actor
 
-from .helpers import get_crawler_actor_config, get_description_from_kvstore, is_description_suitable
+from .helpers import get_crawler_actor_config, get_description_from_kvstore, is_description_suitable, normalize_url
 from .renderer import render_llms_txt
 
 if TYPE_CHECKING:
@@ -23,6 +23,7 @@ async def main() -> None:
         if url is None:
             msg = 'Missing "startUrl" attribute in input!'
             raise ValueError(msg)
+        url_normalized = normalize_url(url)
 
         max_crawl_depth = int(actor_input.get('maxCrawlDepth', 1))
 
@@ -62,7 +63,7 @@ async def main() -> None:
                 logger.warning('Missing "htmlUrl" attribute in dataset item!')
                 continue
 
-            is_root = item_url == url
+            is_root = normalize_url(item_url) == url_normalized
             if is_root:
                 description = await get_description_from_kvstore(run_store, html_url)
                 data['description'] = description if is_description_suitable(description) else None
@@ -84,7 +85,7 @@ async def main() -> None:
         if is_dataset_empty:
             msg = (
                 'No pages were crawled successfully!'
-                'Please check the "apify/website-content-crawler" actor run for more details.'
+                ' Please check the "apify/website-content-crawler" actor run for more details.'
             )
             raise RuntimeError(msg)
 

tests/test_helpers.py

@@ -0,0 +1,14 @@
+from src.helpers import get_hostname_path_string_from_url, normalize_url
+
+
+def test_normalize_url() -> None:
+    url = 'https://example.com/'
+    url_normalized = 'https://example.com'
+    assert normalize_url(url) == url_normalized
+
+def test_get_hostname_path_string_from_url() -> None:
+    url = 'https://example.com/path'
+    assert get_hostname_path_string_from_url(url) == 'example.com/path'
+
+    url2 = 'https://example.com/path/'
+    assert get_hostname_path_string_from_url(url2) == 'example.com/path/'

tests/test_renderer.py

@@ -1,9 +1,16 @@
+from typing import TYPE_CHECKING
+
 from src.renderer import render_llms_txt
 
+if TYPE_CHECKING:
+    from src.mytypes import LLMSData
+
 
 def test_render_llms_txt() -> None:
-    data = {
+    data: LLMSData = {
         'title': 'docs.apify.com',
+        'details': None,
+        'description': None,
         'sections': [
            {
                'title': 'Index',
@@ -27,9 +34,10 @@ def test_render_llms_txt() -> None:
     assert render_llms_txt(data) == expected_output
 
 def test_render_llms_txt_with_description() -> None:
-    data = {
+    data: LLMSData = {
         'title': 'docs.apify.com',
         'description': 'Apify documentation',
+        'details': None,
         'sections': [
            {
                'title': 'Index',
@@ -55,7 +63,7 @@ def test_render_llms_txt_with_description() -> None:
     assert render_llms_txt(data) == expected_output
 
 def test_render_llms_txt_with_description_and_details() -> None:
-    data = {
+    data: LLMSData = {
         'title': 'docs.apify.com',
         'description': 'Apify documentation',
         'details': 'This is the documentation for Apify',
@@ -86,9 +94,11 @@ def test_render_llms_txt_with_description_and_details() -> None:
     assert render_llms_txt(data) == expected_output
 
 def test_render_llms_txt_with_no_sections() -> None:
-    data = {
+    data: LLMSData = {
         'title': 'docs.apify.com',
         'description': 'Apify documentation',
+        'details': None,
+        'sections': []
     }
 
     expected_output = """# docs.apify.com