🐛 only do the news search params and instant search

equinor · Nov 20, 2024 · 5b43441 · 5b43441
1 parent 1f9b518
commit 5b43441
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/web/pages/robots.txt.ts b/web/pages/robots.txt.ts
@@ -20,7 +20,7 @@ export const crawlableDomains = [
 
 const robots = (domain: string) => `User-agent: *
 ${crawlableDomains.includes(domain) ? 'Allow' : 'Disallow'}: /
-${domain === 'www.equinor.com' ? 'Disallow: /search \nDisallow: /search.html \nDisallow: /*?*' : ''}
+${domain === 'www.equinor.com' ? `Disallow: /search \nDisallow: /search.html \nDisallow: /?*topic \nDisallow: /?*country \nDisallow: /?*year \nDisallow: /?*refinementList /?*sortBy` : ''}
 Sitemap: ${domain.startsWith('www') ? `https://${domain}` : domain}/sitemap.xml
 `