diff --git a/web/pages/robots.txt.ts b/web/pages/robots.txt.ts index d480ea862..ee3dd8ffe 100644 --- a/web/pages/robots.txt.ts +++ b/web/pages/robots.txt.ts @@ -20,7 +20,7 @@ export const crawlableDomains = [ const robots = (domain: string) => `User-agent: * ${crawlableDomains.includes(domain) ? 'Allow' : 'Disallow'}: / -${domain === 'www.equinor.com' ? 'Disallow: /search \nDisallow: /search.html \nDisallow: /*?*' : ''} +${domain === 'www.equinor.com' ? `Disallow: /search \nDisallow: /search.html \nDisallow: /?*topic \nDisallow: /?*country \nDisallow: /?*year \nDisallow: /?*refinementList /?*sortBy` : ''} Sitemap: ${domain.startsWith('www') ? `https://${domain}` : domain}/sitemap.xml `