🤖 Add disallow for news and nyheter #2644 (#2645)

millianapia · web-flow · commit c8674f9a7187 · 2024-11-06T12:28:13.000+01:00
* 🤖 Add disallow for news and nyheter #2644 * Add search params #2644 * Change & ? * only search params #2644 * Change to wildcard * Add newline break #2644
diff --git a/web/pages/robots.txt.ts b/web/pages/robots.txt.ts
@@ -20,7 +20,7 @@ export const crawlableDomains = [
 
 const robots = (domain: string) => `User-agent: *
 ${crawlableDomains.includes(domain) ? 'Allow' : 'Disallow'}: /
-${domain === 'www.equinor.com' ? 'Disallow: /search \nDisallow: /search.html' : ''}
+${domain === 'www.equinor.com' ? 'Disallow: /search \nDisallow: /search.html \nDisallow: /*?*' : ''}
 Sitemap: ${domain.startsWith('www') ? `https://${domain}` : domain}/sitemap.xml
 `
 

Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,7 @@ export const crawlableDomains = [`
`20`	`20`
`21`	`21`	const robots = (domain: string) => `User-agent: *
`22`	`22`	`${crawlableDomains.includes(domain) ? 'Allow' : 'Disallow'}: /`
`23`		`-${domain === 'www.equinor.com' ? 'Disallow: /search \nDisallow: /search.html' : ''}`
	`23`	`+${domain === 'www.equinor.com' ? 'Disallow: /search \nDisallow: /search.html \nDisallow: /?' : ''}`
`24`	`24`	Sitemap: ${domain.startsWith('www') ? `https://${domain}` : domain}/sitemap.xml
`25`	`25`	`
`26`	`26`