Remove the single-result endpoints from the robots.txt (#4660)

* Remove the single-result endpoints from the robots.txt

* Set robots meta content to `all`

* Fix playwright e2e tests

Author: AetherUnbound

frontend/src/server-middleware/robots.js

@@ -33,14 +33,10 @@ Crawl-delay: 10
 Disallow: /search/audio/
 Disallow: /search/image/
 Disallow: /search/
-Disallow: /image/
-Disallow: /audio/
 # Disallow the same for all translated routes
 Disallow: /*/search/audio/
 Disallow: /*/search/image/
 Disallow: /*/search/
-Disallow: /*/image/
-Disallow: /*/audio/
 
 ${aiDisallowRules}
       `

frontend/src/utils/og.ts

@@ -19,7 +19,7 @@ export const createDetailPageMeta = ({
     {
       hid: "robots",
       name: "robots",
-      content: "noindex",
+      content: "all",
     },
   ] as (MetaPropertyName | MetaPropertyProperty)[]
   if (title) {

frontend/test/playwright/e2e/seo.spec.ts

@@ -42,7 +42,7 @@ const pages = {
       "/v1/images/da5cb478-c093-4d62-b721-cda18797e3fb/thumb/"
     ),
     ogTitle: "bird",
-    robots: "noindex",
+    robots: "all",
   },
   audioDetail: {
     url: "/audio/7e063ee6-343f-48e4-a4a5-f436393730f6",
@@ -51,7 +51,7 @@ const pages = {
       "/v1/audio/7e063ee6-343f-48e4-a4a5-f436393730f6/thumb/"
     ),
     ogTitle: "I Love My Dog You Love your Cat",
-    robots: "noindex",
+    robots: "all",
   },
   about: {
     url: "/about",