diff --git a/app/public/robots.txt b/app/public/robots.txt new file mode 100644 index 00000000..2b2410e9 --- /dev/null +++ b/app/public/robots.txt @@ -0,0 +1,9 @@ +User-agent: * +Disallow: /datasets?* +Disallow: /tasks?* +Disallow: /flows?* +Disallow: /runs?* +Allow: /datasets$ +Allow: /tasks$ +Allow: /flows$ +Allow: /runs$