From 08aa26de761164b97794bfe2ba1c350c44eda114 Mon Sep 17 00:00:00 2001
From: sarayourfriend <git@sarayourfriend.pictures>
Date: Tue, 9 Apr 2024 13:55:28 +1000
Subject: [PATCH] Add robots.txt and ai.txt to frontend

---
 frontend/src/static/ai.txt     | 60 ++++++++++++++++++++++++++++++++++
 frontend/src/static/robots.txt | 35 ++++++++++++++++++++
 2 files changed, 95 insertions(+)
 create mode 100644 frontend/src/static/ai.txt
 create mode 100644 frontend/src/static/robots.txt

diff --git a/frontend/src/static/ai.txt b/frontend/src/static/ai.txt
new file mode 100644
index 00000000000..1472382b3a8
--- /dev/null
+++ b/frontend/src/static/ai.txt
@@ -0,0 +1,60 @@
+# Spawning AI
+# Prevent datasets from using the following file types
+
+User-Agent: *
+Disallow: *.txt
+Disallow: *.pdf
+Disallow: *.doc
+Disallow: *.docx
+Disallow: *.odt
+Disallow: *.rtf
+Disallow: *.tex
+Disallow: *.wks
+Disallow: *.wpd
+Disallow: *.wps
+Disallow: *.html
+Disallow: *.bmp
+Disallow: *.gif
+Disallow: *.ico
+Disallow: *.jpeg
+Disallow: *.jpg
+Disallow: *.png
+Disallow: *.svg
+Disallow: *.tif
+Disallow: *.tiff
+Disallow: *.webp
+Disallow: *.aac
+Disallow: *.aiff
+Disallow: *.amr
+Disallow: *.flac
+Disallow: *.m4a
+Disallow: *.mp3
+Disallow: *.oga
+Disallow: *.opus
+Disallow: *.wav
+Disallow: *.wma
+Disallow: *.mp4
+Disallow: *.webm
+Disallow: *.ogg
+Disallow: *.avi
+Disallow: *.mov
+Disallow: *.wmv
+Disallow: *.flv
+Disallow: *.mkv
+Disallow: *.py
+Disallow: *.js
+Disallow: *.java
+Disallow: *.c
+Disallow: *.cpp
+Disallow: *.cs
+Disallow: *.h
+Disallow: *.css
+Disallow: *.php
+Disallow: *.swift
+Disallow: *.go
+Disallow: *.rb
+Disallow: *.pl
+Disallow: *.sh
+Disallow: *.sql
+Disallow: /
+Disallow: *
diff --git a/frontend/src/static/robots.txt b/frontend/src/static/robots.txt
new file mode 100644
index 00000000000..2739ca4400a
--- /dev/null
+++ b/frontend/src/static/robots.txt
@@ -0,0 +1,35 @@
+User-agent: GPTBot
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: Omgilibot
+Disallow: /
+
+User-agent: Omgili
+Disallow: /
+
+User-agent: FacebookBot
+Disallow: /
+
+User-agent: Diffbot
+Disallow: /
+
+User-agent: Bytespider
+Disallow: /
+
+User-agent: ImagesiftBot
+Disallow: /
+
+User-agent: cohere-ai
+Disallow: /