From ea7564ad1634f422f0d73b44cfe06b848f9cf0eb Mon Sep 17 00:00:00 2001
From: Madhur Prajapati <madhurbprajapati2@gmail.com>
Date: Sat, 7 Oct 2023 23:05:45 +0530
Subject: [PATCH 1/3] added robots.txt for API endpoints

---
 api/api/templates/robots.txt | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 api/api/templates/robots.txt

diff --git a/api/api/templates/robots.txt b/api/api/templates/robots.txt
new file mode 100644
index 00000000000..811e0134451
--- /dev/null
+++ b/api/api/templates/robots.txt
@@ -0,0 +1,8 @@
+# Block API endpoints
+User-agent: *
+Disallow: /v1/images/
+Disallow: /v1/audio/
+Disallow: /v1/auth/
+
+User-agent: GPTBot
+Disallow: /

From a62e5cb499fd1f82e5265d4f7b047dc0c981bb46 Mon Sep 17 00:00:00 2001
From: Madhur Prajapati <madhurbprajapati2@gmail.com>
Date: Sun, 8 Oct 2023 08:48:40 +0530
Subject: [PATCH 2/3] added path for robots.txt for API endpoints

---
 api/conf/urls/__init__.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/api/conf/urls/__init__.py b/api/conf/urls/__init__.py
index 432bc5aa8f9..029d49fbbe8 100644
--- a/api/conf/urls/__init__.py
+++ b/api/conf/urls/__init__.py
@@ -9,6 +9,7 @@
 from django.contrib import admin
 from django.urls import include, path
 from django.views.generic import RedirectView
+from django.views.generic.base import TemplateView
 from rest_framework.routers import SimpleRouter
 
 from api.views.audio_views import AudioViewSet
@@ -35,6 +36,13 @@
     path("admin/", admin.site.urls),
     path("healthcheck/", HealthCheck.as_view(), name="health"),
     path("v1/", include(versioned_paths)),
+    path(
+        "robots.txt/",
+        TemplateView.as_view(
+            template_name="robots.txt",
+            content_type="text/plain",
+        ),
+    ),
 ]
 
 if settings.ENVIRONMENT == "local":

From dda0245b96d8b95040aed067f7580b3b0672ccc0 Mon Sep 17 00:00:00 2001
From: Dhruv Bhanushali <dhruv_b@live.com>
Date: Sun, 8 Oct 2023 17:55:44 +0000
Subject: [PATCH 3/3] Merge imports

---
 api/conf/urls/__init__.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/api/conf/urls/__init__.py b/api/conf/urls/__init__.py
index 029d49fbbe8..83b3a1ae7f6 100644
--- a/api/conf/urls/__init__.py
+++ b/api/conf/urls/__init__.py
@@ -8,8 +8,7 @@
 from django.conf.urls.static import static
 from django.contrib import admin
 from django.urls import include, path
-from django.views.generic import RedirectView
-from django.views.generic.base import TemplateView
+from django.views.generic import RedirectView, TemplateView
 from rest_framework.routers import SimpleRouter
 
 from api.views.audio_views import AudioViewSet