Added a robots.txt file

config/settings/base.py

@@ -147,6 +147,7 @@
         "BACKEND": "django.template.backends.django.DjangoTemplates",
         "DIRS": [
             str(ROOT_DIR / "grumpy/core/templates"),
+            str(ROOT_DIR / "config/templates"),
         ],
         # 'APP_DIRS': True,  # not needed since "app_directories.Loader" is specified below
         "OPTIONS": {

config/templates/robots.txt

@@ -0,0 +1,4 @@
+# robots.txt
+
+User-Agent: *
+Disallow: /

config/urls.py

@@ -11,6 +11,7 @@
 from django.conf.urls.static import static
 from django.contrib import admin
 from django.urls import include, path
+from django.views.generic import TemplateView
 
 # from allauth.account.decorators import secure_admin_login
 
@@ -48,6 +49,11 @@
 #################
 
 urlpatterns = [
+    # robots.txt
+    path(
+        "robots.txt",
+        TemplateView.as_view(template_name="robots.txt", content_type="text/plain"),
+    ),
     # admin...
     path("admin/", admin.site.urls),
     # auth...

grumpy/core/tests/test_urls.py

@@ -0,0 +1,8 @@
+from http import HTTPStatus
+
+def test_robots_txt(client):
+
+    response = client.get("/robots.txt")
+    assert response.status_code == HTTPStatus.OK
+    assert response["content-type"] == "text/plain"
+    assert response.content.startswith(b"# robots.txt\n")