From 500a00fcc56c945eb1468c648a72f95e9fa08ae2 Mon Sep 17 00:00:00 2001
From: maia arson crimew <maia@crimew.gay>
Date: Sun, 10 Nov 2024 21:21:02 +0100
Subject: [PATCH] autogenerate robots.txt

this automatically generates our robots.txt file, grabbing an updated list of ai scrapers to block each time
---
 src/_data/aibots.js   |  4 ++++
 src/robots.njk        | 16 ++++++++++++++++
 src/static/robots.txt |  5 -----
 3 files changed, 20 insertions(+), 5 deletions(-)
 create mode 100644 src/_data/aibots.js
 create mode 100644 src/robots.njk
 delete mode 100644 src/static/robots.txt

diff --git a/src/_data/aibots.js b/src/_data/aibots.js
new file mode 100644
index 0000000..e956790
--- /dev/null
+++ b/src/_data/aibots.js
@@ -0,0 +1,4 @@
+module.exports = async function() {
+    response = await fetch("https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/refs/heads/main/robots.txt");
+    return response.text()
+}
\ No newline at end of file
diff --git a/src/robots.njk b/src/robots.njk
new file mode 100644
index 0000000..96c308e
--- /dev/null
+++ b/src/robots.njk
@@ -0,0 +1,16 @@
+---
+permalink: /robots.txt
+---
+# omg haiiiii robots ^-^
+# i love robots :3
+
+# AI scrapers
+{{ aibots }}
+
+# everyone else
+User-agent: *
+Allow: /
+
+# sitemaps
+Sitemap: {{ site.url }}/sitemap.xml
+Sitemap: {{ site.url }}/sitemap-news.xml
\ No newline at end of file
diff --git a/src/static/robots.txt b/src/static/robots.txt
deleted file mode 100644
index 229997d..0000000
--- a/src/static/robots.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-User-agent: GPTBot
-Disallow: /
-
-User-agent: Google-Extended
-Disallow: /
\ No newline at end of file