diff --git a/src/_data/aibots.js b/src/_data/aibots.js new file mode 100644 index 0000000..e956790 --- /dev/null +++ b/src/_data/aibots.js @@ -0,0 +1,4 @@ +module.exports = async function() { + response = await fetch("https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/refs/heads/main/robots.txt"); + return response.text() +} \ No newline at end of file diff --git a/src/robots.njk b/src/robots.njk new file mode 100644 index 0000000..96c308e --- /dev/null +++ b/src/robots.njk @@ -0,0 +1,16 @@ +--- +permalink: /robots.txt +--- +# omg haiiiii robots ^-^ +# i love robots :3 + +# AI scrapers +{{ aibots }} + +# everyone else +User-agent: * +Allow: / + +# sitemaps +Sitemap: {{ site.url }}/sitemap.xml +Sitemap: {{ site.url }}/sitemap-news.xml \ No newline at end of file diff --git a/src/static/robots.txt b/src/static/robots.txt deleted file mode 100644 index 229997d..0000000 --- a/src/static/robots.txt +++ /dev/null @@ -1,5 +0,0 @@ -User-agent: GPTBot -Disallow: / - -User-agent: Google-Extended -Disallow: / \ No newline at end of file