diff --git a/layouts/robots.txt b/layouts/robots.txt new file mode 100644 index 00000000..7f11ea21 --- /dev/null +++ b/layouts/robots.txt @@ -0,0 +1,4 @@ +Sitemap: {{"/sitemap.xml" | absURL}} + +User-agent: GPTBot +Disallow: / diff --git a/static/robots.txt b/static/robots.txt deleted file mode 100644 index a6be3783..00000000 --- a/static/robots.txt +++ /dev/null @@ -1,7 +0,0 @@ -Sitemap: /sitemap.xml - -User-agent: GPTBot -Disallow: / - -User-Agent: DuckDuckBot -Allow: /