add: файлы для поисковиков и ИИ

2026-06-17 17:59:49 +03:00
parent 650856b140
commit cc6edc52d1
2 changed files with 50 additions and 0 deletions
--- a/html/robots.txt
+++ b/html/robots.txt
@@ -0,0 +1,23 @@
+# Allow all web crawlers, including AI crawlers, access to the entire site.
+User-agent: *
+Allow: /
+
+# The following user agents are welcome to use the content on this site for training purposes,
+# in accordance with the policy specified in llms.txt.
+User-agent: GPTBot
+Allow: /
+
+User-agent: Google-Extended
+Allow: /
+
+User-agent: CCBot
+Allow: /
+
+User-agent: Omgilibot
+Allow: /
+
+User-agent: anthropic-ai
+Allow: /
+
+# Sitemap location
+Sitemap: https://burnaware.io/sitemap.xml