24 lines
473 B
Plaintext
24 lines
473 B
Plaintext
# Allow all web crawlers, including AI crawlers, access to the entire site.
|
|
User-agent: *
|
|
Allow: /
|
|
|
|
# The following user agents are welcome to use the content on this site for training purposes,
|
|
# in accordance with the policy specified in llms.txt.
|
|
User-agent: GPTBot
|
|
Allow: /
|
|
|
|
User-agent: Google-Extended
|
|
Allow: /
|
|
|
|
User-agent: CCBot
|
|
Allow: /
|
|
|
|
User-agent: Omgilibot
|
|
Allow: /
|
|
|
|
User-agent: anthropic-ai
|
|
Allow: /
|
|
|
|
# Sitemap location
|
|
Sitemap: https://burnaware.io/sitemap.xml
|