User-agent: *
Disallow: /go/

User-agent: DataForSeoBot
Crawl-delay: 5

# Meta AI crawler used to collect public web content for AI model training.
# Blocked because TextGiraffe does not need to provide content for model training.
User-agent: meta-externalagent
Disallow: /

# Meta AI search crawler used to index pages for Meta AI search results and citations.
# Intentionally allowed.
User-agent: meta-webindexer
Disallow: /go/

Sitemap: https://www.textgiraffe.com/sitemap.xml