# Perkusai robots policy
#
# AI assistants / crawlers: use ONLY the dedicated AI endpoints  /api/ai/search  and
# /api/ai/stats. Every OTHER /api/ endpoint is internal (used by the website) and is
# off-limits to bots.
# Discovery: /llms.txt · /openapi.json · /for-ai.html (also .en.html / .ru.html)

# Generic crawlers: site pages yes, the whole API no.
# Content Signals (contentsignals.org): AI may use our content to answer/ground shopping
# queries (ai-input) and surface us in AI search, but not to train models. Adjust as you wish.
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /api/

# AI crawlers / assistants: site pages + the AI endpoint (/api/ai/) are allowed;
# all other /api/ endpoints are disallowed. (Allow is listed first and is the more
# specific path, so /api/ai/ wins over the /api/ Disallow.)
# Grouped user-agents (OpenAI, Anthropic, Google/Gemini, Perplexity, Microsoft/Copilot,
# Apple, Meta, Amazon, ByteDance, DuckDuckGo, You.com, Cohere, Common Crawl):
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: bingbot
User-agent: BingPreview
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: Amazonbot
User-agent: Bytespider
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: cohere-ai
User-agent: CCBot
Allow: /api/ai/
Disallow: /api/

Sitemap: https://perkusai.lt/sitemap.xml