# robots.txt for https://www.sparkscs.co.uk/

# 1. Point AI Agents to your structured summaries
Sitemap: https://www.sparkscs.co.uk/sitemap.xml
Link: <https://www.sparkscs.co.uk/llms.txt>; rel="help"
Link: <https://www.sparkscs.co.uk/llms-full.txt>; rel="help"

# 2. General Rules for all bots
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /login/
Disallow: /checkout/
Disallow: /search?*
Disallow: /private/

# 3. Specific instructions for AI Scrapers (GPTBot, CCBot, etc.)
# We allow them, but want them to prioritize the LLM files
User-agent: GPTBot
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /

# 4. Crawl-delay to prevent server strain during heavy indexing
Crawl-delay: 10