# ADVANCED ROBOTS.TXT FOR CRESCA.XYZ - DUAL SEO & AI OPTIMIZATION
# Last Updated: August 2025
# Optimized for: Google Search, AI Training, Domain Rating tracking
# Strategy: Maximum indexing with controlled access for SEO tools

# === PRIMARY SEARCH ENGINES ===
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 2

# === SOCIAL MEDIA CRAWLERS ===
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

# === AI CRAWLERS & TRAINING BOTS (OPTIMIZED FOR AI SEARCH) ===
# OpenAI Crawlers
User-agent: GPTBot
Allow: /
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 2

# Anthropic Claude
User-agent: ClaudeBot
Allow: /
Crawl-delay: 2

# Common Crawl (Used by multiple AI systems)
User-agent: CCBot
Allow: /
Crawl-delay: 3

# Google Bard/Gemini
User-agent: Google-Extended
Allow: /
Crawl-delay: 2

# Bing AI/Copilot
User-agent: BingBot
Allow: /
Crawl-delay: 1

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Crawl-delay: 3

# Character.AI
User-agent: CharacterAI
Allow: /
Crawl-delay: 5

# Block training for unauthorized AI
User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Allow: /

# === SEO TOOLS & ANALYTICS (CONTROLLED ACCESS) ===
# Allow AhrefsBot for Domain Rating tracking but limit crawl rate
User-agent: AhrefsBot
Allow: /
Crawl-delay: 10
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/

# Allow SemrushBot for SEO analysis but with restrictions
User-agent: SemrushBot
Allow: /
Crawl-delay: 15
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/

# Allow MozBot for SEO metrics
User-agent: rogerbot
Allow: /
Crawl-delay: 10

# Block aggressive unknown crawlers
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SiteAuditBot
Disallow: /

# === GLOBAL RULES FOR ALL BOTS ===
User-agent: *
Allow: /

# === PRIORITY PAGES FOR MAXIMUM INDEXING ===
# Core business pages
Allow: /about
Allow: /features
Allow: /pricing
Allow: /contact
Allow: /blog/
Allow: /compare/
Allow: /email-marketing
Allow: /email-marketing-benchmarks
Allow: /how-to-use
Allow: /login
Allow: /signup

# Landing pages for AI discovery
Allow: /ai-email-marketing
Allow: /automated-campaigns
Allow: /email-templates
Allow: /analytics
Allow: /integrations

# SEO-optimized content pages
Allow: /resources/
Allow: /guides/
Allow: /case-studies/
Allow: /whitepapers/

# Block sensitive areas
Disallow: /admin/
Disallow: /api/
Disallow: /_next/
Disallow: /dashboard/
Disallow: /user/
Disallow: /account/
Disallow: /settings/

# Block tracking parameters for clean URLs
Disallow: /*.json$
Disallow: /*?utm_*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*
Disallow: /*?ref=*
Disallow: /*?source=*

# Block development files
Disallow: /tmp/
Disallow: /temp/
Disallow: /_vercel/
Disallow: /node_modules/
Disallow: /.git/
Disallow: /.env
Disallow: /package.json
Disallow: /package-lock.json

# === SITEMAPS FOR COMPREHENSIVE INDEXING ===
Sitemap: https://cresca.xyz/sitemap.xml
Sitemap: https://cresca.xyz/blog-sitemap.xml
Sitemap: https://cresca.xyz/pages-sitemap.xml

# === HOST DECLARATION ===
Host: https://cresca.xyz

# === OPTIMIZED CACHE & STATIC ASSETS ===
# Allow search engines and AI to access static content
Allow: /images/
Allow: /css/
Allow: /js/
Allow: /_next/static/
Allow: /favicon.ico
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /manifest.json
Allow: /llms.txt

# === AI TRAINING OPTIMIZATION ===
# Specify clean content for AI training
Allow: /blog/*
Allow: /guides/*
Allow: /resources/*
Allow: /documentation/*

# === AI CONTENT GUIDE ===
# LLMs.txt file for AI-friendly content discovery
# https://cresca.xyz/llms.txt

# === CRAWL OPTIMIZATION DIRECTIVES ===
# Request-rate: 1/10s for respectful crawling
# Visit-time: 0200-0800 GMT for off-peak crawling