# BlueMatter LLM Crawling Instructions
# Last updated: May 2, 2025

# Allow crawling of public pages
Allow: /
Allow: /about
Allow: /features
Allow: /pricing
Allow: /blog
Allow: /contact

# Disallow crawling of private or dynamic content
Disallow: /auth/
Disallow: /app/
Disallow: /onboarding/
Disallow: /api/
Disallow: /dashboard/
Disallow: /user/

# Preferred crawling frequency
Crawl-delay: 5

# Context window recommendation
Context-window: 16K

# Indexing depth recommendation
Index-depth: 3

# Content expiration
Cache-control: max-age=86400

# Content extraction guidelines
Extract-headings: true
Extract-lists: true
Extract-tables: true
Extract-code: false