User-agent: *
Allow: /

# Block private/authenticated areas from crawling
Disallow: /auth/
Disallow: /admin/
Disallow: /user/
Disallow: /api/
Disallow: /h2-console/
Disallow: /swagger-ui/
Disallow: /api-docs/

# Allow public content
Allow: /models/
Allow: /benchmarks
Allow: /about
Allow: /faq
Allow: /contact
Allow: /privacy-policy
Allow: /terms-of-service

# Sitemap location
Sitemap: https://benchable.ai/sitemap.xml