User-agent: *
Allow: /

# Sitemap location
Sitemap: https://www.get-tldr.com/sitemap.xml

# Crawl delay for respectful crawling
Crawl-delay: 1

# Allow all major search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# Block access to admin areas
User-agent: *
Disallow: /app/
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /_index
Disallow: /*.json$
Disallow: /*?*

# Allow access to public API documentation
User-agent: *
Allow: /api-docs

# Block temporary or development files
User-agent: *
Disallow: /tmp/
Disallow: /temp/
Disallow: /dev/
Disallow: /test/
Disallow: /*.log$
Disallow: /*.tmp$

# Allow CSS and JS for better rendering
User-agent: *
Allow: /build/
Allow: /assets/
Allow: /css/
Allow: /js/
Allow: /images/
Allow: /fonts/

# Mobile-friendly
User-agent: Googlebot-Mobile
Allow: /

User-agent: Googlebot-Image
Allow: /

# News sitemaps (if applicable)
# Sitemap: https://www.get-tldr.com/news-sitemap.xml