User-agent: *
Allow: /

# Sitemaps
Sitemap: https://freesmartcalculator.com/sitemap.xml

# Crawl-delay for respectful crawling (1 second between requests)
Crawl-delay: 1

# Disallow admin and internal paths
Disallow: /admin/
Disallow: /api/
Disallow: /_next/
Disallow: /scripts/
Disallow: /node_modules/
Disallow: /.git/
Disallow: /calculator-prompts/
Disallow: /enhanced-data/
Disallow: /enhanced-prompts/
Disallow: /analysis-temp/

# Disallow common non-content files
Disallow: /*.json$
Disallow: /*.md$
Disallow: /*.txt$
Disallow: /*.log$
Disallow: /package*.json
Disallow: /tsconfig*.json
Disallow: /next.config.js
Disallow: /tailwind.config.ts
Disallow: /postcss.config.js

# Allow important static files
Allow: /favicon.ico
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /site.webmanifest
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$

# Explicitly allow all calculator categories (for clarity)
Allow: /finance/
Allow: /health/
Allow: /construction/
Allow: /math/
Allow: /conversion/
Allow: /physics/
Allow: /chemistry/
Allow: /biology/
Allow: /ecology/
Allow: /statistics/
Allow: /sports/
Allow: /food/
Allow: /everyday-life/
Allow: /other/

# Allow important pages
Allow: /about-us/
Allow: /contact-us/
Allow: /faq/
Allow: /help/
Allow: /privacy-policy/
Allow: /terms-of-services/

# Special instructions for major search engines
User-agent: Googlebot
Crawl-delay: 0.5
Allow: /

User-agent: Bingbot
Crawl-delay: 1
Allow: /

User-agent: Slurp
Crawl-delay: 2
Allow: /

# Allow SEO analysis bots (they help with visibility and analytics)
User-agent: SemrushBot
Crawl-delay: 2
Allow: /

User-agent: AhrefsBot
Crawl-delay: 2
Allow: /

# Allow other legitimate bots with slower crawl rates
User-agent: MJ12bot
Crawl-delay: 3
Allow: /

User-agent: DotBot
Crawl-delay: 3
Allow: /

# Block only truly problematic bots
User-agent: SemrushBot-SA
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: GPTBot
Disallow: /