# Default robots.txt file for WordPress

# Block all bots from accessing sensitive directories
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-json/
Disallow: /cgi-bin/
Disallow: /xmlrpc.php

# Block specific pages
Disallow: /thank-you
Disallow: /?s=
Disallow: /search/

# Allow admin-ajax.php for dynamic content to work
Allow: /wp-admin/admin-ajax.php

# Sitemap location (update with your actual sitemap URL)
Sitemap: https://www.khaitanpower.com/sitemap_index.xml

# Avoid indexing login and registration pages
Disallow: /wp-login.php
Disallow: /wp-register.php

# Block access to private feeds
Disallow: /feed/
Disallow: /comments/feed/

# Specific search engines
User-agent: Googlebot
Disallow: /thank-you

User-agent: Bingbot
Disallow: /thank-you

# Security headers
User-agent: *
Crawl-delay: 10