# Settlnn - PG and Hostel Finder in Pune
# Robots.txt for SEO optimization

# Allow all search engines to crawl
User-agent: *
Allow: /

# Sitemap location
Sitemap: https://settlnn.com/sitemap.xml

# Disallow sensitive/private areas
Disallow: /admin/
Disallow: /dashboard/
Disallow: /api/
Disallow: /_next/
Disallow: /uploads/
Disallow: /auth/
Disallow: /private/
Disallow: /temp/
Disallow: /*.json$
Disallow: /*.xml$
Disallow: /search?*
Disallow: /properties?*

# Allow important public pages
Allow: /
Allow: /properties
Allow: /pune/
Allow: /college/
Allow: /about
Allow: /contact

# Specific bot instructions
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 2

User-agent: facebookexternalhit
Allow: /

# Block bad bots (optional)
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /