User-agent: *
Allow: /

# Important pages for crawling
Allow: /services
Allow: /pricing
Allow: /about
Allow: /careers
Allow: /blog
Allow: /stories
Allow: /news

# Disallow admin/private areas
Disallow: /admin
Disallow: /api
Disallow: /uploads

# Sitemap location
Sitemap: https://mainstreet.com/sitemap.xml

# Crawl delay for respectful crawling
Crawl-delay: 1