# Stenberg College Robots.txt
# Allow all search engines to crawl public content

User-agent: *
Allow: /

# Block admin areas and internal paths
Disallow: /cp/
Disallow: /cp
Disallow: /admin/
Disallow: /admin
Disallow: /.env
Disallow: /vendor/
Disallow: /storage/
Disallow: /bootstrap/
Disallow: /config/

# Block common development/testing paths
Disallow: /test/
Disallow: /staging/
Disallow: /dev/

# Allow specific important files
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /favicon.ico

# Sitemap location
Sitemap: https://stenbergcollege.com/sitemap.xml

# Crawl delay (optional - only if needed for server performance)
# Crawl-delay: 1