User-agent: *
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /admin-login
Disallow: /admin-login/
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /profile
Disallow: /profile/
Disallow: /receipt
Disallow: /receipt/
Disallow: /digital-payment-success
Disallow: /merchandise/checkout
Disallow: /api/
Disallow: /cart/
Disallow: /checkout/

# AI Search & LLM Crawlers
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Sitemaps
Sitemap: https://www.webitya.com/sitemap.xml
Sitemap: https://www.webitya.com/blogs.xml
Sitemap: https://www.webitya.com/services.xml
Sitemap: https://www.webitya.com/portfolio.xml
Sitemap: https://www.webitya.com/pages.xml
Sitemap: https://www.webitya.com/careers.xml
Sitemap: https://www.webitya.com/courses.xml
Sitemap: https://www.webitya.com/on-job-training.xml
Sitemap: https://www.webitya.com/hire-talent.xml
Sitemap: https://www.webitya.com/notes.xml
Sitemap: https://www.webitya.com/merchandise.xml
Sitemap: https://www.webitya.com/reviews.xml