# =======================================================
# robots.txt for https://netsolonline.com
# Last Updated: Oct 2025
# Author: Netsol Online
# Purpose: Allow full crawling of public business pages while blocking duplicates, system files & internal search.
# =======================================================

# --- Universal Rules (applied to all bots) ---
User-agent: *
Disallow: /App_Data/
Disallow: /bin/
Disallow: /obj/
Disallow: /Scripts/
Disallow: /Content/
Disallow: /admin/
Disallow: /web.config
Disallow: /*.axd$
Disallow: /CaptchaImage.axd

# --- Block thin / duplicate content ---
Disallow: /search
Disallow: /search?
Disallow: /?s=
Disallow: /Default.aspx
Disallow: /*?*

# --- Allow all extensionless and clean URLs ---
Allow: /$
Allow: /About
Allow: /Contact
Allow: /Services
Allow: /Solutions
Allow: /Products

# --- Business Solutions ---
Allow: /ERP
Allow: /DIRBS
Allow: /HRPayroll
Allow: /Inventory
Allow: /REMA

# --- Education Solutions ---
Allow: /iSS
Allow: /LMS

# --- Hospitality Solutions ---
Allow: /RMS
Allow: /ESSMIS
Allow: /PBS

# --- E-Commerce Solutions ---
Allow: /MyCart
Allow: /B2B
Allow: /B2C

# --- Healthcare Solutions ---
Allow: /HMS
Allow: /Rehab
Allow: /EMR
Allow: /Clinic
Allow: /EHR

# --- Membership Management ---
Allow: /Membership

# --- Other Key Pages ---
Allow: /RequestDemo
Allow: /GetQuote
Allow: /FAQ
Allow: /Term-Condition
Allow: /Refund-Policy
Allow: /Hosting-Policy
Allow: /Cookie-Policy
Allow: /Privacy-Policy
Allow: /Service-Policy
Allow: /Sitemap

# --- Sitemap for Search Engines ---
Sitemap: https://netsolonline.com/sitemap.xml

# =======================================================
# --- Search Engine Specific Rules ---
# =======================================================

# Googlebot (Primary Indexer)
User-agent: Googlebot
Allow: /
Disallow: /search
Disallow: /?s=
Disallow: /Default.aspx

# Bingbot (Microsoft Search)
User-agent: Bingbot
Allow: /
Disallow: /search
Disallow: /?s=
Disallow: /Default.aspx

# Yahoo! Slurp
User-agent: Slurp
Allow: /
Disallow: /search

# DuckDuckBot
User-agent: DuckDuckBot
Allow: /
Disallow: /search

# =======================================================
# --- AI Crawler & Research Bots (Responsible Access) ---
# =======================================================

# ChatGPT / OpenAI GPTBot
User-agent: GPTBot
Allow: /About
Allow: /Services
Allow: /Solutions
Allow: /Products
Disallow: /admin/
Disallow: /search

# OpenAI Data Crawler (For Web Analysis)
User-agent: ChatGPT-User
Allow: /About
Allow: /Solutions
Disallow: /search
Disallow: /admin/

# Anthropic Claude AI
User-agent: ClaudeBot
Allow: /About
Allow: /Solutions
Disallow: /search

# Perplexity.ai Crawler
User-agent: PerplexityBot
Allow: /About
Allow: /Solutions
Disallow: /search

# Google-Extended (AI training data)
User-agent: Google-Extended
Disallow: /admin/
Disallow: /search
Allow: /Solutions
Allow: /ERP
Allow: /About

# =======================================================
# --- Notes ---
# - All main service, solution, and informational pages are crawlable.
# - Dynamic or duplicate content is safely blocked.
# - AI crawlers are permitted limited, ethical access for summarization.
# - Sitemap ensures structured discovery.
# =======================================================