# robots.txt for arabicescorts.com
# Cleaned and optimized for Yandex, Google, Bing compliance
# Last updated: 2026-08-08


# ============================================================
# Main rules for all crawlers
# ============================================================
User-Agent: *
Allow: /
Crawl-delay: 1

# Block system and core folders
Disallow: /plugins/
# /libs/ is crawlable so Google can see X-Robots-Tag: noindex on JS/CSS
# (required to clear "Indexed though blocked by robots.txt"). Do NOT remove the
# X-Robots-Tag headers in .htaccess.
Disallow: /includes/
Disallow: /tmp/
Disallow: /backup/
Disallow: /admin/

# Block duplicate content from sorting and filters
Disallow: /print*
Disallow: /*?sort_by=
Disallow: /*&sort_by=
Disallow: /*?sort_type=
Disallow: /*&sort_type=
Disallow: /*claim-listing/?id=
# categoryFilter combo URLs (millions of noindex variants — save crawl budget)
Disallow: /*:*
Disallow: /index.php

# Retired gtranslate machine-translated prefixes (site now uses en/ar/tr only)
# Crawlers that still hit /es/, /ru/, /pt/ get 301 via legacy_seo_redirect.inc.php

# Block specific pages to prevent conflict with SEO URLs
Disallow: /*confirm.html*
Disallow: /*print.html*
Disallow: /404.html*
Disallow: /*listing-remove.html*
Disallow: /*claim-listing.html*
Disallow: /*newsletter.html*
Disallow: /*listing-details.html*
Disallow: /*escorts.html*

# ============================================================

### Rules generated by the Sitemap plugin ###
Sitemap: https://www.arabicescorts.com/sitemap.xml

# Excluded pages:
Disallow: /*confirm.html*
Disallow: /*listing-details.html*
Disallow: /*print.html*
Disallow: /404.html*
Disallow: /*listing-remove.html*
Disallow: /*escorts.html*
Disallow: /*claim-listing.html*
Disallow: /*newsletter.html*