# =============================================================================
# AI Crawlers & Assistants - Explicitly Allowed
# =============================================================================

# OpenAI
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Allow: /

# Anthropic
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
Allow: /

# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /

# ByteDance
User-agent: Bytespider
Allow: /

# Huawei
User-agent: PetalBot
Allow: /

# Apple
User-agent: Applebot
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# Meta
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
Allow: /

# Google AI
User-agent: Google-CloudVertexBot
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# DuckDuckGo
User-agent: DuckAssistBot
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# Other AI Crawlers
User-agent: Anchor Browser
User-agent: Novellum AI Crawl
User-agent: ProRataInc
User-agent: Timpibot
Allow: /

# Internet Archive
User-agent: archive.org_bot
Allow: /

# =============================================================================
# Search Engine Crawlers
# =============================================================================

User-agent: Googlebot
User-agent: Bingbot
Allow: /

# Language parameter from Flutter app - block from indexing
Disallow: /*?lang=*
Disallow: /*&lang=*

# Faceted Navigation - block filter combinations to save crawl budget
# Google Dec 2024: https://developers.google.com/search/docs/crawling-indexing/crawling-managing-faceted-navigation
# Allow clean paths (make/model), block any URL with query parameters (filters)
# Using *? pattern to match any path level + query string
Allow: /catalog-avto/
Disallow: /catalog-avto*?
Allow: /catalog-avto-china/
Allow: /catalog-avto-china-new/
Allow: /catalog-avto-china-used/
Disallow: /catalog-avto-china*?
Disallow: /catalog-avto-china-new*?
Disallow: /catalog-avto-china-used*?
Allow: /catalog-avto-europe/
Disallow: /catalog-avto-europe*?
Allow: /catalog-avto-korea/
Allow: /catalog-avto-korea-new/
Allow: /catalog-avto-korea-used/
Disallow: /catalog-avto-korea*?
Disallow: /catalog-avto-korea-new*?
Disallow: /catalog-avto-korea-used*?
Allow: /catalog-avto-uae/
Disallow: /catalog-avto-uae*?

# Block invalid URLs where lot_number is used as make (e.g. /catalog-avto-china/472166/model-name)
# Make names cannot start with numbers - these are invalid URLs that should not be indexed
# Using * wildcard for explicit matching of all paths starting with digit
Disallow: /catalog-avto/0*
Disallow: /catalog-avto/1*
Disallow: /catalog-avto/2*
Disallow: /catalog-avto/3*
Disallow: /catalog-avto/4*
Disallow: /catalog-avto/5*
Disallow: /catalog-avto/6*
Disallow: /catalog-avto/7*
Disallow: /catalog-avto/8*
Disallow: /catalog-avto/9*
Disallow: /catalog-avto-china/0*
Disallow: /catalog-avto-china/1*
Disallow: /catalog-avto-china/2*
Disallow: /catalog-avto-china/3*
Disallow: /catalog-avto-china/4*
Disallow: /catalog-avto-china/5*
Disallow: /catalog-avto-china/6*
Disallow: /catalog-avto-china/7*
Disallow: /catalog-avto-china/8*
Disallow: /catalog-avto-china/9*
Disallow: /catalog-avto-china-new/0*
Disallow: /catalog-avto-china-new/1*
Disallow: /catalog-avto-china-new/2*
Disallow: /catalog-avto-china-new/3*
Disallow: /catalog-avto-china-new/4*
Disallow: /catalog-avto-china-new/5*
Disallow: /catalog-avto-china-new/6*
Disallow: /catalog-avto-china-new/7*
Disallow: /catalog-avto-china-new/8*
Disallow: /catalog-avto-china-new/9*
Disallow: /catalog-avto-china-used/0*
Disallow: /catalog-avto-china-used/1*
Disallow: /catalog-avto-china-used/2*
Disallow: /catalog-avto-china-used/3*
Disallow: /catalog-avto-china-used/4*
Disallow: /catalog-avto-china-used/5*
Disallow: /catalog-avto-china-used/6*
Disallow: /catalog-avto-china-used/7*
Disallow: /catalog-avto-china-used/8*
Disallow: /catalog-avto-china-used/9*
Disallow: /catalog-avto-europe/0*
Disallow: /catalog-avto-europe/1*
Disallow: /catalog-avto-europe/2*
Disallow: /catalog-avto-europe/3*
Disallow: /catalog-avto-europe/4*
Disallow: /catalog-avto-europe/5*
Disallow: /catalog-avto-europe/6*
Disallow: /catalog-avto-europe/7*
Disallow: /catalog-avto-europe/8*
Disallow: /catalog-avto-europe/9*
Disallow: /catalog-avto-korea/0*
Disallow: /catalog-avto-korea/1*
Disallow: /catalog-avto-korea/2*
Disallow: /catalog-avto-korea/3*
Disallow: /catalog-avto-korea/4*
Disallow: /catalog-avto-korea/5*
Disallow: /catalog-avto-korea/6*
Disallow: /catalog-avto-korea/7*
Disallow: /catalog-avto-korea/8*
Disallow: /catalog-avto-korea/9*
Disallow: /catalog-avto-korea-new/0*
Disallow: /catalog-avto-korea-new/1*
Disallow: /catalog-avto-korea-new/2*
Disallow: /catalog-avto-korea-new/3*
Disallow: /catalog-avto-korea-new/4*
Disallow: /catalog-avto-korea-new/5*
Disallow: /catalog-avto-korea-new/6*
Disallow: /catalog-avto-korea-new/7*
Disallow: /catalog-avto-korea-new/8*
Disallow: /catalog-avto-korea-new/9*
Disallow: /catalog-avto-korea-used/0*
Disallow: /catalog-avto-korea-used/1*
Disallow: /catalog-avto-korea-used/2*
Disallow: /catalog-avto-korea-used/3*
Disallow: /catalog-avto-korea-used/4*
Disallow: /catalog-avto-korea-used/5*
Disallow: /catalog-avto-korea-used/6*
Disallow: /catalog-avto-korea-used/7*
Disallow: /catalog-avto-korea-used/8*
Disallow: /catalog-avto-korea-used/9*
Disallow: /catalog-avto-uae/0*
Disallow: /catalog-avto-uae/1*
Disallow: /catalog-avto-uae/2*
Disallow: /catalog-avto-uae/3*
Disallow: /catalog-avto-uae/4*
Disallow: /catalog-avto-uae/5*
Disallow: /catalog-avto-uae/6*
Disallow: /catalog-avto-uae/7*
Disallow: /catalog-avto-uae/8*
Disallow: /catalog-avto-uae/9*

# System directories
Disallow: /backend/
Disallow: /admin/
Disallow: /api/
Disallow: /vendor/
Disallow: /storage/
Disallow: /node_modules/
Disallow: /resources/
Disallow: /tests/

# =============================================================================
# Yandex - Special Rules
# =============================================================================

User-agent: Yandex
Allow: /
Disallow: /*amp/
Disallow: /*?lang=*
Disallow: /*&lang=*
Disallow: /backend/
Disallow: /admin/
Host: https://westmotors.kz

# =============================================================================
# Baidu
# =============================================================================

User-agent: Baiduspider
Allow: /
Disallow: /backend/
Disallow: /admin/

# =============================================================================
# Default Rules (All Other Bots)
# =============================================================================

User-agent: *
Allow: /

# Language parameter from Flutter app - block from indexing
Disallow: /*?lang=*
Disallow: /*&lang=*

# Faceted Navigation - block filter combinations (any path + query params)
Allow: /catalog-avto/
Disallow: /catalog-avto*?
Allow: /catalog-avto-china/
Allow: /catalog-avto-china-new/
Allow: /catalog-avto-china-used/
Disallow: /catalog-avto-china*?
Disallow: /catalog-avto-china-new*?
Disallow: /catalog-avto-china-used*?
Allow: /catalog-avto-europe/
Disallow: /catalog-avto-europe*?
Allow: /catalog-avto-korea/
Allow: /catalog-avto-korea-new/
Allow: /catalog-avto-korea-used/
Disallow: /catalog-avto-korea*?
Disallow: /catalog-avto-korea-new*?
Disallow: /catalog-avto-korea-used*?
Allow: /catalog-avto-uae/
Disallow: /catalog-avto-uae*?

# Block invalid URLs where lot_number is used as make (e.g. /catalog-avto-china/472166/model-name)
# Make names cannot start with numbers - these are invalid URLs that should not be indexed
# Using * wildcard for explicit matching of all paths starting with digit
Disallow: /catalog-avto/0*
Disallow: /catalog-avto/1*
Disallow: /catalog-avto/2*
Disallow: /catalog-avto/3*
Disallow: /catalog-avto/4*
Disallow: /catalog-avto/5*
Disallow: /catalog-avto/6*
Disallow: /catalog-avto/7*
Disallow: /catalog-avto/8*
Disallow: /catalog-avto/9*
Disallow: /catalog-avto-china/0*
Disallow: /catalog-avto-china/1*
Disallow: /catalog-avto-china/2*
Disallow: /catalog-avto-china/3*
Disallow: /catalog-avto-china/4*
Disallow: /catalog-avto-china/5*
Disallow: /catalog-avto-china/6*
Disallow: /catalog-avto-china/7*
Disallow: /catalog-avto-china/8*
Disallow: /catalog-avto-china/9*
Disallow: /catalog-avto-china-new/0*
Disallow: /catalog-avto-china-new/1*
Disallow: /catalog-avto-china-new/2*
Disallow: /catalog-avto-china-new/3*
Disallow: /catalog-avto-china-new/4*
Disallow: /catalog-avto-china-new/5*
Disallow: /catalog-avto-china-new/6*
Disallow: /catalog-avto-china-new/7*
Disallow: /catalog-avto-china-new/8*
Disallow: /catalog-avto-china-new/9*
Disallow: /catalog-avto-china-used/0*
Disallow: /catalog-avto-china-used/1*
Disallow: /catalog-avto-china-used/2*
Disallow: /catalog-avto-china-used/3*
Disallow: /catalog-avto-china-used/4*
Disallow: /catalog-avto-china-used/5*
Disallow: /catalog-avto-china-used/6*
Disallow: /catalog-avto-china-used/7*
Disallow: /catalog-avto-china-used/8*
Disallow: /catalog-avto-china-used/9*
Disallow: /catalog-avto-europe/0*
Disallow: /catalog-avto-europe/1*
Disallow: /catalog-avto-europe/2*
Disallow: /catalog-avto-europe/3*
Disallow: /catalog-avto-europe/4*
Disallow: /catalog-avto-europe/5*
Disallow: /catalog-avto-europe/6*
Disallow: /catalog-avto-europe/7*
Disallow: /catalog-avto-europe/8*
Disallow: /catalog-avto-europe/9*
Disallow: /catalog-avto-korea/0*
Disallow: /catalog-avto-korea/1*
Disallow: /catalog-avto-korea/2*
Disallow: /catalog-avto-korea/3*
Disallow: /catalog-avto-korea/4*
Disallow: /catalog-avto-korea/5*
Disallow: /catalog-avto-korea/6*
Disallow: /catalog-avto-korea/7*
Disallow: /catalog-avto-korea/8*
Disallow: /catalog-avto-korea/9*
Disallow: /catalog-avto-korea-new/0*
Disallow: /catalog-avto-korea-new/1*
Disallow: /catalog-avto-korea-new/2*
Disallow: /catalog-avto-korea-new/3*
Disallow: /catalog-avto-korea-new/4*
Disallow: /catalog-avto-korea-new/5*
Disallow: /catalog-avto-korea-new/6*
Disallow: /catalog-avto-korea-new/7*
Disallow: /catalog-avto-korea-new/8*
Disallow: /catalog-avto-korea-new/9*
Disallow: /catalog-avto-korea-used/0*
Disallow: /catalog-avto-korea-used/1*
Disallow: /catalog-avto-korea-used/2*
Disallow: /catalog-avto-korea-used/3*
Disallow: /catalog-avto-korea-used/4*
Disallow: /catalog-avto-korea-used/5*
Disallow: /catalog-avto-korea-used/6*
Disallow: /catalog-avto-korea-used/7*
Disallow: /catalog-avto-korea-used/8*
Disallow: /catalog-avto-korea-used/9*
Disallow: /catalog-avto-uae/0*
Disallow: /catalog-avto-uae/1*
Disallow: /catalog-avto-uae/2*
Disallow: /catalog-avto-uae/3*
Disallow: /catalog-avto-uae/4*
Disallow: /catalog-avto-uae/5*
Disallow: /catalog-avto-uae/6*
Disallow: /catalog-avto-uae/7*
Disallow: /catalog-avto-uae/8*
Disallow: /catalog-avto-uae/9*

# System directories
Disallow: /backend/
Disallow: /admin/
Disallow: /api/
Disallow: /*?2amo*
Disallow: /cdn-cgi/l/email-protection
Disallow: /vendor/
Disallow: /storage/
Disallow: /node_modules/
Disallow: /resources/
Disallow: /tests/

# Container tracking - allow main page, disallow subpages
Allow: /container-tracking
Disallow: /container-tracking-map/*
Disallow: /container-tracking/*

# =============================================================================
# Sitemap
# =============================================================================

Sitemap: https://westmotors.kz/sitemap.xml