# robots.txt - ABRAJ Energy Services
# Investor Relations website (Umbraco CMS) - umb-abrajenergy-ir
# https://llmstxt.org/ | https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt
#
# Application host: https://dev.vn.euroland.com:8104
# Production: https://www.abrajenergy.com
# Pre-launch: HTML pages on dev/staging emit noindex (SiteGeo:AllowSearchIndexing).

# -----------------------------------------------------------------------------
# Default rules (all crawlers)
# -----------------------------------------------------------------------------
User-agent: *
Allow: /

# CMS, backoffice, and non-public application paths
Disallow: /umbraco/
Disallow: /App_Plugins/
Disallow: /App_Data/
Disallow: /install/

# Error and utility pages (not useful for indexing)
Disallow: /InternalServerError.html

# -----------------------------------------------------------------------------
# Major search engines
# (Explicit groups for clarity; same public-site rules apply.)
# -----------------------------------------------------------------------------
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
User-agent: Applebot
Allow: /
Disallow: /umbraco/
Disallow: /App_Plugins/
Disallow: /App_Data/
Disallow: /install/
Disallow: /InternalServerError.html

# -----------------------------------------------------------------------------
# AI assistants, LLM crawlers, and answer engines (GEO)
# Public IR content may be crawled and cited.
# Admin paths remain blocked for all agents below.
# -----------------------------------------------------------------------------
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: Bytespider
User-agent: DeepSeekBot
User-agent: ChatGLM-Spider
User-agent: PanguBot
User-agent: PetalBot
User-agent: Baidubot
User-agent: CCBot
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: Applebot-Extended
User-agent: cohere-ai
User-agent: Diffbot
User-agent: YouBot
User-agent: ImagesiftBot
User-agent: Omgilibot

Allow: /
Disallow: /umbraco/
Disallow: /App_Plugins/
Disallow: /App_Data/
Disallow: /install/
Disallow: /InternalServerError.html

# -----------------------------------------------------------------------------
# Sitemap (dynamic - SitemapController /sitemap.xml)
# Update host to production URL before go-live.
# -----------------------------------------------------------------------------
Sitemap: https://dev.vn.euroland.com:8104/sitemap.xml

# -----------------------------------------------------------------------------
# LLM / GEO - curated site summary for AI assistants (https://llmstxt.org/)
# Also linked from HTML: <link rel="llms-txt" href="/llms.txt">
# Note: "LLMs-Txt:" is NOT a valid robots.txt directive (Lighthouse/Google reject it).
# Discoverability is via the HTML link rel above and /llms.txt itself.
# -----------------------------------------------------------------------------
# See also: https://dev.vn.euroland.com:8104/llms.txt