# robots.txt — untangle.bio
# Last updated: August 2026
#
# Policy: all search engines and AI crawlers are welcome (default allow).
# App/auth/admin pages are excluded from indexes via <meta name="robots"
# content="noindex, nofollow"> on the pages themselves — they must stay
# crawlable here so bots can see those tags. Only /api/ is disallowed
# (JSON endpoints, nothing indexable, saves crawl budget).
#
# Note: a bot matching a named User-agent group below ignores the * group,
# so any Disallow must be repeated per group. We keep named groups only
# for bots we block outright.

User-agent: *
Allow: /
Disallow: /api/

# ────────────────────────────────────────
# BLOCK SEO SCRAPERS & LINK HARVESTERS
# ────────────────────────────────────────
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# ────────────────────────────────────────
# SITEMAP
# ────────────────────────────────────────
Sitemap: https://untangle.bio/sitemap.xml