# NameHippo robots.txt
#
# AEO-FRIENDLY VERSION (2026-10-15)
#
# CHANGE LOG
# - 2026-10-15: FIXED a grouping bug. A named User-agent group REPLACES the
# `*` group wholesale; it does not inherit from it. The previous file gave
# AhrefsBot/SemrushBot/MJ12bot a group holding only Crawl-delay, and each
# AI crawler a group holding only `Allow: /`. That silently lifted every
# private-area block for all of them, and they were crawling /checkout/,
# /admin, /dashboard, /api/ and /login. On 15 Oct 2026 alone, 187 requests
# walked /checkout/5 .. /checkout/225 across 79 domain IDs from 57 IPs.
# Every group now carries the full Disallow list.
# - 2026-05-20: Removed all AI crawler blocks (GPTBot, ClaudeBot, CCBot,
# Google-Extended). AI assistants may crawl, index and cite inventory.
# - 2026-05-20: Added /llms.txt and /llms-full.txt references.
# - Earlier: Removed `Disallow: /*?*`; replaced with explicit private blocks.
#
# - Sitemap is the source of truth for what we *want* indexed.
#
# MAINTENANCE RULE: rules are NOT inherited between groups. If you add a
# User-agent block, copy the Disallow list into it, or that bot gets the
# run of the site.
User-agent: *
# Private areas - never crawl
Disallow: /admin
Disallow: /admin/
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /account
Disallow: /account/
Disallow: /checkout
Disallow: /checkout/
Disallow: /transfer
Disallow: /transfer/
Disallow: /seller
Disallow: /seller/
Disallow: /api/
Disallow: /webhook/
Disallow: /cron/
Disallow: /_tools/
Disallow: /_tls/
# UX-only pages that shouldn't dilute the index
# NH 2026-10-23: /search is intentionally NOT blocked.
# The page already sends .
# Blocking it here prevented crawlers from ever READING that directive, so
# any /search URL already in the index could never be removed. Allow the
# crawl, let the noindex be seen and honoured, then re-block later if needed.
Disallow: /explore
Disallow: /explore/
Disallow: /compare
Disallow: /favorites
Disallow: /shortlist
Disallow: /verify-otp
Disallow: /verify-link
Disallow: /verify-email
Disallow: /login-otp
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /login
Disallow: /register
Disallow: /logout
# Static assets (be explicit so paranoid bots don't skip them)
Allow: /favicon.ico
Allow: /favicon-16x16.png
Allow: /favicon-32x32.png
Allow: /apple-touch-icon.png
Allow: /android-chrome-192x192.png
Allow: /android-chrome-512x512.png
Allow: /site.webmanifest
Allow: /robots.txt
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /sitemap-domains.xml
Allow: /sitemap-pages.xml
Allow: /sitemap-blog.xml
Allow: /sitemap-industries.xml
Allow: /sitemap-keywords.xml
Allow: /assets/
# ---------------------------------------------------------------------------
# AI crawlers - welcome on every public page, shut out of private areas.
# `Allow: /` plus the Disallow list: the longest matching rule wins, so each
# listing, category and article stays open while /admin and /checkout stay shut.
# See /llms.txt and /llms-full.txt for structured site context.
# ---------------------------------------------------------------------------
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: cohere-ai
User-agent: Meta-ExternalAgent
User-agent: Bytespider
User-agent: YouBot
User-agent: Diffbot
Allow: /
# Private areas - never crawl
Disallow: /admin
Disallow: /admin/
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /account
Disallow: /account/
Disallow: /checkout
Disallow: /checkout/
Disallow: /transfer
Disallow: /transfer/
Disallow: /seller
Disallow: /seller/
Disallow: /api/
Disallow: /webhook/
Disallow: /cron/
Disallow: /_tools/
Disallow: /_tls/
# UX-only pages that shouldn't dilute the index
# NH 2026-10-23: /search is intentionally NOT blocked.
# The page already sends .
# Blocking it here prevented crawlers from ever READING that directive, so
# any /search URL already in the index could never be removed. Allow the
# crawl, let the noindex be seen and honoured, then re-block later if needed.
Disallow: /explore
Disallow: /explore/
Disallow: /compare
Disallow: /favorites
Disallow: /shortlist
Disallow: /verify-otp
Disallow: /verify-link
Disallow: /verify-email
Disallow: /login-otp
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /login
Disallow: /register
Disallow: /logout
# ---------------------------------------------------------------------------
# SEO / backlink crawlers - paced, and held to the same private-area blocks.
# ---------------------------------------------------------------------------
User-agent: AhrefsBot
User-agent: SemrushBot
Crawl-delay: 10
# Private areas - never crawl
Disallow: /admin
Disallow: /admin/
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /account
Disallow: /account/
Disallow: /checkout
Disallow: /checkout/
Disallow: /transfer
Disallow: /transfer/
Disallow: /seller
Disallow: /seller/
Disallow: /api/
Disallow: /webhook/
Disallow: /cron/
Disallow: /_tools/
Disallow: /_tls/
# UX-only pages that shouldn't dilute the index
# NH 2026-10-23: /search is intentionally NOT blocked.
# The page already sends .
# Blocking it here prevented crawlers from ever READING that directive, so
# any /search URL already in the index could never be removed. Allow the
# crawl, let the noindex be seen and honoured, then re-block later if needed.
Disallow: /explore
Disallow: /explore/
Disallow: /compare
Disallow: /favorites
Disallow: /shortlist
Disallow: /verify-otp
Disallow: /verify-link
Disallow: /verify-email
Disallow: /login-otp
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /login
Disallow: /register
Disallow: /logout
User-agent: MJ12bot
Crawl-delay: 20
# Private areas - never crawl
Disallow: /admin
Disallow: /admin/
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /account
Disallow: /account/
Disallow: /checkout
Disallow: /checkout/
Disallow: /transfer
Disallow: /transfer/
Disallow: /seller
Disallow: /seller/
Disallow: /api/
Disallow: /webhook/
Disallow: /cron/
Disallow: /_tools/
Disallow: /_tls/
# UX-only pages that shouldn't dilute the index
# NH 2026-10-23: /search is intentionally NOT blocked.
# The page already sends .
# Blocking it here prevented crawlers from ever READING that directive, so
# any /search URL already in the index could never be removed. Allow the
# crawl, let the noindex be seen and honoured, then re-block later if needed.
Disallow: /explore
Disallow: /explore/
Disallow: /compare
Disallow: /favorites
Disallow: /shortlist
Disallow: /verify-otp
Disallow: /verify-link
Disallow: /verify-email
Disallow: /login-otp
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /login
Disallow: /register
Disallow: /logout
# Sitemaps
Sitemap: https://namehippo.com/sitemap.xml
# Structured AI context
# /llms.txt - concise summary for AI citations
# /llms-full.txt - comprehensive AI reference with categories and pricing