User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password/
Disallow: /index.php/user/register/
Disallow: /index.php/user/login/
Disallow: /index.php/user/logout/
Disallow: /offer/
# Combinatorial facet paths (crawler trap: unique uncacheable pages)
Disallow: /local-vanity-numbers/*/*
Disallow: /*-phone-numbers/*/*
Disallow: /*-area-code/*/*
Disallow: /on-sale/*/*
Disallow: /is-new/*/*
# AhrefsBot walks the generated preview images one URL at a time - 3665 requests
# and 0.8 GB in a day, 96% of that path's traffic, against 27 from Googlebot.
# Each URL is unique, so no cache ever helps and every one costs a PHP render.
# /img/ is NOT closed to everyone: it is the og:image on /phone-number/* and the
# Merchant Center product image (PNE-506), so Googlebot must keep fetching it.
# A named group REPLACES the * group for that crawler instead of adding to it,
# so the * disallows are repeated below. Do not drop them, and mirror any new
# * disallow here too or AhrefsBot walks straight past it.
User-agent: AhrefsBot
Disallow: /img/
Disallow: /admin/
Disallow: /search/
Disallow: /offer/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /local-vanity-numbers/*/*
Disallow: /*-phone-numbers/*/*
Disallow: /*-area-code/*/*
Disallow: /on-sale/*/*
Disallow: /is-new/*/*
# AI assistants: a curated index of this site is at /llms.txt, usage policy at /ai.txt
Sitemap: https://www.vanitynumber.com/sitemap.xml