# 12andus.com robots.txt

# OLD RULES (commented out for social sharing - 2025-11-17):
# Allow only home page and blog - everything else requires login
# All bots: Block everything except home and blog
# User-agent: *
# Disallow: /
# Allow: /$
# Allow: /blog/

# NEW RULES (for social sharing):
# Explicitly allow Facebook bot (they cache robots.txt for 24h)
User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

# Allow all other bots
User-agent: *
Allow: /
Disallow: /*?*view=rss
Disallow: /action/
# Malformed pagination crawler trap: an '&' directly after the path (no '?')
# produced infinitely-accumulating /blog/all&offset=...&offset=... URLs that 404
# (legacy pagination bug, fixed in code). Real pagination uses /blog/all?offset=
# and is unaffected. See Search Console "Not found (404)" ~1.86M.
Disallow: /blog/all&
Disallow: /blog/all/&
# Edit pages are owner/admin-only and 302-redirect logged-out visitors to
# /login. Googlebot was discovering /blog/edit/<guid> via a leaked entity-menu
# link on every post and crawling ~183K of them ("Page with redirect").
# Block them so crawl budget goes to real content. Also see the entity-menu
# access gate in object/blog.php.
Disallow: /blog/edit
# POST-only social-share tracking endpoint — a crawler GET returns 405
# ("Blocked due to other 4xx issue"). Not a page; block to keep it out of crawl.
Disallow: /track-social
# Internal search-results pages (/search?q=…&search_type=…) — Google flags these
# as Soft 404 and best practice is to keep internal search results out of the
# index. Blocking stops the crawl waste; on-site search is unaffected for users.
Disallow: /search
# Internal file/gallery listing (/file/search?subtype=…&offset=…) — query-string
# listing pages that seeded a ~1.84M-URL lang-permutation 404 trap (now fixed in
# head.php: hreflang block removed, picker is rel=nofollow). Same internal-listing
# category as /search; block to stop crawl waste on the seed pages. Already-known
# URLs age out on Google's own cadence either way.
Disallow: /file/search

# Sitemap (if you have one)
Sitemap: https://12andus.com/sitemap.xml