User-agent: * Sitemap: https://thegreatmerch.com/sitemap.xml ############## # custom # ############## # =========================================== # robots.txt for thegreatmerch.com # =========================================== # Named user-agent blocks override the wildcard block entirely. # Any rule that should apply to Googlebot or Bingbot must be # repeated in their own sections. # ------------------------------------------- # ------------------------------------------- # SEARCH ENGINES - Full access # Search results, sorting/filter URLs and pagination blocked to save # crawl budget and server load; products found via sitemap # ------------------------------------------- User-agent: Googlebot Allow: / Disallow: /website/search Disallow: /*?search= Disallow: /*&search= Disallow: /shop/page/ Disallow: /shop/category/*/page/ Disallow: /*?page= Disallow: /*&page= Disallow: /*?order= Disallow: /*&order= Disallow: /*?attribute_value= Disallow: /*&attribute_value= User-agent: Googlebot-Image Allow: / User-agent: Googlebot-News Allow: / User-agent: AdsBot-Google Allow: / User-agent: Bingbot Allow: / Disallow: /website/search Disallow: /*?search= Disallow: /*&search= Disallow: /shop/page/ Disallow: /shop/category/*/page/ Disallow: /*?page= Disallow: /*&page= Disallow: /*?order= Disallow: /*&order= Disallow: /*?attribute_value= Disallow: /*&attribute_value= User-agent: Applebot Allow: / Disallow: /website/search Disallow: /*?search= Disallow: /*&search= Disallow: /shop/page/ Disallow: /shop/category/*/page/ Disallow: /*?page= Disallow: /*&page= Disallow: /*?order= Disallow: /*&order= Disallow: /*?attribute_value= Disallow: /*&attribute_value= User-agent: Qwantify Allow: / Disallow: /website/search Disallow: /*?search= Disallow: /*&search= # ------------------------------------------- # SOCIAL MEDIA CRAWLERS - Allow for link previews # ------------------------------------------- User-agent: facebookexternalhit Allow: / User-agent: Twitterbot Allow: / User-agent: LinkedInBot Allow: / User-agent: Pinterest Allow: / # ------------------------------------------- # SEO TOOLS - Throttle (not needed for business) # One group with several User-agent lines shares these rules # ------------------------------------------- User-agent: AhrefsBot User-agent: SemrushBot User-agent: DotBot Crawl-delay: 60 Disallow: /website/search Disallow: /*?search= Disallow: /*&search= Disallow: /shop/page/ Disallow: /shop/category/*/page/ Disallow: /*?page= Disallow: /*&page= Disallow: /*?order= Disallow: /*&order= Disallow: /*?attribute_value= Disallow: /*&attribute_value= # ------------------------------------------- # BLOCKED CRAWLERS - No value, waste resources # ------------------------------------------- User-agent: MJ12bot User-agent: Bytespider User-agent: PetalBot User-agent: YandexBot Disallow: / # ------------------------------------------- # DEFAULT RULES - All other bots # (includes AI search bots: OAI-SearchBot, ChatGPT-User, # PerplexityBot, Claude-SearchBot, meta-externalagent) # ------------------------------------------- User-agent: * Allow: / Allow: /shop/ Allow: /web/static/ Allow: /web/css/ Allow: /web/js/ Allow: /web/image/ Allow: /favicon Allow: /favicon.ico Allow: /*.ico$ Allow: /web/image/website/*/favicon Allow: /*/favicon* Disallow: /web/database/ Disallow: /web/login Disallow: /web/session/ Disallow: /web/webclient/ Disallow: /web/action/ Disallow: /my/ Disallow: /admin/ Disallow: /private/ Disallow: /temp/ Disallow: /website/search Disallow: /*?search= Disallow: /*&search= Disallow: /shop/page/ Disallow: /shop/category/*/page/ Disallow: /*?page= Disallow: /*&page= Disallow: /*?order= Disallow: /*&order= Disallow: /*?attribute_value= Disallow: /*&attribute_value= # ------------------------------------------- # SITEMAP # ------------------------------------------- Sitemap: https://thegreatmerch.com/sitemap.xml