User-agent: * Allow: / # Block system & temp folders Disallow: /cgi-bin/ Disallow: /tmp/ Disallow: /private/ Disallow: /administrator/ # Block auth & user flows Disallow: /login/ Disallow: /register/ Disallow: /cart/ Disallow: /checkout/ # Block internal search & parameters Disallow: /search/ Disallow: /*?* Disallow: /*&* # Block spam / auto-generated files Disallow: /*.htm$ Disallow: /*.sei$ # Block fake folders created by bots Disallow: /page01 Disallow: /page03 Disallow: /net/ Disallow: /essitco.com/ Sitemap: https://www.essitco.com/sitemap.xml # ========================================== # Explicit rules for AI / ChatGPT-style crawlers # ========================================== User-agent: GPTBot Allow: / User-agent: CCBot Allow: / User-agent: PerplexityBot Allow: / User-agent: OpenAI Allow: / User-agent: Google-Extended Allow: / User-agent: GoogleOther Allow: / User-agent: Bingbot Allow: / User-agent: BingPreview Allow: / User-agent: MicrosoftCopilot Allow: / # ========================================== # Google Ads and Image Crawlers # ========================================== # Google Ads User-agent: AdsBot-Google Disallow: # Google Images User-agent: Googlebot-Image Disallow: