# Generated by aem-free-waf. Edit waf-config.yaml, not this file. # Rights reservation: text and data mining of this site's content for AI training is # expressly reserved under Art. 4(3) of Directive (EU) 2019/790 (EU DSM Directive). # Everyone else: normal crawling, with stated AI preferences and AEM crawl hygiene User-agent: * Allow: / Allow: /etc.clientlibs/ Disallow: /bin/ Disallow: /libs/ Disallow: /apps/ Disallow: /conf/ Disallow: /etc/ Disallow: /home/ Disallow: /var/ Disallow: /tmp/ Disallow: /crx/ Disallow: /system/ Disallow: /*?q= Disallow: /*&q= Disallow: /*?search= Disallow: /*&search= Disallow: /*?sort= Disallow: /*&sort= Disallow: /*?filter= Disallow: /*&filter= Disallow: /*?facet= Disallow: /*&facet= Content-Signal: search=yes, ai-input=yes, ai-train=no Content-Usage: train-ai=n # Training opt-out tokens: these never crawl, and search visibility is unaffected User-agent: __usage_ai-training User-agent: Applebot-Extended User-agent: Google-Extended Disallow: / # AI crawlers the CDN policy blocks (source: ai.robots.txt) User-agent: AgentTimes User-agent: AI2Bot User-agent: Ai2Bot-Dolma User-agent: aiHitBot User-agent: anthropic-ai User-agent: Awario User-agent: bedrockbot User-agent: Brightbot User-agent: Bytespider User-agent: CCBot User-agent: ChatGLM-Spider User-agent: ClaudeBot User-agent: CloudVertexBot User-agent: cohere-training-data-crawler User-agent: Cotoyogi User-agent: Datenbank User-agent: DeepSeekBot User-agent: Echobot User-agent: ElevenlabsCrawler User-agent: ERNIEBot User-agent: FacebookBot User-agent: Factset_spyderbot User-agent: FriendlyCrawler User-agent: GoogleOther User-agent: GoogleOther-Image User-agent: GoogleOther-Video User-agent: GPTBot User-agent: Hunyuan User-agent: ICC-Crawler User-agent: ImagesiftBot User-agent: imageSpider User-agent: img2dataset User-agent: ISSCyberRiskCrawler User-agent: Kangaroo User-agent: KimiBot User-agent: laion-huggingface-processor User-agent: LAIONDownloader User-agent: LCC User-agent: Lightpanda User-agent: meta-externalagent User-agent: MistralAI-Training User-agent: MoonshotBot User-agent: MyCentralAIScraperBot User-agent: NagetBot User-agent: netEstate User-agent: newsai User-agent: PanguBot User-agent: panscient User-agent: QwenBot User-agent: Scrapy User-agent: Sidetrade User-agent: Spider User-agent: TikTokSpider User-agent: Timpibot User-agent: VelenPublicWebCrawler User-agent: WARDBot User-agent: webzio-extended User-agent: YandexAdditional User-agent: YandexAdditionalBot User-agent: YiBot Disallow: / # Sitemap: set robots.sitemaps in waf-config.yaml (absolute URL)