# dona.org robots.txt # Updated: April 14, 2026 # Purpose: Block abusive bots and rate-limit AI crawlers # -------------------------------------------------- # BLOCK: Abusive bots consuming excessive resources # -------------------------------------------------- # DataForSeoBot - #1 offender, 4,192 requests in 2 days # Scraping /find-a-doula/ pages relentlessly, all cache misses User-agent: DataForSeoBot Disallow: / # MJ12bot (Majestic SEO crawler) User-agent: MJ12bot Disallow: / # BrokenLinksBot - third-party link checker User-agent: BrokenLinksBot Disallow: / # FindFiles.net - burst-crawled 212 pages in a single hour User-agent: findfiles Disallow: / # AhrefsBot - 4,543 requests on April 14 alone, new #1 offender User-agent: AhrefsBot Disallow: / # -------------------------------------------------- # RATE-LIMIT: AI bots (crawl-delay in seconds) # -------------------------------------------------- User-agent: ChatGPT-User Crawl-delay: 10 Disallow: /wp-admin/ Disallow: /wp-login.php User-agent: PerplexityBot Crawl-delay: 10 Disallow: /wp-admin/ Disallow: /wp-login.php User-agent: YouBot Crawl-delay: 10 Disallow: /wp-admin/ Disallow: /wp-login.php User-agent: TikTokSpider Crawl-delay: 10 # -------------------------------------------------- # STANDARD: Allow legitimate search engines # -------------------------------------------------- User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / # Baidu - crawling old/removed pages, generating 404s # Block unless Chinese traffic matters User-agent: Baiduspider Disallow: / # SemrushBot - SEO crawler, moderate volume User-agent: SemrushBot Crawl-delay: 5 # -------------------------------------------------- # DEFAULT: All other bots # -------------------------------------------------- User-agent: * Disallow: /wp-admin/ Disallow: /wp-login.php Disallow: /xmlrpc.php Allow: /wp-admin/admin-ajax.php Sitemap: https://dona.org/sitemap_index.xml
Se han encontrado las siguientes palabras clave. Comprueba si esta página está bien optimizada para cada palabra clave en concreto.
(Deseable)