# ===================================== # GLOBAL PERFORMANCE-FRIENDLY ROBOTS.TXT # ===================================== User-agent: * Disallow: /cgi-bin/ Crawl-delay: 10 # ===================================== # MAJOR SEARCH ENGINES (Allow, Gentle Delay) # ===================================== User-agent: Googlebot Crawl-delay: 15 Disallow: User-agent: Googlebot-Image Crawl-delay: 15 Disallow: User-agent: Bingbot Crawl-delay: 15 Disallow: User-agent: Slurp Crawl-delay: 15 Disallow: User-agent: DuckDuckBot Crawl-delay: 15 Disallow: User-agent: Baiduspider Disallow: / User-agent: Applebot Crawl-delay: 15 Disallow: # ===================================== # ANALYTICS, SEO, SCRAPERS, & OTHER HEAVY BOTS # ===================================== User-agent: AhrefsBot Disallow: / Crawl-delay: 30 User-agent: SemrushBot Disallow: / Crawl-delay: 30 User-agent: DotBot Disallow: / Crawl-delay: 30 User-agent: MJ12bot Disallow: / Crawl-delay: 30 User-agent: BLEXBot Disallow: / Crawl-delay: 30 User-agent: Yandex Disallow: / Crawl-delay: 30 User-agent: PetalBot Disallow: / Crawl-delay: 30 User-agent: CCBot Disallow: / Crawl-delay: 30 User-agent: MauiBot Disallow: / Crawl-delay: 30 User-agent: SeznamBot Disallow: / Crawl-delay: 30 User-agent: Exabot Disallow: / Crawl-delay: 30 User-agent: GrapeshotCrawler Disallow: / Crawl-delay: 30 User-agent: Wget Disallow: / User-agent: curl Disallow: / User-agent: python-requests Disallow: / User-agent: python Disallow: / User-agent: libwww-perl Disallow: / User-agent: HTTrack Disallow: / User-agent: proximic Disallow: / User-agent: Screaming Frog SEO Spider Disallow: / User-agent: Sogou Disallow: / User-agent: MegaIndex Disallow: / User-agent: AspiegelBot Disallow: / User-agent: SiteAuditBot Disallow: / User-agent: linkdexbot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: CensysInspect Disallow: / User-agent: ZoominfoBot Disallow: / User-agent: Bytespider Disallow: / User-agent: GPTBot Disallow: / # ===================================== # SOCIAL & PREVIEW BOTS (Allow, Gentle Delay) # ===================================== User-agent: Twitterbot Crawl-delay: 5 Disallow: User-agent: facebookexternalhit Crawl-delay: 5 Disallow: User-agent: LinkedInBot Crawl-delay: 5 Disallow: User-agent: Pinterest Crawl-delay: 5 Disallow:
Folgende Keywords wurden erkannt. Überprüfe die Optimierung dieser Keywords für Deine Seite.
(Nice to have)