Booster.io - SEO Check

Meta Tags

Medienliste

Überschriftenstruktur

Gefundene Links auf dieser Seite

HTTP-Header

Links von Wikipedia

Robots.txt

robots.txt update

# General rules for all bots
User-agent: *
Disallow: /wp-admin/ # Standard WordPress admin block
Disallow: /wp-content/uploads/wpforms/ # Your specific WPForms upload block
Disallow: /search # Your internal site search results
Disallow: /forums/users/ # Your forum user profiles
# Add any other general disallows for private areas, specific files, or query parameters if needed.
# E.g., Disallow: /checkout/
# E.g., Disallow: /my-account/
# E.g., Disallow: /*?s= (Alternative for site search if /search isn't the only path)

Allow: /wp-admin/admin-ajax.php # Standard WordPress AJAX allow

# --- Specific Search Engine Crawlers (Ensure broad access) ---
User-agent: Googlebot
Disallow: # Allow full access

User-agent: AdsBot-Google
Disallow: # Allow if you run Google Ads

User-agent: Bingbot
Disallow: # Allow full access

User-agent: Slurp
Disallow:

User-agent: DuckDuckBot
Disallow:

# --- LLM & AI Crawlers - Permissive with Stated Terms (via /license page) ---
User-agent: GPTBot 
Disallow: # Allowing OpenAI's web crawler

User-agent: Google-Extended 
Disallow: # Allowing Google's AI user agent

User-agent: ClaudeBot # Anthropic's crawler (monitor for actual user agent)
Disallow: # Allowing for now

User-agent: PerplexityBot 
Disallow: # Allowing Perplexity AI

User-agent: YouBot 
Disallow: # Allowing You.com

User-agent: ChatGPT-User # User-initiated browsing via ChatGPT
Disallow:

# --- Potentially More Restrictive or Bots to Watch ---
User-agent: CCBot # Common Crawl
Disallow: / # Consider disallowing to prevent use in broad, unattributed datasets

User-agent: Amazonbot 
Disallow: / # Consider disallowing if no benefit and high traffic/resource use

# Add other specific known problematic scrapers or LLM training bots you wish to block here.
# Example:
# User-agent: SomeNewAggressiveScraperBot
# Disallow: /

# --- Yoast SEO Sitemap ---
# Yoast typically adds its sitemap reference. If it doesn't add it automatically
# after these changes, or if you want to ensure it's present, you can keep it.
# If Yoast re-adds its entire block, you might need to adjust this structure
# or configure Yoast not to write to robots.txt if it causes conflicts.
# For now, we'll assume Yoast's sitemap line is sufficient if it adds it.
# If not, add your sitemap line here:
Sitemap: https://booster.io/sitemap_index.xml

# It's generally better to have only one User-agent: * block that is comprehensive,
# or ensure more specific user-agent blocks appear AFTER a general User-agent: *
# if you intend for them to be the primary rule for those specific bots.
# The structure above prioritizes specific bot instructions.

Suchvorschau