Stadtwerke-hilden.de - SEO Checker

Meta tags

Media list

Heading structure

Found links on this page

HTTP Response Header

Links from Wikipedia

Robots.txt

robots.txt generated by MKH 052026
User-agent: *
Disallow: 
Disallow: /cgi-bin/

# === Explicitly allowed AI bots ===
# GPTBot (OpenAI)
User-agent: GPTBot
Allow: /

# ChatGPT-User (OpenAI)
User-agent: ChatGPT-User
Allow: /

# OAI-SearchBot (OpenAI)
User-agent: OAI-SearchBot
Allow: /

# ClaudeBot (Anthropic)
User-agent: ClaudeBot
Allow: /

# PerplexityBot (Perplexity AI)
User-agent: PerplexityBot
Allow: /

# Google-Extended (Google)
User-agent: Google-Extended
Allow: /

# Gemini (Google)
User-agent: Google-InspectionTool
Allow: /

# Bytespider (ByteDance)
User-agent: Bytespider
Allow: /

# CCBot (Common Crawl)
User-agent: CCBot/2.0 (https://commoncrawl.org/faq/)
Allow: /

# Amazonbot (Amazon)
User-agent: Amazonbot
Allow: /

# AppleBot-Extended (Apple)
User-agent: Applebot-Extended
Allow: /

# FacebookBot (Meta)
User-agent: FacebookBot
Allow: /

# Meta-ExternalAgent (Meta)
User-agent: Meta-ExternalAgent
Allow: /

# cohere-ai (Cohere)
User-agent: cohere-ai
Allow: /

# YouBot (You.com)
User-agent: YouBot
Allow: /

# Diffbot (Diffbot)
User-agent: Diffbot
Allow: /

# PetalBot (Huawei (Petal Search))
User-agent: PetalBot
Allow: /

# Barkrowler (Babbar)
User-agent: Barkrowler
Allow: /

# Timpibot (Timpi)
User-agent: Timpibot
Allow: /

# Seekr (Seekr Technologies)
User-agent: Seekr
Allow: /

# Kangaroo Bot (Kangaroo LTD)
User-agent: Kangaroo
Allow: /

# Velenpublicwebcrawler (Velen)
User-agent: Velenpublicwebcrawler
Allow: /

# Omgili (Omgili (Webz.io))
User-agent: omgili
Allow: /

# ICC-Crawler (NICT (Japan))
User-agent: ICC-Crawler
Allow: /

# Brightbot (Bright Data)
User-agent: BrightBot
Allow: /

# Scrapy (Open Source (Zyte))
User-agent: Scrapy/2.11 (+https://scrapy.org)
Allow: /

# xAI-Bot (xAI (Elon Musk))
User-agent: xAI-Bot
Allow: /

# DuckAssistBot (DuckDuckGo)
User-agent: DuckAssistBot
Allow: /

# Bingbot (Microsoft)
User-agent: bingbot
Allow: /

# Ai2Bot (Allen Institute for AI (Ai2))
User-agent: Ai2Bot
Allow: /

# MistralBot (Mistral AI)
User-agent: MistralBot
Allow: /

# Googlebot (Google)
User-agent: Googlebot
Allow: /

# Applebot (Apple)
User-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)
Allow: /

# AhrefsBot (Ahrefs)
User-agent: AhrefsBot
Allow: /

# SemrushBot (Semrush)
User-agent: SemrushBot
Allow: /

# YandexBot (Yandex)
User-agent: YandexBot
Allow: /

# Baiduspider (Baidu)
User-agent: Baiduspider
Allow: /

# BraveBot (Brave)
User-agent: BraveBot
Allow: /

# NaverBot (Naver)
User-agent: Yeti
Allow: /

# HuggingFaceBot (Hugging Face)
User-agent: HuggingFaceBot
Allow: /

# DuckDuckBot (DuckDuckGo)
User-agent: DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)
Allow: /

# LinkedInBot (LinkedIn (Microsoft))
User-agent: Mozilla
Allow: /

# archive.org_bot (Internet Archive)
User-agent: archive.org_bot
Allow: /

# TurnitinBot (Turnitin)
User-agent: TurnitinBot/3.0 (https://turnitin.com/robot/crawlerinfo.html)
Allow: /

# iaskspider (iAsk.ai)
User-agent: iaskspider/1.0 (+https://iask.ai/)
Allow: /

# Sogou (Tencent)
User-agent: Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)
Allow: /

# DataForSeoBot (DataForSEO)
User-agent: DataForSeoBot
Allow: /

# MJ12bot (Majestic)
User-agent: MJ12bot/v1.4.8 (http://majestic.com/research/majestic-12?V=1.4.8)
Allow: /

# rogerbot (Moz)
User-agent: rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot- +http://moz.com/help/pro/what-is-rogerbot)
Allow: /

# DeepSeekBot (DeepSeek)
User-agent: DeepSeekBot
Allow: /

# Firecrawl (Mendable / Firecrawl)
User-agent: Firecrawl/1.0 (+https://firecrawl.dev)
Allow: /

# JinaBot (Jina AI)
User-agent: JinaBot
Allow: /

# ExaBot (Exa AI)
User-agent: Exa-Search
Allow: /

# MojeekBot (Mojeek)
User-agent: MojeekBot/0.11 (https://www.mojeek.com/bot.html)
Allow: /

# ApifyBot (Apify)
User-agent: ApifyBot
Allow: /

# QwenBot (Alibaba Cloud)
User-agent: QwenBot
Allow: /

# YandexGPT (Yandex)
User-agent: YandexGPT
Allow: /

# img2dataset (HuggingFace / Community)
User-agent: img2dataset
Allow: /

# NewsPlease (Community / Various)
User-agent: news-please/1.5
Allow: /

# Slurp (Yahoo / Verizon Media)
User-agent: Slurp
Allow: /

# Twitterbot (X (formerly Twitter))
User-agent: Twitterbot
Allow: /

# Default — allow all other crawlers
User-agent: *
Allow: /

Search preview