Capitalhairandbeauty.co.uk - SEO Checker

Meta tags

Media list

Heading structure

Found links on this page

HTTP Response Header

Links from Wikipedia

Robots.txt

#Blocked User Agents
User-agent: AhrefsBot
User-agent: Baiduspider-video
User-agent: Baiduspider-news
User-agent: Baiduspider-favo
User-agent: Baiduspider-cpro
User-agent: Baiduspider-ads
User-agent: MJ12bot
User-agent: BLEXBot
User-agent: Ezooms
User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 7.0) XSpider 7
User-agent: AspiegelBot
Disallow: /

#Rules for User Agents with 30 second crawl delay
User-agent: bingbot
User-agent: Yeti
User-agent: Baiduspider
User-agent: Baiduspider-image
User-agent: sogou spider
User-agent: Slurp China
User-agent: YandexBot
Crawl-delay: 30
Disallow: /*Basket.aspx
Disallow: /*Checkout.aspx 
Disallow: /*Login.aspx 
Disallow: /*checkout
Disallow: /*login
Disallow: /*basket
Disallow: /*?*filter=*
Disallow: /*?*donotindex=*
Disallow: /*?*sortorder=*
Disallow: /*?*listing_pagesize=*
Disallow: /students
Disallow: /*?*searchterm=*
Disallow: /*?*expand=*
Disallow: /leeds-only*
Disallow: */college-kits*
Disallow: /*?*productlist_add=*
Disallow: /my-account*

#Rules for User Agents with 1 second crawl delay
User-agent: Pinterestbot
Crawl-delay: 1
Disallow: /*Basket.aspx
Disallow: /*Checkout.aspx 
Disallow: /*Login.aspx 
Disallow: /*checkout
Disallow: /*login
Disallow: /*basket
Disallow: /*?*filter=*
Disallow: /*?*donotindex=*
Disallow: /*?*sortorder=*
Disallow: /*?*listing_pagesize=*
Disallow: /students
Disallow: /*?*searchterm=*
Disallow: /*?*expand=*
Disallow: /leeds-only*
Disallow: */college-kits*
Disallow: /*?*productlist_add=*
Disallow: /my-account*

#Rules for User Agents with 10 second crawl delay
User-agent: GeedoProductSearch
Crawl-delay: 10
Disallow: /*Basket.aspx
Disallow: /*Checkout.aspx 
Disallow: /*Login.aspx 
Disallow: /*checkout
Disallow: /*login
Disallow: /*basket
Disallow: /*?*filter=*
Disallow: /*?*donotindex=*
Disallow: /*?*sortorder=*
Disallow: /*?*listing_pagesize=*
Disallow: /students
Disallow: /*?*searchterm=*
Disallow: /*?*expand=*
Disallow: /leeds-only*
Disallow: */college-kits*
Disallow: /*?*productlist_add=*
Disallow: /my-account*

#Explicitly allow certain user agents, and also apply standard rules to them
User-agent: ScarabBot/1.0
User-agent: ScarabBot
Allow: /
Disallow: /*Basket.aspx
Disallow: /*Checkout.aspx 
Disallow: /*Login.aspx 
Disallow: /*checkout
Disallow: /*login
Disallow: /*basket
Disallow: /*?*filter=*
Disallow: /*?*donotindex=*
Disallow: /*?*sortorder=*
Disallow: /*?*listing_pagesize=*
Disallow: /students
Disallow: /*?*searchterm=*
Disallow: /*?*expand=*
Disallow: /leeds-only*
Disallow: */college-kits*
Disallow: /*?*productlist_add=*
Disallow: /my-account*

#Standard rules for any other user agent
User-agent: *
Disallow: /*Basket.aspx
Disallow: /*Checkout.aspx 
Disallow: /*Login.aspx 
Disallow: /*checkout
Disallow: /*login
Disallow: /*basket
Disallow: /*?*filter=*
Disallow: /*?*donotindex=*
Disallow: /*?*sortorder=*
Disallow: /*?*listing_pagesize=*
Disallow: /students
Disallow: /*?*searchterm=*
Disallow: /*?*expand=*
Disallow: /leeds-only*
Disallow: */college-kits*
Disallow: /*?*productlist_add=*
Disallow: /my-account*

Sitemap: https://www.capitalhairandbeauty.co.uk/sitemap_index.xml

Search preview