Tutorcare.co.uk - SEO Checker

Meta tags

Media list

Heading structure

Found links on this page

HTTP Response Header

Links from Wikipedia

Robots.txt

# robots.txt for TutorCare / WordPress
# Purpose: keep public SEO pages crawlable while reducing crawl waste from admin,
# search, feeds, duplicate/comment URLs, generated cache/backup/log files, and
# non-essential aggressive crawlers.
# Last updated: 2026-08-02

# -----------------------------------------------------------------------------
# Main rule set for search engines
# -----------------------------------------------------------------------------
User-agent: *

# WordPress admin and login areas
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /wp-cron.php

# WordPress-generated duplicate or low-value crawl paths
Disallow: /search/
Disallow: /*?s=
Disallow: /*&s=
Disallow: /*?replytocom=
Disallow: /*&replytocom=
Disallow: */trackback/
Disallow: */feed/
Disallow: /comments/feed/

# REST/API endpoints that are usually not useful as standalone search results
# Remove these rules if front-end rendering, apps, integrations, or plugins need
# crawlers to access these endpoints.
Disallow: /wp-json/wp/v2/users
Disallow: /wp-json/oembed/
Disallow: /*?rest_route=

# Generated, temporary, backup, cache, log, and diagnostic files/directories
Disallow: /readme.html
Disallow: /license.txt
Disallow: /wp-content/debug.log
Disallow: /wp-content/cache/
Disallow: /wp-content/upgrade/
Disallow: /wp-content/backups/
Disallow: /wp-content/backup/
Disallow: /wp-content/uploads/backups/
Disallow: /wp-content/uploads/backup/
Disallow: /wp-content/uploads/wc-logs/
Disallow: /wp-content/uploads/wpo-plugins-tables-list.json

# Do not index markdown pages for AI scrapers
Disallow: /*.md$

# Keep render-critical assets crawlable.
# Do not block /wp-content/themes/, /wp-content/plugins/, /wp-includes/, CSS, JS,
# fonts, or normal images unless a specific file is confirmed non-essential.
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/
Allow: /wp-includes/

# Sitemap
Sitemap: https://tutorcare.co.uk/sitemap_index.xml

# -----------------------------------------------------------------------------
# Optional anti-scraper / non-essential crawler opt-outs
# Note: robots.txt is advisory. Malicious bots often ignore it. Enforce with WAF,
# server rules, CDN bot management, or rate limiting when traffic is abusive.
# -----------------------------------------------------------------------------

# SEO/intelligence crawlers often not needed unless the client actively uses them
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SerpstatBot
Disallow: /

# AI training / large-scale content harvesting crawlers
# Remove or modify if the client wants visibility in specific AI/search products.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PerplexityBot
Disallow: /

# Additional high-volume international crawlers; block only if these markets are
# not strategically relevant and logs show crawl waste or abuse.
User-agent: PetalBot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: 360Spider
Disallow: /

Search preview