Direct-fireplaces.com - SEO Checker

Meta tags

Media list

Heading structure

Found links on this page

HTTP Response Header

Links from Wikipedia

Robots.txt

User-agent: *
Allow: /

#######################################
##### MAGENTO DIRECTORIES & FILES #####
#######################################

##### Directories #####

# Disallow: /catalog/
Disallow: /*.php$
Disallow: /*SID=
Disallow: /app/
Disallow: /checkout/
Disallow: /customer/
Disallow: /directory/
Disallow: /elasticsuite/tracker/
Disallow: /index.php/
Disallow: /lib/
Disallow: /media/css_secure/
Disallow: /page_cache/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /rest/
Disallow: /review/
# Disallow: /static/*.html
Disallow: /var/

##### Paths (clean URLs) #####

# Disallow: /catalog/category/view/
# Disallow: /catalog/product_compare/
# Disallow: /catalog/product/gallery/
# Disallow: /catalog/product/view/
Disallow: /cart.php
Disallow: /catalogsearch/
Disallow: /checkout/
Disallow: /control/
Disallow: /currency/
Disallow: /customer/
Disallow: /customize/
Disallow: /index.php/
Disallow: /poll/
Disallow: /review/
Disallow: /rss.php
Disallow: /sendfriend/
Disallow: /tag/
Disallow: /wishlist.php
Disallow: /wishlist/

Disallow: /estimateddeliverydate/message/update
Disallow: /pquestiondeferrer/rating/getmetadatamultiple
Disallow: /restriction/message/message
Disallow: /apptrian_facebookpixel/index/index
Disallow: /aw_advancedreports/countViews/

##### Files #####

Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt

#######################################
######## QUERY STRING BLOCKER #########
#######################################

# Magento Parameters
Disallow: *order=*
Disallow: *mode=*
Disallow: *limit=*
Disallow: *dir=*
Disallow: *q=*
Disallow: *price=*
Disallow: *sendfriend*
Disallow: *?awc=*
Disallow: *?source=*
Disallow: *?utm_source=*
Disallow: *?platform=*
Disallow: *?PageSpeed=*
Disallow: *?gclid=*
Disallow: *?product_list_dir=*
Disallow: */mwdownloads*
Disallow: *___store=*

# Other Parameters

Disallow: *control_type=*
Disallow: *ef_heat_output=*
Disallow: *ef_heater_type=*
Disallow: *finish_options=*
Disallow: *fire_type=*
Disallow: *flame_effect=*
Disallow: *fp_style=*
Disallow: *fuel_bed_options=*
Disallow: *guaruntee=*
Disallow: *mantel_width_multiselect=*
Disallow: *manufacturer=*
Disallow: *price=*
Disallow: *price_set=*
Disallow: *product_list_dir=*
Disallow: *product_list_order=*
Disallow: *recess_required=*
Disallow: *suitable_for=*
Disallow: *brand=*
Disallow: *cat=*
Disallow: *control=*
Disallow: *control_type=*
Disallow: *energy_rating=*
Disallow: *external_air_compatible=*
Disallow: *external_combustion_kit=*
Disallow: *finish=*
Disallow: *flame_effect=*
Disallow: *flue_diameter=*
Disallow: *flue_parts=*
Disallow: *fuel_bed=*
Disallow: *gas_options=*
Disallow: *grade=*
Disallow: *output=*
Disallow: *overall_depth=*
Disallow: *overall_height=*
Disallow: *overall_width=*
Disallow: *radiators_filter=*
Disallow: *range=*
Disallow: *source=*
Disallow: *ss_grade=*
Disallow: *suitable_fire_type=*
Disallow: *thermostat=*
Disallow: *s=*
Disallow: *air_vent=*
Disallow: *chimney_type=*
Disallow: *flame_effect=*
Disallow: *fpcolor_filter=*
Disallow: *front_type=*
Disallow: *heat_type=*
Disallow: *safety=*
Disallow: *tv_sizes=*
Disallow: *ef_heat_output=*
Disallow: *est_delivery=*
Disallow: *finish_options=*
Disallow: *fire_type=*
Disallow: *gf_heat_output=*
Disallow: *mantel_width_multiselect=*
Disallow: *manufacturer=*

# Product Parameters
Disallow: *%5B*%5D=*
Disallow: *[*]=*

#######################################
#### WORDPRESS DIRECTORIES & FILES ####
#######################################

Disallow: /resources/wp-content/upgrade/
Disallow: /resources/wp-admin/
Disallow: /resources/wp-includes/
Disallow: /resources/author/admin/
Disallow: *fullsite=*
Disallow: *action=*
Disallow: *post_id*
Disallow: *disabled=*
Disallow: *is_comment=*

# Product Parameters
Disallow: *%5B*%5D=*
Disallow: *[*]=*
# Other Parameters
Disallow: *order=*
Disallow: *mode=*
Disallow: *limit=*
Disallow: *dir=*
Disallow: *q=*
Disallow: *price=*
Disallow: *sendfriend*
Disallow: *?awc=*
Disallow: *?source=*
Disallow: *?utm_source=*
Disallow: *?platform=*
Disallow: *?PageSpeed=*
Disallow: *?gclid=*
Disallow: *?product_list_dir=*
Disallow: */mwdownloads*
Disallow: *control_type=*
Disallow: *ef_heat_output=*
Disallow: *ef_heater_type=*
Disallow: *finish_options=*
Disallow: *fire_type=*
Disallow: *flame_effect=*
Disallow: *fp_style=*
Disallow: *fuel_bed_options=*
Disallow: *guaruntee=*
Disallow: *mantel_width_multiselect=*
Disallow: *manufacturer=*
Disallow: *price=*
Disallow: *price_set=*
Disallow: *product_list_dir=*
Disallow: *product_list_order=*
Disallow: *recess_required=*
Disallow: *suitable_for=*
Disallow: *brand=*
Disallow: *cat=*
Disallow: *control=*
Disallow: *control_type=*
Disallow: *energy_rating=*
Disallow: *external_air_compatible=*
Disallow: *external_combustion_kit=*
Disallow: *finish=*
Disallow: *flame_effect=*
Disallow: *flue_diameter=*
Disallow: *flue_parts=*
Disallow: *fuel_bed=*
Disallow: *gas_options=*
Disallow: *grade=*
Disallow: *output=*
Disallow: *overall_depth=*
Disallow: *overall_height=*
Disallow: *overall_width=*
Disallow: *radiators_filter=*
Disallow: *range=*
Disallow: *source=*
Disallow: *ss_grade=*
Disallow: *suitable_fire_type=*
Disallow: *thermostat=*
Disallow: *s=*
Disallow: *air_vent=*
Disallow: *chimney_type=*
Disallow: *flame_effect=*
Disallow: *fpcolor_filter=*
Disallow: *front_type=*
Disallow: *heat_type=*
Disallow: *safety=*
Disallow: *tv_sizes=*
Disallow: *ef_heat_output=*
Disallow: *est_delivery=*
Disallow: *finish_options=*
Disallow: *fire_type=*
Disallow: *gf_heat_output=*
Disallow: *mantel_width_multiselect=*
Disallow: *manufacturer=*

User-agent: adidxbot
Allow: /

# --- AI & Preview Crawlers: explicit allow-list -----------------
# IMPORTANT: This opts in to some model-training programmes (see notes).
# Flip 'Allow: /' to 'Disallow: /' on any group you DON'T want to permit.

# OpenAI (training + browsing + user-initiated fetches)
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ChatGPT-User/2.0
Allow: /

# Anthropic / Claude (browsing + user fetches + search bot)
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
Allow: /
# Anthropic training crawler (opt-in here)
User-agent: anthropic-ai
Allow: /

# Perplexity (crawler + user-initiated fetcher)
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /

# Google AI training control (not a classic crawler)
User-agent: Google-Extended
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# Apple (classic + training opt-in)
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /

# Meta (Facebook/Instagram link previews + AI fetcher)
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: Meta-ExternalAgent
Allow: /

# LinkedIn link previews
User-agent: LinkedInBot
Allow: /

# ByteDance (TikTok) crawler
User-agent: Bytespider
Allow: /

# DuckDuckGo DuckAssist
User-agent: DuckAssistBot
Allow: /

# Cohere (public UA is not formally documented; harmless to include)
User-agent: cohere-ai
Allow: /

# Open datasets / research crawlers often used for AI training
User-agent: CCBot
User-agent: AI2Bot
User-agent: SemanticScholarBot
User-agent: Diffbot
User-agent: omgili
User-agent: omgilibot
Allow: /

# Smaller AI search engines
User-agent: YouBot
User-agent: Timpibot
User-agent: TimpiBot
Allow: /

user agent: Pinterestbot
Crawl delay: 1
# ---------------------------------------------------------------

Sitemap: https://www.direct-fireplaces.com/pub/media/direct_fireplaces_sitemap_additional_link.xml
Sitemap: https://www.direct-fireplaces.com/pub/media/direct_fireplaces_sitemap_category.xml
Sitemap: https://www.direct-fireplaces.com/pub/media/direct_fireplaces_sitemap_product.xml
Sitemap: https://www.direct-fireplaces.com/pub/media/direct_fireplaces_sitemap_other.xml
Sitemap: https://www.direct-fireplaces.com/pub/media/direct_fireplaces_sitemap_cms.xml

Search preview