This page is not available

This site can not be crawled by the Seobility bot. The robots.txt file restricts access for our bot. No URLs can be crawled that deny access to the user agent Seobility or *.

Problem URL: http://nextplatform.com

# =============================================================================
# robots.txt
# Last updated: 4 June 2026
#
# POLICY: Default-deny. Any crawler not explicitly allowed below is BLOCKED.
#   - Traditional search engines ............ ALLOWED
#   - AI search crawlers that cite + refer ... ALLOWED (conditional on attribution)
#   - Operational bots (ads, social, uptime) . ALLOWED
#   - AI training / scraping crawlers ........ BLOCKED (licence required)
#   - Anything else, including unknown bots .. BLOCKED by default
#
#
# AI companies: to licence our content, see https://situationpublishing.com/ai-licensing/
# Licensing enquiries: [email protected]
#
# If you are an AI company and want to use our content, contact us about
# a licensing arrangement. See: https://situationpublishing.com/ai-licensing/
# =============================================================================

# ---------------------------------------------------------------------------
# Traditional search engines — ALLOWED
# ---------------------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Applebot
Allow: /


# ---------------------------------------------------------------------------
# AI search crawlers that cite and return referral traffic — ALLOWED
# These power AI search answers and link back to source content.
# Permission is conditional on continued referral traffic and attribution.
# ---------------------------------------------------------------------------
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /


# ---------------------------------------------------------------------------
# Operational bots we depend on — ALLOWED
# DO NOT REMOVE. Blocking these breaks programmatic revenue and social sharing.
# ---------------------------------------------------------------------------

# Google advertising crawlers (ad serving / quality)
User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

User-agent: Mediapartners-Google
Allow: /

# Social link-preview / unfurl bots
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Slackbot-LinkExpanding
Allow: /

User-agent: Discordbot
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: WhatsApp
Allow: /

# Ad verification / brand safety / contextual vendors — ALLOWED
# Clients use different verification partners, so all the major ones are listed.
# Ad ops: review periodically and add any niche vendor a client requires.
# (UA matching is case-insensitive.)

# Integral Ad Science (IAS)
User-agent: ias_crawler
Allow: /

User-agent: ias_wombles
Allow: /

User-agent: ias_admantx
Allow: /

# DoubleVerify
User-agent: DoubleVerifyBot
Allow: /

User-agent: DVBot
Allow: /

# Comscore (contextual)
User-agent: proximic
Allow: /

# Peer39 (contextual)
User-agent: peer39_crawler
Allow: /

# Oracle / Grapeshot (contextual — winding down, kept for clients still on it)
User-agent: GrapeshotCrawler
Allow: /

User-agent: grapeshot
Allow: /

# GumGum (contextual)
User-agent: GumGum
Allow: /

# HUMAN Security (brand safety)
User-agent: GlancelotBot
Allow: /

# The Trade Desk
User-agent: TTD-Content
Allow: /

# PubMatic
User-agent: PubMatic Crawler Bot
Allow: /

# Sincera (supply-chain transparency)
User-agent: SinceraSyntheticUser
Allow: /

# Amazon advertising (distinct from Amazonbot AI-training crawler, blocked below)
User-agent: AmazonAdBot
Allow: /

# Other brand-safety / ad-tech crawlers
User-agent: Leikibot
Allow: /

User-agent: SlickBot
Allow: /


# ---------------------------------------------------------------------------
# AI training / scraping crawlers — BLOCKED  (belt-and-suspenders)
# These are already blocked by the default-deny rule at the bottom; listed
# explicitly to document intent. Contact us for licensing terms.
# ---------------------------------------------------------------------------

# OpenAI (training)
User-agent: GPTBot
Disallow: /

# Anthropic
User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Google (AI training — separate from Googlebot search indexing)
User-agent: Google-Extended
Disallow: /

User-agent: Google-CloudVertexBot
Disallow: /

# Meta
User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: FacebookBot
Disallow: /

# Apple (AI training — separate from standard Applebot)
User-agent: Applebot-Extended
Disallow: /

# Amazon
User-agent: Amazonbot
Disallow: /

# ByteDance / TikTok
User-agent: Bytespider
Disallow: /

# Common Crawl
User-agent: CCBot
Disallow: /

# Cohere
User-agent: cohere-ai
Disallow: /

# Diffbot
User-agent: Diffbot
Disallow: /

# You.com
User-agent: YouBot
Disallow: /

# DuckDuckGo AI assistant (distinct from DuckDuckBot search)
User-agent: DuckAssistBot
Disallow: /

# Timpi
User-agent: Timpibot
Disallow: /

# Omgili / Webz.io
User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

# ImagesiftBot
User-agent: ImagesiftBot
Disallow: /

# xAI / Grok
User-agent: GrokBot
Disallow: /

# Allen Institute for AI (AI2)
User-agent: AI2Bot
Disallow: /

# Firecrawl (dev-tool scraper)
User-agent: FirecrawlAgent
Disallow: /

# Scrapy-based generic crawlers
User-agent: Scrapy
Disallow: /


# ---------------------------------------------------------------------------
# DEFAULT — everything else is BLOCKED
# Any crawler not explicitly allowed above is denied. Review server logs /
# TollBit WEEKLY and add new beneficial crawlers (a new AI search engine, or a
# new ad-verification vendor a client brings) to the ALLOWED sections above.
#
# EXCEPTION: ads.txt and app-ads.txt must stay readable by ALL bots —
# programmatic demand depends on it — so they are allowed even under default-deny.
# ---------------------------------------------------------------------------
User-agent: *
Allow: /ads.txt
Allow: /app-ads.txt
Disallow: /


# ---------------------------------------------------------------------------
# Sitemaps
# ---------------------------------------------------------------------------
Sitemap: https://www.nextplatform.com/sitemap.xml