# robots.txt for https://www.misar.blog
#
# Content is free to crawl, index, and surface in search/AI answers.
# See /llms.txt for the full AI access policy.

# ── Default: allow all crawlers ─────────────────────────────────────────────
User-agent: *
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /_next/
Disallow: /admin/
Disallow: /private/
Disallow: /editor/
Disallow: /preview/
Disallow: /onboarding/
Disallow: /auth/callback
Disallow: /auth/invitation
Disallow: /search
Disallow: /*opengraph-image*
Disallow: /*twitter-image*

# ── Search Engine Crawlers ───────────────────────────────────────────────────
User-agent: Googlebot
User-agent: Bingbot
User-agent: bingbot
User-agent: Applebot
User-agent: TerracottaBot
User-agent: Terracotta
User-agent: YandexBot
User-agent: Slurp
User-agent: DuckDuckBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /_next/
Disallow: /admin/
Disallow: /private/
Disallow: /editor/
Disallow: /preview/
Disallow: /onboarding/
Disallow: /auth/callback
Disallow: /auth/invitation
Disallow: /search
Disallow: /*opengraph-image*
Disallow: /*twitter-image*

# ── AI Crawlers: ALLOWED (search/answer/summarizer bots that cite + drive traffic)
# These fetch to ANSWER live queries and cite the source — they do NOT train
# foundation models. Blocking them would cost citations in ChatGPT Search,
# Perplexity, and Claude's search tool with no privacy upside.
# NOTE: the model-TRAINING crawlers (GPTBot, ClaudeBot, Google-Extended,
# Applebot-Extended) are intentionally NOT here — they are in the DISALLOWED
# group below. AI Overviews / SGE still work: they crawl via Googlebot, not
# Google-Extended.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Claude-SearchBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agents.md
Allow: /.well-known/ai-plugin.json
Disallow: /api/
Disallow: /dashboard/
Disallow: /_next/
Disallow: /admin/
Disallow: /private/
Disallow: /editor/
Disallow: /preview/
Disallow: /onboarding/
Disallow: /auth/callback
Disallow: /auth/invitation
Disallow: /search
Disallow: /*opengraph-image*
Disallow: /*twitter-image*
Crawl-delay: 1

# ── AI Crawlers: METERED (licensed access — crawl allowed, access is priced) ──
# BAN OR BILL, NEVER BOTH.
#
# These were previously served a blanket Disallow, which made the paid-crawl gate
# unreachable: a robots-compliant crawler never requests an article, so it never
# receives the 402 and can never pay. Over three days that produced 83 gate hits
# against ~7,400 bot pageviews/day — and $0. We were telling every prospective
# payer to go away and then waiting for them to pay.
#
# They are now allowed at the robots layer so the request actually arrives, and
# the licensing decision is made per-request by the paid-crawl gate
# (src/lib/monetization/crawler-policy.ts + middleware.ts), which can price,
# meter, or wave through based on class policy. robots.txt states *reachability*;
# the gate states *terms*.
#
# Anything that CANNOT pay is in the DISALLOWED group below — the test for
# membership here is "is there a counterparty that could hold a licence", not
# "do we like this crawler".
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Google-CloudVertexBot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: AI2Bot
User-agent: Bytespider
User-agent: CCBot
User-agent: Cohere-AI
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: Omgilibot
User-agent: Omgili
User-agent: ImagesiftBot
User-agent: Timpibot
User-agent: PanguBot
User-agent: FriendlyCrawler
User-agent: Kangaroo Bot
User-agent: PetalBot
User-agent: YouBot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /_next/
Disallow: /admin/
Disallow: /private/
Disallow: /editor/
Disallow: /preview/
Disallow: /onboarding/
Disallow: /auth/callback
Disallow: /auth/invitation
Disallow: /search
Disallow: /*opengraph-image*
Disallow: /*twitter-image*
Crawl-delay: 1

# ── AI Crawlers: DISALLOWED (no counterparty — cannot be billed) ─────────────
# Kept banned deliberately: each is either a tool/library UA with no company
# behind it to hold a licence, or a service whose use of the content generates
# no licensable value for us. Unbanning these would give away content with no
# prospect of payment, which is the opposite of the change above.
#   Scrapy, img2dataset — generic scraping libraries, not an identifiable payer
#   TurnitinBot         — plagiarism detection; no licensing relationship
#   peer39_crawler      — ad-verification/brand-safety scraping; no payer
User-agent: Scrapy
User-agent: img2dataset
User-agent: TurnitinBot
User-agent: peer39_crawler
Disallow: /

# ── AI Assistants (user-triggered fetches — allowed) ─────────────────────────
# A person asked their assistant to open/summarize THIS url. Not bulk training.
# (ChatGPT-User / OAI-SearchBot / PerplexityBot are declared in the ALLOWED group
#  above — a UA must appear in only one group, so they are intentionally not repeated.)
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: MistralAI-User
User-agent: Meta-ExternalFetcher
User-agent: DuckAssistBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agents.md
Disallow: /api/
Disallow: /dashboard/
Disallow: /_next/
Disallow: /admin/
Disallow: /private/
Disallow: /editor/
Disallow: /preview/
Disallow: /onboarding/
Disallow: /auth/callback
Disallow: /auth/invitation
Disallow: /search
Disallow: /*opengraph-image*
Disallow: /*twitter-image*

# ── Archivers ────────────────────────────────────────────────────────────────
User-agent: archive.org_bot
User-agent: ia_archiver
User-agent: Wayback
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /_next/
Disallow: /admin/
Disallow: /private/
Disallow: /editor/
Disallow: /preview/
Disallow: /onboarding/
Disallow: /auth/callback
Disallow: /auth/invitation
Disallow: /search
Disallow: /*opengraph-image*
Disallow: /*twitter-image*
Crawl-delay: 2

# ── Infrastructure Crawlers ──────────────────────────────────────────────────
User-agent: Cloudflare-Healthchecker
User-agent: CloudflareBot
Allow: /

# ── Google Image Crawler ─────────────────────────────────────────────────────
# Googlebot-Image is a separate bot — must be explicitly allowed to crawl
# images for Google Images and Visual Search (Lens). Inherits * disallows
# unless overridden here.
User-agent: Googlebot-Image
Allow: /
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.avif$
# opengraph-image and twitter-image paths are still blocked (social cards only)
Disallow: /*opengraph-image*
Disallow: /*twitter-image*

# ── Googlebot-News: not required ────────────────────────────────────────────
# MisarBlog is not a news publisher registered in Google News Publisher Center
# and does not carry NewsArticle schema, so Googlebot-News inherits from *
# (Allow: /) without a dedicated block — no action required.

# ── Discovery files ──────────────────────────────────────────────────────────
# Advertise ONLY the sitemap index. It references every sub-sitemap
# (sitemap.xml, sitemap-static/programmatic/content/images/videos.xml), so
# listing them individually here is redundant and risks crawlers treating the
# same URLs as competing top-level sitemaps. One canonical entry point only.
# ── Machine-readable licensing ───────────────────────────────────────────────
# Two competing declaration formats, both free, neither with teeth today. We
# emit both because the cost is two lines and there is no convergence to bet on:
# Cloudflare/RSL say "ai-train", the IETF aipref draft says "train-ai", and that
# draft's attachment half expired in Oct 2025 with the vocabulary still marked
# "DO NOT REFLECT CONSENSUS".
#
# Content-Signal (Cloudflare, Sept 2025) — a preference, explicitly NOT a
# technical countermeasure. It embeds a reservation of rights under Art. 4 of
# EU Directive 2019/790, which is the actual reason to publish it.
#
# These values must stay consistent with /license.xml, with the content-signal
# header on our HTTP 402, and with DEFAULT_CLASS_POLICY in crawler-policy.ts:
# search and answer-engine use is free (citations are a discovery channel worth
# more than the toll), model training requires a licence.
Content-Signal: search=yes, ai-input=yes, ai-train=no
License: https://www.misar.blog/license.xml

Sitemap: https://www.misar.blog/sitemap-index.xml