# Analyzing Market · robots.txt
# Crypto market data, analysis & education · Bedrock WordPress

# ── Sitemaps (news first = freshness signal) ──
Sitemap: https://analyzingmarket.com/news-sitemap.xml
Sitemap: https://analyzingmarket.com/sitemap.xml

# ── Universal policy ──
User-agent: *
Allow: /app/uploads/
Allow: /app/themes/analyzingmarket/dist/
Allow: /app/themes/analyzingmarket/resources/
Allow: /wp/wp-admin/admin-ajax.php
Disallow: /wp/wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /app/plugins/
Disallow: /app/mu-plugins/
# Crawl-budget noise (internal search + tracking params)
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/
Disallow: /*?replytocom=
Disallow: /*?utm_source=
Disallow: /*?utm_medium=
Disallow: /*?utm_campaign=
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /trackback/
Disallow: /*/trackback/
Disallow: /comments/feed/

# ── Blocked scrapers (resource drain, no traffic value) ──
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: NPbot
Disallow: /

# ── International + social + AI-citation engines: full access ──
User-agent: Googlebot
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Bingbot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

# ── SEO-audit crawlers: allow (needed for our own audits) ──
User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: rogerbot
Allow: /

User-agent: DotBot
Allow: /

User-agent: DataForSeoBot
Allow: /

User-agent: ScreamingFrogSEOSpider
Allow: /

Sitemap: http://analyzingmarket.com/sitemap.xml

Disallow: /*?k=
Disallow: /*&k=