# Robots.txt for IndieBible.com
# Updated: October 2025
# Purpose: WordPress optimization, privacy protection, and AI/LLM guidelines

# ====================
# GENERAL RULES FOR ALL BOTS
# ====================

User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/woocommerce_uploads/
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/
Disallow: /category/*/*
Disallow: */trackback
Disallow: */feed
Disallow: */comments
Disallow: /*?*
Disallow: /*?
Allow: /wp-content/uploads/
Allow: /wp-admin/admin-ajax.php

# ====================
# WOOCOMMERCE SPECIFIC
# ====================

# Protect customer accounts and checkout
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wc-api/

# Protect order tracking and customer data
Disallow: /order-tracking/
Disallow: /addons/
Disallow: /*.php$

# Allow product pages and shop
Allow: /shop/
Allow: /product/
Allow: /product-category/

# ====================
# PRIVATE & DOWNLOADABLE FILES
# ====================

# Protect all downloadable products
Disallow: /wp-content/uploads/woocommerce_uploads/
Disallow: /downloads/
Disallow: /*.pdf$
Disallow: /*.zip$
Disallow: /*.mp3$
Disallow: /*.mp4$
Disallow: /*.doc$
Disallow: /*.docx$
Disallow: /*.xls$
Disallow: /*.xlsx$

# Protect private member content
Disallow: /members-only/
Disallow: /subscriber-content/
Disallow: /premium/

# ====================
# WORDPRESS TECHNICAL FILES
# ====================

Disallow: /cgi-bin/
Disallow: /wp-*.php
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /wp-signup.php
Disallow: /readme.html
Disallow: /license.txt

# ====================
# AI CRAWLERS & LLMs - BALANCED APPROACH
# ====================

# OpenAI GPTBot - Allow indexing for suggestions, block content scraping
User-agent: GPTBot
Allow: /
Disallow: /wp-content/
Disallow: /blog/*
Disallow: /articles/*
Disallow: /resources/*
Disallow: /*.pdf$
Disallow: /*.zip$
Disallow: /shop/
Disallow: /product/
Disallow: /product-category/
Crawl-delay: 10

# Google AI (Bard/Gemini) - Allow indexing for suggestions, block content
User-agent: Google-Extended
Allow: /
Disallow: /wp-content/
Disallow: /blog/*
Disallow: /articles/*
Disallow: /resources/*
Disallow: /*.pdf$
Disallow: /*.zip$
Disallow: /shop/
Disallow: /product/
Disallow: /product-category/
Crawl-delay: 10

# Anthropic Claude - Allow indexing for suggestions, block content
User-agent: anthropic-ai
Allow: /
Disallow: /wp-content/
Disallow: /blog/*
Disallow: /articles/*
Disallow: /resources/*
Disallow: /*.pdf$
Disallow: /*.zip$
Disallow: /shop/
Disallow: /product/
Disallow: /product-category/
Crawl-delay: 10

# Common Crawl - Allow limited access
User-agent: CCBot
Allow: /
Disallow: /wp-content/
Disallow: /blog/*
Disallow: /articles/*
Disallow: /resources/*
Disallow: /shop/
Disallow: /product/
Crawl-delay: 10

# Cohere AI - Allow indexing for suggestions, block content
User-agent: cohere-ai
Allow: /
Disallow: /wp-content/
Disallow: /blog/*
Disallow: /articles/*
Disallow: /resources/*
Disallow: /*.pdf$
Disallow: /*.zip$
Crawl-delay: 10

# Omgili/Webz.io - Block completely
User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

# Bytespider (TikTok) - Block completely
User-agent: Bytespider
Disallow: /

# Yandex - Block AI training
User-agent: YandexBot
Allow: /
Disallow: /wp-content/
Disallow: /blog/*
Disallow: /articles/*
Crawl-delay: 10

# ====================
# AGGRESSIVE SCRAPERS - BLOCK COMPLETELY
# ====================

# Block known bad bots and scrapers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: facebookexternalhit
Disallow: /

# Block email harvesters
User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailWolf
Disallow: /

# ====================
# GOOD BOTS - ALLOW FULL ACCESS
# ====================

# Google - Full access for search
User-agent: Googlebot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wp-content/uploads/woocommerce_uploads/

# Google Images
User-agent: Googlebot-Image
Allow: /wp-content/uploads/
Disallow: /wp-content/uploads/woocommerce_uploads/

# Bing - Full access
User-agent: Bingbot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

# ====================
# SITEMAP LOCATION
# ====================

Sitemap: https://indiebible.com/sitemap.xml
Sitemap: https://indiebible.com/sitemap_index.xml