# Libraryminds - Searchable Video Knowledge Library
# https://libraryminds.com
# Last updated: 2026-03

# =============================================
# GROUP 1: BLOCKED — High-cost scrapers (bandwidth protection)
# =============================================
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: Amazonbot
User-agent: PetalBot
User-agent: MJ12bot
Disallow: /

# =============================================
# GROUP 2: GENERAL — All other crawlers
# =============================================
User-agent: *

# === ALLOWED: E-E-A-T Core (Experience, Expertise, Authoritativeness, Trust) ===
Allow: /
Allow: /what-is-libraryminds
Allow: /features
Allow: /pricing

# === ALLOWED: E-E-A-T Authoritativeness — Author & Founder pages ===
Allow: /authors/
Allow: /authors/libraryminds-team

# === ALLOWED: Developer API docs (technical authority) ===
Allow: /developers

# === ALLOWED: Use Cases (conversion + topical authority) ===
Allow: /use-cases

# === ALLOWED: Milestones (trust + E-E-A-T) ===
Allow: /milestones/

# === ALLOWED: E-E-A-T Expertise — Blog (topical authority) ===
Allow: /blog
Allow: /blog/

# === ALLOWED: E-E-A-T Experience — Resource articles ===
Allow: /resources/
Allow: /resources/why-you-forget-what-you-watch
Allow: /resources/build-your-video-library
Allow: /resources/search-across-your-videos

# === ALLOWED: E-E-A-T Expertise — Free tools (demonstrable expertise) ===
Allow: /free-tools
Allow: /free-tools/
Allow: /free-tools/srt-validator
Allow: /free-tools/text-to-speech
Allow: /free-tools/audio-to-text
Allow: /free-tools/word-counter
Allow: /free-tools/character-counter
Allow: /free-tools/duration-calculator
Allow: /free-tools/timestamp-generator

# === ALLOWED: E-E-A-T Trust — Legal & support ===
Allow: /privacy
Allow: /terms
Allow: /help
Allow: /contact

# === ALLOWED: Public transcripts (user-generated content / social proof) ===
Allow: /t/

# === ALLOWED: Static assets ===
Allow: /logo.png
Allow: /favicon.png
Allow: /opengraph.jpg
Allow: /llms.txt

# === DISALLOWED: Private / authenticated pages ===
Disallow: /dashboard
Disallow: /admin
Disallow: /login
Disallow: /register
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /u/
Disallow: /jobs
Disallow: /transcript

# === DISALLOWED: API endpoints ===
Disallow: /api/

# =============================================
# GROUP 3: AI AGENTS — Explicit E-E-A-T access
# =============================================
User-agent: Google-Extended
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Bytespider
User-agent: cohere-ai

# === ALLOWED: E-E-A-T pages for AI training / search ===
Allow: /
Allow: /what-is-libraryminds
Allow: /features
Allow: /pricing
Allow: /developers
Allow: /use-cases
Allow: /milestones/
Allow: /authors/
Allow: /blog
Allow: /blog/
Allow: /resources/
Allow: /free-tools
Allow: /free-tools/
Allow: /t/
Allow: /privacy
Allow: /terms
Allow: /help
Allow: /contact
Allow: /llms.txt

# === DISALLOWED: Private content ===
Disallow: /dashboard
Disallow: /admin
Disallow: /login
Disallow: /register
Disallow: /verify-email
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /u/
Disallow: /jobs
Disallow: /transcript
Disallow: /api/

# =============================================
# SITEMAPS
# =============================================
Sitemap: https://libraryminds.com/sitemap.xml