# ClipNova AI - Robots.txt
# Last updated: June 16, 2026
# Website: https://clipnova.in
#
# IMPORTANT (Jun-16-2026 fix): a crawler obeys ONLY its single most-specific
# User-agent group - groups are NOT merged. Previously the auth/internal Disallow
# rules sat under the last-declared bot ("User-agent: Slackbot") and a redundant
# "User-agent: Googlebot / Allow: /" group made Googlebot ignore every block.
# All shared rules now live under "User-agent: *" so EVERY crawler (Googlebot,
# Bingbot, etc.) honours them. Only genuinely-different bots (the AI crawlers)
# keep their own group. Crawl-delay removed (Google ignores it; it was flagged in
# Search Console and we want faster crawling, not throttling).

# =============================================
# Default - applies to ALL crawlers without a more-specific group below
# =============================================
User-agent: *
Allow: /

# --- High-value public pages (Allow: / already covers these; explicit for clarity) ---
Allow: /discover/
Allow: /watch/
Allow: /video-detail/
Allow: /youtube-shorts-maker/
Allow: /instagram-reels-creator/
Allow: /text-to-video-ai/
Allow: /free-stock-videos/
Allow: /ai-video-maker/
Allow: /ai-shorts-generator/
Allow: /horror-story-video-maker/
Allow: /faceless-youtube-channel/
Allow: /ai-motivation-video-maker/
Allow: /mythology-video-maker/
Allow: /news-video-creator/
Allow: /educational-video-maker/
Allow: /product-ad-creator/
Allow: /ai-voiceover-generator/
Allow: /tiktok-video-maker/
Allow: /comedy-video-maker/
Allow: /facts-video-maker/
Allow: /mystery-video-maker/
Allow: /viral-video-maker/
Allow: /multilingual-video-maker/
Allow: /youtube-auto-publish/
Allow: /youtube-autopilot-channel/
Allow: /psychology-video-maker/
Allow: /money-video-maker/
Allow: /love-story-video-maker/
Allow: /crime-story-video-maker/
Allow: /history-video-maker/
Allow: /sci-fi-video-maker/
Allow: /kids-story-video-maker/

# --- Block: auth-gated and internal pages (focus crawl budget on indexable money pages) ---
Disallow: /account
Disallow: /user-videos
Disallow: /user-video/
Disallow: /admin
Disallow: /welcome
# /home 301s to / (nginx) - block to stop the recurring "Page with redirect" churn.
Disallow: /home$
Disallow: /story/customize/
Disallow: /story/generate
Disallow: /story/result
Disallow: /api/
Disallow: /private/
Disallow: /node_modules/

# --- Assets needed for rendering ---
Allow: /manifest.json
Allow: /assets/
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$

# --- Sitemaps (proxied through nginx, not under /api/) ---
Allow: /sitemap.xml
Allow: /sitemap-videos.xml

# =============================================
# AI crawlers — ALL ALLOWED for GEO (Generative Engine Optimization).
# [Jun-17-2026] Reversed the old "block training scrapers" policy: GA4 shows
# "AI Assistant" is a top + fastest-growing channel (+82% in 7 days). Being in
# the LLMs' knowledge is HOW we get recommended when users ask ChatGPT / Gemini /
# Perplexity / Claude "best free AI video maker". Blocking GPTBot / CCBot /
# Claude-Web was throttling our own fastest-growing source. For a growth-stage
# consumer app, discoverability >> protecting a video-generator's "content".
# Public URLs only; private/auth pages are auth-gated + low-value to these agents.
# =============================================
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# =============================================
# Sitemaps
# [Jun-17-2026] sitemap-videos.xml REMOVED — it advertised ~9,900 thin auto-generated /watch/ video pages
# that Google rejects ("Crawled – currently not indexed") and that dilute the domain's quality reputation,
# keeping the money pages stuck at positions 26-40. Those /watch/ pages are now noindex,follow (backend),
# so submitting them was actively harmful. Only the 78 curated landing pages (sitemap.xml) are submitted.
# (Also removed from Search Console.) `Allow: /watch/` is kept so Google can still CRAWL these pages and
# SEE the noindex — do NOT Disallow them until they've dropped from the index.
# =============================================
Sitemap: https://clipnova.in/sitemap.xml