# ============================================================
#  robots.txt for VideoHub YouTube Gallery
# ============================================================

User-agent: *
Allow: /
Disallow: /save-data.php
Disallow: /videos.json
Disallow: /*.json$
Disallow: /*.sql$
Disallow: /*.log$
Disallow: /config
Disallow: /.env

# Sitemap
Sitemap: https://youtubeseo.learnwithtest.pro/sitemap.php

# ── Googlebot ──────────────────────────────────────────────
User-agent: Googlebot
Allow: /
Allow: /video/
Disallow: /save-data.php
Crawl-delay: 1

# ── Bingbot ────────────────────────────────────────────────
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# ── Google Image Bot ───────────────────────────────────────
User-agent: Googlebot-Image
Allow: /
Disallow: /save-data.php

# ── Google Video Bot ───────────────────────────────────────
User-agent: Googlebot-Video
Allow: /
Allow: /video/

# ── Block resource-wasting crawlers ────────────────────────
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# ── Clean URL parameters (Google Search Console hint) ──────
Clean-param: utm_source&utm_medium&utm_campaign&utm_term&utm_content&ref&source /