# =============================================================
# mytortenland.de — robots.txt
# Son güncelleme: 2026
# Strateji:
#   - Genel kural: site açık
#   - Parametre sayfaları (kopya içerik) kapatılır
#   - Google/Bing/Yandex için tag sayfaları açık bırakılır
#   - AI botlarına izin verilir
#   - Agresif scraper/SEO botları kapatılır
# =============================================================

# -------------------------------------------------------
# 1) GENEL KURAL — tüm bilinmeyen botlar
# -------------------------------------------------------
User-agent: *
Allow: /

# Parametre kaynaklı kopya sayfalar (duplicate content)
Disallow: /*?page=
Disallow: /*?fpc=
Disallow: /*&page=
Disallow: /*?sort=
Disallow: /*&sort=
Disallow: /*?order=
Disallow: /*&order=
Disallow: /*?limit=
Disallow: /*&limit=
Disallow: /*?filter_name=
Disallow: /*&filter_name=
Disallow: /*?filter_sub_category=
Disallow: /*&filter_sub_category=
Disallow: /*?filter_description=
Disallow: /*&filter_description=

# tag= parametreli sayfalar — genel botlara kapalı
# (Google/Bing/Yandex için aşağıda ayrıca açılır)
Disallow: /*?tag=
Disallow: /*&tag=

# Sistem / yönetim dizinleri
Disallow: /admin/
Disallow: /yonetim/
Disallow: /system/
Disallow: /install/
Disallow: /download/
Disallow: /cache/

# -------------------------------------------------------
# 2) GOOGLE BOTLARI
# tag= sayfalarına izin verilir (SEO değeri var)
# -------------------------------------------------------
User-agent: Googlebot
Allow: /*?tag=
Allow: /*&tag=
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Storebot-Google
Allow: /

User-agent: Google-InspectionTool
Allow: /

# Google AI eğitim botu — izin verilir
User-agent: Google-Extended
Allow: /

# -------------------------------------------------------
# 3) BING BOTLARI
# tag= sayfalarına izin verilir
# -------------------------------------------------------
User-agent: Bingbot
Allow: /*?tag=
Allow: /*&tag=
Allow: /

User-agent: AdIdxBot
Allow: /

User-agent: BingPreview
Allow: /

# -------------------------------------------------------
# 4) YANDEX BOTLARI
# tag= sayfalarına izin verilir
# -------------------------------------------------------
User-agent: YandexBot
Allow: /*?tag=
Allow: /*&tag=
Allow: /

User-agent: YandexImages
Allow: /

User-agent: YandexVideo
Allow: /

User-agent: YandexMobileBot
Allow: /

# -------------------------------------------------------
# 5) DuckDuckGo
# -------------------------------------------------------
User-agent: DuckDuckBot
Allow: /*?tag=
Allow: /*&tag=
Allow: /

# -------------------------------------------------------
# 6) AI / LLM BOTLARI — izin verilir
# -------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# -------------------------------------------------------
# 7) AGRESİF SEO / SCRAPER BOTLARI — tamamen kapalı
# (htaccess'te de 403 alıyorlar, çift katman güvenlik)
# -------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: HTTrack
Disallow: /

# -------------------------------------------------------
# 8) AMAZON BOTLARI — kapalı
# -------------------------------------------------------
User-agent: Amazonbot
Disallow: /

User-agent: Amazon-AdBot
Disallow: /

# -------------------------------------------------------
# SİTEMAP
# -------------------------------------------------------
Sitemap: https://mytortenland.de/sitemap.xml