# ═══════════════════════════════════════════════════════════════════════
# RevivalRO Freya - robots.txt
# Best Ragnarok Online Renewal Private Server
# https://freya.ragnarevival.com
# ═══════════════════════════════════════════════════════════════════════

# ─── GLOBAL RULE: ALLOW ALL CRAWLERS ─────────────────────────────────
User-agent: *
Allow: /
Disallow: /data/
Disallow: /lib/
Disallow: /includes/
Disallow: /tmp/
Disallow: /?module=account&action=login
Disallow: /?module=account&action=logout
Disallow: /*?action=logout
Crawl-delay: 1

# ─── GOOGLE (Search, Images, Video, News, Mobile, AdsBot) ───────────
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Storebot-Google
Allow: /

User-agent: Google-InspectionTool
Allow: /

User-agent: GoogleOther
Allow: /

# ─── BING / MSN (Microsoft) ─────────────────────────────────────────
User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: msnbot
Allow: /

User-agent: msnbot-media
Allow: /

User-agent: BingPreview
Allow: /

User-agent: adidxbot
Allow: /

# ─── YAHOO ──────────────────────────────────────────────────────────
User-agent: Slurp
Allow: /

# ─── YANDEX (Russia - #1 Russian Search Engine) ─────────────────────
User-agent: Yandex
Allow: /
Crawl-delay: 1

User-agent: YandexBot
Allow: /

User-agent: YandexImages
Allow: /

User-agent: YandexVideo
Allow: /

User-agent: YandexMedia
Allow: /

User-agent: YandexMobileBot
Allow: /

User-agent: YandexRenderResourcesBot
Allow: /

# ─── BAIDU (China - #1 Chinese Search Engine) ───────────────────────
User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: Baiduspider-image
Allow: /

User-agent: Baiduspider-video
Allow: /

User-agent: Baiduspider-news
Allow: /

User-agent: Baiduspider-render
Allow: /

# ─── SOGOU (China - #2 Chinese Search Engine) ───────────────────────
User-agent: Sogou
Allow: /

User-agent: Sogou web spider
Allow: /

User-agent: Sogou inst spider
Allow: /

# ─── SHENMA / UC Browser (China - Mobile Search) ────────────────────
User-agent: YisouSpider
Allow: /

User-agent: UCBrowser
Allow: /

# ─── 360 SEARCH / HAOSOU (China - #3 Chinese Search Engine) ─────────
User-agent: 360Spider
Allow: /

User-agent: HaoSouSpider
Allow: /

# ─── SEZNAM (Czech Republic) ────────────────────────────────────────
User-agent: SeznamBot
Allow: /

# ─── DUCKDUCKGO ─────────────────────────────────────────────────────
User-agent: DuckDuckBot
Allow: /

# ─── QWANT (Europe / France) ────────────────────────────────────────
User-agent: Qwant
Allow: /

User-agent: Qwantify
Allow: /

# ─── ECOSIA (Germany / Green Search) ────────────────────────────────
User-agent: Ecosia
Allow: /

# ─── MOJEEK (UK Independent Engine) ─────────────────────────────────
User-agent: MojeekBot
Allow: /

# ─── BRAVE SEARCH ───────────────────────────────────────────────────
User-agent: Brave
Allow: /

# ─── SWISSCOWS (Switzerland) ────────────────────────────────────────
User-agent: Swisscows
Allow: /

# ─── PETAL SEARCH (Huawei - Huge in Asia) ───────────────────────────
User-agent: PetalBot
Allow: /
Crawl-delay: 2

# ─── COCCOC (Vietnam - #1 Vietnamese Search Engine) ─────────────────
User-agent: coccocbot-web
Allow: /

User-agent: coccocbot-image
Allow: /

# ─── MAIL.RU (Russia - #2 Russian Search Engine) ────────────────────
User-agent: Mail.RU_Bot
Allow: /

# ─── RAMBLER (Russia) ───────────────────────────────────────────────
User-agent: Rambler
Allow: /

# ─── SPUTNIK (Russia - Government Search) ───────────────────────────
User-agent: SputnikBot
Allow: /

# ─── GOGO (Indonesia) ───────────────────────────────────────────────
User-agent: Linguee Bot
Allow: /

# ─── YAHOO JAPAN ────────────────────────────────────────────────────
User-agent: Y!J-ASR
Allow: /

User-agent: Y!J-BRJ
Allow: /

User-agent: Y!J-BSC
Allow: /

# ─── LINE (Japan / Thailand / Taiwan Messaging + Search) ────────────
User-agent: Linespider
Allow: /

# ─── APPLE ──────────────────────────────────────────────────────────
User-agent: Applebot
Allow: /

User-agent: AppleNewsBot
Allow: /

# ─── FACEBOOK / META (Social Sharing Preview) ───────────────────────
User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

# ─── TWITTER / X ────────────────────────────────────────────────────
User-agent: Twitterbot
Allow: /

# ─── DISCORD ────────────────────────────────────────────────────────
User-agent: Discordbot
Allow: /

# ─── TELEGRAM ───────────────────────────────────────────────────────
User-agent: TelegramBot
Allow: /

# ─── WHATSAPP ───────────────────────────────────────────────────────
User-agent: WhatsApp
Allow: /

# ─── LINKEDIN ───────────────────────────────────────────────────────
User-agent: LinkedInBot
Allow: /

# ─── PINTEREST ──────────────────────────────────────────────────────
User-agent: Pinterest
Allow: /

User-agent: Pinterestbot
Allow: /

# ─── REDDIT ─────────────────────────────────────────────────────────
User-agent: Redditbot
Allow: /

# ─── SLACK ──────────────────────────────────────────────────────────
User-agent: Slackbot
Allow: /

User-agent: Slackbot-LinkExpanding
Allow: /

# ─── SNAPCHAT ───────────────────────────────────────────────────────
User-agent: Snapchat
Allow: /

# ─── SKYPE ──────────────────────────────────────────────────────────
User-agent: SkypeUriPreview
Allow: /

# ─── VIBER ──────────────────────────────────────────────────────────
User-agent: Viber
Allow: /

# ─── SEO / ANALYTICS TOOLS (Let them crawl for SEO insights) ────────
User-agent: AhrefsBot
Allow: /
Crawl-delay: 5

User-agent: SemrushBot
Allow: /
Crawl-delay: 5

User-agent: MJ12bot
Allow: /
Crawl-delay: 5

User-agent: DotBot
Allow: /
Crawl-delay: 5

User-agent: rogerbot
Allow: /
Crawl-delay: 5

User-agent: Screaming Frog SEO Spider
Allow: /

# ─── ARCHIVE.ORG (Web Archive - Good for Trust) ─────────────────────
User-agent: ia_archiver
Allow: /

User-agent: archive.org_bot
Allow: /

# ─── AI BOTS (Allowed - More Visibility) ────────────────────────────
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Bytespider
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Omgili
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: YouBot
Allow: /

# ─── BLOCK KNOWN BAD BOTS ───────────────────────────────────────────
User-agent: MJ12bot
Crawl-delay: 10

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: Seekport
Disallow: /

# ─── SITEMAPS ────────────────────────────────────────────────────────
Sitemap: https://freya.ragnarevival.com/sitemap.xml

# ─── HOST DIRECTIVE (Yandex) ────────────────────────────────────────
Host: https://freya.ragnarevival.com