# 기본 정책 — 일반 크롤러
User-agent: *
Allow: /
Crawl-delay: 10

# 개인 정보가 포함된 경로는 크롤링 제외
Disallow: /console/
Disallow: /signin
Disallow: /onboarding/

# SPA rewrite 목적지. 실제 콘텐츠가 아니라 빈 껍데기라 색인할 이유가 없다.
# (/faq/* 등은 이 셸을 받아가지만 URL 이 달라 여기 영향을 받지 않는다)
Disallow: /shell/

# ── 검색 엔진 (허용) ──────────────────────────────
User-agent: Yeti
Allow: /
Crawl-delay: 2

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /
Crawl-delay: 2

# ── AI 크롤러 (GEO 목적, 허용) ────────────────────
# 실제 추천·인용 트래픽으로 이어지는 것만 남긴다.
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

# ── 차단: 대량 크롤러 (국내 유입 가치 없음) ────────
User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: YisouSpider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: 360Spider
Disallow: /

User-agent: Sosospider
Disallow: /

# ── 차단: SEO 스파이 / 백링크 분석 ────────────────
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: SEOkicks
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: spbot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: 80legs
Disallow: /

User-agent: WBSearchBot
Disallow: /

User-agent: Riddler
Disallow: /

User-agent: Aboundex
Disallow: /

# ── 차단: 스크래퍼 / 스캐너 ───────────────────────
User-agent: Screaming Frog SEO Spider
Disallow: /

User-agent: scrapy
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: SiteExplorer
Disallow: /

User-agent: Offline Navigator
Disallow: /

User-agent: Black Hole
Disallow: /

User-agent: Nikto
Disallow: /

User-agent: sqlmap
Disallow: /

User-agent: masscan
Disallow: /

User-agent: ZmEu
Disallow: /

User-agent: Morfeus Fucking Scanner
Disallow: /

# 사이트맵 위치
Sitemap: https://myonbi.kr/sitemap.xml