# Block aggressive SEO bots
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# Rate limit social media crawlers
User-agent: facebookexternalhit
Crawl-delay: 10

User-agent: meta-externalagent
Crawl-delay: 10

# Allow legitimate search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Yandex
Allow: /

# Claude SearchBot (Anthropic)
User-agent: Claude-SearchBot
Allow: /
Crawl-delay: 5

# Default rules for all other bots
User-agent: *
Disallow: /admin
Disallow: /api
Disallow: /*?product_url=*
Crawl-delay: 2
Allow: /shop/*
Allow: /product/*

Sitemap: https://oncube.biz/sitemap.xml