# Search Engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Applebot
Allow: /

# Social media crawlers for link previews
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# Block other social media crawlers
User-agent: WhatsApp
Disallow: /

User-agent: TelegramBot
Disallow: /

User-agent: SkypeUriPreview
Disallow: /

User-agent: Discordbot
Disallow: /

User-agent: Slackbot
Disallow: /

User-agent: redditbot
Disallow: /

User-agent: PinterestBot
Disallow: /

User-agent: Snapchat
Disallow: /

User-agent: TikTokBot
Disallow: /

User-agent: InstagramBot
Disallow: /

# Block AI crawlers
User-agent: CCBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: OpenAI-SearchBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

# SEO Tools
User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: MJ12bot
Allow: /

User-agent: DotBot
Allow: /

User-agent: Screaming Frog SEO Spider
Allow: /

User-agent: SEOkicks
Allow: /

User-agent: BLEXBot
Allow: /

User-agent: MegaIndex
Allow: /

# Block archive crawlers
User-agent: archive.org_bot
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: Wayback
Disallow: /

# Block automated tools
User-agent: curl
Disallow: /

User-agent: wget
Disallow: /

User-agent: python-requests
Disallow: /

User-agent: libwww-perl
Disallow: /

User-agent: Go-http-client
Disallow: /

User-agent: node-fetch
Disallow: /

# Default rule for unlisted crawlers
User-agent: *
Disallow: /admin/
Disallow: /private/
Disallow: /tmp/

# Sitemap location
Sitemap: https://scntix.com/sitemap.xml