#General
User-agent: *
Disallow: /candidates-privacy-policy/ 
Disallow: /?s=
Disallow: /search/
Disallow: /blog/tag/*

#Known crawlers
User-agent: LinkedInBot
User-agent: bingbot
User-agent: RSiteAuditor
User-agent: SiteAuditBot
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: SemrushBot-SI
User-agent: SemrushBot-BA
User-agent: SemrushBot-SWA
User-agent: SemrushBot-CT
User-agent: SplitSignalBot
User-agent: SemrushBot-COUB
User-agent: AhrefsBot
User-agent: AhrefsSiteAudit
Crawl-delay: 30
User-agent: Yandex
User-agent: YandexBot
Crawl-delay: 300
#Feeds crawlers
User-agent: Feedly
User-agent: feeder.co
User-agent: FlipboardRSS
User-agent: omgili
User-agent: FeedBurner
User-agent: UniversalFeedParser
User-agent: Competitors.app
Allow: /feed/
Crawl-delay: 30

#Blocked crawlers
User-agent: RyteBot
User-agent: OnPageBot
User-agent: dotbot
User-agent: rogerbot
user-agent: MJ12bot
User-agent: SeznamBot
User-agent: DataForSeoBot
User-agent: PetalBot
User-agent: coccocbot-web
User-agent: SEOkicks
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: SerendeputyBot
User-agent: MojeekBot
User-agent: Scrapy
User-agent: Orbbot
User-agent: MauiBot
User-Agent: Neevabot
User-agent: x09Mozilla
User-agent: x22Mozilla
Disallow: /

Sitemap: https://www.bairesdev.com/api/sitemap/index.xml
Sitemap: https://www.bairesdev.com/blog/sitemap_index.xml
Sitemap: https://www.bairesdev.com/tools/tessel/sitemap_index.xml
Sitemap: https://www.bairesdev.com/tools/pwned/sitemap_index.xml