User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SERankingBacklinksBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: SemrushBot-SI
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: SemrushBot-CT
Disallow: /

User-agent: SemrushBot-OCOB
Disallow: /

User-agent: SemrushBot-FT
Disallow: /

User-agent: SplitSignalBot
Disallow: /

User-agent: AhrefsSiteAudit
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: AwarioBot
Disallow: /

User-agent: Dataprovider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: webzio-extended
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: img2dataset
Disallow: /

User-agent: panscient
Disallow: /

User-agent: ImageMetaBot
Disallow: /

# AI training crawlers: the whole site except the law library.
User-agent: CCBot
User-agent: FacebookBot
User-agent: Bytespider
User-agent: TikTokSpider
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Reflectionbot
User-agent: PanguBot
Disallow: /api/lexlint/case-law/
Disallow: /l/
Disallow: /law/
Disallow: /frameworks/
Disallow: /case/

User-agent: *
Allow: /
Disallow: /api/lexlint/case-law/
Sitemap: https://lexlint.io/sitemap.xml