User-agent: SMTBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: Cliqzbot
Disallow: /

User-agent: Amazonbot               # Amazon's user agent
Disallow: /           # disallow this directory

User-agent: MJ12bot
Disallow: /

User-agent: DomainStatsBot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: spbot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SeekportBot
Crawl-delay: 2

User-agent: *
Allow: /

Disallow: /articletype/

Disallow: /comment/list/*
Allow: /comment/list/$