User-agent: *
Disallow: /*?paged=
Disallow: /wp-admin/
Disallow: /*blackhole
Disallow: /?blackhole
Allow: /wp-admin/admin-ajax.php

# Block common scraping bots
User-agent: Scrapy
Disallow: /

User-agent: ScrapyBot
Disallow: /

User-agent: Wget
Disallow: /

User-agent: curl
Disallow: /

# Allow good bots
User-agent: Googlebot
Allow: /

User-agent: bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /