# https://varvy.com/robottxt.html
# If your crawler does not respect robots.txt, your CIDR, country, region or continent
# will go to iptables blacklist. Policy is deny allow by default


# 1 day = 86400
Crawl-delay: 86400

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: yacybot
Allow: /

User-agent: Slurp
Allow: /

User-agent: bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: facebot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: ia_archiver
Allow: /

# Disallow

User-agent: *
User-agent: AlphaSeoBot
User-agent: DotBot
User-agent: FemtosearchBot

Disallow: /