# www.robotstxt.org/
# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449

User-agent: *
Disallow: /openx*
Disallow: /email-alert*
Disallow: /resetting*
Disallow: /register* 
Disallow: /login*
Disallow: /jobs?title=*
Disallow: /my-jobs/*
Disallow: /connect/*
Sitemap: https://www.jobkralle.de/sitemap/de/sitemap.xml

User-agent: msnbot
Crawl-delay: 5

User-agent: bingbot
Crawl-delay: 5

User-agent: amazonaws
Crawl-delay: 5

User-agent: AhrefsBot 
Crawl-delay: 5

User-agent: SemrushBot
Crawl-delay: 5

User-agent: BLEXBot
Crawl-delay: 5

User-agent: dotbot
Crawl-delay: 5

User-agent: Yandex
Crawl-delay: 5

User-agent: MegaIndex.ru
Crawl-delay: 5

User-agent: SeekportBot
Crawl-delay: 5

User-agent: SeznamBot
Crawl-delay: 5

User-agent: FreshBot
Crawl-delay: 5

User-agent: DataForSeoBot
Crawl-delay: 5

User-agent: PetalBot 
Crawl-delay: 5

User-agent: Applebot
Crawl-delay: 5