# -----------------------------------------------------------------------------
# robots.txt for the AutoShrine.com Network
# Issues? Contact webmaster@autoshrine.com
# -----------------------------------------------------------------------------

# -----------------------------------------------------------------------------
# Restricted crawlers
# -----------------------------------------------------------------------------

User-agent: Baiduspider
User-agent: CCBot
User-agent: Cliqzbot
User-agent: CrsspxlBot
User-agent: discoverybot
User-agent: DomainCrawler
User-agent: DotBot
User-agent: ExtLinksBot
User-agent: Ezooms
User-agent: Grapeshot
User-agent: istellabot
User-agent: JikeSpider
User-agent: LinkpadBot
User-agent: magpie-crawler
User-agent: MauiBot
User-agent: MegaIndex.ru
User-agent: MJ12bot
User-agent: proximic
User-agent: SEOkicks-Robot
User-agent: Sistrix
User-agent: Sogou
User-agent: spbot
User-agent: VelenPublicWebCrawler
User-agent: WBSearchBot
User-agent: WeSEE
User-agent: Yandex
User-agent: YYSpider
User-agent: ZumBot
Disallow: /journal/
Disallow: /member/
Disallow: /modelinfo/
Disallow: /phorum/
Disallow: /forum/
Disallow: /registry/

# -----------------------------------------------------------------------------
# Banned crawlers
# -----------------------------------------------------------------------------

User-agent: AhrefsBot
User-agent: AlphaBot
User-agent: AlphaSeoBot
User-agent: AlphaSeoBot-SA
User-agent: Barkrowler
User-agent: BLEXBot
User-agent: BomboraBot
User-agent: careerbot
User-agent: GumGum
User-agent: GumGum-Bot
User-agent: Linguee
User-agent: ip-web-crawler.com
User-agent: PetalBot
User-agent: SemrushBot
User-agent: SemrushBot-BA
User-agent: SemrushBot-SA
User-agent: SeznamBot
User-agent: TurnitinBot
User-Agent: trendictionbot
User-Agent: ZoominfoBot
Disallow: /

# -----------------------------------------------------------------------------
# Delays
# -----------------------------------------------------------------------------

# Nutch is the VigLink crawler
User-agent: Nutch
User-agent: VigLink
Crawl-delay: 2

# Suggested 3s
User-agent: bingbot
User-agent: msnbot-media
Crawl-delay: 3

User-agent: Seekport
Crawl-delay: 5

# Crawl-delay not supported
#User-agent: Googlebot

# -----------------------------------------------------------------------------
# All other crawlers
# -----------------------------------------------------------------------------

User-agent: *
Crawl-delay: 5
Disallow: /ads/click.php
Disallow: /calendar/attend.php
Disallow: /member/activate
Disallow: /member/map.php
Disallow: /member/password_reset.php
Disallow: /member/reset
Disallow: /member/pp_result.php
Disallow: /modelinfo/
Disallow: /newsletter/
Disallow: /phorum/control.php
Disallow: /phorum/feed.php
Disallow: /phorum/login.php
Disallow: /phorum/moderation.php
Disallow: /phorum/pm.php
Disallow: /phorum/post.php
Disallow: /phorum/posting.php
Disallow: /phorum/random.php
Disallow: /phorum/read.php?*,printview,*
Disallow: /phorum/read.php?*,quote=1
Disallow: /phorum/read.php?*,newer
Disallow: /phorum/read.php?*,older
Disallow: /phorum/report.php
Disallow: /phorum/search.php
Disallow: /registry/imgr_reset_pass.php
Disallow: /registry/browse.php
Disallow: /registry/transfer.php
Disallow: /toggle_comment.php