#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html
#
# For syntax checking, see:
# http://www.frobee.com/robots-txt-check

User-agent: *
Crawl-delay: 10
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
Disallow: /kreanet
Disallow: /kreanet/
Disallow: /kreanet/*
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /vip/register/
Disallow: /vip/password/
Disallow: /vip/login/
Disallow: /vip/logout/
Disallow: /vip/register
Disallow: /vip/password
Disallow: /vip/login
Disallow: /vip/logout
Disallow: /vip/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/
Disallow: /?q=vip/password/
Disallow: /?q=vip/register/
Disallow: /?q=vip/login/
Disallow: /?q=vip/logout/
Disallow: /?q=vip/password
Disallow: /?q=vip/register
Disallow: /?q=vip/login
Disallow: /?q=vip/logout
Disallow: /?q=vip/

#badbots
User-agent: 008
User-agent: crawler4j
User-agent: crawler4j (http://code.google.com/p/crawler4j/)
User-agent: Curious George - www.analyticsseo.com/crawler
User-agent: Curious George - www.analyticsseo.com
User-agent: MegaIndex.ru/2.0
User-agent: MegaIndex.ru/
User-agent: +https://www.megaindex.ru/
User-agent: Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)
User-agent: Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)
User-agent: MJ12
User-agent: Mozilla/5.0 (compatible; Steeler/3.5; http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)
User-agent: Steeler
User-agent: Microsoft-WebDAV-MiniRedir/6.1.7601
User-agent: Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)
User-agent: Findxbot
User-agent: Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)
User-agent: SeznamBot
User-agent: Seznam
User-agent: Mozilla/4.0 (compatible;  Vagabondo/4.0; http://webagent.wise-guys.nl/)
User-agent: Mozilla/4.0 (compatible;  Vagabondo/4.0; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/; http://www.wise-guys.nl/)
User-agent: Vagabondo
User-agent: Jakarta Commons-HttpClient/3.0.1
User-agent: LWP::Simple/5.822
User-agent: Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)
User-agent: SMTBot
User-agent: Mozilla/5.0 (compatible; memoryBot/1.22.56 +http://internetmemory.org/en/)
User-agent: memoryBot
User-agent: Mozilla/5.0 (compatible; smrjbot/0.0.20)
User-agent: smrjbot
User-agent: Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )
User-agent: spbot
User-agent: MJ12bot
User-agent: BaiDuSpider
User-agent: Exabot
User-agent: Yandex
User-agent: BSpider
Disallow:/