# robots.txt for http://www.worldsrecords.com/

User-agent: *
Disallow: /catalog/temp/ # scripts only
Disallow: /_logs/ # test area
Disallow: /manager/ # 
Disallow: /plugins/ # 
# Disallow: /cgi-bin/ # 
Disallow: /vdsbackup/ # 
Disallow: /audio/ # 
Disallow: /cgi-bin/storeR.cgi/* # added * 10/2/08

User-agent: Mediapartners-Google*
Crawl-Delay: 10
# is 5 seconds
#Disallow:

User-agent: ShopWiki
Crawl-Delay: 50
Disallow: /cgi-bin/storeR.cgi/* # added * 10/2/08

User-agent: VoilaBot
Crawl-Delay: 30
Disallow: /cgi-bin/storeR.cgi/* # added * 10/2/08

User-agent: Slurp #yahoo
Disallow: /cgi-bin/storeR.cgi/* # added * 10/2/08
Crawl-delay: 60

User-agent: Fatbot #thefind
Crawl-delay: 40
Disallow: /cgi-bin/storeR.cgi/* # added * 10/2/08

User-agent: msnbot
Disallow: /cgi-bin/storeR.cgi/* # added * 10/2/08
Crawl-delay: 40

User-Agent: Twiceler
Crawl-delay: 40
Disallow: /cgi-bin/storeR.cgi/* # added * 10/2/08

User-Agent: Googlebot
Disallow: /cgi-bin/storeR.cgi/* # added * 10/2/08
Crawl-delay: 20

User-Agent: Charlotte  #searchme trying 
Crawl-delay: 50
Disallow: /cgi-bin/storeR.cgi/* # added * 10/2/08

User-agent: ia_archiver
#Disallow: /cgi-bin
Disallow: /cgi-bin/storeR.cgi/* # added * 10/2/08

User-agent: Teoma
#Disallow: /cgi-bin
Disallow: /cgi-bin/storeR.cgi/* # added * 10/2/08

User-agent: NuSearch Spider
Disallow: /cgi-bin/storeR.cgi/* # added * 10/2/08

User-agent: Gigabot
Crawl-Delay: 30
#Disallow: /cgi-bin  maybe have to deactivate this
Disallow: /cgi-bin/storeR.cgi/* # added * 10/2/08