# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-Agent: *
# Disallow: /

# banning ldspider and oBot/2.3.1 from entire site because they do not honor HTTP response format specifications and keep requesting
# responses in unsupported formats thus throwing 500 errors 
# ldspider - http://code.google.com/p/ldspider/
User-Agent: ldspider
Disallow: /

User-Agent: oBot/2.3.1
Disallow: /