# /robots.txt - 2/6/96 CJ
# 20070206 - Let the Google Mini in  -  Andy
#
# Keeps Webcrawlers and Spiders (a.k.a robots) from 
# accessing webpages
#

User-agent: gsa-crawler
Disallow:

#This keeps all robots out of everything
 
User-agent: *
Disallow: /
 
User-agent: MOMspider
Disallow:
Disallow: /cgi-bin/lwgate/

#This keeps all robots out of anything that starts 
#with the given path
 
#User-agent: *
#Disallow: /<path relative to web root>
 
#This lets all robots go anywhere in site
 
#User-agent: *
#Disallow: 
 
#
#Note:  Only one "User-agent: *" can be uncommented. 
#Robots may respond differently when more than one choice
#is available.  If you wish to allow a certain robot, 
#use a case-insensitive substring of that robot's name
#for "User-agent: "
#
 
#This must be in the ROOT directory of a website.  If 
#not, it will not be acknowledged
#i.e. it must be http://www.hmc.edu/robots.txt,
#not in a subdirectory like 
#http://www.hmc.edu/robots/robots.txt