# # robots.txt for http://www.arch.ttu.edu/ # # $Id: robots.txt, 26/06/2003 # #User-agent: *scooter* #This is altavista's bot, allow it #User-Agent: T-Rex #this is lycos' spider, allow it User-agent: *slurp* Disallow: / #don't allow anyone to use wget (for those honoring robots.txt) User-agent: *wget* Disallow: / #disallow curl for same reason as wget User-agent: *curl* Disallow: / # exclude some access-controlled areas User-agent: * Disallow: /model_database/ Disallow: /arc/database/ Disallow: /facultysearch/ Disallow: /fluidflow/ Disallow: /Form_Engine/ Disallow: /schedules/calendar/ Disallow: /resources/jobs/ Disallow: /architecture/resources/rooms/ Disallow: /resources/forum/