# robots.txt for www.nrao.edu
# How come this doesn't work?  See, e.g.
# http://suche.fireball.de/fcgi/query.fcg?action=query&pg=express&q=%2bmercator+%2bbrowser&submit=suchen&what=web
# which was snarfed with Scooter2_Mercator_url-1.0... 
# Modified 2001.03.27 PPM: remove /cgi* and spell out no-go areas.
User-agent: gsa-crawler
Disallow: /cgi-bin/
Disallow: /cgi-internal/
Disallow: /php/cal/

User-agent: *
Disallow: /cgi-bin/
Disallow: /cgi-internal/
Disallow: /internal
Disallow: /pubcompdocs/
Disallow: /gold/
Disallow: /php/