#this file prohibits robots/crawlers/spiders etc. from 'finding' and indexing
#certain webpages
# for further info see 
#
#  http://www.utoronto.ca/ic/ultraseek/webserv.html
#------------------------------------------------------------

User-Agent: *    # applies to all robots
Disallow: /internal/
Disallow: /tempfiles/
Disallow: /archives/
Disallow: /DDO/internal/
Disallow: /DDO/DDOshop/
Disallow: /ddold/
Disallow: /ddold2/


# these last two contain the files from the old DDO server- all the
# useful ones have been copied out to DDO but these are being kept 
# for the time being in case I missed any

# the following are obsolete/ redirect files


Disallow: journals.core.html
Disallow: papers.html
Disallow: staffdir.html
Disallow: library.html
Disallow: hsh.html
Disallow: reviews1.html

# the following are trial files

Disallow: astrograd.html  # Christine Clement's tmp file
Disallow: ungrad1.html  # Christine Clement's tmp file


# the following are individual requests

Disallow: reviews_search.html
Disallow: reviews_qtr.html
Disallow: reviews_yr.html

# block cgi-bin directory

Disallow: /cgi-bin/

# block Bill Clarke's web upon his request

Disallow: /users/bclarke