###############################
#
# sample robots.txt file for this website 
#
# addresses all robots by using wild card *
#
User-agent: *
# list folders robots are not allowed to index

#Disallow: /
Disallow: /THDL/profiles
Disallow: /THDL/reg_pool
Disallow: /THDL/include
Disallow: /THDL/xsl
Disallow: /THDL/OnlineTools
Disallow: /THDL/OfflineTools
Disallow: /THDL/temp
Disallow: /datemap
Disallow: /feedback
Disallow: /termdb

#Disallow: /tutorials/meta/
#Disallow: /tutorials/images/
#Disallow: /tutorials/assets/
#Disallow: /tutorials/404redirect/
#
# list specific files robots are not allowed to index
#
#Disallow: /tutorials/meta_tags.html
#Disallow: /tutorials/custom_error_page.html
#
# End of robots.txt file
#
###############################

# To disallow all user agents to access all subdirectories, just use the following setting
# User-agent: * Disallow: /