###############################
#
#
User-agent: *
#
# list folders robots are not allowed to index 
#
Disallow: /admin/
Disallow: /ssi/
Disallow: /images/
Disallow: /user/
Disallow: /english/commentary/
Disallow: /bbs/
#
# list specific files robots are not allowed to index
#
Disallow: /search_records.jsp
Disallow: /logout.jsp
Disallow: /login.jsp
Disallow: /userhome.jsp
Disallow: /printable.jsp
Disallow: /sgfview.jsp
Disallow: /english/sgfview.jsp
Disallow: /english/printable.jsp
Disallow: /english/byplayer2.jsp
Disallow: /english/bytournament2.jsp
Disallow: /error.jsp
Disallow: /english/error.jsp
Disallow: /english/record_request.jsp
Disallow: /record_request.jsp
Disallow: /v2/modules/collection/sgfview.php
Disallow: /v3/modules/collection/sgfview.php
Disallow: /v2/modules/collection/printable.php
Disallow: /v3/modules/collection/printable.php
Disallow: /v2/modules/collection/record_request.php
Disallow: /v3/modules/collection/record_request.php
Disallow: /v2/modules/newbb/viewtopic.php
Disallow: /v2/modules/collection/game_rss.php
Disallow: /v2/modules/collection/byplayer2.php
Disallow: /v3/modules/collection/byplayer2.php
Disallow: /servlet/net.go4go.RecordPrint_v2
#
###############################
#
User-agent: http://www.almaden.ibm.com/cs/crawler 
Disallow: /
User-agent: fast
Disallow: /
User-agent: ia_archiver
Disallow: /
User-agent: Webdup
Disallow: /
User-agent: Sogou
Disallow: /
User-agent: SimplePie
Disallow: /
User-agent: Yandex
Disallow: /