# robots.txt for http://idiom.com/~chrisber/ # User-agent: Slurp Crawl-delay: 600 User-agent: Yahoo-MMCrawler User-agent: Yahoo-MMCrawler/3.x User-agent: Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html) User-agent: Yahoo-MMCrawler/3.x (mms dash mmcrawler dash support at yahoo dash inc dot com) User-agent: Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp) User-agent: Yahoo! Slurp User-agent: Yahoo Slurp User-agent: Slurp Disallow: / User-agent: * Disallow: cgi-bin/refresh.cgi Disallow: cgi-bin/test/