User-Agent: *
Disallow: /cgi-bin/
Disallow: /css/
Disallow: /data/
Disallow: /face/
Disallow: /image/
Disallow: /reg1/
Disallow: /reg2/
Disallow: /mb/css/
Disallow: /mb/face/
Disallow: /mb/img/
Disallow: /mb/include/
Disallow: /mb/mbmain/
Disallow: /analyzer.html
Disallow: /google_analytics.html
Disallow: /mb_analyze.txt

Sitemap: http://www.nyacyc.org/sitemap.xml
Sitemap: http://www.nyacyc.org/mobile-sitemap.xml
Sitemap: http://www.nyacyc.org/urllist.txt

User-agent:slurp
Allow: /y_key_ff0446defb8235c0.html
Allow: /sitemaps.xml
Allow: /urllist.txt

User-agent:googlebot
Allow: /sitemaps.xml
Allow: /urllist.txt

User-agent:msnbot
Allow: /LiveSearchSiteAuth.xml
Allow: /sitemaps.xml
Allow: /urllist.txt



User-agent: Slurp
Crawl-delay: 8

User-agent: msnbot
Crawl-delay: 5

User-agent: nutch
Crawl-delay: 5

User-agent: yeti
Crawl-delay: 8

User-agent: Yeti/1.0 (NHN Corp.; http://help.naver.com/robots/)
Crawl-delay: 8

User-agent: naverbot
Crawl-delay: 8

User-agent: googlebot
Crawl-delay: 5

User-agent: dotbot
Crawl-delay: 5

User-agent: twiceler
Crawl-delay: 8

User-agent: yandex
Crawl-delay: 5

User-agent: Teoma
Crawl-delay: 5

User-agent: BoardReader
Crawl-delay: 8

User-agent: Exabot
Crawl-delay: 10