# Don't index any admin pages
User-agent: *
Disallow: /mls/admin/
Disallow: /hfengine/admin/
Disallow: /mls/tpv/
Disallow: /mls/agentlogin/
Disallow: /mls/ihfadmin/
Disallow: /idx/admin/
Disallow: /idx/tpv/
Disallow: /idx/agentlogin/
Disallow: /idx/ihfadmin/
Disallow: /idx/modal/
Disallow: /idx/controllers/*Map*
Disallow: /idx/controllers/*map*
Disallow: /idx/mlsportal/admin/
Disallow: /member-admin/

# Don't index CT_CMLS terms doc, per request from client 103977, Desk case 174097
Disallow: /idx/includes/imgincludes/

User-agent: TruliaBot
Disallow: /

User-agent: PropsmartCrawler
Disallow: /

User-agent: Mp3Bot
Disallow: /

# Don't index any property pages
User-agent: Mozilla*
Disallow: /idx/
Disallow: /member-admin/

User-agent: oodlebot
Disallow: /idx/
Disallow: /member-admin/

User-agent: IRLbot
Disallow: /idx/
Disallow: /member-admin/

User-agent: contype
Disallow: /idx/
Disallow: /member-admin/

User-agent: curl
Disallow: /idx/
Disallow: /member-admin/

User-agent: Slurp
Crawl-delay: 80
Disallow: /mls/admin/
Disallow: /hfengine/admin/
Disallow: /mls/tpv/
Disallow: /mls/agentlogin/
Disallow: /mls/ihfadmin/
Disallow: /idx/admin/
Disallow: /idx/tpv/
Disallow: /idx/agentlogin/
Disallow: /idx/ihfadmin/
Disallow: /idx/modal/
Disallow: /idx/controllers/*Map*
Disallow: /idx/controllers/*map*
Disallow: /idx/mlsportal/admin/
Disallow: /member-admin/

User-agent: msnbot
Crawl-delay: 20
Disallow: /mls/admin/
Disallow: /hfengine/admin/
Disallow: /mls/tpv/
Disallow: /mls/agentlogin/
Disallow: /mls/ihfadmin/
Disallow: /idx/admin/
Disallow: /idx/tpv/
Disallow: /idx/agentlogin/
Disallow: /idx/ihfadmin/
Disallow: /idx/modal/
Disallow: /idx/controllers/*Map*
Disallow: /idx/controllers/*map*
Disallow: /idx/mlsportal/admin/
Disallow: /member-admin/

User-agent: SimilarPages/Nutch-1.0-dev
Disallow: /

User-agent: SimilarPages
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: discobot
Disallow: /

User-agent: CatchBot
Disallow: /

User-agent: trovitBot
Disallow: /

User-agent: twiceler
Crawl-delay: 80
Disallow: /mls/admin/
Disallow: /hfengine/admin/
Disallow: /mls/tpv/
Disallow: /mls/agentlogin/
Disallow: /mls/ihfadmin/
Disallow: /idx/admin/
Disallow: /idx/tpv/
Disallow: /idx/agentlogin/
Disallow: /idx/ihfadmin/
Disallow: /idx/modal/
Disallow: /idx/controllers/*Map*
Disallow: /idx/controllers/*map*
Disallow: /idx/mlsportal/admin/
Disallow: /member-admin/

User-agent: 008
Disallow: /

user-agent: sitebot
disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Aboundexbot
Disallow: /

User-agent: AddThis.com
Disallow: /

User-agent: Yandex
Disallow: /

user-agent: AhrefsBot
disallow: /

# Internet Archive (Wayback machine)
User-agent: ia_archiver
Disallow: /

User-agent: Zing-BottaBot
Disallow: /