################################### # # Robot.txt file # Live - No roots for staging and sub domains with Brinkster # # User-agent: * # # list folders robots are not allowed to index # # Disallow: /folder/ # # list specific files robots are not allowed to index # # Disallow: /directory/foo.html # #################################### User-agent: Titan Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: ExtractorPro Disallow: / User-agent: * Disallow: /bab/ Disallow: /bbg/ Disallow: /bevan/ Disallow: /cgi-bin/ Disallow: /css/ Disallow: /domains/ Disallow: /domains2/ Disallow: /domain-management/ Disallow: /images/ Disallow: /inc/ Disallow: /js/ Disallow: /query/ Disallow: /phpsysinfo/ Disallow: /staging/ Disallow: /tpp/ Disallow: /resources/index.html Disallow: /demos/ Disallow: /quest/ Disallow: /rss/