################################### # # Robot.txt file # Live - No roots for staging and sub domains with Brinkster # # User-agent: * # # list folders robots are not allowed to index # # Disallow: /folder/ # # list specific files robots are not allowed to index # # Disallow: /directory/foo.html # #################################### User-agent: Titan Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: ExtractorPro Disallow: / User-agent: * Disallow: /bab/ Disallow: /bevan/ Disallow: /cgi-bin/ Disallow: /css/ Disallow: /domain/ Disallow: /images/ Disallow: /inc/ Disallow: /js/ Disallow: /query/ Disallow: /phpsysinfo/ Disallow: /staging/ Disallow: /tpp/ Disallow: /resources/index.html Disallow: /demos/ Disallow: /vidmov/ Disallow: /_forms_/ Disallow: /_quest_/ Disallow: /_rss_/ Disallow: /_vidmov_/