Repositorio para la herramienta Tupale completo
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

270 lines
4.3KB

  1. User-agent: MJ12bot
  2. Disallow: /
  3. User-agent: Vagabondo
  4. Disallow: /
  5. User-agent: BaiDuSpider
  6. Disallow: /
  7. User-agent: Exabot
  8. Disallow: /
  9. User-agent: Yandex
  10. Disallow: /
  11. User-agent: BSpider
  12. Disallow: /
  13. User-agent: MegaIndex
  14. Disallow: /
  15. User-agent: AhrefsBot
  16. Crawl-Delay: 3600
  17. # Slow down bots
  18. User-agent: *
  19. Crawl-delay: 10
  20. # Disallow: Sistrix
  21. User-agent: sistrix
  22. Disallow: /
  23. # Disallow: Sistrix
  24. User-agent: SISTRIX Crawler
  25. Disallow: /
  26. # Disallow: Sistrix
  27. User-agent: SISTRIX
  28. Disallow: /
  29. # Disallow: SEOkicks-Robot
  30. User-agent: SEOkicks-Robot
  31. Disallow: /
  32. # Disallow: jobs.de-Robot
  33. User-agent: jobs.de-Robot
  34. Disallow: /
  35. # Backlink Analysis
  36. user-agent: AhrefsBot
  37. disallow: /
  38. # Bot der Leipziger Unister Holding GmbH
  39. user-agent: UnisterBot
  40. disallow: /
  41. # http://moz.com/products
  42. User-agent: dotbot
  43. Disallow: /
  44. # http://www.searchmetrics.com
  45. User-agent: SearchmetricsBot
  46. Disallow: /
  47. # http://www.majestic12.co.uk/projects/dsearch/mj12bot.php
  48. User-agent: MJ12bot
  49. Disallow: /
  50. # http://www.domaintools.com/webmasters/surveybot.php
  51. User-agent: SurveyBot
  52. Disallow: /
  53. # http://www.seodiver.com/bot
  54. user-agent: SEOdiver
  55. disallow: /
  56. # http://openlinkprofiler.org/bot
  57. User-agent: spbot
  58. Disallow: /
  59. # http://www.wotbox.com/bot/
  60. User-agent: wotbox
  61. Disallow: /
  62. # http://www.opensiteexplorer.org/dotbot
  63. # http://moz.com/researchtools/ose/dotbot
  64. User-agent: dotbot
  65. Disallow: /
  66. # http://www.meanpath.com/meanpathbot.html
  67. User-agent: meanpathbot
  68. Disallow: /
  69. # http://www.backlinktest.com/crawler.html
  70. User-agent: BacklinkCrawler
  71. Disallow: /
  72. # http://www.brandwatch.com/magpie-crawler/
  73. User-agent: magpie-crawler
  74. Disallow: /
  75. # http://filterdb.iss.net/crawler/
  76. User-agent: oBot
  77. Disallow: /
  78. User-agent: fr-crawler
  79. Disallow: /
  80. # http://webmeup-crawler.com
  81. User-agent: BLEXBot
  82. Disallow: /
  83. # https://megaindex.com/crawler
  84. User-agent: MegaIndex.ru
  85. Disallow: /
  86. User-agent: megaindex.com
  87. Disallow: /
  88. # http://www.cloudservermarket.com
  89. User-Agent: CloudServerMarketSpider
  90. Disallow: /
  91. # http://www.trendiction.de/de/publisher/bot
  92. User-Agent: trendictionbot
  93. Disallow: /
  94. # http://www.exalead.com
  95. User-agent: Exabot
  96. Disallow: /
  97. # http://www.career-x.de/bot.html
  98. User-agent: careerbot
  99. Disallow: /
  100. # https://www.lipperhey.com/en/about/
  101. User-agent: Lipperhey-Kaus-Australis
  102. Disallow: /
  103. User-agent: seoscanners.net
  104. Disallow: /
  105. User-agent: MetaJobBot
  106. Disallow: /
  107. User-agent: Spiderbot
  108. Disallow: /
  109. User-agent: LinkStats
  110. Disallow: /
  111. User-agent: JobboerseBot
  112. Disallow: /
  113. User-agent: ICCrawler
  114. Disallow: /
  115. User-agent: Plista
  116. Disallow: /
  117. User-agent: Domain Re-Animator Bot
  118. Disallow: /
  119. # https://www.lipperhey.com/en/about/
  120. User-agent: Lipperhey-Kaus-Australis
  121. Disallow: /
  122. # https://turnitin.com/robot/crawlerinfo.html
  123. User-agent: turnitinbot
  124. Disallow: /
  125. # http://help.coccoc.com/
  126. User-agent: coccoc
  127. Disallow: /
  128. # ubermetrics-technologies.com
  129. User-agent: um-IC
  130. Disallow: /
  131. # datenbutler.de
  132. User-agent: mindUpBot
  133. Disallow: /
  134. # http://searchgears.de/uber-uns/crawling-faq.html
  135. User-agent: sg-Orbiter
  136. Disallow: /
  137. # http://commoncrawl.org/faq/
  138. User-agent: CCBot
  139. Disallow: /
  140. # https://www.qwant.com/
  141. User-agent: Qwantify
  142. Disallow: /
  143. # http://linkfluence.net/
  144. User-agent: Kraken
  145. Disallow: /
  146. # http://www.botje.com/plukkie.htm
  147. User-agent: plukkie
  148. Disallow: /
  149. # https://www.safedns.com/searchbot
  150. User-agent: SafeDNSBot
  151. Disallow: /
  152. # http://www.haosou.com/help/help_3_2.html
  153. User-agent: 360Spider
  154. Disallow: /
  155. # http://www.haosou.com/help/help_3_2.html
  156. User-agent: HaosouSpider
  157. Disallow: /
  158. # http://www.moz.com/dp/rogerbot
  159. User-agent: rogerbot
  160. Disallow: /
  161. # http://www.openhose.org/bot.html
  162. User-agent: OpenHoseBot
  163. Disallow: /
  164. # http://www.screamingfrog.co.uk/seo-spider/
  165. User-agent: Screaming Frog SEO Spider
  166. Disallow: /
  167. # http://thumbsniper.com
  168. User-agent: ThumbSniper
  169. Disallow: /
  170. # http://www.radian6.com/crawler
  171. User-agent: R6_CommentReader
  172. Disallow: /
  173. User-agent: ImplisenseBot
  174. Disallow: /
  175. # http://cliqz.com/company/cliqzbot
  176. User-agent: Cliqzbot
  177. Disallow: /
  178. # https://www.aihitdata.com/about
  179. User-agent: aiHitBot
  180. Disallow: /
  181. # http://www.trendiction.com/en/publisher/bot
  182. User-Agent: trendictionbot
  183. Disallow: /
  184. # http://warebay.com/bot.html
  185. User-agent: WBSearchBot
  186. Disallow: /
  187. # Block MegaIndex.ru
  188. User-agent: MegaIndex.ru/2.0
  189. Disallow: /
  190. User-agent: MegaIndex.ru
  191. Disallow: /
  192. User-agent: megaIndex.ru
  193. Disallow: /
  194. User-agent: Slurp
  195. Disallow: /
  196. User-agent: MJ12bot
  197. Disallow: /