From a0697e1bb582cd157206c0ca437bcada3b4f3158 Mon Sep 17 00:00:00 2001
From: Matt Wells <mattdwells@hotmail.com>
Date: Wed, 26 Feb 2014 10:26:09 -0800
Subject: [PATCH] do not allow custom crawls to spider the web any more.
---
Collectiondb.cpp | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/Collectiondb.cpp b/Collectiondb.cpp
index d19d9800..3c1b912a 100644
--- a/Collectiondb.cpp
+++ b/Collectiondb.cpp
@@ -2137,11 +2137,13 @@ bool CollectionRec::rebuildUrlFilters ( ) {
i++;
// 2nd default filter
- if ( m_restrictDomain ) {
- m_regExs[i].set("!isonsamedomain && !ismanualadd");
- m_spiderPriorities [i] = SPIDER_PRIORITY_FILTERED;
- i++;
- }
+ // always turn this on for now. they need to add domains they want
+ // to crawl as seeds so they do not spider the web.
+ //if ( m_restrictDomain ) {
+ m_regExs[i].set("!isonsamedomain && !ismanualadd");
+ m_spiderPriorities [i] = SPIDER_PRIORITY_FILTERED;
+ i++;
+ //}
m_regExs[i].set("errorcount>=1 && !hastmperror");
m_spiderPriorities [i] = 15;