Forks/privacore-open-source-search-engine
1
0
Fork 0
forked from Mirrors/privacore-open-source-search-engine
Code Pull Requests Activity

Merge branch 'diffbot' into diffbot-testing

Browse Source
This commit is contained in:
Matt Wells
2015-05-03 00:34:07 -07:00
parent b15b7cdb94 821c6cb424
commit 52e7c75970
3 changed files with 18 additions and 1 deletions

1
Conf.h

@ -681,6 +681,7 @@ class Conf {
bool m_logDebugSummary ;
bool m_logDebugSpider ;
bool m_logDebugMsg13 ;
bool m_diffbotMsg13Hack ;
bool m_logDebugUrlAttempts ;
bool m_logDebugTcp ;
bool m_logDebugThread ;

8
Msg13.cpp

@ -281,6 +281,12 @@ bool Msg13::forwardRequest ( ) {
//
int32_t nh = g_hostdb.m_numHosts;
int32_t hostId = hash32h(((uint32_t)r->m_firstIp >> 8), 0) % nh;
// avoid host #0 for diffbot hack which is dropping some requests
// because of the streaming bug methinks
if ( hostId == 0 && nh >= 2 && g_conf.m_diffbotMsg13Hack )
hostId = 1;
// get host to send to from hostId
Host *h = NULL;
// . pick first alive host, starting with "hostId" as the hostId
@ -3014,7 +3020,7 @@ bool addToHammerQueue ( Msg13Request *r ) {
CollectionRec *cr = g_collectiondb.getRec ( r->m_collnum );
bool canUseProxies = false;
if ( cr->m_automaticallyUseProxies ) canUseProxies = true;
if ( cr && cr->m_automaticallyUseProxies ) canUseProxies = true;
if ( r->m_forceUseFloaters ) canUseProxies = true;
//if ( g_conf.m_useProxyIps ) canUseProxies = true;
//if ( g_conf.m_automaticallyUseProxyIps ) canUseProxies = true;

10
Parms.cpp

@ -19542,6 +19542,16 @@ void Parms::init ( ) {
m->m_obj = OBJ_CONF;
m++;
m->m_title = "disable host0 for msg13 reception hack";
m->m_cgi = "dmth";
m->m_off = (char *)&g_conf.m_diffbotMsg13Hack - g;
m->m_type = TYPE_BOOL;
m->m_def = "0";
m->m_priv = 1;
m->m_page = PAGE_LOG;
m->m_obj = OBJ_CONF;
m++;
m->m_title = "log debug spider proxies";
m->m_cgi = "ldspr";
m->m_off = (char *)&g_conf.m_logDebugProxies - g;