fix a couple of cores.
reduce memory usage significantly by not pre-allocating some per-collection hashtables.
This commit is contained in:
6
Msg4.cpp
6
Msg4.cpp
@ -863,7 +863,11 @@ bool sendBuffer ( long hostId , long niceness ) {
|
||||
|
||||
// sanity check. our clock must be in sync with host #0's or with
|
||||
// a host from his group, group #0
|
||||
if ( ! isClockInSync() ) { char *xx=NULL ; *xx=0; }
|
||||
if ( ! isClockInSync() ) {
|
||||
log("msg4: msg4: warning sending out adds but clock not in "
|
||||
"sync with host #0");
|
||||
//char *xx=NULL ; *xx=0; }
|
||||
}
|
||||
// try to keep all zids unique, regardless of their group
|
||||
static uint64_t s_lastZid = 0;
|
||||
// select a "zid", a sync id
|
||||
|
@ -1889,7 +1889,9 @@ bool sendPageCrawlbot ( TcpSocket *socket , HttpRequest *hr ) {
|
||||
// i guess bail if not there?
|
||||
if ( ! cr ) {
|
||||
log("crawlbot: missing coll rec for coll %s",collName);
|
||||
char *msg = "invalid or missing collection rec";
|
||||
//char *msg = "invalid or missing collection rec";
|
||||
char *msg = "Could not create job because missing seeds or "
|
||||
"urls.";
|
||||
return sendErrorReply2 (socket,fmt,msg);
|
||||
}
|
||||
|
||||
|
@ -1248,16 +1248,16 @@ bool SpiderColl::load ( ) {
|
||||
// a restart request from crawlbottesting for this collnum which
|
||||
// calls Collectiondb::resetColl2() which calls deleteSpiderColl()
|
||||
// on THIS spidercoll, but our m_loading flag is set
|
||||
if (!m_sniTable.set ( 4,8,5000,NULL,0,false,MAX_NICENESS,"snitbl") )
|
||||
if (!m_sniTable.set ( 4,8,0,NULL,0,false,MAX_NICENESS,"snitbl") )
|
||||
return false;
|
||||
if (!m_cdTable.set (4,4,3000,NULL,0,false,MAX_NICENESS,"cdtbl"))
|
||||
if (!m_cdTable.set (4,4,0,NULL,0,false,MAX_NICENESS,"cdtbl"))
|
||||
return false;
|
||||
// doledb seems to have like 32000 entries in it
|
||||
long numSlots = 0; // was 128000
|
||||
if(!m_doleIpTable.set(4,4,numSlots,NULL,0,false,MAX_NICENESS,"doleip"))
|
||||
return false;
|
||||
// this should grow dynamically...
|
||||
if (!m_waitingTable.set (4,8,100,NULL,0,false,MAX_NICENESS,"waittbl"))
|
||||
if (!m_waitingTable.set (4,8,16,NULL,0,false,MAX_NICENESS,"waittbl"))
|
||||
return false;
|
||||
// . a tree of keys, key is earliestSpiderTime|ip (key=12 bytes)
|
||||
// . earliestSpiderTime is 0 if unknown
|
||||
|
@ -2557,7 +2557,12 @@ bool XmlDoc::indexDoc2 ( ) {
|
||||
// error?
|
||||
if ( ! metaList ) {
|
||||
// sanity check. g_errno must be set
|
||||
if ( ! g_errno ) { char *xx=NULL;*xx=0; }
|
||||
if ( ! g_errno ) {
|
||||
log("build: Error UNKNOWN error spidering. setting "
|
||||
"to bad engineer.");
|
||||
g_errno = EBADENGINEER;
|
||||
//char *xx=NULL;*xx=0; }
|
||||
}
|
||||
log("build: Error spidering for doc %s: %s",
|
||||
m_firstUrl.m_url,mstrerror(g_errno));
|
||||
return true;
|
||||
|
Reference in New Issue
Block a user