fix a couple of cores.

reduce memory usage significantly by not
pre-allocating some per-collection hashtables.
This commit is contained in:
Matt Wells
2014-11-05 09:36:42 -08:00
parent 4eed706bd4
commit 23c565afc8
4 changed files with 17 additions and 6 deletions

@ -863,7 +863,11 @@ bool sendBuffer ( long hostId , long niceness ) {
// sanity check. our clock must be in sync with host #0's or with
// a host from his group, group #0
if ( ! isClockInSync() ) { char *xx=NULL ; *xx=0; }
if ( ! isClockInSync() ) {
log("msg4: msg4: warning sending out adds but clock not in "
"sync with host #0");
//char *xx=NULL ; *xx=0; }
}
// try to keep all zids unique, regardless of their group
static uint64_t s_lastZid = 0;
// select a "zid", a sync id

@ -1889,7 +1889,9 @@ bool sendPageCrawlbot ( TcpSocket *socket , HttpRequest *hr ) {
// i guess bail if not there?
if ( ! cr ) {
log("crawlbot: missing coll rec for coll %s",collName);
char *msg = "invalid or missing collection rec";
//char *msg = "invalid or missing collection rec";
char *msg = "Could not create job because missing seeds or "
"urls.";
return sendErrorReply2 (socket,fmt,msg);
}

@ -1248,16 +1248,16 @@ bool SpiderColl::load ( ) {
// a restart request from crawlbottesting for this collnum which
// calls Collectiondb::resetColl2() which calls deleteSpiderColl()
// on THIS spidercoll, but our m_loading flag is set
if (!m_sniTable.set ( 4,8,5000,NULL,0,false,MAX_NICENESS,"snitbl") )
if (!m_sniTable.set ( 4,8,0,NULL,0,false,MAX_NICENESS,"snitbl") )
return false;
if (!m_cdTable.set (4,4,3000,NULL,0,false,MAX_NICENESS,"cdtbl"))
if (!m_cdTable.set (4,4,0,NULL,0,false,MAX_NICENESS,"cdtbl"))
return false;
// doledb seems to have like 32000 entries in it
long numSlots = 0; // was 128000
if(!m_doleIpTable.set(4,4,numSlots,NULL,0,false,MAX_NICENESS,"doleip"))
return false;
// this should grow dynamically...
if (!m_waitingTable.set (4,8,100,NULL,0,false,MAX_NICENESS,"waittbl"))
if (!m_waitingTable.set (4,8,16,NULL,0,false,MAX_NICENESS,"waittbl"))
return false;
// . a tree of keys, key is earliestSpiderTime|ip (key=12 bytes)
// . earliestSpiderTime is 0 if unknown

@ -2557,7 +2557,12 @@ bool XmlDoc::indexDoc2 ( ) {
// error?
if ( ! metaList ) {
// sanity check. g_errno must be set
if ( ! g_errno ) { char *xx=NULL;*xx=0; }
if ( ! g_errno ) {
log("build: Error UNKNOWN error spidering. setting "
"to bad engineer.");
g_errno = EBADENGINEER;
//char *xx=NULL;*xx=0; }
}
log("build: Error spidering for doc %s: %s",
m_firstUrl.m_url,mstrerror(g_errno));
return true;