Forks/open-source-search-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'testing'

Browse Source
This commit is contained in:
Matt 2015-02-18 07:26:01 -07:00
commit 913102c48c
38 changed files with 1705 additions and 459 deletions

7
Collectiondb.cpp

@ -1084,6 +1084,9 @@ bool Collectiondb::setRecPtr ( collnum_t collnum , CollectionRec *cr ) {
// set it
m_recs = (CollectionRec **)m_recPtrBuf.getBufStart();
// tell spiders to re-upadted the active list
g_spiderLoop.m_activeListValid = false;
// a delete?
if ( ! cr ) {
// sanity
@ -2103,6 +2106,10 @@ bool CollectionRec::rebuildUrlFilters2 ( ) {
// addDefault = true;
if ( ! rebuild ) return true;
// tell spider loop to update active list
g_spiderLoop.m_activeListValid = false;
if ( !strcmp(s,"shallow" ) )
return rebuildShallowRules();

8
Collectiondb.h

@ -27,6 +27,7 @@ public:
char *m_parmEnd;
class UdpSlot *m_slot;
bool m_doRebuilds;
bool m_rebuildActiveList;
bool m_doProxyRebuild;
bool m_updatedRound;
collnum_t m_collnum;
@ -309,6 +310,9 @@ class CollectionRec {
public:
// active linked list of collectionrecs used by spider.cpp
class CollectionRec *m_nextActive;
// these just set m_xml to NULL
CollectionRec();
virtual ~CollectionRec();
@ -387,8 +391,8 @@ class CollectionRec {
// for regular crawls
bool rebuildUrlFilters2();
// for diffbot crawl or bulk jobs
bool rebuildUrlFiltersDiffbot();
// for diffbot crawl or bulk jobs
bool rebuildUrlFiltersDiffbot();
bool rebuildLangRules( char *lang , char *tld );

20
Conf.cpp

@ -174,20 +174,32 @@ bool Conf::isCollAdmin2 ( TcpSocket *sock ,
// . only need to be from root IP *OR* have password, not both
bool Conf::isMasterAdmin ( TcpSocket *socket , HttpRequest *hr ) {
bool isAdmin = false;
// totally open access?
//if ( m_numConnectIps <= 0 && m_numMasterPwds <= 0 )
if ( m_connectIps.length() <= 0 &&
m_masterPwds.length() <= 0 )
return true;
isAdmin = true;
// coming from root gets you in
if ( isMasterIp ( socket->m_ip ) ) return true;
if ( socket && isMasterIp ( socket->m_ip ) )
isAdmin = true;
//if ( isConnectIp ( socket->m_ip ) ) return true;
if ( hasMasterPwd ( hr ) ) return true;
if ( hasMasterPwd ( hr ) )
isAdmin = true;
return false;
if ( ! isAdmin )
return false;
// default this to true so if user specifies &admin=0 then it
// cancels our admin view
if ( hr && ! hr->getLong("admin",1) )
return false;
return true;
}

19
HttpServer.cpp

@ -1299,11 +1299,11 @@ bool HttpServer::sendReply ( TcpSocket *s , HttpRequest *r , bool isAdmin) {
mnew ( f, sizeof(File), "HttpServer");
// note that
if ( g_conf.m_logDebugTcp )
log("tcp: new file=0x%"PTRFMT"",(PTRTYPE)f);
log("tcp: new filestate=0x%"PTRFMT"",(PTRTYPE)f);
// don't honor HUGE requests
if ( pathLen > 100 ) {
if ( g_conf.m_logDebugTcp )
log("tcp: deleting file=0x%"PTRFMT" [1]",
log("tcp: deleting filestate=0x%"PTRFMT" [1]",
(PTRTYPE)f);
mdelete ( f, sizeof(File), "HttpServer");
delete (f);
@ -1328,7 +1328,7 @@ bool HttpServer::sendReply ( TcpSocket *s , HttpRequest *r , bool isAdmin) {
// use default page if does not exist under host-specific path
if (pathLen == 11 && strncmp ( path , "/index.html" ,11 ) ==0){
if ( g_conf.m_logDebugTcp )
log("tcp: deleting file=0x%"PTRFMT" [2]",
log("tcp: deleting filestate=0x%"PTRFMT" [2]",
(PTRTYPE)f);
mdelete ( f, sizeof(File), "HttpServer");
delete (f);
@ -1349,7 +1349,7 @@ bool HttpServer::sendReply ( TcpSocket *s , HttpRequest *r , bool isAdmin) {
// if f STILL does not exist (or error) then send a 404
if ( f->doesExist() <= 0 ) {
if ( g_conf.m_logDebugTcp )
log("tcp: deleting file=0x%"PTRFMT" [3]",
log("tcp: deleting filestate=0x%"PTRFMT" [3]",
(PTRTYPE)f);
mdelete ( f, sizeof(File), "HttpServer");
delete (f);
@ -1376,7 +1376,8 @@ bool HttpServer::sendReply ( TcpSocket *s , HttpRequest *r , bool isAdmin) {
// . set g_errno on error
if ( bytesToSend > 0 && ! f->open(O_RDONLY | O_NONBLOCK | O_ASYNC)) {
if ( g_conf.m_logDebugTcp )
log("tcp: deleting file=0x%"PTRFMT" [4]",(PTRTYPE)f);
log("tcp: deleting filestate=0x%"PTRFMT" [4]",
(PTRTYPE)f);
mdelete ( f, sizeof(File), "HttpServer");
delete (f);
return sendErrorReply ( s , 404 , "Not Found" );
@ -1424,7 +1425,8 @@ bool HttpServer::sendReply ( TcpSocket *s , HttpRequest *r , bool isAdmin) {
log("http: got ZET request and am not proxy");
// bail
if ( g_conf.m_logDebugTcp )
log("tcp: deleting file=0x%"PTRFMT" [5]",(PTRTYPE)f);
log("tcp: deleting filestate=0x%"PTRFMT" [5]",
(PTRTYPE)f);
mdelete ( f, sizeof(File), "HttpServer");
delete (f);
g_errno = EBADREQUEST;
@ -1439,7 +1441,8 @@ bool HttpServer::sendReply ( TcpSocket *s , HttpRequest *r , bool isAdmin) {
char *sendBuf = (char *) mmalloc ( sendBufSize ,"HttpServer" );
if ( ! sendBuf ) {
if ( g_conf.m_logDebugTcp )
log("tcp: deleting file=0x%"PTRFMT" [6]",(PTRTYPE)f);
log("tcp: deleting filestate=0x%"PTRFMT" [6]",
(PTRTYPE)f);
mdelete ( f, sizeof(File), "HttpServer");
delete (f);
return sendErrorReply(s,500,mstrerror(g_errno));
@ -1688,7 +1691,7 @@ void cleanUp ( void *state , TcpSocket *s ) {
true );
}
if ( g_conf.m_logDebugTcp )
log("tcp: deleting file=0x%"PTRFMT" fd=%"INT32" [7] "
log("tcp: deleting filestate=0x%"PTRFMT" fd=%"INT32" [7] "
"s=0x%"PTRFMT"", (PTRTYPE)f,fd,(PTRTYPE)s);
// this should also close f
mdelete ( f, sizeof(File), "HttpServer");

33
Linkdb.cpp

@ -799,8 +799,21 @@ void handleRequest25 ( UdpSlot *slot , int32_t netnice ) {
if ( g_errno == ETRYAGAIN ) { char *xx=NULL;*xx=0; }
// wait for msg5 to be done reading list. this happens somehow,
// i'm not 100% sure how. code has too many indirections.
if ( m25->m_gettingList ) {
log("linkdb: avoiding core");
return;
}
// sanity
if ( m25->m_msg5.m_msg3.m_numScansCompleted <
m25->m_msg5.m_msg3.m_numScansStarted ) { char *xx=NULL;*xx=0; }
if ( g_errno )
log("linkdb: error getting linkinfo: %s",mstrerror(g_errno));
// else
// log("linkdb: got link info without blocking");
// it did not block... g_errno will be set on error so sendReply()
// should in that case send an error reply.
@ -1146,7 +1159,12 @@ bool Msg25::doReadLoop ( ) {
m_gettingList = false;
// debug log
if ( g_conf.m_logDebugBuild )
log("build: msg25 call to msg0 did not block");
log("build: msg25 call to msg5 did not block");
// sanity
if ( m_msg5.m_msg3.m_numScansCompleted <
m_msg5.m_msg3.m_numScansStarted ) { char *xx=NULL;*xx=0; }
// return true on error
if ( g_errno ) {
log("build: Had error getting linkers to url %s : %s.",
@ -1203,6 +1221,10 @@ bool Msg25::gotList() {
// reset # of docIds linking to us
//m_numDocIds = 0;
// sanity
if ( m_msg5.m_msg3.m_numScansCompleted <
m_msg5.m_msg3.m_numScansStarted ) { char *xx=NULL;*xx=0; }
//log("debug: entering gotlist this=%"XINT32"",(int32_t)this);
// return true on error
@ -1980,6 +2002,10 @@ bool Msg25::gotLinkText ( Msg20Request *req ) { // LinkTextReply *linkText ) {
g_errno == ENOSLOTS ) {
m_errors++;
if ( m_numReplies < m_numRequests ) return false;
if ( m_gettingList ) {
log("linkdb: gotLinkText: gettinglist1");
return false;
}
return true;
}
// otherwise, keep going, but this reply can not vote
@ -2249,6 +2275,11 @@ bool Msg25::gotLinkText ( Msg20Request *req ) { // LinkTextReply *linkText ) {
// wait for all replies to come in
if ( m_numReplies < m_numRequests ) return false;
if ( m_gettingList ) {
log("linkdb: gotLinkText: gettinglist2");
return false;
}
//
//
// READ MORE FROM LINKDB to avoid truncation

227
Loop.cpp

@ -52,6 +52,8 @@ int32_t g_numVTAlarms = 0;
int32_t g_numQuickPolls = 0;
int32_t g_missedQuickPolls = 0;
int32_t g_numSigChlds = 0;
int32_t g_numSigPipes = 0;
int32_t g_numSigIOs = 0;
int32_t g_numSigQueues = 0;
int32_t g_numSigOthers = 0;
@ -138,19 +140,18 @@ void Loop::unregisterReadCallback ( int fd, void *state ,
bool silent ){
if ( fd < 0 ) return;
// from reading
unregisterCallback ( m_readSlots , fd , state , callback ,
silent );
unregisterCallback ( m_readSlots,fd, state , callback, silent,true );
}
void Loop::unregisterWriteCallback ( int fd, void *state ,
void (* callback)(int fd,void *state)){
// from writing
unregisterCallback ( m_writeSlots , fd , state , callback );
unregisterCallback ( m_writeSlots , fd , state,callback,false,false);
}
void Loop::unregisterSleepCallback ( void *state ,
void (* callback)(int fd,void *state)){
unregisterCallback (m_readSlots,MAX_NUM_FDS,state,callback);
unregisterCallback (m_readSlots,MAX_NUM_FDS,state,callback,false,true);
}
static fd_set s_selectMaskRead;
@ -164,7 +165,7 @@ static int32_t s_numWriteFds = 0;
void Loop::unregisterCallback ( Slot **slots , int fd , void *state ,
void (* callback)(int fd,void *state) ,
bool silent ) {
bool silent , bool forReading ) {
// bad fd
if ( fd < 0 ) {log(LOG_LOGIC,
"loop: fd to unregister is negative.");return;}
@ -177,6 +178,7 @@ void Loop::unregisterCallback ( Slot **slots , int fd , void *state ,
// . sleep a min of 40ms so g_now is somewhat up to date
int32_t min = 40; // 0x7fffffff;
int32_t lastMin = min;
// chain through all callbacks registerd with this fd
while ( s ) {
// get the next slot (NULL if no more)
@ -196,23 +198,32 @@ void Loop::unregisterCallback ( Slot **slots , int fd , void *state ,
// so and clear a bit so doPoll() function is fast
if ( slots[fd] == s && s->m_next == NULL ) {
for (int32_t i = 0; i < s_numReadFds ; i++ ) {
if ( ! forReading ) break;
if ( s_readFds[i] != fd ) continue;
s_readFds[i] = s_readFds[s_numReadFds-1];
s_numReadFds--;
// remove from select mask too
FD_CLR(fd,&s_selectMaskRead );
if ( g_conf.m_logDebugLoop ||
g_conf.m_logDebugTcp )
log("loop: unregistering read "
"callback for fd=%i",fd);
break;
}
for (int32_t i = 0; i < s_numWriteFds ; i++ ) {
if ( forReading ) break;
if ( s_writeFds[i] != fd ) continue;
s_writeFds[i] = s_writeFds[s_numWriteFds-1];
s_numWriteFds--;
// remove from select mask too
FD_CLR(fd,&s_selectMaskWrite);
if ( g_conf.m_logDebugLoop )
log("loop: clearing fd=%"INT32" from "
"write #wrts=%"INT32""
,(int32_t)fd,(int32_t)s_numWriteFds);
if ( g_conf.m_logDebugLoop ||
g_conf.m_logDebugTcp )
log("loop: unregistering write "
"callback for fd=%"INT32" from "
"write #wrts=%"INT32"",
(int32_t)fd,
(int32_t)s_numWriteFds);
// FD_CLR(fd,&s_selectMaskExcept);
break;
}
@ -302,6 +313,12 @@ bool Loop::addSlot ( bool forReading , int fd, void *state,
log("loop: bad fd of %"INT32"",(int32_t)fd);
char *xx=NULL;*xx=0;
}
// debug note
if ( forReading && (g_conf.m_logDebugLoop || g_conf.m_logDebugTcp) )
log("loop: registering read callback sd=%i",fd);
else if ( g_conf.m_logDebugLoop || g_conf.m_logDebugTcp )
log("loop: registering write callback sd=%i",fd);
// . ensure fd not already registered with this callback/state
// . prevent dups so you can keep calling register w/o fear
Slot *s;
@ -312,7 +329,7 @@ bool Loop::addSlot ( bool forReading , int fd, void *state,
s->m_state == state ) {
// don't set g_errno for this anymore, just bitch
//g_errno = EBADENGINEER;
log(LOG_LOGIC,"loop: fd %i is already registered.",fd);
log(LOG_LOGIC,"loop: fd=%i is already registered.",fd);
return true;
}
s = s->m_next;
@ -448,9 +465,9 @@ void Loop::callCallbacks_ass ( bool forReading , int fd , int64_t now ,
int saved_errno = g_errno;
// get the first Slot in the chain that is waiting on this fd
Slot *s ;
//if ( forReading ) s = m_readSlots [ fd ];
//else s = m_writeSlots [ fd ];
s = m_readSlots [ fd ];
if ( forReading ) s = m_readSlots [ fd ];
else s = m_writeSlots [ fd ];
//s = m_readSlots [ fd ];
// ensure we called something
int32_t numCalled = 0;
@ -589,7 +606,7 @@ Loop::Loop ( ) {
// set all callbacks to NULL so we know they're empty
for ( int32_t i = 0 ; i < MAX_NUM_FDS+2 ; i++ ) {
m_readSlots [i] = NULL;
//m_writeSlots[i] = NULL;
m_writeSlots[i] = NULL;
}
// the extra sleep slots
//m_readSlots [ MAX_NUM_FDS ] = NULL;
@ -639,6 +656,16 @@ void sigHandlerQueue_r ( int x , siginfo_t *info , void *v ) {
return;
}
if ( info->si_signo == SIGPIPE ) {
g_numSigPipes++;
return;
}
if ( info->si_signo == SIGIO ) {
g_numSigIOs++;
return;
}
if ( info->si_code == SI_QUEUE ) {
g_numSigQueues++;
//log("admin: got sigqueue");
@ -647,6 +674,8 @@ void sigHandlerQueue_r ( int x , siginfo_t *info , void *v ) {
return;
}
// wtf is this?
g_numSigOthers++;
@ -1197,6 +1226,7 @@ void sigalrmHandler ( int x , siginfo_t *info , void *y ) {
g_profiler.getStackFrame(0);
}
/*
static sigset_t s_rtmin;
void maskSignals() {
@ -1228,7 +1258,7 @@ void unmaskSignals() {
return;
}
}
*/
// shit, we can't make this realtime!! RdbClose() cannot be called by a
// real time sig handler
@ -1870,6 +1900,8 @@ void Loop::doPoll ( ) {
// based it only goes off when that much "cpu time" has elapsed.
else v.tv_usec = QUICKPOLL_INTERVAL * 1000;
//int32_t count = v.tv_usec;
// set descriptors we should watch
// MDW: no longer necessary since we have s_selectMaskRead, etc.
// for ( int32_t i = 0 ; i < MAX_NUM_FDS ; i++ ) {
@ -1882,12 +1914,23 @@ void Loop::doPoll ( ) {
// FD_SET ( i , &exceptfds );
// }
// }
again:
// gotta copy to our own since bits get cleared by select() function
fd_set readfds;
fd_set writefds;
fd_set exceptfds;
// only register write callbacks if TcpServer.cpp failed to write
// the # of bytes that it wanted to a socket descriptor. and it
// should unregister the writecallback as soon as it is able to
// write the bytes it wanted to write.
//FD_ZERO ( &writefds );
//int64_t startTime = gettimeofdatInMillisecondsLocal();
again:
//fd_set exceptfds;
gbmemcpy ( &readfds, &s_selectMaskRead , sizeof(fd_set) );
gbmemcpy ( &writefds, &s_selectMaskWrite , sizeof(fd_set) );
//gbmemcpy ( &exceptfds, &s_selectMaskExcept , sizeof(fd_set) );
@ -1895,7 +1938,7 @@ void Loop::doPoll ( ) {
// what is the point of fds for writing... its for when we
// get a new socket via accept() it is read for writing...
//FD_ZERO ( &writefds );
FD_ZERO ( &exceptfds );
//FD_ZERO ( &exceptfds );
if ( g_conf.m_logDebugLoop )
log("loop: in select");
@ -1916,18 +1959,26 @@ void Loop::doPoll ( ) {
// . poll the fd's searching for socket closes
// . the sigalrms and sigvtalrms and SIGCHLDs knock us out of this
// select() with n < 0 and errno equal to EINTR
// select() with n < 0 and errno equal to EINTR.
// . crap the sigalarms kick us out here every 1ms. i noticed
// then when running disableTimer() above and we don't get
// any EINTRs... can we mask those out here? it only seems to be
// the SIGALRMs not the SIGVTALRMs that interrupt us.
n = select (MAX_NUM_FDS,
&readfds,
&writefds,
&exceptfds,
NULL,//&exceptfds,
&v );
g_inWaitState = false;
if ( n >= 0 ) errno = 0;
if ( g_conf.m_logDebugLoop )
log("loop: out select n=%"INT32" errno=%"INT32" errnomsg=%s",
(int32_t)n,(int32_t)errno,mstrerror(errno));
log("loop: out select n=%"INT32" errno=%"INT32" errnomsg=%s "
"ms_wait=%i",
(int32_t)n,(int32_t)errno,mstrerror(errno),
(int)v.tv_sec*1000);
if ( n < 0 ) {
// valgrind
@ -1935,13 +1986,25 @@ void Loop::doPoll ( ) {
// got it. if we get a sig alarm or vt alarm or
// SIGCHLD (from Threads.cpp) we end up here.
//log("loop: got errno=%"INT32"",(int32_t)errno);
// if shutting own was it a sigterm ?
// if not linux we have to decrease this by 1ms
//count -= 1000;
// and re-assign to wait less time. we are
// assuming SIGALRM goes off once per ms and if
// that is not what interrupted us we may end
// up exiting early
//if ( count <= 0 && m_shutdown ) return;
// wait less this time around
//v.tv_usec = count;
// if shutting down was it a sigterm ?
if ( m_shutdown ) goto again;
// handle returned threads for niceness 0
g_threads.timedCleanUp(-3,0); // 3 ms
if ( g_threads.m_needsCleanup )
g_threads.timedCleanUp(-3,0); // 3 ms
if ( m_inQuickPoll ) goto again;
// high niceness threads
g_threads.timedCleanUp(-4,MAX_NICENESS); // 3 ms
if ( g_threads.m_needsCleanup )
g_threads.timedCleanUp(-4,MAX_NICENESS); //3 ms
goto again;
}
g_errno = errno;
@ -1959,19 +2022,21 @@ void Loop::doPoll ( ) {
if ( g_conf.m_logDebugLoop)
logf(LOG_DEBUG,"loop: Got %"INT32" fds waiting.",n);
if ( g_conf.m_logDebugLoop) {
for ( int32_t i = 0 ; i < MAX_NUM_FDS ; i++ ) {
for ( int32_t i = 0 ;
(g_conf.m_logDebugLoop || g_conf.m_logDebugTcp) && i<MAX_NUM_FDS;
i++){
// continue if not set for reading
if ( FD_ISSET ( i , &readfds ) )
log("loop: fd %"INT32" is on for read",i);
if ( FD_ISSET ( i , &readfds ) )
log("loop: fd=%"INT32" is on for read qp=%i",i,
(int)m_inQuickPoll);
if ( FD_ISSET ( i , &writefds ) )
log("loop: fd %"INT32" is on for write",i);
if ( FD_ISSET ( i , &exceptfds ) )
log("loop: fd %"INT32" is on for except",i);
log("loop: fd=%"INT32" is on for write qp=%i",i,
(int)m_inQuickPoll);
// if ( FD_ISSET ( i , &exceptfds ) )
// log("loop: fd %"INT32" is on for except",i);
// debug
// if niceness is not -1, handle it below
}
}
// . reset the need to poll flag if everything is caught up now
@ -2026,35 +2091,28 @@ void Loop::doPoll ( ) {
// if niceness is not 0, handle it below
if ( s && s->m_niceness > 0 ) continue;
// must be set
if ( FD_ISSET ( fd , &readfds ) ) {
if ( g_conf.m_logDebugLoop )
log("loop: calling cback0 niceness=%"INT32" fd=%i"
, s->m_niceness , fd );
calledOne = true;
callCallbacks_ass (true/*forReading?*/,fd, g_now,0);
}
// fds are always ready for writing so take this out.
// our read callbacks always try to do a write as well.
if ( FD_ISSET ( fd , &writefds ) ) {
if ( g_conf.m_logDebugLoop )
log("loop: calling cback0 niceness=%"INT32" fd=%i"
, s->m_niceness , fd );
calledOne = true;
callCallbacks_ass (false/*forReading?*/,fd, g_now,0);
}
if ( ! FD_ISSET ( fd , &readfds ) ) continue;
if ( g_conf.m_logDebugLoop || g_conf.m_logDebugTcp )
log("loop: calling cback0 niceness=%"INT32" "
"fd=%i", s->m_niceness , fd );
calledOne = true;
callCallbacks_ass (true,fd, g_now,0);//read?
}
for ( int32_t i = 0 ; i < s_numWriteFds ; i++ ) {
if ( n == 0 ) break;
int fd = s_writeFds[i];
s = m_writeSlots [ fd ];
// if niceness is not 0, handle it below
if ( s && s->m_niceness > 0 ) continue;
// fds are always ready for writing so take this out.
if ( ! FD_ISSET ( fd , &writefds ) ) continue;
if ( g_conf.m_logDebugLoop || g_conf.m_logDebugTcp )
log("loop: calling wcback0 niceness=%"INT32" fd=%i"
, s->m_niceness , fd );
calledOne = true;
callCallbacks_ass (false,fd, g_now,0);//false=forRead?
}
// for ( int32_t i = 0 ; i < s_numWriteFds ; i++ ) {
// if ( n == 0 ) break;
// int fd = s_writeFds[i];
// s = m_writeSlots [ fd ];
// // if niceness is not 0, handle it below
// if ( s && s->m_niceness > 0 ) continue;
// // must be set
// if ( FD_ISSET ( fd , &writefds ) )
// callCallbacks_ass (false/*forReading?*/,fd, g_now,1);
// }
// handle returned threads for niceness 0
g_threads.timedCleanUp(-3,0); // 3 ms
@ -2068,36 +2126,31 @@ void Loop::doPoll ( ) {
if ( n == 0 ) break;
int fd = s_readFds[i];
s = m_readSlots [ fd ];
// if niceness is not 0, handle it below
// if niceness is <= 0 we did it above
if ( s && s->m_niceness <= 0 ) continue;
// must be set
if ( FD_ISSET ( fd , &readfds ) ) {
if ( g_conf.m_logDebugLoop )
log("loop: calling cback1 niceness=%"INT32" fd=%i"
, s->m_niceness , fd );
calledOne = true;
callCallbacks_ass (true/*forReading?*/,fd, g_now,1);
}
// fds are always ready for writing so take this out.
// our read callbacks always try to do a write as well.
if ( FD_ISSET ( fd , &writefds ) ) {
if ( g_conf.m_logDebugLoop )
log("loop: calling cback1 niceness=%"INT32" fd=%i"
, s->m_niceness , fd );
calledOne = true;
callCallbacks_ass (false/*forReading?*/,fd, g_now,1);
}
if ( ! FD_ISSET ( fd , &readfds ) ) continue;
if ( g_conf.m_logDebugLoop || g_conf.m_logDebugTcp )
log("loop: calling cback1 niceness=%"INT32" "
"fd=%i", s->m_niceness , fd );
calledOne = true;
callCallbacks_ass (true,fd, g_now,1);//read?
}
for ( int32_t i = 0 ; i < s_numWriteFds ; i++ ) {
if ( n == 0 ) break;
int fd = s_writeFds[i];
s = m_writeSlots [ fd ];
// if niceness is <= 0 we did it above
if ( s && s->m_niceness <= 0 ) continue;
// must be set
if ( ! FD_ISSET ( fd , &writefds ) ) continue;
if ( g_conf.m_logDebugLoop || g_conf.m_logDebugTcp )
log("loop: calling wcback1 niceness=%"INT32" "
"fd=%i", s->m_niceness , fd );
calledOne = true;
callCallbacks_ass (false,fd, g_now,1);//forread?
}
// for ( int32_t i = 0 ; i < s_numWriteFds ; i++ ) {
// if ( n == 0 ) break;
// int fd = s_writeFds[i];
// s = m_writeSlots [ fd ];
// // if niceness is not 0, handle it below
// if ( s && s->m_niceness <= 0 ) continue;
// // must be set
// if ( FD_ISSET ( fd , &writefds ) )
// callCallbacks_ass (false/*forReading?*/,fd, g_now,1);
// }
//if ( ! calledOne )
// log("loop: select returned n=%"INT32" but nothing called.",n);

5
Loop.h

@ -96,6 +96,8 @@ extern int32_t g_numQuickPolls;
extern int32_t g_numSigChlds;
extern int32_t g_numSigQueues;
extern int32_t g_numSigPipes;
extern int32_t g_numSigIOs;
extern int32_t g_numSigOthers;
@ -201,7 +203,8 @@ class Loop {
void unregisterCallback ( Slot **slots , int fd , void *state ,
void (* callback)(int fd,void *state) ,
bool silent = false );
bool silent , // = false );
bool forReading );
bool addSlot ( bool forReading , int fd , void *state ,
void (* callback)(int fd , void *state ) ,

15
Msg13.cpp

@ -1034,6 +1034,17 @@ void downloadTheDocForReals3b ( Msg13Request *r ) {
// into the hammercache
r->m_downloadStartTimeMS = nowms;
// prevent core from violating MAX_DGRAMS #defined in UdpSlot.h
int32_t maxDocLen1 = r->m_maxTextDocLen;
int32_t maxDocLen2 = r->m_maxOtherDocLen;
// fix core in UdpServer.cpp from sending back a too big reply
if ( maxDocLen1 < 0 || maxDocLen1 > MAX_ABSDOCLEN )
maxDocLen1 = MAX_ABSDOCLEN;
if ( maxDocLen2 < 0 || maxDocLen2 > MAX_ABSDOCLEN )
maxDocLen2 = MAX_ABSDOCLEN;
// . download it
// . if m_proxyIp is non-zero it will make requests like:
// GET http://xyz.com/abc
@ -1047,8 +1058,8 @@ void downloadTheDocForReals3b ( Msg13Request *r ) {
30*1000 , // 30 sec timeout
r->m_proxyIp ,
r->m_proxyPort ,
r->m_maxTextDocLen ,
r->m_maxOtherDocLen ,
maxDocLen1,//r->m_maxTextDocLen ,
maxDocLen2,//r->m_maxOtherDocLen ,
agent ,
DEFAULT_HTTP_PROTO , // "HTTP/1.0"
false , // doPost?

4
Msge1.cpp

@ -116,7 +116,7 @@ bool Msge1::getFirstIps ( TagRec **grv ,
if ( ! launchRequests ( 0 ) ) return false;
// save it? might be a page parser
//if ( ! strcmp(m_coll,"qatest123") ) saveTestBuf();
if ( m_coll && ! strcmp(m_coll,"qatest123") ) saveTestBuf("qa");
// none blocked, we are done
return true;
@ -340,7 +340,7 @@ void gotMsgCWrapper ( void *state , int32_t ip ) {
if ( ! THIS->launchRequests(i) ) return;
// . save it if we should. might be a page parser
// . mdw i uncommented this when we cored all the time
//if ( ! strcmp(THIS->m_coll,"qatest123")) saveTestBuf();
if ( THIS->m_coll&&!strcmp(THIS->m_coll,"qatest123"))saveTestBuf("qa");
// must be all done, call the callback
THIS->m_callback ( THIS->m_state );
}

4
PageBasic.cpp

@ -97,6 +97,10 @@ bool updateSiteListBuf ( collnum_t collnum ,
CollectionRec *cr = g_collectiondb.getRec ( collnum );
if ( ! cr ) return true;
// tell spiderloop to update the active list in case this
// collection suddenly becomes active
g_spiderLoop.m_activeListValid = false;
// this might make a new spidercoll...
SpiderColl *sc = g_spiderCache.getSpiderColl ( cr->m_collnum );

4
PageHosts.cpp

@ -553,7 +553,7 @@ skipReplaceHost:
,h->m_pingInfo.m_currentSpiders
);
if ( format == FORMAT_HTML ) {
if ( format == FORMAT_HTML && h->m_pingInfo.m_udpSlotsInUse ) {
char *f1 = "";
char *f2 = "";
if ( h->m_pingInfo.m_udpSlotsInUse >= 200 ) {
@ -576,7 +576,7 @@ skipReplaceHost:
);
}
if ( format == FORMAT_HTML ) {
if ( format == FORMAT_HTML && h->m_pingInfo.m_tcpSocketsInUse){
char *f1 = "";
char *f2 = "";
if ( h->m_pingInfo.m_tcpSocketsInUse >= 100 ) {

95
PageResults.cpp

@ -2057,6 +2057,51 @@ bool printLeftNavColumn ( SafeBuf &sb, State0 *st ) {
return true;
}
bool printIgnoredWords ( SafeBuf *sb , SearchInput *si ) {
// mention ignored query terms
// we need to set another Query with "keepAllSingles" set to false
Query *qq2 = &si->m_q;
//qq2.set ( q , qlen , NULL , 0 , si->m_boolFlag , false );
bool firstIgnored = true;
for ( int32_t i = 0 ; i < qq2->m_numWords ; i++ ) {
//if ( si->m_xml ) break;
QueryWord *qw = &qq2->m_qwords[i];
// only print out words ignored cuz they were stop words
if ( qw->m_ignoreWord != IGNORE_QSTOP ) continue;
// print header -- we got one
if ( firstIgnored ) {
if ( si->m_format == FORMAT_XML )
sb->safePrintf ("\t<ignoredWords><![CDATA[");
else if ( si->m_format == FORMAT_JSON )
sb->safePrintf ("\t\"ignoredWords\":\"");
else if ( si->m_format == FORMAT_HTML )
sb->safePrintf ("<br><font "
"color=\"#707070\">The "
"following query words "
"were ignored: "
"<b>");
firstIgnored = false;
}
// print the word
char *t = qw->m_word;
int32_t tlen = qw->m_wordLen;
sb->utf8Encode2 ( t , tlen );
sb->safePrintf (" ");
}
// print tail if we had ignored terms
if ( ! firstIgnored ) {
sb->incrementLength(-1);
if ( si->m_format == FORMAT_XML )
sb->safePrintf("]]></ignoredWords>\n");
else if ( si->m_format == FORMAT_JSON )
sb->safePrintf("\",\n");
else if ( si->m_format == FORMAT_HTML )
sb->safePrintf ("</b>. Preceed each with a '+' or "
"wrap in "
"quotes to not ignore.</font>");
}
return true;
}
bool printSearchResultsHeader ( State0 *st ) {
@ -2421,6 +2466,8 @@ bool printSearchResultsHeader ( State0 *st ) {
"<![CDATA[%s]]>"
"</queryLanguage>\n"
, getLanguageString(si->m_queryLangId) );
// print query words we ignored, like stop words
printIgnoredWords ( sb , si );
for ( int i = 0 ; i < q->m_numTerms ; i++ ) {
sb->safePrintf("\t\t<term>\n");
QueryTerm *qt = &q->m_qterms[i];
@ -2494,6 +2541,8 @@ bool printSearchResultsHeader ( State0 *st ) {
sb->safePrintf("\t\"queryLanguage\":\"");
sb->jsonEncode ( getLanguageString(si->m_queryLangId) );
sb->safePrintf("\",\n");
// print query words we ignored, like stop words
printIgnoredWords ( sb , si );
sb->safePrintf("\t\"terms\":[\n");
for ( int i = 0 ; i < q->m_numTerms ; i++ ) {
sb->safePrintf("\t\t{\n");
@ -2720,8 +2769,8 @@ bool printSearchResultsHeader ( State0 *st ) {
ulltoa ( inbuf , docsInColl );
Query qq3;
Query *qq2;
bool firstIgnored;
//Query *qq2;
//bool firstIgnored;
//bool isAdmin = si->m_isMasterAdmin;
bool isAdmin = (si->m_isMasterAdmin || si->m_isCollAdmin);
if ( si->m_format != FORMAT_HTML ) isAdmin = false;
@ -2981,44 +3030,10 @@ bool printSearchResultsHeader ( State0 *st ) {
}
*/
// mention ignored query terms
// we need to set another Query with "keepAllSingles" set to false
qq2 = &si->m_q;
//qq2.set ( q , qlen , NULL , 0 , si->m_boolFlag , false );
firstIgnored = true;
for ( int32_t i = 0 ; i < qq2->m_numWords ; i++ ) {
//if ( si->m_xml ) break;
QueryWord *qw = &qq2->m_qwords[i];
// only print out words ignored cuz they were stop words
if ( qw->m_ignoreWord != IGNORE_QSTOP ) continue;
// print header -- we got one
if ( firstIgnored ) {
if ( si->m_format == FORMAT_XML )
sb->safePrintf ("\t<ignoredWords><![CDATA[");
else if ( si->m_format == FORMAT_HTML )
sb->safePrintf (" &nbsp; <font "
"color=\"#707070\">The "
"following query words "
"were ignored: "
"<b>");
firstIgnored = false;
}
// print the word
char *t = qw->m_word;
int32_t tlen = qw->m_wordLen;
sb->utf8Encode2 ( t , tlen );
sb->safePrintf (" ");
}
// print tail if we had ignored terms
if ( ! firstIgnored ) {
sb->incrementLength(-1);
if ( si->m_format == FORMAT_XML )
sb->safePrintf("]]></ignoredWords>\n");
else if ( si->m_format == FORMAT_HTML )
sb->safePrintf ("</b>. Preceed each with a '+' or "
"wrap in "
"quotes to not ignore.</font>");
}
printIgnoredWords ( sb , si );
if ( si->m_format == FORMAT_HTML ) sb->safePrintf("<br><br>");

17
PageStats.cpp

@ -580,6 +580,7 @@ bool sendPageStats ( TcpSocket *s , HttpRequest *r ) {
caches[2] = g_dns.getCache();
caches[3] = g_dns.getCacheLocal();
caches[4] = resultsCache;
caches[5] = &g_spiderLoop.m_winnerListCache;
//caches[5] = &g_termListCache;
//caches[6] = &g_genericCache[SEORESULTS_CACHEID];
//caches[5] = &g_qtable;
@ -590,7 +591,7 @@ bool sendPageStats ( TcpSocket *s , HttpRequest *r ) {
//caches[6] = &g_forcedCache;
//caches[9] = &g_msg20Cache;
//caches[10] = &g_tagdb.m_listCache;
int32_t numCaches = 5;
int32_t numCaches = 6;
if ( format == FORMAT_HTML )
p.safePrintf (
@ -832,6 +833,8 @@ bool sendPageStats ( TcpSocket *s , HttpRequest *r ) {
"<tr class=poo><td><b>SIGCHLDS</b></td><td>%"INT32"</td></tr>\n"
"<tr class=poo><td><b>SIGQUEUES</b></td><td>%"INT32"</td></tr>\n"
"<tr class=poo><td><b>SIGPIPES</b></td><td>%"INT32"</td></tr>\n"
"<tr class=poo><td><b>SIGIOS</b></td><td>%"INT32"</td></tr>\n"
"<tr class=poo><td><b>SIGOTHERS</b></td><td>%"INT32"</td></tr>\n"
//"<tr class=poo><td><b>read signals</b></td><td>%"INT64"</td></tr>\n"
@ -859,6 +862,8 @@ bool sendPageStats ( TcpSocket *s , HttpRequest *r ) {
g_numSigChlds,
g_numSigQueues,
g_numSigPipes,
g_numSigIOs,
g_numSigOthers,
//g_stats.m_readSignals,
@ -1818,9 +1823,9 @@ bool sendPageStats ( TcpSocket *s , HttpRequest *r ) {
g_tagdb.getRdb(),
g_clusterdb.getRdb(),
g_linkdb.getRdb(),
g_cachedb.getRdb(),
g_serpdb.getRdb(),
g_monitordb.getRdb(),
//g_cachedb.getRdb(),
//g_serpdb.getRdb(),
//g_monitordb.getRdb(),
g_statsdb.getRdb(),
g_catdb.getRdb()
//g_placedb.getRdb() ,
@ -2288,8 +2293,8 @@ bool sendPageStats ( TcpSocket *s , HttpRequest *r ) {
"<br><i>Note: twins may differ in rec counts but still have "
"the same data because they dump at different times which "
"leads to different reactions. To see if truly equal, "
"do a 'gb ddump' then when that finishes, a, 'gb imerge'"
"and a 'gb tmerge'\n");
"do a 'gb ddump' then when that finishes, a, 'gb pmerge'"
"for posdb or a 'gb tmerge' for titledb.\n");
// print the final tail
//p += g_httpServer.printTail ( p , pend - p );

59
Pages.cpp

@ -1446,20 +1446,25 @@ bool Pages::printAdminTop (SafeBuf *sb ,
int32_t doneCount = 0;
int32_t activeCount = 0;
int32_t pauseCount = 0;
int32_t betweenRoundsCount = 0;
uint32_t nowGlobal = (uint32_t)getTimeGlobal();
for (int32_t i = 0 ; i < g_collectiondb.m_numRecs ; i++ ) {
CollectionRec *cc = g_collectiondb.m_recs[i];
if ( ! cc ) continue;
CrawlInfo *ci = &cc->m_globalCrawlInfo;
if ( cc->m_spideringEnabled &&
! ci->m_hasUrlsReadyToSpider &&
ci->m_urlsHarvested )
if ( cc->m_spideringEnabled &&
nowGlobal < cc->m_spiderRoundStartTime )
betweenRoundsCount++;
else if ( cc->m_spideringEnabled &&
! ci->m_hasUrlsReadyToSpider &&
ci->m_urlsHarvested )
emptyCount++;
else if ( ! ci->m_hasUrlsReadyToSpider )
doneCount++;
else if (cc->m_spideringEnabled && ci->m_hasUrlsReadyToSpider )
activeCount++;
else if (!cc->m_spideringEnabled && ci->m_hasUrlsReadyToSpider)
pauseCount++;
else if (cc->m_spideringEnabled && ci->m_hasUrlsReadyToSpider )
activeCount++;
}
@ -1470,29 +1475,42 @@ bool Pages::printAdminTop (SafeBuf *sb ,
"<b>Key</b>"
"<br>"
"<br>"
"\n"
);
sb->safePrintf(
"<font color=black>"
"&#x25cf;</font> spider is done (%"INT32")"
"<br>"
"\n"
"<font color=orange>"
"&#x25cf;</font> spider is paused (%"INT32")"
"<br>"
"\n"
"<font color=green>"
"&#x25cf;</font> spider is active (%"INT32")"
"<br>"
"\n"
"<font color=gray>"
"&#x25cf;</font> spider queue empty (%"INT32")"
"<br>"
"\n"
"<font color=blue>"
"&#x25cf;</font> between rounds (%"INT32")"
"<br>"
"\n"
"</div>"
,doneCount
,pauseCount
,activeCount
,emptyCount
,betweenRoundsCount
);
@ -2638,13 +2656,27 @@ bool Pages::printCollectionNavBar ( SafeBuf *sb ,
".e{background-color:#e0e0e0;}"
"</style>\n");
int32_t showAll = hr->getLong("showall",0);
int32_t row = 0;
uint32_t nowGlobal = (uint32_t)getTimeGlobal();
int32_t numPrinted = 0;
bool printMsg = false;
//for ( int32_t i = a ; i < b ; i++ ) {
for ( int32_t i = 0 ; i < g_collectiondb.m_numRecs ; i++ ) {
CollectionRec *cc = g_collectiondb.m_recs[i];
if ( ! cc ) continue;
if ( numPrinted >= 20 && ! showAll ) {
printMsg = true;
break;
}
// count it
numPrinted++;
//
// CLOUD SEARCH ENGINE SUPPORT
//
@ -2694,6 +2726,11 @@ bool Pages::printCollectionNavBar ( SafeBuf *sb ,
ci->m_urlsHarvested )
bcolor = "gray";
if ( cc->m_spideringEnabled &&
nowGlobal < cc->m_spiderRoundStartTime )
bcolor = "blue";
sb->safePrintf("<font color=%s>&#x25cf;</font> ",bcolor);
if ( i != collnum || ! highlight )// || ! coll || ! coll[0])
@ -2717,8 +2754,20 @@ bool Pages::printCollectionNavBar ( SafeBuf *sb ,
sb->safePrintf("</div>\n");
else
sb->safePrintf("<br>\n");
}
if ( showAll ) return status;
// convert our current page number to a path
if ( printMsg ) {
char *path = s_pages[page].m_filename;
sb->safePrintf("<a href=\"/%s?c=%s&showall=1\">"
"...show all...</a><br>"
, path , coll );
}
//sb->safePrintf ( "</center><br/>" );
return status;

24
Parms.cpp

@ -621,6 +621,8 @@ bool CommandParserTestInit ( char *rec ) {
CollectionRec *cr = g_collectiondb.getRec("qatest123");
// turn on spiders
if ( cr ) cr->m_spideringEnabled = 1;
// tell spider loop to update active list
g_spiderLoop.m_activeListValid = false;
// if we are not host 0, turn on spiders for testing
if ( g_hostdb.m_myHost->m_hostId != 0 ) return true;
// start the test loop to inject urls for parsing/spidering
@ -641,6 +643,8 @@ bool CommandSpiderTestInit ( char *rec ) {
CollectionRec *cr = g_collectiondb.getRec("qatest123");
// turn on spiders
if ( cr ) cr->m_spideringEnabled = 1;
// tell spider loop to update active list
g_spiderLoop.m_activeListValid = false;
// if we are not host 0, turn on spiders for testing
if ( g_hostdb.m_myHost->m_hostId != 0 ) return true;
// start the test loop to inject urls for parsing/spidering
@ -659,6 +663,8 @@ bool CommandSpiderTestCont ( char *rec ) {
CollectionRec *cr = g_collectiondb.getRec("qatest123");
// turn on spiders
if ( cr ) cr->m_spideringEnabled = 1;
// tell spider loop to update active list
g_spiderLoop.m_activeListValid = false;
// done
return true;
}
@ -3280,6 +3286,11 @@ void Parms::setParm ( char *THIS , Parm *m , int32_t mm , int32_t j , char *s ,
oldVal,
newVal);
// if they turn spiders on or off then tell spiderloop to update
// the active list
//if ( strcmp(m->m_cgi,"cse") )
// g_spiderLoop.m_activeListValid = false;
// only send email alerts if we are host 0 since everyone syncs up
// with host #0 anyway
if ( g_hostdb.m_hostId != 0 ) return;
@ -3295,6 +3306,7 @@ void Parms::setParm ( char *THIS , Parm *m , int32_t mm , int32_t j , char *s ,
if ( strcmp(m->m_cgi,"se")==0 && g_conf.m_spideringEnabled )
return;
char tmp[1024];
Host *h0 = g_hostdb.getHost ( 0 );
int32_t ip0 = 0;
@ -16216,7 +16228,10 @@ void Parms::init ( ) {
m->m_def = "1";
m->m_page = PAGE_SPIDER;
m->m_obj = OBJ_COLL;
m->m_flags = PF_CLONE;
// this linked list of colls is in Spider.cpp and used to only
// poll the active spider colls for spidering. so if coll
// gets paused/unpaused we have to update it.
m->m_flags = PF_CLONE | PF_REBUILDACTIVELIST;
m++;
m->m_title = "site list";
@ -21255,6 +21270,9 @@ void handleRequest3fLoop ( void *weArg ) {
if ( parm->m_flags & PF_REBUILDPROXYTABLE )
we->m_doProxyRebuild = true;
if ( parm->m_flags & PF_REBUILDACTIVELIST )
we->m_rebuildActiveList = true;
// get collnum i guess
if ( parm->m_type != TYPE_CMD )
we->m_collnum = getCollnumFromParmRec ( rec );
@ -21333,6 +21351,9 @@ void handleRequest3fLoop ( void *weArg ) {
cx->rebuildUrlFilters();
}
if ( we->m_rebuildActiveList && cx )
g_spiderLoop.m_activeListValid = false;
// if user changed the list of proxy ips rebuild the binary
// array representation of the proxy ips we have
if ( we->m_doProxyRebuild )
@ -21378,6 +21399,7 @@ void handleRequest3f ( UdpSlot *slot , int32_t niceness ) {
we->m_parmEnd = parmEnd;
we->m_errno = 0;
we->m_doRebuilds = false;
we->m_rebuildActiveList = false;
we->m_updatedRound = false;
we->m_doProxyRebuild = false;
we->m_collnum = -1;

1
Parms.h

@ -234,6 +234,7 @@ class GigablastRequest {
#define PF_CLONE 0x20000
#define PF_PRIVATE 0x40000 // for password to not show in api
#define PF_SMALLTEXTAREA 0x80000
#define PF_REBUILDACTIVELIST 0x100000
class Parm {
public:

45
Process.cpp

@ -1402,13 +1402,13 @@ bool Process::save2 ( ) {
// . Msg1 requests will get ETRYAGAIN error replies
// . this is instantaneous because all tree mods happen in this
// main process, not in a thread
disableTreeWrites();
disableTreeWrites( false );
bool useThreads = true;
// . tell all rdbs to save trees
// . will return true if no rdb tree needs a save
if ( ! saveRdbTrees ( useThreads ) ) return false;
if ( ! saveRdbTrees ( useThreads , false ) ) return false;
// . save all rdb maps if they need it
// . will return true if no rdb map needs a save
@ -1440,7 +1440,7 @@ bool Process::save2 ( ) {
g_cacheWritesEnabled = true;
// reenable tree writes since saves were completed
enableTreeWrites();
enableTreeWrites( false );
log(LOG_INFO,"gb: Saved data to disk. Re-enabling Writes.");
@ -1463,7 +1463,8 @@ bool Process::shutdown2 ( ) {
if ( g_threads.amThread() ) return true;
if ( m_urgent )
log(LOG_INFO,"gb: Shutting down urgently. Try #%"INT32".",m_try++);
log(LOG_INFO,"gb: Shutting down urgently. Try #%"INT32".",
m_try++);
else
log(LOG_INFO,"gb: Shutting down. Try #%"INT32".",m_try++);
@ -1513,11 +1514,11 @@ bool Process::shutdown2 ( ) {
// . Msg1 requests will get ECLOSING error msgs
// . this is instantaneous because all tree mods happen in this
// main process, not in a thread
disableTreeWrites();
disableTreeWrites( true );
// . tell all rdbs to save trees
// . will return true if no rdb tree needs a save
if ( ! saveRdbTrees ( useThreads ) )
if ( ! saveRdbTrees ( useThreads , true ) )
if ( ! m_urgent ) return false;
// save this right after the trees in case we core
@ -1652,12 +1653,19 @@ bool Process::shutdown2 ( ) {
return true;
}
void Process::disableTreeWrites ( ) {
void Process::disableTreeWrites ( bool shuttingDown ) {
// loop over all Rdbs
for ( int32_t i = 0 ; i < m_numRdbs ; i++ ) {
Rdb *rdb = m_rdbs[i];
// if we save doledb while spidering it screws us up
// because Spider.cpp can not directly write into the
// rdb tree and it expects that to always be available!
if ( ! shuttingDown && rdb->m_rdbId == RDB_DOLEDB )
continue;
rdb->disableWrites();
}
// don't save spider related trees if not shutting down
if ( ! shuttingDown ) return;
// disable all spider trees and tables
for ( int32_t i = 0 ; i < g_collectiondb.m_numRecs ; i++ ) {
SpiderColl *sc = g_spiderCache.getSpiderCollIffNonNull(i);
@ -1669,12 +1677,14 @@ void Process::disableTreeWrites ( ) {
}
void Process::enableTreeWrites ( ) {
void Process::enableTreeWrites ( bool shuttingDown ) {
// loop over all Rdbs
for ( int32_t i = 0 ; i < m_numRdbs ; i++ ) {
Rdb *rdb = m_rdbs[i];
rdb->enableWrites();
}
// don't save spider related trees if not shutting down
if ( ! shuttingDown ) return;
// enable all waiting trees
for ( int32_t i = 0 ; i < g_collectiondb.m_numRecs ; i++ ) {
SpiderColl *sc = g_spiderCache.getSpiderCollIffNonNull(i);
@ -1707,14 +1717,21 @@ bool Process::isRdbMerging ( ) {
// . returns false if blocked, true otherwise
// . calls callback when done saving
bool Process::saveRdbTrees ( bool useThread ) {
bool Process::saveRdbTrees ( bool useThread , bool shuttingDown ) {
// never if in read only mode
if ( g_conf.m_readOnlyMode ) return true;
// no thread if shutting down
if ( shuttingDown ) useThread = false;
// turn off statsdb until everyone is done
//g_statsdb.m_disabled = true;
// loop over all Rdbs and save them
for ( int32_t i = 0 ; ! m_calledSave && i < m_numRdbs ; i++ ) {
Rdb *rdb = m_rdbs[i];
// if we save doledb while spidering it screws us up
// because Spider.cpp can not directly write into the
// rdb tree and it expects that to always be available!
if ( ! shuttingDown && rdb->m_rdbId == RDB_DOLEDB )
continue;
rdb->saveTree ( useThread );
}
@ -1728,7 +1745,7 @@ bool Process::saveRdbTrees ( bool useThread ) {
// launched.
// . and sets m_isSaving=false on SpiderCache::doneSaving when they
// are all done.
g_spiderCache.save ( useThread );
if ( shuttingDown ) g_spiderCache.save ( useThread );
// do not re-save the stuff we just did this round
m_calledSave = true;
@ -1737,6 +1754,8 @@ bool Process::saveRdbTrees ( bool useThread ) {
// check if any need to finish saving
for ( int32_t i = 0 ; i < m_numRdbs ; i++ ) {
Rdb *rdb = m_rdbs[i];
// do not return until all saved if we are shutting down
if ( shuttingDown ) break;
//if ( rdb->needsSave ( ) ) return false;
// we disable the tree while saving so we can't really add recs
// to one rdb tree while saving, but for crawlbot
@ -1744,11 +1763,15 @@ bool Process::saveRdbTrees ( bool useThread ) {
if ( rdb->isSavingTree ( ) ) return false;
}
// only save spiderdb based trees if shutting down so we can
// still write to them without writes being disabled
if ( ! shuttingDown ) return true;
// . check spider cache files (doleiptable waitingtree etc.)
// . this should return true if it still has some files that haven't
// saved to disk yet... so if it returns true we return false
// indicating that we are still waiting!
if ( g_spiderCache.needsSave () ) return false;
if ( ! shuttingDown && g_spiderCache.needsSave () ) return false;
// reset for next call
m_calledSave = false;

6
Process.h

@ -38,11 +38,11 @@ class Process {
bool isAnyTreeSaving ( ) ;
bool save2 ( ) ;
bool shutdown2 ( ) ;
void disableTreeWrites ( ) ;
void enableTreeWrites ( ) ;
void disableTreeWrites ( bool shuttingDown ) ;
void enableTreeWrites ( bool shuttingDown ) ;
bool isRdbDumping ( ) ;
bool isRdbMerging ( ) ;
bool saveRdbTrees ( bool useThread ) ;
bool saveRdbTrees ( bool useThread , bool shuttingDown ) ;
bool saveRdbMaps ( bool useThread ) ;
bool saveRdbCaches ( bool useThread ) ;
bool saveBlockingFiles1 ( ) ;

170
Rdb.cpp

@ -34,6 +34,8 @@ void attemptMergeAll ( int fd , void *state ) ;
Rdb::Rdb ( ) {
m_lastReclaim = -1;
m_cacheLastTime = 0;
m_cacheLastTotal = 0LL;
@ -1176,6 +1178,12 @@ bool Rdb::dumpTree ( int32_t niceness ) {
if ( g_conf.m_isWikipedia && m_rdbId == RDB_INDEXDB )
return true;
// never dump doledb any more. it's rdbtree only.
if ( m_rdbId == RDB_DOLEDB )
return true;
// if we are in a quickpoll do not initiate dump.
// we might have been called by handleRequest4 with a niceness of 0
// which was niceness converted from 1
@ -1870,7 +1878,7 @@ bool Rdb::addList ( collnum_t collnum , RdbList *list,
// . if we don't have enough room to store list, initiate a dump and
// return g_errno of ETRYAGAIN
// . otherwise, we're guaranteed to have room for this list
if ( ! hasRoom(list) ) {
if ( ! hasRoom(list,niceness) ) {
// stop it
m_inAddList = false;
// if tree is empty, list will never fit!!!
@ -2093,7 +2101,7 @@ bool Rdb::needsDump ( ) {
return false;
}
bool Rdb::hasRoom ( RdbList *list ) {
bool Rdb::hasRoom ( RdbList *list , int32_t niceness ) {
// how many nodes will tree need?
int32_t numNodes = list->getNumRecs( );
if ( !m_useTree && !m_buckets.hasRoom(numNodes)) return false;
@ -2107,6 +2115,23 @@ bool Rdb::hasRoom ( RdbList *list ) {
// does tree have room for these nodes?
if ( m_useTree && m_tree.getNumAvailNodes() < numNodes ) return false;
// if we are doledb, we are a tree-only rdb, so try to reclaim
// memory from deleted nodes. works by condesing the used memory.
if ( m_rdbId == RDB_DOLEDB &&
// if there is no room left in m_mem (RdbMem class)...
m_mem.m_ptr2 - m_mem.m_ptr1 < dataSpace &&
//m_mem.m_ptr1 - m_mem.m_mem > 1024 ) {
// and last time we tried this, if any, it reclaimed 1MB+
( m_lastReclaim > 1024*1024 || m_lastReclaim == -1 ) ) {
// reclaim the memory now. returns -1 and sets g_errno on error
int32_t reclaimed = reclaimMemFromDeletedTreeNodes(niceness);
// ignore errors for now
g_errno = 0;
// how much did we free up?
if ( reclaimed >= 0 )
m_lastReclaim = reclaimed;
}
// does m_mem have room for "dataSpace"?
if ( (int64_t)m_mem.getAvailMem() < dataSpace ) return false;
// otherwise, we do have room
@ -2401,7 +2426,7 @@ bool Rdb::addRecord ( collnum_t collnum,
// . we NO LONGER annihilate with him. why?
// . freeData should be true, the tree doesn't own the data
// so it shouldn't free it really
m_tree.deleteNode ( n , true ); // false =freeData?);
m_tree.deleteNode3 ( n , true ); // false =freeData?);
// mark as changed
//if ( ! m_needsSave ) {
// m_needsSave = true;
@ -2644,7 +2669,7 @@ bool Rdb::addRecord ( collnum_t collnum,
"errCode = %s",
rr->getUrlHash48(),
mstrerror(indexCode));
m_tree.deleteNode(tn,false);
m_tree.deleteNode3(tn,false);
}
}
// clear errors from adding to SpiderCache
@ -2756,6 +2781,24 @@ int64_t Rdb::getNumTotalRecs ( bool useCache ) {
return total;
}
int64_t Rdb::getCollNumTotalRecs ( collnum_t collnum ) {
if ( collnum < 0 ) return 0;
CollectionRec *cr = g_collectiondb.m_recs[collnum];
if ( ! cr ) return 0;
// if swapped out, this will be NULL, so skip it
RdbBase *base = cr->getBasePtr(m_rdbId);
if ( ! base ) {
log("rdb: getcollnumtotalrecs: base swapped out");
return 0;
}
return base->getNumTotalRecs();
}
// . how much mem is alloced for all of our maps?
// . we have one map per file
int64_t Rdb::getMapMemAlloced () {
@ -3203,3 +3246,122 @@ bool Rdb::needsSave() {
if(m_useTree) return m_tree.m_needsSave;
else return m_buckets.needsSave();
}
// if we are doledb, we are a tree-only rdb, so try to reclaim
// memory from deleted nodes. works by condesing the used memory.
// returns how much we reclaimed.
int32_t Rdb::reclaimMemFromDeletedTreeNodes( int32_t niceness ) {
log("rdb: reclaiming tree mem for doledb");
// this only works for non-dumped RdbMem right now, i.e. doledb only
if ( m_rdbId != RDB_DOLEDB ) { char *xx=NULL;*xx=0; }
// start scanning the mem pool
char *p = m_mem.m_mem;
char *pend = m_mem.m_ptr1;
char *dst = p;
int32_t inUseOld = pend - p;
char *pstart = p;
int32_t marked = 0;
// mark the data of unoccupied nodes somehow
int32_t nn = m_tree.m_minUnusedNode;
for ( int i = 0 ; i < nn ; i++ ) {
QUICKPOLL ( niceness );
// count occupied skip empty nodes in tree
if ( m_tree.m_parents[i] != -2 ) continue;
// get the data
char *data = m_tree.m_data[i];
// skip if somehow null already
if ( ! data ) continue;
// sanity, ensure legit
if ( data < pstart ) { char *xx=NULL;*xx=0; }
// now mark the spiderrequest key as 00000's
memset ( data , 0 , sizeof(SPIDERDBKEY) );
// make it NULL
m_tree.m_data[i] = NULL;
marked++;
}
HashTableX ht;
if (!ht.set ( 4,
4,
m_tree.m_numUsedNodes*2,
NULL , 0 ,
false ,
niceness ,
"trectbl",
true )) // useMagic? yes..
return -1;
int32_t noticed = 0;
// the spider requests should be linear in there. so we can scan
// them. then put their offset into a map that maps it to the new
// offset after doing the memmove().
for ( ; p < pend ; ) {
QUICKPOLL ( niceness );
SpiderRequest *sreq = (SpiderRequest *)p;
int32_t oldOffset = p - pstart;
int32_t recSize = sreq->getRecSize();
// if it has been expunged, skip the copy of it
if ( sreq->m_key.n0 == 0LL &&
sreq->m_key.n1 == 0LL ) {
p += recSize;
noticed++;
continue;
}
// otherwise, copy it over if still in tree
gbmemcpy ( dst , p , recSize );
int32_t newOffset = dst - pstart;
// store in map
ht.addKey ( &oldOffset , &newOffset );
dst += recSize;
p += recSize;
}
if ( noticed != marked ) { char *xx=NULL;*xx=0; }
// sanity
if(ht.getNumSlotsUsed()!=m_tree.m_numUsedNodes){char *xx=NULL;*xx=0;}
int32_t inUseNew = dst - pstart;
// update mem class as well
m_mem.m_ptr1 = dst;
// how much did we reclaim
int32_t reclaimed = inUseOld - inUseNew;
if ( reclaimed < 0 ) { char *xx=NULL;*xx=0; }
if ( reclaimed == 0 && marked ) { char *xx=NULL;*xx=0;}
// now update data ptrs in the tree, m_data[]
for ( int i = 0 ; i < nn ; i++ ) {
QUICKPOLL ( niceness );
// skip empty nodes in tree
if ( m_tree.m_parents[i] == -2 ) continue;
// update the data otherwise
char *data = m_tree.m_data[i];
// sanity, ensure legit
if ( data < pstart ) { char *xx=NULL;*xx=0; }
int32_t offset = data - pstart;
int32_t *newOffsetPtr = (int32_t *)ht.getValue ( &offset );
if ( ! newOffsetPtr ) { char *xx=NULL;*xx=0; }
char *newData = pstart + *newOffsetPtr;
m_tree.m_data[i] = newData;
}
log("rdb: reclaimed %"INT32" bytes after scanning %"INT32" "
"undeleted nodes and %"INT32" deleted nodes for doledb"
,reclaimed,nn,marked);
// return # of bytes of mem we reclaimed
return reclaimed;
}

7
Rdb.h

@ -146,7 +146,10 @@ class Rdb {
return addRecord(coll,(char *)&key,data,dataSize, niceness);};
// returns false if no room in tree or m_mem for a list to add
bool hasRoom ( RdbList *list );
bool hasRoom ( RdbList *list , int32_t niceness );
int32_t reclaimMemFromDeletedTreeNodes( int32_t niceness ) ;
int32_t m_lastReclaim;
// . returns false on error and sets errno
// . return true on success
@ -234,6 +237,8 @@ class Rdb {
// positive minus negative
int64_t getNumTotalRecs ( bool useCache = false ) ;
int64_t getCollNumTotalRecs ( collnum_t collnum );
int64_t getNumRecsOnDisk ( );
int64_t getNumGlobalRecs ( );

3
RdbCache.cpp

@ -1332,7 +1332,7 @@ void RdbCache::clearAll ( ) {
// . this just clears the contents of the cache
// . used when deleting a collection in Rdb::delColl() and used in
// Rdb::updateToRebuild() when updating/setting the rdb to a rebuilt rdb
/*
// . try it again now with new 64-bit logic updates (MDW 2/10/2015)
void RdbCache::clear ( collnum_t collnum ) {
// bail if no writing ops allowed now
if ( ! g_cacheWritesEnabled ) { char *xx=NULL;*xx=0; }
@ -1348,7 +1348,6 @@ void RdbCache::clear ( collnum_t collnum ) {
*(collnum_t *)m_ptrs[i] = -1;
}
}
*/
bool RdbCache::load ( ) {
return load ( m_dbname );

16
RdbTree.cpp

@ -713,7 +713,7 @@ int32_t RdbTree::deleteNode ( collnum_t collnum , char *key , bool freeData ) {
//log("db: deleting n1=%"XINT64" n0=%"XINT64" node=%"INT32".",
// *(int64_t *)(key+8), *(int64_t *)(key+0),node);
if ( node == -1 ) return -1;
deleteNode(node,freeData);
deleteNode3(node,freeData);
return node;
}
@ -739,7 +739,7 @@ void RdbTree::deleteNodes ( collnum_t collnum ,
if ( m_collnums[node] != collnum ) break;
//if ( m_keys [node] > endKey ) return;
if ( KEYCMP(m_keys,node,endKey,0,m_ks) > 0 ) break;
deleteNode ( node , freeData );
deleteNode3 ( node , freeData );
// rotation in setDepths() will cause him to be replaced
// with one of his kids, unless he's a leaf node
//node = next;
@ -753,7 +753,7 @@ void RdbTree::deleteNodes ( collnum_t collnum ,
// . deletes node i from the tree
// . i's parent should point to i's left or right kid
// . if i has no parent then his left or right kid becomes the new top node
void RdbTree::deleteNode ( int32_t i , bool freeData ) {
void RdbTree::deleteNode3 ( int32_t i , bool freeData ) {
// sanity check
if ( ! m_isWritable ) {
log("db: Can not delete record from tree because "
@ -1107,7 +1107,7 @@ void RdbTree::deleteOrderedList ( collnum_t collnum ,
//if ( m_keys [ node ] == key && m_collnums [ node ] == collnum ) {
if ( KEYCMP(m_keys,node,key,0,m_ks)==0 && m_collnums[node] == collnum){
// trim the node from the tree
deleteNode ( node , true /*freeData?*/ );
deleteNode3 ( node , true /*freeData?*/ );
// get next node in tree
node = getNextNode ( node ) ;
// . point to next key in list to delete
@ -1718,7 +1718,7 @@ bool RdbTree::getList ( collnum_t collnum ,
// don't core, but i think i fixed it here.
m_gettingList++;
// stop when we've hit or jsut exceed minRecSizes
// stop when we've hit or just exceed minRecSizes
// or we're out of nodes
for ( ; node >= 0 && list->getListSize() < minRecSizes ;
node = getNextNode ( node ) ) {
@ -2765,7 +2765,7 @@ bool RdbTree::fastLoad ( BigFile *f , RdbMem *stack ) {
log("got one");
// make it negative
m_keys[i].n0 &= 0xfffffffffffffffeLL;
//deleteNode ( i , true ); // freeData?
//deleteNode3 ( i , true ); // freeData?
//goto again;
}
log("REMOVED %"INT32"",count);
@ -3116,12 +3116,12 @@ void RdbTree::cleanTree ( ) { // char **bases ) {
g_collectiondb.m_recs[m_collnums[i]] ) continue;
// if it is negtiave, remove it, that is wierd corruption
if ( m_collnums[i] < 0 )
deleteNode ( i , true );
deleteNode3 ( i , true );
// remove it otherwise
// don't actually remove it!!!! in case collection gets
// moved accidentally.
// no... otherwise it can clog up the tree forever!!!!
deleteNode ( i , true );
deleteNode3 ( i , true );
count++;
// save it
collnum = m_collnums[i];

6
RdbTree.h

@ -164,10 +164,10 @@ class RdbTree {
// . returns true iff was found and deleted
// . returns false iff not found
// . frees m_data[node] if freeIt is true
void deleteNode ( int32_t node , bool freeData );
//int32_t deleteNode ( collnum_t collnum , key_t &key , bool freeData ) ;
void deleteNode3 ( int32_t node , bool freeData );
//int32_t deleteNode ( collnum_t collnum , key_t &key,bool freeData) ;
int32_t deleteNode ( collnum_t collnum , char *key , bool freeData ) ;
int32_t deleteNode ( collnum_t collnum , key_t &key , bool freeData ) {
int32_t deleteNode ( collnum_t collnum , key_t &key , bool freeData) {
return deleteNode ( collnum , (char *)&key , freeData ); };
// delete all nodes with keys in [startKey,endKey]

663
Spider.cpp

File diff suppressed because it is too large Load Diff

28
Spider.h

@ -1066,6 +1066,8 @@ extern class Doledb g_doledb;
#define MAX_DOLEREC_SIZE (MAX_BEST_REQUEST_SIZE+sizeof(key_t)+4)
#define MAX_SP_REPLY_SIZE (sizeof(SpiderReply))
#define OVERFLOWLISTSIZE 200
// we have one SpiderColl for each collection record
class SpiderColl {
public:
@ -1149,9 +1151,10 @@ class SpiderColl {
char m_sendLocalCrawlInfoToHost[MAX_HOSTS];
Msg4 m_msg4x;
Msg4 m_msg4;
Msg1 m_msg1;
bool m_msg1Avail;
//Msg4 m_msg4;
//Msg1 m_msg1;
//bool m_msg1Avail;
RdbList m_tmpList;
bool isInDupCache ( SpiderRequest *sreq , bool addToCache ) ;
@ -1163,6 +1166,7 @@ class SpiderColl {
bool addToDoleTable ( SpiderRequest *sreq ) ;
bool addDoleBufIntoDoledb (bool isFromCache,uint32_t cachedTimestamp);
bool updateSiteNumInlinksTable ( int32_t siteHash32,int32_t sni,
time_t tstamp); // time_t
@ -1296,6 +1300,13 @@ class SpiderColl {
bool printStats ( SafeBuf &sb ) ;
bool isFirstIpInOverflowList ( int32_t firstIp ) ;
int32_t *m_overflowList;
int64_t m_totalNewSpiderRequests;
int64_t m_lastSreqUh48;
int32_t m_lastOverflowFirstIp;
private:
class CollectionRec *m_cr;
};
@ -1553,10 +1564,19 @@ class SpiderLoop {
// save on msg12 lookups! keep somewhat local...
RdbCache m_lockCache;
RdbCache m_winnerListCache;
//bool m_gettingLocks;
// for round robining in SpiderLoop::doleUrls(), etc.
int32_t m_cri;
//int32_t m_cri;
void buildActiveList ( ) ;
class CollectionRec *m_crx;
class CollectionRec *m_activeList;
bool m_activeListValid;
uint32_t m_recalcTime;
bool m_recalcTimeValid;
int64_t m_doleStart;

8
Syncdb.cpp

@ -122,7 +122,7 @@ bool Syncdb::gotMetaListRequest ( char *req , int32_t reqSize , uint32_t sid ) {
int32_t dn = m_qt.getNode ( 0, (char *)&ck ) ;
// hey, we annihilated with the positive key. i guess our
// twin is ahead of us!
if ( dn >= 0 ) m_qt.deleteNode( dn , false );
if ( dn >= 0 ) m_qt.deleteNode3 ( dn , false );
// . ADD the B KEY
// . we did not annihilate with a negative key
else if ( m_qt.addKey ( &k5 ) < 0 ) { char *xx=NULL;*xx=0; }
@ -329,7 +329,7 @@ bool Syncdb::loop2 ( ) {
m_qt.deleteNode ( 0 , (char *)&k , true );
// . and delete that node (freeData = true)
// . no! not until we got all the checkoff requests in!
// stree->deleteNode ( n , true );
// stree->deleteNode3 ( n , true );
// advance on success
m_ia++;
// go back for more
@ -459,7 +459,7 @@ void Syncdb::loop3 ( ) {
// must be there!
if ( ! dn ) { char *xx=NULL;*xx=0; }
// nuke it
m_qt.deleteNode ( dn , true );
m_qt.deleteNode3 ( dn , true );
}
// . success
// . do not recall until big loop completes a round
@ -814,7 +814,7 @@ void handleRequest5d ( UdpSlot *slot , int32_t netnice ) {
int32_t nn = g_syncdb.m_qt.getNode ( 0 , (char *)&pk );
// if yes, nuke it. they annihilate.
if ( nn >= 0 ) {
g_syncdb.m_qt.deleteNode ( nn , true );
g_syncdb.m_qt.deleteNode3 ( nn , true );
continue;
}
// . otherwise, add right to the tree

356
TcpServer.cpp

@ -15,36 +15,28 @@ static void readTimeoutPollWrapper ( int sd , void *state ) ;
static void acceptSocketWrapper ( int sd , void *state ) ;
static void timePollWrapper ( int fd , void *state ) ;
static void logSSLError(SSL *ssl, int ret) {
static char *getSSLError(SSL *ssl, int ret) {
switch (SSL_get_error(ssl, ret)) {
case SSL_ERROR_NONE:
log("net: ssl: No Error.");
break;
return "No SSL Error.";
case SSL_ERROR_ZERO_RETURN:
log ("net: ssl: Error: Zero Return");
break;
return "Zero Return";
case SSL_ERROR_WANT_READ:
log ("net: ssl: Error: Want Read");
break;
return "Want Read";
case SSL_ERROR_WANT_WRITE:
log ("net: ssl: Error: Want Write");
break;
return "Want Write";
case SSL_ERROR_WANT_CONNECT:
log ("net: ssl: Error: Want Connect");
break;
//case SSL_ERROR_WANT_ACCEPT:
// log ("net: ssl: Error: Want Accept");
// break;
return "Want Connect";
case SSL_ERROR_WANT_ACCEPT:
return "Want Accept";
case SSL_ERROR_WANT_X509_LOOKUP:
log ("net: ssl: Error: Want X509 Lookup");
break;
return "Want X509 Lookup";
case SSL_ERROR_SYSCALL:
log ("net: ssl: Error: Syscall");
break;
return "Syscall";
case SSL_ERROR_SSL:
log ("net: ssl: Error: SSL");
break;
return "SSL Library failure";
}
return "Unknown SSL Error";
}
// free all TcpSockets and their bufs
@ -713,7 +705,12 @@ bool TcpServer::sendMsg ( TcpSocket *s ,
// . destroy the socket on error
// . this will also unregister all our callbacks for the socket
// . TODO: deleting nodes from under Loop::callCallbacks is dangerous!!
if ( g_errno ) { destroySocket ( s ); return true; }
if ( g_errno ) {
if ( g_conf.m_logDebugTcp )
log("tcp: writeSocket error: %s",mstrerror(g_errno));
destroySocket ( s );
return true;
}
// if in streaming mode just return true, do not set sockState
// to ST_NEEDS_CLOSE lest it be destroyed. streaming mode needs
@ -976,6 +973,10 @@ TcpSocket *TcpServer::wrapSocket ( int sd , int32_t niceness , bool isIncoming )
// so let's go to 100 minutes so we can deal with reranked queries
// (Msg3b) that take like an hour.
s->m_timeout = 1000*60*1000;
if ( g_conf.m_logDebugTcp )
log("tcp: wrapping sd=%i",sd);
// a temp thang
//int parm;
// . TODO: make sure this sd will NEVER exist!!
@ -1028,10 +1029,11 @@ TcpSocket *TcpServer::wrapSocket ( int sd , int32_t niceness , bool isIncoming )
// if a write we did failed because the buffer was full?.
// let's do this after the connection request is accepted!
// MDW: try this again
if(!g_loop.registerWriteCallback(sd,this,writeSocketWrapper,niceness)){
g_loop.unregisterReadCallback(sd,this , readSocketWrapper );
goto hadError;
}
//if(!g_loop.registerWriteCallback(sd,this,writeSocketWrapper,
//niceness)){
// g_loop.unregisterReadCallback(sd,this , readSocketWrapper );
// goto hadError;
// }
// return "s" on success
return s;
// otherwise, free "s" and return NULL
@ -1184,7 +1186,13 @@ void readSocketWrapper ( int sd , void *state ) {
// -for-write fds on the select() call in Loop.cpp on the same fds
// we are waiting for reads on. so if we get a signal it could really
// be a ready-for-write signal, so try this writing just in case.
writeSocketWrapper ( sd , state );
// ok, since i added writecallbacks back, no need for this now
// MDW 2/16/2015
// plus, when this was here and we called readSocketWrapper from
// the readTimeoutPoll() function the readSocketWrapper2() would
// sometimes close the socket so 'sd' would be invalid and
// cause getSocket() called by writeSocketWrapper() to return NULL.
//writeSocketWrapper ( sd , state );
}
@ -1214,6 +1222,23 @@ void readSocketWrapper2 ( int sd , void *state ) {
THIS->destroySocket ( s );
return;
}
if ( s->m_sockState == ST_SSL_HANDSHAKE ) {
int r = THIS->sslHandshake ( s );
// return if it blocked
if ( r == 0 ) return;
// error?
if ( r == -1 ) {
log("tcp: ssl handshake2 error sd=%i",s->m_sd);
THIS->makeCallback ( s );
THIS->destroySocket ( s );
return;
}
// it went through, should be ST_WRITING so go below
THIS->writeSocket ( s );
return;
}
// . if this socket was connecting than call connectSocket()
// . it returns false if blocked,true otherwise and sets g_errno on err
if ( s->isConnecting() ) {
@ -1224,6 +1249,9 @@ void readSocketWrapper2 ( int sd , void *state ) {
if ( status == 1 ) status = THIS->writeSocket ( s );
// destroy socket and call callback on connect error
if ( status == -1 ) {
if ( g_conf.m_logDebugTcp )
log("tcp: connectSocket error2: %s",
mstrerror(g_errno));
// i saw
// ssl: Error on Connect
// ssl: Error: Syscall
@ -1411,11 +1439,23 @@ int32_t TcpServer::readSocket ( TcpSocket *s ) {
// deal with errors
if ( n < 0 ) {
// valgrind
if ( errno == EINTR ) goto loop;
if ( errno == EINTR ) {
if ( g_conf.m_logDebugTcp )
log("tcp: readsocket: read got interrupted");
goto loop;
}
// copy errno to g_errno
g_errno = errno;
if ( g_errno == EAGAIN || g_errno == 0 ||
g_errno == EILSEQ) { g_errno = 0; return 0; }
if ( g_errno == EAGAIN ||
g_errno == 0 ||
g_errno == EILSEQ) {
if ( g_conf.m_logDebugTcp )
log("tcp: readsocket: read got error "
"(avail=%i) %s",
(int)avail,mstrerror(g_errno));
g_errno = 0;
return 0;
}
log("tcp: Failed to read on socket: %s.", mstrerror(g_errno));
return -1;
}
@ -1572,6 +1612,23 @@ void writeSocketWrapper ( int sd , void *state ) {
THIS->destroySocket ( s );
return;
}
if ( s->m_sockState == ST_SSL_HANDSHAKE ) {
int r = THIS->sslHandshake ( s );
// return if it blocked. write callback shoud be
// registered... or read, depending on the the handshake return
if ( r == 0 ) return;
// error?
if ( r == -1 ) {
log("tcp: ssl handshake error sd=%i",s->m_sd);
THIS->makeCallback ( s );
THIS->destroySocket ( s );
return;
}
// it went through, should be ST_WRITING so go below
}
// . if loop notified us of an error on this socket then destroy it
// . like -- pollhup, socket closed
if ( g_errno == ESOCKETCLOSED ) {
@ -1658,6 +1715,9 @@ void writeSocketWrapper ( int sd , void *state ) {
// set to false, so don't destroy socket just yet...
if ( wasStreaming ) return;
// skip if we already destroyed in writeSocket()
if ( s->m_sockState == ST_CLOSE_CALLED ) return;
// . destroy the socket on error, recycle on transaction completion
// . this will also unregister all our callbacks for the socket
if ( status == -1 ) THIS->destroySocket ( s );
@ -1681,7 +1741,10 @@ int32_t TcpServer::writeSocket ( TcpSocket *s ) {
// we only register write callback to see when it is connected so
// we can do a write, so we should not need this now
//g_loop.unregisterWriteCallback(s->m_sd,this,writeSocketWrapper);
if ( s->m_writeRegistered ) {
g_loop.unregisterWriteCallback(s->m_sd,this,writeSocketWrapper);
s->m_writeRegistered = false;
}
loop:
@ -1714,15 +1777,19 @@ int32_t TcpServer::writeSocket ( TcpSocket *s ) {
msg = s->m_sendBuf + tunnelRequestSize;
s->m_totalToSend = s->m_sendBufUsed - tunnelRequestSize;
toSend = s->m_sendBufUsed - tunnelRequestSize -s->m_sendOffset;
// enter write mode after this
s->m_tunnelMode = 3;
// reset this so we do not truncate the NEXT reply!
s->m_totalToRead = 0;
//
// use ssl now
//
int r = sslHandshake ( s );
// error? g_errno will be set, return -1
if ( r == -1 ) return -1;
// return 0 if it would block
if ( r == 0 ) return 0;
// i guess it completed. will be ST_WRITING mode now.
// enter write tunnel mode too
s->m_tunnelMode = 3;
}
if ( s->m_tunnelMode == 3 ) {
@ -1776,14 +1843,53 @@ int32_t TcpServer::writeSocket ( TcpSocket *s ) {
return -1;
}
g_errno = 0;
//log("........... TcpServer write blocked on %i\n",
//s->m_sd);
if ( g_conf.m_logDebugTcp )
log("........... TcpServer write blocked on %i\n",
s->m_sd);
// need to listen for writability now since our write
// failed to write everythin gout
if ( ! s->m_writeRegistered &&
! g_loop.registerWriteCallback(s->m_sd,
this,
writeSocketWrapper,
s->m_niceness)){
log("tcp: failed to reg write callback1 for "
"sd=%i", s->m_sd);
return -1;
}
// do not keep doing it otherwise select() goes crazy
s->m_writeRegistered = true;
return 0;
}
// debug msg
if ( g_conf.m_logDebugTcp )
log("........... TcpServer wrote %i bytes on %i\n",
n,s->m_sd);
// if we did not write all the bytes we wanted we have to register
// a write callback
if ( n < toSend ) {
// another debug
//if ( g_conf.m_logDebugTcp )
log("tcp: only wrote %"INT32" of %"INT32" bytes "
"tried.",n,toSend);
// need to listen for writability now since our write
// failed to write everythin gout
if ( ! s->m_writeRegistered &&
! g_loop.registerWriteCallback(s->m_sd,
this,
writeSocketWrapper,
s->m_niceness)){
log("tcp: failed to reg write callback1 for "
"sd=%i", s->m_sd);
return -1;
}
// do not keep doing it otherwise select() goes crazy
s->m_writeRegistered = true;
}
// return 0 if we blocked on this write
if ( n == 0 ) return 0;
// update last action time stamp
@ -1803,7 +1909,9 @@ int32_t TcpServer::writeSocket ( TcpSocket *s ) {
if ( s->m_totalSent < s->m_totalToSend ) {
// note that
if ( g_conf.m_logDebugTcp )
log(".... Tcpserver: only sent fraction. looping.");
log(".... Tcpserver: only sent fraction %"INT32" of "
"%"INT32" bytes. looping.",
s->m_totalSent , s->m_totalToSend);
// . refill the sendBuf
// . this might set m_sendBufUsed, m_sendBufOffset, ...
// . it may also block in which case nothing will be changed
@ -1845,6 +1953,13 @@ int32_t TcpServer::writeSocket ( TcpSocket *s ) {
s->m_sockState = ST_READING;
return 1 ;
}
if ( s->m_streamingMode ) return true;
// close it. without this here the socket only gets
// closed for real in the timeout loop.
destroySocket ( s );
// . otherwise, we finished sending a reply
// . our caller should call recycleSocket ( s ) to keep it alive
return true ;
@ -1873,6 +1988,13 @@ int32_t TcpServer::connectSocket ( TcpSocket *s ) {
if ( g_conf.m_logDebugTcp )
log("........... TcpServer connected %i to %s "
"port %i\n", s->m_sd, iptoa(s->m_ip), s->m_port );
// don't listen for writing any more
if ( s->m_writeRegistered ) {
g_loop.unregisterWriteCallback(s->m_sd,
this,
writeSocketWrapper);
s->m_writeRegistered = false;
}
// hey it was successful!
goto connected;
}
@ -1880,24 +2002,50 @@ int32_t TcpServer::connectSocket ( TcpSocket *s ) {
if ( errno == EINTR ) goto retry3;
// copy errno to g_errno
g_errno = errno;
// hey! it's alrady connected!
// hey! it's alrady connected!
// i'm not sure that is what this means... MDW 2/16/2015
// i think it means the connect request was already initiated.
if ( g_errno == EALREADY ) {
if ( g_conf.m_logDebugTcp )
log("........... TcpServer got EALREADY, "
"so must be in progress sd=%i",s->m_sd);
g_errno = EINPROGRESS;
/*
// debug msg
if ( g_conf.m_logDebugTcp )
log("........... TcpServer already connected %i to "
"%s port %i\n", s->m_sd, iptoa(s->m_ip),s->m_port);
g_loop.unregisterWriteCallback(s->m_sd,
this,
writeSocketWrapper);
// we are already connected, stop waiting for writability
if ( s->m_writeRegistered ) {
g_loop.unregisterWriteCallback(s->m_sd,
this,
writeSocketWrapper);
s->m_writeRegistered = false;
}
g_errno = 0;
goto connected;
*/
}
// we blocked with the EINPROGRESS g_errno
if ( g_errno == EINPROGRESS ) {
g_errno = 0;
return 0;
g_errno = 0;
// note that
if ( g_conf.m_logDebugTcp )
log("tcp: connection is in progress sd=%i",s->m_sd);
// according to 'man connect' select() needs to listen
// for writability
if ( s->m_writeRegistered ) return 0;
// make select() listen on this fd for when it can write
if ( ! g_loop.registerWriteCallback(s->m_sd,
this,
writeSocketWrapper,
s->m_niceness)){
log("tcp: failed to reg write callback2 for "
"sd=%i", s->m_sd);
return -1;
}
s->m_writeRegistered = true;
return 0;
}
// return -1 on real error
if ( g_conf.m_logDebugTcp )
@ -1906,14 +2054,23 @@ int32_t TcpServer::connectSocket ( TcpSocket *s ) {
return -1;
connected:
// change state so this doesn't get called again
s->m_sockState = ST_WRITING;
// connect ssl
if ( m_useSSL ) {
// enter handshake mode now
s->m_sockState = ST_SSL_HANDSHAKE;
// i guess state is special
int r = sslHandshake ( s );
// there was an error
if (r == -1 ) return -1;
// i guess it would block. this should register
// the write callback if we need to do a write operation still
if ( r == 0 ) return 0;
}
// change state so this doesn't get called again
s->m_sockState = ST_WRITING;
return 1;
}
@ -1932,6 +2089,11 @@ void TcpServer::destroySocket ( TcpSocket *s ) {
//char *xx=NULL;*xx=0; }
}
if ( s->m_sockState == ST_CLOSE_CALLED ) {
log("tcp: destroy already called for sock=%i",s->m_sd);
return;
}
// sanity check
if ( s->m_udpSlot ) {
log("tcp: sending back error on udp slot err=%s",
@ -2032,6 +2194,7 @@ void TcpServer::destroySocket ( TcpSocket *s ) {
}
*/
SSL_free(s->m_ssl);
s->m_ssl = NULL;
}
// ssl debug!
@ -2073,7 +2236,10 @@ void TcpServer::destroySocket ( TcpSocket *s ) {
// always free the sendBuf
if ( s->m_sendBuf ) mfree (s->m_sendBuf, s->m_sendBufSize,"TcpServer");
// unregister it with Loop so we don't get any calls about it
g_loop.unregisterWriteCallback ( sd , this , writeSocketWrapper );
if ( s->m_writeRegistered ) {
g_loop.unregisterWriteCallback ( sd, this, writeSocketWrapper);
s->m_writeRegistered = false;
}
g_loop.unregisterReadCallback ( sd , this , readSocketWrapper );
// debug msg
//log("unregistering sd=%"INT32"",sd);
@ -2158,6 +2324,8 @@ void readTimeoutPollWrapper ( int sd , void *state ) {
void TcpServer::readTimeoutPoll ( ) {
// get the time now in seconds
int64_t now = gettimeofdayInMilliseconds();
if ( g_conf.m_logDebugTcp )
log("tcp: entering timeout loop");
// send the msg that is mostly caught up with it's acks first.
// "ackWait" is how many more acks we need to complete the transmission
for ( int32_t i = 0 ; i <= m_lastFilled ; i++ ) {
@ -2181,6 +2349,9 @@ void TcpServer::readTimeoutPoll ( ) {
// . if he is sending, that sticks too, so try it!
// . or if we're connecting to him...
if ( s->isSending() || s->isConnecting() ) {
if ( g_conf.m_logDebugTcp )
log("tcp: timeloop: calling writesock on sd=%i"
,s->m_sd);
writeSocketWrapper ( s->m_sd , this );
s = m_tcpSockets[i];
if ( ! s ) continue;
@ -2189,6 +2360,9 @@ void TcpServer::readTimeoutPoll ( ) {
// . HACK: this fixes the problem, albeit not the best way
// . or if he's connecting to us...
if ( s->isReading() || s->isConnecting() ) {
if ( g_conf.m_logDebugTcp )
log("tcp: timeloop: calling readsock on sd=%i"
,s->m_sd);
readSocketWrapper ( s->m_sd , this );
s = m_tcpSockets[i];
if ( ! s ) continue;
@ -2253,6 +2427,9 @@ void TcpServer::readTimeoutPoll ( ) {
continue;
}
if ( g_conf.m_logDebugTcp )
log("tcp: timeloop: timing out sd=%i" ,s->m_sd);
//log("tcp: timeout=%"INT32" fd=%"INT32"",sockTimeout,s->m_sd);
// uncomment this if you want to close a socket if they havent
@ -2270,6 +2447,8 @@ void TcpServer::readTimeoutPoll ( ) {
// reset g_errno so we can continue
g_errno = 0;
}
if ( g_conf.m_logDebugTcp )
log("tcp: exiting timeout loop");
}
// . sd should be m_sock
@ -2589,20 +2768,25 @@ bool TcpServer::sendChunk ( TcpSocket *s ,
}
// returns -1 on error with g_errno set
// returns -1 on error with g_errno set. returns 0 if would block. 1 if done.
int TcpServer::sslHandshake ( TcpSocket *s ) {
if ( s->m_sockState != ST_SSL_HANDSHAKE ) { char *xx=NULL;*xx=0; }
// steal from ssl tcp server i guess in case we are not it
s->m_ssl = SSL_new(g_httpServer.m_ssltcp.m_ctx);
if ( ! s->m_ssl ) {
s->m_ssl = SSL_new(g_httpServer.m_ssltcp.m_ctx);
SSL_set_fd(s->m_ssl, s->m_sd);
//int64_t now1 = gettimeofdayInMilliseconds();
SSL_set_connect_state(s->m_ssl);
}
SSL_set_fd(s->m_ssl, s->m_sd);
//int64_t now1 = gettimeofdayInMilliseconds();
SSL_set_connect_state(s->m_ssl);
int r = SSL_connect(s->m_ssl);
if ( g_conf.m_logDebugTcp )
log("tcp: connecting with ssl on sd=%"INT32"",(int32_t)s->m_sd);
log("tcp: ssl handshake on sd=%"INT32" r=%i",
(int32_t)s->m_sd,r);
//int64_t now2 = gettimeofdayInMilliseconds();
//int64_t took = now2 - now1 ;
//if ( took >= 2 ) log("tcp: ssl_connect took %"INT64"ms", took);
@ -2610,23 +2794,81 @@ int TcpServer::sslHandshake ( TcpSocket *s ) {
log("ssl: SSL is NULL after connect.");
char *xx = NULL; *xx = 0;
}
if ( r > 0 ) return r;
// if the connection happened return r, should be 1
if ( r > 0 ) {
if ( g_conf.m_logDebugTcp )
log("tcp: ssl handshake done. entering writing mode "
"sd=%i",s->m_sd);
// ok, it completed, go into writing mode
s->m_sockState = ST_WRITING;
return r;
}
int sslError = SSL_get_error(s->m_ssl, r);
char *sslMsg = getSSLError(s->m_ssl, r);
// if ( sslError == SSL_ERROR_SSL ) {
// log("tcp: ssl handhshake already completed sd=%i?",s->m_sd);
// // ok, it completed, go into writing mode
// s->m_sockState = ST_WRITING;
// return 1;
// }
if ( sslError != SSL_ERROR_WANT_READ &&
sslError != SSL_ERROR_WANT_WRITE &&
sslError != SSL_ERROR_NONE ) {
logSSLError(s->m_ssl, r);
log("net: ssl: Error on Connect (%"INT32"). ip=%s",
(int32_t)sslError,iptoa(s->m_ip));
log("tcp: ssl: Error on Connect (%"INT32"). r=%i ip=%s msg=%s",
(int32_t)sslError,r,iptoa(s->m_ip),sslMsg);
g_errno = ESSLERROR;
// make sure read callback is registered
// g_loop.registerReadCallback (s->m_sd,this,readSocketWrapper,
// s->m_niceness);
// crap, if we return 1 here then
// it will call THIS->writeSocket() which
// will return -1 and not set g_errno
return -1;
}
if ( g_conf.m_logDebugTcp )
log("tcp: ssl: Error on Connect (%"INT32"). r=%i ip=%s msg=%s",
(int32_t)sslError,r,iptoa(s->m_ip),sslMsg);
if ( sslError <= 0 ) { char *xx=NULL;*xx=0; }
if ( g_conf.m_logDebugTcp )
log("tcp: ssl handshake returned r=%i",r);
// read callbacks are always registered and if we need a read
// hopefully it will be called. TODO: verify this...
if ( sslError == SSL_ERROR_WANT_READ ) {
//log("tcp: ssl handshake is not want write sd=%i",s->m_sd);
//logSSLError(s->m_ssl, r);
return 0;
}
// returns 0 if it would block
// need to listen for writability now since our write
// failed to write everything out.
// if it is EWANTREAD we are already listening on the
// read for all file descriptors at all times. it is only
// writes we have to turn on and off.
if ( ! s->m_writeRegistered &&
! g_loop.registerWriteCallback(s->m_sd,
this,
writeSocketWrapper,
s->m_niceness)){
log("tcp: failed to reg write callback3 for "
"sd=%i", s->m_sd);
return -1;
}
// do not keep doing it otherwise select() goes crazy
s->m_writeRegistered = true;
log("tcp: registered write callback for sslHandshake");
// we would block
return 0;
}

3
TcpSocket.h

@ -26,6 +26,7 @@
#define ST_CLOSE_CALLED 7
#define ST_SSL_ACCEPT 8
#define ST_SSL_SHUTDOWN 9
#define ST_SSL_HANDSHAKE 10
// hack to repopulate the socket's send buf when its done sending
// it's current sendbuf in order to transmit large amounts of data that
// can't all fit in memory at the same time:
@ -123,6 +124,8 @@ class TcpSocket {
char m_niceness;
char m_streamingMode;
bool m_writeRegistered;
int32_t m_shutdownStart;
// SSL members

4
TopTree.cpp

@ -380,7 +380,7 @@ bool TopTree::addNode ( TopNode *t , int32_t tnn ) {
// delete him from the top tree now as well
//deleteNode ( nn , domHash );
// then delete him from the m_t2 tree
m_t2.deleteNode ( min , false );
m_t2.deleteNode3 ( min , false );
//logf(LOG_DEBUG,"deleting1 %"INT32"",min);
}
// if we have not violated the ridiculous max, just add ourselves
@ -489,7 +489,7 @@ bool TopTree::addNode ( TopNode *t , int32_t tnn ) {
// get next node from t2
int32_t next = m_t2.getNextNode ( min );
// delete from m_t2
m_t2.deleteNode ( min , false );
m_t2.deleteNode3 ( min , false );
// skip if not th emin
if ( m_domMinNode[domHash2] != min ) continue;
// if we were the last, that's it

7
UdpSlot.cpp

@ -295,9 +295,12 @@ bool UdpSlot::sendSetup ( char *msg ,
int32_t maxMsgSize = m_maxDgramSize * MAX_DGRAMS;
log(LOG_LOGIC,"udp: Msg size of %"INT32" bytes is too big "
"to send. Max dgram size = %"INT32". Max dgrams = "
"%"INT32". Max msg size = %"INT32".",
"%"INT32". Max msg size = %"INT32" msgtype=0x%hhx. Please "
"increase the #define MAX_DGRAMS in UdpSlot.h and "
"recompile to fix this.",
(int32_t)msgSize,(int32_t)m_maxDgramSize,
(int32_t)MAX_DGRAMS,maxMsgSize);
(int32_t)MAX_DGRAMS,maxMsgSize,
msgType);
char *xx=NULL; *xx=0;
return false;
//msgSize = MAX_DGRAMS * DGRAM_SIZE;

2
UdpSlot.h

@ -79,6 +79,8 @@
#define MAX_DGRAMS (((180*1024*1024) / DGRAM_SIZE_LB) + 1)
//#endif
#define MAX_ABSDOCLEN ((MAX_DGRAMS * DGRAM_SIZE_LB)-50000)
// . the max size of an incoming request for a hot udp server
// . we cannot call malloc so it must fit in here
// . i increased this from 5k to 10k to better support Msg17's caching requests

32
XmlDoc.cpp

@ -189,6 +189,8 @@ static int64_t s_lastTimeStart = 0LL;
void XmlDoc::reset ( ) {
m_linkOverflows = 0;
m_isImporting = false;
m_printedMenu = false;
@ -9446,7 +9448,7 @@ int32_t getTopGigabits ( HashTableX *ht ,
// sanity check
if ( tn < 0 ) { char *xx=NULL;*xx=0; }
// kick out smallest
tree.deleteNode ( tn , false );
tree.deleteNode3 ( tn , false );
// get new smallest
tn = tree.getLowestNode();
// set the new minkey
@ -12758,9 +12760,10 @@ int32_t *XmlDoc::getSiteNumInlinks ( ) {
else if ( ns < 100 ) maxAge = 60;
// if index size is tiny then maybe we are just starting to
// build something massive, so reduce the cached max age
if ( g_titledb.m_rdb.getNumGlobalRecs() < 100000000 ) // 100M
int64_t nt = g_titledb.m_rdb.getCollNumTotalRecs(m_collnum);
if ( nt < 100000000 ) //100M
maxAge = 3;
if ( g_titledb.m_rdb.getNumGlobalRecs() < 10000000 ) // 10M
if ( nt < 10000000 ) //10M
maxAge = 1;
// for every 100 urls you already got, add a day!
sni = atol(tag->getTagData());
@ -19998,10 +20001,16 @@ bool XmlDoc::logIt ( SafeBuf *bb ) {
(int32_t)m_links.hasRSSOutlink() );
if ( m_numOutlinksAddedValid )
sb->safePrintf("outlinksadded=%04"INT32" ",(int32_t)m_numOutlinksAdded);
sb->safePrintf("outlinksadded=%04"INT32" ",
(int32_t)m_numOutlinksAdded);
if ( m_linkOverflows )
sb->safePrintf("linkoverflows=%04"INT32" ",
(int32_t)m_linkOverflows);
if ( m_metaListValid )
sb->safePrintf("addlistsize=%05"INT32" ",(int32_t)m_metaListSize);
sb->safePrintf("addlistsize=%05"INT32" ",
(int32_t)m_metaListSize);
else
sb->safePrintf("addlistsize=%05"INT32" ",(int32_t)0);
@ -20259,7 +20268,7 @@ bool XmlDoc::logIt ( SafeBuf *bb ) {
}
if ( m_httpStatusValid && m_httpStatus != 200 )
sb->safePrintf("httpstatus=%"INT32,(int32_t)m_httpStatus);
sb->safePrintf("httpstatus=%"INT32" ",(int32_t)m_httpStatus);
if ( m_isDupValid && m_isDup )
sb->safePrintf("dupofdocid=%"INT64" ",m_docIdWeAreADupOf);
@ -25161,7 +25170,7 @@ char *XmlDoc::addOutlinkSpiderRecsToMetaList ( ) {
bool ignore = false;
if ( mbuf[0] == '1' ) ignore = true;
//SpiderColl *sc = g_spiderCache.getSpiderCollIffNonNull ( m_collnum );
SpiderColl *sc = g_spiderCache.getSpiderCollIffNonNull ( m_collnum );
//
// serialize each link into the metalist now
@ -25176,6 +25185,15 @@ char *XmlDoc::addOutlinkSpiderRecsToMetaList ( ) {
//int32_t hc = hcv[i];
// ip lookup failed? do not add to spiderdb then
if ( firstIp == 0 || firstIp == -1 ) continue;
// if firstIp is in the SpiderColl::m_overflowFirstIps list
// then do not add any more links to it. it already has
// more than 500MB worth.
if ( sc && sc->isFirstIpInOverflowList ( firstIp ) ) {
m_linkOverflows++;
continue;
}
// sanity check
//if ( firstIp == 0x03 ) {char *xx=NULL;*xx=0; }
// get flags

1
XmlDoc.h

@ -2032,6 +2032,7 @@ class XmlDoc {
char m_isFiltered;
int32_t m_urlFilterNum;
int32_t m_numOutlinksAdded;
int32_t m_linkOverflows;
int32_t m_numOutlinksAddedFromSameDomain;
int32_t m_numOutlinksFiltered;
int32_t m_numOutlinksBanned;

32
fctypes.cpp

@ -1910,14 +1910,28 @@ int64_t gettimeofdayInMillisecondsSynced() {
log("xml: clock not in sync with host #0 yet!!!!!!");
//char *xx = NULL; *xx = 0; }
}
//if ( ! g_clockInSync )
// log("gb: Getting global time but clock not in sync.");
// this isn't async signal safe...
struct timeval tv;
gettimeofday ( &tv , NULL );
int64_t now=(int64_t)(tv.tv_usec/1000)+((int64_t)tv.tv_sec)*1000;
int64_t now;
// the real tiem sigalrm interrupt in Loop.cpp sets this to
// true once per millisecond
if ( ! g_clockNeedsUpdate ) {
now = g_now;
}
else {
//if ( ! g_clockInSync )
// log("gb: Getting global time but clock not in sync.");
// this isn't async signal safe...
struct timeval tv;
gettimeofday ( &tv , NULL );
now = (int64_t)(tv.tv_usec/1000)+((int64_t)tv.tv_sec)*1000;
}
// update g_nowLocal
if ( now > g_now ) g_now = now;
g_clockNeedsUpdate = false;
// adjust from Msg0x11 time adjustments
now += s_adjustment;
// update g_now if it is more accurate
@ -1992,6 +2006,12 @@ int64_t gettimeofdayInMilliseconds() {
return now;
}
int64_t gettimeofdayInMilliseconds_force ( ) {
g_clockNeedsUpdate = true;
return gettimeofdayInMilliseconds();
}
time_t getTime () {
return getTimeLocal();
}

1
fctypes.h

@ -177,6 +177,7 @@ int64_t gettimeofdayInMillisecondsSynced() ; // synced with host #0
int64_t gettimeofdayInMillisecondsLocal () ;// this is local now
int64_t gettimeofdayInMilliseconds() ;// this is local now
uint64_t gettimeofdayInMicroseconds(void) ;
int64_t gettimeofdayInMilliseconds_force ( ) ;
// . get time in seconds since epoch
// . use this instead of call to time(NULL) cuz it uses adjustment

2
html/faq.html

@ -270,7 +270,7 @@ For RedHat do a <b>yum install gcc-c++ glibc-static libstdc++-static openssl-sta
</ul>
-->
<b>1.0</b> For <u>Ubuntu 12.02 or 14.04</u>: do <b>sudo apt-get update ; apt-get install make g++ libssl-dev</b>
<b>1.0</b> For <u>Ubuntu 12.02 or 14.04</u>: do <b>sudo apt-get update ; apt-get install make g++ libssl-dev binutils</b>
<br><br>

222
main.cpp

@ -3349,12 +3349,12 @@ int main2 ( int argc , char *argv[] ) {
// linkdb
if ( ! g_linkdb.init() ) {
log("db: Linkdb init failed." ); return 1; }
if ( ! g_cachedb.init() ) {
log("db: Cachedb init failed." ); return 1; }
if ( ! g_serpdb.init() ) {
log("db: Serpdb init failed." ); return 1; }
if ( ! g_monitordb.init() ) {
log("db: Monitordb init failed." ); return 1; }
// if ( ! g_cachedb.init() ) {
// log("db: Cachedb init failed." ); return 1; }
// if ( ! g_serpdb.init() ) {
// log("db: Serpdb init failed." ); return 1; }
// if ( ! g_monitordb.init() ) {
// log("db: Monitordb init failed." ); return 1; }
// use sectiondb again for its immense voting power for detecting and
// removing web page chrome, categories, etc. only use if
// CollectionRec::m_isCustomCrawl perhaps to save space.
@ -10743,7 +10743,7 @@ bool bucketstest ( char* dbname ) {
keySize) ) // own data?
return log("speedTest: tree init failed.");
// add to regular tree
int64_t t = gettimeofdayInMilliseconds();
int64_t t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < numKeys * keySize; i += keySize ) {
char* key = k+i;
KEYSET(oppKey,key,keySize);
@ -10751,14 +10751,14 @@ bool bucketstest ( char* dbname ) {
int32_t n;
n = rt.getNode ( 0, oppKey );
if ( n >= 0 ) {
rt.deleteNode ( n , true );
rt.deleteNode3 ( n , true );
}
if ( rt.addNode ( 0, key , NULL , 0 ) < 0 )
return log("speedTest: rdb tree addNode "
"failed");
}
// print time it took
int64_t e = gettimeofdayInMilliseconds();
int64_t e = gettimeofdayInMilliseconds_force();
log("db: added %"INT32" keys to rdb tree in %"INT64" ms, "
"now have %"INT32" keys",numKeys,e - t, rt.getNumUsedNodes());
@ -10772,7 +10772,7 @@ bool bucketstest ( char* dbname ) {
}
rdbb.testAndRepair();
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
log("db: added %"INT32" keys to rdb buckets in %"INT64" ms, "
"now have %"INT32" keys, mem used: %"INT32"",
numKeys,t - e, rdbb.getNumKeys(),rdbb.getMemOccupied());
@ -10781,22 +10781,22 @@ bool bucketstest ( char* dbname ) {
log("db: saving and loading buckets.");
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
rdbb.fastSave ( ".",
false,
NULL, NULL);
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
log("db: saved rdbbuckets in %"INT64" ms",t - e);
//rdbb.setNeedsSave(false);
rdbb.clear();
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
rdbb.loadBuckets ( "TestBuckets" );
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
log("db: loaded rdbbuckets in %"INT64" ms", t - e);
rdbb.selfTest(true, true);
@ -10827,14 +10827,14 @@ bool bucketstest ( char* dbname ) {
int32_t found = 0;
for ( int32_t i = 0 ; i < 0; i++ ) {
int32_t j = (rand() % numKeys) * keySize;
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
int32_t nodeNum = rt.getNode ( 0 , (char*)&k[j]);
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
ttook += t - e;
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
char* foundKey = rdbb.getKeyVal ( 0 , (char*)&k[j], NULL, NULL);
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
btook += t - e;
@ -10870,10 +10870,10 @@ bool bucketstest ( char* dbname ) {
// sort the list of keys
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
gbsort ( k , numKeys , sizeof(key_t) , keycmp );
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("db: sorted %"INT32" in %"INT64" ms",numKeys,e - t);
@ -10910,7 +10910,7 @@ bool bucketstest ( char* dbname ) {
ekey = key1;
}
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
rt.getList ( (collnum_t)0 ,
skey,
ekey,
@ -10919,7 +10919,7 @@ bool bucketstest ( char* dbname ) {
&numPosRecs,
&numNegRecs,
true ); //use half keys
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
ttook += t - e;
tgot += treelist.getNumRecs();
@ -10940,7 +10940,7 @@ bool bucketstest ( char* dbname ) {
t - e );
*/
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
rdbb.getList ( (collnum_t)0 ,
skey,
ekey,
@ -10949,7 +10949,7 @@ bool bucketstest ( char* dbname ) {
&numPosRecs,
&numNegRecs,
true ); //use half keys
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
btook += t - e;
bgot += bucketlist.getNumRecs();
@ -11080,7 +11080,7 @@ bool bucketstest ( char* dbname ) {
for ( int32_t i = 0 ; i < numKeys ; i++ ) {
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
char* key = &k[i*keySize];
KEYSET(oppKey,key,keySize);
@ -11089,19 +11089,19 @@ bool bucketstest ( char* dbname ) {
collnum_t collnum = rand() % 10;
n = rt.getNode ( collnum , oppKey );
if ( n >= 0 ) rt.deleteNode ( n , true );
if ( n >= 0 ) rt.deleteNode3 ( n , true );
if ( rt.addNode (collnum, key, NULL , 0 ) < 0 )
return log("speedTest: rdb tree addNode "
"failed");
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
tAddTook += t - e;
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
if ( rdbb.addNode(collnum, key, NULL, 0 ) < 0 )
return log("speedTest: rdb buckets addNode "
"failed");
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
bAddTook += t - e;
if(i % 100 != 0) continue;
@ -11134,7 +11134,7 @@ bool bucketstest ( char* dbname ) {
}
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
rt.getList ( collnum,
skey,
ekey,
@ -11143,7 +11143,7 @@ bool bucketstest ( char* dbname ) {
&numPosRecs,
&numNegRecs,
true ); //use half keys
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
tgetListTook += t - e;
tgot += treelist.getNumRecs();
@ -11170,7 +11170,7 @@ bool bucketstest ( char* dbname ) {
t - e ,tgetListTook );
}
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
rdbb.getList ( collnum,
skey,
ekey,
@ -11179,7 +11179,7 @@ bool bucketstest ( char* dbname ) {
&numPosRecs,
&numNegRecs,
true ); //use half keys
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
bgetListTook += t - e;
bgot += bucketlist.getNumRecs();
@ -11258,10 +11258,10 @@ bool bucketstest ( char* dbname ) {
log("db: removing %"INT32" nodes from tree. "
"tree currently has %"INT32" keys",
treelist.getNumRecs(), rt.getNumUsedNodes ( ));
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
rt.deleteList(collnum, &treelist, true);
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
tdelListTook += t - e;
log("db: Now tree has %"INT32" keys", rt.getNumUsedNodes());
@ -11269,9 +11269,9 @@ bool bucketstest ( char* dbname ) {
log("db: removing %"INT32" nodes from buckets. "
"buckets currently has %"INT32" keys",
bucketlist.getNumRecs(), rdbb.getNumKeys(0));
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
rdbb.deleteList(collnum, &bucketlist);
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
bdelListTook += t - e;
log("db: Now buckets has %"INT32" keys", rdbb.getNumKeys(0));
@ -11298,13 +11298,13 @@ bool bucketstest ( char* dbname ) {
//int32_t n = rt.getNextNode ( (collnum_t)0, (char *)&kk );
int32_t n = rt.getFirstNode();
// loop it
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
int32_t count = 0;
while ( n >= 0 ) {
n = rt.getNextNode ( n );
count++;
}
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("db: getList for %"INT32" nodes in %"INT64" ms",count,e - t);
#endif
rt.m_needsSave = false;
@ -11317,7 +11317,7 @@ bool treetest ( ) {
int32_t numKeys = 500000;
log("db: speedtest: generating %"INT32" random keys.",numKeys);
// seed randomizer
srand ( (int32_t)gettimeofdayInMilliseconds() );
srand ( (int32_t)gettimeofdayInMilliseconds_force() );
// make list of one million random keys
key_t *k = (key_t *)mmalloc ( sizeof(key_t) * numKeys , "main" );
if ( ! k ) return log("speedtest: malloc failed");
@ -11342,7 +11342,7 @@ bool treetest ( ) {
"tree-test" ) )
return log("speedTest: tree init failed.");
// add to regular tree
int64_t t = gettimeofdayInMilliseconds();
int64_t t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < numKeys ; i++ ) {
//if ( k[i].n1 == 1234567 )
// fprintf(stderr,"i=%"INT32"\n",i);
@ -11351,14 +11351,14 @@ bool treetest ( ) {
"failed");
}
// print time it took
int64_t e = gettimeofdayInMilliseconds();
int64_t e = gettimeofdayInMilliseconds_force();
log("db: added %"INT32" keys to rdb tree in %"INT64" ms",numKeys,e - t);
// sort the list of keys
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
gbsort ( k , numKeys , sizeof(key_t) , keycmp );
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("db: sorted %"INT32" in %"INT64" ms",numKeys,e - t);
// get the list
@ -11368,13 +11368,13 @@ bool treetest ( ) {
kk.n1 = 1234567;
int32_t n = rt.getNextNode ( (collnum_t)0, (char *)&kk );
// loop it
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
int32_t count = 0;
while ( n >= 0 && --first >= 0 ) {
n = rt.getNextNode ( n );
count++;
}
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("db: getList for %"INT32" nodes in %"INT64" ms",count,e - t);
return true;
}
@ -11386,7 +11386,7 @@ bool hashtest ( ) {
int32_t numKeys = 1000000;
log("db: speedtest: generating %"INT32" random keys.",numKeys);
// seed randomizer
srand ( (int32_t)gettimeofdayInMilliseconds() );
srand ( (int32_t)gettimeofdayInMilliseconds_force() );
// make list of one million random keys
key_t *k = (key_t *)mmalloc ( sizeof(key_t) * numKeys , "main" );
if ( ! k ) return log("speedtest: malloc failed");
@ -11397,22 +11397,22 @@ bool hashtest ( ) {
HashTable ht;
ht.set ( (int32_t)(1.1 * numKeys) );
// add to regular tree
int64_t t = gettimeofdayInMilliseconds();
int64_t t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < numKeys ; i++ )
if ( ! ht.addKey ( r[i] , 1 ) )
return log("hashtest: add key failed.");
// print time it took
int64_t e = gettimeofdayInMilliseconds();
int64_t e = gettimeofdayInMilliseconds_force();
// add times
log("db: added %"INT32" keys in %"INT64" ms",numKeys,e - t);
// do the delete test
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < numKeys ; i++ )
if ( ! ht.removeKey ( r[i] ) )
return log("hashtest: add key failed.");
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
// add times
log("db: deleted %"INT32" keys in %"INT64" ms",numKeys,e - t);
@ -11474,7 +11474,7 @@ bool thrutest ( char *testdir , int64_t fileSize ) {
// write 2 gigs to the file, 1M at a time
{
int64_t t1 = gettimeofdayInMilliseconds();
int64_t t1 = gettimeofdayInMilliseconds_force();
int32_t numLoops = fileSize / bufSize;
int64_t off = 0LL;
int32_t next = 0;
@ -11487,7 +11487,7 @@ bool thrutest ( char *testdir , int64_t fileSize ) {
if ( i + 1 < numLoops && next < 100000000 ) continue;
next = 0;
// print speed every X seconds
int64_t t2 = gettimeofdayInMilliseconds();
int64_t t2 = gettimeofdayInMilliseconds_force();
float mBps = (float)off / (float)(t2-t1) / 1000.0 ;
fprintf(stderr,"wrote %"INT64" bytes in %"INT64" ms (%.1f MB/s)\n",
off,t2-t1,mBps);
@ -11497,7 +11497,7 @@ bool thrutest ( char *testdir , int64_t fileSize ) {
doreadtest:
{
int64_t t1 = gettimeofdayInMilliseconds();
int64_t t1 = gettimeofdayInMilliseconds_force();
int32_t numLoops = fileSize / bufSize;
int64_t off = 0LL;
int32_t next = 0;
@ -11510,7 +11510,7 @@ bool thrutest ( char *testdir , int64_t fileSize ) {
if ( i + 1 < numLoops && next < 100000000 ) continue;
next = 0;
// print speed every X seconds
int64_t t2 = gettimeofdayInMilliseconds();
int64_t t2 = gettimeofdayInMilliseconds_force();
float mBps = (float)off / (float)(t2-t1) / 1000.0 ;
fprintf(stderr,"read %"INT64" bytes in %"INT64" ms (%.1f MB/s)\n",
off,t2-t1,mBps);
@ -11623,7 +11623,7 @@ skip:
//pthread_t tid1 ; //, tid2;
// set time
s_startTime = gettimeofdayInMilliseconds();
s_startTime = gettimeofdayInMilliseconds_force();
int32_t stksize = 1000000 ;
int32_t bufsize = stksize * s_numThreads ;
@ -11702,12 +11702,12 @@ void *startUp ( void *state , ThreadEntry *t ) {
size = s_maxReadSize;
//if ( size < 32*1024 ) size = 32*1024;
// time it
int64_t start = gettimeofdayInMilliseconds();
int64_t start = gettimeofdayInMilliseconds_force();
//fprintf(stderr,"%"INT32") i=%"INT32" start\n",id,i );
//pread ( s_fd1 , buf , size , off );
s_f.read ( buf , size , off );
//fprintf(stderr,"%"INT32") i=%"INT32" done\n",id,i );
int64_t now = gettimeofdayInMilliseconds();
int64_t now = gettimeofdayInMilliseconds_force();
#undef usleep
usleep(0);
#define usleep(a) { char *xx=NULL;*xx=0; }
@ -12247,20 +12247,20 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
// 0.35 ms to getText()
// speed test
int64_t t = gettimeofdayInMilliseconds();
int64_t t = gettimeofdayInMilliseconds_force();
for ( int32_t k = 0 ; k < 100 ; k++ )
xd.set2 (rec, listSize, coll , NULL , 0 );
int64_t e = gettimeofdayInMilliseconds();
int64_t e = gettimeofdayInMilliseconds_force();
logf(LOG_DEBUG,"build: Took %.3f ms to set title rec.",
(float)(e-t)/100.0);
// speed test
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t k = 0 ; k < 100 ; k++ ) {
char *mm = (char *)mmalloc ( 300*1024 , "ztest");
mfree ( mm , 300*1024 ,"ztest");
}
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
logf(LOG_DEBUG,"build: Took %.3f ms to do mallocs.",
(float)(e-t)/100.0);
@ -12270,7 +12270,7 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
// loop parse
Xml xml;
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ )
if ( ! xml.set ( content , contentLen ,
false, 0, false, xd.m_version ,
@ -12280,7 +12280,7 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
return log("build: speedtestxml: xml set: %s",
mstrerror(g_errno));
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Xml::set() took %.3f ms to parse docId %"INT64".",
(double)(e - t)/100.0,docId);
double bpms = contentLen/((double)(e-t)/100.0);
@ -12289,7 +12289,7 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
xml.reset();
// loop parse
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ )
if ( ! xml.set ( content , contentLen ,
false, 0, false, xd.m_version , false ,
@ -12297,7 +12297,7 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
return log("build: xml(setparents=false): %s",
mstrerror(g_errno));
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Xml::set(setparents=false) took %.3f ms to "
"parse docId %"INT64".", (double)(e - t)/100.0,docId);
@ -12308,44 +12308,44 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
}
Words words;
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ )
if ( ! words.set ( &xml , true , true ) )
return log("build: speedtestxml: words set: %s",
mstrerror(g_errno));
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Words::set(xml,computeIds=true) took %.3f ms for %"INT32" words"
" (precount=%"INT32") for docId %"INT64".",
(double)(e - t)/100.0,words.m_numWords,words.m_preCount,docId);
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ )
if ( ! words.set2 ( &xml , true , true ) )
return log("build: speedtestxml: words set: %s",
mstrerror(g_errno));
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Words::set2(xml,computeIds=true) took %.3f ms for %"INT32" "\
"words (precount=%"INT32") for docId %"INT64".",
(double)(e - t)/100.0,words.m_numWords,words.m_preCount,docId);
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ )
if ( ! words.set ( &xml , true , false ) )
return log("build: speedtestxml: words set: %s",
mstrerror(g_errno));
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Words::set(xml,computeIds=false) "
"took %.3f ms for %"INT32" words"
" (precount=%"INT32") for docId %"INT64".",
(double)(e - t)/100.0,words.m_numWords,words.m_preCount,docId);
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ )
//if ( ! words.set ( &xml , true , true ) )
if ( ! words.set ( content , TITLEREC_CURRENT_VERSION,
@ -12353,7 +12353,7 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
return log("build: speedtestxml: words set: %s",
mstrerror(g_errno));
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Words::set(content,computeIds=true) "
"took %.3f ms for %"INT32" words "
"for docId %"INT64".",
@ -12363,14 +12363,14 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
Pos pos;
// computeWordIds from xml
words.set ( &xml , true , true ) ;
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ )
//if ( ! words.set ( &xml , true , true ) )
if ( ! pos.set ( &words , NULL ) )
return log("build: speedtestxml: pos set: %s",
mstrerror(g_errno));
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Pos::set() "
"took %.3f ms for %"INT32" words "
"for docId %"INT64".",
@ -12380,14 +12380,14 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
Bits bits;
// computeWordIds from xml
words.set ( &xml , true , true ) ;
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ )
//if ( ! words.set ( &xml , true , true ) )
if ( ! bits.setForSummary ( &words ) )
return log("build: speedtestxml: Bits set: %s",
mstrerror(g_errno));
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Bits::setForSummary() "
"took %.3f ms for %"INT32" words "
"for docId %"INT64".",
@ -12405,7 +12405,7 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
bits.set ( &words ,TITLEREC_CURRENT_VERSION, 0);
Phrases phrases;
phrases.set ( &words,&bits,true,true,TITLEREC_CURRENT_VERSION,0);
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ )
//if ( ! words.set ( &xml , true , true ) )
// do not supply xd so it will be set from scratch
@ -12423,7 +12423,7 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
mstrerror(g_errno));
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Scores::set() "
"took %.3f ms for %"INT32" words "
"for docId %"INT64".",
@ -12432,7 +12432,7 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
//Phrases phrases;
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ )
if ( ! phrases.set ( &words ,
&bits ,
@ -12443,7 +12443,7 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
return log("build: speedtestxml: Phrases set: %s",
mstrerror(g_errno));
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Phrases::set() "
"took %.3f ms for %"INT32" words "
"for docId %"INT64".",
@ -12461,7 +12461,7 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
//LinkInfo info1;
//LinkInfo info2;
// computeWordIds from xml
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ )
//if ( ! words.set ( &xml , true , true ) )
if ( ! weights.set (&words ,
@ -12482,7 +12482,7 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
return log("build: speedtestxml: Weights set: %s",
mstrerror(g_errno));
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Weights::set() "
"took %.3f ms for %"INT32" words "
"for docId %"INT64".",
@ -12494,31 +12494,31 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
log("Unicode initialization failed!");
return 1;
}
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ )
if ( ! words.set ( &xml , true , true ) )
return log("build: speedtestxml: words set: %s",
mstrerror(g_errno));
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Words::set(computeIds=true) took %.3f ms for %"INT32" words "
"for docId %"INT64".",
(double)(e - t)/100.0,words.m_numWords,docId);
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ )
if ( ! words.set ( &xml , false , true ) )
return log("build: speedtestxml: words set: %s",
mstrerror(g_errno));
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Words::set(computeIds=false) took %.3f ms for docId %"INT64".",
(double)(e - t)/100.0,docId);
*/
char *buf = (char *)mmalloc(contentLen*2+1,"main");
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ )
if ( ! xml.getText ( buf , contentLen*2+1 ,
0 , // startNode
@ -12531,13 +12531,13 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
return log("build: speedtestxml: getText: %s",
mstrerror(g_errno));
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Xml::getText(computeIds=false) took %.3f ms for docId "
"%"INT64".",(double)(e - t)/100.0,docId);
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ ) {
int32_t bufLen = xml.getText ( buf , contentLen*2+1 ,
0 , // startNode
@ -12555,7 +12555,7 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
}
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Xml::getText(computeIds=false) w/ word::set() "
"took %.3f ms for docId "
"%"INT64".",(double)(e - t)/100.0,docId);
@ -12568,7 +12568,7 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
q.set2 ( query , langUnknown , false );
matches.setQuery ( &q );
words.set ( &xml , true , 0 ) ;
t = gettimeofdayInMilliseconds();
t = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < 100 ; i++ ) {
matches.reset();
if ( ! matches.addMatches ( &words ) )
@ -12576,7 +12576,7 @@ bool parseTest ( char *coll , int64_t docId , char *query ) {
mstrerror(g_errno));
}
// print time it took
e = gettimeofdayInMilliseconds();
e = gettimeofdayInMilliseconds_force();
log("build: Matches::set() took %.3f ms for %"INT32" words"
" (precount=%"INT32") for docId %"INT64".",
(double)(e - t)/100.0,words.m_numWords,words.m_preCount,docId);
@ -12624,7 +12624,7 @@ bool summaryTest1 ( char *rec , int32_t listSize, char *coll , int64_t docId ,
// CollectionRec *cr = g_collectiondb.getRec ( coll );
// start the timer
int64_t t = gettimeofdayInMilliseconds();
int64_t t = gettimeofdayInMilliseconds_force();
//int32_t titleMaxLen = cr->m_titleMaxLen;
//bool considerTitlesFromBody = false;
@ -12680,7 +12680,7 @@ bool summaryTest1 ( char *rec , int32_t listSize, char *coll , int64_t docId ,
}
// print time it took
int64_t e = gettimeofdayInMilliseconds();
int64_t e = gettimeofdayInMilliseconds_force();
log("build: V1 Summary/Title/Gigabits generation took %.3f ms for docId "
"%"INT64".",
(double)(e - t)/100.0,docId);
@ -12698,7 +12698,7 @@ bool summaryTest2 ( char *rec , int32_t listSize, char *coll , int64_t docId ,
CollectionRec *cr = g_collectiondb.getRec ( coll );
// start the timer
int64_t t = gettimeofdayInMilliseconds();
int64_t t = gettimeofdayInMilliseconds_force();
int32_t titleMaxLen = cr->m_titleMaxLen;
int32_t summaryMaxLen = cr->m_summaryMaxLen;
@ -12804,7 +12804,7 @@ bool summaryTest2 ( char *rec , int32_t listSize, char *coll , int64_t docId ,
}
// print time it took
int64_t e = gettimeofdayInMilliseconds();
int64_t e = gettimeofdayInMilliseconds_force();
log("build: V2 Summary/Title/Gigabits generation took %.3f ms for "
"docId %"INT64".",
(double)(e - t)/100.0,docId);
@ -12823,7 +12823,7 @@ bool summaryTest3 ( char *rec , int32_t listSize, char *coll , int64_t docId ,
CollectionRec *cr = g_collectiondb.getRec ( coll );
// start the timer
int64_t t = gettimeofdayInMilliseconds();
int64_t t = gettimeofdayInMilliseconds_force();
int32_t titleMaxLen = cr->m_titleMaxLen;
int32_t summaryMaxLen = cr->m_summaryMaxLen;
@ -12961,7 +12961,7 @@ bool summaryTest3 ( char *rec , int32_t listSize, char *coll , int64_t docId ,
}
// print time it took
int64_t e = gettimeofdayInMilliseconds();
int64_t e = gettimeofdayInMilliseconds_force();
log("build: V3 Summary/Title/Gigabits generation took %.3f ms for "
"docId %"INT64".",
(double)(e - t)/100.0,docId);
@ -14229,7 +14229,7 @@ bool pingTest ( int32_t hid , uint16_t clientPort ) {
int32_t ip = h->m_ip;
ip = atoip("127.0.0.1",9);
startTime = gettimeofdayInMilliseconds();
startTime = gettimeofdayInMilliseconds_force();
to.sin_family = AF_INET;
to.sin_addr.s_addr = h->m_ip;
to.sin_port = ntohs(h->m_port);
@ -14255,7 +14255,7 @@ bool pingTest ( int32_t hid , uint16_t clientPort ) {
int32_t msgSize = 3; // indicates a debug ping packet to PingServer.cpp
up->setHeader ( dgram, msgSize, 0x11, dnum, transId, true, false , 0 );
int32_t size = up->getHeaderSize(0) + msgSize;
int64_t start = gettimeofdayInMilliseconds();
int64_t start = gettimeofdayInMilliseconds_force();
// debug
//log("db: sending %"INT32" bytes",size);
n = sendto(sock,dgram,size,0,(struct sockaddr *)&to,sizeof(to));
@ -14266,7 +14266,7 @@ bool pingTest ( int32_t hid , uint16_t clientPort ) {
readLoop2:
// loop until we read something
n = recvfrom (sock,dgram,DGRAM_SIZE,0,(sockaddr *)&from, &fromLen);
if (gettimeofdayInMilliseconds() - start>2000) {lost++; goto sendLoop;}
if (gettimeofdayInMilliseconds_force() - start>2000) {lost++; goto sendLoop;}
if ( n <= 0 ) goto readLoop2; // { sched_yield(); goto readLoop2; }
// for what transId?
int32_t tid = up->getTransId ( dgram , n );
@ -14289,7 +14289,7 @@ bool pingTest ( int32_t hid , uint16_t clientPort ) {
// debug
//log("db: read %"INT32" bytes",n);
// mark the time
int64_t took = gettimeofdayInMilliseconds()-start;
int64_t took = gettimeofdayInMilliseconds_force()-start;
if ( took > 1 ) log("net: pingtest: got reply #%"INT32" (tid=%"INT32") "
"in %"INT64" ms",replies,transId,took);
// make average
@ -15984,7 +15984,7 @@ void injectedWrapper ( void *state , TcpSocket *s ) {
}
void saveRdbs ( int fd , void *state ) {
int64_t now = gettimeofdayInMilliseconds();
int64_t now = gettimeofdayInMilliseconds_force();
int64_t last;
Rdb *rdb ;
// . try saving every 10 minutes from time of last write to disk
@ -16343,7 +16343,7 @@ void membustest ( int32_t nb , int32_t loops , bool readf ) {
g_clockNeedsUpdate = true;
// time stamp
int64_t t = gettimeofdayInMilliseconds();
int64_t t = gettimeofdayInMilliseconds_force();
fprintf(stderr,"memtest: start = %"INT64"\n",t);
@ -16400,7 +16400,7 @@ void membustest ( int32_t nb , int32_t loops , bool readf ) {
g_clockNeedsUpdate = true;
// completed
int64_t now = gettimeofdayInMilliseconds();
int64_t now = gettimeofdayInMilliseconds_force();
fprintf(stderr,"memtest: now = %"INT64"\n",t);
// multiply by 4 since these are int32_ts
char *op = "read";
@ -16981,7 +16981,7 @@ void countdomains( char* coll, int32_t numRecs, int32_t verbosity, int32_t outpu
g_titledb.getRdb()->addRdbBase1(coll );
log( LOG_INFO, "cntDm: parms: %s, %"INT32"", coll, numRecs );
int64_t time_start = gettimeofdayInMilliseconds();
int64_t time_start = gettimeofdayInMilliseconds_force();
// turn off threads
g_threads.disableThreads();
@ -17722,7 +17722,7 @@ void countdomains( char* coll, int32_t numRecs, int32_t verbosity, int32_t outpu
mfree( dom_table, numRecs * sizeof(struct dom_info *), "main-dcfdt" );
int64_t time_end = gettimeofdayInMilliseconds();
int64_t time_end = gettimeofdayInMilliseconds_force();
log( LOG_INFO, "cntDm: Took %"INT64"ms to count domains in %"INT32" recs.",
time_end-time_start, countDocs );
log( LOG_INFO, "cntDm: %"INT32" bytes of Total Memory Used.",
@ -18643,7 +18643,7 @@ void rmTest() {
// now delete
fprintf(stderr,"Deleting files\n");
int64_t now = gettimeofdayInMillisecondsLocal();
int64_t now = gettimeofdayInMilliseconds_force();
for ( int32_t i = 0 ; i < max ; i++ ) {
SafeBuf fn;
@ -18653,7 +18653,7 @@ void rmTest() {
f.unlink();
}
int64_t took = gettimeofdayInMillisecondsLocal() - now;
int64_t took = gettimeofdayInMilliseconds_force() - now;
fprintf(stderr,"Deleting files took %"INT64" ms\n",took);

8
qa.cpp

@ -26,6 +26,7 @@ void wait( float seconds ) {
NULL , // state
qatestWrapper,//m_masterLoop
0 )) {// niceness
log("qa: waiting %i milliseconds",(int)delay);
s_registered = true;
// wait for it, return -1 since we blocked
return;
@ -721,7 +722,7 @@ bool qainject1 ( ) {
if ( ! s_flags[21] ) {
wait(2.0);
wait(6.0);
s_flags[21] = true;
return false;
}
@ -1071,18 +1072,20 @@ bool qaSyntax ( ) {
return false;
}
static int s_i;
// now query check
//static bool s_y4 = false;
if ( ! s_flags[4] ) {
wait(1.5);
s_flags[4] = true;
s_i = 0;
return false;
}
//
// now run a bunch of queries
//
static int s_i = 0;
static char *s_q[] ={"cat dog",
"+cat",
"mp3 \"take five\"",
@ -2830,6 +2833,7 @@ bool qatest ( ) {
if ( s_registered ) {
g_loop.unregisterSleepCallback(NULL,qatestWrapper);
s_registered = false;
log("qa: done waiting");
}
if ( ! s_callback ) s_callback = qatest;