mirror of
https://github.com/privacore/open-source-search-engine.git
synced 2025-07-15 02:36:08 -04:00
fixed bad deletenode call causing dups in
winnertree.
This commit is contained in:
4
Rdb.cpp
4
Rdb.cpp
@ -2426,7 +2426,7 @@ bool Rdb::addRecord ( collnum_t collnum,
|
||||
// . we NO LONGER annihilate with him. why?
|
||||
// . freeData should be true, the tree doesn't own the data
|
||||
// so it shouldn't free it really
|
||||
m_tree.deleteNode ( n , true ); // false =freeData?);
|
||||
m_tree.deleteNode3 ( n , true ); // false =freeData?);
|
||||
// mark as changed
|
||||
//if ( ! m_needsSave ) {
|
||||
// m_needsSave = true;
|
||||
@ -2669,7 +2669,7 @@ bool Rdb::addRecord ( collnum_t collnum,
|
||||
"errCode = %s",
|
||||
rr->getUrlHash48(),
|
||||
mstrerror(indexCode));
|
||||
m_tree.deleteNode(tn,false);
|
||||
m_tree.deleteNode3(tn,false);
|
||||
}
|
||||
}
|
||||
// clear errors from adding to SpiderCache
|
||||
|
14
RdbTree.cpp
14
RdbTree.cpp
@ -713,7 +713,7 @@ int32_t RdbTree::deleteNode ( collnum_t collnum , char *key , bool freeData ) {
|
||||
//log("db: deleting n1=%"XINT64" n0=%"XINT64" node=%"INT32".",
|
||||
// *(int64_t *)(key+8), *(int64_t *)(key+0),node);
|
||||
if ( node == -1 ) return -1;
|
||||
deleteNode(node,freeData);
|
||||
deleteNode3(node,freeData);
|
||||
return node;
|
||||
}
|
||||
|
||||
@ -739,7 +739,7 @@ void RdbTree::deleteNodes ( collnum_t collnum ,
|
||||
if ( m_collnums[node] != collnum ) break;
|
||||
//if ( m_keys [node] > endKey ) return;
|
||||
if ( KEYCMP(m_keys,node,endKey,0,m_ks) > 0 ) break;
|
||||
deleteNode ( node , freeData );
|
||||
deleteNode3 ( node , freeData );
|
||||
// rotation in setDepths() will cause him to be replaced
|
||||
// with one of his kids, unless he's a leaf node
|
||||
//node = next;
|
||||
@ -753,7 +753,7 @@ void RdbTree::deleteNodes ( collnum_t collnum ,
|
||||
// . deletes node i from the tree
|
||||
// . i's parent should point to i's left or right kid
|
||||
// . if i has no parent then his left or right kid becomes the new top node
|
||||
void RdbTree::deleteNode ( int32_t i , bool freeData ) {
|
||||
void RdbTree::deleteNode3 ( int32_t i , bool freeData ) {
|
||||
// sanity check
|
||||
if ( ! m_isWritable ) {
|
||||
log("db: Can not delete record from tree because "
|
||||
@ -1107,7 +1107,7 @@ void RdbTree::deleteOrderedList ( collnum_t collnum ,
|
||||
//if ( m_keys [ node ] == key && m_collnums [ node ] == collnum ) {
|
||||
if ( KEYCMP(m_keys,node,key,0,m_ks)==0 && m_collnums[node] == collnum){
|
||||
// trim the node from the tree
|
||||
deleteNode ( node , true /*freeData?*/ );
|
||||
deleteNode3 ( node , true /*freeData?*/ );
|
||||
// get next node in tree
|
||||
node = getNextNode ( node ) ;
|
||||
// . point to next key in list to delete
|
||||
@ -2765,7 +2765,7 @@ bool RdbTree::fastLoad ( BigFile *f , RdbMem *stack ) {
|
||||
log("got one");
|
||||
// make it negative
|
||||
m_keys[i].n0 &= 0xfffffffffffffffeLL;
|
||||
//deleteNode ( i , true ); // freeData?
|
||||
//deleteNode3 ( i , true ); // freeData?
|
||||
//goto again;
|
||||
}
|
||||
log("REMOVED %"INT32"",count);
|
||||
@ -3116,12 +3116,12 @@ void RdbTree::cleanTree ( ) { // char **bases ) {
|
||||
g_collectiondb.m_recs[m_collnums[i]] ) continue;
|
||||
// if it is negtiave, remove it, that is wierd corruption
|
||||
if ( m_collnums[i] < 0 )
|
||||
deleteNode ( i , true );
|
||||
deleteNode3 ( i , true );
|
||||
// remove it otherwise
|
||||
// don't actually remove it!!!! in case collection gets
|
||||
// moved accidentally.
|
||||
// no... otherwise it can clog up the tree forever!!!!
|
||||
deleteNode ( i , true );
|
||||
deleteNode3 ( i , true );
|
||||
count++;
|
||||
// save it
|
||||
collnum = m_collnums[i];
|
||||
|
@ -164,10 +164,10 @@ class RdbTree {
|
||||
// . returns true iff was found and deleted
|
||||
// . returns false iff not found
|
||||
// . frees m_data[node] if freeIt is true
|
||||
void deleteNode ( int32_t node , bool freeData );
|
||||
//int32_t deleteNode ( collnum_t collnum , key_t &key , bool freeData ) ;
|
||||
void deleteNode3 ( int32_t node , bool freeData );
|
||||
//int32_t deleteNode ( collnum_t collnum , key_t &key,bool freeData) ;
|
||||
int32_t deleteNode ( collnum_t collnum , char *key , bool freeData ) ;
|
||||
int32_t deleteNode ( collnum_t collnum , key_t &key , bool freeData ) {
|
||||
int32_t deleteNode ( collnum_t collnum , key_t &key , bool freeData) {
|
||||
return deleteNode ( collnum , (char *)&key , freeData ); };
|
||||
|
||||
// delete all nodes with keys in [startKey,endKey]
|
||||
|
88
Spider.cpp
88
Spider.cpp
@ -1641,6 +1641,8 @@ bool SpiderColl::makeWaitingTree ( ) {
|
||||
if ( g_spiderdb.isSpiderReply ( (key128_t *)rec ) ) continue;
|
||||
// get request
|
||||
SpiderRequest *sreq = (SpiderRequest *)rec;
|
||||
// skip if not assigned to us
|
||||
if ( ! isAssignedToUs ( sreq->m_firstIp ) ) continue;
|
||||
// get first ip
|
||||
int32_t firstIp = sreq->m_firstIp;
|
||||
// skip if in dole ip table
|
||||
@ -1663,7 +1665,7 @@ bool SpiderColl::makeWaitingTree ( ) {
|
||||
// add to table now since its in the tree
|
||||
if ( ! m_waitingTable.addKey ( &firstIp , &fakeone ) ) {
|
||||
log("spider: makeWaitTree2: %s",mstrerror(g_errno));
|
||||
m_waitingTree.deleteNode ( wn , true );
|
||||
m_waitingTree.deleteNode3 ( wn , true );
|
||||
//log("sper: 6 del node %"INT32" for %s",wn,iptoa(firstIp));
|
||||
return false;
|
||||
}
|
||||
@ -2587,7 +2589,7 @@ bool SpiderColl::addToWaitingTree ( uint64_t spiderTimeMS , int32_t firstIp ,
|
||||
(uint32_t)(spiderTimeMS/1000LL),
|
||||
iptoa(firstIp));
|
||||
// remove from tree so we can add it below
|
||||
m_waitingTree.deleteNode ( tn , false );
|
||||
m_waitingTree.deleteNode3 ( tn , false );
|
||||
//log("spider: 4 del node %"INT32" for %s",tn,iptoa(firstIp));
|
||||
}
|
||||
else {
|
||||
@ -2661,7 +2663,7 @@ bool SpiderColl::addToWaitingTree ( uint64_t spiderTimeMS , int32_t firstIp ,
|
||||
// add to table now since its in the tree
|
||||
if ( ! m_waitingTable.addKey ( &firstIp , &spiderTimeMS ) ) {
|
||||
// remove from tree then
|
||||
m_waitingTree.deleteNode ( wn , false );
|
||||
m_waitingTree.deleteNode3 ( wn , false );
|
||||
//log("spider: 5 del node %"INT32" for %s",wn,iptoa(firstIp));
|
||||
return false;
|
||||
}
|
||||
@ -2734,7 +2736,7 @@ int32_t SpiderColl::getNextIpFromWaitingTree ( ) {
|
||||
// these operations should fail if writes have been disabled
|
||||
// and becase the trees/tables for spidercache are saving
|
||||
// in Process.cpp's g_spiderCache::save() call
|
||||
m_waitingTree.deleteNode ( node , true );
|
||||
m_waitingTree.deleteNode3 ( node , true );
|
||||
//log("spdr: 8 del node node %"INT32" for %s",node,iptoa(firstIp));
|
||||
// note it
|
||||
if ( g_conf.m_logDebugSpider )
|
||||
@ -3347,7 +3349,7 @@ key128_t makeUfnTreeKey ( int32_t firstIp ,
|
||||
// key bitmap (192 bits):
|
||||
//
|
||||
// ffffffff ffffffff ffffffff ffffffff f=firstIp
|
||||
// pppppppp pppppppp HHHHHHHH HHHHHHHH p=priority H=hopcount
|
||||
// pppppppp pppppppp HHHHHHHH HHHHHHHH p=255-priority H=hopcount
|
||||
// tttttttt tttttttt tttttttt tttttttt t=spiderTimeMS
|
||||
// tttttttt tttttttt tttttttt tttttttt h=urlHash48
|
||||
// hhhhhhhh hhhhhhhh hhhhhhhh hhhhhhhh
|
||||
@ -3798,7 +3800,7 @@ bool SpiderColl::scanListForWinners ( ) {
|
||||
sb.getBufStart(),
|
||||
iptoa(firstIp));
|
||||
// delete the exact node #
|
||||
m_waitingTree.deleteNode ( wn , false );
|
||||
m_waitingTree.deleteNode3 ( wn , false );
|
||||
}
|
||||
*/
|
||||
//char *xx=NULL;*xx=0; }
|
||||
@ -4310,6 +4312,9 @@ bool SpiderColl::scanListForWinners ( ) {
|
||||
spiderTimeMS ,
|
||||
uh48 );
|
||||
|
||||
// if ( uh48 == 110582802025376LL )
|
||||
// log("hey");
|
||||
|
||||
// if this url is already in the winnerTree then either we
|
||||
// replace it or we skip ourselves.
|
||||
//
|
||||
@ -4325,13 +4330,14 @@ bool SpiderColl::scanListForWinners ( ) {
|
||||
oldwk = (key192_t *)m_winnerTable.
|
||||
getDataFromSlot ( winSlot );
|
||||
// are we lower priority? (or equal)
|
||||
// smaller keys are HIGHER priority.
|
||||
if(KEYCMP((char *)&wk,(char *)oldwk,
|
||||
sizeof(key192_t))<=0)
|
||||
sizeof(key192_t))>=0)
|
||||
continue;
|
||||
// from table too. no it's a dup uh48!
|
||||
//m_winnerTable.deleteKey ( &uh48 );
|
||||
// otherwise we supplant it. remove old key from tree.
|
||||
m_winnerTree.deleteNode ( 0 , oldwk );
|
||||
m_winnerTree.deleteNode ( 0 , (char *)oldwk , false);
|
||||
// supplant in table and tree... just add below...
|
||||
}
|
||||
|
||||
@ -4393,7 +4399,7 @@ bool SpiderColl::scanListForWinners ( ) {
|
||||
// from table too
|
||||
m_winnerTable.removeKey ( &m_tailUh48 );
|
||||
// delete the tail so new spiderrequest can enter
|
||||
m_winnerTree.deleteNode ( tailNode , true );
|
||||
m_winnerTree.deleteNode3 ( tailNode , true );
|
||||
|
||||
}
|
||||
|
||||
@ -4454,10 +4460,72 @@ bool SpiderColl::scanListForWinners ( ) {
|
||||
(char *)newMem ,
|
||||
need );
|
||||
|
||||
// log("adding wk uh48=%llu #usednodes=%i",
|
||||
// uh48,m_winnerTree.m_numUsedNodes);
|
||||
|
||||
// sanity
|
||||
//SpiderRequest *sreq2 = (SpiderRequest *)m_winnerTree.
|
||||
//getData ( nn );
|
||||
|
||||
// //////////////////////
|
||||
// // MDW dedup test
|
||||
// HashTableX dedup;
|
||||
// int32_t ntn = m_winnerTree.getNumNodes();
|
||||
// char dbuf[3*MAX_WINNER_NODES*(8+1)];
|
||||
// dedup.set ( 8,
|
||||
// 0,
|
||||
// (int32_t)2*ntn, // # slots to initialize to
|
||||
// dbuf,
|
||||
// (int32_t)(3*MAX_WINNER_NODES*(8+1)),
|
||||
// false,
|
||||
// MAX_NICENESS,
|
||||
// "windt");
|
||||
// for ( int32_t node = m_winnerTree.getFirstNode() ;
|
||||
// node >= 0 ;
|
||||
// node = m_winnerTree.getNextNode ( node ) ) {
|
||||
// // get data for that
|
||||
// SpiderRequest *sreq2;
|
||||
// sreq2 = (SpiderRequest *)m_winnerTree.getData ( node );
|
||||
// // parse it up
|
||||
// int32_t winIp;
|
||||
// int32_t winPriority;
|
||||
// int32_t winHopCount;
|
||||
// int64_t winSpiderTimeMS;
|
||||
// int64_t winUh48;
|
||||
// key192_t *winKey = (key192_t *)m_winnerTree.getKey ( node );
|
||||
// parseWinnerTreeKey ( winKey ,
|
||||
// &winIp ,
|
||||
// &winPriority,
|
||||
// &winHopCount,
|
||||
// &winSpiderTimeMS ,
|
||||
// &winUh48 );
|
||||
// // sanity
|
||||
//if(winUh48 != sreq2->getUrlHash48() ) { char *xx=NULL;*xx=0;}
|
||||
// // make the doledb key
|
||||
// key_t doleKey = g_doledb.makeKey ( winPriority,
|
||||
// // convert to secs from ms
|
||||
// winSpiderTimeMS / 1000 ,
|
||||
// winUh48 ,
|
||||
// false );
|
||||
// // dedup. if we add dups the problem is is that they
|
||||
// // overwrite the key in doledb yet the doleiptable count
|
||||
// // remains undecremented and doledb is empty and never
|
||||
// // replenished because the firstip can not be added to
|
||||
// // waitingTree because doleiptable count is > 0. this was
|
||||
// // causing spiders to hang for collections. i am not sure
|
||||
// // why we should be getting dups in winnertree because they
|
||||
// // have the same uh48 and that is the key in the tree.
|
||||
// if ( dedup.isInTable ( &winUh48 ) ) {
|
||||
// log("spider: got dup uh48=%"UINT64" dammit", winUh48);
|
||||
// char *xx=NULL;*xx=0;
|
||||
// continue;
|
||||
// }
|
||||
// // do not allow dups
|
||||
// dedup.addKey ( &winUh48 );
|
||||
// }
|
||||
// // end dedup test
|
||||
//////////////////////////
|
||||
|
||||
// set new tail priority and time for next compare
|
||||
if ( m_winnerTree.getNumUsedNodes() >= maxWinners ) {
|
||||
// for the worst node in the tree...
|
||||
@ -4957,7 +5025,7 @@ bool SpiderColl::addDoleBufIntoDoledb ( bool isFromCache ,
|
||||
//int32_t wn = m_waitingTree.getNode(0,(char *)&m_waitingTreeKey);
|
||||
//if ( wn < 0 ) { char *xx=NULL;*xx=0; }
|
||||
if ( wn >= 0 ) {
|
||||
m_waitingTree.deleteNode (wn,false );
|
||||
m_waitingTree.deleteNode3 (wn,false );
|
||||
//log("spdr: 2 del node %"INT32" for %s",wn,iptoa(firstIp));
|
||||
}
|
||||
|
||||
|
@ -122,7 +122,7 @@ bool Syncdb::gotMetaListRequest ( char *req , int32_t reqSize , uint32_t sid ) {
|
||||
int32_t dn = m_qt.getNode ( 0, (char *)&ck ) ;
|
||||
// hey, we annihilated with the positive key. i guess our
|
||||
// twin is ahead of us!
|
||||
if ( dn >= 0 ) m_qt.deleteNode( dn , false );
|
||||
if ( dn >= 0 ) m_qt.deleteNode3 ( dn , false );
|
||||
// . ADD the B KEY
|
||||
// . we did not annihilate with a negative key
|
||||
else if ( m_qt.addKey ( &k5 ) < 0 ) { char *xx=NULL;*xx=0; }
|
||||
@ -329,7 +329,7 @@ bool Syncdb::loop2 ( ) {
|
||||
m_qt.deleteNode ( 0 , (char *)&k , true );
|
||||
// . and delete that node (freeData = true)
|
||||
// . no! not until we got all the checkoff requests in!
|
||||
// stree->deleteNode ( n , true );
|
||||
// stree->deleteNode3 ( n , true );
|
||||
// advance on success
|
||||
m_ia++;
|
||||
// go back for more
|
||||
@ -459,7 +459,7 @@ void Syncdb::loop3 ( ) {
|
||||
// must be there!
|
||||
if ( ! dn ) { char *xx=NULL;*xx=0; }
|
||||
// nuke it
|
||||
m_qt.deleteNode ( dn , true );
|
||||
m_qt.deleteNode3 ( dn , true );
|
||||
}
|
||||
// . success
|
||||
// . do not recall until big loop completes a round
|
||||
@ -814,7 +814,7 @@ void handleRequest5d ( UdpSlot *slot , int32_t netnice ) {
|
||||
int32_t nn = g_syncdb.m_qt.getNode ( 0 , (char *)&pk );
|
||||
// if yes, nuke it. they annihilate.
|
||||
if ( nn >= 0 ) {
|
||||
g_syncdb.m_qt.deleteNode ( nn , true );
|
||||
g_syncdb.m_qt.deleteNode3 ( nn , true );
|
||||
continue;
|
||||
}
|
||||
// . otherwise, add right to the tree
|
||||
|
@ -380,7 +380,7 @@ bool TopTree::addNode ( TopNode *t , int32_t tnn ) {
|
||||
// delete him from the top tree now as well
|
||||
//deleteNode ( nn , domHash );
|
||||
// then delete him from the m_t2 tree
|
||||
m_t2.deleteNode ( min , false );
|
||||
m_t2.deleteNode3 ( min , false );
|
||||
//logf(LOG_DEBUG,"deleting1 %"INT32"",min);
|
||||
}
|
||||
// if we have not violated the ridiculous max, just add ourselves
|
||||
@ -489,7 +489,7 @@ bool TopTree::addNode ( TopNode *t , int32_t tnn ) {
|
||||
// get next node from t2
|
||||
int32_t next = m_t2.getNextNode ( min );
|
||||
// delete from m_t2
|
||||
m_t2.deleteNode ( min , false );
|
||||
m_t2.deleteNode3 ( min , false );
|
||||
// skip if not th emin
|
||||
if ( m_domMinNode[domHash2] != min ) continue;
|
||||
// if we were the last, that's it
|
||||
|
@ -9448,7 +9448,7 @@ int32_t getTopGigabits ( HashTableX *ht ,
|
||||
// sanity check
|
||||
if ( tn < 0 ) { char *xx=NULL;*xx=0; }
|
||||
// kick out smallest
|
||||
tree.deleteNode ( tn , false );
|
||||
tree.deleteNode3 ( tn , false );
|
||||
// get new smallest
|
||||
tn = tree.getLowestNode();
|
||||
// set the new minkey
|
||||
|
4
main.cpp
4
main.cpp
@ -10751,7 +10751,7 @@ bool bucketstest ( char* dbname ) {
|
||||
int32_t n;
|
||||
n = rt.getNode ( 0, oppKey );
|
||||
if ( n >= 0 ) {
|
||||
rt.deleteNode ( n , true );
|
||||
rt.deleteNode3 ( n , true );
|
||||
}
|
||||
if ( rt.addNode ( 0, key , NULL , 0 ) < 0 )
|
||||
return log("speedTest: rdb tree addNode "
|
||||
@ -11089,7 +11089,7 @@ bool bucketstest ( char* dbname ) {
|
||||
collnum_t collnum = rand() % 10;
|
||||
|
||||
n = rt.getNode ( collnum , oppKey );
|
||||
if ( n >= 0 ) rt.deleteNode ( n , true );
|
||||
if ( n >= 0 ) rt.deleteNode3 ( n , true );
|
||||
|
||||
if ( rt.addNode (collnum, key, NULL , 0 ) < 0 )
|
||||
return log("speedTest: rdb tree addNode "
|
||||
|
Reference in New Issue
Block a user