fixed bad deletenode call causing dups in

winnertree.
This commit is contained in:
Matt
2015-02-12 16:12:23 -08:00
parent 579a08d287
commit 24eac820d5
8 changed files with 99 additions and 31 deletions

@ -2426,7 +2426,7 @@ bool Rdb::addRecord ( collnum_t collnum,
// . we NO LONGER annihilate with him. why?
// . freeData should be true, the tree doesn't own the data
// so it shouldn't free it really
m_tree.deleteNode ( n , true ); // false =freeData?);
m_tree.deleteNode3 ( n , true ); // false =freeData?);
// mark as changed
//if ( ! m_needsSave ) {
// m_needsSave = true;
@ -2669,7 +2669,7 @@ bool Rdb::addRecord ( collnum_t collnum,
"errCode = %s",
rr->getUrlHash48(),
mstrerror(indexCode));
m_tree.deleteNode(tn,false);
m_tree.deleteNode3(tn,false);
}
}
// clear errors from adding to SpiderCache

@ -713,7 +713,7 @@ int32_t RdbTree::deleteNode ( collnum_t collnum , char *key , bool freeData ) {
//log("db: deleting n1=%"XINT64" n0=%"XINT64" node=%"INT32".",
// *(int64_t *)(key+8), *(int64_t *)(key+0),node);
if ( node == -1 ) return -1;
deleteNode(node,freeData);
deleteNode3(node,freeData);
return node;
}
@ -739,7 +739,7 @@ void RdbTree::deleteNodes ( collnum_t collnum ,
if ( m_collnums[node] != collnum ) break;
//if ( m_keys [node] > endKey ) return;
if ( KEYCMP(m_keys,node,endKey,0,m_ks) > 0 ) break;
deleteNode ( node , freeData );
deleteNode3 ( node , freeData );
// rotation in setDepths() will cause him to be replaced
// with one of his kids, unless he's a leaf node
//node = next;
@ -753,7 +753,7 @@ void RdbTree::deleteNodes ( collnum_t collnum ,
// . deletes node i from the tree
// . i's parent should point to i's left or right kid
// . if i has no parent then his left or right kid becomes the new top node
void RdbTree::deleteNode ( int32_t i , bool freeData ) {
void RdbTree::deleteNode3 ( int32_t i , bool freeData ) {
// sanity check
if ( ! m_isWritable ) {
log("db: Can not delete record from tree because "
@ -1107,7 +1107,7 @@ void RdbTree::deleteOrderedList ( collnum_t collnum ,
//if ( m_keys [ node ] == key && m_collnums [ node ] == collnum ) {
if ( KEYCMP(m_keys,node,key,0,m_ks)==0 && m_collnums[node] == collnum){
// trim the node from the tree
deleteNode ( node , true /*freeData?*/ );
deleteNode3 ( node , true /*freeData?*/ );
// get next node in tree
node = getNextNode ( node ) ;
// . point to next key in list to delete
@ -2765,7 +2765,7 @@ bool RdbTree::fastLoad ( BigFile *f , RdbMem *stack ) {
log("got one");
// make it negative
m_keys[i].n0 &= 0xfffffffffffffffeLL;
//deleteNode ( i , true ); // freeData?
//deleteNode3 ( i , true ); // freeData?
//goto again;
}
log("REMOVED %"INT32"",count);
@ -3116,12 +3116,12 @@ void RdbTree::cleanTree ( ) { // char **bases ) {
g_collectiondb.m_recs[m_collnums[i]] ) continue;
// if it is negtiave, remove it, that is wierd corruption
if ( m_collnums[i] < 0 )
deleteNode ( i , true );
deleteNode3 ( i , true );
// remove it otherwise
// don't actually remove it!!!! in case collection gets
// moved accidentally.
// no... otherwise it can clog up the tree forever!!!!
deleteNode ( i , true );
deleteNode3 ( i , true );
count++;
// save it
collnum = m_collnums[i];

@ -164,10 +164,10 @@ class RdbTree {
// . returns true iff was found and deleted
// . returns false iff not found
// . frees m_data[node] if freeIt is true
void deleteNode ( int32_t node , bool freeData );
//int32_t deleteNode ( collnum_t collnum , key_t &key , bool freeData ) ;
void deleteNode3 ( int32_t node , bool freeData );
//int32_t deleteNode ( collnum_t collnum , key_t &key,bool freeData) ;
int32_t deleteNode ( collnum_t collnum , char *key , bool freeData ) ;
int32_t deleteNode ( collnum_t collnum , key_t &key , bool freeData ) {
int32_t deleteNode ( collnum_t collnum , key_t &key , bool freeData) {
return deleteNode ( collnum , (char *)&key , freeData ); };
// delete all nodes with keys in [startKey,endKey]

@ -1641,6 +1641,8 @@ bool SpiderColl::makeWaitingTree ( ) {
if ( g_spiderdb.isSpiderReply ( (key128_t *)rec ) ) continue;
// get request
SpiderRequest *sreq = (SpiderRequest *)rec;
// skip if not assigned to us
if ( ! isAssignedToUs ( sreq->m_firstIp ) ) continue;
// get first ip
int32_t firstIp = sreq->m_firstIp;
// skip if in dole ip table
@ -1663,7 +1665,7 @@ bool SpiderColl::makeWaitingTree ( ) {
// add to table now since its in the tree
if ( ! m_waitingTable.addKey ( &firstIp , &fakeone ) ) {
log("spider: makeWaitTree2: %s",mstrerror(g_errno));
m_waitingTree.deleteNode ( wn , true );
m_waitingTree.deleteNode3 ( wn , true );
//log("sper: 6 del node %"INT32" for %s",wn,iptoa(firstIp));
return false;
}
@ -2587,7 +2589,7 @@ bool SpiderColl::addToWaitingTree ( uint64_t spiderTimeMS , int32_t firstIp ,
(uint32_t)(spiderTimeMS/1000LL),
iptoa(firstIp));
// remove from tree so we can add it below
m_waitingTree.deleteNode ( tn , false );
m_waitingTree.deleteNode3 ( tn , false );
//log("spider: 4 del node %"INT32" for %s",tn,iptoa(firstIp));
}
else {
@ -2661,7 +2663,7 @@ bool SpiderColl::addToWaitingTree ( uint64_t spiderTimeMS , int32_t firstIp ,
// add to table now since its in the tree
if ( ! m_waitingTable.addKey ( &firstIp , &spiderTimeMS ) ) {
// remove from tree then
m_waitingTree.deleteNode ( wn , false );
m_waitingTree.deleteNode3 ( wn , false );
//log("spider: 5 del node %"INT32" for %s",wn,iptoa(firstIp));
return false;
}
@ -2734,7 +2736,7 @@ int32_t SpiderColl::getNextIpFromWaitingTree ( ) {
// these operations should fail if writes have been disabled
// and becase the trees/tables for spidercache are saving
// in Process.cpp's g_spiderCache::save() call
m_waitingTree.deleteNode ( node , true );
m_waitingTree.deleteNode3 ( node , true );
//log("spdr: 8 del node node %"INT32" for %s",node,iptoa(firstIp));
// note it
if ( g_conf.m_logDebugSpider )
@ -3347,7 +3349,7 @@ key128_t makeUfnTreeKey ( int32_t firstIp ,
// key bitmap (192 bits):
//
// ffffffff ffffffff ffffffff ffffffff f=firstIp
// pppppppp pppppppp HHHHHHHH HHHHHHHH p=priority H=hopcount
// pppppppp pppppppp HHHHHHHH HHHHHHHH p=255-priority H=hopcount
// tttttttt tttttttt tttttttt tttttttt t=spiderTimeMS
// tttttttt tttttttt tttttttt tttttttt h=urlHash48
// hhhhhhhh hhhhhhhh hhhhhhhh hhhhhhhh
@ -3798,7 +3800,7 @@ bool SpiderColl::scanListForWinners ( ) {
sb.getBufStart(),
iptoa(firstIp));
// delete the exact node #
m_waitingTree.deleteNode ( wn , false );
m_waitingTree.deleteNode3 ( wn , false );
}
*/
//char *xx=NULL;*xx=0; }
@ -4310,6 +4312,9 @@ bool SpiderColl::scanListForWinners ( ) {
spiderTimeMS ,
uh48 );
// if ( uh48 == 110582802025376LL )
// log("hey");
// if this url is already in the winnerTree then either we
// replace it or we skip ourselves.
//
@ -4325,13 +4330,14 @@ bool SpiderColl::scanListForWinners ( ) {
oldwk = (key192_t *)m_winnerTable.
getDataFromSlot ( winSlot );
// are we lower priority? (or equal)
// smaller keys are HIGHER priority.
if(KEYCMP((char *)&wk,(char *)oldwk,
sizeof(key192_t))<=0)
sizeof(key192_t))>=0)
continue;
// from table too. no it's a dup uh48!
//m_winnerTable.deleteKey ( &uh48 );
// otherwise we supplant it. remove old key from tree.
m_winnerTree.deleteNode ( 0 , oldwk );
m_winnerTree.deleteNode ( 0 , (char *)oldwk , false);
// supplant in table and tree... just add below...
}
@ -4393,7 +4399,7 @@ bool SpiderColl::scanListForWinners ( ) {
// from table too
m_winnerTable.removeKey ( &m_tailUh48 );
// delete the tail so new spiderrequest can enter
m_winnerTree.deleteNode ( tailNode , true );
m_winnerTree.deleteNode3 ( tailNode , true );
}
@ -4454,10 +4460,72 @@ bool SpiderColl::scanListForWinners ( ) {
(char *)newMem ,
need );
// log("adding wk uh48=%llu #usednodes=%i",
// uh48,m_winnerTree.m_numUsedNodes);
// sanity
//SpiderRequest *sreq2 = (SpiderRequest *)m_winnerTree.
//getData ( nn );
// //////////////////////
// // MDW dedup test
// HashTableX dedup;
// int32_t ntn = m_winnerTree.getNumNodes();
// char dbuf[3*MAX_WINNER_NODES*(8+1)];
// dedup.set ( 8,
// 0,
// (int32_t)2*ntn, // # slots to initialize to
// dbuf,
// (int32_t)(3*MAX_WINNER_NODES*(8+1)),
// false,
// MAX_NICENESS,
// "windt");
// for ( int32_t node = m_winnerTree.getFirstNode() ;
// node >= 0 ;
// node = m_winnerTree.getNextNode ( node ) ) {
// // get data for that
// SpiderRequest *sreq2;
// sreq2 = (SpiderRequest *)m_winnerTree.getData ( node );
// // parse it up
// int32_t winIp;
// int32_t winPriority;
// int32_t winHopCount;
// int64_t winSpiderTimeMS;
// int64_t winUh48;
// key192_t *winKey = (key192_t *)m_winnerTree.getKey ( node );
// parseWinnerTreeKey ( winKey ,
// &winIp ,
// &winPriority,
// &winHopCount,
// &winSpiderTimeMS ,
// &winUh48 );
// // sanity
//if(winUh48 != sreq2->getUrlHash48() ) { char *xx=NULL;*xx=0;}
// // make the doledb key
// key_t doleKey = g_doledb.makeKey ( winPriority,
// // convert to secs from ms
// winSpiderTimeMS / 1000 ,
// winUh48 ,
// false );
// // dedup. if we add dups the problem is is that they
// // overwrite the key in doledb yet the doleiptable count
// // remains undecremented and doledb is empty and never
// // replenished because the firstip can not be added to
// // waitingTree because doleiptable count is > 0. this was
// // causing spiders to hang for collections. i am not sure
// // why we should be getting dups in winnertree because they
// // have the same uh48 and that is the key in the tree.
// if ( dedup.isInTable ( &winUh48 ) ) {
// log("spider: got dup uh48=%"UINT64" dammit", winUh48);
// char *xx=NULL;*xx=0;
// continue;
// }
// // do not allow dups
// dedup.addKey ( &winUh48 );
// }
// // end dedup test
//////////////////////////
// set new tail priority and time for next compare
if ( m_winnerTree.getNumUsedNodes() >= maxWinners ) {
// for the worst node in the tree...
@ -4957,7 +5025,7 @@ bool SpiderColl::addDoleBufIntoDoledb ( bool isFromCache ,
//int32_t wn = m_waitingTree.getNode(0,(char *)&m_waitingTreeKey);
//if ( wn < 0 ) { char *xx=NULL;*xx=0; }
if ( wn >= 0 ) {
m_waitingTree.deleteNode (wn,false );
m_waitingTree.deleteNode3 (wn,false );
//log("spdr: 2 del node %"INT32" for %s",wn,iptoa(firstIp));
}

@ -122,7 +122,7 @@ bool Syncdb::gotMetaListRequest ( char *req , int32_t reqSize , uint32_t sid ) {
int32_t dn = m_qt.getNode ( 0, (char *)&ck ) ;
// hey, we annihilated with the positive key. i guess our
// twin is ahead of us!
if ( dn >= 0 ) m_qt.deleteNode( dn , false );
if ( dn >= 0 ) m_qt.deleteNode3 ( dn , false );
// . ADD the B KEY
// . we did not annihilate with a negative key
else if ( m_qt.addKey ( &k5 ) < 0 ) { char *xx=NULL;*xx=0; }
@ -329,7 +329,7 @@ bool Syncdb::loop2 ( ) {
m_qt.deleteNode ( 0 , (char *)&k , true );
// . and delete that node (freeData = true)
// . no! not until we got all the checkoff requests in!
// stree->deleteNode ( n , true );
// stree->deleteNode3 ( n , true );
// advance on success
m_ia++;
// go back for more
@ -459,7 +459,7 @@ void Syncdb::loop3 ( ) {
// must be there!
if ( ! dn ) { char *xx=NULL;*xx=0; }
// nuke it
m_qt.deleteNode ( dn , true );
m_qt.deleteNode3 ( dn , true );
}
// . success
// . do not recall until big loop completes a round
@ -814,7 +814,7 @@ void handleRequest5d ( UdpSlot *slot , int32_t netnice ) {
int32_t nn = g_syncdb.m_qt.getNode ( 0 , (char *)&pk );
// if yes, nuke it. they annihilate.
if ( nn >= 0 ) {
g_syncdb.m_qt.deleteNode ( nn , true );
g_syncdb.m_qt.deleteNode3 ( nn , true );
continue;
}
// . otherwise, add right to the tree

@ -380,7 +380,7 @@ bool TopTree::addNode ( TopNode *t , int32_t tnn ) {
// delete him from the top tree now as well
//deleteNode ( nn , domHash );
// then delete him from the m_t2 tree
m_t2.deleteNode ( min , false );
m_t2.deleteNode3 ( min , false );
//logf(LOG_DEBUG,"deleting1 %"INT32"",min);
}
// if we have not violated the ridiculous max, just add ourselves
@ -489,7 +489,7 @@ bool TopTree::addNode ( TopNode *t , int32_t tnn ) {
// get next node from t2
int32_t next = m_t2.getNextNode ( min );
// delete from m_t2
m_t2.deleteNode ( min , false );
m_t2.deleteNode3 ( min , false );
// skip if not th emin
if ( m_domMinNode[domHash2] != min ) continue;
// if we were the last, that's it

@ -9448,7 +9448,7 @@ int32_t getTopGigabits ( HashTableX *ht ,
// sanity check
if ( tn < 0 ) { char *xx=NULL;*xx=0; }
// kick out smallest
tree.deleteNode ( tn , false );
tree.deleteNode3 ( tn , false );
// get new smallest
tn = tree.getLowestNode();
// set the new minkey

@ -10751,7 +10751,7 @@ bool bucketstest ( char* dbname ) {
int32_t n;
n = rt.getNode ( 0, oppKey );
if ( n >= 0 ) {
rt.deleteNode ( n , true );
rt.deleteNode3 ( n , true );
}
if ( rt.addNode ( 0, key , NULL , 0 ) < 0 )
return log("speedTest: rdb tree addNode "
@ -11089,7 +11089,7 @@ bool bucketstest ( char* dbname ) {
collnum_t collnum = rand() % 10;
n = rt.getNode ( collnum , oppKey );
if ( n >= 0 ) rt.deleteNode ( n , true );
if ( n >= 0 ) rt.deleteNode3 ( n , true );
if ( rt.addNode (collnum, key, NULL , 0 ) < 0 )
return log("speedTest: rdb tree addNode "