forked from Mirrors/privacore-open-source-search-engine
92 lines
2.6 KiB
C++
92 lines
2.6 KiB
C++
#include "Doledb.h"
|
|
#include "SpiderCache.h"
|
|
#include "SpiderLoop.h"
|
|
#include "SpiderColl.h"
|
|
#include "ScopedLock.h"
|
|
|
|
|
|
Doledb g_doledb;
|
|
|
|
|
|
/////////////////////////
|
|
///////////////////////// DOLEDB
|
|
/////////////////////////
|
|
|
|
// reset rdb
|
|
void Doledb::reset() { m_rdb.reset(); }
|
|
|
|
bool Doledb::init ( ) {
|
|
// . what's max # of tree nodes?
|
|
// . assume avg spider rec size (url) is about 45
|
|
// . 45 + 33 bytes overhead in tree is 78
|
|
// . use 5MB for the tree
|
|
int32_t maxTreeMem = 150000000; // 150MB
|
|
int32_t maxTreeNodes = maxTreeMem / 78;
|
|
|
|
// initialize our own internal rdb
|
|
return m_rdb.init ( "doledb" ,
|
|
-1 , // fixedDataSize
|
|
2 , // MinFilesToMerge
|
|
maxTreeMem ,
|
|
maxTreeNodes ,
|
|
false, // half keys?
|
|
12, // key size
|
|
false); //useIndexFile
|
|
}
|
|
|
|
//
|
|
// remove all recs from doledb for the given collection
|
|
//
|
|
static void nukeDoledbWrapper ( int fd , void *state ) {
|
|
g_loop.unregisterSleepCallback ( state , nukeDoledbWrapper );
|
|
collnum_t collnum = *(collnum_t *)state;
|
|
nukeDoledb ( collnum );
|
|
}
|
|
|
|
void nukeDoledb ( collnum_t collnum ) {
|
|
// in case we changed url filters for this collection #
|
|
{
|
|
RdbCacheLock rcl(g_spiderLoop.m_winnerListCache);
|
|
g_spiderLoop.m_winnerListCache.clear ( collnum );
|
|
}
|
|
|
|
// . nuke doledb for this collnum
|
|
// . it will unlink the files and maps for doledb for this collnum
|
|
// . it will remove all recs of this collnum from its tree too
|
|
if (g_doledb.getRdb()->isSavingTree()) {
|
|
g_loop.registerSleepCallback(100, &collnum, nukeDoledbWrapper, "Doledb::nukeDoledbWrapper");
|
|
return;
|
|
}
|
|
|
|
// . ok, tree is not saving, it should complete entirely from this call
|
|
g_doledb.getRdb()->deleteAllRecs(collnum);
|
|
|
|
SpiderColl *sc = g_spiderCache.getSpiderCollIffNonNull(collnum);
|
|
if ( sc ) {
|
|
// . make sure to nuke m_doledbIpTable as well
|
|
sc->clearDoledbIpTable();
|
|
// need to recompute this!
|
|
//sc->m_ufnMapValid = false;
|
|
|
|
{
|
|
ScopedLock sl(sc->m_waitingTree.getLock());
|
|
|
|
// log it
|
|
log("spider: rebuilding %s from doledb nuke", sc->getCollName());
|
|
// activate a scan if not already activated
|
|
sc->m_waitingTreeNeedsRebuild = true;
|
|
// if a scan is ongoing, this will re-set it
|
|
sc->resetWaitingTreeNextKey();
|
|
// clear it?
|
|
sc->m_waitingTree.clear_unlocked();
|
|
sc->clearWaitingTable();
|
|
}
|
|
// kick off the spiderdb scan to rep waiting tree and doledb
|
|
sc->populateWaitingTreeFromSpiderdb(false);
|
|
}
|
|
|
|
// note it
|
|
log("spider: finished nuking doledb for coll (%" PRId32")",
|
|
(int32_t)collnum);
|
|
}
|