Forks/privacore-open-source-search-engine
1
0
Fork 0
forked from Mirrors/privacore-open-source-search-engine
Code Pull Requests Activity

Removed unused ColectionRec::m_seedHashTable

Browse Source
This commit is contained in:
Ivan Skytte Jørgensen
2017-01-23 17:18:19 +01:00
parent beae4dfbc6
commit 3d62d9c2db
3 changed files with 0 additions and 28 deletions

7
Collectiondb.cpp

@ -768,13 +768,6 @@ bool Collectiondb::resetColl2( collnum_t oldCollnum, collnum_t newCollnum, bool
cr->m_spiderStatus = SP_INITIALIZING; // this is 0
//cr->m_spiderStatusMsg = NULL;
// reset seed buf
if ( purgeSeeds ) {
// reset seed dedup table
HashTableX *ht = &cr->m_seedHashTable;
ht->reset();
}
// so XmlDoc.cpp can detect if the collection was reset since it
// launched its spider:
cr->m_lastResetCount++;

2
Collectiondb.h

@ -455,8 +455,6 @@ class CollectionRec {
int32_t m_overflow;
int32_t m_overflow2;
HashTableX m_seedHashTable;
int32_t m_maxAddUrlsPerIpDomPerDay;
// . max content length of text/html or text/plain document

19
PageCrawlBot.cpp

@ -114,25 +114,6 @@ bool printCrawlDetails2 (SafeBuf *sb , CollectionRec *cx , char format ) {
// . do not add dups into m_diffbotSeeds safebuf
// . return 0 if not in table, 1 if in table. -1 on error adding to table.
static int32_t isInSeedBuf(CollectionRec *cr, const char *url, int len) {
HashTableX *ht = &cr->m_seedHashTable;
// if table is empty, populate it
if ( ht->m_numSlotsUsed <= 0 ) {
// initialize the hash table
if ( ! ht->set(8,0,1024,NULL,0,false,"seedtbl") )
return -1;
}
// is this url in the hash table?
int64_t u64 = hash64 ( url, len );
if ( ht->isInTable ( &u64 ) ) return 1;
// add it to hashtable
if ( ! ht->addKey ( &u64 ) ) return -1;
// WAS not in table
return 0;
}