Forks/privacore-open-source-search-engine
1
0
Fork 0
forked from Mirrors/privacore-open-source-search-engine
Code Pull Requests Activity

more core stability fixes. prevent core dumps

Browse Source
This commit is contained in:
Matt Wells
2014-07-16 12:07:39 -07:00
parent 9347b1fc79
commit 6b797f5023
8 changed files with 47 additions and 37 deletions

20
Datedb.cpp

@ -8,7 +8,7 @@
// a global class extern'd in .h file
Datedb g_datedb;
Datedb g_datedb2;
//Datedb g_datedb2;
// resets rdb
void Datedb::reset() { m_rdb.reset(); }
@ -42,7 +42,7 @@ bool Datedb::init ( ) {
long maxCacheNodes = g_conf.m_datedbMaxCacheMem / 600;
long pageSize = GB_INDEXDB_PAGE_SIZE;
// long pageSize = GB_INDEXDB_PAGE_SIZE;
// we now use a disk page cache as opposed to the
// old rec cache. i am trying to do away with the Rdb::m_cache rec
// cache in favor of cleverly used disk page caches, because
@ -57,13 +57,13 @@ bool Datedb::init ( ) {
if ( g_hostdb.m_useTmpCluster ) pcmem = 0;
// . init the page cache
// . MDW: "minimize disk seeks" not working otherwise i'd enable it!
if ( ! m_pc.init ( "datedb",
RDB_DATEDB,
pcmem ,
pageSize ,
true , // use RAM disk?
false )) // minimize disk seeks?
return log("db: Datedb pc init failed.");
// if ( ! m_pc.init ( "datedb",
// RDB_DATEDB,
// pcmem ,
// pageSize ,
// true , // use RAM disk?
// false )) // minimize disk seeks?
// return log("db: Datedb pc init failed.");
// . set our own internal rdb
// . max disk space for bin tree is same as maxTreeMem so that we
@ -85,7 +85,7 @@ bool Datedb::init ( ) {
maxCacheNodes ,
true , // use half keys?
g_conf.m_datedbSaveCache ,
&m_pc ,
NULL,//&m_pc ,
false , // is titledb?
false , // preload dskpagecache
16 ); // key size

6
Datedb.h

@ -172,13 +172,13 @@ class Datedb {
Rdb m_rdb;
DiskPageCache *getDiskPageCache ( ) { return &m_pc; };
//DiskPageCache *getDiskPageCache ( ) { return &m_pc; };
DiskPageCache m_pc;
//DiskPageCache m_pc;
};
extern class Datedb g_datedb;
extern class Datedb g_datedb2;
//extern class Datedb g_datedb2;
#endif

2
PageResults.cpp

@ -2028,7 +2028,7 @@ bool printSearchResultsHeader ( State0 *st ) {
sb->safePrintf ("   "
"<font color=red><b>"
"<a href=\"/admin/api?&c=%s>api</a></b>"
"<a href=\"/admin/api?&c=%s\">api</a></b>"
, coll );
sb->safePrintf (" &nbsp; "
"<font color=red><b>"

5
Pages.cpp

@ -2602,6 +2602,11 @@ bool printApiForPage ( SafeBuf *sb , long PAGENUM , CollectionRec *cr ) {
if ( PAGENUM == PAGE_NONE ) return true;
if ( ! cr ) {
log("api: no collection provided");
return true;
}
char *pageStr = s_pages[PAGENUM].m_filename;
// unknown?

8
Posdb.cpp

@ -4553,8 +4553,12 @@ bool PosdbTable::setQueryTermInfo ( ) {
// i'm feeling if a boolean query put this in there too, the
// hashtable that maps each docid to its boolean bit vector
// where each bit stands for an operand so we can quickly evaluate
// the bit vector in a truth table
long maxSlots = maxDocIds * 2;
// the bit vector in a truth table.
// CRAP, can't use min list size because it might be behind a
// NOT operator!!! then we core trying to realloc m_bt in a thread
// below when trying to grow it. they could all be OR'd together
// so alloc the most!
long maxSlots = (grand/12) * 2;
// get total operands we used
//long numOperands = m_q->m_numWords;//Operands;
// a quoted phrase counts as a single operand

37
Rdb.cpp

@ -10,7 +10,7 @@
#include "Posdb.h"
#include "Cachedb.h"
#include "Monitordb.h"
#include "Datedb.h"
//#include "Datedb.h"
#include "Titledb.h"
#include "Spider.h"
//#include "Tfndb.h"
@ -164,8 +164,8 @@ bool Rdb::init ( char *dir ,
if ( m_rdbId == RDB2_INDEXDB2 ) m_pageSize = GB_INDEXDB_PAGE_SIZE;
if ( m_rdbId == RDB_POSDB ) m_pageSize = GB_INDEXDB_PAGE_SIZE;
if ( m_rdbId == RDB2_POSDB2 ) m_pageSize = GB_INDEXDB_PAGE_SIZE;
if ( m_rdbId == RDB_DATEDB ) m_pageSize = GB_INDEXDB_PAGE_SIZE;
if ( m_rdbId == RDB2_DATEDB2 ) m_pageSize = GB_INDEXDB_PAGE_SIZE;
//if ( m_rdbId == RDB_DATEDB ) m_pageSize = GB_INDEXDB_PAGE_SIZE;
//if ( m_rdbId == RDB2_DATEDB2 ) m_pageSize = GB_INDEXDB_PAGE_SIZE;
if ( m_rdbId == RDB_SECTIONDB ) m_pageSize = GB_INDEXDB_PAGE_SIZE;
if ( m_rdbId == RDB_PLACEDB ) m_pageSize = GB_INDEXDB_PAGE_SIZE;
if ( m_rdbId == RDB2_SECTIONDB2) m_pageSize = GB_INDEXDB_PAGE_SIZE;
@ -227,9 +227,9 @@ bool Rdb::init ( char *dir ,
(m_rdbId == RDB_INDEXDB ||
m_rdbId == RDB2_INDEXDB2 ||
m_rdbId == RDB_POSDB ||
m_rdbId == RDB2_POSDB2 ||
m_rdbId == RDB_DATEDB ||
m_rdbId == RDB2_DATEDB2
m_rdbId == RDB2_POSDB2
//m_rdbId == RDB_DATEDB ||
//m_rdbId == RDB2_DATEDB2
//m_rdbId == RDB_LINKDB ||
//m_rdbId == RDB2_LINKDB2))
))
@ -1642,7 +1642,7 @@ void attemptMergeAll ( int fd , void *state ) {
//g_checksumdb2.getRdb()->attemptMerge ( 1 , false , !state);
//g_indexdb2.getRdb()->attemptMerge ( 1 , false , !state);
g_posdb2.getRdb()->attemptMerge ( 1 , false , !state);
g_datedb2.getRdb()->attemptMerge ( 1 , false , !state);
//g_datedb2.getRdb()->attemptMerge ( 1 , false , !state);
//g_sectiondb2.getRdb()->attemptMerge ( 1 , false , !state);
g_titledb2.getRdb()->attemptMerge ( 1 , false , !state);
//g_tfndb2.getRdb()->attemptMerge ( 1 , false , !state);
@ -1714,7 +1714,7 @@ bool Rdb::addList ( collnum_t collnum , RdbList *list,
m_rdbId == RDB_TFNDB ||
m_rdbId == RDB_INDEXDB ||
m_rdbId == RDB_POSDB ||
m_rdbId == RDB_DATEDB ||
//m_rdbId == RDB_DATEDB ||
m_rdbId == RDB_CLUSTERDB ||
m_rdbId == RDB_LINKDB ||
//m_rdbId == RDB_CHECKSUMDB ||
@ -2700,7 +2700,7 @@ Rdb *getRdbFromId ( uint8_t rdbId ) {
//s_table9 [ RDB_TFNDB ] = g_tfndb.getRdb();
s_table9 [ RDB_CLUSTERDB ] = g_clusterdb.getRdb();
s_table9 [ RDB_CATDB ] = g_catdb.getRdb();
s_table9 [ RDB_DATEDB ] = g_datedb.getRdb();
//s_table9 [ RDB_DATEDB ] = g_datedb.getRdb();
s_table9 [ RDB_LINKDB ] = g_linkdb.getRdb();
s_table9 [ RDB_CACHEDB ] = g_cachedb.getRdb();
s_table9 [ RDB_SERPDB ] = g_serpdb.getRdb();
@ -2718,7 +2718,7 @@ Rdb *getRdbFromId ( uint8_t rdbId ) {
s_table9 [ RDB2_SPIDERDB2 ] = g_spiderdb2.getRdb();
//s_table9 [ RDB2_TFNDB2 ] = g_tfndb2.getRdb();
s_table9 [ RDB2_CLUSTERDB2 ] = g_clusterdb2.getRdb();
s_table9 [ RDB2_DATEDB2 ] = g_datedb2.getRdb();
//s_table9 [ RDB2_DATEDB2 ] = g_datedb2.getRdb();
s_table9 [ RDB2_LINKDB2 ] = g_linkdb2.getRdb();
s_table9 [ RDB2_REVDB2 ] = g_revdb2.getRdb();
s_table9 [ RDB2_TAGDB2 ] = g_tagdb2.getRdb();
@ -2733,7 +2733,7 @@ char getIdFromRdb ( Rdb *rdb ) {
if ( rdb == g_catdb.getRdb () ) return RDB_CATDB;
if ( rdb == g_indexdb.getRdb () ) return RDB_INDEXDB;
if ( rdb == g_posdb.getRdb () ) return RDB_POSDB;
if ( rdb == g_datedb.getRdb () ) return RDB_DATEDB;
//if ( rdb == g_datedb.getRdb () ) return RDB_DATEDB;
if ( rdb == g_titledb.getRdb () ) return RDB_TITLEDB;
if ( rdb == g_sectiondb.getRdb () ) return RDB_SECTIONDB;
if ( rdb == g_placedb.getRdb () ) return RDB_PLACEDB;
@ -2754,7 +2754,7 @@ char getIdFromRdb ( Rdb *rdb ) {
if ( rdb == g_catdb.getRdb () ) return RDB_CATDB;
if ( rdb == g_indexdb2.getRdb () ) return RDB2_INDEXDB2;
if ( rdb == g_posdb2.getRdb () ) return RDB2_POSDB2;
if ( rdb == g_datedb2.getRdb () ) return RDB2_DATEDB2;
//if ( rdb == g_datedb2.getRdb () ) return RDB2_DATEDB2;
if ( rdb == g_tagdb2.getRdb () ) return RDB2_TAGDB2;
if ( rdb == g_titledb2.getRdb () ) return RDB2_TITLEDB2;
if ( rdb == g_sectiondb2.getRdb () ) return RDB2_SECTIONDB2;
@ -2778,7 +2778,7 @@ char isSecondaryRdb ( uint8_t rdbId ) {
case RDB2_CATDB2 : return true;
case RDB2_INDEXDB2 : return true;
case RDB2_POSDB2 : return true;
case RDB2_DATEDB2 : return true;
//case RDB2_DATEDB2 : return true;
case RDB2_TAGDB2 : return true;
case RDB2_TITLEDB2 : return true;
case RDB2_SECTIONDB2 : return true;
@ -2809,14 +2809,14 @@ char getKeySizeFromRdbId ( uint8_t rdbId ) {
// assume 12
long ks = 12;
// only these are 16 as of now
if ( i == RDB_DATEDB ||
if ( //i == RDB_DATEDB ||
i == RDB_SPIDERDB ||
i == RDB_TAGDB ||
i == RDB_SYNCDB ||
i == RDB_SECTIONDB ||
i == RDB_PLACEDB ||
i == RDB2_DATEDB2 ||
//i == RDB2_DATEDB2 ||
i == RDB2_SPIDERDB2 ||
i == RDB2_TAGDB2 ||
i == RDB2_SECTIONDB2 ||
@ -2853,7 +2853,7 @@ long getDataSizeFromRdbId ( uint8_t rdbId ) {
i == RDB_INDEXDB ||
i == RDB_TFNDB ||
i == RDB_CLUSTERDB ||
i == RDB_DATEDB ||
//i == RDB_DATEDB ||
//i == RDB_FAKEDB ||
i == RDB_LINKDB )
ds = 0;
@ -2878,8 +2878,9 @@ long getDataSizeFromRdbId ( uint8_t rdbId ) {
i == RDB2_INDEXDB2 ||
i == RDB2_TFNDB2 ||
i == RDB2_CLUSTERDB2 ||
i == RDB2_LINKDB2 ||
i == RDB2_DATEDB2 )
i == RDB2_LINKDB2
//i == RDB2_DATEDB2 )
)
ds = 0;
else if ( i == RDB2_TITLEDB2 ||
i == RDB2_REVDB2 ||

4
SafeBuf.cpp

@ -3411,8 +3411,8 @@ bool SafeBuf::base64Decode ( char *src , long srcLen , long niceness ) {
for ( unsigned char c = '0' ; c <= '9'; c++ )
s_bmap[c] = val++;
if ( val != 62 ) { char *xx=NULL;*xx=0; }
s_bmap['+'] = 62;
s_bmap['/'] = 63;
s_bmap[(unsigned char)'+'] = 62;
s_bmap[(unsigned char)'/'] = 63;
}
// reserve twice as much space i guess

2
SpiderProxy.cpp

@ -706,7 +706,7 @@ void handleRequest54 ( UdpSlot *udpSlot , long niceness ) {
goto redo;
}
long long oldest = 0x7fffffffffffffff;
long long oldest = 0x7fffffffffffffffLL;
SpiderProxy *winnersp = NULL;
// now find the best proxy wih the minCount
for ( long i = 0 ; i < s_iptab.getNumSlots() ; i++ ) {