Mirrors/privacore-open-source-search-engine
1
0
Fork 1
mirror of https://github.com/privacore/open-source-search-engine.git synced 2025-07-15 02:36:08 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:privacore/open-source-search-engine

Browse Source
This commit is contained in:
Ivan Skytte Jørgensen
2018-01-29 15:25:20 +01:00
parent 419e0f0baf 289c2cdbcf
commit 9bb06c8d7c
5 changed files with 56 additions and 38 deletions

4
Conf.cpp

@ -229,7 +229,9 @@ Conf::Conf ( ) {
m_logTraceLanguageResultOverride = false;
m_logTraceMem = false;
m_logTraceMsg0 = false;
m_logTraceMsg4 = false;
m_logTraceMsg4In = false;
m_logTraceMsg4Out = false;
m_logTraceMsg4OutData = false;
m_logTraceMsg25 = false;
m_logTracePageLinkdbLookup = false;
m_logTracePageSpiderdbLookup = false;

4
Conf.h

@ -376,7 +376,9 @@ class Conf {
bool m_logTraceLanguageResultOverride;
bool m_logTraceMem;
bool m_logTraceMsg0;
bool m_logTraceMsg4;
bool m_logTraceMsg4In;
bool m_logTraceMsg4Out;
bool m_logTraceMsg4OutData;
bool m_logTraceMsg25;
bool m_logTracePageLinkdbLookup;
bool m_logTracePageSpiderdbLookup;

12
Msg4In.cpp

@ -34,7 +34,7 @@ static GbThreadQueue s_incomingThreadQueue;
// all these parameters should be preset
bool Msg4In::registerHandler() {
logTrace( g_conf.m_logTraceMsg4, "BEGIN" );
logTrace( g_conf.m_logTraceMsg4In, "BEGIN" );
// register ourselves with the udp server
if ( ! g_udpServer.registerHandler ( msg_type_4, handleRequest4 ) ) {
@ -42,7 +42,7 @@ bool Msg4In::registerHandler() {
return false;
}
logTrace( g_conf.m_logTraceMsg4, "END - returning true");
logTrace( g_conf.m_logTraceMsg4In, "END - returning true");
return true;
}
@ -65,7 +65,7 @@ void Msg4In::finalizeIncomingThread() {
static void Msg4In::processMsg4(void *item) {
UdpSlot *slot = static_cast<UdpSlot*>(item);
logTrace( g_conf.m_logTraceMsg4, "BEGIN" );
logTrace( g_conf.m_logTraceMsg4In, "BEGIN" );
// extract what we read
char *readBuf = slot->m_readBuf;
@ -75,14 +75,14 @@ static void Msg4In::processMsg4(void *item) {
logError("calling sendErrorReply error='%s'", mstrerror(g_errno));
g_udpServer.sendErrorReply(slot,g_errno);
logTrace(g_conf.m_logTraceMsg4, "END - addMetaList returned false. g_errno=%d", g_errno);
logTrace(g_conf.m_logTraceMsg4In, "END - addMetaList returned false. g_errno=%d", g_errno);
return;
}
// good to go
g_udpServer.sendReply(NULL, 0, NULL, 0, slot);
logTrace(g_conf.m_logTraceMsg4, "END - OK");
logTrace(g_conf.m_logTraceMsg4In, "END - OK");
}
static void Msg4In::handleRequest4(UdpSlot *slot, int32_t /*netnice*/) {
@ -164,7 +164,7 @@ static void Msg4In::handleRequest4(UdpSlot *slot, int32_t /*netnice*/) {
logError("call sendErrorReply");
g_udpServer.sendErrorReply(slot,g_errno);
logTrace( g_conf.m_logTraceMsg4, "END - ETRYAGAIN. Waiting to sync with host #0" );
logTrace( g_conf.m_logTraceMsg4In, "END - ETRYAGAIN. Waiting to sync with host #0" );
return;
}

54
Msg4Out.cpp

@ -123,7 +123,7 @@ bool Msg4::initializeOutHandling() {
// . returns false on failure
bool rc = g_loop.registerSleepCallback(MSG4_WAIT, NULL, sleepCallback4, "Msg4Out::sleepCallback4");
logTrace( g_conf.m_logTraceMsg4, "END - returning %s", rc?"true":"false");
logTrace( g_conf.m_logTraceMsg4Out, "END - returning %s", rc?"true":"false");
return rc;
}
@ -173,13 +173,13 @@ Msg4::~Msg4() {
// used by Repair.cpp to make sure we are not adding any more data ("writing")
bool hasAddsInQueue() {
logTrace( g_conf.m_logTraceMsg4, "BEGIN" );
logTrace( g_conf.m_logTraceMsg4Out, "BEGIN" );
// if there is an outstanding multicast...
{
ScopedLock sl(s_mtxMcasts);
if ( s_multicastInUseCount>0 ) {
logTrace( g_conf.m_logTraceMsg4, "END - multicast waiting, returning true" );
logTrace( g_conf.m_logTraceMsg4Out, "END - multicast waiting, returning true" );
return true;
}
}
@ -188,7 +188,7 @@ bool hasAddsInQueue() {
{
ScopedLock sl(s_mtxQueuedMsg4s);
if(!s_queuedMsg4s.empty()) {
logTrace( g_conf.m_logTraceMsg4, "END - msg4 waiting, returning true" );
logTrace( g_conf.m_logTraceMsg4Out, "END - msg4 waiting, returning true" );
return true;
}
}
@ -201,13 +201,13 @@ bool hasAddsInQueue() {
}
if ( *(int32_t *)s_hostBufs[i] > 4 ) {
logTrace( g_conf.m_logTraceMsg4, "END - hostbuf waiting, returning true" );
logTrace( g_conf.m_logTraceMsg4Out, "END - hostbuf waiting, returning true" );
return true;
}
}
// otherwise, we have nothing queued up to add
logTrace( g_conf.m_logTraceMsg4, "END - nothing queued, returning false" );
logTrace( g_conf.m_logTraceMsg4Out, "END - nothing queued, returning false" );
return false;
}
@ -358,7 +358,7 @@ void Msg4::prepareMetaList() {
VALGRIND_CHECK_MEM_IS_DEFINED(key,p-key);
#endif
logTrace(g_conf.m_logTraceMsg4, " rdb=%s key=%s keySize=%" PRId32" isDel=%d dataSize=%" PRId32" shardNum=%" PRId32" hostId=%" PRId32,
logTrace(g_conf.m_logTraceMsg4OutData, " rdb=%s key=%s keySize=%" PRId32" isDel=%d dataSize=%" PRId32" shardNum=%" PRId32" hostId=%" PRId32,
getDbnameFromId(rdbId), KEYSTR(key, ks), ks, del, dataSize, shardNum, hostId);
int32_t recSize = p - key;
@ -373,7 +373,7 @@ void Msg4::prepareMetaList() {
}
bool Msg4::processMetaList() {
logTrace( g_conf.m_logTraceMsg4, "BEGIN" );
logTrace( g_conf.m_logTraceMsg4Out, "BEGIN" );
// reserve necessary space
for (; m_destHostId < m_destHosts.size(); ++m_destHostId) {
@ -389,13 +389,13 @@ bool Msg4::processMetaList() {
// check size/flush buffer
if (!checkBufferSize(m_destHostId, destHost.first)) {
logTrace(g_conf.m_logTraceMsg4, "Unable to flush buffer");
logTrace(g_conf.m_logTraceMsg4Out, "Unable to flush buffer");
return false;
}
// prepare buffer
if (!prepareBuffer(m_destHostId, neededSpaceForBuf)) {
logTrace(g_conf.m_logTraceMsg4, "Unable to prepare buffer");
logTrace(g_conf.m_logTraceMsg4Out, "Unable to prepare buffer");
return false;
}
@ -408,7 +408,7 @@ bool Msg4::processMetaList() {
}
}
logTrace( g_conf.m_logTraceMsg4, "END - OK, true" );
logTrace( g_conf.m_logTraceMsg4Out, "END - OK, true" );
return true;
}
@ -789,10 +789,10 @@ bool saveAddsInProgress(const char *prefix) {
// . returns false on an unrecoverable error, true otherwise
// . sets g_errno on error
bool loadAddsInProgress(const char *prefix) {
logTrace( g_conf.m_logTraceMsg4, "BEGIN" );
logTrace( g_conf.m_logTraceMsg4Out, "BEGIN" );
if ( g_conf.m_readOnlyMode ) {
logTrace( g_conf.m_logTraceMsg4, "END - Read-only mode. Returning true" );
logTrace( g_conf.m_logTraceMsg4Out, "END - Read-only mode. Returning true" );
return true;
}
@ -807,18 +807,18 @@ bool loadAddsInProgress(const char *prefix) {
if ( ! prefix ) prefix = "";
sprintf ( filename, "%s%saddsinprogress.dat", g_hostdb.m_dir , prefix );
logTrace( g_conf.m_logTraceMsg4, "filename [%s]", filename);
logTrace( g_conf.m_logTraceMsg4Out, "filename [%s]", filename);
int32_t fd = open ( filename, O_RDONLY );
if ( fd < 0 ) {
if(errno==ENOENT) {
logTrace( g_conf.m_logTraceMsg4, "END - not found, returning true" );
logTrace( g_conf.m_logTraceMsg4Out, "END - not found, returning true" );
return true;
}
log(LOG_ERROR, "%s:%s: Failed to open %s for reading: %s",__FILE__,__func__,filename,strerror(errno));
g_errno = errno;
logTrace( g_conf.m_logTraceMsg4, "END - returning false" );
logTrace( g_conf.m_logTraceMsg4Out, "END - returning false" );
return false;
}
@ -842,7 +842,7 @@ bool loadAddsInProgress(const char *prefix) {
if ( nb != 4 ) {
close ( fd );
logError("Read of message size returned %" PRId32 " bytes instead of 4", nb);
logTrace( g_conf.m_logTraceMsg4, "END - returning false" );
logTrace( g_conf.m_logTraceMsg4Out, "END - returning false" );
return false;
}
@ -867,7 +867,7 @@ bool loadAddsInProgress(const char *prefix) {
if ( nb != 4 ) {
close ( fd );
logError("Read of message size returned %" PRId32 " bytes instead of 4", nb);
logTrace( g_conf.m_logTraceMsg4, "END - returning false" );
logTrace( g_conf.m_logTraceMsg4Out, "END - returning false" );
return false;
}
p += 4;
@ -891,7 +891,7 @@ bool loadAddsInProgress(const char *prefix) {
{
close ( fd );
log(LOG_ERROR,"build: Could not alloc %" PRId32" bytes for reading %s",allocSize,filename);
logTrace( g_conf.m_logTraceMsg4, "END - returning false" );
logTrace( g_conf.m_logTraceMsg4Out, "END - returning false" );
return false;
}
@ -905,7 +905,7 @@ bool loadAddsInProgress(const char *prefix) {
*(int32_t *)buf = 4;
// return false
log(LOG_ERROR,"%s:%s: error reading addsinprogress.dat: %s", __FILE__, __func__, mstrerror(errno));
logTrace( g_conf.m_logTraceMsg4, "END - returning false" );
logTrace( g_conf.m_logTraceMsg4Out, "END - returning false" );
return false;
}
// skip over it
@ -936,7 +936,7 @@ bool loadAddsInProgress(const char *prefix) {
if ( nb != 4 ) {
close ( fd );
logError("Read of message size returned %" PRId32 " bytes instead of 4", nb);
logTrace( g_conf.m_logTraceMsg4, "END - returning false" );
logTrace( g_conf.m_logTraceMsg4Out, "END - returning false" );
return false;
}
@ -948,7 +948,7 @@ bool loadAddsInProgress(const char *prefix) {
close (fd);
log(LOG_ERROR, "%s:%s: bad msg4 hostid %" PRId32,__FILE__,__func__,hostId);
logTrace( g_conf.m_logTraceMsg4, "END - returning false" );
logTrace( g_conf.m_logTraceMsg4Out, "END - returning false" );
return false;
}
@ -957,7 +957,7 @@ bool loadAddsInProgress(const char *prefix) {
if ( nb != 4 ) {
close ( fd );
logError("Read of message size returned %" PRId32 " bytes instead of 4", nb);
logTrace( g_conf.m_logTraceMsg4, "END - returning false" );
logTrace( g_conf.m_logTraceMsg4Out, "END - returning false" );
return false;
}
p += 4;
@ -968,7 +968,7 @@ bool loadAddsInProgress(const char *prefix) {
close ( fd );
log(LOG_ERROR, "%s:%s: could not alloc msg4 buf",__FILE__,__func__);
logTrace( g_conf.m_logTraceMsg4, "END - returning false" );
logTrace( g_conf.m_logTraceMsg4Out, "END - returning false" );
return false;
}
@ -977,7 +977,7 @@ bool loadAddsInProgress(const char *prefix) {
if ( nb != numBytes ) {
close ( fd );
log(LOG_ERROR,"%s:%s: build: bad msg4 buf read", __FILE__, __func__ );
logTrace( g_conf.m_logTraceMsg4, "END - returning false" );
logTrace( g_conf.m_logTraceMsg4Out, "END - returning false" );
return false;
}
p += numBytes;
@ -989,7 +989,7 @@ bool loadAddsInProgress(const char *prefix) {
log(LOG_WARN, "%s:%s: could not resend reload buf: %s",
__FILE__,__func__,mstrerror(g_errno));
logTrace( g_conf.m_logTraceMsg4, "END - returning false" );
logTrace( g_conf.m_logTraceMsg4Out, "END - returning false" );
return false;
}
}
@ -998,6 +998,6 @@ bool loadAddsInProgress(const char *prefix) {
// all done
close ( fd );
logTrace( g_conf.m_logTraceMsg4, "END - OK, returning true" );
logTrace( g_conf.m_logTraceMsg4Out, "END - OK, returning true" );
return true;
}

20
Parms.cpp

@ -9013,9 +9013,23 @@ void Parms::init ( ) {
m->m_page = PAGE_LOG;
m++;
m->m_title = "log trace info for Msg4";
m->m_cgi = "ltrc_msgfour";
simple_m_set(Conf,m_logTraceMsg4);
m->m_title = "log trace info for Msg4In";
m->m_cgi = "ltrc_msgfour_in";
simple_m_set(Conf,m_logTraceMsg4In);
m->m_def = "0";
m->m_page = PAGE_LOG;
m++;
m->m_title = "log trace info for Msg4Out";
m->m_cgi = "ltrc_msgfour_out";
simple_m_set(Conf,m_logTraceMsg4Out);
m->m_def = "0";
m->m_page = PAGE_LOG;
m++;
m->m_title = "log trace info for Msg4Out data";
m->m_cgi = "ltrc_msgfourdat";
simple_m_set(Conf,m_logTraceMsg4OutData);
m->m_def = "0";
m->m_page = PAGE_LOG;
m++;