Mirrors/privacore-open-source-search-engine
1
0
Fork 1
mirror of https://github.com/privacore/open-source-search-engine.git synced 2025-07-14 02:36:06 -04:00
Code Issues Packages Projects Releases Wiki Activity

Make docIdSplits range configurable, defaulting to [5..15]

Browse Source
This commit is contained in:
Ivan Skytte Jørgensen
2016-03-18 22:15:11 +01:00
parent 6108f06e65
commit fcb3a5ff3f
3 changed files with 36 additions and 7 deletions

2
Conf.h

@ -151,6 +151,8 @@ class Conf {
int32_t m_maxDocsWanted; //maximum number of results in one go. Puts a limit on SearchInput::m_docsWanted
int32_t m_maxFirstResultNum; //maximum document offset / result-page. Puts a limit on SearchInput::m_firstResultNum
int32_t min_docid_splits; //minimum number of DocId splits using Msg40
int32_t max_docid_splits; //maximum number of DocId splits using Msg40
int64_t m_msg40_msg39_timeout; //timeout for entire get-docid-list phase, in milliseconds.
int64_t m_msg3a_msg39_network_overhead; //additional latency/overhead of sending reqeust+response over network.

14
Msg40.cpp

@ -335,21 +335,21 @@ bool Msg40::federatedLoop ( ) {
if ( numDocIdSplits <= 0 ) numDocIdSplits = 1;
// and mult based on index size
numDocIdSplits *= mult;
// prevent going OOM for type:article AND html
if ( numDocIdSplits < 5 ) numDocIdSplits = 5;
log(LOG_DEBUG,"query: Msg40::federatedLoop: numDocIdSplits=%d", numDocIdSplits);
//}
if ( cr ) mr.m_maxQueryTerms = cr->m_maxQueryTerms;
else mr.m_maxQueryTerms = 100;
// prevent going OOM for type:article AND html
// special oom hack fix
if ( cr && cr->m_isCustomCrawl && numDocIdSplits < 4 )
numDocIdSplits = 4;
// limit to 10
if ( numDocIdSplits > 15 )
numDocIdSplits = 15;
// limit numDocIdSplits to the range specified in the configuration
if ( numDocIdSplits < g_conf.min_docid_splits )
numDocIdSplits = g_conf.min_docid_splits;
if ( numDocIdSplits > g_conf.max_docid_splits )
numDocIdSplits = g_conf.max_docid_splits;
log(LOG_DEBUG,"query: Msg40::federatedLoop: numDocIdSplits=%d", numDocIdSplits);
// store it in the reuquest now
mr.m_numDocIdSplits = numDocIdSplits;

27
Parms.cpp

@ -8703,6 +8703,33 @@ void Parms::init ( ) {
m++;
m->m_title = "Min DocId splits";
m->m_desc = "Minimum number of Docid splits when deciding how many 'chunks' to use for limiting memory use while intersecting lists";
m->m_cgi = "min_docid_splits";
m->m_off = offsetof(Conf,min_docid_splits);
m->m_xml = "min_docid_splits";
m->m_type = TYPE_LONG;
m->m_page = PAGE_SEARCH;
m->m_obj = OBJ_CONF;
m->m_def = "5";
m->m_min = 1;
m->m_flags = 0;
m++;
m->m_title = "Max DocId splits";
m->m_desc = "Maximum number of Docid splits when deciding how many 'chunks' to use for limiting memory use while intersecting lists";
m->m_cgi = "max_docid_splits";
m->m_off = offsetof(Conf,max_docid_splits);
m->m_xml = "max_docid_splits";
m->m_type = TYPE_LONG;
m->m_page = PAGE_SEARCH;
m->m_obj = OBJ_CONF;
m->m_def = "15";
m->m_min = 1;
m->m_flags = 0;
m++;
m->m_title = "msg40->39 timeout";
m->m_desc = "Timeout for Msg40/Msg3a to collect candidate docids with Msg39. In milliseconds";
m->m_cgi = "msgfourty_msgthirtynine_timeout";