mirror of
https://github.com/privacore/open-source-search-engine.git
synced 2025-07-16 02:46:08 -04:00
Add logic to limit number of msg7s to 100 per hosts, then we drop the
requests.
This commit is contained in:
@ -590,7 +590,7 @@ void sendUdpReply7 ( void *state ) {
|
||||
|
||||
uint32_t statColor = 0xccffcc;
|
||||
if(xd->m_indexCode) {
|
||||
statColor = 0x4e99e9;
|
||||
statColor = 0xaaddaa;//0x4e99e9;
|
||||
}
|
||||
g_stats.addStat_r ( xd->m_rawUtf8ContentSize,
|
||||
xd->m_injectStartTime,
|
||||
|
20
Statsdb.cpp
20
Statsdb.cpp
@ -65,14 +65,12 @@ static Label s_labels[] = {
|
||||
// . 300MB/s is max read rate regardless to stop graph shrinkage
|
||||
// . use 1KB as the min resolution per pixel
|
||||
// . stored in Bps so use 1/1000 as scalar to get into KBps
|
||||
{ GRAPH_QUANTITY,200,"disk_read",1,"%.0f MBps",1.0/(1000.0*1000.0),0x000000,
|
||||
"disk read"},
|
||||
{ GRAPH_QUANTITY,200,"disk_read",1,"%.0f MBps",1.0/(1000.0*1000.0),0x000000,"disk read"},
|
||||
|
||||
// . 300MB/s is max write rate regardless to stop graph shrinkage
|
||||
// . use 1KB as the min resolution per pixel
|
||||
// . stored in Bps so use 1/1000 as scalar to get into KBps
|
||||
{GRAPH_QUANTITY,200,"disk_write",1,"%.0f Mbps",1.0/(1000.0*1000.0), 0xff0000,
|
||||
"disk write"},
|
||||
{GRAPH_QUANTITY,200,"disk_write",1,"%.0f Mbps",1.0/(1000.0*1000.0), 0xff0000, "disk write"},
|
||||
|
||||
// . 20 is the max dps regardless to stop graph shrinkage
|
||||
// . use .03 qps as the min resolution per pixel
|
||||
@ -247,6 +245,8 @@ void flushStatsWrapper ( int fd , void *state ) {
|
||||
void Statsdb::addDocsIndexed ( ) {
|
||||
|
||||
if ( ! isClockInSync() ) return;
|
||||
if ( g_hostdb.hasDeadHost() ) return;
|
||||
|
||||
|
||||
// only once per five seconds
|
||||
int32_t now = getTimeLocal();
|
||||
@ -273,13 +273,17 @@ void Statsdb::addDocsIndexed ( ) {
|
||||
int32_t docsIndexedInInterval = total - s_lastTotal;
|
||||
float docsPerSecond = docsIndexedInInterval / (float)interval;
|
||||
|
||||
s_lastTotal = total;
|
||||
log("build: total docs indexed: %f. docs per second %f %i %i", (float)total, docsPerSecond, docsIndexedInInterval, interval);
|
||||
|
||||
// add it if changed though
|
||||
int64_t nowms = gettimeofdayInMillisecondsGlobal();
|
||||
addStat ( MAX_NICENESS,"docs_indexed", nowms, nowms, (float)total );
|
||||
addStat ( MAX_NICENESS,"docs_per_second", nowms, nowms, docsPerSecond );
|
||||
// Prevent a datapoint which adds all of the docs indexed to date.
|
||||
if( s_lastTotal != 0 ) {
|
||||
addStat ( MAX_NICENESS,"docs_per_second", nowms, nowms, docsPerSecond );
|
||||
}
|
||||
|
||||
s_lastTotal = total;
|
||||
}
|
||||
|
||||
// . m_key bitmap in statsdb:
|
||||
@ -991,7 +995,7 @@ char *Statsdb::plotGraph ( char *pstart ,
|
||||
float y2 = *(float *)p; p += 4;
|
||||
|
||||
// scale it right away
|
||||
y2 *= yscalar;
|
||||
y2 = (y2 - ymin) * yscalar;
|
||||
|
||||
// adjust
|
||||
if ( y2 > ymax ) y2 = ymax;
|
||||
@ -1011,7 +1015,7 @@ char *Statsdb::plotGraph ( char *pstart ,
|
||||
y2 = ((float)DY2 * (y2 - ymin)) / (ymax-ymin);
|
||||
}
|
||||
|
||||
//log("label is %s point is %f", label->m_label, y2);
|
||||
//log("label is %s point is %f %f- %f %f", label->m_label, (float)y2, (float)ymin, (float)ymax, (float)yscalar);
|
||||
// set lasts for next iteration of this loop
|
||||
lastx = x2;
|
||||
lasty = y2;
|
||||
|
@ -286,6 +286,7 @@ bool UdpServer::init ( uint16_t port, UdpProtocol *proto, int32_t niceness,
|
||||
// no requests waiting yet
|
||||
m_requestsInWaiting = 0;
|
||||
// special count
|
||||
m_msg07sInWaiting = 0;
|
||||
m_msg10sInWaiting = 0;
|
||||
m_msgc1sInWaiting = 0;
|
||||
//m_msgDsInWaiting = 0;
|
||||
@ -1005,7 +1006,7 @@ UdpSlot *UdpServer::getBestSlotToSend ( int64_t now ) {
|
||||
UdpSlot *maxi = NULL;
|
||||
int32_t score;
|
||||
//UdpSlot *slot;
|
||||
// . we send dgrams with the lowest "score" first
|
||||
// . we send dgrams with the lowest "score" first
|
||||
// . the "score" is just number of ACKs you're waiting for
|
||||
// . that way transmissions that are the most caught up to their ACKs
|
||||
// are considered faster so we send to them first
|
||||
@ -1482,6 +1483,9 @@ int32_t UdpServer::readSock_ass ( UdpSlot **slotPtr , int64_t now ) {
|
||||
// rate, these are pretty lightweight. msg 0x10 reply gen times
|
||||
// are VERY low. MDW
|
||||
bool getSlot = true;
|
||||
if ( msgType == 0x07 && m_msg07sInWaiting >= 100 )
|
||||
getSlot = false;
|
||||
|
||||
if ( msgType == 0x10 && m_msg10sInWaiting >= 50 )
|
||||
getSlot = false;
|
||||
// crawl update info from Spider.cpp
|
||||
@ -1669,6 +1673,7 @@ int32_t UdpServer::readSock_ass ( UdpSlot **slotPtr , int64_t now ) {
|
||||
// if we connected to a request slot, count it
|
||||
m_requestsInWaiting++;
|
||||
// special count
|
||||
if ( msgType == 0x07 ) m_msg07sInWaiting++;
|
||||
if ( msgType == 0x10 ) m_msg10sInWaiting++;
|
||||
if ( msgType == 0xc1 ) m_msgc1sInWaiting++;
|
||||
//if ( msgType == 0xd ) m_msgDsInWaiting++;
|
||||
@ -3120,6 +3125,7 @@ void UdpServer::destroySlot ( UdpSlot *slot ) {
|
||||
// one less request in waiting
|
||||
m_requestsInWaiting--;
|
||||
// special count
|
||||
if ( slot->m_msgType == 0x07 ) m_msg07sInWaiting--;
|
||||
if ( slot->m_msgType == 0x10 ) m_msg10sInWaiting--;
|
||||
if ( slot->m_msgType == 0xc1 ) m_msgc1sInWaiting--;
|
||||
//if ( slot->m_msgType == 0xd ) m_msgDsInWaiting--;
|
||||
|
@ -390,6 +390,7 @@ class UdpServer {
|
||||
int32_t m_requestsInWaiting;
|
||||
|
||||
// like m_requestsInWaiting but requests which spawn other requests
|
||||
int32_t m_msg07sInWaiting;
|
||||
int32_t m_msg10sInWaiting;
|
||||
int32_t m_msgc1sInWaiting;
|
||||
//int32_t m_msgDsInWaiting;
|
||||
|
Reference in New Issue
Block a user