Add logic to limit number of msg7s to 100 per hosts, then we drop the

requests.
This commit is contained in:
Zak Betz
2015-09-03 22:17:16 -06:00
parent b4a0668b21
commit e7f1c75855
4 changed files with 21 additions and 10 deletions

@ -590,7 +590,7 @@ void sendUdpReply7 ( void *state ) {
uint32_t statColor = 0xccffcc;
if(xd->m_indexCode) {
statColor = 0x4e99e9;
statColor = 0xaaddaa;//0x4e99e9;
}
g_stats.addStat_r ( xd->m_rawUtf8ContentSize,
xd->m_injectStartTime,

@ -65,14 +65,12 @@ static Label s_labels[] = {
// . 300MB/s is max read rate regardless to stop graph shrinkage
// . use 1KB as the min resolution per pixel
// . stored in Bps so use 1/1000 as scalar to get into KBps
{ GRAPH_QUANTITY,200,"disk_read",1,"%.0f MBps",1.0/(1000.0*1000.0),0x000000,
"disk read"},
{ GRAPH_QUANTITY,200,"disk_read",1,"%.0f MBps",1.0/(1000.0*1000.0),0x000000,"disk read"},
// . 300MB/s is max write rate regardless to stop graph shrinkage
// . use 1KB as the min resolution per pixel
// . stored in Bps so use 1/1000 as scalar to get into KBps
{GRAPH_QUANTITY,200,"disk_write",1,"%.0f Mbps",1.0/(1000.0*1000.0), 0xff0000,
"disk write"},
{GRAPH_QUANTITY,200,"disk_write",1,"%.0f Mbps",1.0/(1000.0*1000.0), 0xff0000, "disk write"},
// . 20 is the max dps regardless to stop graph shrinkage
// . use .03 qps as the min resolution per pixel
@ -247,6 +245,8 @@ void flushStatsWrapper ( int fd , void *state ) {
void Statsdb::addDocsIndexed ( ) {
if ( ! isClockInSync() ) return;
if ( g_hostdb.hasDeadHost() ) return;
// only once per five seconds
int32_t now = getTimeLocal();
@ -273,13 +273,17 @@ void Statsdb::addDocsIndexed ( ) {
int32_t docsIndexedInInterval = total - s_lastTotal;
float docsPerSecond = docsIndexedInInterval / (float)interval;
s_lastTotal = total;
log("build: total docs indexed: %f. docs per second %f %i %i", (float)total, docsPerSecond, docsIndexedInInterval, interval);
// add it if changed though
int64_t nowms = gettimeofdayInMillisecondsGlobal();
addStat ( MAX_NICENESS,"docs_indexed", nowms, nowms, (float)total );
addStat ( MAX_NICENESS,"docs_per_second", nowms, nowms, docsPerSecond );
// Prevent a datapoint which adds all of the docs indexed to date.
if( s_lastTotal != 0 ) {
addStat ( MAX_NICENESS,"docs_per_second", nowms, nowms, docsPerSecond );
}
s_lastTotal = total;
}
// . m_key bitmap in statsdb:
@ -991,7 +995,7 @@ char *Statsdb::plotGraph ( char *pstart ,
float y2 = *(float *)p; p += 4;
// scale it right away
y2 *= yscalar;
y2 = (y2 - ymin) * yscalar;
// adjust
if ( y2 > ymax ) y2 = ymax;
@ -1011,7 +1015,7 @@ char *Statsdb::plotGraph ( char *pstart ,
y2 = ((float)DY2 * (y2 - ymin)) / (ymax-ymin);
}
//log("label is %s point is %f", label->m_label, y2);
//log("label is %s point is %f %f- %f %f", label->m_label, (float)y2, (float)ymin, (float)ymax, (float)yscalar);
// set lasts for next iteration of this loop
lastx = x2;
lasty = y2;

@ -286,6 +286,7 @@ bool UdpServer::init ( uint16_t port, UdpProtocol *proto, int32_t niceness,
// no requests waiting yet
m_requestsInWaiting = 0;
// special count
m_msg07sInWaiting = 0;
m_msg10sInWaiting = 0;
m_msgc1sInWaiting = 0;
//m_msgDsInWaiting = 0;
@ -1005,7 +1006,7 @@ UdpSlot *UdpServer::getBestSlotToSend ( int64_t now ) {
UdpSlot *maxi = NULL;
int32_t score;
//UdpSlot *slot;
// . we send dgrams with the lowest "score" first
// . we send dgrams with the lowest "score" first
// . the "score" is just number of ACKs you're waiting for
// . that way transmissions that are the most caught up to their ACKs
// are considered faster so we send to them first
@ -1482,6 +1483,9 @@ int32_t UdpServer::readSock_ass ( UdpSlot **slotPtr , int64_t now ) {
// rate, these are pretty lightweight. msg 0x10 reply gen times
// are VERY low. MDW
bool getSlot = true;
if ( msgType == 0x07 && m_msg07sInWaiting >= 100 )
getSlot = false;
if ( msgType == 0x10 && m_msg10sInWaiting >= 50 )
getSlot = false;
// crawl update info from Spider.cpp
@ -1669,6 +1673,7 @@ int32_t UdpServer::readSock_ass ( UdpSlot **slotPtr , int64_t now ) {
// if we connected to a request slot, count it
m_requestsInWaiting++;
// special count
if ( msgType == 0x07 ) m_msg07sInWaiting++;
if ( msgType == 0x10 ) m_msg10sInWaiting++;
if ( msgType == 0xc1 ) m_msgc1sInWaiting++;
//if ( msgType == 0xd ) m_msgDsInWaiting++;
@ -3120,6 +3125,7 @@ void UdpServer::destroySlot ( UdpSlot *slot ) {
// one less request in waiting
m_requestsInWaiting--;
// special count
if ( slot->m_msgType == 0x07 ) m_msg07sInWaiting--;
if ( slot->m_msgType == 0x10 ) m_msg10sInWaiting--;
if ( slot->m_msgType == 0xc1 ) m_msgc1sInWaiting--;
//if ( slot->m_msgType == 0xd ) m_msgDsInWaiting--;

@ -390,6 +390,7 @@ class UdpServer {
int32_t m_requestsInWaiting;
// like m_requestsInWaiting but requests which spawn other requests
int32_t m_msg07sInWaiting;
int32_t m_msg10sInWaiting;
int32_t m_msgc1sInWaiting;
//int32_t m_msgDsInWaiting;