Forks/open-source-search-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

clean out proxy load table more often to keep things fast.

Browse Source
This commit is contained in:
Matt Wells 2015-09-12 13:33:42 -07:00
parent fb332a23c4
commit 583974093e
1 changed files with 8 additions and 4 deletions

12
SpiderProxy.cpp

@ -5,7 +5,10 @@
#include "HttpServer.h"
#include "SpiderProxy.h"
#define LOADPOINT_EXPIRE_MS (10*60*1000)
//#define LOADPOINT_EXPIRE_MS (10*60*1000)
// make it 15 seconds not 10 minutes otherwise it gets too full with dup
// keys and really clogs things up
#define LOADPOINT_EXPIRE_MS (15*1000)
//
// BASIC DETAILS
@ -927,7 +930,7 @@ void handleRequest54 ( UdpSlot *udpSlot , int32_t niceness ) {
// and the loadbucket id
//*(int32_t *)p = bb.m_id; p += 4;
int32_t sanityCount = 0;//s_loadTable.getNumSlots();
//int32_t sanityCount = 0;//s_loadTable.getNumSlots();
top:
// now remove old entries from the load table. entries that
@ -941,11 +944,11 @@ void handleRequest54 ( UdpSlot *udpSlot , int32_t niceness ) {
if ( pp->m_downloadEndTimeMS == 0LL ) continue;
// delta t
int64_t took = nowms - pp->m_downloadEndTimeMS;
// < 10 mins?
// < 10 mins? now it's < 15 seconds to prevent clogging.
if ( took < LOADPOINT_EXPIRE_MS ) continue;
// 100 at a time
if ( sanityCount++ > 100 ) break;
//if ( sanityCount++ > 100 ) break;
// ok, its too old, nuke it to save memory
s_loadTable.removeSlot(i);
@ -1046,6 +1049,7 @@ bool initSpiderProxyStuff() {
128,
NULL,
0,
// this slows us down
true, // allow dups?
MAX_NICENESS,
"lbtab",