Merge branch 'diffbot-testing' of github.com:gigablast/open-source-search-engine into diffbot-testing

Loop.cpp

@@ -52,6 +52,8 @@ int32_t g_numVTAlarms = 0;
 int32_t g_numQuickPolls = 0;
 int32_t g_missedQuickPolls = 0;
 int32_t g_numSigChlds = 0;
+int32_t g_numSigPipes = 0;
+int32_t g_numSigIOs = 0;
 int32_t g_numSigQueues = 0;
 int32_t g_numSigOthers = 0;
 
@@ -639,6 +641,16 @@ void sigHandlerQueue_r ( int x , siginfo_t *info , void *v ) {
 		return;
 	}
 
+	if ( info->si_signo == SIGPIPE ) {
+		g_numSigPipes++;
+		return;
+	}
+
+	if ( info->si_signo == SIGIO ) {
+		g_numSigIOs++;
+		return;
+	}
+
 	if ( info->si_code == SI_QUEUE ) {
 		g_numSigQueues++;
 		//log("admin: got sigqueue");
@@ -647,6 +659,8 @@ void sigHandlerQueue_r ( int x , siginfo_t *info , void *v ) {
 		return;
 	}
 
+
+
 	// wtf is this?
 	g_numSigOthers++;
 

Loop.h

@@ -96,6 +96,8 @@ extern int32_t g_numQuickPolls;
 
 extern int32_t g_numSigChlds;
 extern int32_t g_numSigQueues;
+extern int32_t g_numSigPipes;
+extern int32_t g_numSigIOs;
 extern int32_t g_numSigOthers;
 
 

PageStats.cpp

@@ -832,6 +832,8 @@ bool sendPageStats ( TcpSocket *s , HttpRequest *r ) {
 
 			      "<tr class=poo><td><b>SIGCHLDS</b></td><td>%"INT32"</td></tr>\n"
 			      "<tr class=poo><td><b>SIGQUEUES</b></td><td>%"INT32"</td></tr>\n"
+			      "<tr class=poo><td><b>SIGPIPES</b></td><td>%"INT32"</td></tr>\n"
+			      "<tr class=poo><td><b>SIGIOS</b></td><td>%"INT32"</td></tr>\n"
 			      "<tr class=poo><td><b>SIGOTHERS</b></td><td>%"INT32"</td></tr>\n"
 
 			      //"<tr class=poo><td><b>read signals</b></td><td>%"INT64"</td></tr>\n"
@@ -859,6 +861,8 @@ bool sendPageStats ( TcpSocket *s , HttpRequest *r ) {
 
 			      g_numSigChlds,
 			      g_numSigQueues,
+			      g_numSigPipes,
+			      g_numSigIOs,
 			      g_numSigOthers,
 
 			      //g_stats.m_readSignals,

Rdb.cpp

@@ -2756,6 +2756,24 @@ int64_t Rdb::getNumTotalRecs ( bool useCache ) {
 	return total;
 }
 
+
+int64_t Rdb::getCollNumTotalRecs ( collnum_t collnum ) {
+
+	if ( collnum < 0 ) return 0;
+
+	CollectionRec *cr = g_collectiondb.m_recs[collnum];
+	if ( ! cr ) return 0;
+	// if swapped out, this will be NULL, so skip it
+	RdbBase *base = cr->getBasePtr(m_rdbId);
+	if ( ! base ) {
+		log("rdb: getcollnumtotalrecs: base swapped out");
+		return 0;
+	}
+	return base->getNumTotalRecs();
+}
+
+
+
 // . how much mem is alloced for all of our maps?
 // . we have one map per file
 int64_t Rdb::getMapMemAlloced () {

Rdb.h

@@ -234,6 +234,8 @@ class Rdb {
 	// positive minus negative
 	int64_t getNumTotalRecs ( bool useCache = false ) ;
 
+	int64_t getCollNumTotalRecs ( collnum_t collnum );
+
 	int64_t getNumRecsOnDisk ( );
 
 	int64_t getNumGlobalRecs ( );

XmlDoc.cpp

@@ -12758,9 +12758,10 @@ int32_t *XmlDoc::getSiteNumInlinks ( ) {
 		else if ( ns < 100 ) maxAge = 60;
 		// if index size is tiny then maybe we are just starting to 
 		// build something massive, so reduce the cached max age
-		if ( g_titledb.m_rdb.getNumGlobalRecs() < 100000000 ) // 100M
+		int64_t nt = g_titledb.m_rdb.getCollNumTotalRecs(m_collnum);
+		if ( nt < 100000000 ) //100M
 			maxAge = 3;
-		if ( g_titledb.m_rdb.getNumGlobalRecs() < 10000000 ) // 10M
+		if ( nt < 10000000 ) //10M
 			maxAge = 1;
 		// for every 100 urls you already got, add a day!
 		sni = atol(tag->getTagData());

fctypes.cpp

@@ -1910,14 +1910,28 @@ int64_t gettimeofdayInMillisecondsSynced() {
 		log("xml: clock not in sync with host #0 yet!!!!!!");
 		//char *xx = NULL; *xx = 0; }
 	}
-	//if ( ! g_clockInSync ) 
-	//	log("gb: Getting global time but clock not in sync.");
-	// this isn't async signal safe...
-	struct timeval tv;
-	gettimeofday ( &tv , NULL );
-	int64_t now=(int64_t)(tv.tv_usec/1000)+((int64_t)tv.tv_sec)*1000;
+
+	int64_t now;
+
+	// the real tiem sigalrm interrupt in Loop.cpp sets this to
+	// true once per millisecond
+	if ( ! g_clockNeedsUpdate ) {
+		now = g_now;
+	}
+	else {
+		//if ( ! g_clockInSync ) 
+		//	log("gb: Getting global time but clock not in sync.");
+		// this isn't async signal safe...
+		struct timeval tv;
+		gettimeofday ( &tv , NULL );
+		now = (int64_t)(tv.tv_usec/1000)+((int64_t)tv.tv_sec)*1000;
+	}
+
 	// update g_nowLocal
 	if ( now > g_now ) g_now = now;
+
+	g_clockNeedsUpdate = false;
+
 	// adjust from Msg0x11 time adjustments
 	now += s_adjustment;
 	// update g_now if it is more accurate

html/faq.html

@@ -218,7 +218,7 @@ For RedHat do a <b>yum install gcc-c++ glibc-static libstdc++-static openssl-sta
 </ul>
 -->
 
-<b>1.0</b> For <u>Ubuntu 12.02 or 14.04</u>: do <b>sudo apt-get update ; apt-get install make g++ libssl-dev</b>
+<b>1.0</b> For <u>Ubuntu 12.02 or 14.04</u>: do <b>sudo apt-get update ; apt-get install make g++ libssl-dev binutils</b>
 
 <br><br>