Merge branch 'testing' of github.com:gigablast/open-source-search-engine into testing

HashTableX.cpp

@@ -17,6 +17,7 @@ void HashTableX::constructor() {
 	m_txtBuf = NULL;
 	m_useKeyMagic = false;
 	m_ks = 0;
+	m_allowGrowth = true;
 }
 
 void HashTableX::destructor() {
@@ -93,6 +94,7 @@ void HashTableX::reset ( ) {
 	m_numSlotsUsed = 0;
 	m_addIffNotUnique = false;
 	m_maskKeyOffset = 0;
+	m_allowGrowth = true;
 	//m_useKeyMagic = false;
 	// we should free it in reset()
 	if ( m_doFree && m_txtBuf ) {
@@ -189,6 +191,13 @@ bool HashTableX::addKey ( void *key , void *val , long *slot ) {
 	}
 	// never got initialized? call HashTableX::init()
 	if ( m_ks <= 0 ){ char *xx=NULL; *xx=0; }
+
+	if ( ! m_allowGrowth && m_numSlotsUsed + 1 > m_numSlots ) {
+		log("hashtable: hit max ceiling of hashtable of %li slots. "
+		    "and can not grow because in thread.",m_numSlotsUsed);
+		return false;
+	}
+
 	// check to see if we should grow the table. now we grow
 	// when 25% full to make operations faster so getLongestString()
 	// doesn't return such big numbers!

HashTableX.h

@@ -53,6 +53,10 @@ class HashTableX {
 	// how many keys are dups
 	long getNumDups();
 
+	// if in a thread to dont allow it to grow
+	void setNonGrow() { m_allowGrowth = false; }
+	bool m_allowGrowth;
+
 	bool addFloat ( long *wid , float score ) {
 		long slot = getSlot ( wid );
                 if ( slot<0 ) return addKey( wid ,&score,&slot);

PageInject.cpp

@@ -817,6 +817,7 @@ void ImportState::reset() {
 
 // . call this when gb startsup
 // . scan collections to see if any imports were active
+// . returns false and sets g_errno on failure
 bool resumeImports ( ) {
 
 	for ( long i = 0 ; i < g_collectiondb.m_numRecs ; i++ ) {
@@ -830,9 +831,9 @@ bool resumeImports ( ) {
 		try { is = new (ImportState); }
 		catch ( ... ) { 
 			g_errno = ENOMEM;
-			log("PageInject: new(%i): %s", 
-			    (int)sizeof(ImportState),mstrerror(g_errno));
-			return NULL;
+			log("PageInject: new(%li): %s", 
+			    (long)sizeof(ImportState),mstrerror(g_errno));
+			return false;
 		}
 		mnew ( is, sizeof(ImportState) , "isstate");
 		// assign to cr as well

Posdb.cpp

@@ -838,6 +838,8 @@ bool PosdbTable::allocTopTree ( ) {
 						  slots,NULL,0,false,
 						  0,"qfht" ) )
 			return false;
+		// make it nongrowable because we'll be in a thread
+		qt->m_facetHashTable.setNonGrow();
 	}
 	return true;
 }

UdpServer.cpp

@@ -82,7 +82,7 @@ UdpServer g_udpServer2;
 
 
 static void readPollWrapper_ass ( int fd , void *state ) ;
-static void sendPollWrapper_ass ( int fd , void *state ) ;
+//static void sendPollWrapper_ass ( int fd , void *state ) ;
 static void timePollWrapper     ( int fd , void *state ) ;
 static void defaultCallbackWrapper ( void *state , UdpSlot *slot );
 

main.cpp

@@ -4605,23 +4605,34 @@ int install ( install_flag_konst_t installFlag , long hostId , char *dir ,
 
 	long maxOut = 6;
 
-	// this is a big rcp so only do one at a time...
+	// this is a big scp so only do two at a time...
 	if  ( installFlag == ifk_install ) maxOut = 1;
 
 	// same with this. takes too long on gk144, jams up
-	if  ( installFlag == ifk_installgb ) maxOut = 1;
+	if  ( installFlag == ifk_installgb ) maxOut = 2;
+
+	if  ( installFlag == ifk_installgbrcp ) maxOut = 4;
+
+	long maxOutPerIp = 6;
 
 	// go through each host
 	for ( long i = 0 ; i < g_hostdb.getNumHosts() ; i++ ) {
 		Host *h2 = g_hostdb.getHost(i);
 
-		// if host ip is like the 10th occurence then do
-		// not do ampersand
-		iptab.addScore((long *)&h2->m_ip);
-		long score = iptab.getScore32(&h2->m_ip);
-		char *amp = " &";
-		if ( (score % maxOut) == 0 ) amp = "";
+		char *amp = " ";
+
+		// if i is NOT multiple of maxOut then use '&'
+		// even if all all different machines (IPs) scp chokes and so
+		// does rcp a little. so restrict to maxOut at a time.
+		if ( (i+1) % maxOut ) amp = "&";
 			
+
+		// if host ip is like the 10th occurence then do
+		// not do ampersand. this is for hosts on the same IP.
+		//long score = iptab.getScore32(&h2->m_ip);
+		//if ( (score % maxOutPerIp) ) amp = "&";
+		//iptab.addScore((long *)&h2->m_ip);
+
 		// limit install to this hostId if it is >= 0
 		//if ( hostId >= 0 && h2->m_hostId != hostId ) continue;
 		if ( hostId >= 0 && hostId2 == -1 ) {