catdb is now generated successfully.

Catdb.cpp

@@ -29,7 +29,11 @@ bool Catdb::init (  ) {
 	// . what's max # of tree nodes?
 	// . assume avg tagdb rec size (siteUrl) is about 82 bytes we get:
 	// . NOTE: 32 bytes of the 82 are overhead
-	long treeMem = g_conf.m_catdbMaxTreeMem;
+	//long treeMem = g_conf.m_catdbMaxTreeMem;
+	// speed up gen catdb, use 15MB. later maybe once gen is complete
+	// we can free this tree or something...
+	// TODO!
+	long treeMem = 15000000;
 	//long treeMem = 100000000;
 	//long maxTreeNodes = g_conf.m_catdbMaxTreeMem / 82;
 	long maxTreeNodes = treeMem / 82;

Msg1.cpp

@@ -355,6 +355,12 @@ bool Msg1::sendSomeOfList ( ) {
 	if ( m_list->m_fixedDataSize != getDataSizeFromRdbId(m_rdbId) ) {
 		char *xx=NULL;*xx=0; }
 
+	// little debug thing for genCatdb from msg9b's huge list add
+	if ( m_list->m_listSize > 10000000 )
+		log("msg1: adding chunk @ %li of %li bytes",
+		    dataStart - m_list->m_list ,
+		    m_list->m_listSize );
+
 	// . now send this list to the host
 	// . this returns false if blocked, true otherwise
 	// . it also sets g_errno on error
@@ -447,7 +453,9 @@ bool Msg1::sendData ( unsigned long groupId, char *listData , long listSize ) {
 		if ( ! g_errno ) sendToSelf = false;
 		else {
 			log("rdb: msg1 had error: %s",mstrerror(g_errno));
-			return true;
+			// this is messing up generate catdb's huge rdblist add
+			// why did we put it in there??? from msg9b.cpp
+			//return true;
 		}
 		
  		QUICKPOLL(m_niceness);

Msg2a.cpp

@@ -328,6 +328,12 @@ bool Msg2a::makeCatdb( char  *coll,
 		// null terminate
 		m_urls[urlp] = '\0';
 		currUrl++;
+		// debug
+		//SafeBuf sb;
+		//sb.safeMemcpy(&m_urls[urlp-urlLen],urlLen);
+		//sb.nullTerm();
+		//log("gencat: url=%s",sb.getBufStart());
+
 	}
 	log(LOG_INFO, "db: Wrote %li urls to update (%li)\n",
 		      currUrl - m_numRemoveUrls, m_numUpdateIndexes);

Msg8b.cpp

@@ -45,7 +45,7 @@ bool Msg8b::getCatRec  ( Url     *url              ,
 	// clear g_errno
 	g_errno = 0;
 	// warning
-	if ( ! coll ) log(LOG_LOGIC,"net: NULL collection. msg8b.");
+	//if ( ! coll ) log(LOG_LOGIC,"net: NULL collection. msg8b.");
 	// store the calling parameters in this class for retrieval by callback
 	m_state          = state;
 	m_callback       = callback;

Msg9b.cpp

@@ -123,7 +123,10 @@ bool Msg9b::addCatRecs ( char *urls        ,
 		}
 		else if ( ! m_list.addRecord ( key, dataSize, data ) )
 			return true;
-		
+
+		// debug
+		//log("gencat: adding url=%s",sr.m_url);
+
 		//skip:
 		// now advance p to e
 		p = e;
@@ -133,7 +136,8 @@ bool Msg9b::addCatRecs ( char *urls        ,
 		
 		QUICKPOLL((niceness));
 	}
-	log ( LOG_INFO, "Msg9b: %li sites and %li links added", k , c );
+	log ( LOG_INFO, "Msg9b: %li sites and %li links added. "
+	      "listSize=%li", k , c , m_list.m_listSize );
 	// . now add the m_list to tagdb using msg1
 	// . use high priority (niceness of 0)
 	// . i raised niceness from 0 to 1 so multicast does not use the