fixed a couple really nasty mem leak bugs from new facet code

Mem.cpp

@@ -1276,6 +1276,8 @@ int Mem::printMem ( ) {
 	// print out table sorted by sizes
 	for ( int32_t i = 0 ; i < np ; i++ ) {
 		int32_t a = p[i];
+		//if ( strcmp((char *)&s_labels[a*16],"umsg20") == 0 )
+		//	log("hey");
 		log(LOG_INFO,"mem: %05"INT32") %"INT32" 0x%"PTRFMT" %s", 
 		    i,s_sizes[a] , (PTRTYPE)s_mptrs[a] , &s_labels[a*16] );
 	}

Msg20.cpp

@@ -15,6 +15,8 @@ void Msg20::constructor () {
 	m_inProgress = false;
 	m_launched = false;
 	m_ii = -1;
+	m_owningParent = (void *)0x012345;
+	m_constructedId = 5;
 	reset();
 	m_mcast.constructor();
 }
@@ -24,6 +26,10 @@ void Msg20::destructor  () { reset(); m_mcast.destructor(); }
 #include "Process.h"
 
 void Msg20::freeReply() {
+
+	//log("msg20: freeing reply for msg20=0x%"PTRFMT" reply=0x%"PTRFMT"",
+	//    (PTRTYPE)this,(PTRTYPE)m_r);
+
 	if ( ! m_r ) return;
 	// sometimes the msg20 reply carries an merged bffer from
 	// msg40 that is a constructed ptr_eventSummaryLines from a
@@ -36,16 +42,25 @@ void Msg20::freeReply() {
 }
 
 void Msg20::reset() { 
+
+	//log("msg20: resetting msg20=0x%"PTRFMT" reply=0x%"PTRFMT"",
+	//    (PTRTYPE)this,(PTRTYPE)m_r);
+
+	if ( ! m_owningParent ) { char *xx=NULL;*xx=0; }
+
 	// not allowed to reset one in progress
 	if ( m_inProgress ) { 
 		// do not core on abrupt exits!
-		if (g_process.m_mode == EXIT_MODE ) return;
+		if (g_process.m_mode == EXIT_MODE ) {
+			log("msg20: msg20 not being freed because exiting.");
+			return;
+		}
 		// otherwise core
 		char *xx=NULL;*xx=0; 
 	}
 	m_launched = false;
 	if ( m_request && m_request   != m_requestBuf )
-		mfree ( m_request , m_requestSize  , "Msg20rb" );
+		mfree ( m_request , m_requestSize  , "Msg20rb1" );
 	freeReply();
 	//if ( m_r ) m_r->destructor();
 	//if ( m_r && m_ownReply ) //&& (char *)m_r != m_replyBuf )
@@ -110,6 +125,8 @@ bool Msg20::getSummary ( Msg20Request *req ) {
 	// reset ourselves in case recycled
 	reset();
 
+	if ( ! m_owningParent ) { char *xx=NULL;*xx=0; }
+
 	// consider it "launched"
 	m_launched = true;
 
@@ -281,7 +298,7 @@ void Msg20::gotReply ( UdpSlot *slot ) {
 
 	// free our serialized request buffer to save mem
 	if ( m_request && m_request   != m_requestBuf ) {
-		mfree ( m_request , m_requestSize  , "Msg20rb" );
+		mfree ( m_request , m_requestSize  , "Msg20rb2" );
 		m_request = NULL;
 	}
 
@@ -313,7 +330,8 @@ void Msg20::gotReply ( UdpSlot *slot ) {
 	//if( rp != m_replyBuf )
 	relabel( rp , m_replyMaxSize, "Msg20-mcastGBR" );
 
-	// sanity check
+	// sanity check. make sure multicast is not going to free the
+	// slot's m_readBuf... we need to own it.
 	if ( freeit ) {
 		log(LOG_LOGIC,"query: msg20: gotReply: Bad engineer.");
 		char *xx = NULL; *xx = 0;
@@ -327,12 +345,22 @@ void Msg20::gotReply ( UdpSlot *slot ) {
 
 	// cast it
 	m_r = (Msg20Reply *)rp;
+
+	m_r->m_parentOwner = (void *)this;
+	m_r->m_constructorId = 2;
+
 	// reset this since constructor never called
 	m_r->m_tmp = 0;
 	// we own it now
 	m_ownReply = true;
 	// deserialize it, sets g_errno on error??? not yet TODO!
 	m_r->deserialize();
+
+	// log("msg20: got msg20=0x%"PTRFMT" msg20reply=0x%"PTRFMT" slot=0x%"PTRFMT" s_tmp=%i"
+	//     ,(PTRTYPE)this
+	//     ,(PTRTYPE)m_r
+	//     ,(PTRTYPE)slot
+	//     ,s_tmp);
 }
 
 // . this is called
@@ -356,6 +384,9 @@ void handleRequest20 ( UdpSlot *slot , int32_t netnice ) {
 		return;
 	}
 
+	//log("query: got umsg20 %i 0x%"PTRFMT"",slot->m_readBufMaxSize,
+	//    (PTRTYPE)slot->m_readBuf);
+
 	// parse the request
 	Msg20Request *req = (Msg20Request *)slot->m_readBuf;
 
@@ -473,6 +504,9 @@ Msg20Reply::Msg20Reply ( ) {
 	//ptr_eventSummaryLines = NULL;
 	m_tmp = 0;
 
+	m_parentOwner = NULL;
+	m_constructorId = 0;
+
 	// seems to be an issue... caused a core with bogus size_dbuf
 	int32_t *sizePtr = &size_tbuf;
 	int32_t *sizeEnd = &size_note;
@@ -570,6 +604,8 @@ int32_t Msg20::deserialize ( char *buf , int32_t bufSize ) {
 	if ( bufSize < (int32_t)sizeof(Msg20Reply) ) {
 		g_errno = ECORRUPTDATA; return -1; }
 	m_r = (Msg20Reply *)buf;
+	m_r->m_parentOwner = (void *)this;
+	m_r->m_constructorId = 1;
 	// do not free "buf"/"m_r"
 	m_ownReply = false;
 	return m_r->deserialize ( );

Msg20.h

@@ -404,6 +404,8 @@ public:
 	int32_t       m_midDomHash          ; // set for m_getLinkText
 	int32_t       m_adIdHash            ; // set for m_getLinkText
 	int32_t       m_timeLinkSpam        ; // set for m_getLinkText
+	void         *m_parentOwner;
+	char          m_constructorId;
 	char       m_inlinkWeight        ; // set for m_getLinkText
 	char       m_isLinkSpam          ; // set for m_getLinkText
 	char       m_isAnomaly           ; // set for m_getLinkText
@@ -820,6 +822,9 @@ class Msg20 {
 	void *m_state2;
 	void *m_state3;
 
+	void *m_owningParent;
+	char m_constructedId;
+
 	// PostQueryRerank storage area for printing out in PageResults.cpp
 	float m_pqr_old_score        ;
 	float m_pqr_factor_diversity ;

Msg3a.cpp

@@ -963,8 +963,11 @@ bool Msg3a::mergeLists ( ) {
 		// facethashlists from each shard into
 		//int64_t tid  = m_q->m_qterms[i].m_termId;
 		// we hold all the facet values
+		// m_q is a ptr to State0::m_si.m_q from PageResults.cpp
+		// and Msg40.cpp ultimately.
 		HashTableX *ht = &qt->m_facetHashTable;
-		// we have to manually cal this
+		// we have to manually call this because Query::constructor()
+		// might have been called explicitly
 		ht->constructor();
 		// 4 byte key, 4 byte score for counting facet values
 		if ( ! ht->set(4,sizeof(FacetEntry),

Msg40.cpp

@@ -706,6 +706,10 @@ bool Msg40::federatedLoop ( ) {
 		// launch a search request
 		m_num3aRequests++;
 		// this returns false if it would block and will call callback
+		// m_si is actually contained in State0 in PageResults.cpp
+		// and Msg40::m_si points to that. so State0's destructor
+		// should call SearchInput's destructor which calls
+		// Query's destructor to destroy &m_si->m_q here when done.
 		if(!mp->getDocIds(&mp->m_rrr,&m_si->m_q,this,gotDocIdsWrapper))
 			continue;
 		if ( g_errno && ! m_errno ) 
@@ -1060,6 +1064,9 @@ bool Msg40::reallocMsg20Buf ( ) {
 			p += sizeof(Msg20);
 			// init it
 			tmp[i]->constructor();
+			// set this now
+			tmp[i]->m_owningParent = (void *)this;
+			tmp[i]->m_constructedId = 1;
 			// count it
 			pcount++;
 			// skip it if it is a new docid, we do not have a Msg20
@@ -1209,6 +1216,9 @@ bool Msg40::reallocMsg20Buf ( ) {
 		m_msg20[i] = (Msg20 *)p;
 		// call its constructor
 		m_msg20[i]->constructor();
+		// set this now
+		m_msg20[i]->m_owningParent = (void *)this;
+		m_msg20[i]->m_constructedId = 2;
 		// point to the next Msg20
 		p += sizeof(Msg20);
 		// remember num to free in reset() function
@@ -5951,6 +5961,7 @@ bool Msg40::printJsonItemInCSV ( State0 *st , int32_t ix ) {
 	return true;
 }
 
+// this is a safebuf of msg20s for doing facet string lookups
 Msg20 *Msg40::getUnusedMsg20 ( ) {
 
 	// make a safebuf of 50 of them if we haven't yet
@@ -5959,8 +5970,14 @@ Msg20 *Msg40::getUnusedMsg20 ( ) {
 			return NULL;
 		}
 		Msg20 *ma = (Msg20 *)m_unusedBuf.getBufStart();
-		for ( int32_t i = 0 ; i < (int32_t)MAX2 ; i++ )
+		for ( int32_t i = 0 ; i < (int32_t)MAX2 ; i++ ) {
 			ma[i].constructor();
+			ma[i].m_owningParent = (void *)this;
+			ma[i].m_constructedId = 3;
+			// if we don't update length then Msg40::resetBuf2() 
+			// will fail to call Msg20::destructor on them
+			m_unusedBuf.m_length += sizeof(Msg20);
+		}
 	}
 		
 

PageReindex.cpp

@@ -305,6 +305,9 @@ bool Msg1c::reindexQuery ( char *query ,
 	// langunknown?
 	m_qq.set2 ( query , langId , true ); // /*bool flag*/ );
 
+	// a debug thing
+	m_qq.m_containingParent = (void *)this;
+
 	//CollectionRec *cr = g_collectiondb.getRec ( collnum );
 
 	// sanity fix

Query.cpp

@@ -30,6 +30,8 @@ void Query::constructor ( ) {
 	m_qwordsAllocSize      = 0;
 	//m_expressionsAllocSize = 0;
 	m_qwords               = NULL;
+	m_numTerms = 0;
+	m_containingParent = NULL;
 	//m_expressions          = NULL;
 	reset ( );
 }
@@ -43,6 +45,16 @@ Query::~Query ( ) {
 }
 
 void Query::reset ( ) {
+
+	// if Query::constructor() was called explicitly then we have to
+	// call destructors explicitly as well...
+	for ( long i = 0 ; i < m_numTerms ; i++ ) {
+	 	// get it
+		QueryTerm *qt = &m_qterms[i];
+		HashTableX *ht = &qt->m_facetHashTable;
+		ht->reset();
+	}
+
 	m_docIdRestriction = 0LL;
 	m_groupThatHasDocId = NULL;
 	m_bufLen      = 0;

Query.h

@@ -996,6 +996,8 @@ class Query {
 	bool m_hasSynonyms;
 
 	SafeBuf m_debugBuf;
+
+	void *m_containingParent;
 };
 /*
 class QueryScores {

SearchInput.cpp

@@ -469,6 +469,8 @@ bool SearchInput::set ( TcpSocket *sock , HttpRequest *r ) { //, Query *q ) {
 		return false;
 	}
 
+	m_q.m_containingParent = (void *)this;
+
 	if ( m_q.m_truncated && m_q.m_isBoolean ) {
 		g_errno = EQUERYTOOBIG;
 		g_msg = " (error: query is too long)";

XmlDoc.cpp

@@ -42470,6 +42470,10 @@ SafeBuf *XmlDoc::getMatchingQueriesScoredForFullQuery ( ) {
 	// . set the query class for msg3a
 	// . queryExpansion = true
 	m_query3a->set2 ( qstr , langId , true );
+
+	// a debug thing
+	m_query3a->m_containingParent = (void *)this;
+
 	// secret variable latchon
 	m_msg3a->m_hack = this;