more 64bit fixes

Dates.cpp

@@ -18238,7 +18238,7 @@ bool initDateTypes ( ) {
 	s_init98 = true;
 	// set the keysize to 8 (wid) maps to a dv ptr
 	// use 0 for niceness
-	if ( !s_dvt.set(8,4,300,s_dvbuf,10000,true,0,"dvts")) 
+	if ( !s_dvt.set(8,sizeof(DateVal *),300,s_dvbuf,10000,true,0,"dvts")) 
 		return false;
 	// mark this
 	char localBuf[1000];

Makefile

@@ -97,14 +97,16 @@ else
 #
 # add -m32 flag to this line if you need to make a 32-bit gb.
 #
-CPPFLAGS = -g -Wall -pipe -fno-stack-protector -Wno-write-strings -Wstrict-aliasing=0 -Wno-uninitialized -DPTHREADS -Wno-unused-but-set-variable
+CPPFLAGS = -g -Wall -pipe -fno-stack-protector -Wno-write-strings -Wstrict-aliasing=0 -Wno-uninitialized -DPTHREADS -Wno-unused-but-set-variable -static
 #LIBS= -L. ./libz.a ./libssl.a ./libcrypto.a ./libiconv.a ./libm.a ./libstdc++.a -lpthread
 # use this for compiling on CYGWIN: (only for 32bit cygwin right now and
 # you have to install the packages that have these libs.
 #LIBS= -lz -lm -lpthread -lssl -lcrypto -liconv
 
-# apt-get install libssl-dev
-LIBS= -lz -lm -lpthread -lssl -lcrypto -liconv
+# apt-get install libssl-dev (to provide libssl and libcrypto)
+# to build static libiconv.a do a './configure --enable-static' then 'make'
+# in the iconv directory
+LIBS=  -lm -lpthread -lssl -lcrypto ./libiconv64.a ./libz64.a
 
 endif
 

Mem.cpp

@@ -1988,7 +1988,7 @@ void *getElecMem ( int32_t size ) {
 	//   THEN possibly another MEMPAGESIZE-1 bytes to hit the next page
 	//   boundary for protecting the "freed" mem below, but can get
 	//   by with (MEMPAGESIZE-(size%MEMPAGESIZE)) more
-	int32_t need = size + 8 + MEMPAGESIZE + MEMPAGESIZE ;
+	int32_t need = size + sizeof(char *)*2 + MEMPAGESIZE + MEMPAGESIZE ;
 	// want to end on a page boundary too!
 	need += (MEMPAGESIZE-(size%MEMPAGESIZE));
 	// get that
@@ -2013,8 +2013,8 @@ void *getElecMem ( int32_t size ) {
 	// save this
 	char *returnMem = p;
 	// store the ptrs
-	*(char **)(returnMem- 4) = realMem;
-	*(char **)(returnMem- 8) = realMemEnd;
+	*(char **)(returnMem- sizeof(char *)) = realMem;
+	*(char **)(returnMem- sizeof(char *)*2) = realMemEnd;
 	// protect that after we wrote our ptr
 	if ( mprotect ( protMem , MEMPAGESIZE , PROT_NONE) < 0 )
 		log("mem: mprotect failed: %s",mstrerror(errno));
@@ -2035,7 +2035,8 @@ void *getElecMem ( int32_t size ) {
 	return returnMem;
 #else
 	// how much to alloc
-	int32_t need = size + 8 + MEMPAGESIZE + MEMPAGESIZE + MEMPAGESIZE;
+	int32_t need = size + MEMPAGESIZE + MEMPAGESIZE + MEMPAGESIZE;
+	need += sizeof(char *)*2;
 	// get that
 	char *realMem = (char *)sysmalloc ( need );	
 	if ( ! realMem ) return NULL;
@@ -2061,10 +2062,10 @@ void *getElecMem ( int32_t size ) {
 	// after we "free" it below
 	if ( p < realMem ) { char *xx=NULL;*xx=0; }
 	// store mem ptrs before protecting
-	*(char **)(returnMem- 4) = realMem;
-	*(char **)(returnMem- 8) = realMemEnd;
+	*(char **)(returnMem- sizeof(char *)  ) = realMem;
+	*(char **)(returnMem- sizeof(char *)*2) = realMemEnd;
 	// sanity
-	if ( returnMem - 8 < realMem ) { char *xx=NULL;*xx=0; }
+	if ( returnMem - sizeof(char *)*2 < realMem ) { char *xx=NULL;*xx=0; }
 	// protect that after we wrote our ptr
 	if ( mprotect ( protMem , MEMPAGESIZE , PROT_NONE) < 0 )
 		log("mem: mprotect failed: %s",mstrerror(errno));
@@ -2123,9 +2124,9 @@ void freeElecMem ( void *fakeMem ) {
 
 	// now original memptr is right before "p" and we can
 	// read it now that we are unprotected
-	char *realMem    = *(char **)(cp-4);
+	char *realMem    = *(char **)(cp-sizeof(char *));
 	// set real mem end (no!?)
-	char *realMemEnd = *(char **)(cp-8);
+	char *realMemEnd = *(char **)(cp-sizeof(char *)*2);
 
 	// set it all to 0x99
 	memset ( realMem , 0x99 , realMemEnd - realMem );

Msg13.cpp

@@ -9,6 +9,7 @@
 #include "Test.h"
 #include "Speller.h"
 #include "SpiderProxy.h" // OP_GETPROXY OP_RETPROXY
+#include "zlib.h"
 
 char *g_fakeReply = 
 	"HTTP/1.0 200 (OK)\r\n"
@@ -127,7 +128,7 @@ bool Msg13::registerHandler ( ) {
 
 	// . set up the request table (aka wait in line table)
 	// . allowDups = "true"
-	if ( ! s_rt.set ( 8 , 4 , 0 , NULL , 0 , true,0,"wait13tbl") )
+	if ( ! s_rt.set ( 8 ,sizeof(UdpSlot *),0,NULL,0,true,0,"wait13tbl") )
 		return false;
 
 	if ( ! g_loop.registerSleepCallback(10,NULL,scanHammerQueue) )
@@ -1780,7 +1781,9 @@ void gotHttpReply2 ( void *state ,
 					 (unsigned char*)reply, 
 					 replySize);
 		if(zipErr != Z_OK) {
-			log("spider: had error zipping Msg13 reply.");
+			log("spider: had error zipping Msg13 reply. %s "
+			    "(%"INT32")",
+			    zError(zipErr),(int32_t)zipErr);
 			mfree (compressedBuf, need, "Msg13ZipError");
 			g_errno = ECORRUPTDATA;
 			g_udpServer.sendErrorReply(slot,g_errno);

Msg40.cpp

@@ -622,9 +622,9 @@ bool Msg40::federatedLoop ( ) {
 	mr.size_query                  = m_si->m_q.m_origLen+1;
 	//mr.ptr_whiteList               = m_si->m_whiteListBuf.getBufStart();
 	//mr.size_whiteList              = m_si->m_whiteListBuf.length()+1;
-	int32_t slen = 0; if ( m_si->m_sites ) slen = gbstrlen(m_si->m_sites);
+	int32_t slen = 0; if ( m_si->m_sites ) slen=gbstrlen(m_si->m_sites)+1;
 	mr.ptr_whiteList               = m_si->m_sites;
-	mr.size_whiteList              = slen + 1;
+	mr.size_whiteList              = slen;
 	mr.m_timeout                   = -1; // auto-determine based on #terms
 	// make sure query term counts match in msg39
 	mr.m_maxQueryTerms             = m_si->m_maxQueryTerms; 

Process.cpp

@@ -165,7 +165,7 @@ char *g_files[] = {
 	"libnetpbm.so.10",
 	"libpng12.so.0",
 	"libtiff.so.4",
-	"libz.so.1",
+	//"libz.so.1",
 	"LICENSE",
 	"pngtopnm",
 	"pnmscale",

RdbCache.cpp

@@ -353,7 +353,8 @@ void RdbCache::addLong ( collnum_t collnum ,
 	k.n1 = key;
 	// sanity check
 	if ( m_cks > (int32_t)sizeof(key_t) ) { char *xx = NULL; *xx = 0; }
-	addRecord ( collnum , (char *)&k , NULL , 0 , (char *)&value , 4 ,
+	addRecord ( collnum , (char *)&k , NULL , 0 , (char *)&value , 
+		    sizeof(char *), // 4 , now 8 for 64 bit archs
 		    0 , // timestamp=now
 		    retRecPtr );
 	// clear error in case addRecord set it
@@ -472,6 +473,7 @@ bool RdbCache::getRecord ( collnum_t collnum   ,
 		if ( incCounts ) m_numMisses++;
 		return false;
 	}
+	// skip timestamp
 	p += 4;
 	// store data size if our recs are var length or we cache lists of
 	// fixed length recs, and those lists need a dataSize
@@ -1216,7 +1218,7 @@ void RdbCache::removeKey ( collnum_t collnum , char *key , char *rec ) {
 		m_ptrs[n] = NULL;
 		// undo stats
 		m_numPtrsUsed--;
-		m_memOccupied -= 4;
+		m_memOccupied -= sizeof(char *);//4;
 		// re-hash it back to possibly fill the "gap"
 		addKey ( *(collnum_t *)ptr , kptr , ptr );
 		if ( ++n >= m_numPtrsMax ) n = 0;
@@ -1244,7 +1246,7 @@ void RdbCache::addKey ( collnum_t collnum , char *key , char *ptr ) {
 	// if already there don't inc the count
 	if ( ! m_ptrs[n] ) {
 		m_numPtrsUsed++;
-		m_memOccupied += 4;
+		m_memOccupied += sizeof(char *);
 		// debug msg 
 		//key_t *k = (key_t *)key;
 		//log("cache: %s added key.n1=%"UINT32" key.n0=%"UINT64" to slot #%"INT32" "

Sections.cpp

@@ -261,7 +261,7 @@ bool Sections::set ( Words     *w                       ,
 	m_sectionPtrBuf.setLabel("psectbuf");
 
 	// separate buf now for section ptr for each word
-	if ( ! m_sectionPtrBuf.reserve ( nw *4 ) ) return true;
+	if ( ! m_sectionPtrBuf.reserve ( nw *sizeof(Section *)) ) return true;
 	m_sectionPtrs = (Section **)m_sectionPtrBuf.getBufStart();
 	m_sectionPtrsEnd = (Section **)m_sectionPtrBuf.getBufEnd();
 

Words.cpp

@@ -521,7 +521,11 @@ bool Words::addWords(char *s,int32_t nodeLen,bool computeWordIds, int32_t nicene
 // common to Unicode and ISO-8859-1
 bool Words::allocateWordBuffers(int32_t count, bool tagIds) {
 	// alloc if we need to (added 4 more for m_nodes[])
-	int32_t wordSize = 20;
+	int32_t wordSize = 0;
+	wordSize += sizeof(char *);
+	wordSize += sizeof(int32_t);
+	wordSize += sizeof(int64_t);
+	wordSize += sizeof(int32_t);
 	if ( tagIds ) wordSize += sizeof(nodeid_t);
 	m_bufSize = wordSize * count;
 	if(m_bufSize < 0) return log("build: word count overflow %"INT32" "
@@ -558,6 +562,8 @@ bool Words::allocateWordBuffers(int32_t count, bool tagIds) {
 		p += sizeof(nodeid_t) * count;
 	}
 
+	if ( p > m_buf + m_bufSize ) { char *xx=NULL;*xx=0; }
+
 	return true;
 }
 

XmlDoc.cpp

@@ -31585,7 +31585,10 @@ int gbcompress ( unsigned char *dest      ,
 	// cygwin uses the system libz.a which is not hacked for our quickpoll
 #ifndef CYGWIN	
 	// tell deflat() to call quickpoll
-	//setQuickPoll ( (char *)&g_loop.m_needsToQuickPoll , deflateQuickPoll );
+
+	// MDW: 11/14/2014 don't do this for the 64bit zlib for now just to
+	// save some time. do it later when it proves to be an issue.
+	//setQuickPoll ( (char *)&g_loop.m_needsToQuickPoll, deflateQuickPoll);
 #endif
 
 	err = deflate(&stream, Z_FINISH);

qa.cpp

@@ -756,6 +756,7 @@ bool qainject2 ( ) {
 		SafeBuf ubuf;
 		ubuf.load("./injectme3");
 		sb.urlEncode(ubuf.getBufStart());
+		sb.nullTerm();
 		if ( ! getUrl ( "/admin/inject",
 				// check reply, seems to have only a single 
 				// docid in it
@@ -2007,6 +2008,7 @@ bool qajson ( ) {
 			      "&urls="//www.walmart.com+ibm.com"
 			      );
 		sb.urlEncode ( s_ubuf4 );
+		sb.nullTerm();
 		// . now a list of websites we want to spider
 		// . the space is already encoded as +
 		if ( ! getUrl ( "/admin/addurl",0,sb.getBufStart()) )
@@ -2227,6 +2229,7 @@ bool qaxml ( ) {
 			      "&urls="//www.walmart.com+ibm.com"
 			      );
 		sb.urlEncode ( s_ubuf5 );
+		sb.nullTerm();
 		// . now a list of websites we want to spider
 		// . the space is already encoded as +
 		if ( ! getUrl ( "/admin/addurl",0,sb.getBufStart()) )