Moved Test.* to misc and removed all references to it and g_test from the main source code

Doledb.cpp

@@ -12,7 +12,6 @@
 #include "CountryCode.h"
 #include "DailyMerge.h"
 #include "Process.h"
-#include "Test.h" // g_test
 #include "Threads.h"
 #include "XmlDoc.h"
 #include "HttpServer.h"

Makefile

@@ -61,7 +61,7 @@ OBJS =  UdpSlot.o Rebalance.o \
 	CountryCode.o DailyMerge.o Tagdb.o \
 	Images.o Wiki.o Wiktionary.o \
 	Timezone.o Sections.o SiteGetter.o qa.o \
-	Test.o Synonyms.o \
+	Synonyms.o \
 	PageCrawlBot.o Json.o PageBasic.o \
 	Punycode.o Version.o \
 	HighFrequencyTermShortcuts.o \

Msg12.cpp

@@ -14,7 +14,6 @@
 #include "CountryCode.h"
 #include "DailyMerge.h"
 #include "Process.h"
-#include "Test.h" // g_test
 #include "Threads.h"
 #include "XmlDoc.h"
 #include "HttpServer.h"

Msg13.cpp

@@ -6,7 +6,6 @@
 #include "Stats.h"
 #include "HashTableX.h"
 #include "XmlDoc.h"
-#include "Test.h"
 #include "SpiderProxy.h" // OP_GETPROXY OP_RETPROXY
 #include "zlib.h"
 

Msge1.cpp

@@ -1,7 +1,6 @@
 #include "gb-include.h"
 
 #include "Msge1.h"
-#include "Test.h"
 
 // utility functions
 bool getTestIp ( char *url , int32_t *retIp , bool *found , int32_t niceness ,
@@ -544,7 +543,6 @@ bool getTestIp ( char *url , int32_t *retIp , bool *found , int32_t niceness ,
 			char *xx=NULL;*xx=0; }
 		// null it out now, we freed it
 		s_testBuf = NULL;
-		//char *testDir = g_test.getTestDir();
 		// filename
 		char fn[100]; 
 		sprintf(fn,"%s/%s/ips.txt",g_hostdb.m_dir,testDir);
@@ -718,7 +716,6 @@ void makeQADir();
 // . save it back to disk
 // . we should call this from Test.cpp when the run is completed!!
 bool saveTestBuf ( char *testDir ) {
-	//char *testDir = g_test.getTestDir();
 	// ensure ./qa/ subdir exsts. in qa.cpp
 	makeQADir();
 	// filename

Parms.cpp

@@ -23,7 +23,6 @@
 #include "PingServer.h"
 #include "Proxy.h"
 #include "hash.h"
-#include "Test.h"
 #include "Rebalance.h"
 #include "SpiderProxy.h" // buildProxyTable()
 #include "PageInject.h" // InjectionRequest
@@ -547,50 +546,6 @@ bool CommandResetColl ( char *rec , WaitEntry *we ) {
 #endif
 
 
-bool CommandParserTestInit ( char *rec ) {
-	// enable testing for all other hosts
-	g_conf.m_testParserEnabled = 1;
-	// reset all files
-	g_test.removeFiles();
-	// turn spiders on globally
-	g_conf.m_spideringEnabled = 1;
-	//g_conf.m_webSpideringEnabled = 1;
-	// turn on for test coll too
-	CollectionRec *cr = g_collectiondb.getRec("qatest123");
-	// turn on spiders
-	if ( cr ) cr->m_spideringEnabled = 1;
-	// tell spider loop to update active list
-	g_spiderLoop.m_activeListValid = false;
-	// if we are not host 0, turn on spiders for testing
-	if ( g_hostdb.m_myHost->m_hostId != 0 ) return true;
-	// start the test loop to inject urls for parsing/spidering
-	g_test.initTestRun();
-	// done 
-	return true;
-}
-
-bool CommandSpiderTestInit ( char *rec ) {
-	// enable testing for all other hosts
-	g_conf.m_testSpiderEnabled = 1;
-	// reset all files
-	g_test.removeFiles();
-	// turn spiders on globally
-	g_conf.m_spideringEnabled = 1;
-	//g_conf.m_webSpideringEnabled = 1;
-	// turn on for test coll too
-	CollectionRec *cr = g_collectiondb.getRec("qatest123");
-	// turn on spiders
-	if ( cr ) cr->m_spideringEnabled = 1;
-	// tell spider loop to update active list
-	g_spiderLoop.m_activeListValid = false;
-	// if we are not host 0, turn on spiders for testing
-	if ( g_hostdb.m_myHost->m_hostId != 0 ) return true;
-	// start the test loop to inject urls for parsing/spidering
-	g_test.initTestRun();
-	// done 
-	return true;
-}
-
 bool CommandSpiderTestCont ( char *rec ) {
 	// enable testing for all other hosts
 	g_conf.m_testSpiderEnabled = 1;
@@ -5594,59 +5549,6 @@ void Parms::init ( ) {
 	m->m_obj   = OBJ_CONF;
         m++;
 
-	m->m_title = "init QA tests";
-	m->m_desc  = "If initiated gb performs some integrity tests "
-		"to ensure injecting, spidering and searching works "
-		"properly. Uses ./test/ subdirectory. Injects "
-		"urls in ./test/inject.txt. Spiders urls "
-		"in ./test/spider.txt. "
-		"Each of those two files is essentially a simple format of "
-		"a url followed by the http reply received from the server "
-		"for that url. "
-		// TODO: generate these files
-		;
-	m->m_cgi   = "qasptei";
-	m->m_type  = TYPE_CMD;
-	m->m_func  = CommandSpiderTestInit;
-	m->m_def   = "1";
-	m->m_cast  = 1;
-	m->m_group = false;
-	m->m_flags = PF_HIDDEN | PF_NOSAVE;
-	m->m_page  = PAGE_MASTER;
-	m->m_obj   = OBJ_CONF;
-	m++;
-
-
-	m->m_title = "init parser test run";
-	m->m_desc  = "If enabled gb injects the urls in the "
-		"./test-parser/urls.txt "
-		"file and outputs ./test-parser/qa.html";
-	m->m_cgi   = "qaptei";
-	m->m_type  = TYPE_CMD;
-	m->m_func  = CommandParserTestInit;
-	m->m_def   = "1";
-	m->m_cast  = 1;
-	m->m_flags = PF_HIDDEN | PF_NOSAVE;
-	m->m_page  = PAGE_MASTER;
-	m->m_obj   = OBJ_CONF;
-	m++;
-
-
-	m->m_title = "init spider test run";
-	m->m_desc  = "If enabled gb injects the urls in "
-		"./test-spider/spider.txt "
-		"and spiders links.";
-	m->m_cgi   = "qasptei";
-	m->m_type  = TYPE_CMD;
-	m->m_func  = CommandSpiderTestInit;
-	m->m_def   = "1";
-	m->m_cast  = 1;
-	m->m_group = false;
-	m->m_flags = PF_HIDDEN | PF_NOSAVE;
-	m->m_page  = PAGE_MASTER;
-	m->m_obj   = OBJ_CONF;
-	m++;
-
 	m->m_title = "continue spider test run";
 	m->m_desc  = "Resumes the test.";
 	m->m_cgi   = "qaspter";

PingServer.cpp

@@ -22,7 +22,6 @@ int32_t klogctl( int, char *,int ) { return 0; }
 #include "Spider.h"
 #include "SpiderColl.h"
 #include "SpiderLoop.h"
-#include "Test.h"
 #include "Rebalance.h"
 #include "Version.h"
 
@@ -896,22 +895,6 @@ void handleRequest11 ( UdpSlot *slot , int32_t niceness ) {
 	}
 	// always send back an empty reply
 	g_udpServer.sendReply_ass( reply, replySize, NULL, 0, slot, NULL, NULL, 500, 1000, true );
-
-	// . now in PingServer.cpp for hostid 0 it checks
-	//   the urlsindexed from each host if g_conf.m_testParserEnabled
-	//   is true to see if we should call g_test.stopIt()
-	// . add up each hosts urls indexed
-	if ( ! g_conf.m_testParserEnabled     ) return;
-	if ( g_hostdb.m_myHost->m_hostId != 0 ) return;
-	int32_t total = 0;
-	for ( int32_t i = 0 ; i < g_hostdb.getNumHosts() ; i++ ) {
-		Host *h = &g_hostdb.m_hosts[i];
-		total += h->m_urlsIndexed;
-	}
-	// all done?
-	if ( total >= g_test.m_urlsAdded ) g_test.stopIt();
-
-	return;
 }
 
 // . sets m_minRepairMode

Spider.cpp

@@ -28,7 +28,6 @@
 #include "CountryCode.h"
 #include "DailyMerge.h"
 #include "Process.h"
-#include "Test.h" // g_test
 #include "Threads.h"
 #include "XmlDoc.h"
 #include "HttpServer.h"

SpiderColl.cpp

@@ -13,7 +13,6 @@
 #include "CountryCode.h"
 #include "DailyMerge.h"
 #include "Process.h"
-#include "Test.h" // g_test
 #include "Threads.h"
 #include "XmlDoc.h"
 #include "HttpServer.h"

SpiderLoop.cpp

@@ -13,7 +13,6 @@
 #include "CountryCode.h"
 #include "DailyMerge.h"
 #include "Process.h"
-#include "Test.h" // g_test
 #include "Threads.h"
 #include "XmlDoc.h"
 #include "HttpServer.h"
@@ -1751,9 +1750,7 @@ bool SpiderLoop::spiderUrl9 ( SpiderRequest *sreq ,
 
 	// sanity check
 	// core dump? just re-run gb and restart the parser test...
-	if ( //g_test.m_isRunning && 
-	     //! g_test.m_spiderLinks &&
-	     g_conf.m_testParserEnabled &&
+	if ( g_conf.m_testParserEnabled &&
 	     ! sreq->m_isInjecting ) { 
 		char *xx=NULL;*xx=0; }
 
@@ -2272,88 +2269,6 @@ if( 	g_conf.m_logTraceSpider ) log(LOG_TRACE,"%s:%s:%d: BEGIN", __FILE__, __func
 	bool respider = false;
 	if ( xd->m_oldDocValid && xd->m_oldDoc ) respider = true;
 
-	// . dump it out to a file in the "qatest123" subdir
-	// . but only the first time we spider it...
-	/*
-	if ( ! strcmp(xd->m_coll,"qatest123") && ! respider &&
-	     // no longer need this when qa testing spider, not parser
-	     g_conf.m_testParserEnabled ) {
-		// save the buffers
-		//saveTestBuf();
-		// get it
-		//SafeBuf *pbuf = xd->m_pbuf;
-		SafeBuf sb;
-		// get it
-		xd->printDoc ( &sb );
-		// get the first url
-		Url *u = xd->getFirstUrl();
-		// . get its hash
-		// . should be same hash we use to store doc.%"UINT64".html in
-		//   XmlDoc.cpp/Msg13.cpp stuff (getTestDoc())
-		int64_t h = hash64 ( u->getUrl() , u->getUrlLen() );
-		char *testDir = g_test.getTestDir();
-		// make filename to dump out to
-		char fn[1024]; 
-		sprintf(fn,"%s/%s/parse.%"UINT64".%"UINT32".html",
-			g_hostdb.m_dir,testDir,h,g_test.m_runId);
-		// . dump it out to a file
-		// . WATCH OUT. g_errno is set on internal errors, like OOM
-		//   or whatever, so don't save in those cases...???????
-		sb.dumpToFile ( fn );
-		// just dump the <div class=shotdisplay> tags into this file
-		sprintf(fn,"%s/%s/parse-shortdisplay.%"UINT64".%"UINT32".html",
-			g_hostdb.m_dir,testDir,h,g_test.m_runId);
-		// output to a special file
-		SafeBuf tmp;
-		// insert this
-		tmp.safeStrcpy("<meta http-equiv=\"Content-Type\" "
-			       "content=\"text/html; "
-			       "charset=utf-8\">\n");
-		// header stuff
-		tmp.safePrintf("<html><body>\n");
-		// put the onclick script in there
-		tmp.safeStrcpy ( xd->getCheckboxScript() );
-		// concatenate just these sections in "sb" to "tmp"
-		tmp.cat2 ( sb , 
-			   "<div class=shortdisplay>" ,
-			   "</div class=shortdisplay>" );
-		// header stuff
-		tmp.safePrintf("\n</body></html>\n");
-		// then dump
-		tmp.dumpToFile ( fn );
-		// if it had critical errors from XmlDoc::validateOutput()
-		// then create that file!
-		//if ( xd->m_validateMisses > 0 || xd->m_validateFlagged ) {
-		// make the critical file filename
-		char cf[1024];
-		sprintf (cf,"%s/%s/critical.%"UINT64".%"UINT32".txt",
-			 g_hostdb.m_dir,testDir,h,g_test.m_runId);
-		// save to that
-		ttt.dumpToFile ( cf );
-		//char cmd[256];
-		//sprintf(cmd,"touch %s/test/critical.%"UINT64".%"UINT32".txt",
-		//	g_hostdb.m_dir,h,g_test.m_runId);
-		//system(cmd);
-
-		// note it
-		//log("crazyin: %s",u->m_url );
-		// note it
-		//g_test.m_urlsAdded--;
-		g_test.m_urlsIndexed++;
-
-		// now in PingServer.cpp for hostid 0 it checks
-		// the urlsindexed from each host if g_conf.m_testParserEnabled
-		// is true to see if we should call g_test.stopIt()
-
-		// if that is zero we are done
-		//if ( g_test.m_urlsAdded == 0 && ! g_test.m_isAdding &&
-		//     // only stop if not spidering links
-		//     //! g_test.m_spiderLinks )
-		//     g_conf.m_testParserEnabled )
-		//	// wrap things up
-		//	g_test.stopIt();
-	}
-	*/
 
 	// note it
 	// this should not happen any more since indexDoc() will take

XmlDoc.cpp

@@ -22,7 +22,6 @@
 #include "Wiki.h"
 #include "Speller.h"
 #include "SiteGetter.h"
-#include "Test.h"
 #include "Synonyms.h"
 #include "PageInject.h"
 #include "HttpServer.h"

main.cpp

@@ -68,7 +68,6 @@
 #include "Process.h"
 #include "sort.h"
 #include "RdbBuckets.h"
-#include "Test.h"
 #include "SpiderProxy.h"
 #include "HashTable.h"
 
@@ -2447,8 +2446,6 @@ int main2 ( int argc , char *argv[] ) {
 	// the spider cache used by SpiderLoop
 	if ( ! g_spiderCache.init() ) {
 		log("db: SpiderCache init failed." ); return 1; }
-	if ( ! g_test.init() ) {
-		log("db: test init failed" ); return 1; }
 
 	// ensure clusterdb tree is big enough for quicker generation
 	//if ( strcmp ( cmd, "genclusterdb" ) == 0 ) {