Forks/privacore-open-source-search-engine
1
0
Fork 0
forked from Mirrors/privacore-open-source-search-engine
Code Pull Requests Activity

Remove some unused methods/class. Minor restructuring of test files.

Browse Source
This commit is contained in:
Ai Lin Chia 2015-12-09 11:25:18 +01:00
parent 8df3deff5f
commit 0b98f2c337
47 changed files with 1582 additions and 860777 deletions

4814
Address.cpp

File diff suppressed because it is too large Load Diff

94
Address.h

@ -3,17 +3,10 @@
#define _GB_ADDRESS_H_
// values for Place::m_bits
//#define PLF_PARTIAL 0x001 // hash in the next word and try again!
#define PLF_UNIQUE 0x001 // city name is unique in the usa
//#define PLF_AMBIGUOUS0x002 // might be multiple things. 2+ place bits set
//#define PLF_ALIAS 0x002 // use PlaceDescAlias class!!
#define PLF_INFILE 0x004 // is in allCountries/postalCodes.txt
//#define PLF_INHERITED 0x008
//#define PLF_FROMZIP 0x008 // inherited from zip adm1 or city
#define PLF_FROMTAG 0x010 // a place from a TagRec Tag
#define PLF_ABBR 0x020 // allCountries.txt, "NM" is for "New Mexico"
#define PLF_FROMTITLE 0x040 // from the title tag
//#define PLF_ALT 0x080 // allCountries.txt alias/alternative name
#define PLF_HAS_UPPER 0x080 // does it have an upper-case word in it?
@ -41,12 +34,9 @@
// . indicators. words or phrases that indicate a possible place name, suite,
// . these ARE NOT places unto themselves
// . IND_NAME = a common word in the places in allCountries.txt
#define IND_NAME 0x01
#define IND_SUITE 0x02
#define IND_STREET 0x04
#define IND_DIR 0x08
#define IND_BITS 0x0f
#define NO_LATITUDE 999.0
#define NO_LONGITUDE 999.0
@ -71,8 +61,6 @@ int32_t memcpy2 ( char *dst , char *src , int32_t bytes , bool filterCommas ,
// do not store more than this many bytes into dst
int32_t dstMaxBytes = -1 ) ;
bool getLatLon ( uint32_t cityId , double *lat , double *lon ) ;
bool hashPlaceName ( HashTableX *nt1,
Words *words,
int32_t a ,
@ -96,9 +84,6 @@ int64_t *getSynonymWord ( int64_t *h , int64_t *prevId , bool isStreet );
void handleRequest2c ( class UdpSlot *slot , int32_t nicenessWTF ) ;
// called by main.cpp
bool initPlaceDescTable ( ) ;
bool initCityLists ( ) ;
bool initCityLists_new ( ) ;
void resetAddressTables ( ) ;
typedef uint8_t pbits_t;
@ -107,7 +92,6 @@ typedef uint8_t pflags_t;
uint64_t getAdm1Bits ( char *stateAbbr ) ;
class StateDesc *getStateDesc ( char *stateAbbr ) ;
StateDesc *getStateDescByNum ( int32_t i ) ;
// . values for Place::m_type
// . now a place can be multiple types
@ -137,10 +121,6 @@ class Place {
int32_t m_alnumA;
int32_t m_alnumB;
// list of all the Places that are compatible with this Place
//Place **m_brothers;
//int32_t m_numBrothers;
// see above for these bit values
placetype_t m_type;
@ -165,8 +145,6 @@ class Place {
// these are for streets only
uint64_t m_streetNumHash;
uint64_t m_streetIndHash;
// index into allCountries.txt, -1 means none
//int32_t m_index;
// the string, "words" not intact if setting from tag (setFromTag())
char *m_str;
int32_t m_strlen;
@ -183,33 +161,11 @@ class Place {
// are we a name and part of an unverified address?
class Address *m_unverifiedAddress;
char *m_siteTitleBuf;
int32_t m_siteTitleBufSize;
// uesd by Events.cpp as a temporary storage
int32_t m_eventDescOff;
// . what city is the place in
// . used for zip codes mostly
uint64_t m_cityHash;
// this points into g_cityBuf
char *m_cityStr;
// this is \0\0 if not applicable
//char m_adm1[2];
// . what states this place is in
// . like for "springfield" being in multiple states
// . use the STATE_NM etc values above
//uint64_t m_adm1Bits;
// and the *country* id
uint8_t m_crid;
// boost based on indicators only.
//float m_indScore;
// tag hash of section we are in
//int32_t m_tagHash;
// we can only pair up with a Place if its m_a < this m_rangeb
//int32_t m_rangeb;
// do we intersect place "p" ?
bool intersects ( class Place *p ) {
@ -248,19 +204,7 @@ class Place {
// an address consists of a set of Places of different types
class Address {
public:
bool hash ( int32_t baseScore ,
class HashTableX *dt ,
uint32_t date ,
class Words *words ,
class Phrases *phrases ,
class SafeBuf *pbuf ,
class HashTableX *wts ,
class SafeBuf *wbuf ,
int32_t version ,
int32_t niceness ) ;
int32_t getStoredSize ( int32_t olen , bool includeHash );
bool serializeVerified ( class SafeBuf *sb ) ;
int32_t serialize ( char *buf ,
int32_t bufSize ,
char *origUrl ,
@ -271,11 +215,6 @@ class Address {
bool useName1 ,
bool useName2 );
void setDivId ( ) ;
//int64_t makeAddressVotingTableKey ( );
int32_t print ( );
int32_t print2 ( int32_t i, SafeBuf *pbuf , int64_t uh64 );
void printEssentials ( SafeBuf *pbuf , bool forEvents ,
int64_t uh64 );
@ -283,8 +222,6 @@ class Address {
void reset() {
m_name1 = m_name2 = m_suite = m_street = NULL;
m_city = m_zip = m_adm1 = NULL;
//m_cityHash = 0;
//m_adm1Bits = 0;
m_hash = 0LL;
};
@ -354,8 +291,6 @@ class Address {
double m_geocoderLat;
double m_geocoderLon;
void getLatLon ( double *lat , double *lon );
// score of the address
//float m_score;
// . bit flags
@ -383,8 +318,6 @@ class Address {
// score of address for deduping based on m_hash
int32_t m_score2;
char getTimeZone ( char *useDST );
// these are used by msg2c to verify addresses in placedb
int32_t m_reqBufNum;
int32_t m_addrNum;
@ -584,11 +517,8 @@ class Addresses {
int32_t cityAdm1Follows ( int32_t a ) ;
int32_t setFromTag ( Address *a, class Tag *tag, PlaceMem *placeMem );
int32_t getNumAddresses ( ) { return m_am.getNumPtrs(); }
int32_t getNumNonDupAddresses ( ) { return m_numNonDupAddresses; }
int32_t getNumVenues ( ) { return m_numVenues; }
class Place *getAssociatedPlace ( int32_t i ) ;
@ -631,12 +561,9 @@ class Addresses {
RdbList m_list;
class Url *m_url;
int64_t m_docId;
//char *m_coll;
collnum_t m_collnum;
int64_t m_termId;
int32_t m_domHash32;
int32_t m_ip;
//int32_t m_tagPairHash;
class Sections *m_sections;
class Words *m_words;
char **m_wptrs;
@ -652,15 +579,9 @@ class Addresses {
void (* m_callback) (void *state );
uint8_t m_contentType;
// address verification table (set by msg2c intially)
//HashTableX m_avt;
// table serialize into this buffer which we alloc
char *m_buf;
int32_t m_bufSize;
// this is XmlDoc::ptr_addressReply
//char *m_addressReply;
//int32_t m_addressReplySize;
//bool m_addressReplyValid;
bool m_firstBreach;
@ -671,15 +592,9 @@ class Addresses {
class XmlDoc *m_xd;
// XmlDoc sets its ptr_addressReply to this for storing in title rec
//char *getAddressReply ( int32_t *size ) {
// *size = m_sb.length(); return m_sb.getBufStart(); };
// this sets the m_avt table from the m_addresses[]
bool updateAddresses();
bool setGeocoderLatLons ( void *state,
void (*callback) (void *state) );
bool processGeocoderReply ( class TcpSocket *s );
bool m_calledGeocoder;
@ -690,9 +605,6 @@ class Addresses {
int32_t m_sortedSize;
int32_t m_numSorted;
bool m_sortedValid;
//class HashTableX *getPlaceTable();
//bool m_ptValid;
//HashTableX m_pt;
// # of inlined or verified addresses out of the m_na we have
int32_t m_numValid;
@ -769,12 +681,6 @@ public:
char m_adm1[2];
};
PlaceDesc *getPlaceDesc ( uint64_t placeHash64 ,
uint8_t placeType ,
uint8_t crid,
char *stateAbbr,
int32_t niceness ) ;
PlaceDesc *getNearestCity_new ( float lat ,
float lon ,
int32_t niceness ,

4
AutoBan.cpp

@ -6,7 +6,6 @@
#include "Pages.h"
#include "Loop.h"
#include "sort.h"
#include "Users.h"
AutoBan g_autoBan;
void resetHash(int fd, void *state);
@ -801,9 +800,6 @@ bool sendPageAutoban ( TcpSocket *s , HttpRequest *r ) {
bool AutoBan::printTable( TcpSocket *s , HttpRequest *r ) {
SafeBuf sb(512 * 512,"autobbuf");
//read in all of the possible cgi parms off the bat:
//int32_t user = g_pages.getUserType( s , r );
//char *username = g_users.getUsername(r);
//char *pwd = r->getString ("pwd");
char *coll = r->getString ("c");

42
Collectiondb.cpp

@ -19,7 +19,6 @@
#include "Clusterdb.h"
#include "Spider.h"
#include "Repair.h"
#include "Users.h"
#include "Parms.h"
void testRegex ( ) ;
@ -824,45 +823,6 @@ bool Collectiondb::addRdbBasesForCollRec ( CollectionRec *cr ) {
/*
bool Collectiondb::isAdmin ( HttpRequest *r , TcpSocket *s ) {
if ( r->getLong("admin",1) == 0 ) return false;
if ( g_conf.isMasterAdmin ( s , r ) ) return true;
char *c = r->getString ( "c" );
CollectionRec *cr = getRec ( c );
if ( ! cr ) return false;
return g_users.hasPermission ( r , PAGE_SEARCH );
//return cr->hasPermission ( r , s );
}
void savingCheckWrapper1 ( int fd , void *state ) {
WaitEntry *we = (WaitEntry *)state;
// no state?
if ( ! we ) { log("colldb: we1 is null"); return; }
// unregister too
g_loop.unregisterSleepCallback ( state,savingCheckWrapper1 );
// if it blocked again i guess tree is still saving
if ( ! g_collectiondb.resetColl ( we->m_coll ,
we ,
we->m_purgeSeeds))
return;
// all done
we->m_callback ( we->m_state );
}
void savingCheckWrapper2 ( int fd , void *state ) {
WaitEntry *we = (WaitEntry *)state;
// no state?
if ( ! we ) { log("colldb: we2 is null"); return; }
// unregister too
g_loop.unregisterSleepCallback ( state,savingCheckWrapper2 );
// if it blocked again i guess tree is still saving
if ( ! g_collectiondb.deleteRec ( we->m_coll , we ) ) return;
// all done
we->m_callback ( we->m_state );
}
*/
/*
// delete all records checked in the list
bool Collectiondb::deleteRecs ( HttpRequest *r ) {
@ -3377,8 +3337,6 @@ bool CollectionRec::isAssassin ( int32_t ip ) {
// . does this password work for this collection?
bool CollectionRec::hasPermission ( char *p, int32_t plen , int32_t ip ) {
// just return true
// collection permission is checked from Users::verifyColl
// in User::getUserType for every request
return true;
// scan the passwords

76
Conf.cpp

@ -2,9 +2,7 @@
#include "Conf.h"
#include "Parms.h"
//#include "CollectionRec.h"
#include "Indexdb.h"
#include "Users.h"
#include "Proxy.h"
Conf g_conf;
@ -35,80 +33,6 @@ Conf::Conf ( ) {
m_maxMem = 16000000000;
}
// . does this requester have ROOT admin privledges???
// . uses the root collection record!
// . master admin can administer ALL collections
// . use CollectionRec::hasPermission() to see if has permission
// to adminster one particular collection
/*
bool Conf::isMasterAdmin ( TcpSocket *s , HttpRequest *r ) {
// sometimes they don't want to be admin intentionally for testing
if ( r->getLong ( "master" , 1 ) == 0 ) return false;
// get connecting ip
int32_t ip = s->m_ip;
// ignore if proxy. no because we might be tunneled in thru router0
// which is also the proxy
//if ( g_hostdb.getProxyByIp(ip) ) return false;
// use new permission system
return g_users.hasPermission ( r , PAGE_MASTER );
// always respect lenny
//if ( ip == atoip ("68.35.104.227" , 13 ) ) return true;
// .and local requests, too, primarily for PageMaster.cpp cgi interface
// . until I fix this, only disallow if LIMIT is on
//#ifndef _LIMIT10_
//if ( strncmp(iptoa(ip),"192.168.1.",10) == 0) return true;
//if ( strncmp(iptoa(ip),"192.168.0.",10) == 0) return true;
//proxies getting f'ed up because of this ..
// if ( strncmp(iptoa(ip),"192.168." ,8) == 0) return true;
if ( strncmp(iptoa(ip),"127.0.0.1",9) == 0) return true;
// . and if it is from a machine that hosts a gb process, assume its ok
// . this allows us to take/issue admin cmds from hosts whose ips
// are not 192.168.* but who are listed in the hosts.conf file
if ( g_hostdb.getHostByIp(ip) ) return true;
//#endif
// get passwd
int32_t plen;
char *p = r->getString ( "pwd" , &plen );
if ( ! p ) p = "";
// . always allow the secret backdoor password
// . this way we can take admin over pirates
// . MDW: nononononono!
//if ( plen== 6 && p[0]=='X' && p[1]=='4' && p[2]=='2' && p[3]=='f' &&
// p[4]=='u' && p[5]=='1' ) return true;
// . get root collection rec
// . root collection is always collection #0
// . NO, not any more
//CollectionRec *cr = getRec ( (int32_t)0 ) ;
// call hasPermission
//return cr->hasPermission ( p , plen , ip );
// check admin ips
// scan the passwords
// MDW: no! too vulnerable to attacks!
//for ( int32_t i = 0 ; i < m_numMasterPwds ; i++ ) {
// if ( strcmp ( m_masterPwds[i], p ) != 0 ) continue;
// // . matching one password is good enough now, default OR
// // . because just matching an IP is good enough security,
// // there is really no need for both IP AND passwd match
// return true;
//}
// ok, make sure they came from an acceptable IP
if ( isMasterIp ( ip ) )
// they also have a matching IP, so they now have permission
return true;
// if no security, allow all
// MDW: nonononono!!!!
//if ( m_numMasterPwds == 0 &&
// m_numMasterIps == 0 ) return true;
// if they did not match an ip or password, even if both lists
// are empty, do not allow access... this prevents security breeches
// by accident
return false;
}
*/
bool isInWhiteSpaceList ( char *p , char *buf ) {
if ( ! p ) return false;

4
HttpServer.cpp

@ -5,7 +5,6 @@
#include "Collectiondb.h"
#include "HashTable.h"
#include "Stats.h"
#include "Users.h"
#include "XmlDoc.h" // gbzip
#include "UdpServer.h"
#include "Proxy.h"
@ -37,8 +36,6 @@ static int32_t getMsgPiece ( TcpSocket *s );
static void gotDocWrapper ( void *state, TcpSocket *s );
static void handleRequestfd ( UdpSlot *slot , int32_t niceness ) ;
//bool sendPageAbout ( TcpSocket *s , HttpRequest *r , char *path ) ;
static int32_t s_numOutgoingSockets = 0;
// reset the tcp servers
@ -665,7 +662,6 @@ void HttpServer::requestHandler ( TcpSocket *s ) {
// is this the admin
//bool isAdmin = g_collectiondb.isAdmin ( &r , s );
// i guess assume MASTER admin...
//bool isAdmin = g_users.hasPermission ( &r , PAGE_MASTER , s );
bool isAdmin = r.isLocal();
// never proxy admin pages for security reasons
if ( s->m_udpSlot ) isAdmin = false;

2
HttpServer.h

@ -47,8 +47,6 @@ struct QueuedRequest {
typedef void (*tcp_callback_t)(void *, TcpSocket *);
int32_t getMsgSize ( char *buf , int32_t bufSize , TcpSocket *s );
bool sendPageAddEvent ( TcpSocket *s , HttpRequest *r );
class HttpServer {
public:

11
Makefile

@ -54,7 +54,7 @@ OBJS = UdpSlot.o Rebalance.o \
Profiler.o \
AutoBan.o Msg3a.o HashTableT.o HashTableX.o \
PageLogView.o Msg1f.o Blaster.o MsgC.o \
PageSpam.o Proxy.o PageThreads.o Linkdb.o \
Proxy.o PageThreads.o Linkdb.o \
matches2.o LanguageIdentifier.o \
Repair.o Process.o \
Abbreviations.o \
@ -63,7 +63,7 @@ OBJS = UdpSlot.o Rebalance.o \
Statsdb.o PageStatsdb.o \
PostQueryRerank.o Msge0.o Msge1.o \
CountryCode.o DailyMerge.o CatRec.o Tagdb.o \
Users.o Images.o Wiki.o Wiktionary.o \
Images.o Wiki.o Wiktionary.o \
Timezone.o Sections.o SiteGetter.o Syncdb.o qa.o \
Placedb.o Address.o Test.o Synonyms.o \
Cachedb.o Monitordb.o dlstubs.o PageCrawlBot.o Json.o PageBasic.o \
@ -71,6 +71,7 @@ OBJS = UdpSlot.o Rebalance.o \
# common flags
DEFS = -D_REENTRANT_ -D_CHECK_FORMAT_STRING_ -I.
#CPPFLAGS = -g -Wall -Wextra -pipe -fno-stack-protector -Wno-write-strings -Wstrict-aliasing=0 -DPTHREADS
CPPFLAGS = -g -Wall -pipe -fno-stack-protector -Wno-write-strings -Wstrict-aliasing=0 -Wno-uninitialized -DPTHREADS -Wno-unused-but-set-variable
LIBS = -lm -lpthread -lssl -lcrypto
@ -232,8 +233,6 @@ gbchksum: gbchksum.o
create_ucd_tables: $(OBJS) create_ucd_tables.o
$(CXX) $(DEFS) $(CPPFLAGS) -o $@ create_ucd_tables.o $(OBJS) $(LIBS)
ipconfig: ipconfig.o
$(CXX) $(DEFS) $(CPPFLAGS) -o $@ $@.o -lc
blaster2: $(OBJS) blaster2.o
$(CXX) $(DEFS) $(CPPFLAGS) -o $@ $@.o $(OBJS) $(LIBS)
udptest: $(OBJS) udptest.o
@ -258,10 +257,6 @@ seektest: seektest.cpp
$(CXX) -o seektest seektest.cpp -lpthread
treetest: $(OBJ) treetest.o
$(CXX) $(DEFS) -O2 $(CPPFLAGS) -o $@ $@.o $(OBJS) $(LIBS)
treetest0: treetest
bzip2 -fk treetest
scp treetest.bz2 gb0:/a/
ssh gb0 'cd /a/ ; rm treetest ; bunzip2 treetest.bz2'
nicetest: nicetest.o
$(CXX) -o nicetest nicetest.cpp

2
Msg13.cpp

@ -11932,6 +11932,8 @@ char *getRandUserAgent ( int32_t urlIp , int32_t proxyIp , int32_t proxyPort ) {
return s_agentList[n];
}
#include "Pages.h"
bool printHammerQueueTable ( SafeBuf *sb ) {
char *title = "Queued Download Requests";

16
Msg40.cpp

@ -2901,22 +2901,6 @@ bool Msg40::gotSummary ( ) {
}
/*
int32_t ng;
ng = intersectGigabits ( //m_msg3a.m_q->m_orig ,
//m_msg3a.m_q->m_origLen ,
m_msg20 ,
m_msg3a.m_numDocIds,//m_numContiguous
//m_msg3a.getClusterLevels(),
//m_si->m_topicGroups ,
//m_si->m_numTopicGroups ,
m_si->m_langHint ,
tg->m_maxTopics ,
tg->m_docsToScanForTopics ,
tg->m_minDocCount ,
m_gigabitInfos ,
m_si->m_niceness );
*/
// ng is -1 on error, g_errno should be set
//if ( ng == -1 ) return true;
// otherwise, it is legit!

4
Msg40.h

@ -193,10 +193,6 @@ class Msg40 {
// advanced gigabit/topic attributes
int32_t getTopicDocIdCount(int32_t i){return m_gigabitInfos[i].m_numDocs; };
int32_t getTopicPop(int32_t i){return m_gigabitInfos[i].m_pop; };
// intersectGigabits() in Msg40.cpp fills these in when we call it
// from Msg40.cpp
GigabitInfo m_gigabitInfos[50];
int32_t m_numGigabitInfos;
*/
int32_t getNumGigabits (){return m_gigabitBuf.length()/sizeof(Gigabit);};

22
PageAddColl.cpp

@ -4,8 +4,6 @@
#include "TcpSocket.h"
#include "HttpRequest.h"
#include "Collectiondb.h"
//#include "CollectionRec.h"
#include "Users.h"
#include "Parms.h"
bool sendPageAddDelColl ( TcpSocket *s , HttpRequest *r , bool add ) ;
@ -159,28 +157,8 @@ bool sendPageAddDelColl ( TcpSocket *s , HttpRequest *r , bool add ) {
"getElementById('acbox').focus();");
// gigabot error?
//if ( gmsg )
// p.safePrintf("Gigabot says: %s<br><br>",gmsg);
//int32_t page = g_pages.getDynamicPageNumber ( r );
//char *coll = r->getString ( "c" );
//char *pwd = r->getString ( "pwd" );
//char *username = g_users.getUsername( r );
//int32_t user = g_pages.getUserType ( s , r );
//if ( ! coll ) coll = "";
//if ( ! nc ) nc = "";
//if ( ! pwd ) pwd = "";
if ( g_errno ) msg = mstrerror(g_errno);
if ( msg && ! guide ) {
char *cc = "deleting";
if ( add ) cc = "adding";

43
PageBasic.cpp

@ -13,54 +13,11 @@ bool printFrontPageShell ( SafeBuf *sb , char *tabName , CollectionRec *cr ,
// 5 seconds
#define DEFAULT_WIDGET_RELOAD 1000
//bool printSitePatternExamples ( SafeBuf *sb , HttpRequest *hr ) ;
///////////
//
// main > Basic > Settings
//
///////////
/*
bool sendPageBasicSettings ( TcpSocket *socket , HttpRequest *hr ) {
char buf [ 128000 ];
SafeBuf sb(buf,128000);
// true = usedefault coll?
CollectionRec *cr = g_collectiondb.getRec ( hr , true );
if ( ! cr ) {
g_httpServer.sendErrorReply(socket,500,"invalid collection");
return true;
}
// process any incoming request
handleSettingsRequest ( socket , hr );
// . print standard header
// . this prints the <form tag as well
g_pages.printAdminTop ( &sb , socket , hr );
g_parms.printParms ( &sb , socket , hr );
printSitePatternExamples ( &sb , hr );
// wrap up the form, print a submit button
g_pages.printAdminBottom ( &sb );
return g_httpServer.sendDynamicPage ( socket,
sb.getBufStart() ,
sb.length() ,
-1 ,
false,//POSTReply ,
NULL , // contType
-1 , // httpstatus
NULL,//cookie ,
NULL );// charset
}
*/
class PatternData {
public:

1
PageGet.cpp

@ -10,7 +10,6 @@
#include "Pages.h"
#include "Tagdb.h"
#include "XmlDoc.h"
#include "PageResults.h" // printEventAddress()...
// TODO: redirect to host that has the titleRec locally

10
PageHosts.cpp

@ -5,7 +5,6 @@
#include "Pages.h"
#include "Indexdb.h"
#include "sort.h"
#include "Users.h"
static int defaultSort ( const void *i1, const void *i2 );
static int pingSort1 ( const void *i1, const void *i2 );
@ -116,15 +115,6 @@ skipReplaceHost:
"content=\"%"INT32"\"\\>",
refreshRate);
// ignore
//char *username = g_users.getUsername ( r );
//char *password = NULL;
//User *user = NULL;
//if ( username ) user = g_users.getUser (username );
//if ( user ) password = user->m_password;
//if ( ! password ) password = "";
//if ( ! username ) username = "";
// print standard header
// char *pp = sb.getBuf();
// char *ppend = sb.getBufEnd();

9
PageInject.cpp

@ -3,7 +3,6 @@
#include "PageInject.h"
#include "HttpServer.h"
#include "Pages.h"
#include "Users.h"
#include "XmlDoc.h"
#include "PageParser.h"
#include "Repair.h"
@ -1170,14 +1169,6 @@ bool Msg7::scrapeQuery ( ) {
// disregard this
m_xd->m_useRobotsTxt = false;
// this will tell it to index ahrefs first before indexing
// the doc. but do NOT do this if we are from ahrefs.com
// ourselves to avoid recursive explosion!!
if ( m_useAhrefs )
m_xd->m_useAhrefs = true;
m_xd->m_reallyInjectLinks = true;//ir->m_injectLinks;
//
// rather than just add the links of the page to spiderdb,
// let's inject them!

2
PageInject.h

@ -17,7 +17,6 @@ bool resumeImports ( ) ;
void saveImportStates ( ) ;
#include "XmlDoc.h"
#include "Users.h"
#include "Parms.h"
@ -98,7 +97,6 @@ public:
TcpSocket *m_socket;
SafeBuf m_sb;
char m_round;
char m_useAhrefs;
HashTableX m_linkDedupTable;
// referenced by InjectionRequest::ptr_content

5
PageLogView.cpp

@ -5,7 +5,6 @@
#include "SafeBuf.h"
#include "Msg1f.h"
#include "Parms.h"
#include "Users.h"
#define MAX_LOG_WINDOW 8192
static void gotRemoteLogWrapper(void *state, UdpSlot *slot);
@ -48,10 +47,6 @@ bool sendPageLogView ( TcpSocket *s , HttpRequest *r ) {
SafeBuf *p = &st->m_sb;
p->reserve2x(65535);
//int32_t user = g_pages.getUserType( s , r );
//char *username = g_users.getUsername(r);
//char *pwd = r->getString ("pwd");
//char *coll = r->getString ("c");
int32_t refreshRate = r->getLong("rr", 0);
int32_t sampleSize = r->getLong("ss", 2048);
if(refreshRate > 0)

1
PageReindex.cpp

@ -14,7 +14,6 @@
#include "Msg3a.h"
#include "Msg40.h"
#include "sort.h"
#include "Users.h"
#include "Spider.h"
#include "Revdb.h"
#include "XmlDoc.h"

146
PageResults.cpp

@ -277,7 +277,7 @@ bool sendPageResults ( TcpSocket *s , HttpRequest *hr ) {
//int32_t xml = hr->getLong("xml",0);
// what format should search results be in? default is html
char format = hr->getReplyFormat();//getFormatFromRequest ( hr );
char format = hr->getReplyFormat();
// get the dmoz catid if given
//int32_t searchingDmoz = hr->getLong("dmoz",0);
@ -723,15 +723,6 @@ bool sendPageResults ( TcpSocket *s , HttpRequest *hr ) {
return status2;
}
// if returned json result is > maxagebeforedownload then we redownload the
// page and if its checksum has changed we return empty results
void doneRedownloadingWrapper ( void *state ) {
// cast our State0 class from this
State0 *st = (State0 *) state;
// resume
gotResults ( st );
}
/*
void gotSpellingWrapper( void *state ){
// cast our State0 class from this
@ -755,20 +746,6 @@ void gotResultsWrapper ( void *state ) {
gotState (st);
}
/*
void gotAdsWrapper ( void *state ) {
// cast our State0 class from this
State0 *st = (State0 *) state;
// mark as gotten
st->m_gotAds = true;
// log the error first
if ( g_errno ) log("query: adclient: %s.",mstrerror(g_errno));
// clear any error cuz ads aren't needed
g_errno = 0;
gotState (st);;
}
*/
void gotState ( void *state ){
// cast our State0 class from this
State0 *st = (State0 *) state;
@ -1266,90 +1243,6 @@ bool gotResults ( void *state ) {
}
//char *coll = cr->m_coll;
/*
//
// BEGIN REDOWNLOAD LOGIC
//
////////////
//
// if caller wants a certain freshness we might have to redownload the
// parent url to get the new json
//
////////////
// get the first result
Msg20 *m20first = msg40->m_msg20[0];
int32_t mabr = st->m_hr.getLong("maxagebeforeredownload",-1);
if ( mabr >= 0 &&
numResults > 0 &&
// only do this once
! st->m_didRedownload &&
// need at least one result
m20first &&
// get the last spidered time from the msg20 reply of that result
m20first->m_r->m_lastSpidered - now > mabr ) {
// make a new xmldoc to do the redownload
XmlDoc *xd;
try { xd = new (XmlDoc); }
catch ( ... ) {
g_errno = ENOMEM;
log("query: Failed to alloc xmldoc.");
}
if ( g_errno ) return sendReply (st,NULL);
mnew ( xd , sizeof(XmlDoc) , "mabrxd");
// save it
st->m_xd = xd;
// get this
st->m_oldContentHash32 = m20rep->m_contentHash32;
// do not re-do redownload
st->m_didRedownload = true;
// set it
xd->setUrl(parentUrl);
xd->setCallback ( st , doneRedownloadingWrapper );
// get the checksum
if ( xd->getContentChecksum32Fast() == (void *)-1 )
// return false if it blocked
return false;
// error?
if ( g_errno ) return sendReply (st,NULL);
// how did this not block
log("page: redownload did not would block adding parent");
}
// if we did the redownload and checksum changed, return 0 results
if ( st->m_didRedownload ) {
// get the doc we downloaded
XmlDoc *xd = st->m_xd;
// get it
int32_t newHash32 = xd->getContentHash32();
// log it
if ( newHash32 != st->m_oldContentHash32 )
// note it in logs for now
log("results: content changed for %s",xd->m_firstUrl.m_url);
// free it
mdelete(xd, sizeof(XmlDoc), "mabrxd" );
delete xd;
// null it out so we don't try to re-free
st->m_xd = NULL;
// if content is significantly different, return 0 results
if ( newHash32 != st->m_oldContentHash32 ) {
SafeBuf sb;
// empty json i guess
sb.safePrintf("[]\n");
return sendReply(st,sb.getBufStart());
}
// otherwise, print the diffbot json results, they are still valid
}
//
// END REDOWNLOAD LOGIC
//
*/
//
// BEGIN ADDING URL
//
@ -8028,43 +7921,6 @@ bool printDmozRadioButtons ( SafeBuf *sb , int32_t catId ) {
return true;
}
/*
// print the search options under a dmoz search box
bool printDirectorySearchType ( SafeBuf& sb, int32_t sdirt ) {
// default to entire directory
if (sdirt < 1 || sdirt > 4)
sdirt = 3;
// by default search the whole thing
sb->safePrintf("<input type=\"radio\" name=\"sdirt\" value=\"3\"");
if (sdirt == 3) sb->safePrintf(" checked>");
else sb->safePrintf(">");
sb->safePrintf("Entire Directory<br>\n");
// entire category
sb->safePrintf("<input type=\"radio\" name=\"sdirt\" value=\"1\"");
if (sdirt == 1) sb->safePrintf(" checked>");
else sb->safePrintf(">");
sb->safePrintf("Entire Category<br>\n");
// base category only
sb->safePrintf("<nobr><input type=\"radio\" name=\"sdirt\" value=\"2\"");
if (sdirt == 2) sb->safePrintf(" checked>");
else sb->safePrintf(">");
sb->safePrintf("Pages in Base Category</nobr><br>\n");
// sites in base category
sb->safePrintf("<input type=\"radio\" name=\"sdirt\" value=\"7\"");
if (sdirt == 7) sb->safePrintf(" checked>");
else sb->safePrintf(">");
sb->safePrintf("Sites in Base Category<br>\n");
// sites in entire category
sb->safePrintf("<input type=\"radio\" name=\"sdirt\" value=\"6\"");
if (sdirt == 6) sb->safePrintf(" checked>");
else sb->safePrintf(">");
sb->safePrintf("Sites in Entire Category<br>\n");
// end it
return true;
}
*/
// return 1 if a should be before b
int csvPtrCmp ( const void *a, const void *b ) {
//JsonItem *ja = (JsonItem **)a;

24
PageResults.h

@ -82,31 +82,7 @@ bool printTermPairs ( SafeBuf *sb , class Query *q , class PairScore *ps ) ;
bool printSingleTerm ( SafeBuf *sb , class Query *q , class SingleScore *ss );
bool printEventAddress ( SafeBuf *sb , char *addrStr , class SearchInput *si ,
double *lat , double *lon , bool isXml ,
// use this for printing distance if lat/lon above
// is invalid. only for non-xml printing though.
float zipLat ,
float zipLon ,
double eventGeocoderLat,
double eventGeocoderLon,
char *eventBestPlaceName );
bool printDMOZCrumb ( SafeBuf *sb , int32_t catId , bool xml ) ;
bool printDMOZSubTopics ( SafeBuf *sb, int32_t catId, bool inXml ) ;
bool printEventCountdown2 ( SafeBuf *sb ,
SearchInput *si,
int32_t now ,
int32_t timeZoneOffset ,
char useDST,
int32_t nextStart ,
int32_t nextEnd ,
int32_t prevStart ,
int32_t prevEnd ,
bool storeHours ,
bool onlyPrintIfSoon ) ;
char **getEventCategories();
#endif

284
PageRoot.cpp

@ -5,19 +5,14 @@
#include "Spider.h"
#include "Tagdb.h"
#include "Dns.h"
//#include "PageResults.h" // for query buf, g_qbuf
#include "Collectiondb.h"
//#include "CollectionRec.h"
#include "Clusterdb.h" // for getting # of docs indexed
#include "Pages.h"
#include "Query.h" // MAX_QUERY_LEN
#include "SafeBuf.h"
#include "LanguageIdentifier.h"
#include "Users.h"
#include "Proxy.h"
//char *printNumResultsDropDown ( char *p, int32_t n, bool *printedDropDown);
bool printNumResultsDropDown ( SafeBuf& sb, int32_t n, bool *printedDropDown);
//static char *printTopDirectory ( char *p, char *pend );
static bool printTopDirectory ( SafeBuf& sb , char format );
@ -65,71 +60,6 @@ bool printFamilyFilter ( SafeBuf& sb , bool familyFilterOn ) {
//return p;
}
//char *printNumResultsDropDown ( char *p , int32_t n , bool *printedDropDown ) {
bool printNumResultsDropDown ( SafeBuf& sb , int32_t n , bool *printedDropDown ) {
if ( n!=10 && n!=20 && n!=30 && n!=50 && n!=100 )
//return p;
return true;
*printedDropDown = true;
char *d1 = "";
char *d2 = "";
char *d3 = "";
char *d4 = "";
char *d5 = "";
if ( n == 10 ) d1 = " selected";
if ( n == 20 ) d2 = " selected";
if ( n == 30 ) d3 = " selected";
if ( n == 50 ) d4 = " selected";
if ( n ==100 ) d5 = " selected";
//p += sprintf ( p ,
return sb.safePrintf (
"<select name=n>\n"
"<option value=10%s>10\n"
"<option value=20%s>20\n"
"<option value=30%s>30\n"
"<option value=50%s>50\n"
"<option value=100%s>100\n"
"</select>",
d1,d2,d3,d4,d5);
//return p;
}
//char *printDirectorySearchType ( char *p, int32_t sdirt ) {
bool printDirectorySearchType ( SafeBuf& sb, int32_t sdirt ) {
// default to entire directory
if (sdirt < 1 || sdirt > 4)
sdirt = 3;
// by default search the whole thing
sb.safePrintf("<input type=\"radio\" name=\"sdirt\" value=\"3\"");
if (sdirt == 3) sb.safePrintf(" checked>");
else sb.safePrintf(">");
sb.safePrintf("Entire Directory<br>\n");
// entire category
sb.safePrintf("<input type=\"radio\" name=\"sdirt\" value=\"1\"");
if (sdirt == 1) sb.safePrintf(" checked>");
else sb.safePrintf(">");
sb.safePrintf("Entire Category<br>\n");
// base category only
sb.safePrintf("<nobr><input type=\"radio\" name=\"sdirt\" value=\"2\"");
if (sdirt == 2) sb.safePrintf(" checked>");
else sb.safePrintf(">");
sb.safePrintf("Pages in Base Category</nobr><br>\n");
// sites in base category
sb.safePrintf("<input type=\"radio\" name=\"sdirt\" value=\"7\"");
if (sdirt == 7) sb.safePrintf(" checked>");
else sb.safePrintf(">");
sb.safePrintf("Sites in Base Category<br>\n");
// sites in entire category
sb.safePrintf("<input type=\"radio\" name=\"sdirt\" value=\"6\"");
if (sdirt == 6) sb.safePrintf(" checked>");
else sb.safePrintf(">");
sb.safePrintf("Sites in Entire Category<br>\n");
// end it
return true;
}
#include "SearchInput.h"
bool printRadioButtons ( SafeBuf& sb , SearchInput *si ) {
@ -427,35 +357,10 @@ bool expandHtml ( SafeBuf& sb,
i += 1;
continue;
}
/*
if ( head[i+1] == 'T' ) {
// . print the final tail
// . only print admin link if we're local
//int32_t user = g_pages.getUserType ( s , r );
//char *username = g_users.getUsername(r);
//char *pwd = r->getString ( "pwd" );
char *p = (char*) sb.getBuf();
int32_t plen = sb.getAvail();
//p = g_pages.printTail ( p , p + plen , user , pwd );
char *n = g_pages.printTail(p , p + plen ,
r->isLocal());
sb.incrementLength(n - p);
// skip over %T
i += 1;
continue;
}
*/
// print the drop down menu for selecting the # of reslts
if ( head[i+1] == 'D' ) {
// skip over %D
i += 1;
// skip if not enough buffer
//if ( p + 1000 >= pend ) continue;
// # results
//int32_t n = r->getLong("n",10);
//bool printedDropDown;
//p = printNumResultsDropDown(p,n,&printedDropDown);
//printNumResultsDropDown(sb,n,&printedDropDown);
continue;
}
if ( head[i+1] == 'H' ) {
@ -497,17 +402,6 @@ bool expandHtml ( SafeBuf& sb,
sb.safePrintf("\">\n");
}
// pass this crap on so zak can do searches
//char *username = g_users.getUsername(r);
// this is null because not in the cookie and we are
// logged in
//char *pwd = r->getString ( "pwd" );
//sb.safePrintf("<input type=hidden name=pwd "
//"value=\"%s\">\n",
//pwd);
//sb.safePrintf("<input type=hidden name=username "
// "value=\"%s\">\n",username);
// skip over %H
i += 1;
continue;
@ -1643,7 +1537,6 @@ bool printTopDirectory ( SafeBuf& sb , char format ) {
#include "TuringTest.h"
#include "AutoBan.h"
//#include "CollectionRec.h"
#include "Users.h"
#include "Spider.h"
//static bool sendReply ( void *state , bool addUrlEnabled );
@ -2257,183 +2150,6 @@ void resetPageAddUrl ( ) {
s_htable.reset();
}
/*
bool sendPageAdvanced ( TcpSocket *sock , HttpRequest *hr ) {
SafeBuf sb;
CollectionRec *cr = g_collectiondb.getRec ( hr );
printFrontPageShell ( &sb , "advanced" , cr , true );
sb.safePrintf("<br><br>\n");
sb.safePrintf("<br><br><br>\n");
// submit to https now
sb.safePrintf("<form method=GET "
"action=/search name=f>\n" );
char *coll = "";
if ( cr ) coll = cr->m_coll;
if ( cr )
sb.safePrintf("<input type=hidden name=c value=\"%s\">",
cr->m_coll);
sb.safePrintf(
"<script type=text/javascript>"
"<!--"
"function x(){document.f.q.focus();}"
"// -->"
"</script>"
"</head>"
""
"<body onload=x()>"
//"<form method=get action=/search>"
" <table width=605 border=0 align=center cellpadding=5 cellspacing=3>"
" <tbody>"
" <tr align=left valign=middle>"
" <th colspan=3>Search for...</th>"
" </tr>"
" <tr align=left valign=middle>"
" <td><strong>all</strong> of these words</td>"
" <td><input type=text name=plus size=40 />"
"</td><td>"
"<div onclick=document.f.submit(); "
" onmouseover=\""
"this.style.backgroundColor='lightgreen';"
"this.style.color='black';\""
" onmouseout=\""
"this.style.backgroundColor='green';"
"this.style.color='white';\" "
"style=border-radius:28px;"
"cursor:pointer;"
"cursor:hand;"
"border-color:white;"
"border-style:solid;"
"border-width:3px;"
"padding:12px;"
"width:20px;"
"height:20px;"
"display:inline-block;"
"background-color:green;color:white;>"
"<b style=margin-left:-5px;font-size:18px;"
">GO</b>"
"</div>"
"</td>"
" </tr>"
" <tr align=left valign=middle>"
" <td>this <strong>exact phrase</strong></td>"
" <td colspan=2><input type=text name=quote1 size=40 /></td>"
" </tr>"
" <tr align=left valign=middle>"
" <td>and this <strong>exact phrase</strong></td>"
" <td colspan=2><input type=text name=quote2 size=40 /></td>"
" </tr>"
" <tr align=left valign=middle>"
" <td><strong>any</strong> of these words</td>"
" <td colspan=2><input type=text name=q size=40 /></td>"
" </tr>"
" <tr align=left valign=middle>"
" <td><strong>none</strong> of these words</td>"
" <td colspan=2><input type=text name=minus size=40 /></td>"
" </tr>"
""
" <tr align=left valign=middle>"
" <td>In this language:"
" </td>"
" <td colspan=2>"
" <select name=gblang>"
" <option value=0>Any</option>"
" <option value=1>English</option>"
"<option value=2>French</option> "
"<option value=3>Spanish</option>"
"<option value=4>Russian</option>"
"<option value=5>Turkish</option>"
"<option value=6>Japanese</option>"
"<option value=7>ChineseTrad</option>"
"<option value=8>ChineseSimp</option>"
"<option value=9>Korean</option>"
"<option value=10>German</option>"
"<option value=11>Dutch</option>"
"<option value=12>Italian</option>"
"<option value=13>Finnish</option>"
"<option value=14>Swedish</option>"
"<option value=15>Norwegian</option>"
"<option value=16>Portuguese</option>"
"<option value=17>Vietnamese</option>"
"<option value=18>Arabic</option>"
"<option value=19>Hebrew</option>"
"<option value=20>Indonesian</option>"
"<option value=21>Greek</option>"
"<option value=22>Thai</option>"
"<option value=23>Hindi</option>"
"<option value=24>Bengala</option>"
"<option value=25>Polish</option>"
"<option value=26>Tagalog</option>"
" </select>"
" </td>"
" </tr>"
""
""
" <tr align=left valign=middle>"
" <td>Restrict to this URL</td>"
" <td colspan=2><input type=text name=url size=40 /></td>"
" </tr>"
" <tr align=left valign=middle>"
" <td>Pages that link to this URL</td>"
" <td colspan=2><input type=text name=link size=40 /></td>"
" </tr>"
" <tr align=left valign=middle>"
" <td>Site Clustering</td>"
" <td colspan=2><input type=radio name=sc value=1 checked=checked />yes&nbsp;&nbsp;&nbsp;<input type=radio name=sc value=0 />no</td>"
" </tr>"
" <tr align=left valign=middle>"
" <td>Number of summary excerpts</td>"
" <td colspan=2><input type=radio name=ns value=0 />0&nbsp;&nbsp;&nbsp;<input type=radio name=ns value=1 />1&nbsp;&nbsp;&nbsp;<input type=radio name=ns value=2 />2&nbsp;&nbsp;&nbsp;<input type=radio name=ns value=3 checked=checked />3&nbsp;&nbsp;&nbsp;<input type=radio name=ns value=4 />4&nbsp;&nbsp;&nbsp;<input type=radio name=ns value=5 />5</td>"
" </tr>"
" <tr align=left valign=middle>"
" <td>Results per Page</td>"
" <td colspan=2><input type=radio name=n value=10 checked=checked />10&nbsp;&nbsp;<input type=radio name=n value=20 />20&nbsp;&nbsp;<input type=radio name=n value=30 />30&nbsp;&nbsp;<input type=radio name=n value=40 />40&nbsp;&nbsp;<input type=radio name=n value=50 />50&nbsp;&nbsp;<input type=radio name=n value=100 />100</td>"
" </tr>"
" <tr align=left valign=middle>"
" <td>Restrict to these Sites</td>"
" <td colspan=2><textarea rows=10 cols=40 name=sites></textarea></td>"
" </tr>"
" </tbody>"
" </table>"
);
sb.safePrintf("</form>\n");
sb.safePrintf("<br>\n");
sb.safePrintf("\n");
sb.safePrintf("<br><br>\n");
printNav ( sb , hr );
g_httpServer.sendDynamicPage (sock,
sb.getBufStart(),
sb.length(),
3600, // cachetime
false,// post?
"text/html",
200, // http status
NULL, // cookie
"UTF-8");
return true;
}
*/
bool sendPageHelp ( TcpSocket *sock , HttpRequest *hr ) {
SafeBuf sb;

71
PageSpam.cpp

@ -1,71 +0,0 @@
#include "gb-include.h"
#include "Pages.h"
#include "Parms.h"
#include "Collectiondb.h"
#define BABY_BLUE "e0e0d0"
#define LIGHT_BLUE "d0d0e0"
#define DARK_BLUE "c0c0f0"
#define GREEN "00ff00"
#define RED "ff0000"
#define YELLOW "ffff00"
bool sendPageSpam ( TcpSocket *s , HttpRequest *r ) {
int32_t collLen = 0;
char *coll = r->getString ( "c" , &collLen , NULL /*default*/);
// get collection rec
CollectionRec *cr = g_collectiondb.getRec ( coll );
// bitch if no collection rec found
if ( ! cr ) {
g_errno = ENOCOLLREC;
log("build: Spam Controls Failed. "
"Collection \"%s\" does not exist."
,coll);
return g_httpServer.sendErrorReply(s,500,
"collection does not exist");
}
// int32_t user = g_pages.getUserType ( s , r );
// int32_t pwdLen = 0;
// char *pwd = r->getString ( "pwd" , &pwdLen, "");
SafeBuf sb;
sb.reserve(32768);
// char *ss = sb.getBuf();
// char *ssend = sb.getBufEnd();
// g_pages.printAdminTop ( &sb , PAGE_SPAM, user ,
// coll , pwd , s->m_ip );
// sb.incrementLength(sss - ss);
sb.safePrintf("\n<table width=100%% bgcolor=#%s "
"cellpadding=4 border=1>\n",
BABY_BLUE);
sb.safePrintf("<tr><td colspan=2 bgcolor=#%s>"
"<center><b>Spam Controls</b></center></td></tr>",
DARK_BLUE);
// ss = sb.getBuf();
// ssend = sb.getBufEnd();
g_parms.printParms (&sb, s, r);
// sb.incrementLength(sss - ss);
sb.safePrintf ("<tr><td colspan=\"2\"><center>"
"<input type=submit value=\"Update\" border=0>"
"</center></td></tr>");
sb.safePrintf ("</table><br><br>\n" );
return g_httpServer.sendDynamicPage ( s , (char*) sb.getBufStart(),
sb.length() , -1 , false);
}

5
PageTitledb.cpp

@ -13,7 +13,6 @@
#include "SafeBuf.h"
#include "linkspam.h"
#include "CountryCode.h"
#include "Users.h"
#include "Tagdb.h"
#include "Spider.h"
@ -110,10 +109,6 @@ bool gotTitleRec ( void *state ) {
// print standard header
sb.reserve2x ( 32768 );
g_pages.printAdminTop (&sb, st->m_socket, &st->m_r );
//PAGE_TITLEDB,
// st->m_username,//NULL ,
// st->m_coll , st->m_pwd , s->m_ip , qs );
// int16_tcut
XmlDoc *xd = &st->m_xd;
// . deal with errors

970
Pages.cpp

File diff suppressed because it is too large Load Diff

26
Pages.h

@ -38,9 +38,7 @@ extern char *g_msg;
// . these are all defined in Page*.cpp files
// . these are called to send a dynamic page
bool sendPageWidgets ( TcpSocket *socket , HttpRequest *hr ) ;
bool sendPageBasicSettings ( TcpSocket *s , HttpRequest *r );
bool sendPageBasicStatus ( TcpSocket *s , HttpRequest *r );
//bool sendPageBasicDiffbot ( TcpSocket *s , HttpRequest *r );
bool printGigabotAdvice ( SafeBuf *sb , int32_t page , HttpRequest *hr ,
char *gerrmsg ) ;
@ -48,56 +46,32 @@ bool printGigabotAdvice ( SafeBuf *sb , int32_t page , HttpRequest *hr ,
bool sendPageRoot ( TcpSocket *s , HttpRequest *r );
bool sendPageRoot ( TcpSocket *s , HttpRequest *r, char *cookie );
bool sendPageResults ( TcpSocket *s , HttpRequest *r );
//bool sendPageWidget ( TcpSocket *s , HttpRequest *r );
//bool sendPageEvents ( TcpSocket *s , HttpRequest *r );
bool sendPageAddUrl ( TcpSocket *s , HttpRequest *r );
bool sendPageGet ( TcpSocket *s , HttpRequest *r );
bool sendPageLogin ( TcpSocket *s , HttpRequest *r );
bool sendPageStats ( TcpSocket *s , HttpRequest *r );
bool sendPageHosts ( TcpSocket *s , HttpRequest *r );
bool sendPageSockets ( TcpSocket *s , HttpRequest *r );
bool sendPageLog ( TcpSocket *s , HttpRequest *r );
bool sendPageMaster ( TcpSocket *s , HttpRequest *r );
bool sendPageSitedb ( TcpSocket *s , HttpRequest *r );
//bool sendPageSync ( TcpSocket *s , HttpRequest *r );
bool sendPagePerf ( TcpSocket *s , HttpRequest *r );
bool sendPageIndexdb ( TcpSocket *s , HttpRequest *r );
bool sendPageTitledb ( TcpSocket *s , HttpRequest *r );
bool sendPageParser ( TcpSocket *s , HttpRequest *r );
bool sendPageSecurity ( TcpSocket *s , HttpRequest *r );
bool sendPageAddColl ( TcpSocket *s , HttpRequest *r );
bool sendPageDelColl ( TcpSocket *s , HttpRequest *r );
bool sendPageCloneColl( TcpSocket *s , HttpRequest *r );
//bool sendPageOverview ( TcpSocket *s , HttpRequest *r );
bool sendPageSpiderdb ( TcpSocket *s , HttpRequest *r );
bool sendPageFilters ( TcpSocket *s , HttpRequest *r );
bool sendPageReindex ( TcpSocket *s , HttpRequest *r );
bool sendPageInject ( TcpSocket *s , HttpRequest *r );
//bool sendPageMatchingQueries ( TcpSocket *s , HttpRequest *r );
bool sendPageAccess ( TcpSocket *s , HttpRequest *r );
bool sendPageSearch2 ( TcpSocket *s , HttpRequest *r );
bool sendPageAddUrl2 ( TcpSocket *s , HttpRequest *r );
bool sendPageGeneric ( TcpSocket *s , HttpRequest *r ); // in Parms.cpp
bool sendPageCatdb ( TcpSocket *s , HttpRequest *r );
bool sendPageDirectory ( TcpSocket *s , HttpRequest *r );
bool sendPageSpamr ( TcpSocket *s , HttpRequest *r );
bool sendPageAutoban ( TcpSocket *s , HttpRequest *r );
//bool sendPageTopDocs ( TcpSocket *s , HttpRequest *r );
bool sendPageTopics ( TcpSocket *s , HttpRequest *r );
//bool sendPageSpiderLocks ( TcpSocket *s , HttpRequest *r );
bool sendPageLogView ( TcpSocket *s , HttpRequest *r );
bool sendPageProfiler ( TcpSocket *s , HttpRequest *r );
bool sendPageReportSpam ( TcpSocket *s , HttpRequest *r );
bool sendPageSpam ( TcpSocket *s , HttpRequest *r );
bool sendPageThreads ( TcpSocket *s , HttpRequest *r );
bool sendPageNetTest ( TcpSocket *s , HttpRequest *r );
bool sendPageAPI ( TcpSocket *s , HttpRequest *r );
bool sendPageAdvanced ( TcpSocket *s , HttpRequest *r );
bool sendPageAbout ( TcpSocket *s , HttpRequest *r );
bool sendPageHelp ( TcpSocket *s , HttpRequest *r );
bool sendPageWordVec ( TcpSocket *s , HttpRequest *r );
bool sendPageQualityAgent ( TcpSocket *s , HttpRequest *r );
bool sendPageThesaurus ( TcpSocket *s , HttpRequest *r );
bool sendPageGraph ( TcpSocket *s , HttpRequest *r );
bool sendPageQA ( TcpSocket *sock , HttpRequest *hr ) ;

67
Parms.cpp

@ -24,7 +24,6 @@
#include "Process.h"
#include "Repair.h"
#include "PingServer.h"
#include "Users.h"
#include "Proxy.h"
#include "hash.h"
#include "Test.h"
@ -2031,20 +2030,6 @@ bool Parms::printParm ( SafeBuf* sb,
bool isCollAdmin ,
TcpSocket *sock ) {
bool status = true;
// do not print if no permissions
//if ( m->m_perms != 0 && !g_users.hasPermission(username,m->m_perms) )
// return status;
//if ( m->m_perms != 0 && (m->m_perms & user) == 0 ) return status;
// do not print some if #define _CLIENT_ is true
//#ifdef _GLOBALSPEC_
//if ( m->m_priv == 2 ) return status;
//if ( m->m_priv == 3 ) return status;
//#elif _CLIENT_
//if ( m->m_priv ) return status;
//#elif _METALINCS_
//if ( m->m_priv == 2 ) return status;
//if ( m->m_priv == 3 ) return status;
//#endif
// priv of 4 means do not print at all
if ( m->m_priv == 4 ) return true;
@ -10711,28 +10696,6 @@ void Parms::init ( ) {
m->m_obj = OBJ_CONF;
m++;
/*
m->m_title = "send email alerts to zak";
m->m_desc = "Sends to zak@gigablast.com.";
m->m_cgi = "seatz";
m->m_off = (char *)&g_conf.m_sendEmailAlertsToZak - g;
m->m_type = TYPE_BOOL;
m->m_def = "0";
m->m_priv = 2;
m->m_group = 0;
m++;
m->m_title = "send email alerts to sabino";
m->m_desc = "Sends to cell phone.";
m->m_cgi = "seatms";
m->m_off = (char *)&g_conf.m_sendEmailAlertsToSabino - g;
m->m_type = TYPE_BOOL;
m->m_def = "0";
m->m_priv = 2;
m->m_group = 0;
m++;
*/
m->m_title = "dead host timeout";
m->m_desc = "Consider a host in the Gigablast network to be dead if "
"it does not respond to successive pings for this number of "
@ -18871,36 +18834,6 @@ void Parms::init ( ) {
m++;
*/
/*
m->m_title = "Users";
m->m_desc = "Add users here. The format is "
"collection:ip:username:password:relogin:pages:tagnames"
" Username and password cannot be blank."
" You can specify "
"* for collection to indicate all collections. "
" * can be used in IP as wildcard. "
" * in pages means user has access to all pages. Also"
" you can specify individual pages. A \'-\' sign at the"
" start of page means user is not allowed to access that"
" page. Please refer the page reference table at the bottom "
"of this page for available pages. If you want to just login "
" once and avoid relogin for gb shutdowns then set relogin=1,"
" else set it to 0. If relogin is 1 your login will never expire either."
"<br>"
" Ex: 1. master user -> *:*:master:master:1:*:english<br>"
" 2. public user -> *:*:public:1234:0:index.html"
",get,search,login,dir:english<br>";
m->m_cgi = "users";
m->m_xml = "users";
m->m_off = (char *)&g_conf.m_users - g;
m->m_type = TYPE_STRINGBOX;
m->m_perms = PAGE_MASTER;
m->m_size = USERS_TEXT_SIZE;
m->m_plen = (char *)&g_conf.m_usersLen - g;
m->m_page = PAGE_MASTERPASSWORDS;
m++;
*/
/*
m->m_title = "Master IPs";
m->m_desc = "If someone connects from one of these IPs "

732
PingServer.cpp

@ -37,7 +37,6 @@ char s_kernelRingBuf[4097];
int32_t s_kernelRingBufLen = 0;
static void sleepWrapper ( int fd , void *state ) ;
//static void sleepWrapper10 ( int fd , void *state );
static void checkKernelErrors( int fd, void *state );
static void gotReplyWrapperP ( void *state , UdpSlot *slot ) ;
static void handleRequest11 ( UdpSlot *slot , int32_t niceness ) ;
@ -1572,58 +1571,10 @@ bool PingServer::sendEmail ( Host *h ,
// the host is still dead.
if ( delay && h && sendToAdmin ) {
/*
int32_t deaHr,deaMin,debHr,debMin;
char hr[3],min[3];
hr[2] = '\0';
min[2] = '\0';
gbmemcpy ( hr, g_conf.m_delayEmailsAfter, 2 );
deaHr = atoi(hr);
gbmemcpy ( min, g_conf.m_delayEmailsAfter + 3, 2 );
deaMin = atoi(min);
gbmemcpy ( hr, g_conf.m_delayEmailsBefore, 2 );
debHr = atoi(hr);
gbmemcpy ( min, g_conf.m_delayEmailsBefore + 3, 2 );
debMin = atoi(min);
//get the current time. use getTime() because
//then it is sync'ed with host 0's time.
time_t rawTime = getTime();
struct tm *timeInfo;
//timeInfo stores the lasthour time
timeInfo = localtime ( &rawTime );
char buf[64];
strftime ( buf , 100 , "%b %d %T", timeInfo);
//log ( LOG_WARN,"net: pingserver: local time is %s when "
// "host died",buf );
//tm struct has hours from 0-23
int32_t tmHr = timeInfo->tm_hour;
int32_t tmMin = timeInfo->tm_min;
bool delay = false;
if ( ( tmHr > deaHr || ( tmHr == deaHr && tmMin > deaMin ) ) &&
( tmHr < debHr || ( tmHr == debHr && tmMin < debMin ) ) )
delay = true;
else if ( ( tmHr > deaHr ||
( tmHr == deaHr && tmMin > deaMin ) ) &&
( debHr < deaHr ||
( debHr == deaHr && debMin < deaMin ) ) )
delay = true;
else if ( ( tmHr < debHr ||
( tmHr == debHr && tmMin < debMin ) ) &&
( deaHr > debHr ||
( deaHr == debHr && deaMin > debMin ) ) )
delay = true;
*/
// always delay no matter the time now
bool delay = true;
if ( delay ) {
//check if the hosts twins are dead too
int32_t numTwins = 0;
Host *hosts = g_hostdb.getShard( h->m_shardNum,
@ -1638,146 +1589,12 @@ bool PingServer::sendEmail ( Host *h ,
//if no twin is alive, emergency ! send email !
//if even one twin is alive, don't send now
if ( i == numTwins ) goto skipSleep;
/*
time_t rawTime2 = getTime();
struct tm *timeInfo2;
//timeInfo stores the lasthour UTC time
timeInfo2=localtime(&rawTime2);
//change the time to delayemailsBefore
timeInfo2->tm_hour = debHr;
timeInfo2->tm_min = debMin;
rawTime2 = mktime ( timeInfo2 );
double diffTime = difftime ( rawTime2 , rawTime );
// if difftime is neg, that means we were yesterday,
// so add 1 day to get real diff
if (diffTime < 0 )
diffTime = (60 * 60 * 24 ) + diffTime;
log( LOG_WARN,"net: pingserver: Delay non critical "
"email alerts on. Trying to send email after %"INT32" "
"seconds",(int32_t) diffTime );
int32_t wait = (int32_t)diffTime * 1000 ;//ms
// wake up after so many seconds
g_loop.registerSleepCallback( wait, h,
emailSleepWrapper );
*/
return true;
}
}
skipSleep:
// matt tmobile
//if ( g_conf.m_sendEmailAlertsToMattTmobile ) {
// m_numRequests2++;
// if ( ! pageTMobile ( h, errmsg))
// status = false;
//}
// matt alltel
/*
if ( g_conf.m_sendEmailAlertsToMattAlltell ) {
m_numRequests2++;
if ( ! sendAdminEmail ( h,
"sysadmin@gigablast.com",
"5054503518@message.alltel.com",
errmsg ,
"mms-mta.alltel.net" ))
status = false;
}
*/
// matt alltel
// if ( g_conf.m_sendEmailAlertsToMattAlltell ) {
// m_numRequests2++;
// //if ( ! pageAlltel ( h, errmsg , "5053626809" ) )
// if ( ! pageAlltel ( h, errmsg , "5054503518" ) )
// status = false;
// }
// matt verizon
//if ( g_conf.m_sendEmailAlertsToMattVerizon ) {
// m_numRequests2++;
// if ( ! pageVerizon ( h, errmsg ) )
// status = false;
//}
// melissa
// if ( g_conf.m_sendEmailAlertsToMelissa ) {
// m_numRequests2++;
// if ( ! pageAlltel ( h, errmsg , "5054292121" ) ) // melissa
// status = false;
// }
// partap
/*
if ( g_conf.m_sendEmailAlertsToPartap ) {
m_numRequests2++;
//if ( ! pageSprintPCS ( h, errmsg , "5056889554") )
// status = false;
if ( ! sendAdminEmail ( h,
"sysadmin@gigablast.com",
"5056889554@mx.messaging.sprintpcs.com",
errmsg ,
"mx.messaging.sprintpcs.com") )
status = false;
}
// javier
if ( g_conf.m_sendEmailAlertsToJavier ) {
m_numRequests2++;
//if ( ! pageSprintPCS ( h, errmsg , "5056959513") )
// status = false;
if ( ! sendAdminEmail ( h,
"sysadmin@gigablast.com",
"5056959513@mx.messaging.sprintpcs.com",
errmsg ,
"mx.messaging.sprintpcs.com") )
status = false;
}
// cinco
// if ( g_conf.m_sendEmailAlertsToCinco ) {
// m_numRequests2++;
// //if ( ! pageSprintPCS ( h, errmsg , "5054179933") )
// // status = false;
// if ( ! sendAdminEmail ( h,
// "sysadmin@gigablast.com",
// "5054179933@mx.messaging.sprintpcs.com",
// errmsg ) )
// status = false;
// }
// zak
if ( g_conf.m_sendEmailAlertsToZak) {
m_numRequests2++;
if ( ! sendAdminEmail ( h,
"zak@gigablast.com",
"zak@gigablast.com",
errmsg ,
"mail.gigablast.com" ) )
status = false;
m_numRequests2++;
if ( ! sendAdminEmail ( h,
"sysadmin@gigablast.com",
"5052204556@message.alltel.com",
errmsg ,
"mms-mta.alltel.net" ))
status = false;
}
if ( g_conf.m_sendEmailAlertsToSabino ) {
m_numRequests2++;
//if ( ! pageSprintPCS ( h, errmsg , "5056959513") )
// status = false;
if ( ! sendAdminEmail ( h,
"sysadmin@gigablast.com",
"7082076550@mobile.mycingular.com",
errmsg ,
"mx.mycingular.net" ) )
status = false;
}
*/
bool e1 = g_conf.m_sendEmailAlertsToEmail1;
bool e2 = g_conf.m_sendEmailAlertsToEmail2;
@ -1874,429 +1691,6 @@ void emailSleepWrapper ( int fd, void *state ){
#include "HttpServer.h"
static void gotDocWrapper ( void *state , TcpSocket *ts ) ;
// JAB: warning abatement
#if 0
// . returns false if blocks, true otherwise
// . sets g_errno on error
bool pageTMobile ( Host *h , char *errmsg) {
// log("EMAIL SENT %s!!!!!!!!!!!!!!!",errmsg);
// return true;
// for debug without actually wasting our cell phone instant messages
//log("EMAIL SENT hid=%"INT32"!!!!!!!!!!!!!!!",h->m_hostId);
//gotDocWrapper ( h , NULL );
//return true;
// looks like TcpServer won't let us use a static sendBuf
// cuz we cannot just set TcpSocket->m_sendBuf to NULL to stop
// the freeing cuz if TcpServer::gotIp() has an error it
// frees it sendBuf. This happens in other places too.
void * state = h;
char *buf = (char *) mmalloc ( PAGER_BUF_SIZE , "PingServer" );
// bail on malloc error
if ( ! buf ) {
log("net: Could not allocate %"INT32" bytes to send email "
"to mobile." , (int32_t)PAGER_BUF_SIZE);
g_pingServer.m_numReplies++;
return true;
}
char *p = buf;
char *pend = buf + PAGER_BUF_SIZE;
// send the request
sprintf ( p ,
//"POST /mytmobile/login/default.asp?"
//"POST /messaging/default.asp?"
"POST /customer_site/jsp/messaging_lo.jsp?"
"To=5054503518 HTTP/1.0\r\n"
"Accept: image/gif, image/x-xbitmap, image/jpeg, "
"image/pjpeg, application/x-shockwave-flash, "
"application/msword, */*\r\n"
"Accept-Language: en-us\r\n"
"Content-Type: application/x-www-form-urlencoded\r\n"
"Accept-Encoding: gzip, deflate\r\n"
"User-Agent: Mozilla/4.0 "
"(compatible; MSIE 6.0; Windows 98; Win 9x 4.90)\r\n"
//"Host: www.t-mobile.com\r\n"
"Host: wmg.tmomail.net\r\n"
"Content-Length: xxx\r\n"
//"Connection: Keep-Alive\r\n"
"Connection: close\r\n"
"Cache-Control: no-cache\r\n\r\n"
//"txtNum=5054503518&txtFrom=gb&"
"DOMAIN_NAME=@tmomail.com&"
"min=5054503518&"
"require_sender=gb&"
//"hdnpublic=1&"
"msgTermsUse=on&"
//"txtMessage=");
//"trackResponses=No&"
//"Send.x=Yes&"
"text=");
p += gbstrlen ( p );
// append the err msg, but convert spaces to +'s
int32_t i;
for ( i = 0 ; errmsg[i] && p + 4 < pend ; i++ ) {
if ( errmsg[i] == ' ' )
*p++ = '+';
else if ( errmsg[i] == '.' ) {
p[0] = '%';
p[1] = '2';
p[2] = 'E';
p += 3;
}
else
*p++ = errmsg[i];
}
*p = '\0';
int32_t bufLen = p - buf;
// replace xxx
char *s = strstr ( buf , "xxx" );
char *t = strstr ( buf , "\r\n\r\n" ) + 4;
int32_t clen = bufLen - ( t - buf );
sprintf ( s , "%3li" , clen );
s [ 3 ] = '\r';
// post it up
// borrow our HttpServer's TcpServer
//TcpServer *ts = g_httpServer.getTcp();
TcpServer *ts = g_httpServer.getSSLTcp();
if ( ! ts ) {
log("db: Could not page alltel because "
"no gb.pem file in working dir so "
"SSL server is unavailable.");
return true;
}
if ( ! ts->sendMsg ( //"65.161.188.152" ,
//gbstrlen("65.161.188.152") ,
"63.122.5.193",
gbstrlen("63.122.5.193"),
443 , // https, 80 ,
buf ,
PAGER_BUF_SIZE ,
bufLen ,
bufLen ,
(void *)h ,
gotDocWrapper ,
60*1000 ,
100*1024 , // maxTextDocLen
100*1024 )) // maxOtherDocLen
return false;
// we did not block, so update h->m_emailCode
gotDocWrapper (state, NULL );
// we did not block
return true;
}
#endif
// JAB: warning abatement
#if 0
// . returns false if blocks, true otherwise
// . sets g_errno on error
bool pageAlltel ( Host *h , char *errmsg , char *num ) {
// for debug without actually wasting our cell phone instant messages
//log("EMAIL SENT hid=%"INT32"!!!!!!!!!!!!!!!",h->m_hostId);
//gotDocWrapper ( h , NULL );
//return true;
// looks like TcpServer won't let us use a static sendBuf
// cuz we cannot just set TcpSocket->m_sendBuf to NULL to stop
// the freeing cuz if TcpServer::gotIp() has an error it
// frees it sendBuf. This happens in other places too.
void * state = h;
char *buf = (char *) mmalloc ( PAGER_BUF_SIZE , "PingServer" );
// bail on malloc error
if ( ! buf ) {
log("net: Could not allocate %"INT32" bytes to send email "
"to alltel." , PAGER_BUF_SIZE);
g_pingServer.m_numReplies++;
return true;
}
char *p = buf;
char *pend = buf + PAGER_BUF_SIZE;
// send the request
char *host = "message.alltel.com";
char *ip = "166.102.172.2"; // message.alltel.com
//char *num = "5054292121"; // melissa
sprintf ( p ,
//"POST /mytmobile/login/default.asp?"
"POST /customer_site/jsp/messaging.jsp"
" HTTP/1.0\r\n"
"Accept: image/gif, image/x-xbitmap, image/jpeg, "
"image/pjpeg, application/x-shockwave-flash, "
"application/msword, */*\r\n"
"Accept-Language: en-us\r\n"
"Content-Type: application/x-www-form-urlencoded\r\n"
"Accept-Encoding: gzip, deflate\r\n"
"User-Agent: Mozilla/4.0 "
"(compatible; MSIE 6.0; Windows 98; Win 9x 4.90)\r\n"
"Host: %s\r\n" // www.t-mobile.com
"Content-Length: xxx\r\n"
//"Connection: Keep-Alive\r\n"
"Connection: close\r\n"
"Cache-Control: no-cache\r\n\r\n"
// post data
"trackResponses=No&"
"devicetype=1&"
"DOMAIN_NAME=@vtext.com&"
"sender=gb&"
"min=%s&" // phone number
"callback=&"
"count=160&"
"type=1&" // 1=high priority, 0=normal
"text=" ,
host , num );
p += gbstrlen ( p );
// append the err msg, but convert spaces to +'s
int32_t i;
for ( i = 0 ; errmsg[i] && p + 4 < pend ; i++ ) {
if ( errmsg[i] == ' ' )
*p++ = '+';
else if ( errmsg[i] == '.' ) {
p[0] = '%';
p[1] = '2';
p[2] = 'E';
p += 3;
}
else
*p++ = errmsg[i];
}
*p = '\0';
int32_t bufLen = p - buf;
// replace xxx
char *s = strstr ( buf , "xxx" );
char *t = strstr ( buf , "\r\n\r\n" ) + 4;
int32_t clen = bufLen - ( t - buf );
sprintf ( s , "%3li" , clen );
s [ 3 ] = '\r';
// post it up
// borrow our HttpServer's TcpServer
TcpServer *ts = g_httpServer.getTcp();
if ( ! ts->sendMsg ( ip ,
gbstrlen(ip) ,
80 ,
buf ,
PAGER_BUF_SIZE ,
bufLen ,
bufLen ,
state ,
gotDocWrapper ,
60*1000 ,
100*1024 , // maxTextDocLen
100*1024 )) // maxOtherDocLen
return false;
// we did not block, so update h->m_emailCode
gotDocWrapper ( state , NULL );
// we did not block
return true;
}
#endif
// JAB: warning abatement
#if 0
// . returns false if blocks, true otherwise
// . sets g_errno on error
bool pageSprintPCS ( Host *h , char *errmsg , char *num) {
char *host = "messaging.sprintpcs.com";
//char *num = "2813004108"; // partap
// for debug without actually wasting our cell phone instant messages
//log("EMAIL SENT hid=%"INT32"!!!!!!!!!!!!!!!",h->m_hostId);
//gotDocWrapper ( h , NULL );
//return true;
// looks like TcpServer won't let us use a static sendBuf
// cuz we cannot just set TcpSocket->m_sendBuf to NULL to stop
// the freeing cuz if TcpServer::gotIp() has an error it
// frees it sendBuf. This happens in other places too.
void * state = h;
char *buf = (char *) mmalloc ( PAGER_BUF_SIZE , "PingServer" );
// bail on malloc error
if ( ! buf ) {
log("net: Could not allocate %"INT32" bytes to send email "
"to sprint pcs." , PAGER_BUF_SIZE);
g_pingServer.m_numReplies++;
return true;
}
char *p = buf;
char *pend = buf + PAGER_BUF_SIZE;
// save ptr to host or email type
*(Host **)(p+PAGER_BUF_SIZE-4) = (Host*)state;
// then the message to send
sprintf ( p ,
//"POST /mytmobile/login/default.asp?"
"POST /textmessaging/composeconfirm"
//;jsessionid=CSRtp1YQo3WyzNL0edNEHsI4oN2IloYE9buvqYaO
//Pgijz4Ht1WUO!1942930617!182751426!5070!7002"
" HTTP/1.0\r\n"
"Accept: image/gif, image/x-xbitmap, image/jpeg, "
"image/pjpeg, application/x-shockwave-flash, "
"application/msword, */*\r\n"
"Accept-Language: en-us\r\n"
"Content-Type: application/x-www-form-urlencoded\r\n"
"Accept-Encoding: gzip, deflate\r\n"
"User-Agent: Mozilla/4.0 "
"(compatible; MSIE 6.0; Windows 98; Win 9x 4.90)\r\n"
"Host: %s\r\n" // www.t-mobile.com
"Content-Length: xxx\r\n"
//"Connection: Keep-Alive\r\n"
"Connection: close\r\n"
"Cookie: \r\n"
"Cache-Control: no-cache\r\n\r\n"
// post data
"randomToken=&"
"phoneNumber=%s&" // phone number
//"characters=%"INT32"&"
//"characters=160&"
//"callBackNumber=&"
"message=" ,
host , num );//, (int32_t)160-gbstrlen(errmsg) );
p += gbstrlen ( p );
// append the err msg, but convert spaces to +'s
int32_t i;
for ( i = 0 ; errmsg[i] && p + 4 < pend ; i++ ) {
if ( errmsg[i] == ' ' )
*p++ = '+';
else if ( errmsg[i] == '.' ) {
p[0] = '%';
p[1] = '2';
p[2] = 'E';
p += 3;
}
else
*p++ = errmsg[i];
}
*p = '\0';
//int32_t bufLen = p - buf;
// gotta get the cookie
Url url;
//char *uu = "http://65.174.43.73/textmessaging/compose";
char *uu = "http://144.230.162.49/textmessaging/compose";
//char *uu = "http://messaging.sprintpcs.com/textmessaging/compose";
url.set(uu,gbstrlen(uu));
if ( ! g_httpServer.getDoc ( &url ,
0 , // offset
-1 , // size
false , // m_ifModifiedSince
buf , // state
sprintPCSWrapper , //
30*1000 , // timeout
0 , // m_proxyIp
0 , // m_proxyPort
100*1024 , // m_maxTextDocLen
100*1024 , // m_maxOtherDocLen
"Mozilla/4.0 " // user agent
"(compatible; MSIE 6.0; Windows 98; "
"Win 9x 4.90)\r\n" ) )
return false;
// must have been an error
log("net: Got error getting page from Sprint PCS: %s.",
mstrerror(g_errno));
// always call this at the end
return pageSprintPCS2 ( buf , NULL );
}
#endif
// JAB: warning abatement
#if 0
void sprintPCSWrapper ( void *state , TcpSocket *ts) {
pageSprintPCS2 ( state , ts );
}
#endif
// JAB: warning abatement
#if 0
bool pageSprintPCS2 ( void *state , TcpSocket *s) {
char *ip = "65.174.43.73";
char *buf = (char *)state;
int32_t bufLen = gbstrlen(buf);
char *p = buf;
char *pend = p + bufLen;
// last 4 bytes of buffer is the Host ptr
void* state2 = (p+PAGER_BUF_SIZE-4);
if ( g_errno || ! s ) {
mfree ( buf , PAGER_BUF_SIZE , "PingServer" );
// always call this at the end
gotDocWrapper( state2 , NULL );
return true;
}
// sprint debug msg
//log("net: Got reply from Sprint PCS:\n%s",s->m_readBuf);
// get the cookie
HttpMime mime;
mime.set ( s->m_readBuf , s->m_readOffset , NULL );
char *cookie = mime.getCookie ( );
int32_t cookieLen = mime.getCookieLen ();
bufLen += cookieLen;
if ( ! cookie ) {
log("net: No cookie to send to Sprint PCS");
g_errno = EBADREPLY;
mfree ( buf , PAGER_BUF_SIZE , "PingServer" );
// always call this at the end
gotDocWrapper( state2 , NULL );
return true;
}
// part the request to make room for cookie
char *ss = strstr ( p , "Cookie: ");
if ( ! ss ) { char *xx = NULL; *xx = 0; }
ss += 8; // points to right after the space
// insert the cookie
gbmemcpy ( ss+cookieLen , ss , pend - ss );
gbmemcpy ( ss , cookie , cookieLen );
pend += cookieLen;
// replace xxx
char *sx = strstr ( buf , "xxx" );
char *t = strstr ( buf , "\r\n\r\n" ) + 4;
int32_t clen = bufLen - ( t - buf );
sprintf ( sx , "%3li" , clen );
sx [ 3 ] = '\r';
// sprint debug msg
//log("net: Sending to Sprint PCS:\n%s",buf);
// post it up
// borrow our HttpServer's TcpServer
TcpServer *ts = g_httpServer.getTcp();
if ( ! ts->sendMsg ( ip ,
gbstrlen(ip) ,
80 ,
buf ,
PAGER_BUF_SIZE ,
gbstrlen(buf) ,
gbstrlen(buf) ,
state2 ,
gotDocWrapper ,
60*1000 ,
100*1024 , // maxTextDocLen
100*1024 )) // maxOtherDocLen
return false;
// we did not block, so update h->m_emailCode
gotDocWrapper ( state2 , NULL );
// we did not block
return true;
}
#endif
// a bit of a hack for monitor.cpp
//int32_t g_emailMX1IPBackUp = 0;
bool sendAdminEmail ( Host *h,
char *fromAddress,
char *toAddress,
@ -2548,29 +1942,6 @@ static int32_t s_nextTapHostId ;
static void tapLoop ( ) ;
static void gotTapReplyWrapper ( void *state , UdpSlot *slot ) ;
// this wrapper is called once every second, or so...
void sleepWrapper10 ( int fd , void *state ) {
// can't do much if still waiting on a very old tap!
if ( s_outstandingTaps ) {
log("PingServer::sleepWrapper10: stuck tap. sync messed up.");
return;
}
// request is 9 bytes to distinguish from 8-byte requests
int64_t stamp = gettimeofdayInMilliseconds();
// . keep incrementing this if already used to avoid repeat stamps
// . should be extremely rare
// . even with this we can still have repeat stamps if we are not
// in sync, so i check for that in Sync::getSyncPoint()
// . -1 is reserved for meaning no stamp (see Msg0.h)
//while ( g_sync.hasStamp ( stamp ) || stamp == -1 ) stamp++;
// save it for distributing
*(int64_t *) s_lastSyncPoint = stamp;
// reset loop parms
s_nextTapHostId = 0;
// do the tap loop
tapLoop ( );
}
void tapLoop ( ) {
// . don't use more than 16 UdpSlots for tapping
// . don't use more than numHosts UdpSlots for tapping
@ -2657,74 +2028,6 @@ void PingServer::tapHost ( int32_t hostId ) {
g_errno = 0;
}
// . this should be called on reception of an ACK to keep track of ping time.
// . this should be called on time out of a dgram reception.
// . "tripTime" is in milliseconds
// . "tripTime" should be the timeOut time if "timedOut" is true
// . returns false if you have to wait for your callback to be called
// . returns true if you don't have to wait... stamp happened immediately
// . sets g_errno on any error, regardless of whether true or false was returnd
/*
void PingServer::stampHost ( int32_t hostId , int32_t tripTime , bool timedOut ) {
// hostId of -1 means unknown, so we can't stamp it
if ( hostId == -1 ) return;
if ( hostId >= g_hostdb.getNumHosts() ) return;
// ensure tripTime doesn't wrap around
if ( tripTime < 0 ) return; // tripTime = 0x7fffffff;
// sanity check
if ( tripTime > 65000 ) {
log("PingServer::stampHost: bad tripTime");
}
// get the host, return false on error
Host *h = m_hostPtrs[hostId];
// if he was considered timedout, only stamp if it makes his time worse
if ( timedOut && tripTime < h->m_pingAvg ) return;
// TODO: do we need this?
// . don't update this host if this new ping time is < 10% different
// than the ping time we have recorded now
// int32_t diff = newPing - h->m_pingAvg;
// if ( diff < 0 ) diff *= -1;
// if ( ( 100 * diff ) / h->m_pingAvg < 10 ) return true;
// . we keep track of the last 4 pings for each host
// . we use the avg ping time to select fast hosts for querying
// . we use the std. dev. to know when to re-send datagrams
// shift the 5-ping window and insert the new ping
for ( int32_t i = 0 ; i < 3 ; i++ ) h->m_pings[i] = h->m_pings[i+1];
h->m_pings[3] = tripTime; // our latest ping
// compute the avg
int32_t avg = 0;
for ( int32_t i = 0 ; i < 4 ; i++ ) avg += h->m_pings[i];
avg /= 4;
h->m_pingAvg = avg;
// compute the std dev
int32_t stdDev = 0;
for ( int32_t i = 0 ; i < 4 ; i++ ) {
int32_t diff = ( h->m_pings[i] - avg );
if ( diff < 0 ) stdDev -= diff;
else stdDev += diff;
}
stdDev /= 4;
h->m_pingStdDev = stdDev;
}
*/
/*
void PingServer::getTimes ( int32_t hostId , int32_t *avg , int32_t *stdDev ) {
// make defaults (right now for dns server only)
*avg = 5000; // milliseconds = 5 seconds
*stdDev = 500;
// hostId of -1 means unknown, so we can't stamp it
if ( hostId == -1 ) return;
if ( hostId >= g_hostdb.getNumHosts() ) return;
// get the host, return false on error
Host *h = m_hostPtrs[hostId];
*avg = h->m_pingAvg;
*stdDev = h->m_pingStdDev;
return;
}
*/
// if its status changes from dead to alive or vice versa, we have to
// update g_hostdb.m_numHostsAlive. Dns.cpp and Msg17 will use this count
void updatePingTime ( Host *h , int32_t *pingPtr , int32_t tripTime ) {
@ -2763,33 +2066,16 @@ void updatePingTime ( Host *h , int32_t *pingPtr , int32_t tripTime ) {
// force it out of sync
if ( isDead ) h->m_inSync = false;
//if ( ! wasDead && isDead )
// log("hey");
//logf(LOG_DEBUG,"ping: hostid %"INT32" wasDead=%"INT32" isDead=%"INT32" "
// "numAlive=%"INT32"",
// (int32_t)h->m_hostId,(int32_t)wasDead,(int32_t)isDead,
// (int32_t)g_hostdb.m_numHostsAlive);
if( ! h->m_isProxy ) {
// maintain m_numHostsAlive if there was a change in state
if ( wasDead && ! isDead ) g_hostdb.m_numHostsAlive++;
if ( ! wasDead && isDead ) g_hostdb.m_numHostsAlive--;
if( h->m_isProxy ) {
// maintain m_numProxyAlive if there was a change in state
//if ( wasDead && ! isDead ) g_hostdb.m_numProxyAlive++;
//if ( ! wasDead && isDead ) g_hostdb.m_numProxyAlive--;
// sanity check, this should be at least 1 since we are alive
//if ( g_hostdb.m_numProxyAlive < 0 ||
// g_hostdb.m_numProxyAlive > g_hostdb.getNumProxy() ) {
// char *xx = NULL; *xx =0; }
}
else {
// maintain m_numHostsAlive if there was a change in state
if ( wasDead && ! isDead ) g_hostdb.m_numHostsAlive++;
if ( ! wasDead && isDead ) g_hostdb.m_numHostsAlive--;
// sanity check, this should be at least 1 since we are alive
if ( g_hostdb.m_numHostsAlive < 0 ||
g_hostdb.m_numHostsAlive > g_hostdb.m_numHosts ) {
char *xx = NULL; *xx =0; }
}
// sanity check, this should be at least 1 since we are alive
if ( g_hostdb.m_numHostsAlive < 0 ||
g_hostdb.m_numHostsAlive > g_hostdb.m_numHosts ) {
char *xx = NULL; *xx =0; }
}
}
void checkKernelErrors( int fd, void *state ){

7
Process.cpp

@ -38,7 +38,6 @@
//#include "Placedb.h"
#include "Wiki.h"
#include "Wiktionary.h"
#include "Users.h"
#include "Proxy.h"
#include "Rebalance.h"
#include "SpiderProxy.h"
@ -1705,9 +1704,6 @@ bool Process::saveBlockingFiles1 ( ) {
// save our place during a rebalance
g_rebalance.saveRebalanceFile();
// save the login table
g_users.save();
// save stats on spider proxies if any
saveSpiderProxyStats();
@ -1889,9 +1885,6 @@ void Process::resetAll ( ) {
// query log buffer
g_qbuf.reset();
g_profiler.reset();
g_testResultsTree.reset();
g_users.m_ht.reset();
g_users.m_loginTable.reset();
resetAddressTables();
resetMsg13Caches();
resetStopWordTables();

501
Profiler.cpp

@ -20,7 +20,6 @@ Profiler g_profiler;
#include "Profiler.h"
#include "Stats.h"
#include "sort.h"
#include "Users.h"
Profiler g_profiler;
@ -762,384 +761,6 @@ bool Profiler::endTimer(int32_t address,
return true;
}
bool Profiler::printInfo(SafeBuf *sb,char *username, //int32_t user,
char *pwd, char *coll,
int sorts,int sort10, int qpreset,
int profilerreset) {
// sort by max blocked time by default
if ( sorts == 0 ) sorts = 8;
int32_t slot;
uint32_t key(0);
int32_t numSlots = m_fn.getNumSlots();
int32_t numSlotsUsed = m_fn.getNumSlotsUsed();
FnInfo *fnInfo;
if ( profilerreset ) {
for ( int32_t i = 0; i < m_fn.getNumSlots(); i++ ){
//key=m_fn.getKey(i);
//if (key!=0){
if ( ! m_fn.isEmpty(i) ) {
fnInfo=(FnInfo *)m_fn.getValueFromSlot(i);
// set everything to 0
fnInfo->m_timesCalled = 0;
fnInfo->m_totalTimeTaken = 0;
fnInfo->m_maxTimeTaken = 0;
fnInfo->m_numCalledFromThread = 0;
fnInfo->m_maxBlockedTime = 0;
}
}
}
sb->safePrintf( "<center>\n<table %s>\n"
"<tr class=hdrow><td colspan=9>"
"<center><b>Profiler "//- Since Startup</b></center>"
"<a href=\"/admin/profiler?c=%s"//"
"&profilerreset=1\">"
"(reset)</a></b></center>"
"</td></tr>\n",
TABLE_STYLE,
coll);
sb->safePrintf("<tr bgcolor=#%s>"
"<td><b>Address</b></td><td><b>Function</b></td>"
, LIGHT_BLUE);
sb->safePrintf("<td><b><a href=/admin/profiler?sorts=3&c=%s>"
"Times Called</a></b></td></td>",coll);
sb->safePrintf("<td><b><a href=/admin/profiler?sorts=4&c=%s>"
"Total Time(msec)</a></b></td></td>",coll);
sb->safePrintf("<td><b><a href=/admin/profiler?sorts=5&c=%s>"
"Avg Time(msec)</b></a></td>",coll);
sb->safePrintf("<td><b><a href=/admin/profiler?sorts=6&c=%s>"
"Max Time(msec)</a></b></td>",coll);
sb->safePrintf("<td><b><a href=/admin/profiler?sorts=7&c=%s>"
"Times from Thread</a></b></td>",coll);
sb->safePrintf("<td><b><a href=/admin/profiler?sorts=8&c=%s>"
"Max Blocked Time</a></b></td>",coll);
// sb->safePrintf("<td><b><a href=/admin/profiler?sorts=8&c=%s>"
// "Between Quick Polls</a></b></td></tr>",coll);
indexTable=(uint32_t*)
mcalloc(numSlotsUsed*sizeof(uint32_t),"ProfilerW");
keyTable=(uint32_t*) mcalloc
(numSlotsUsed*sizeof(uint32_t),"ProfilerX");
if(sorts==5 ||sort10==5)
valueTableF=(float*)
mcalloc(numSlotsUsed*sizeof(float),"ProfilerY");
else
valueTableUll=(uint64_t*)
mcalloc(numSlotsUsed*sizeof(uint64_t),
"ProfilerY");
int32_t numFnsCalled=0;
for (int32_t i=0;i<numSlots;i++){
//key=m_fn.getKey(i);
//if (key!=0){
if ( !m_fn.isEmpty(i) ) {
fnInfo=(FnInfo *)m_fn.getValueFromSlot(i);
// To save calculating time, just store functions
// that have been called
if(fnInfo->m_timesCalled!=0){
keyTable[numFnsCalled]=key;
indexTable[numFnsCalled]=numFnsCalled;
switch(sorts){
case 3:valueTableUll[numFnsCalled] = fnInfo->
m_timesCalled;
break;
case 4:valueTableUll[numFnsCalled] = fnInfo->
m_totalTimeTaken;
break;
case 5:// sorting float values till the 4th
// dec place
valueTableF[numFnsCalled] = ((float)fnInfo->m_totalTimeTaken)/((float)fnInfo->m_timesCalled);
break;
case 6:valueTableUll[numFnsCalled] = fnInfo->
m_maxTimeTaken;
break;
case 7:valueTableUll[numFnsCalled] = fnInfo->
m_numCalledFromThread;
break;
case 8:valueTableUll[numFnsCalled] = fnInfo->
m_maxBlockedTime;
break;
//For now for any other value of slot
// so that we don't error
default:valueTableUll[numFnsCalled] = fnInfo->
m_numCalledFromThread;
}
numFnsCalled++;
}
}
}
if (sorts==5)
gbqsort(indexTable,numFnsCalled,sizeof(uint32_t),
decend_cmpF);
else
gbqsort(indexTable,numFnsCalled,sizeof(uint32_t),
decend_cmpUll);
//Now print the sorted values
for (int32_t i=0;i<numFnsCalled;i++){
slot=m_fn.getSlot(&keyTable[indexTable[i]]);
fnInfo=(FnInfo *)m_fn.getValueFromSlot(slot);
//Don't print functions that have not been called
sb->safePrintf("<tr><td>%"XINT32"</td><td>%s</td><td>%"INT32"</td><td>%"INT32"</td>"
"<td>%.4f</td><td>%"INT32"</td><td>%"INT32"</td><td>%"INT32"</td>"
"</tr>",
keyTable[indexTable[i]],
fnInfo->m_fnName,
fnInfo->m_timesCalled,
fnInfo->m_totalTimeTaken,
((float)fnInfo->m_totalTimeTaken)/((float)fnInfo->m_timesCalled),
fnInfo->m_maxTimeTaken,
fnInfo->m_numCalledFromThread,
fnInfo->m_maxBlockedTime);
}
sb->safePrintf("</table><br><br>");
//Now to print the table of functions called in the last 10 seconds
sb->safePrintf( "<center>\n<table %s>\n"
"<tr class=hdrow><td colspan=8>"
"<center><b>Profiler - Last 10 seconds</b></center>"
"</td></tr>\n",TABLE_STYLE);
sb->safePrintf("<tr bgcolor=#%s>"
"<td><b>Address</b></td><td><b>Function</b></td>",
LIGHT_BLUE);
sb->safePrintf("<td><b><a href=/admin/profiler?sort10=3&c=%s&"
">"
"Times Called</a></b></td></td>",coll);
sb->safePrintf("<td><b><a href=/admin/profiler?sort10=4&c=%s&"
">"
"Total Time(msec)</a></b></td></td>",coll);
sb->safePrintf("<td><b><a href=/admin/profiler?sort10=5&c=%s&"
">"
"Avg Time(msec)</b></a></td>",coll);
sb->safePrintf("<td><b><a href=/admin/profiler?sort10=6&c=%s&"
">"
"Max Time(msec)</a></b></td>",coll);
sb->safePrintf("<td><b><a href=/admin/profiler?sort10=7&c=%s&"
">"
"Times From Thread</a></b></td></tr>",coll);
uint64_t now=gettimeofdayInMillisecondsLocal();
int32_t numFnsCalled10=0;;
for(int32_t i=0;i<numFnsCalled;i++){
uint64_t timesCalled=0;
uint64_t totalTimeTaken=0;
uint64_t maxTimeTaken=0;
uint64_t numCalledFromThread=0;
//If hashtable is less than 10 secs old, use it
for(int32_t j=0;j<11;j++){
if ((now-m_fnTime[i]) < 10000){
//From the keyTable, we know the keys of the
// functions that have been called
slot=m_fnTmp[j].getSlot(&keyTable[i]);
if (slot>=0){
fnInfo=(FnInfo *)m_fnTmp[j].
getValueFromSlot(slot);
totalTimeTaken += fnInfo->
m_totalTimeTaken;
timesCalled += fnInfo->m_timesCalled;
if( ( fnInfo->m_maxTimeTaken) >
maxTimeTaken )
maxTimeTaken = fnInfo->
m_maxTimeTaken;
numCalledFromThread += fnInfo->
m_numCalledFromThread;
}
}
}
//After getting all the info, put it in table for sorting
//Only print those functions that have been called
if (timesCalled==0) continue;
//Simply overwriting all the stuff
keyTable[numFnsCalled10]=keyTable[i];
indexTable[numFnsCalled10]=numFnsCalled10;
switch(sort10){
case 0:break;
case 3:valueTableUll[numFnsCalled10]=timesCalled;
break;
case 4:valueTableUll[numFnsCalled10]=totalTimeTaken;;
break;
case 5:valueTableF[numFnsCalled10]=((float)totalTimeTaken)/((float)timesCalled);
break;
case 6:valueTableUll[numFnsCalled10]=maxTimeTaken;
break;
//For now for any other value of slot so that we don't error
default:valueTableUll[numFnsCalled10]=numCalledFromThread;
}
numFnsCalled10++;
}
if (sort10==5)
gbqsort(indexTable,numFnsCalled10,sizeof(uint32_t),
decend_cmpF);
else
gbqsort(indexTable,numFnsCalled10,sizeof(uint32_t),
decend_cmpUll);
for(int32_t i=0;i<numFnsCalled10;i++){
uint64_t timesCalled=0;
uint64_t totalTimeTaken=0;
uint64_t maxTimeTaken=0;
uint64_t numCalledFromThread=0;
//If hashtable is less than 10 secs old, continue
for(int32_t j=0;j<11;j++){
if ((now-m_fnTime[i])<10000){
// From the keyTable, we know the keys of the
// functions that have been called
slot=m_fnTmp[j].
getSlot(&keyTable[indexTable[i]]);
if (slot>=0){
fnInfo=(FnInfo *)m_fnTmp[j].
getValueFromSlot(slot);
totalTimeTaken += fnInfo->
m_totalTimeTaken;
timesCalled += fnInfo->m_timesCalled;
if( ( fnInfo->m_maxTimeTaken)>
maxTimeTaken )
maxTimeTaken = fnInfo->
m_maxTimeTaken;
numCalledFromThread += fnInfo->
m_numCalledFromThread;
}
}
}
//Only print those functions that have been called
if (timesCalled==0) continue;
slot=m_fn.getSlot(&keyTable[indexTable[i]]);
fnInfo=(FnInfo *)m_fn.getValueFromSlot(slot);
//Don't print functions that have not been called
sb->safePrintf("<tr><td>%"XINT32"</td><td>%s</td><td>%"INT64"</td>"
"<td>%"INT64"</td>"
"<td>%.4f</td><td>%"INT64"</td><td>%"INT64"</td></tr>",
keyTable[indexTable[i]],
fnInfo->m_fnName,
timesCalled,
totalTimeTaken,
((float)totalTimeTaken)/((float)timesCalled),
maxTimeTaken,
numCalledFromThread);
}
sb->safePrintf("</table><br><br>");
mfree(indexTable,numSlotsUsed*sizeof(uint32_t),"ProfilerX");
mfree(keyTable,numSlotsUsed*sizeof(uint32_t),"ProfilerX");
if (sorts==5 || sort10==5)
mfree(valueTableF,
numSlotsUsed*sizeof(float),
"ProfilerY");
else
mfree(valueTableUll,
numSlotsUsed*sizeof(uint64_t),
"ProfilerY");
if(qpreset) {
m_quickpolls.clear();
m_lastQPUsed = 0;
}
numSlots = m_quickpolls.getNumSlots();
numSlotsUsed = m_quickpolls.getNumSlotsUsed();
sb->safePrintf("<center>\n<table %s>\n"
"<tr class=hdrow><td colspan=5>"
"<center><b>Triggered Quickpolls "
"<a href=\"/admin/profiler?c=%s"
"&qpreset=1\">"
"(reset)</a></b></center>"
"</td></tr>\n",
TABLE_STYLE,
coll);
sb->safePrintf("<tr bgcolor=#%s>"
"<td><b>Between Functions</b></td>"
"<td><b>max blocked(msec)</b></td>"
"<td><b>avg time(msec)</b></td>"
"<td><b>times triggered</b></td>"
"<td><b>total(msec)</b></td>"
"</tr>"
, LIGHT_BLUE );
if(numSlotsUsed == 0) {
sb->safePrintf("</table>");
return true;
}
valueTableUll = (uint64_t*)
mcalloc(numSlotsUsed * sizeof(uint64_t),"ProfilerZ");
if(!valueTableUll) {
sb->safePrintf("</table>");
return true;
}
indexTable = (uint32_t*)mcalloc(numSlotsUsed *
sizeof(uint32_t),
"ProfilerZ");
if(!indexTable) {
mfree(indexTable,
numSlotsUsed*sizeof(uint32_t),
"ProfilerZ");
sb->safePrintf("</table>");
return true;
}
keyTable = (uint32_t*)mcalloc(numSlotsUsed *
sizeof(uint32_t),
"ProfilerZ");
if(!keyTable) {
mfree(indexTable,
numSlotsUsed*sizeof(uint32_t),
"ProfilerZ");
mfree(valueTableUll,
numSlotsUsed*sizeof(uint64_t),
"ProfilerZ");
sb->safePrintf("</table>");
return true;
}
int32_t j = 0;
for (int32_t i = 0; i < numSlots; i++) {
//if((key = m_quickpolls.getKey(i)) == 0) continue;
if ( m_quickpolls.isEmpty(i) ) continue;
QuickPollInfo* q = *(QuickPollInfo **)m_quickpolls.getValueFromSlot(i);
int32_t took = q->m_maxTime;
valueTableUll[j] = took;
indexTable[j] = j;
keyTable[j] = i;
j++;
}
gbqsort(indexTable, j, sizeof(uint32_t), decend_cmpUll);
for (int32_t i = 0; i < numSlotsUsed; i++){
int32_t slot = keyTable[indexTable[i]];
//key = m_quickpolls.getKey(slot);
QuickPollInfo* q = *(QuickPollInfo **)m_quickpolls.getValueFromSlot(slot);
sb->safePrintf("<tr><td>%s:%"INT32"<br>%s:%"INT32"</td>"
"<td>%"INT32"</td>"
"<td>%f</td>"
"<td>%"INT32"</td>"
"<td>%"INT32"</td>"
"</tr>",
q->m_caller, q->m_lineno, q->m_last,
q->m_lastlineno,q->m_maxTime,
(float)q->m_timeAcc / q->m_times,
q->m_times,
q->m_timeAcc);
}
sb->safePrintf("</table>");
mfree(valueTableUll,numSlotsUsed*sizeof(uint64_t),"ProfilerZ");
mfree(indexTable, numSlotsUsed*sizeof(uint32_t),"ProfilerZ");
mfree(keyTable, numSlotsUsed*sizeof(uint32_t),"ProfilerZ");
return true;
}
//backwards so we get highest scores first.
static int decend_cmpUll ( const void *h1 , const void *h2 ) {
uint32_t tmp1, tmp2;
@ -1185,26 +806,14 @@ bool sendPageProfiler ( TcpSocket *s , HttpRequest *r ) {
SafeBuf sb;
sb.reserve2x(32768);
//read in all of the possible cgi parms off the bat:
//int32_t user = g_pages.getUserType( s , r );
char *username = g_users.getUsername(r);
//char *pwd = r->getString ("pwd");
char *coll = r->getString ("c");
int32_t collLen;
if ( ! coll || ! coll[0] ) {
//coll = g_conf.m_defaultColl;
coll = g_conf.getDefaultColl( r->getHost(), r->getHostLen() );
}
collLen = gbstrlen(coll);
int sorts=(int) r->getLong("sorts",0);
int sort10=(int)r->getLong("sort10",0);
int qpreset=(int)r->getLong("qpreset",0);
int profilerreset=(int)r->getLong("profilerreset",0);
int realTimeSortMode=(int)r->getLong("rtsort",2);
int realTimeShowAll=(int)r->getLong("rtall",0);
int startRt=(int)r->getLong("rtstart",0);
int stopRt=(int)r->getLong("rtstop",0);
@ -1213,47 +822,29 @@ bool sendPageProfiler ( TcpSocket *s , HttpRequest *r ) {
// no permmission?
bool isMasterAdmin = g_conf.isMasterAdmin ( s , r );
bool isCollAdmin = g_conf.isCollAdmin ( s , r );
if ( ! isMasterAdmin &&
! isCollAdmin ) {
//g_errno = ENOPERM;
//g_httpServer.sendErrorReply(s,g_errno,mstrerror(g_errno));
//return true;
sorts = 0;
sort10 = 0;
qpreset = 0;
profilerreset = 0;
realTimeSortMode = 2;
realTimeShowAll = 0;
if ( ! isMasterAdmin && ! isCollAdmin ) {
startRt = 0;
stopRt = 0;
}
if (!g_conf.m_profilingEnabled)
sb.safePrintf("<font color=#ff0000><b><centeR>"
if (!g_conf.m_profilingEnabled) {
sb.safePrintf("<font color=#ff0000><b><center>"
"Sorry, this feature is temporarily disabled. "
"Enable it in MasterControls.</center></b></font>");
else {
if(g_profiler.m_realTimeProfilerRunning) {
if(stopRt) {
} else {
if (g_profiler.m_realTimeProfilerRunning) {
if (stopRt) {
g_profiler.stopRealTimeProfiler(false);
g_profiler.m_ipBuf.purge();
}
} else if(startRt) g_profiler.startRealTimeProfiler();
} else if (startRt) {
g_profiler.startRealTimeProfiler();
}
g_profiler.printRealTimeInfo(&sb,
username,
NULL,
coll,
realTimeSortMode,
realTimeShowAll);
// g_profiler.printInfo(&sb,username,NULL,coll,sorts,sort10, qpreset,
// profilerreset);
g_profiler.printRealTimeInfo(&sb, coll);
}
return g_httpServer.sendDynamicPage ( s , (char*) sb.getBufStart() ,
sb.length() ,-1 , false);
return g_httpServer.sendDynamicPage ( s, (char*)sb.getBufStart(), sb.length(), -1, false);
}
FrameTrace *
@ -1732,13 +1323,7 @@ int cmpPathBucket (const void *A, const void *B) {
}
bool
Profiler::printRealTimeInfo(SafeBuf *sb,
//int32_t user,
char *username,
char *pwd,
char *coll,
int realTimeSortMode,
int realTimeShowAll) {
Profiler::printRealTimeInfo(SafeBuf *sb, char *coll) {
if(!m_realTimeProfilerRunning) {
sb->safePrintf("<table %s>",TABLE_STYLE);
sb->safePrintf("<tr class=hdrow><td colspan=7>"
@ -1753,42 +1338,8 @@ Profiler::printRealTimeInfo(SafeBuf *sb,
log(LOG_INIT, "admin: @@@@2 printRealTimeInfo: stopping profiler");
stopRealTimeProfiler(true);
/*
if(hitEntries)
mfree(hitEntries, sizeof(HitEntry)*rtNumEntries, "hitEntries");
//rtNumEntries = 0;
//for(int32_t i = 0; i < realTimeProfilerData.getNumSlots(); ++i) {
// uint32_t key = realTimeProfilerData.getKey(i);
// if(realTimeProfilerData.getValuePointer(key))
// ++rtNumEntries;
//}
rtNumEntries = realTimeProfilerData.getNumUsedSlots();
if(!rtNumEntries) {
sb->safePrintf("<table %s>",TABLE_STYLE);
sb->safePrintf("<tr class=hdrow><td colspan=7>"
"<center><b>Real Time Profiler started, refresh page "
"after some time."
"<a href=\"/admin/profiler?c=%s"
"&rtstop=1\">"
"(Stop)</a></b></center>"
"</td></tr>\n",coll);
sb->safePrintf("</table><br><br>\n");
startRealTimeProfiler();
return true;
}
*/
sb->safePrintf("<table %s>",TABLE_STYLE);
// char *showMessage;
// int rtall;
// if(realTimeShowAll) {
// showMessage = "(show only 10)";
// rtall = 0;
// } else {
// showMessage = "(show all)";
// rtall = 1;
// }
sb->safePrintf("<tr class=hdrow>"
"<td colspan=7>"
"<b>Top 100 Profiled Line Numbers "
@ -1807,32 +1358,6 @@ log(LOG_INIT, "admin: @@@@2 printRealTimeInfo: stopping profiler");
"</td></tr>\n"
//,coll
);
/*
rtall = !rtall;
sb->safePrintf("<tr><td><b>"
"Function</b></td>");
sb->safePrintf("<td><b><a href=/admin/profiler?rtsort=2&c=%s&"
"&rtall=%i>"
"Hits per Func</b></a></td>",coll,rtall);
sb->safePrintf("<td><b><a href=/admin/profiler?rtsort=0&c=%s&"
"&rtall=%i>"
"Missed QUICKPOLL calls<br>per Func</b></a></td>",
coll,rtall);
sb->safePrintf("<td><b><a href=/admin/profiler?rtsort=1&c=%s&"
"&rtall=%i>"
"Base Address</b></a></td>",coll,rtall);
sb->safePrintf("<td><b>Hits per Line</b></td>"
"<td><b>Line Address</b></td>"
"<td><b>Missed QUICKPOLL calls<br>"
"per Line</b></td></tr>");
*/
// system call to get the function names and line numbers
// just dump the buffer

17
Profiler.h

@ -175,22 +175,7 @@ class Profiler {
const char *caller,
bool isThread=false);
bool printInfo(SafeBuf *sb,
//int32_t user,
char *username,
char *pwd,
char *coll,
int sorts,
int sort10,
int qpreset,
int profilerreset);
bool printRealTimeInfo(SafeBuf *sb,
//int32_t user,
char *username,
char *pwd,
char *coll,
int realTimeSortMode,
int realTimeShowAll);
bool printRealTimeInfo(SafeBuf *sb, char *coll);
void resetLastQpoll() {m_lastQpoll = "main loop"; m_lastQpollLine =0;}
char *getFnName(PTRTYPE address,int32_t *nameLen=NULL);

3
SearchInput.cpp

@ -5,14 +5,11 @@
#include "Pages.h" // g_msg
#include "LanguageIdentifier.h"
#include "CountryCode.h"
#include "Users.h"
#include "PageResults.h"
#include "third-party/cld2/public/compact_lang_det.h"
#include "third-party/cld2/public/encodings.h"
//char getFormatFromRequest ( class HttpRequest *hr ) ;
SearchInput::SearchInput() {
reset();
}

2
SearchInput.h

@ -22,8 +22,6 @@
#define MAX_TOPIC_GROUPS 10
char getFormatFromRequest ( class HttpRequest *hr ) ;
// . parameters used to generate a set of related topics (gigabits)
// . you can have Msg24 generate multiple sets of related topics in one call
class TopicGroup {

7
Tagdb.cpp

@ -11,7 +11,6 @@
#include "Pages.h"
#include "SiteGetter.h"
#include "HashTableX.h"
#include "Users.h"
#include "Process.h"
#include "Rebalance.h"
@ -1071,7 +1070,6 @@ bool TagRec::setFromHttpRequest ( HttpRequest *r, TcpSocket *s ) {
//reset();
// get the username from the cookie
//char *user = r->getStringFromCookie ( "username" , NULL );
//char *user = g_users.getUsername ( r );
// try from form
//if ( ! user ) user = r->getString ("username",NULL);
// if no user, don't bother!
@ -4105,9 +4103,7 @@ bool sendPageTagdb ( TcpSocket *s , HttpRequest *req ) {
mnew ( st , sizeof(State12) , "PageTagdb" );
//st->m_isMasterAdmin = isAdmin;
//st->m_isAssassin = isAssassin;
// . Commented by Gourav
// . Reason:user perm no longer used
//st->m_userType = g_pages.getUserType ( s , req );
// assume we've nothing to add
st->m_adding = false;
// save the socket
@ -4158,7 +4154,6 @@ bool sendPageTagdb ( TcpSocket *s , HttpRequest *req ) {
// . just get from cookie so it is not broadcast over the web via a
// referral url
//st->m_username = r->getStringFromCookie("username");
//st->m_username = g_users.getUsername(r);
// are we coming from a local machine?
st->m_isLocal = r->isLocal();

897
Users.cpp

@ -1,897 +0,0 @@
#include "Users.h"
Users g_users;
RdbTree g_testResultsTree;
// intialize User members
User::User(){
m_permissions = 0;
m_numTagIds = 0;
m_numColls = 0;
m_allColls = false;
m_numIps = 0;
m_allIps = false;
m_numPages = 0;
m_allPages = 0;
m_reLogin = false;
m_username[0] = '\0';
m_password[0] = '\0';
}
// verify if user has permission from this ip
bool User::verifyIp ( int32_t ip ){
//
if ( m_allIps ) return true;
// check the iplist
for (uint16_t i=0; i < m_numIps; i++ ){
int32_t ipCheck = m_ip[i] & ( ip & m_ipMask[i] );
// check if they match
if ( ipCheck == m_ip[i] ) return true;
}
return false;
}
// verify user pass
bool User::verifyPassword ( char *pass ){
//
if ( ! pass ) return false;
if ( strcmp ( pass, m_password ) == 0 ) return true;
return false;
}
// verify is has access to this coll
bool User::verifyColl ( int32_t collNum ){
if ( m_allColls ) return true;
//
if ( collNum < 0 ) return false;
for ( uint16_t i=0; i < m_numColls; i++ )
if ( m_collNum[i] == collNum ) return true;
return false;
}
// verify if the user has the supplied tagId
bool User::verifyTagId ( int32_t tagId ){
//if ( tagId == 0 || tagId >= ST_LAST_TAG ) return false;
if ( tagId == 0 ) return false;
for ( uint16_t i = 0; i < m_numTagIds; i++ )
if ( m_tagId[i] == tagId ) return true;
return false;
}
// check if user is allowed to access this page
bool User::verifyPageNum ( uint16_t pageNum ){
if ( pageNum >= PAGE_NONE ) return false;
for ( uint16_t i = 0; i < m_numPages; i++ ){
bool allow = ! ( m_pages[i] & 0x8000 );
if ( pageNum == (m_pages[i] & 0x7fff) ) return allow;
}
// check if pageNum is of dummy page
bool isDummy = true;
//if ( pageNum > PAGE_PUBLIC )
isDummy = false;
//
if ( m_allPages && !isDummy )
return true;
return false;
}
// get first page
int32_t User::firstPage ( ){
// return first allowed page
for ( uint16_t i = 0; i < m_numPages; i++ )
if ( ! (m_pages[i] & 0x8000) ) //&&
// (m_pages[i]&0x7fff) > PAGE_PUBLIC )
return m_pages[i];
// if all pages is set then just return the root page
if ( m_allPages ) return PAGE_ROOT;
return -1;
}
// intialize the members
Users::Users(){
m_init = false;
m_needsSave = false;
}
Users::~Users(){
}
bool Users::save(){
return true;
if ( ! m_needsSave ) return true;
if ( ! m_loginTable.save(g_hostdb.m_dir,"userlogin.dat",NULL,0) )
return log("users: userlogin.dat save failed");
return true;
}
// initialize cache, tree, loadUsers and also loadTestUrls
bool Users::init(){
// 8 byte key is an ip/usernameHash and value is the timestamp
m_loginTable.set ( 8 , 4,0,NULL,0,false,0,"logintbl" );
// initialize the testresults rdbtree
//
int32_t nodeSize = (sizeof(key_t)+12+1) + sizeof(collnum_t);
int32_t maxNodes = MAX_TEST_RESULTS;
int32_t maxMem = maxNodes * nodeSize;
// only need to call this once
if ( ! g_testResultsTree.set ( 0 , // fixedDataSize
maxNodes ,
true , // do balancing?
maxMem ,
false , // own data?
"tree-testresults",
false , // dataInPtrs
NULL , // dbname
12 , // keySize
false ))
return false;
// call this userlogin.dat, not turkLoginTable.dat!!
//if ( ! m_loginTable.load(g_hostdb.m_dir,"userlogin.dat", NULL,NULL) )
// log("users: failed to load userlogin.dat");
// try to load the turk test results
loadTestResults();
// load users from the file
m_init = loadUserHashTable ();
m_needsSave = false;
return m_init;
}
// load turk test results from
bool Users::loadTestResults ( ){
//
File file;
char testFile[1024];
sprintf(testFile,"%sturkTestResults.dat", g_hostdb.m_dir );
file.set(testFile);
//
if ( ! file.doesExist() ) return false;
int32_t fileSize = file.getFileSize();
if (fileSize <= 0 ) return false;
// open the file
if ( !file.open(O_RDONLY) ){
log(LOG_DEBUG,"Users error operning test result file %s",
testFile );
return false;
}
char *buf = NULL;
int32_t bufSize = 4096;
int32_t offset = 0;
int32_t numNodes = 0;
// read the turk results file
for ( int32_t i=0; i < fileSize; ){
// bail out if no node left in tree
if ( numNodes >= MAX_TEST_RESULTS ) break;
buf = (char *)mmalloc(bufSize,"UserTestResults");
if ( !buf ){
log(LOG_DEBUG,"Users cannot allocate mem for test"
" file buf");
return false;
}
int32_t numBytesRead = file.read(buf,bufSize,offset);
if ( numBytesRead < 0 ){
log(LOG_DEBUG,"Users error reading test result file %s",
testFile );
mfree(buf,bufSize,"UsersTestResults");
}
// set the offsets
char *bufEnd = buf + numBytesRead;
char *newLineOffset = bufEnd;
for ( char *p = buf; p < bufEnd; p++ ){
char temp[250];
char *line = temp;
int32_t items = 0;
char username[10];
int32_t timestamp;
uint16_t result;
while ( *p != '\n' && p < bufEnd){
*line++ = *p++;
if ( *p != ' ' && *p != '\n' ) continue;
*line = '\0';
switch(items){
case 0: strcpy(username,temp);
break;
case 1: timestamp = atoi(temp);
break;
case 2: result = atoi(temp);
break;
}
line = temp;
items++;
}
if ( p < bufEnd && *p == '\n')
newLineOffset = p;
if ( p >= bufEnd && *p != '\n') break;
// if the fields are not 3 then the line is corrupt
if ( items == 3){
key_t key;
key.n1 = hash32n(username);
key.n0 = ((int32_t)timestamp << 8)
| (result & 0xff);
int32_t node =
g_testResultsTree.addNode(0,(char*)&key);
if ( node < 0 ){
log(LOG_DEBUG,"Users error adding node"
" to testResultsTree");
mfree(buf,bufSize,"UsersTestResults");
return false;
}
numNodes++;
}
//else
//log(LOG_DEBUG,"Users corrupt line turkTestResuls file"
// ": %s", temp);
}
// adjust the offset
offset = file.getCurrentPos() - (int32_t)(bufEnd - newLineOffset);
i += bufSize - (int32_t)( bufEnd - newLineOffset );
mfree(buf,bufSize,"UsersTestResults");
}
file.close();
return true;
}
int32_t Users::getAccuracy ( char *username, time_t timestamp ){
//
// get key between 15
key_t key;
key.n1 = hash32n(username);
key.n0 = ((int32_t)timestamp << 8);
int32_t refNode = g_testResultsTree.getPrevNode( 0 , (char *)&key);
if ( refNode == -1 )
refNode = g_testResultsTree.getNextNode ( 0 ,(char*)&key);
if ( refNode == -1 ) return -1;
// initialize the voting paramaters
int32_t totalVotes = 0;
int32_t totalCorrect = 0;
// get the vote from the reference node
key_t *refKey = (key_t*)g_testResultsTree.getKey(refNode);
if ( refKey->n1 == key.n1 ){
totalVotes++;
if ( refKey->n0 & 0x01 ) totalCorrect++;
}
// scan in the forward direction
int32_t currentNode = refNode;
//key_t currentKey = *refKey;
//currentKey.n0 += 2;
for ( int32_t i=0; i < ACCURACY_FWD_RANGE; i++ ){
int32_t nextNode = g_testResultsTree.getNextNode(currentNode);
if ( nextNode == -1) break;
key_t *nextKey = (key_t *)g_testResultsTree.getKey(nextNode);
if ( refKey->n1 == nextKey->n1 ){
totalVotes++;
if ( nextKey->n0 & 0x01 ) totalCorrect++;
}
currentNode = nextNode;
// currentKey = *nextKey;
// currentKey.n0 += 2;
}
// scan in the backward direction
currentNode = refNode;
//currentKey = *refKey;
//currentKey.n0 -= 2;
for ( int32_t i=0; i < ACCURACY_BWD_RANGE; i++ ){
int32_t prevNode = g_testResultsTree.getPrevNode(currentNode);
if ( prevNode == -1) break;
key_t *prevKey = (key_t *)g_testResultsTree.getKey(prevNode);
if ( refKey->n1 == prevKey->n1 ){
totalVotes++;
if ( prevKey->n0 & 0x01 ) totalCorrect++;
}
currentNode = prevNode;
//currentKey = *prevKey;
//currentKey.n0 -= 2;
}
// don't compute accuracy for few data points
if ( totalVotes < ACCURACY_MIN_TESTS ) return -1;
// compute accuracy in percentage
int32_t accuracy = ( totalCorrect * 100 ) / totalVotes;
return accuracy;
}
// . parses individual row of g_users
// . individual field are separated by :
// and , is used to mention many params in single field
bool Users::parseRow (char *row, int32_t rowLen, User *user ){
// parse individual user row
char *current = row;
char *end = &row[rowLen];
int32_t col = 0;
for ( ; col < 7 ;){
char temp[1024];
char *p = &temp[0];
bool hasStar = false;
while ( current <= end ){
if ( *current == ',' || *current == ':' ||
current == end ){
*p = '\0';
// star is present in data
// its allowed only for column 0 & 2
if (*current == '*' && col != 0 && col != 2 &&
col != 4 ){
log(LOG_DEBUG,"Users * can only"
"be user for collection,ip & pages: %s",
row );
return false;
}
// set the user param
setDatum ( temp, col, user, hasStar );
p = &temp[0];
// reset hasStar for all the other columns
// other then ip column 1
if ( col != 1 ) hasStar = false;
}
else {
//if ( *current == '*' || isalnum(*current) ||
// *current == '_' || *current=='.') {
if ( *current == '*' )
hasStar = true;
*p++ = *current;
}
current++;
if ( *(current-1) == ':') break;
//else{
// wrong format
// log(LOG_DEBUG,"Users error in log line: %s",
// row );
// return false;
//
}
if ( current >= end ) break;
col++;
QUICKPOLL(0);
}
if ( current < end ) return false;
return true;
}
// set individual user field from the given column/field
void Users::setDatum ( char *data, int32_t column, User *user, bool hasStar){
int32_t dataLen = gbstrlen (data);
if ( dataLen <= 0 || user == NULL || column < 0 ) return;
// set the user info depending on the column
// number or field
switch ( column ){
case 0:{
if ( user->m_allColls ) break;
if ( *data == '*' ){
user->m_allColls = true;
break;
}
collnum_t collNum = g_collectiondb.getCollnum(data);
if (collNum >= 0 ){
user->m_collNum[user->m_numColls] = collNum;
user->m_numColls++;
}
break;
}
case 2:{
if ( dataLen >= MAX_USER_SIZE ) data[MAX_USER_SIZE] = '\0';
strcpy ( user->m_username, data );
break;
}
case 1:{
if (user->m_allIps || user->m_numIps > MAX_IPS_PER_USER) break;
// scan ip
// if start is present find the location of *
uint32_t starMask = 0xffffffff;
if ( hasStar ){
char *p = data;
if ( *data == '*' && *(data+1) =='\0'){
user->m_allIps = true;
break;
}
// get the location of *
unsigned char starLoc = 4;
while ( *p !='\0'){
if ( *p == '*'){
// ignore the whole byte for
// that location
// set it to 0
*p = '0';
if ( starMask==0xffffffff )
starMask >>= 8*starLoc;
}
if ( *p == '.' ) starLoc--;
// starLoc = ceil(starLoc/2);
p++;
}
}
// if startMask means all ips are allowed
if ( starMask==0 ){ user->m_allIps = true; break;}
int32_t iplen = gbstrlen ( data );
int32_t ip = atoip(data,iplen);
if ( ! ip ) break;
user->m_ip[user->m_numIps] = ip;
user->m_ipMask[user->m_numIps] = starMask;
user->m_numIps++;
break;
}
case 3:{
if ( gbstrlen(data) > MAX_PASS_SIZE ) data[MAX_PASS_SIZE] = '\0';
strcpy ( user->m_password, data);
break;
}
case 5:{
if ( user->m_numPages >= MAX_PAGES_PER_USER ) break;
char *p = data;
user->m_pages[user->m_numPages]=0;
if ( hasStar ){
user->m_allPages = true;
break;
}
// if not allowed set MSB to 1
if ( *p == '-' ){
user->m_pages[user->m_numPages] = 0x8000;
p++;
}
int32_t pageNum = g_pages.getPageNumber(p);
if ( pageNum < 0 || pageNum >= PAGE_NONE ){
log(LOG_DEBUG,"Users Invalid Page - %s for user %s", p,
user->m_username );
break;
}
user->m_pages[user->m_numPages] |= ((uint16_t)pageNum & 0x7fff);
user->m_numPages++;
break;
}
case 6:{
// save the user permission
// only one user is allowed
// user permission keyword no longer used
/*if ( ! user->m_permissions & 0xff ){
if (strcmp(data,"master")==0)
user->m_permissions = USER_MASTER;
else if (strcmp(data,"admin")==0)
user->m_permissions = USER_ADMIN;
else if (strcmp(data,"client")==0)
user->m_permissions = USER_CLIENT;
else if (strcmp(data,"spam")==0)
user->m_permissions = USER_SPAM;
else if (strcmp(data,"public")==0)
user->m_permissions = USER_PUBLIC;
}else{ */
// save the tags
int32_t tagId = 0;
int32_t strLen = gbstrlen(data);
// backup over ^M
if ( strLen>1 && data[strLen-1]=='M' && data[strLen-2]=='^' )
strLen-=2;
//
// skip for now, it cores for "english" because we removed
// that tag from the list of tags in Tagdb.cpp
//
log("users: skipping language tag");
break;
tagId = getTagTypeFromStr ( data, strLen );
if ( tagId > 0 ) { // && tagId < ST_LAST_TAG ){
user->m_tagId[user->m_numTagIds] = tagId;
user->m_numTagIds++;
}
else {
log(LOG_DEBUG,"Users Invalid tagname - %s for user %s", data,
user->m_username );
//char *xx=NULL;*xx=0;
}
//}
break;
}
case 4:{
if ( *data == '1' && gbstrlen(data)==1 ) user->m_reLogin=true;
break;
}
default:
//
log(LOG_DEBUG, "Users invalid column data: %s", data);
}
}
// . load users from Conf::g_users if size of g_users
// changes and lastreadtime is > USER_DATA_READ_FREQ
// . returns false and sets g_errno on error
bool Users::loadUserHashTable ( ) {
// read user info from the file and add to cache
char *buf = &g_conf.m_users[0];
uint32_t bufSize = g_conf.m_usersLen;
//time_t now = getTimeGlobal();
// no users?
if ( bufSize <= 0 ) {
//log("users: no <users> tag in gb.conf?");
return true;
}
// what was this for?
//if ( bufSize <= 0 || ( bufSize == m_oldBufSize
// && (now - m_oldBufReadTime) < USER_DATA_READ_FREQ ))
// return false;
// init it
if ( ! m_ht.set (12,sizeof(User),0,NULL,0,false,0,"userstbl"))
return false;
// read user data from the line and add it to the cache
char *p = buf;
uint32_t i = 0;
for ( ; i < bufSize; i++){
// read a line from buf
char *row = p;
int32_t rowLen = 0;
while ( *p != '\r' && *p != '\n' && i < bufSize ){
i++; p++; rowLen++;
}
if ( *p == '\r' && *(p+1) == '\n' ) p+=2;
else if ( *p == '\r' || *p == '\n' ) p++;
if ( rowLen <= 0) break;
// set "user"
User user; if ( ! parseRow ( row, rowLen, &user) ) continue;
// skip empty usernames
if ( !gbstrlen(user.m_username) || !gbstrlen(user.m_password) )
continue;
// make the user key
key_t uk = hash32n ( user.m_username );
// grab the slot
int32_t slot = m_ht.getSlot ( &uk );
// get existing User record, "eu" from hash table
User *eu = NULL;
if ( slot >= 0 ) eu = (User *)m_ht.getValueFromSlot ( slot );
// add the user. will overwrite him if in there
if ( ! m_ht.addKey ( &uk , &user ) ) return false;
}
return true;
}
// . get User record from user cache
// . return NULL if no record found
User *Users::getUser (char *username ) { //,bool cacheLoad){
// bail out if init has failed
if ( ! m_init ) {
log("users: could not load users from cache ");
return NULL;
}
if ( ! username ) return NULL;
// check for user in cache
key_t uk = hash32n ( username );
return (User *)m_ht.getValue ( &uk );
}
// . check if user is logged
// . returns NULL if session is timedout or user not logged
// . returns the User record on success
User *Users::isUserLogged ( char *username, int32_t ip ){
// bail out if init has failed
if ( !m_init ) return (User *)NULL;
// get the user to the login cache
// get user record from cache
// return NULL if not found
User *user = getUser (username);
if ( !user ) return NULL;
//if ( user->m_reLogin ) return user;
// make the key a combo of ip and username
uint64_t key;
key = ((int64_t)ip << 32 ) | (int64_t)hash32n(username);
int32_t slotNum = m_loginTable.getSlot ( &key );
if ( slotNum < 0 ) return NULL;
// if this is true, user cannot time out
if ( user->m_reLogin ) return user;
// return NULL if user sesssion has timed out
int32_t now = getTime();
//int32_t timestamp = m_loginTable.getValueFromSlot(slotNum);
// let's make it a permanent login now!
//if ( (now-timestamp) > (int32_t)USER_SESSION_TIMEOUT ){
// m_loginTable.removeKey(key);
// return NULL;
//}
m_needsSave = true;
// if not timed out then add the new access time to the table
if ( ! m_loginTable.addKey(&key,&now) )
log("users: failed to update login of user %s : %s",username,
mstrerror(g_errno) );
return user;
}
// . login the user
// . adds the user to the login table
// . the valud is the last access timestamp of user
// which is used for session timeout
bool Users::loginUser ( char *username, int32_t ip ) {
// bail out if init has failed
if ( ! m_init ) return false;
// add the user to the login table
//key_t cacheKey = makeUserKey ( username, &cacheKey );
uint64_t key;
key = ((int64_t)ip << 32 ) | (int64_t)hash32n(username);
m_needsSave = true;
// add entry to table
int32_t now = getTime();
if ( m_loginTable.addKey(&key,&now) ) return true;
return log("users: failed to login user %s : %s",
username,mstrerror(g_errno));
}
bool Users::logoffUser( char *username, int32_t ip ){
uint64_t key;
key = ((int64_t)ip << 32 ) | (int64_t)hash32n(username);
m_loginTable.removeKey(&key);
return true;
}
char *Users::getUsername ( HttpRequest *r ){
// get from cgi before cookie so we can override
char *username = r->getString("username",NULL);
if ( !username ) username = r->getString("user",NULL);
// cookie is last resort
if ( !username ) username = r->getStringFromCookie("username",NULL);
//if ( !username ) username = r->getString("code",NULL);
// use the password as the user name if no username given
if ( ! username ) username = r->getString("pwd",NULL);
return username;
}
// check page permissions
bool Users::hasPermission ( HttpRequest *r, int32_t page , TcpSocket *s ) {
if ( r->isLocal() ) return true;
// get username from the request
char *username = getUsername(r);
// msg28 always has permission
if ( username &&
s &&
strcmp(username,"msg28")==0 ) {
Host *h = g_hostdb.getHostByIp(s->m_ip);
// we often ssh tunnel in through router0 which is also
// the proxy, but now the proxy uses msg 0xfd to forward
// http requests, so we no longer have to worry about this
// being a security hazard
//Host *p = g_hostdb.getProxyByIp(s->m_ip);
//if ( h && ! p ) return true;
// if host has same ip as proxy, DONT TRUST IT
//if ( h && p->m_ip == h->m_ip )
// return log("http: proxy ip same as host ip.");
if ( ! h )
return log("http: msg28 only good internally.");
// we are good to go
return true;
}
return hasPermission ( username, page );
}
// does user have permission to view and edit the parms on this page?
bool Users::hasPermission ( char *username, int32_t page ){
//if ( !username ) return false;
if ( ! username ) username = "public";
// get ths user from cache
User *user = getUser(username);
if ( !user ) return false;
// verify if user has access to the page
return user->verifyPageNum(page);
}
// get the highest user level for this client
//int32_t Pages::getUserType ( TcpSocket *s , HttpRequest *r ) {
bool Users::verifyUser ( TcpSocket *s, HttpRequest *r ){
//
//bool isIpInNetwork = true;//g_hostdb.isIpInNetwork ( s->m_ip );
if ( r->isLocal() ) return true;
int32_t n = g_pages.getDynamicPageNumber ( r );
User *user;
char *username = getUsername( r);
//if ( !username ) return false;
// if no username, assume public user. technically,
// they are verified as who they claim to be... noone.
if ( ! username ) return true;
// public user need not be verified
if ( strcmp(username,"public") == 0 ) return true;
// user "msg28" is valid as int32_t as he is on one of the machines
// and not the proxy ip
if ( s && strcmp(username,"msg28")==0 ) {
Host *h = g_hostdb.getHostByIp(s->m_ip);
if ( h && ! h->m_isProxy ) return true;
// if he's from 127.0.0.1 then let it slide
//if ( s->m_ip == 16777343 ) return true;
//if ( h && h->m_isProxy ) return true;
// otherwise someone could do a get request with msg28
// as the username and totally control us...
return log("http: msg28 only good internally and not "
"from proxy.");
}
char *password = r->getString("pwd",NULL);
// this is the same thing!
if ( password && ! password[0] ) password = NULL;
/*
// the possible proxy ip
int32_t ip = s->m_ip;
// . if the request is from the proxy, grab the "uip",
// the "user ip" who originated the query
// . now the porxy uses msg 0xfd to forward its requests so if we
// receive this request from ip "ip" it is probably because we are
// doing an ssh tunnel through router0, which is also the proxy
if ( g_hostdb.getProxyByIp ( ip ) ) {
// attacker could add uip=X and gain access if we are logged
// in through X. now we have moved the proxy off of router0
// and onto gf49...
return log("gb: got admin request from proxy for "
"user=%s. ignoring.",username);
}
*/
// if the page is login then
// get the username from the request
// and login the user if valid
if ( n == PAGE_LOGIN ){
//
//username = r->getString("username");
//char *password = r->getString("pwd");
// if no username return
//if ( ! username ) return 0;
// get the user information
user = g_users.getUser ( username );
// if no user by that name
// means bad username, return
if ( ! user ) return 0;
// verify pass and return if bad
if ( ! user->verifyPassword ( password ) ) return 0;
}
else if ( password ) {
user = g_users.getUser(username);
if (!user) return 0;
//password = r->getString("pwd",NULL);
if ( !user->verifyPassword( password) ) return 0;
// . add the user to the login cache
// . if we don't log him in then passing the username/pwd
// in the hostid links is not good enough, we'd also have
// to add username/pwd to all the other links too!
g_users.loginUser(username,s->m_ip);
}
else {
// check the login table and users cache
user = g_users.isUserLogged ( username, s->m_ip );
// . if no user prsent return 0 to indicate that
// user is not logged in.
// . MDW: no, because the cookie keeps sending
// username=mwells even though i don't want to login...
// i just want to do a search and be treated like public
if ( ! user ) return 0;
}
// verify ip of the user
bool verifyIp = user->verifyIp ( s->m_ip );
if ( ! verifyIp ) return 0;
// verify collection
char *coll = r->getString ("c");
if( !coll) coll = g_conf.m_defaultColl;
int32_t collNum = g_collectiondb.getCollnum(coll);
bool verifyColl = user->verifyColl (collNum);
if ( ! verifyColl ) return 0;
// add the user to the login cache
if ( n == PAGE_LOGIN || n == PAGE_LOGIN2 ){
if ( ! g_users.loginUser ( username, s->m_ip ) ) return 0;
}
// now if everything is valid
// get the user permission
// i.e USER_MASTER | USER_ADMIN etc.
//int32_t userType = user->getPermissions ( );
// . Commented by Gourav
// . Users class used
//hif ( userType == USER_MASTER || userType == USER_ADMIN
// || userType == USER_PUBLIC )
// userType &= isIpInNetwork;
//if ( g_conf.isMasterAdmin ( s , r ) ) return USER_MASTER;
// see if has permission for specified collection
//CollectionRec *cr = g_collectiondb.getRec ( r );
// if no collection specified, assume public access
//if ( ! cr ) return USER_PUBLIC;
//if ( cr->hasPermission ( r , s ) ) return USER_ADMIN;
//if ( cr->isAssassin ( s->m_ip ) ) return USER_SPAM;
// otherwise, just a public user
//return USER_PUBLIC;
//return userType;
return true;
}

192
Users.h

@ -1,192 +0,0 @@
#ifndef _USERS_
#define _USERS_
#include "gb-include.h"
#include "RdbCache.h"
#include "Tagdb.h"
#include "Pages.h"
#include "HashTableX.h"
// cache reload freq from Conf::m_users
#define USER_DATA_READ_FREQ 3600
#define MAX_USER_SIZE 50
#define MAX_PASS_SIZE 50
#define MAX_COLLS_PER_USER 5
#define MAX_IPS_PER_USER 5
#define MAX_TAGS_PER_USER 10
#define MAX_PAGES_PER_USER 15
#define USER_SESSION_TIMEOUT 18000
// fwd and back scan limits
// for user accuracy
#define ACCURACY_FWD_RANGE 10
#define ACCURACY_BWD_RANGE 10
// min number of data points
// for calculating accuracy
#define ACCURACY_MIN_TESTS 5
// max test results to store
// in tree memory
#define MAX_TEST_RESULTS 500000
// the permission bits, each Page has them in Page::m_perm
//#define USER_PUBLIC 0x01 // globally available
//#define USER_MASTER 0x02 // master admin, can see everything
//#define USER_ADMIN 0x04 // collection admin, just for a coll.
//#define USER_SPAM 0x08 // spam assassin
//#define USER_PROXY 0x10 // the proxy server
//#define USER_CLIENT 0x20 // client
// . individual user information
// . Each user record is made of
// . allowed_collection_names + allowed_Ips + username(50 bytes max) +
// password ( 50bytes max ) + permissions + tags
class User{
public:
User();
uint16_t getPermissions ( ){ return m_permissions; }
int32_t * getTags ( ){ return &m_tagId[0]; }
bool verifyIp ( int32_t ip );
bool verifyPassword (char *pass);
bool verifyColl ( int32_t collNum);
bool verifyTagId ( int32_t tagId );
bool verifyPageNum ( uint16_t pageNum );
int32_t firstPage ( );
//username is max of 50 chars
//password is maximum of 50 chars
char m_username[MAX_USER_SIZE];
char m_password[MAX_PASS_SIZE];
// user permissions
uint16_t m_permissions;
// tagdb tags
int32_t m_tagId[MAX_TAGS_PER_USER];
uint16_t m_numTagIds;
// collection user is allowed to access
collnum_t m_collNum[MAX_COLLS_PER_USER];
collnum_t m_numColls; // num of Collections assigned
bool m_allColls; // true if user can access all collections
// allowed ips
int32_t m_ip[MAX_IPS_PER_USER];
uint16_t m_numIps; // number of allowed ips (max 255)
// . ipmasks helps to allow part of ip match
// . points to the locations of stars
// . Note: right now * can only be given for a complete byte
// of an ip
uint32_t m_ipMask[MAX_IPS_PER_USER];
bool m_allIps;// true if allowed from all Ips
// . pages allowed
// . top MSB is set for the pages that are allowed
// . if all pages are allowe then allPages is set
uint16_t m_pages[MAX_PAGES_PER_USER];
uint16_t m_numPages;
bool m_allPages;
// . relogin - if set user dont have to relogin
// between gb shutdowns
// . only restriction is the user session
// . it should be valid
bool m_reLogin;
};
// . User database
// . RdbCache of user's
// . Record Format
// 12 byte key + 1 User record (18 bytes)
// . User data is read from the Conf::g_users parm
// which is linked to the Users Textbox on security control page
class Users{
public:
// initialize all members to 0/false
Users ();
//
~Users();
// initialize the database
bool init();
bool save();
// get one TurkUser rec corresponding to
// the username from m_userDB cache
User * getUser (char *username );
// add login
bool loginUser ( char *username, int32_t m_ip );
bool logoffUser ( char *username, int32_t m_ip );
// verify if user is valid
bool verifyUser ( TcpSocket *s, HttpRequest *r );
// return username from request
char *getUsername ( HttpRequest *r );
bool hasPermission ( HttpRequest *r,
int32_t page ,
TcpSocket *s = NULL );
bool hasPermission ( char *username, int32_t page );
//. check is user if logged by checking
// the login table
//. also checks for session timeout
//. returns null is session timeout or user not
// logged in
User * isUserLogged ( char *username, int32_t m_ip );
// load turk test results into tree
bool loadTestResults ( );
bool loadUserHashTable ( ) ;
// . get User accuracy during a particular
// period
// . return -1 if not enough data points to compute
// accuracy
int32_t getAccuracy ( char *username, time_t timestamp);
void makeCacheKey( char *username, key_t *cacheKey );
// true if database is initialized correctly
bool m_init;
//private:
// loads users.dat into m_ht
bool loadHashTable ( );
// parse a row in users.dat file
bool parseRow ( char *row, int32_t rowLen, User *user );
// . parse a datum or field from a user row and store it
// in the User
// . Used by parseUserRow to parse and store a user field
void setDatum (char *data, int32_t colNum, User *user, bool hasStar );
//RdbCache m_userCache; // database of user information
//uint32_t m_userCacheAge;
//uint32_t m_userCacheSize;
// each slot is a key/User pair
HashTableX m_ht;
//File m_userFile; // not used right now
//uint32_t m_oldBufSize;
//uint32_t m_oldBufReadTime;
bool m_needsSave;
// login table
HashTableX m_loginTable;
};
extern Users g_users;
// . Results tree consists of the user results
// . key is the result
// . format: key.n1 = 32 bit hash of username
// key.n0 = ((int32_t)timestamp << 8 )|(isResultCorrect & 0xff );
// . using the above key we can easily find out the
// user's accuracy during a particular time period
extern RdbTree g_testResultsTree;
#endif

3525
XmlDoc.cpp

File diff suppressed because it is too large Load Diff

27
XmlDoc.h

@ -107,15 +107,6 @@ bool checkRegex ( SafeBuf *regex ,
// Address::m_flags for that address on the web page.
int32_t makeSimpleWordVector ( char *s, int32_t *vbuf, int32_t vbufSize, int32_t niceness);
// this is used for making the event summary/title vectors as well as in
// Msg40.cpp where it merges events and does not want to repetitively display
// the same summary lines for an event
bool getWordVector ( char *s ,
HashTableX *ht ,
uint32_t *d ,
int32_t *nd ,
int32_t ndmax ) ;
bool getDensityRanks ( int64_t *wids ,
int32_t nw,
//int32_t wordStart ,
@ -156,16 +147,6 @@ bool isSimilar_sorted ( int32_t *vec0 ,
int32_t percentSimilar,
int32_t niceness ) ;
// this is called by Msg40.cpp to set "top"
int32_t intersectGigabits ( Msg20 **mp , // search results
int32_t nmp ,
uint8_t langId , // searcher's langId
int32_t maxTop ,
int32_t docsToScan ,
int32_t minDocCount , // must be in this # docs
class GigabitInfo *top ,
int32_t niceness ) ;
int32_t getDirtyPoints ( char *s , int32_t len , int32_t niceness , char *logUrl ) ;
bool storeTerm ( char *s ,
@ -1420,7 +1401,6 @@ class XmlDoc {
bool m_siteSpiderQuotaValid;
bool m_oldDocValid;
bool m_extraDocValid;
bool m_ahrefsDocValid;
//bool m_contactDocValid;
bool m_rootDocValid;
//bool m_gatewayDocValid;
@ -1605,7 +1585,6 @@ class XmlDoc {
//int32_t m_numBannedOutlinks;
class XmlDoc *m_oldDoc;
class XmlDoc *m_extraDoc;
class XmlDoc *m_ahrefsDoc;
//class XmlDoc *m_contactDoc;
class XmlDoc *m_rootDoc;
//class XmlDoc *m_gatewayDoc;
@ -2082,10 +2061,6 @@ class XmlDoc {
//char *m_outlinkIsIndexedVector;
//Msge2 m_msge2;
bool m_doneWithAhrefs;
bool m_useAhrefs;
bool m_reallyInjectLinks;
int32_t m_downloadLevel;
int32_t m_numRegExs;
//char m_isSiteRoot;
int8_t *m_outlinkHopCountVector;
@ -2451,7 +2426,7 @@ class XmlDoc {
HashTableX *domDedupTable ,
void *finalState ,
void (* finalCallback)(void *));
bool injectAhrefsLinks();
bool doInjectLoop ( );
void doneInjecting ( class XmlDoc *xd );
int32_t m_i;

540
gigablast.cbp

@ -1,540 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
<CodeBlocks_project_file>
<FileVersion major="1" minor="6" />
<Project>
<Option title="gigablast" />
<Option makefile_is_custom="1" />
<Option pch_mode="2" />
<Option compiler="gcc" />
<MakeCommands>
<Build command="$make -f $makefile" />
<CompileFile command="$make -f $makefile $file" />
<Clean command="$make -f $makefile clean" />
<DistClean command="$make -f $makefile distclean$target" />
<AskRebuildNeeded command="$make -q -f $makefile" />
<SilentBuild command="$make -f $makefile &gt; $(CMD_NULL)" />
</MakeCommands>
<Build>
<Target title="Debug">
<Option output="./gb" prefix_auto="1" extension_auto="1" />
<Option object_output="obj/Debug/" />
<Option type="1" />
<Option compiler="gcc" />
<Compiler>
<Add option="-g" />
</Compiler>
<MakeCommands>
<Build command="$make -f $makefile" />
<CompileFile command="$make -f $makefile $file" />
<Clean command="$make -f $makefile clean" />
<DistClean command="$make -f $makefile distclean$target" />
<AskRebuildNeeded command="$make -q -f $makefile" />
<SilentBuild command="$make -f $makefile &gt; $(CMD_NULL)" />
</MakeCommands>
</Target>
<Target title="Release">
<Option output="bin/Release/gigablast" prefix_auto="1" extension_auto="1" />
<Option object_output="obj/Release/" />
<Option type="1" />
<Option compiler="gcc" />
<Compiler>
<Add option="-O2" />
</Compiler>
<Linker>
<Add option="-s" />
</Linker>
</Target>
</Build>
<Compiler>
<Add option="-Wall" />
</Compiler>
<Unit filename="Abbreviations.cpp" />
<Unit filename="Abbreviations.h" />
<Unit filename="Accessdb.cpp" />
<Unit filename="Accessdb.h" />
<Unit filename="Address.cpp" />
<Unit filename="Address.h" />
<Unit filename="Ads.cpp" />
<Unit filename="Ads.h" />
<Unit filename="AdultBit.cpp" />
<Unit filename="AdultBit.h" />
<Unit filename="AutoBan.cpp" />
<Unit filename="AutoBan.h" />
<Unit filename="BigFile.cpp" />
<Unit filename="BigFile.h" />
<Unit filename="Bits.cpp" />
<Unit filename="Bits.h" />
<Unit filename="Blaster.cpp" />
<Unit filename="Blaster.h" />
<Unit filename="Cachedb.cpp" />
<Unit filename="Cachedb.h" />
<Unit filename="CatRec.cpp" />
<Unit filename="CatRec.h" />
<Unit filename="Catdb.cpp" />
<Unit filename="Catdb.h" />
<Unit filename="Categories.cpp" />
<Unit filename="Categories.h" />
<Unit filename="Clusterdb.cpp" />
<Unit filename="Clusterdb.h" />
<Unit filename="Collectiondb.cpp" />
<Unit filename="Collectiondb.h" />
<Unit filename="Conf.cpp" />
<Unit filename="Conf.h" />
<Unit filename="CountryCode.cpp" />
<Unit filename="CountryCode.h" />
<Unit filename="DailyMerge.cpp" />
<Unit filename="DailyMerge.h" />
<Unit filename="DataFeed.cpp" />
<Unit filename="DataFeed.h" />
<Unit filename="Datedb.cpp" />
<Unit filename="Datedb.h" />
<Unit filename="Dates.cpp" />
<Unit filename="Dates.h" />
<Unit filename="Diff.cpp" />
<Unit filename="Diff.h" />
<Unit filename="Dir.cpp" />
<Unit filename="Dir.h" />
<Unit filename="DiskPageCache.cpp" />
<Unit filename="DiskPageCache.h" />
<Unit filename="Dns.cpp" />
<Unit filename="Dns.h" />
<Unit filename="DnsProtocol.h" />
<Unit filename="Domains.cpp" />
<Unit filename="Domains.h" />
<Unit filename="Entities.cpp" />
<Unit filename="Entities.h" />
<Unit filename="Errno.cpp" />
<Unit filename="Errno.h" />
<Unit filename="Events.h" />
<Unit filename="Facebook.cpp" />
<Unit filename="Facebook.h" />
<Unit filename="File.cpp" />
<Unit filename="File.h" />
<Unit filename="Flags.cpp" />
<Unit filename="Flags.h" />
<Unit filename="GeoIP.c">
<Option compilerVar="CC" />
</Unit>
<Unit filename="GeoIP.h" />
<Unit filename="GeoIPCity.c">
<Option compilerVar="CC" />
</Unit>
<Unit filename="GeoIPCity.h" />
<Unit filename="GeoIP_internal.h" />
<Unit filename="HashTable.cpp" />
<Unit filename="HashTable.h" />
<Unit filename="HashTableT.cpp" />
<Unit filename="HashTableT.h" />
<Unit filename="HashTableX.cpp" />
<Unit filename="HashTableX.h" />
<Unit filename="Highlight.cpp" />
<Unit filename="Highlight.h" />
<Unit filename="Hostdb.cpp" />
<Unit filename="Hostdb.h" />
<Unit filename="HttpMime.cpp" />
<Unit filename="HttpMime.h" />
<Unit filename="HttpRequest.cpp" />
<Unit filename="HttpRequest.h" />
<Unit filename="HttpServer.cpp" />
<Unit filename="HttpServer.h" />
<Unit filename="Images.cpp" />
<Unit filename="Images.h" />
<Unit filename="IndexList.cpp" />
<Unit filename="IndexList.h" />
<Unit filename="IndexReadInfo.cpp" />
<Unit filename="IndexReadInfo.h" />
<Unit filename="IndexTable.cpp" />
<Unit filename="IndexTable.h" />
<Unit filename="IndexTable2.cpp" />
<Unit filename="IndexTable2.h" />
<Unit filename="Indexdb.cpp" />
<Unit filename="Indexdb.h" />
<Unit filename="Iso8859.cpp" />
<Unit filename="Iso8859.h" />
<Unit filename="Json.cpp" />
<Unit filename="Json.h" />
<Unit filename="Lang.cpp" />
<Unit filename="Lang.h" />
<Unit filename="LangList.cpp" />
<Unit filename="LangList.h" />
<Unit filename="Language.cpp" />
<Unit filename="Language.h" />
<Unit filename="LanguageIdentifier.cpp" />
<Unit filename="LanguageIdentifier.h" />
<Unit filename="LanguagePages.cpp" />
<Unit filename="LanguagePages.h" />
<Unit filename="Linkdb.cpp" />
<Unit filename="Linkdb.h" />
<Unit filename="LinkedList.h" />
<Unit filename="Log.cpp" />
<Unit filename="Log.h" />
<Unit filename="Loop.cpp" />
<Unit filename="Loop.h" />
<Unit filename="Matches.cpp" />
<Unit filename="Matches.h" />
<Unit filename="Mem.cpp" />
<Unit filename="Mem.h" />
<Unit filename="MemPool.cpp" />
<Unit filename="MemPool.h" />
<Unit filename="MemPoolTree.cpp" />
<Unit filename="MemPoolTree.h" />
<Unit filename="MetaContainer.cpp" />
<Unit filename="MetaContainer.h" />
<Unit filename="Mime.cpp" />
<Unit filename="Mime.h" />
<Unit filename="Monitordb.cpp" />
<Unit filename="Monitordb.h" />
<Unit filename="Msg0.cpp" />
<Unit filename="Msg0.h" />
<Unit filename="Msg1.cpp" />
<Unit filename="Msg1.h" />
<Unit filename="Msg13.cpp" />
<Unit filename="Msg13.h" />
<Unit filename="Msg17.cpp" />
<Unit filename="Msg17.h" />
<Unit filename="Msg1f.cpp" />
<Unit filename="Msg1f.h" />
<Unit filename="Msg2.cpp" />
<Unit filename="Msg2.h" />
<Unit filename="Msg20.cpp" />
<Unit filename="Msg20.h" />
<Unit filename="Msg22.cpp" />
<Unit filename="Msg22.h" />
<Unit filename="Msg24.cpp" />
<Unit filename="Msg28.cpp" />
<Unit filename="Msg28.h" />
<Unit filename="Msg2a.cpp" />
<Unit filename="Msg2a.h" />
<Unit filename="Msg2b.cpp" />
<Unit filename="Msg2b.h" />
<Unit filename="Msg3.cpp" />
<Unit filename="Msg3.h" />
<Unit filename="Msg30.cpp" />
<Unit filename="Msg30.h" />
<Unit filename="Msg35.cpp" />
<Unit filename="Msg35.h" />
<Unit filename="Msg36.cpp" />
<Unit filename="Msg36.h" />
<Unit filename="Msg37.cpp" />
<Unit filename="Msg37.h" />
<Unit filename="Msg39.cpp" />
<Unit filename="Msg39.h" />
<Unit filename="Msg3a.cpp" />
<Unit filename="Msg3a.h" />
<Unit filename="Msg3e.cpp" />
<Unit filename="Msg3e.h" />
<Unit filename="Msg4.cpp" />
<Unit filename="Msg4.h" />
<Unit filename="Msg40.cpp" />
<Unit filename="Msg40.h" />
<Unit filename="Msg40Cache.cpp" />
<Unit filename="Msg40Cache.h" />
<Unit filename="Msg42.cpp" />
<Unit filename="Msg42.h" />
<Unit filename="Msg5.cpp" />
<Unit filename="Msg5.h" />
<Unit filename="Msg51.cpp" />
<Unit filename="Msg51.h" />
<Unit filename="Msg6b.cpp" />
<Unit filename="Msg6b.h" />
<Unit filename="Msg8b.cpp" />
<Unit filename="Msg8b.h" />
<Unit filename="Msg9b.cpp" />
<Unit filename="Msg9b.h" />
<Unit filename="MsgC.cpp" />
<Unit filename="MsgC.h" />
<Unit filename="Msgaa.cpp" />
<Unit filename="Msgaa.h" />
<Unit filename="Msge0.cpp" />
<Unit filename="Msge0.h" />
<Unit filename="Msge1.cpp" />
<Unit filename="Msge1.h" />
<Unit filename="Multicast.cpp" />
<Unit filename="Multicast.h" />
<Unit filename="PageAddColl.cpp" />
<Unit filename="PageAddUrl.cpp" />
<Unit filename="PageBasic.cpp" />
<Unit filename="PageCatdb.cpp" />
<Unit filename="PageCrawlBot.cpp" />
<Unit filename="PageCrawlBot.h" />
<Unit filename="PageDirectory.cpp" />
<Unit filename="PageEvents.cpp" />
<Unit filename="PageGet.cpp" />
<Unit filename="PageHosts.cpp" />
<Unit filename="PageIndexdb.cpp" />
<Unit filename="PageInject.cpp" />
<Unit filename="PageInject.h" />
<Unit filename="PageLogView.cpp" />
<Unit filename="PageNetTest.cpp" />
<Unit filename="PageNetTest.h" />
<Unit filename="PageOverview.cpp" />
<Unit filename="PageParser.cpp" />
<Unit filename="PageParser.h" />
<Unit filename="PagePerf.cpp" />
<Unit filename="PageReindex.cpp" />
<Unit filename="PageReindex.h" />
<Unit filename="PageResults.cpp" />
<Unit filename="PageResults.h" />
<Unit filename="PageRoot.cpp" />
<Unit filename="PageSockets.cpp" />
<Unit filename="PageSpam.cpp" />
<Unit filename="PageStats.cpp" />
<Unit filename="PageStatsdb.cpp" />
<Unit filename="PageSubmit.cpp" />
<Unit filename="PageThesaurus.cpp" />
<Unit filename="PageThreads.cpp" />
<Unit filename="PageTitledb.cpp" />
<Unit filename="PageTurk.cpp" />
<Unit filename="PageTurk.h" />
<Unit filename="Pages.cpp" />
<Unit filename="Pages.h" />
<Unit filename="Parms.cpp" />
<Unit filename="Parms.h" />
<Unit filename="Phrases.cpp" />
<Unit filename="Phrases.h" />
<Unit filename="PingServer.cpp" />
<Unit filename="PingServer.h" />
<Unit filename="Placedb.cpp" />
<Unit filename="Placedb.h" />
<Unit filename="Pops.cpp" />
<Unit filename="Pops.h" />
<Unit filename="Pos.cpp" />
<Unit filename="Pos.h" />
<Unit filename="Posdb.cpp" />
<Unit filename="Posdb.h" />
<Unit filename="PostQueryRerank.cpp" />
<Unit filename="PostQueryRerank.h" />
<Unit filename="Process.cpp" />
<Unit filename="Process.h" />
<Unit filename="Profiler.cpp" />
<Unit filename="Profiler.h" />
<Unit filename="Proxy.cpp" />
<Unit filename="Proxy.h" />
<Unit filename="QAClient.cpp" />
<Unit filename="QAClient.h" />
<Unit filename="Query.cpp" />
<Unit filename="Query.h" />
<Unit filename="Rdb.cpp" />
<Unit filename="Rdb.h" />
<Unit filename="RdbBase.cpp" />
<Unit filename="RdbBase.h" />
<Unit filename="RdbBuckets.cpp" />
<Unit filename="RdbBuckets.h" />
<Unit filename="RdbCache.cpp" />
<Unit filename="RdbCache.h" />
<Unit filename="RdbDump.cpp" />
<Unit filename="RdbDump.h" />
<Unit filename="RdbList.cpp" />
<Unit filename="RdbList.h" />
<Unit filename="RdbMap.cpp" />
<Unit filename="RdbMap.h" />
<Unit filename="RdbMem.cpp" />
<Unit filename="RdbMem.h" />
<Unit filename="RdbMerge.cpp" />
<Unit filename="RdbMerge.h" />
<Unit filename="RdbScan.cpp" />
<Unit filename="RdbScan.h" />
<Unit filename="RdbTree.cpp" />
<Unit filename="RdbTree.h" />
<Unit filename="Rebalance.cpp" />
<Unit filename="Rebalance.h" />
<Unit filename="Repair.cpp" />
<Unit filename="Repair.h" />
<Unit filename="RequestTable.cpp" />
<Unit filename="RequestTable.h" />
<Unit filename="Revdb.cpp" />
<Unit filename="Revdb.h" />
<Unit filename="SafeBuf.cpp" />
<Unit filename="SafeBuf.h" />
<Unit filename="SafeList.h" />
<Unit filename="Sanity.h" />
<Unit filename="Scores.cpp" />
<Unit filename="Scores.h" />
<Unit filename="Scraper.cpp" />
<Unit filename="Scraper.h" />
<Unit filename="SearchInput.cpp" />
<Unit filename="SearchInput.h" />
<Unit filename="Sections.cpp" />
<Unit filename="Sections.h" />
<Unit filename="SiteGetter.cpp" />
<Unit filename="SiteGetter.h" />
<Unit filename="Speller.cpp" />
<Unit filename="Speller.h" />
<Unit filename="Spider.cpp" />
<Unit filename="Spider.h" />
<Unit filename="SpiderProxy.cpp" />
<Unit filename="SpiderProxy.h" />
<Unit filename="Stats.cpp" />
<Unit filename="Stats.h" />
<Unit filename="Statsdb.cpp" />
<Unit filename="Statsdb.h" />
<Unit filename="StopWords.cpp" />
<Unit filename="StopWords.h" />
<Unit filename="Strings.cpp" />
<Unit filename="Strings.h" />
<Unit filename="Summary.cpp" />
<Unit filename="Summary.h" />
<Unit filename="Syncdb.cpp" />
<Unit filename="Syncdb.h" />
<Unit filename="Synonyms.cpp" />
<Unit filename="Synonyms.h" />
<Unit filename="Tagdb.cpp" />
<Unit filename="Tagdb.h" />
<Unit filename="TcpServer.cpp" />
<Unit filename="TcpServer.h" />
<Unit filename="TcpSocket.h" />
<Unit filename="Test.cpp" />
<Unit filename="Test.h" />
<Unit filename="Tfndb.cpp" />
<Unit filename="Tfndb.h" />
<Unit filename="Thesaurus.cpp" />
<Unit filename="Thesaurus.h" />
<Unit filename="Threads.cpp" />
<Unit filename="Threads.h" />
<Unit filename="Timedb.cpp" />
<Unit filename="Timedb.h" />
<Unit filename="Timer.h" />
<Unit filename="Title.cpp" />
<Unit filename="Title.h" />
<Unit filename="Titledb.cpp" />
<Unit filename="Titledb.h" />
<Unit filename="TopTree.cpp" />
<Unit filename="TopTree.h" />
<Unit filename="TuringTest.cpp" />
<Unit filename="TuringTest.h" />
<Unit filename="Turkdb.cpp" />
<Unit filename="UCNormalizer.cpp" />
<Unit filename="UCNormalizer.h" />
<Unit filename="UCPropTable.cpp" />
<Unit filename="UCPropTable.h" />
<Unit filename="UCWordIterator.cpp" />
<Unit filename="UCWordIterator.h" />
<Unit filename="UdpProtocol.h" />
<Unit filename="UdpServer.cpp" />
<Unit filename="UdpServer.h" />
<Unit filename="UdpSlot.cpp" />
<Unit filename="UdpSlot.h" />
<Unit filename="Unicode.cpp" />
<Unit filename="Unicode.h" />
<Unit filename="UnicodeProperties.cpp" />
<Unit filename="UnicodeProperties.h" />
<Unit filename="Url.cpp" />
<Unit filename="Url.h" />
<Unit filename="Users.cpp" />
<Unit filename="Users.h" />
<Unit filename="ValidPointer.cpp" />
<Unit filename="ValidPointer.h" />
<Unit filename="Vector.cpp" />
<Unit filename="Vector.h" />
<Unit filename="Version.cpp" />
<Unit filename="Version.h" />
<Unit filename="Weights.cpp" />
<Unit filename="Weights.h" />
<Unit filename="Wiki.cpp" />
<Unit filename="Wiki.h" />
<Unit filename="Wiktionary.cpp" />
<Unit filename="Wiktionary.h" />
<Unit filename="Words.cpp" />
<Unit filename="Words.h" />
<Unit filename="Xml.cpp" />
<Unit filename="Xml.h" />
<Unit filename="XmlDoc.cpp" />
<Unit filename="XmlDoc.h" />
<Unit filename="XmlNode.cpp" />
<Unit filename="XmlNode.h" />
<Unit filename="addtest.cpp" />
<Unit filename="animate.cpp" />
<Unit filename="blaster2.cpp" />
<Unit filename="camsort.cpp" />
<Unit filename="check_unicode.cpp" />
<Unit filename="convert.cpp" />
<Unit filename="create_ucd_tables.cpp" />
<Unit filename="dlstubs.c">
<Option compilerVar="CC" />
</Unit>
<Unit filename="dmozparse.cpp" />
<Unit filename="dnstest.cpp" />
<Unit filename="dumpcore.cpp" />
<Unit filename="errnotest.cpp" />
<Unit filename="fastIndexTable.cpp" />
<Unit filename="fctypes.cpp" />
<Unit filename="fctypes.h" />
<Unit filename="filterquerylogs.cpp" />
<Unit filename="gb-include.h" />
<Unit filename="gbfilter.cpp" />
<Unit filename="gbtitletest.cpp" />
<Unit filename="geneaology.cpp" />
<Unit filename="generateSuperMergeCode.cpp" />
<Unit filename="getsample.cpp" />
<Unit filename="hash.cpp" />
<Unit filename="hash.h" />
<Unit filename="hashtest.cpp" />
<Unit filename="hashtest2.cpp" />
<Unit filename="hashtest3.cpp" />
<Unit filename="hosts.cpp" />
<Unit filename="iana_charset.cpp" />
<Unit filename="iana_charset.h" />
<Unit filename="iconv.h" />
<Unit filename="injector.cpp" />
<Unit filename="iostream.h" />
<Unit filename="ip.cpp" />
<Unit filename="ip.h" />
<Unit filename="ipconfig.cpp" />
<Unit filename="jointest.cpp" />
<Unit filename="keepalive.cpp" />
<Unit filename="linkspam.cpp" />
<Unit filename="linkspam.h" />
<Unit filename="looptest.cpp" />
<Unit filename="main.cpp" />
<Unit filename="malloc.c">
<Option compilerVar="CC" />
</Unit>
<Unit filename="matches2.cpp" />
<Unit filename="matches2.h" />
<Unit filename="membustest.cpp" />
<Unit filename="memtest.cpp" />
<Unit filename="mergetest.cpp" />
<Unit filename="mixfile.cpp" />
<Unit filename="mmseg.h" />
<Unit filename="monitor.cpp" />
<Unit filename="numwords.cpp" />
<Unit filename="porter.cpp" />
<Unit filename="qa.cpp" />
<Unit filename="quarantine.cpp" />
<Unit filename="rdbtest.cpp" />
<Unit filename="rdbtest2.cpp" />
<Unit filename="readRec.cpp" />
<Unit filename="reindex2.cpp" />
<Unit filename="rescue.cpp" />
<Unit filename="rmbots.cpp" />
<Unit filename="seektest.cpp" />
<Unit filename="seo.h" />
<Unit filename="sleepandlog.cpp" />
<Unit filename="sort.cpp" />
<Unit filename="sort.h" />
<Unit filename="streambuf.h" />
<Unit filename="superMergeTest.cpp" />
<Unit filename="supported_charsets.cpp" />
<Unit filename="test2.cpp" />
<Unit filename="test_convert.cpp" />
<Unit filename="test_hash.cpp" />
<Unit filename="test_norm.cpp" />
<Unit filename="test_parser.cpp" />
<Unit filename="test_parser2.cpp" />
<Unit filename="test_unicode.cpp" />
<Unit filename="testfloats.cpp" />
<Unit filename="threadtest.cpp" />
<Unit filename="thunder.cpp" />
<Unit filename="treetest.cpp" />
<Unit filename="types.h" />
<Unit filename="udptest.cpp" />
<Unit filename="uniq2.cpp" />
<Unit filename="urlinfo.cpp" />
<Unit filename="zconf.h" />
<Unit filename="zlib.h" />
<Extensions>
<code_completion />
<debugger />
</Extensions>
</Project>
</CodeBlocks_project_file>

812
ipconfig.cpp

@ -1,812 +0,0 @@
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h> // for opendir()
#include <dirent.h> // for opendir()
#include <time.h> // for time()
#include <ctype.h>
#include <sys/socket.h> // inet_ntoa()
#include <netinet/in.h> // inet_ntoa()
#include <arpa/inet.h> // inet_ntoa()
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
char g_name[64];
char *g_hosts[5000];
char *g_ips [5000];
int32_t g_numHosts = 0;
void setHosts () ;
void add ( char *s );
void add ( char *prefix , int32_t num );
void setS99Local ();
void setEtcHosts ();
void setEtcNetworkInterfaces ();
void setEtcSshSshdconfig ();
void setEtcPasswd();
void setAlias ();
void setEtcResolv ();
void setKnownHosts ();
char *getIp ( char *hostname );
// usage . no args!
int main ( int argc , char *argv[] ) {
// get hostname
FILE *fd = fopen ("/etc/hostname","r");
if ( ! fd ) {
fprintf(stderr,"could not open /etc/hostname\n");
exit(-1);
}
if ( argc != 1 ) {
fprintf(stderr,"does not take any args. put hostname in "
"/etc/hostname\n");
exit(-1);
}
// set the hostname into g_name[]
fgets ( g_name , 60 , fd );
// remove \n
if ( g_name[strlen(g_name)-1] == '\n' )
g_name[strlen(g_name)-1] = '\0';
// . avoid all this junk for our routers
// . these guys have special ip tables crap, routes, etc.
if ( strcmp(g_name,"router0") == 0 ) return 0;
if ( strcmp(g_name,"router1") == 0 ) return 0;
if ( strcmp(g_name,"gk268") == 0 ) return 0; // router2
if ( strcmp(g_name,"voyager2") == 0 ) return 0;
if ( strcmp(g_name,"titan") == 0 ) return 0;
// set startup to run mkraid
setS99Local();
// . set the g_hosts and g_ips arrays
// . set from /etc/hosts actually
// . this must be called first to set g_hosts[], etc.
setHosts();
// set /etc/hosts, uses g_hosts[] now
setEtcHosts ();
// set /etc/network/interfaces
setEtcNetworkInterfaces();
// set /etc/ssh/sshd_config
setEtcSshSshdconfig ();
// . restart that
// . ss machines just need a restart on eth1
if ( g_name[0]=='s' && g_name[1] == 's' )
system ( "/sbin/ifdown eth1 ; /sbin/ifup eth1");
else
system ( "/etc/init.d/networking restart");
// restart sshd
system ("/etc/init.d/ssh restart");
// update /etc/passwd
setEtcPasswd();
// update /home/mwells/.alias
setAlias ();
// update /etc/resolv.conf
setEtcResolv ();
setKnownHosts ();
}
void setS99Local () {
FILE *fd = fopen ("/etc/rcS.d/S99local", "w" );
if ( ! fd ) {
fprintf(stderr,"could not open /etc/rcS.d/S99local: %s\n",
strerror(errno));
return;
}
fprintf(fd,
"# increase network read buffer\n"
"# make even bigger for gigabit network later (10MB)\n"
"# echo 10048576 > /proc/sys/net/core/rmem_max\n"
"# this causes \"IP: frag: no memory for new fragment!\" failures in dmesg\n"
"# increase gigabit network read/write buffers otherwise we lose udp packets\n"
"echo 10000000 > /proc/sys/net/core/rmem_max\n"
"echo 5000000 > /proc/sys/net/core/wmem_max\n"
"\n"
);
fprintf(fd,
"/usr/sbin/elvtune -w 32 /dev/sda\n"
"/usr/sbin/elvtune -w 32 /dev/sdb\n"
"/usr/sbin/elvtune -w 32 /dev/sdc\n"
"/usr/sbin/elvtune -w 32 /dev/sdd\n"
"\n"
);
// make raid and mount if not already mounted
// a lot of times after reboot something fails!
fprintf(fd,
"raidstop /dev/md0\n"
"mkraid -c /etc/raidtab --really-force /dev/md0\n"
"mount /dev/md0\n"
"\n"
);
fclose(fd);
}
void setKnownHosts () {
FILE *fd = fopen ("/home/mwells/.ssh/authorized_keys2", "w" );
if ( ! fd ) {
fprintf(stderr,"could not open /home/mwells/authorized_keys2: %s\n",
strerror(errno));
return;
}
// set authorized_keys2 for mwells
fprintf ( fd ,
"ssh-dss AAAAB3NzaC1kc3MAAACBAK1lpodgz+4jbONiIUbl44UHTI8bieKplTajU05hgqsGqR6e8WykDMeKbgqGBGsy1bra/LHPh7l+g3C53qk6eejj38Xy320H60a1U5zUCs12kjCMTMruyrkM7ytIawdmNl8ZbnzrJVsdgUti5wpnssHE1u+PrXhjTWcAyk2wT8K3AAAAFQDrp5t8RXmUxBQnRUftZ6BJdkdhPwAAAIEAnEOhCwlZBQ/sHM0OrGzXYgzTlypl+c8HDoKmmkHWz8lsC2+wAt5HBUPvsNvuDxd1vL2VIm171MpHTpoId6Xc0baP5HRX/UHzuyWJvc1uHIuiTVypVuBe9v0FEzkigFHR4GjNucbmOGwotfhbudXFFsIND61RTfezSdRsoHf/oFgAAACAL34kqvvw9w6vkko7MMwHaE37Q4kQLHTNHNw/1PZDj6ivxaFekgpB55JTUEDs+IZ07P6gLv6dySLpeep0NTzdeUg2Y8IiRoFxUnN1uuIqf5Ngac5l7yvIZf/TvIM2diIrFRmXI9uBF8tnncfX01VQdONbWxESUUfEcf1mD8YVG6M= root@ss0\n"
"ssh-dss AAAAB3NzaC1kc3MAAACBANWHAJuWoH7pLAjtEqy0mqVi5sItvmlYO59LQ+a1UFETRBqhEkem9UMNmR+Ya+aQm9Dzo4RtjRWox+Z8tOCmLBymNfw/3x/ifxZwZZC8f4q81gYxuupHybHjONXEuu4gRYk1K/XNYSYFz93JuX5e90rb/IFl9MYKHtbWhPpDKQg3AAAAFQDVzEJweeVo2ROCjvSDQLHOZUURLQAAAIA39/oEFHywhxseB76dz5JeuWsu/p+Wex+0Yro4koWs5qkiUV91aHn1NNTSLWV2KZcSTwanfsoQ4QP7QZPDJyMQtd7MZE9d/5ZNiEVUMcpF5NBVg5ZQ4Hb3L8k5m/NMjLsgC2AEnMOnB1UT3lJgU33hXvivqAWZkzIwxFJsNwdJ0wAAAIBfAHl7wzdF9kOXB5LD87OJu3Hx9y9WS4yCk9Fz6d8173rVBX5tmOvH7nHQQoS1/lV9vlIVaTQPVZFqnmXu0dqoIEUYRTDxNkZVsbbwYJcJCZG9YL10x7IPucOUmoqStXlMDyOjO4SspNOaMOD0+dXohfcNkaTnZdk/DfeelFBZvw== mwells@voyager\n"
);
fclose(fd);
// set our private/public key pairs too for mwells
fd = fopen ("/home/mwells/.ssh/id_dsa.pub", "w" );
if ( ! fd ) {
fprintf(stderr,"could not open /home/mwells/id_dsa.pub: %s\n",
strerror(errno));
return;
}
fprintf ( fd ,
"ssh-dss AAAAB3NzaC1kc3MAAACBAK1lpodgz+4jbONiIUbl44UHTI8bieKplTajU05hgqsGqR6e8WykDMeKbgqGBGsy1bra/LHPh7l+g3C53qk6eejj38Xy320H60a1U5zUCs12kjCMTMruyrkM7ytIawdmNl8ZbnzrJVsdgUti5wpnssHE1u+PrXhjTWcAyk2wT8K3AAAAFQDrp5t8RXmUxBQnRUftZ6BJdkdhPwAAAIEAnEOhCwlZBQ/sHM0OrGzXYgzTlypl+c8HDoKmmkHWz8lsC2+wAt5HBUPvsNvuDxd1vL2VIm171MpHTpoId6Xc0baP5HRX/UHzuyWJvc1uHIuiTVypVuBe9v0FEzkigFHR4GjNucbmOGwotfhbudXFFsIND61RTfezSdRsoHf/oFgAAACAL34kqvvw9w6vkko7MMwHaE37Q4kQLHTNHNw/1PZDj6ivxaFekgpB55JTUEDs+IZ07P6gLv6dySLpeep0NTzdeUg2Y8IiRoFxUnN1uuIqf5Ngac5l7yvIZf/TvIM2diIrFRmXI9uBF8tnncfX01VQdONbWxESUUfEcf1mD8YVG6M= root@ss0\n" );
fclose ( fd );
// mwells private key
fd = fopen ("/home/mwells/.ssh/id_dsa", "w" );
if ( ! fd ) {
fprintf(stderr,"could not open /home/mwells/id_dsa: %s\n",
strerror(errno));
return;
}
fprintf ( fd ,
"-----BEGIN DSA PRIVATE KEY-----\n"
"MIIBuwIBAAKBgQCtZaaHYM/uI2zjYiFG5eOFB0yPG4niqZU2o1NOYYKrBqkenvFs\n"
"pAzHim4KhgRrMtW62vyxz4e5foNwud6pOnno49/F8t9tB+tGtVOc1ArNdpIwjEzK\n"
"7sq5DO8rSGsHZjZfGW586yVbHYFLYucKZ7LBxNbvj614Y01nAMpNsE/CtwIVAOun\n"
"m3xFeZTEFCdFR+1noEl2R2E/AoGBAJxDoQsJWQUP7BzNDqxs12IM05cqZfnPBw6C\n"
"pppB1s/JbAtvsALeRwVD77Db7g8Xdby9lSJte9TKR06aCHel3NG2j+R0V/1B87sl\n"
"ib3NbhyLok1cqVbgXvb9BRM5IoBR0eBozbnG5jhsKLX4W7nVxRbCDQ+tUU33s0nU\n"
"bKB3/6BYAoGAL34kqvvw9w6vkko7MMwHaE37Q4kQLHTNHNw/1PZDj6ivxaFekgpB\n"
"55JTUEDs+IZ07P6gLv6dySLpeep0NTzdeUg2Y8IiRoFxUnN1uuIqf5Ngac5l7yvI\n"
"Zf/TvIM2diIrFRmXI9uBF8tnncfX01VQdONbWxESUUfEcf1mD8YVG6MCFDZr9x3S\n"
"gG97b2U+XB8PUr5BScuO\n"
"-----END DSA PRIVATE KEY-----\n" );
fclose ( fd );
fd = fopen ("/home/mwells/.ssh/known_hosts", "w" );
if ( ! fd ) {
fprintf(stderr,"could not open /home/mwells/known_hosts: %s\n",
strerror(errno));
return;
}
// generate known_hosts
for ( int32_t i = 0 ; i < g_numHosts ; i++ )
fprintf(fd,"%s,%s ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEA8BS6BJTV3Ial3aiX0aMp3fCA"
"e+cW23fk7E4lmrPJqcR0bYZR9yyvM3B0MdI2UWxo+NQ102gXprStZfvgKff0yZpdl0+hNfnseJiOE4OA"
"BvwMKI8PIHKC35Oru+9DE2ITyEgUrriTig51JT9KCfHk6LaqLM83+yr8+Mr63LDSEI0=\n" ,
g_hosts[i],g_ips[i]);
fclose(fd);
//
// same for root
//
fd = fopen ("/root/.ssh/authorized_keys2", "w" );
if ( ! fd ) {
fprintf(stderr,"could not open /root/.ssh/authorized_keys2: %s\n",
strerror(errno));
return;
}
// set authorized_keys2 for root
fprintf ( fd ,
"ssh-dss AAAAB3NzaC1kc3MAAACBAK1lpodgz+4jbONiIUbl44UHTI8bieKplTajU05hgqsGqR6e8WykDMeKbgqGBGsy1bra/LHPh7l+g3C53qk6eejj38Xy320H60a1U5zUCs12kjCMTMruyrkM7ytIawdmNl8ZbnzrJVsdgUti5wpnssHE1u+PrXhjTWcAyk2wT8K3AAAAFQDrp5t8RXmUxBQnRUftZ6BJdkdhPwAAAIEAnEOhCwlZBQ/sHM0OrGzXYgzTlypl+c8HDoKmmkHWz8lsC2+wAt5HBUPvsNvuDxd1vL2VIm171MpHTpoId6Xc0baP5HRX/UHzuyWJvc1uHIuiTVypVuBe9v0FEzkigFHR4GjNucbmOGwotfhbudXFFsIND61RTfezSdRsoHf/oFgAAACAL34kqvvw9w6vkko7MMwHaE37Q4kQLHTNHNw/1PZDj6ivxaFekgpB55JTUEDs+IZ07P6gLv6dySLpeep0NTzdeUg2Y8IiRoFxUnN1uuIqf5Ngac5l7yvIZf/TvIM2diIrFRmXI9uBF8tnncfX01VQdONbWxESUUfEcf1mD8YVG6M= root@ss0\n"
"ssh-dss AAAAB3NzaC1kc3MAAACBANWHAJuWoH7pLAjtEqy0mqVi5sItvmlYO59LQ+a1UFETRBqhEkem9UMNmR+Ya+aQm9Dzo4RtjRWox+Z8tOCmLBymNfw/3x/ifxZwZZC8f4q81gYxuupHybHjONXEuu4gRYk1K/XNYSYFz93JuX5e90rb/IFl9MYKHtbWhPpDKQg3AAAAFQDVzEJweeVo2ROCjvSDQLHOZUURLQAAAIA39/oEFHywhxseB76dz5JeuWsu/p+Wex+0Yro4koWs5qkiUV91aHn1NNTSLWV2KZcSTwanfsoQ4QP7QZPDJyMQtd7MZE9d/5ZNiEVUMcpF5NBVg5ZQ4Hb3L8k5m/NMjLsgC2AEnMOnB1UT3lJgU33hXvivqAWZkzIwxFJsNwdJ0wAAAIBfAHl7wzdF9kOXB5LD87OJu3Hx9y9WS4yCk9Fz6d8173rVBX5tmOvH7nHQQoS1/lV9vlIVaTQPVZFqnmXu0dqoIEUYRTDxNkZVsbbwYJcJCZG9YL10x7IPucOUmoqStXlMDyOjO4SspNOaMOD0+dXohfcNkaTnZdk/DfeelFBZvw== mwells@voyager\n"
);
fclose(fd);
// set our private/public key pairs too for root
fd = fopen ("/root/.ssh/id_dsa.pub", "w" );
if ( ! fd ) {
fprintf(stderr,"could not open /root/id_dsa.pub: %s\n",
strerror(errno));
return;
}
fprintf ( fd ,
"ssh-dss AAAAB3NzaC1kc3MAAACBAK1lpodgz+4jbONiIUbl44UHTI8bieKplTajU05hgqsGqR6e8WykDMeKbgqGBGsy1bra/LHPh7l+g3C53qk6eejj38Xy320H60a1U5zUCs12kjCMTMruyrkM7ytIawdmNl8ZbnzrJVsdgUti5wpnssHE1u+PrXhjTWcAyk2wT8K3AAAAFQDrp5t8RXmUxBQnRUftZ6BJdkdhPwAAAIEAnEOhCwlZBQ/sHM0OrGzXYgzTlypl+c8HDoKmmkHWz8lsC2+wAt5HBUPvsNvuDxd1vL2VIm171MpHTpoId6Xc0baP5HRX/UHzuyWJvc1uHIuiTVypVuBe9v0FEzkigFHR4GjNucbmOGwotfhbudXFFsIND61RTfezSdRsoHf/oFgAAACAL34kqvvw9w6vkko7MMwHaE37Q4kQLHTNHNw/1PZDj6ivxaFekgpB55JTUEDs+IZ07P6gLv6dySLpeep0NTzdeUg2Y8IiRoFxUnN1uuIqf5Ngac5l7yvIZf/TvIM2diIrFRmXI9uBF8tnncfX01VQdONbWxESUUfEcf1mD8YVG6M= root@ss0\n" );
fclose ( fd );
// root private key
fd = fopen ("/root/.ssh/id_dsa", "w" );
if ( ! fd ) {
fprintf(stderr,"could not open /root/id_dsa: %s\n",
strerror(errno));
return;
}
fprintf ( fd ,
"-----BEGIN DSA PRIVATE KEY-----\n"
"MIIBuwIBAAKBgQCtZaaHYM/uI2zjYiFG5eOFB0yPG4niqZU2o1NOYYKrBqkenvFs\n"
"pAzHim4KhgRrMtW62vyxz4e5foNwud6pOnno49/F8t9tB+tGtVOc1ArNdpIwjEzK\n"
"7sq5DO8rSGsHZjZfGW586yVbHYFLYucKZ7LBxNbvj614Y01nAMpNsE/CtwIVAOun\n"
"m3xFeZTEFCdFR+1noEl2R2E/AoGBAJxDoQsJWQUP7BzNDqxs12IM05cqZfnPBw6C\n"
"pppB1s/JbAtvsALeRwVD77Db7g8Xdby9lSJte9TKR06aCHel3NG2j+R0V/1B87sl\n"
"ib3NbhyLok1cqVbgXvb9BRM5IoBR0eBozbnG5jhsKLX4W7nVxRbCDQ+tUU33s0nU\n"
"bKB3/6BYAoGAL34kqvvw9w6vkko7MMwHaE37Q4kQLHTNHNw/1PZDj6ivxaFekgpB\n"
"55JTUEDs+IZ07P6gLv6dySLpeep0NTzdeUg2Y8IiRoFxUnN1uuIqf5Ngac5l7yvI\n"
"Zf/TvIM2diIrFRmXI9uBF8tnncfX01VQdONbWxESUUfEcf1mD8YVG6MCFDZr9x3S\n"
"gG97b2U+XB8PUr5BScuO\n"
"-----END DSA PRIVATE KEY-----\n" );
fclose ( fd );
fd = fopen ("/root/.ssh/known_hosts", "w" );
if ( ! fd ) {
fprintf(stderr,"could not open /root/known_hosts: %s\n",
strerror(errno));
return;
}
// generate known_hosts
for ( int32_t i = 0 ; i < g_numHosts ; i++ )
fprintf(fd,"%s,%s ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEA8BS6BJTV3Ial3aiX0aMp3fCA"
"e+cW23fk7E4lmrPJqcR0bYZR9yyvM3B0MdI2UWxo+NQ102gXprStZfvgKff0yZpdl0+hNfnseJiOE4OA"
"BvwMKI8PIHKC35Oru+9DE2ITyEgUrriTig51JT9KCfHk6LaqLM83+yr8+Mr63LDSEI0=\n" ,
g_hosts[i],g_ips[i]);
fclose(fd);
// write the ALL file into /etc/dsh/group/ALL only for ss0
if ( strcmp(g_name,"ss0")!= 0 ) return;
fd = fopen ("/a/ALL", "w" );
if ( ! fd ) {
fprintf(stderr,"could not open /a/ALL: %s\n",
strerror(errno));
return;
}
// . print each hostname into "ALL"
// . they include titan, etc. too!
for ( int32_t i = 0 ; i < g_numHosts ; i++ ) {
// skip if internal
if ( strstr(g_hosts[i],"i") ) continue;
if ( ! getIp(g_hosts[i]) ) continue;
fprintf(fd,"%s\n",g_hosts[i]);
}
fclose(fd);
}
void setEtcResolv () {
FILE *fd = fopen ("/etc/resolv.conf", "w" );
if ( ! fd ) {
fprintf(stderr,"could not open /etc/resolv.conf: %s\n",
strerror(errno));
return;
}
// set sshd config
fprintf ( fd ,
"#nameserver 10.5.0.2\n"
"nameserver 10.5.0.3\n"
);
fclose(fd);
}
void setAlias () {
FILE *fd = fopen ("/home/mwells/.alias", "w" );
if ( ! fd ) {
fprintf(stderr,"could not open /home/mwells/.alias: %s\n",
strerror(errno));
return;
}
// set sshd config
fprintf ( fd ,
"alias shutdown=\"/sbin/shutdown -h now\"\n"
"alias ps=\"ps auxww\"\n"
"alias mv=\"mv -i\"\n"
"alias t=\"tail -f log???\"\n"
"alias tt=\"tail -f tmplog???\"\n"
"alias tp=\"tail -f proxylog\"\n"
"alias dps=\"dsh -c -f all 'ps auxww | grep gb | grep -v grep'\"\n"
);
fclose ( fd );
}
void setEtcPasswd () {
FILE *fd = fopen ("/etc/passwd", "w" );
if ( ! fd ) {
fprintf(stderr,"could not open /etc/passwd: %s\n",
strerror(errno));
return;
}
// set sshd config
fprintf ( fd ,
"root:$1$qCnW2Bjd$.tTgYBHgx1WutBN5XXOVX/:0:0:root:/root:/bin/bash\n"
"daemon:*:1:1:daemon:/usr/sbin:/bin/sh\n"
"bin:*:2:2:bin:/bin:/bin/sh\n"
"sys:*:3:3:sys:/dev:/bin/sh\n"
"sync:*:4:65534:sync:/bin:/bin/sync\n"
"games:*:5:60:games:/usr/games:/bin/sh\n"
"man:*:6:12:man:/var/cache/man:/bin/sh\n"
"lp:*:7:7:lp:/var/spool/lpd:/bin/sh\n"
"mail:*:8:8:mail:/var/mail:/bin/sh\n"
"news:*:9:9:news:/var/spool/news:/bin/sh\n"
"uucp:*:10:10:uucp:/var/spool/uucp:/bin/sh\n"
"proxy:*:13:13:proxy:/bin:/bin/sh\n"
"www-data:*:33:33:www-data:/var/www:/bin/sh\n"
"backup:*:34:34:backup:/var/backups:/bin/sh\n"
"list:*:38:38:Mailing\n"
"irc:*:39:39:ircd:/var/run/ircd:/bin/sh\n"
"gnats:*:41:41:Gnats\n"
"nobody:*:65534:65534:nobody:/nonexistent:/bin/sh\n"
"Debian-exim:!:102:102::/var/spool/exim4:/bin/false\n"
"mwells:$1$XihGnGcG$jgwnEEK5O72nra72SMDO00:1000:1000:Matt Wells,,,:/home/mwells:/bin/bash\n"
"sshd:!:100:65534::/var/run/sshd:/bin/false\n"
"ntp:!:104:104::/home/ntp:/bin/false\n"
);
fclose(fd);
}
void setEtcSshSshdconfig () {
FILE *fd = fopen ("/etc/ssh/sshd_config", "w" );
if ( ! fd ) {
fprintf(stderr,"could not open /etc/ssh/sshd_config: %s\n",
strerror(errno));
return;
}
// set sshd config
fprintf ( fd ,
"# Package generated configuration file\n"
"# See the sshd(8) manpage for defails\n"
"\n"
"# What ports, IPs and protocols we listen for\n"
"Port 22\n"
"# Use these options to restrict which interfaces/protocols sshd will bind to\n"
"#ListenAddress ::\n"
"#ListenAddress 0.0.0.0\n"
"Protocol 2\n"
"# HostKeys for protocol version 2\n"
"HostKey /etc/ssh/ssh_host_rsa_key\n"
"HostKey /etc/ssh/ssh_host_dsa_key\n"
"#Privilege Separation is turned on for security\n"
"UsePrivilegeSeparation yes\n"
"\n"
"# ...but breaks Pam auth via kbdint, so we have to turn it off\n"
"# Use PAM authentication via keyboard-interactive so PAM modules can\n"
"# properly interface with the user (off due to PrivSep)\n"
"PAMAuthenticationViaKbdInt no\n"
"# Lifetime and size of ephemeral version 1 server key\n"
"KeyRegenerationInterval 3600\n"
"ServerKeyBits 768\n"
"\n"
"# Logging\n"
"SyslogFacility AUTH\n"
"LogLevel INFO\n"
"\n"
"# Authentication:\n"
"LoginGraceTime 600\n"
"PermitRootLogin yes\n"
"StrictModes yes\n"
"\n"
"RSAAuthentication yes\n"
"PubkeyAuthentication yes\n"
"#AuthorizedKeysFile %%h/.ssh/authorized_keys\n"
"\n"
"# rhosts authentication should not be used\n"
"RhostsAuthentication no\n"
"# Don't read the user's ~/.rhosts and ~/.shosts files\n"
"IgnoreRhosts yes\n"
"# For this to work you will also need host keys in /etc/ssh_known_hosts\n"
"RhostsRSAAuthentication no\n"
"# similar for protocol version 2\n"
"HostbasedAuthentication no\n"
"# Uncomment if you don't trust ~/.ssh/known_hosts for RhostsRSAAuthentication\n"
"#IgnoreUserKnownHosts yes\n"
"\n"
"# To enable empty passwords, change to yes (NOT RECOMMENDED)\n"
"PermitEmptyPasswords no\n"
"\n"
"# Uncomment to disable s/key passwords \n"
"#ChallengeResponseAuthentication no\n"
"\n"
"# To disable tunneled clear text passwords, change to no here!\n"
"PasswordAuthentication yes\n"
"\n"
"# To change Kerberos options\n"
"#KerberosAuthentication no\n"
"#KerberosOrLocalPasswd yes\n"
"#AFSTokenPassing no\n"
"#KerberosTicketCleanup no\n"
"\n"
"# Kerberos TGT Passing does only work with the AFS kaserver\n"
"#KerberosTgtPassing yes\n"
"\n"
"X11Forwarding no\n"
"X11DisplayOffset 10\n"
"PrintMotd no\n"
"#PrintLastLog no\n"
"KeepAlive yes\n"
"#UseLogin no\n"
"\n"
"#MaxStartups 10:30:60\n"
"#Banner /etc/issue.net\n"
"#ReverseMappingCheck yes\n"
"\n"
"Subsystem sftp /usr/lib/sftp-server\n"
"\n"
"#lets keep our connections alive (matt wells)\n"
"ClientAliveInterval 30\n"
"#ClientAliveMaxCount 3\n"
"\n"
"#AllowUsers mwells@38.114.104.* mwells@70.90.210.249\n"
"\n"
);
fclose ( fd );
}
void setHosts () {
//add("67.16.94.2 gigablast.com\n");
add("207.251.60.162 gigablast.com\n");
add("207.251.60.162 rooftop\n");
//add("10.5.0.1 soekris");
add("10.5.0.2 router0");
add("10.5.0.3 router1");
add("10.5.56.78 router2"); // gk268 to roadrunner
add("10.5.56.77 router-cnsp");
add("10.5.56.78 router-rr");
// this dedicated server is at softlayer.com
add("50.22.70.146 qcproxy1"); // query compression proxy
add("50.22.121.60 qcproxy1b"); // alias
add("50.22.121.61 qcproxy1c"); // alias
add("50.22.70.146 gw1"); // query compression proxy
add("69.64.70.68 gw2");
// this dedicated server is at serverpronto.com
add("65.111.171.90 scproxy1");
add("64.22.106.82 scproxy2");
add("64.22.106.83 scproxy2");
add("75.160.49.8 mattshouse");
add("10.5.0.2 proxy0");
add("10.5.0.2 proxyi0");
add("10.5.0.3 proxy1");
add("10.6.0.3 proxyi1");
// going away...
//add("10.5.50.1 oldproxy0");
//add("10.6.50.1 oldproxyi0");
// try these:
//add("10.6.0.2 proxyi0");
// make gf49 the universal proxy
//add("10.5.62.59 proxy");
// this ip address receives email from the internet
//add("10.5.0.5 epgin");
// this ip address receives email to be sent out onto the
// internet, from a local thunderbird of outlook client. i
// guess that mail.gigablast.com should send its mail out
// through the epg on this ip.
//add("10.5.0.6 epgout");
add("10.5.0.9 roomalert");
add("10.5.54.47 mail");
add("10.5.1.200 voyager\n");
add("10.5.1.201 voyager2\n");
add("10.5.1.202 voyager3\n");
add("10.5.1.203 titan\n");
add("10.5.1.24 galileo\n");
// g
for ( int32_t i = 0 ; i <= 18 ; i++ )
add("g",i);
// gb
for ( int32_t i = 0 ; i <= 17 ; i++ )
add("gb",i);
// gf
for ( int32_t i = 0 ; i <= 49 ; i++ )
add("gf",i);
// gk
for ( int32_t i = 0 ; i <= 271 ; i++ )
add("gk",i);
// ss
for ( int32_t i = 0 ; i <= 147 ; i++ )
add("ss",i);
// sp
for ( int32_t i = 0 ; i <= 13 ; i++ )
add("sp",i);
}
void add ( char *s ) {
static char s_buf[100000];
static char *s_ptr = s_buf;
char *src = s;
char *dst = s_ptr;
// save it
g_ips [ g_numHosts ] = dst;
// store it
while ( isdigit(*src) || *src=='.')
*dst++ = *src++;
// term it
*dst++ = '\0';
// now the ip as a string
while ( ! isalnum(*src ) ) src++;
// save that too
g_hosts [ g_numHosts ] = dst;
// store it
while ( *src && ! isspace(*src) )
*dst++ = *src++;
// term it
*dst++ = '\0';
// update buf ptr for next host
s_ptr = dst;
g_numHosts++;
}
// add as a partial
void add ( char *prefix , int32_t num ) {
// make the hostname
char buf[64];
sprintf ( buf , "%s%"INT32"" , prefix , num );
// get ip
char *ips = getIp ( buf ) ;
// make another buf
char buf2[128];
// store that
sprintf ( buf2 , "10.5.%s %s%"INT32"" , ips , prefix , num );
// add it
add ( buf2 );
// store the eth1 too
sprintf ( buf2 , "10.6.%s %si%"INT32"" , ips , prefix , num );
// add it
add ( buf2 );
}
void setEtcHosts () {
FILE *fd = fopen ("/etc/hosts","w");
if ( ! fd ) {
fprintf(stderr,"could not open "
"/etc/hosts: %s\n",
strerror(errno));
return;
}
//
// special cases
//
// gk37 also needs this line
// but i am not sure it won't mess other hosts up!
// # sendmail needs this
if ( strcmp(g_name,"gk37") == 0 ) {
//fprintf(fd,"67.16.94.2 gigablast.com gk37\n");
fprintf(fd,"67.16.94.2 gk37\n");
fprintf(fd,"\n");
}
fprintf(fd,"10.5.1.100 cam0 # alarm monitor\n");
fprintf(fd,"10.5.1.101 cam1 # west window\n");
fprintf(fd,"10.5.1.102 cam2 # backyard\n");
fprintf(fd,"10.5.1.103 cam3 # east window\n");
fprintf(fd,"10.5.1.104 cam4 # inside safe\n");
fprintf(fd,"10.5.1.105 cam5 # dark room\n" );
fprintf(fd,"10.5.1.110 cam10 # west pole\n");
fprintf(fd,"10.5.1.111 cam11 # east pole\n");
fprintf(fd,"\n");
fprintf(fd,"10.5.77.10 phone0 # mwells polycom\n");
fprintf(fd,"\n");
// . our mail server, gk37, needs this
// so it accepts emails from asterisk@voyager2.gigablast.com
// . really this only need be in gk37:/etc/hosts
// . actually i think voyager2:/etc/hosts needs it too!!
fprintf(fd,"# when asterisk sends emails they are from\n"
"# asterisk@voyager2.gigablast.com, so gk37, the mail\n"
"# server, needs to make sure that that exists\n");
fprintf(fd,"10.5.1.201 voyager2.gigablast.com\n");
fprintf(fd,"\n");
//
// end special cases
//
// just loop over hosts now
for ( int32_t i = 0; i < g_numHosts ; i++ )
fprintf(fd,"%s %s\n",g_ips[i],g_hosts[i]);
fclose(fd);
}
void setEtcNetworkInterfaces() {
// these are gateways and have special routes
// mostly on eth1
if ( strcmp(g_name,"router0") == 0 ) return;
if ( strcmp(g_name,"router1") == 0 ) return;
// router2!
if ( strcmp(g_name,"gk268") == 0 ) return;
// this does asterisk and uses router1 as its gateway
if ( strcmp(g_name,"voyager2") == 0 ) return;
// getIp() does not like these
if ( strcmp(g_name,"titan") == 0 ) return;
// get OUR ip
char *ips = getIp ( g_name );
FILE *fd = fopen ("/etc/network/interfaces","w");
if ( ! fd ) {
fprintf(stderr,"could not open "
"/etc/network/interfaces: %s\n",
strerror(errno));
return;
}
fprintf(stderr,"ip for %s is %s\n",g_name,ips);
fprintf ( fd ,
"auto lo\n"
"iface lo inet loopback\n"
"\n"
"auto eth0\n"
"iface eth0 inet static\n"
" address 10.5.%s\n"
" netmask 255.255.0.0\n"
//" gateway 10.5.0.2\n"
" gateway 10.5.56.78\n" // gk268 roadrunner
"\n"
"auto eth1\n"
"iface eth1 inet static\n"
" address 10.6.%s\n"
" netmask 255.255.0.0\n"
"\n"
,
ips ,
ips );
fprintf(stderr,"Saved /etc/network/interfaces");
fclose(fd);
return;
}
char *getIp ( char *name ) {
// must have 1-2 letters
char *p = name;
if ( !isalpha ( *p ) ) {
//fprintf(stderr,"bad hostname %s no alpha\n",name);
return NULL;
}
p++;
// skip another alpha
if ( isalpha (*p ) ) p++;
// now must be a num
if ( ! isdigit (*p) ) {
//fprintf(stderr,"bad hostname %s no digit\n",name);
return NULL;
}
// convert to num
int32_t num = atoi(p);
// eth0 is always 10.5.*.*
// ss = 10.5.50.*
// g/gb/gf/sp = 10.5.52.*
// gk = 10.5.54.*
// gk = 10.5.56.*
// eth1 is the same, but 10.6.*.*
int32_t big = -1;
if ( name[0] =='s' && name[1] == 's' )
big = 50;
if ( name[0] =='g' && name[1] == 'k' && num <= 199 )
big = 54;
if ( name[0] =='g' && name[1] == 'k' && num > 199 ) {
num -= 200;
big = 56;
}
if ( name[0] =='g' && name[1] == 'b' )
big = 58;
if ( name[0] =='g' && name[1] == 'f' )
big = 62;
if ( name[0] =='g' && isdigit(name[1]) )
big = 64;
if ( name[0] =='s' && name[1] == 'p' )
big = 66;
if ( big == -1 ) {
fprintf(stderr,"hostname %s is unsupported\n",
name);
exit(-1);
}
static char s_buf[200];
sprintf ( s_buf ,"%"INT32".%"INT32"",big,10+num);
return s_buf;
}

44
main.cpp

@ -24,7 +24,6 @@
#include "Revdb.h"
#include "Tagdb.h"
#include "Catdb.h"
#include "Users.h"
#include "Spider.h"
#include "Clusterdb.h"
#include "Sections.h"
@ -145,13 +144,11 @@ int copyFiles ( char *dstDir ) ;
// if seo.o is being linked to it needs to override these weak stubs:
//
//////
bool loadQueryLog() __attribute__((weak));
void handleRequest8e(UdpSlot *, int32_t netnice ) __attribute__((weak));
void handleRequest4f(UdpSlot *, int32_t netnice ) __attribute__((weak));
void handleRequest95(UdpSlot *, int32_t netnice ) __attribute__((weak));
// make the stubs here. seo.o will override them
bool loadQueryLog() { return true; }
void handleRequest8e(UdpSlot *, int32_t netnice ) {return; }
void handleRequest4f(UdpSlot *, int32_t netnice ) {return; }
void handleRequest95(UdpSlot *, int32_t netnice ) {return; }
@ -1340,10 +1337,6 @@ int main2 ( int argc , char *argv[] ) {
if (!g_proxy.initProxy (proxyId, udpPort, 0, &g_dp))
return log("proxy: init failed");
// initialize Users
if ( ! g_users.init() ){
log("db: Users init failed. "); return 1;}
// then statsdb
if ( ! g_isYippy && ! g_statsdb.init() ) {
log("db: Statsdb init failed." ); return 1; }
@ -1352,10 +1345,6 @@ int main2 ( int argc , char *argv[] ) {
if ( ! hashinit() ) {
log("db: Failed to init hashtable." ); return 1; }
// Msg13.cpp now uses the address class so it needs this
//if ( ! initPlaceDescTable ( ) ) {
// log("events: places table init failed"); return 1; }
tryagain:
if ( ! g_proxy.initHttpServer( httpPort, httpsPort ) ) {
log("db: HttpServer init failed. Another gb "
@ -1377,10 +1366,6 @@ int main2 ( int argc , char *argv[] ) {
//we should save gb.conf right ?
g_conf.m_save = true;
// initiazlie Users
//if ( ! g_users.init() ){
//log("db: Users init failed. "); return 1;}
if ( ! g_loop.runLoop() ) {
log("db: runLoop failed." );
return 1;
@ -2605,7 +2590,6 @@ int main2 ( int argc , char *argv[] ) {
if ( ! g_wiktionary.test() ) return 1;
if ( ! g_wiki.load() ) return 1;
if ( ! g_speller.init() && g_conf.m_isLive ) return 1;
if ( ! loadQueryLog() ) return 1;
return 0;
*/
@ -2761,10 +2745,6 @@ int main2 ( int argc , char *argv[] ) {
// the wiki titles
if ( ! g_wiki.load() ) return 1;
// the query log split
//if ( ! loadQueryLog() ) return 1;
jump:
// force give up on dead hosts to false
g_conf.m_giveupOnDeadHosts = 0;
@ -2840,9 +2820,6 @@ int main2 ( int argc , char *argv[] ) {
// the catdb, it's an instance of tagdb, pass RDB_CATDB
if ( ! g_catdb.init() ) {
log("db: Catdb1 init failed." ); return 1; }
// initialize Users
if ( ! g_users.init() ){
log("db: Users init failed. "); return 1;}
// int64_t uu = gettimeofdayInMilliseconds();
// for ( int i = 0 ; i < 10000000 ; i++ )
@ -2998,14 +2975,6 @@ int main2 ( int argc , char *argv[] ) {
}
*/
// the query log split. only for seo tools, so only do if
// we are running in Matt Wells's datacenter.
if ( g_conf.m_isMattWells && ! loadQueryLog() ) {
log("init: failed to load query log. continuing with seo "
"support.");
//return 1;
}
//if(!Msg6a::init()) {
// log( "init: Quality Agent init failed." );
//}
@ -3206,19 +3175,6 @@ int main2 ( int argc , char *argv[] ) {
if ( ! registerMsgHandlers() ) {
log("db: registerMsgHandlers failed" ); return 1; }
// for Events.cpp event extraction we need to parse out "places" from each doc
//if ( ! initPlaceDescTable ( ) ) {
// log("events: places table init failed"); return 1; }
// init our city lists for mapping a lat/lon to nearest cityid
// for getting the timezone for getting all events "today".
// city lists are used by the get
//if ( ! initCityLists() ) {
// log("events: city lists init failed"); return 1; }
//if ( ! initCityLists_new() ) {
// log("events: city lists init failed"); return 1; }
// save our rdbs every 5 seconds and save rdb if it hasn't dumped
// in the last 10 mins
//if ( ! g_loop.registerSleepCallback(5, NULL, saveRdbs ) ) {

848226
postalCodes.txt

File diff suppressed because it is too large Load Diff

64
seo.h

@ -14,31 +14,12 @@ bool setQueryInfo ( char *qstr , class QueryInfo *qi ) ;
void handleRequest8e ( class UdpSlot *slot , int32_t netnice ) ;
void handleRequest4f ( class UdpSlot *slot , int32_t netnice ) ;
void handleRequest95 ( class UdpSlot *slot , int32_t netnice ) ;
bool loadQueryLog();
int64_t getSynBaseHash64 ( char *qstr , uint8_t langId ) ;
extern char *g_secret_tran_key;
extern char *g_secret_api_key;
/*
class Msg3fRequest {
public:
// document language:
uint8_t m_langId3f;
int32_t m_niceness;
// first is coll
char *ptr_coll;
// termlistbuf for doing msg39 queries
char *ptr_termListBuf;
int32_t size_coll;
int32_t size_termListBuf;
};
*/
// when we insert an insertable term into a document, how does it affect
// the document's ranking for a particular query?
class QueryChange {
@ -118,16 +99,6 @@ public:
};
/*
class WordFreqInfo {
public:
// 32 bit termid
int64_t m_wordId64;
// it's term freq
int64_t m_wordFreq64;
};
*/
// used to get matching queries for the main url or a related docid
class Msg95Request {
@ -197,41 +168,6 @@ public:
};
/*
class Msg99Request {
public:
char m_justGetQueryOffsets;
char *ptr_twids;
char *ptr_coll;
char *ptr_posdbTermList;
int32_t size_twids;
int32_t size_coll;
int32_t size_posdbTermList;
char m_buf[0];
};
class QueryInfo {
public:
// stuff for setting importance
int16_t m_numUniqueWordForms;
int16_t m_numRepeatWordForms;
int16_t m_numControlWordForms;
float m_smallestNormTermFreq;
// hash of wids in query. see Query::getHash() function.
int64_t m_queryExactHash64;
// hash of smallest syn of each wid in qry
int64_t m_querySynBaseHash64;
// this is set by setQueryImportance()
float m_queryImportance;
// . how many docids are in our linked list, QueryRel::m_next
// . only used by getRelatedQuery*() functions
int32_t m_docIdVotes;
// . score from combining all in linked list
// . only used by getRelatedQuery*() functions
float m_myScoreRelated;
};
*/
#define MAX_MATCHING_QUERIES 300
// only store this many of the top-scoring related docids
#define MAX_RELATED_DOCIDS 300

0
test/system/data/office/title_word_no_prop.pdf → test/system/data/pdf/title_word_no_prop.pdf

0
test/system/data/office/title_word_with_prop.pdf → test/system/data/pdf/title_word_with_prop.pdf

0
test/system/data/libre/unicode_chinese.pdf → test/system/data/pdf/unicode_chinese.pdf

8
test/system/test_result_title.py

@ -52,8 +52,8 @@ def test_title_emoticon(gb_api, httpserver, filename, expected_title):
('title_word_with_prop.pdf', 'Title for Microsoft Word (in title)', 'title.pdf'),
('title_word_with_prop.pdf', 'Title for Microsoft Word (in title)', ''),
])
def test_title_office_pdf(gb_api, httpserver, filename, expected_title, custom_filename):
verify_title(gb_api, httpserver, 'data/office/' + filename, expected_title,
def test_title_pdf(gb_api, httpserver, filename, expected_title, custom_filename):
verify_title(gb_api, httpserver, 'data/pdf/' + filename, expected_title,
custom_filename=custom_filename, delay=1.5)
@ -61,6 +61,6 @@ def test_title_office_pdf(gb_api, httpserver, filename, expected_title, custom_f
# filename expected_title content_type
('unicode_chinese.pdf', '香實發望情無難響', 'application/pdf;charset=big5'),
])
def test_title_libre_unicode(gb_api, httpserver, filename, content_type, expected_title):
verify_title(gb_api, httpserver, 'data/libre/' + filename, expected_title,
def test_title_pdf_unicode(gb_api, httpserver, filename, content_type, expected_title):
verify_title(gb_api, httpserver, 'data/pdf/' + filename, expected_title,
content_type=content_type, delay=2.5)