Forks/open-source-search-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wtf i did not modify these files.

Browse Source
This commit is contained in:
Matt Wells 2013-12-18 15:26:50 -08:00
parent 2b10a3327d
commit 356836812d
10 changed files with 405 additions and 676 deletions

5
CollectionRec.cpp

@ -600,11 +600,6 @@ bool CollectionRec::hasSearchPermission ( TcpSocket *s , long encapIp ) {
bool CollectionRec::rebuildUrlFilters ( ) {
// only for diffbot custom crawls
if ( m_isCustomCrawl != 1 && // crawl api
m_isCustomCrawl != 2 ) // bulk api
return true;
char *ucp = m_diffbotUrlCrawlPattern.getBufStart();
if ( ucp && ! ucp[0] ) ucp = NULL;

91
Collectiondb.cpp

@ -641,6 +641,12 @@ bool Collectiondb::deleteRec ( char *coll , WaitEntry *we ) {
// "spiders are enabled or active.");
// return false;
//}
// do not allow this if in repair mode
if ( g_repairMode > 0 ) {
log("admin: Can not delete collection while in repair mode.");
g_errno = EBADENGINEER;
return true;
}
// ensure it's not NULL
if ( ! coll ) {
log(LOG_LOGIC,"admin: Collection name to delete is NULL.");
@ -649,21 +655,11 @@ bool Collectiondb::deleteRec ( char *coll , WaitEntry *we ) {
}
// find the rec for this collection
collnum_t collnum = getCollnum ( coll );
return deleteRec2 ( collnum , we );
}
bool Collectiondb::deleteRec2 ( collnum_t collnum , WaitEntry *we ) {
// do not allow this if in repair mode
if ( g_repairMode > 0 ) {
log("admin: Can not delete collection while in repair mode.");
g_errno = EBADENGINEER;
return true;
}
// bitch if not found
if ( collnum < 0 ) {
g_errno = ENOTFOUND;
log(LOG_LOGIC,"admin: Collection #%li is bad, "
"delete failed.",(long)collnum);
log(LOG_LOGIC,"admin: Collection \"%s\" not found, "
"delete failed.",coll);
return true;
}
CollectionRec *cr = m_recs [ collnum ];
@ -694,11 +690,8 @@ bool Collectiondb::deleteRec2 ( collnum_t collnum , WaitEntry *we ) {
// "spiders and wait for them to exit.");
// return false;
//}
char *coll = cr->m_coll;
// note it
log("coll: deleting coll \"%s\"",coll);
log("coll: deleting coll \"%s\"",cr->m_coll);
// we need a save
m_needsSave = true;
@ -778,31 +771,15 @@ bool Collectiondb::deleteRec2 ( collnum_t collnum , WaitEntry *we ) {
// . returns false if blocked and will call callback
bool Collectiondb::resetColl ( char *coll , WaitEntry *we , bool purgeSeeds) {
// save parms in case we block
we->m_purgeSeeds = purgeSeeds;
// ensure it's not NULL
if ( ! coll ) {
log(LOG_LOGIC,"admin: Collection name to delete is NULL.");
g_errno = ENOCOLLREC;
return true;
}
// get the CollectionRec for "test"
CollectionRec *cr = getRec ( coll ); // "test" );
// must be there. if not, we create test i guess
if ( ! cr ) {
log("db: could not get coll rec \"%s\" to reset", coll);
char *xx=NULL;*xx=0;
}
return resetColl2 ( cr->m_collnum,we,purgeSeeds);
}
bool Collectiondb::resetColl2(collnum_t collnum,WaitEntry *we,bool purgeSeeds){
// save parms in case we block
we->m_purgeSeeds = purgeSeeds;
// now must be "test" only for now
//if ( strcmp(coll,"test") ) { char *xx=NULL;*xx=0; }
// no spiders can be out. they may be referencing the CollectionRec
@ -813,7 +790,6 @@ bool Collectiondb::resetColl2(collnum_t collnum,WaitEntry *we,bool purgeSeeds){
// "spiders are enabled or active.");
// return false;
//}
// do not allow this if in repair mode
if ( g_repairMode > 0 ) {
log("admin: Can not delete collection while in repair mode.");
@ -821,7 +797,7 @@ bool Collectiondb::resetColl2(collnum_t collnum,WaitEntry *we,bool purgeSeeds){
return true;
}
log("admin: resetting collnum %li",(long)collnum);
log("admin: resetting coll \"%s\"",coll);
// CAUTION: tree might be in the middle of saving
// we deal with this in Process.cpp now
@ -840,6 +816,14 @@ bool Collectiondb::resetColl2(collnum_t collnum,WaitEntry *we,bool purgeSeeds){
// get the CollectionRec for "test"
CollectionRec *cr = getRec ( coll ); // "test" );
// must be there. if not, we create test i guess
if ( ! cr ) {
log("db: could not get coll rec \"%s\" to reset", coll);
char *xx=NULL;*xx=0;
}
// inc the rec ptr buf i guess
long need = (m_numRecs+1)*sizeof(CollectionRec *);
@ -857,8 +841,6 @@ bool Collectiondb::resetColl2(collnum_t collnum,WaitEntry *we,bool purgeSeeds){
// update length of used bytes
m_recPtrBuf.setLength ( need );
CollectionRec *cr = m_recs[collnum];
if ( ! cr ) { char *xx=NULL;*xx=0; }
/*
// make sure an update not in progress
@ -1003,7 +985,7 @@ bool Collectiondb::resetColl2(collnum_t collnum,WaitEntry *we,bool purgeSeeds){
if ( dir ) {
//g_errno = EEXIST;
log("admin: Trying to create collection %s but "
"directory %s already exists on disk.",cr->m_coll,dname);
"directory %s already exists on disk.",coll,dname);
}
if ( ::mkdir ( dname ,
S_IRUSR | S_IWUSR | S_IXUSR |
@ -1201,29 +1183,10 @@ collnum_t Collectiondb::getCollnum ( char *coll , long clen ) {
*/
}
//collnum_t Collectiondb::getNextCollnum ( collnum_t collnum ) {
// for ( long i = (long)collnum + 1 ; i < m_numRecs ; i++ )
// if ( m_recs[i] ) return i;
// // no next one, use -1
// return (collnum_t) -1;
//}
// what collnum will be used the next time a coll is added?
collnum_t Collectiondb::reserveCollNum ( ) {
if ( m_numRecs < 0x7fff ) {
collnum_t next = m_numRecs;
m_numRecs++;
return next;
}
// search for an empty slot
for ( long i = 0 ; i < m_numRecs ; i++ ) {
if ( ! m_recs[i] ) return (collnum_t)i;
}
log("colldb: no new collnum available. consider upping collnum_t");
// none available!!
return -1;
collnum_t Collectiondb::getNextCollnum ( collnum_t collnum ) {
for ( long i = (long)collnum + 1 ; i < m_numRecs ; i++ )
if ( m_recs[i] ) return i;
// no next one, use -1
return (collnum_t) -1;
}

14
Collectiondb.h

@ -20,14 +20,8 @@ public:
char *m_coll;
bool m_purgeSeeds;
class CollectionRec *m_cr;
// ptr to list of parm recs for Parms.cpp
char *m_parmPtr;
char *m_parmEnd;
class UdpSlot *m_slot;
bool m_doRebuilds;
collnum_t m_collnum;
bool m_registered;
long m_errno;
};
class Collectiondb {
@ -84,10 +78,7 @@ class Collectiondb {
// . uses the root collection record!
bool isAdmin ( class HttpRequest *r , class TcpSocket *s );
//collnum_t getNextCollnum ( collnum_t collnum );
// what collnum will be used the next time a coll is added?
collnum_t reserveCollNum ( ) ;
collnum_t getNextCollnum ( collnum_t collnum );
long long getLastUpdateTime () { return m_lastUpdateTime; };
// updates m_lastUpdateTime so g_spiderCache know when to reload
@ -117,14 +108,11 @@ class Collectiondb {
// returns false if blocked, true otherwise.
bool deleteRec ( char *coll , WaitEntry *we );
bool deleteRec2 ( collnum_t collnum , WaitEntry *we ) ;
//bool updateRec ( CollectionRec *newrec );
bool deleteRecs ( class HttpRequest *r ) ;
// returns false if blocked, true otherwise.
bool resetColl ( char *coll , WaitEntry *we , bool purgeSeeds );
bool resetColl2 ( collnum_t collnum, WaitEntry *we , bool purgeSeeds );
// . keep up to 128 of them, these reference into m_list
// . COllectionRec now includes m_needsSave and m_lastUpdateTime

12
Hostdb.h

@ -235,9 +235,6 @@ class Host {
// network to save on local loop bandwidth costs
char m_type;
bool isProxy() { return (m_type == HT_PROXY); };
bool isGrunt() { return (m_type == HT_GRUNT); };
// for m_type == HT_QCPROXY, we forward the query to the regular proxy
// at this Ip:Port. we should receive a compressed 0xfd reply and
// we uncompress it and return it to the browser.
@ -267,14 +264,6 @@ class Host {
char m_inSync ;
char m_isPermanentOutOfSync ;
// . used by Parms.cpp for broadcasting parm change requests
// . each parm change request has an id
// . this let's us know which id is in progress and what the last
// id completed was
long m_currentParmIdInProgress;
long m_lastParmIdCompleted;
class ParmNode *m_currentNodePtr;
char m_requestBuf[MAX_PING_SIZE];
};
@ -330,7 +319,6 @@ class Hostdb {
long getMyMachineNum ( ) { return m_myMachineNum; };
unsigned long getLoopbackIp ( ) { return m_loopbackIp; };
Host *getMyHost ( ) { return m_myHost; };
bool amProxy ( ) { return m_myHost->isProxy(); };
Host *getMyShard ( ) { return m_myShard; };
long getMyShardNum ( ) { return m_myHost->m_shardNum; };
bool isMyIp ( unsigned long ip ) {

3
Pages.cpp

@ -425,8 +425,7 @@ void doneBroadcastingParms ( void *state ) {
sock->m_handyBuf.purge();
// set another http request again
HttpRequest r;
//bool status = r.set ( sock->m_readBuf , sock->m_readOffset , sock ) ;
r.set ( sock->m_readBuf , sock->m_readOffset , sock ) ;
bool status = r.set ( sock->m_readBuf , sock->m_readOffset , sock ) ;
// we stored the page # below
WebPage *pg = &s_pages[sock->m_pageNum];
// call the page specifc function which will send data back on socket

853
Parms.cpp

File diff suppressed because it is too large Load Diff

18
Parms.h

@ -132,9 +132,6 @@ class Parm {
//bool (*m_func)(TcpSocket *s , HttpRequest *r,
// bool (*cb)(TcpSocket *s , HttpRequest *r));
bool (*m_func)(char *parmRec);
// some functions can block, like when deleting a coll because
// the tree might be saving, so they take a "we" ptr
bool (*m_func2)(char *parmRec,class WaitEntry *we);
long m_plen; // offset of length for TYPE_STRINGS (m_htmlHeadLen...)
char m_group; // start of a new group of controls?
// m_priv = 1 means gigablast's software license clients cannot see
@ -167,8 +164,6 @@ class Parm {
bool getValueAsBool ( class SearchInput *si ) ;
long getValueAsLong ( class SearchInput *si ) ;
char * getValueAsString ( class SearchInput *si ) ;
long getNumInArray ( collnum_t collnum ) ;
};
#define MAX_PARMS 940
@ -255,8 +250,8 @@ class Parms {
bool (*callback)(TcpSocket *s , HttpRequest *r),
class CollectionRec *newcr = NULL );
bool insertParm ( long i , long an , char *THIS ) ;
bool removeParm ( long i , long an , char *THIS ) ;
//void insertParm ( long i , long an , char *THIS ) ;
//void removeParm ( long i , long an , char *THIS ) ;
void setParm ( char *THIS, Parm *m, long mm, long j, char *s,
bool isHtmlEncoded , bool fromRequest ) ;
@ -327,23 +322,18 @@ class Parms {
bool convertHttpRequestToParmList (HttpRequest *hr,SafeBuf *parmList);
Parm *getParmFast2 ( long cgiHash32 ) ;
Parm *getParmFast1 ( char *cgi , long *occNum ) ;
bool broadcastParmList ( SafeBuf *parmList ,
void *state ,
void (* callback)(void *) ,
bool sendToGrunts = true ,
bool sendToProxies = false );
bool broadcastParmList ( SafeBuf *parmList ) ;
bool doParmSendingLoop ( ) ;
bool syncParmsWithHost0 ( ) ;
bool makeSyncHashList ( SafeBuf *hashList ) ;
long getNumInArray ( collnum_t collnum ) ;
bool addAllParmsToList ( SafeBuf *parmList, collnum_t collnum ) ;
bool updateParm ( char *rec , class WaitEntry *we ) ;
bool updateParm ( char *rec ) ;
//
// end new functions
//
bool m_inSyncWithHost0;
bool m_isDefaultLoaded;

26
Process.cpp

@ -733,29 +733,13 @@ bool Process::gotPower ( TcpSocket *s ) {
log("powermo: sending notice to all hosts.");
SafeBuf parmList;
// add the parm rec as a parm cmd
if ( ! g_parms.addNewParmToList1 ( &parmList,
(collnum_t)-1,
NULL, // parmval (argument)
-1, // collnum (-1 -> globalconf)
"poweron") ) // CommandPowerOn()!
return true;
// . use the broadcast call here so things keep their order!
// . we do not need a callback when they have been completely
// broadcasted to all hosts so use NULL for that
g_parms.broadcastParmList ( &parmList , NULL , NULL );
// . turn off spiders
// . also show that power is off now!
//if ( ! m_msg28.massConfig ( m_r.getRequest() ,
// NULL , // state
// doneCmdWrapper ) )
// // return false if this blocked
// return false;
if ( ! m_msg28.massConfig ( m_r.getRequest() ,
NULL , // state
doneCmdWrapper ) )
// return false if this blocked
return false;
// . hmmm.. it did not block
// . this does not block either
doneCmdWrapper ( NULL );

4
Spider.cpp

@ -4347,8 +4347,8 @@ void doneSendingNotification ( void *state ) {
// we have to send these two parms to all in cluster now
SafeBuf parmList;
g_parms.addCurrentParmToList1 ( &parmList , cr , "spiderRoundNum" );
g_parms.addCurrentParmToList1 ( &parmList , cr , "spiderRoundStart" );
g_parms.addParmToList2 ( &parmList , cr , "spiderRoundNum" );
g_parms.addParmToList2 ( &parmList , cr , "spiderRoundStart" );
// this uses msg4 so parm ordering is guaranteed
g_parms.broadcastParmList ( &parmList , NULL , NULL );

55
main.cpp

@ -79,7 +79,7 @@
//#include "Msg34.h"
#include "Msg35.h"
//#include "Msg24.h"
//#include "Msg28.h"
#include "Msg28.h"
//#include "Msg30.h"
//#include "MsgB.h"
//#include "Msg3e.h"
@ -3381,6 +3381,7 @@ long checkDirPerms ( char *dir ) {
// save them all
static void doCmdAll ( int fd, void *state ) ;
static void doneCmdAll ( void *state );
static bool s_sendToHosts;
static bool s_sendToProxies;
static long s_hostId;
@ -3426,45 +3427,9 @@ bool doCmd ( const char *cmd , long hostId , char *filename ,
return true;
}
//static Msg28 s_msg28;
//static TcpSocket s_s;
void doneCmdAll ( void *state ) {
/*
if ( s_sendToProxies ){
if ( ! g_loop.registerSleepCallback(1, NULL, doCmdAll,0 ) ){
log("admin: Loop init failed.");
exit ( 0 );
}
return;
}
*/
log("cmd: completed command");
exit ( 0 );
}
static Msg28 s_msg28;
static TcpSocket s_s;
void doCmdAll ( int fd, void *state ) {
SafeBuf parmList;
// returns false and sets g_errno on error
if ( ! g_parms.convertHttpRequestToParmList ( &s_r , &parmList ) ) {
log("cmd: error converting command: %s",mstrerror(g_errno));
return;
}
// returns true with g_errno set on error
if ( g_parms.broadcastParmList ( &parmList ,
NULL ,
doneCmdAll , // callback when done
s_sendToHosts ,
s_sendToProxies ) ) {
log("cmd: error sending command: %s",mstrerror(g_errno));
return;
}
// wait for it
log("cmd: sent command");
/*
bool status = true;
if ( s_sendToHosts ){
s_sendToHosts = false;
@ -3481,8 +3446,18 @@ void doCmdAll ( int fd, void *state ) {
g_loop.unregisterSleepCallback ( NULL, doCmdAll );
// if we did not block, call the callback directly
if ( status ) doneCmdAll(NULL);
*/
}
void doneCmdAll ( void *state ) {
if ( s_sendToProxies ){
if ( ! g_loop.registerSleepCallback(1, NULL, doCmdAll,0 ) ){
log("admin: Loop init failed.");
exit ( 0 );
}
return;
}
exit ( 0 );
}
// copy a collection from one network to another (defined by 2 hosts.conf's)
int collcopy ( char *newHostsConf , char *coll , long collnum ) {