Merge branch 'master' into nomerge2

This commit is contained in:
Ai Lin Chia
2016-12-14 10:56:44 +01:00
10 changed files with 81 additions and 101 deletions

@ -503,7 +503,7 @@ bool Collectiondb::addRdbBasesForCollRec ( CollectionRec *cr ) {
/// this deletes the collection, not just part of a reset.
bool Collectiondb::deleteRec2 ( collnum_t collnum ) {
// do not allow this if in repair mode
if ( g_repair.isRepairActive() && g_repair.m_collnum == collnum ) {
if ( g_repair.isRepairActive() && g_repair.isRepairingColl(collnum) ) {
log(LOG_WARN, "admin: Can not delete collection while in repair mode.");
g_errno = EBADENGINEER;
return true;
@ -723,7 +723,7 @@ bool Collectiondb::setRecPtr ( collnum_t collnum , CollectionRec *cr ) {
// . returns true with g_errno set on error
bool Collectiondb::resetColl2( collnum_t oldCollnum, collnum_t newCollnum, bool purgeSeeds ) {
// do not allow this if in repair mode
if ( g_repair.isRepairActive() && g_repair.m_collnum == oldCollnum ) {
if ( g_repair.isRepairActive() && g_repair.isRepairingColl(oldCollnum) ) {
log(LOG_WARN, "admin: Can not delete collection while in repair mode.");
g_errno = EBADENGINEER;
return true;

@ -278,7 +278,8 @@ Conf::Conf ( ) {
m_bypassValidation = false;
m_maxCallbackDelay = 0;
m_repairingEnabled = false;
m_maxRepairSpiders = 0;
m_maxRepairinjections = 0;
m_rebuildHost = -1;
m_repairMem = 0;
m_fullRebuild = true;
m_rebuildAddOutlinks = false;

3
Conf.h

@ -473,9 +473,10 @@ class Conf {
// used by Repair.cpp
bool m_repairingEnabled;
int32_t m_maxRepairSpiders;
int32_t m_maxRepairinjections;
int64_t m_repairMem;
SafeBuf m_collsToRepair;
int32_t m_rebuildHost;
bool m_fullRebuild;
bool m_rebuildAddOutlinks;
bool m_rebuildRecycleLinkInfo;

@ -8,6 +8,7 @@
#include "Profiler.h"
#include "Repair.h"
#include "Process.h"
#include "UdpServer.h"
#include "Conf.h"
#include "GbMutex.h"
#include "ScopedLock.h"

@ -9197,7 +9197,6 @@ void Parms::init ( ) {
///////////////////////////////////////////
// PAGE REPAIR CONTROLS
///////////////////////////////////////////
#ifndef PRIVACORE_SAFE_VERSION
m->m_title = "rebuild mode enabled";
m->m_desc = "If enabled, gigablast will rebuild the rdbs as "
@ -9215,9 +9214,8 @@ void Parms::init ( ) {
m->m_title = "collection to rebuild";
m->m_xml = "collectionToRebuild";
m->m_desc = "Name of collection to rebuild.";
// m->m_desc = "Comma or space separated list of the collections "
// "to rebuild.";
m->m_desc = "Comma or space separated list of the collections "
"to rebuild.";
m->m_cgi = "rctr"; // repair collections to repair
m->m_off = offsetof(Conf,m_collsToRepair);
m->m_type = TYPE_SAFEBUF;//STRING;
@ -9230,11 +9228,23 @@ void Parms::init ( ) {
m++;
m->m_title = "Limit rebuild to a host";
m->m_desc = "Only rebuild on this host. -1 for all";
m->m_cgi = "rebuild_host";
m->m_off = offsetof(Conf,m_rebuildHost);
m->m_type = TYPE_LONG;
m->m_page = PAGE_REPAIR;
m->m_obj = OBJ_CONF;
m->m_def = "-1";
m->m_group = false;
m++;
m->m_title = "memory to use for rebuild";
m->m_desc = "In bytes.";
m->m_cgi = "rmtu"; // repair mem to use
m->m_off = offsetof(Conf,m_repairMem);
m->m_type = TYPE_LONG;
m->m_type = TYPE_LONG_LONG;
m->m_page = PAGE_REPAIR;
m->m_obj = OBJ_CONF;
m->m_def = "200000000";
@ -9246,7 +9256,7 @@ void Parms::init ( ) {
m->m_desc = "Maximum number of outstanding injections for "
"rebuild.";
m->m_cgi = "mrps";
m->m_off = offsetof(Conf,m_maxRepairSpiders);
m->m_off = offsetof(Conf,m_maxRepairinjections);
m->m_type = TYPE_LONG;
m->m_page = PAGE_REPAIR;
m->m_obj = OBJ_CONF;
@ -9377,7 +9387,6 @@ void Parms::init ( ) {
m->m_group = false;
m++;
#endif
///////////////////////////////////////////
// END PAGE REPAIR //
///////////////////////////////////////////

@ -1504,7 +1504,7 @@ bool Rdb::addList ( collnum_t collnum , RdbList *list, int32_t niceness ) {
// they are bigger. it may add to indexdb/datedb
if ( g_repair.isRepairActive() &&
// but only check for collection we are repairing/rebuilding
collnum == g_repair.m_collnum &&
g_repair.isRepairingColl(collnum) &&
// exception, spider status docs can be deleted from titledb
// if user turns off 'index spider replies' before doing
// the rebuild, when not rebuilding titledb.

@ -21,8 +21,8 @@ static const int32_t s_defaultMaxPendingTimeMs = 5000;
static const uint32_t s_defaultMaxPendingSize = 2000000;
// larger number equals more memory used; but faster generateIndex
// 40000000 * 8 bytes = 320 megabytes
static const uint32_t s_generateMaxPendingSize = 40000000;
// 10000000 * 8 bytes = ~80 megabytes
static const uint32_t s_generateMaxPendingSize = 10000000;
static const int64_t s_rdbIndexCurrentVersion = 0;

@ -22,12 +22,6 @@
#include <fcntl.h>
static void repairWrapper ( int fd , void *state ) ;
static void loopWrapper ( void *state , RdbList *list , Msg5 *msg5 ) ;
static bool saveAllRdbs ( void *state , void (* callback)(void *state) ) ;
static bool anyRdbNeedsSave ( ) ;
static void doneSavingRdb ( void *state );
char g_repairMode = 0;
@ -81,30 +75,15 @@ static Rdb **getAllRdbs ( int32_t *nsr ) {
Repair::Repair() {
// Coverity
m_completed = false;
m_needsCallback = false;
m_docQuality = 0;
m_docId = 0;
m_isDelete = false;
m_totalMem = 0;
m_stage = 0;
m_tfn = 0;
m_count = 0;
m_updated = false;
m_nextTitledbKey = 0;
m_nextSpiderdbKey = 0;
m_nextPosdbKey = 0;
m_nextLinkdbKey = 0;
m_endKey = 0;
m_uh48 = 0;
m_priority = 0;
m_contentHash = 0;
m_clusterdbKey = 0;
m_spiderdbKey = 0;
memset(m_srBuf, 0, sizeof(m_srBuf));
memset(m_tmpBuf, 0, sizeof(m_tmpBuf));
m_chksum1LongLong = 0;
m_isNew = false;
m_SAVE_START = 0;
m_lastDocId = 0;
m_prevDocId = 0;
@ -141,8 +120,6 @@ Repair::Repair() {
m_rebuildRoots = true;
m_rebuildNonRoots = true;
m_collnum = 0;
m_newCollLen = 0;
m_newCollnum = 0;
m_colli = 0;
m_numColls = 0;
m_SAVE_END = 0;
@ -155,7 +132,6 @@ Repair::Repair() {
m_saveRepairState = false;
m_isRetrying = false;
memset(m_newColl, 0, sizeof(m_newColl));
memset(&m_collOffs, 0, sizeof(m_collOffs));
memset(&m_collLens, 0, sizeof(m_collLens));
}
@ -168,7 +144,6 @@ bool Repair::init ( ) {
m_saveRepairState = false;
m_isRetrying = false;
m_needsCallback = false;
m_completed = false;
if( ! g_loop.registerSleepCallback( 1 , NULL , repairWrapper ) ) {
log(LOG_WARN, "repair: Failed register callback.");
return false;
@ -176,13 +151,13 @@ bool Repair::init ( ) {
return true;
}
bool Repair::isRepairActive() {
bool Repair::isRepairActive() const {
return g_repairMode >= 4;
}
// . call this once every second
// . this is responsible for advancing from one g_repairMode to the next
void repairWrapper ( int fd , void *state ) {
void Repair::repairWrapper(int fd, void *state) {
g_errno = 0;
@ -193,6 +168,20 @@ void repairWrapper ( int fd , void *state ) {
// when dumped, would mess up our scan.
if ( ! g_conf.m_repairingEnabled ) return;
if(g_conf.m_rebuildHost>=0 && getMyHostId()!=g_conf.m_rebuildHost) {
log(LOG_DEBUG, "repair: rebuild/repair is for host %d only and not us", g_conf.m_rebuildHost);
g_conf.m_repairingEnabled = false;
return;
}
#ifndef PRIVACORE_SAFE_VERSION
if(g_conf.m_rebuildHost<0) {
log(LOG_DEBUG, "repair: rebuild/repair specified for all but this is not allowed in this safer version");
g_conf.m_repairingEnabled = false;
return;
}
#endif
// if the power went off
if ( ! g_process.m_powerIsOn ) return;
@ -424,9 +413,6 @@ void repairWrapper ( int fd , void *state ) {
// ready to reset
g_repairMode = 8;
// mark it
g_repair.m_completed = true;
}
// go back to 0 once all hosts do not equal 5
@ -467,10 +453,7 @@ void Repair::initScan ( ) {
// reset some stuff for the titledb scan
m_nextTitledbKey.setMin();
m_nextSpiderdbKey.setMin();
m_lastSpiderdbKey.setMin();
m_nextPosdbKey.setMin ();
m_nextLinkdbKey.setMin ();
m_endKey.setMax();
m_titleRecList.reset();
m_count = 0;
@ -558,7 +541,7 @@ void Repair::initScan ( ) {
// advance the number of collections
m_numColls++;
// get the next collection if under 100 collections still
if ( m_numColls < 100 ) goto collLoop;
if ( m_numColls < maxCollections ) goto collLoop;
}
// split the mem we have available among the rdbs
@ -751,7 +734,7 @@ void Repair::getNextCollToRepair ( ) {
}
void loopWrapper ( void *state , RdbList *list , Msg5 *msg5 ) {
void Repair::loopWrapper(void *state, RdbList *list, Msg5 *msg5) {
Repair *THIS = (Repair *)state;
THIS->m_msg5InUse = false;
THIS->loop(NULL);
@ -837,8 +820,6 @@ bool Repair::load ( ) {
// resume titledb scan?
m_nextTitledbKey = m_lastTitledbKey;
// resume spiderdb scan?
m_nextSpiderdbKey = m_lastSpiderdbKey;
// reinstate the valuable vars
m_cr = g_collectiondb.m_recs [ m_collnum ];
@ -927,7 +908,7 @@ bool Repair::loop ( void *state ) {
if( g_conf.m_logTraceRepairs ) log(LOG_TRACE,"%s:%s:%d: STAGE_TITLEDB_3", __FILE__, __func__, __LINE__);
// if we have maxed out our injects, wait for one to come back
if ( m_numOutstandingInjects >= g_conf.m_maxRepairSpiders ) {
if ( m_numOutstandingInjects >= g_conf.m_maxRepairinjections ) {
m_allowInjectToLoop = true;
return false;
}
@ -940,7 +921,7 @@ bool Repair::loop ( void *state ) {
//return false; // (state)
// try to launch another
if ( m_numOutstandingInjects<g_conf.m_maxRepairSpiders ) {
if ( m_numOutstandingInjects<g_conf.m_maxRepairinjections ) {
m_stage = STAGE_TITLEDB_0;
if( g_conf.m_logTraceRepairs ) log(LOG_TRACE,"%s:%s:%d: Still have more free repair spiders, loop.", __FILE__, __func__, __LINE__);
goto loop1;
@ -1235,7 +1216,6 @@ bool Repair::gotScanRecList ( ) {
m_isDelete = false;
// we need this to compute the tfndb key to add/delete
//m_ext = -1;
m_uh48 = 0LL;
// count the title recs we scan
m_recsScanned++;
@ -1296,8 +1276,6 @@ bool Repair::gotScanRecList ( ) {
if ( m_fn == base->getNumFiles() ) id2 = 255;
else id2 = base->m_fileIds2[m_fn];
// that is the tfn...
m_tfn = id2;
*/
// is it a negative titledb key?
@ -1319,7 +1297,8 @@ bool Repair::gotScanRecList ( ) {
return true;
}
static void doneWithIndexDoc ( XmlDoc *xd ) {
void Repair::doneWithIndexDoc(XmlDoc *xd) {
if( g_conf.m_logTraceRepairs ) log(LOG_TRACE,"%s:%s:%d: BEGIN", __FILE__, __func__, __LINE__);
// preserve
@ -1350,7 +1329,8 @@ static void doneWithIndexDoc ( XmlDoc *xd ) {
if( g_conf.m_logTraceRepairs ) log(LOG_TRACE,"%s:%s:%d: END", __FILE__, __func__, __LINE__);
}
static void doneWithIndexDocWrapper ( void *state ) {
void Repair::doneWithIndexDocWrapper(void *state) {
if( g_conf.m_logTraceRepairs ) log(LOG_TRACE,"%s:%s:%d: BEGIN", __FILE__, __func__, __LINE__);
// clean up
doneWithIndexDoc ( (XmlDoc *)state );
@ -1576,7 +1556,6 @@ bool Repair::printRepairStatus ( SafeBuf *sb , int32_t fromIp ) {
int64_t errors2 = m_spiderRecSetErrors;
const char *newColl = " &nbsp; ";
//if ( m_fullRebuild ) newColl = m_newColl;
const char *oldColl = " &nbsp; ";
if ( m_cr ) oldColl = m_cr->m_coll;
@ -1946,7 +1925,7 @@ static bool s_savingAll = false;
// . return false if blocked, true otherwise
// . will call the callback when all have been saved
// . used by Repair.cpp to save all rdbs before doing repair work
bool saveAllRdbs ( void *state , void (* callback)(void *state) ) {
bool Repair::saveAllRdbs(void *state, void (*callback)(void *state)) {
// only call once
if ( s_savingAll ) {
//log("db: Already saving all.");
@ -1978,7 +1957,7 @@ bool saveAllRdbs ( void *state , void (* callback)(void *state) ) {
}
// return false if one or more is still not closed yet
bool anyRdbNeedsSave ( ) {
bool Repair::anyRdbNeedsSave() {
int32_t count = 0;
int32_t nsr;
Rdb **rdbs = getAllRdbs ( &nsr );
@ -1992,7 +1971,7 @@ bool anyRdbNeedsSave ( ) {
}
// returns false if waiting on some to save
void doneSavingRdb ( void *state ) {
void Repair::doneSavingRdb(void *state) {
if ( ! anyRdbNeedsSave() ) return;
// all done
s_savingAll = false;

@ -5,23 +5,34 @@
#include "RdbList.h"
#include "Msg5.h"
#include "Msg4.h"
#include "Tagdb.h"
#include "max_coll_len.h"
#define SR_BUFSIZE 2048
extern char g_repairMode;
class XmlDoc;
class CollectionRec;
class Repair {
public:
Repair();
// is the scan active and adding recs to the secondary rdbs?
bool isRepairActive() ;
bool isRepairActive() const;
bool isRepairingColl(collnum_t coll) const { return m_collnum==coll; }
bool init();
// if we core, call this so repair can resume where it left off
bool save();
// called by Parms.cpp
bool printRepairStatus ( SafeBuf *sb , int32_t fromIp );
bool linkdbRebuildPending() const { return m_rebuildLinkdb; }
private:
//void allHostsReady();
void initScan();
void resetForNewCollection();
@ -38,51 +49,23 @@ public:
bool getTitleRec ( );
bool injectTitleRec ( ) ; // TitleRec *tr );
// called by Pages.cpp
bool printRepairStatus ( SafeBuf *sb , int32_t fromIp );
// if we core, call these so repair can resume where it left off
bool save();
bool load();
bool m_completed;
// general scan vars
Msg5 m_msg5;
Msg4 m_msg4;
bool m_needsCallback;
char m_docQuality;
RdbList m_titleRecList;
int64_t m_docId;
bool m_isDelete;
RdbList m_ulist;
RdbList m_addlist;
int64_t m_totalMem;
int32_t m_stage ;
int32_t m_tfn;
int32_t m_count;
bool m_updated;
// titledb scan vars
key96_t m_nextTitledbKey;
key96_t m_nextSpiderdbKey;
key96_t m_nextPosdbKey;
key128_t m_nextLinkdbKey;
key96_t m_endKey;
int64_t m_uh48;
int32_t m_priority;
uint64_t m_contentHash;
key96_t m_clusterdbKey ;
key96_t m_spiderdbKey;
char m_srBuf[SR_BUFSIZE];
char m_tmpBuf[32];
RdbList m_linkdbListToAdd;
uint64_t m_chksum1LongLong;
// spiderdb scan vars
bool m_isNew;
TagRec m_tagRec;
// . state info
// . indicator of what we save to disk
@ -129,11 +112,7 @@ public:
bool m_rebuildNonRoots ;
// current collection being repaired
//int32_t m_collLen;
collnum_t m_collnum;
char m_newColl[MAX_COLL_LEN];
int32_t m_newCollLen;
collnum_t m_newCollnum;
// . m_colli is the index into m_colls
// . m_colli is the index into g_collectiondb.m_recs if the list
@ -142,8 +121,9 @@ public:
// list of collections to repair, only valid of g_conf.m_collsToRepair
// is not empty
int32_t m_collOffs[100];
int32_t m_collLens[100];
static const int32_t maxCollections = 100;
int32_t m_collOffs[maxCollections];
int32_t m_collLens[maxCollections];
int32_t m_numColls;
// end the stuff to be saved
char m_SAVE_END;
@ -167,6 +147,15 @@ public:
bool m_saveRepairState;
bool m_isRetrying;
static void repairWrapper(int fd, void *state);
static void loopWrapper(void *state, RdbList *list, Msg5 *msg5);
static bool saveAllRdbs(void *state, void (*callback)(void *state));
static bool anyRdbNeedsSave();
static void doneSavingRdb(void *state);
static void doneWithIndexDoc(XmlDoc *xd);
static void doneWithIndexDocWrapper(void *state);
};
// the global class

@ -6546,7 +6546,7 @@ int32_t *XmlDoc::getSiteNumInlinks ( ) {
// if we have already been through this
if ( m_updatingSiteLinkInfoTags ) valid = false;
// if rebuilding linkdb assume we have no links to sample from!
if ( tag && m_useSecondaryRdbs && g_repair.m_rebuildLinkdb )
if ( tag && m_useSecondaryRdbs && g_repair.linkdbRebuildPending() )
valid = true;
// debug log