forked from Mirrors/privacore-open-source-search-engine
Merge branch 'master' into nomerge2
This commit is contained in:
1
Conf.cpp
1
Conf.cpp
@ -295,7 +295,6 @@ Conf::Conf ( ) {
|
||||
m_maxCallbackDelay = 0;
|
||||
m_repairingEnabled = false;
|
||||
m_maxRepairinjections = 0;
|
||||
m_rebuildHost = -1;
|
||||
m_repairMem = 0;
|
||||
m_fullRebuild = true;
|
||||
m_rebuildAddOutlinks = false;
|
||||
|
1
Conf.h
1
Conf.h
@ -494,7 +494,6 @@ class Conf {
|
||||
int32_t m_maxRepairinjections;
|
||||
int64_t m_repairMem;
|
||||
SafeBuf m_collsToRepair;
|
||||
int32_t m_rebuildHost;
|
||||
bool m_fullRebuild;
|
||||
bool m_rebuildAddOutlinks;
|
||||
bool m_rebuildRecycleLinkInfo;
|
||||
|
@ -1538,6 +1538,7 @@ uint32_t Hostdb::getShardNum(rdbid_t rdbId, const void *k) const {
|
||||
|
||||
case RDB_LINKDB:
|
||||
case RDB2_LINKDB2:
|
||||
// sharded by part of linkee sitehash32
|
||||
return m_map [(*(uint16_t *)((char *)k + 26))>>3];
|
||||
|
||||
case RDB_TITLEDB:
|
||||
|
13
Parms.cpp
13
Parms.cpp
@ -8559,6 +8559,7 @@ void Parms::init ( ) {
|
||||
///////////////////////////////////////////
|
||||
// PAGE REPAIR CONTROLS
|
||||
///////////////////////////////////////////
|
||||
#ifndef PRIVACORE_SAFE_VERSION
|
||||
|
||||
m->m_title = "rebuild mode enabled";
|
||||
m->m_desc = "If enabled, gigablast will rebuild the rdbs as "
|
||||
@ -8585,17 +8586,6 @@ void Parms::init ( ) {
|
||||
m->m_flags = PF_REQUIRED;// | PF_COLLDEFAULT;//| PF_NOHTML;
|
||||
m++;
|
||||
|
||||
|
||||
m->m_title = "Limit rebuild to a host";
|
||||
m->m_desc = "Only rebuild on this host. -1 for all";
|
||||
m->m_cgi = "rebuild_host";
|
||||
simple_m_set(Conf,m_rebuildHost);
|
||||
m->m_page = PAGE_REPAIR;
|
||||
m->m_def = "-1";
|
||||
m->m_group = false;
|
||||
m++;
|
||||
|
||||
|
||||
m->m_title = "memory to use for rebuild";
|
||||
m->m_desc = "In bytes.";
|
||||
m->m_cgi = "rmtu"; // repair mem to use
|
||||
@ -8719,6 +8709,7 @@ void Parms::init ( ) {
|
||||
m->m_group = false;
|
||||
m++;
|
||||
|
||||
#endif
|
||||
///////////////////////////////////////////
|
||||
// END PAGE REPAIR //
|
||||
///////////////////////////////////////////
|
||||
|
@ -503,6 +503,7 @@ void Posdb::printKey(const char *k) {
|
||||
"syn=%01" PRId32" "
|
||||
"densRank=%02" PRId32" "
|
||||
"mult=%02" PRId32" "
|
||||
"shardByTermId=%d "
|
||||
"isDel=%d",
|
||||
KEYSTR(k, sizeof(key144_t)),
|
||||
getTermId(k),
|
||||
@ -516,6 +517,7 @@ void Posdb::printKey(const char *k) {
|
||||
(int32_t)getIsSynonym(k),
|
||||
(int32_t)getDensityRank(k),
|
||||
(int32_t)getMultiplier(k),
|
||||
isShardedByTermId(k),
|
||||
KEYNEG(k));
|
||||
}
|
||||
|
||||
|
1
Posdb.h
1
Posdb.h
@ -7,6 +7,7 @@
|
||||
// wwwwwwww wwwwwwww wwGGGGss ssvvvvFF w = word postion , s = wordspamrank
|
||||
// pppppb1N MMMMLZZD v = diversityrank, p = densityrank
|
||||
// M = multiplier, b = in outlink text
|
||||
// N = shard by termId
|
||||
// L = langIdShiftBit (upper bit)
|
||||
// G: 0 = body
|
||||
// 1 = intitletag
|
||||
|
@ -148,7 +148,7 @@ void RdbList::set(char *list, int32_t listSize, char *alloc, int32_t allocSize,
|
||||
verify_signature();
|
||||
logTrace(g_conf.m_logTraceRdbList, "BEGIN. list=%p listSize=%" PRId32" alloc=%p allocSize=%" PRId32,
|
||||
list, listSize, alloc, allocSize);
|
||||
char logbuf1[50],logbuf2[50];
|
||||
char logbuf1[MAX_KEYSTR_BYTES],logbuf2[MAX_KEYSTR_BYTES];
|
||||
logTrace(g_conf.m_logTraceRdbList, "startKey=%s endKey=%s keySize=%hhu fixedDataSize=%" PRId32,
|
||||
KEYSTR(startKey, keySize,logbuf1), KEYSTR(endKey, keySize,logbuf2), keySize, fixedDataSize);
|
||||
|
||||
|
14
Repair.cpp
14
Repair.cpp
@ -163,20 +163,6 @@ void Repair::repairWrapper(int fd, void *state) {
|
||||
// when dumped, would mess up our scan.
|
||||
if ( ! g_conf.m_repairingEnabled ) return;
|
||||
|
||||
if(g_conf.m_rebuildHost>=0 && getMyHostId()!=g_conf.m_rebuildHost) {
|
||||
log(LOG_DEBUG, "repair: rebuild/repair is for host %d only and not us", g_conf.m_rebuildHost);
|
||||
g_conf.m_repairingEnabled = false;
|
||||
return;
|
||||
}
|
||||
|
||||
#ifdef PRIVACORE_SAFE_VERSION
|
||||
if(g_conf.m_rebuildHost<0) {
|
||||
log(LOG_DEBUG, "repair: rebuild/repair specified for all but this is not allowed in this safer version");
|
||||
g_conf.m_repairingEnabled = false;
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
// if the power went off
|
||||
if ( ! g_process.m_powerIsOn ) return;
|
||||
|
||||
|
@ -176,42 +176,9 @@ bool XmlDoc::hashNoSplit ( HashTableX *tt ) {
|
||||
hi.m_prefix = "gbcontenthash";
|
||||
if ( ! hashString ( cbuf,clen,&hi ) ) return false;
|
||||
|
||||
////
|
||||
//
|
||||
// let's stop here for now, until other stuff is actually used again
|
||||
//
|
||||
////
|
||||
|
||||
// let's bring back image thumbnail support for the widget project
|
||||
//return true;
|
||||
|
||||
|
||||
|
||||
char *host = fu->getHost ();
|
||||
//int32_t hlen = fu->getHostLen ();
|
||||
|
||||
/*
|
||||
setStatus ( "hashing no-split qdom keys" );
|
||||
|
||||
char *dom = fu->getDomain ();
|
||||
int32_t dlen = fu->getDomainLen();
|
||||
|
||||
// desc is NULL, prefix will be used as desc
|
||||
hi.m_prefix = "qdom";
|
||||
if ( ! hashString ( dom,dlen,&hi ) ) return false;
|
||||
|
||||
|
||||
setStatus ( "hashing no-split qhost keys" );
|
||||
|
||||
// desc is NULL, prefix will be used as desc
|
||||
hi.m_prefix = "qhost";
|
||||
if ( ! hashString ( host,hlen,&hi ) ) return false;
|
||||
*/
|
||||
|
||||
|
||||
// now hash the site
|
||||
|
||||
|
||||
setStatus ( "hashing no-split SiteGetter terms");
|
||||
|
||||
//
|
||||
|
1
types.h
1
types.h
@ -10,6 +10,7 @@
|
||||
typedef int16_t collnum_t;
|
||||
|
||||
#define MAX_KEY_BYTES 28
|
||||
#define MAX_KEYSTR_BYTES ((MAX_KEY_BYTES*2)+3)
|
||||
|
||||
class u_int96_t {
|
||||
|
||||
|
Reference in New Issue
Block a user