09de59f026
disk space. added tagdb file cache for better performance, less disk accesses. will help reduce disk load. put file cache sizes in master controls and if they change then update the cache size dynamically.
543 lines
17 KiB
C++
543 lines
17 KiB
C++
#include "gb-include.h"
|
|
|
|
#include <sys/stat.h>
|
|
#include "Catdb.h"
|
|
#include "Categories.h"
|
|
#include "CatRec.h"
|
|
#include "Unicode.h"
|
|
#include "Threads.h"
|
|
|
|
// use this to query delete all the banned sites in tagdb
|
|
// dsh -a '/a/gb -c /a/hosts.conf dump S main 0 -1 1 | grep =19' | awk '{print $6}' | sort | uniq | grep "\." > banned
|
|
// cat banned | awk -F'http://' '{print $2}' | grep "[A-z]" | awk '{print "http://64.62.168.52:8000/admin/reindex?c=main&cast=0&sq=site%3A"$1"&delbox=1&f=-1&srn=0&ern=2000000&sto=0&sp=7&action=OK"}' > URLS
|
|
// cat banned | awk -F'/' '{print $3}' | grep -v "[A-z]" | awk '{print "http://64.62.168.52:8000/admin/reindex?c=main&cast=0&sq=ip%3A"$1"&delbox=1&f=-1&srn=0&ern=2000000&sto=0&sp=7&action=OK"}' >> URLS
|
|
// nohup wget -i /a/URLS -O /dev/null &
|
|
|
|
// a global class extern'd in .h file
|
|
Catdb g_catdb;
|
|
|
|
// reset rdb and Xmls
|
|
void Catdb::reset() {
|
|
m_rdb.reset();
|
|
}
|
|
|
|
|
|
bool Catdb::init ( ) {
|
|
// clear our m_keys/m_bufs arrays
|
|
// memset ( m_xml , 0 , MAXNUMSITEFILES * sizeof(Xml *) );
|
|
//memset ( m_keys, 0 , MAXNUMSITEFILES * sizeof(int64_t) );
|
|
// . what's max # of tree nodes?
|
|
// . assume avg tagdb rec size (siteUrl) is about 82 bytes we get:
|
|
// . NOTE: 32 bytes of the 82 are overhead
|
|
//int32_t treeMem = g_conf.m_catdbMaxTreeMem;
|
|
// speed up gen catdb, use 15MB. later maybe once gen is complete
|
|
// we can free this tree or something...
|
|
// TODO!
|
|
int32_t treeMem = 15000000;
|
|
//int32_t treeMem = 100000000;
|
|
//int32_t maxTreeNodes = g_conf.m_catdbMaxTreeMem / 82;
|
|
int32_t maxTreeNodes = treeMem / 82;
|
|
// do not use any page cache if doing tmp cluster in order to
|
|
// prevent swapping
|
|
// int32_t pcmem = g_conf.m_catdbMaxDiskPageCacheMem;
|
|
// if ( g_hostdb.m_useTmpCluster ) pcmem = 0;
|
|
|
|
// pcmem = 0;
|
|
// each entry in the cache is usually just a single record, no lists,
|
|
// unless a hostname has multiple sites in it. has 24 bytes more
|
|
// overhead in cache.
|
|
//int32_t maxCacheNodes = g_conf.m_tagdbMaxCacheMem / 106;
|
|
// we now use a page cache
|
|
// if ( ! m_pc.init ("catdb",RDB_CATDB,pcmem,
|
|
// GB_TFNDB_PAGE_SIZE) )
|
|
// return log("db: Catdb init failed.");
|
|
|
|
// . initialize our own internal rdb
|
|
// . i no longer use cache so changes to tagdb are instant
|
|
// . we still use page cache however, which is good enough!
|
|
//if ( this == &g_catdb )
|
|
if ( ! m_rdb.init ( g_hostdb.m_dir ,
|
|
"catdb" ,
|
|
true , // dedup same keys?
|
|
-1 , // fixed record size
|
|
//g_hostdb.m_groupMask ,
|
|
//g_hostdb.m_groupId ,
|
|
2,//g_conf.m_catdbMinFilesToMerge ,
|
|
treeMem ,//g_conf.m_catdbMaxTreeMem ,
|
|
maxTreeNodes ,
|
|
// now we balance so Sync.cpp can ordered huge list
|
|
true , // balance tree?
|
|
0 , //g_conf.m_tagdbMaxCacheMem ,
|
|
0 , //maxCacheNodes ,
|
|
false , // half keys?
|
|
false , //m_tagdbSaveCache
|
|
NULL, // &m_pc ,
|
|
false,
|
|
false,
|
|
12, // keysize
|
|
false,
|
|
true )) // is collectionless?
|
|
return false;
|
|
|
|
// normally Collectiondb.addColl() will call Rdb::addColl() which
|
|
// will init the CollectionRec::m_rdbBase, which is what
|
|
// Rdb::getBase(collnum_t) will return. however, for collectionless
|
|
// rdb databases we set Rdb::m_collectionlessBase special here.
|
|
// This was in Rdb.cpp::init().
|
|
return m_rdb.addRdbBase1 ( NULL );
|
|
}
|
|
|
|
bool Catdb::init2 ( int32_t treeMem ) {
|
|
// . what's max # of tree nodes?
|
|
// . assume avg tagdb rec size (siteUrl) is about 82 bytes we get:
|
|
// . NOTE: 32 bytes of the 82 are overhead
|
|
int32_t maxTreeNodes = 0;
|
|
|
|
return m_rdb.init ( g_hostdb.m_dir ,
|
|
"tagdbRebuild" ,
|
|
true , // dedup same keys?
|
|
-1 , // fixed record size
|
|
10 , // min to merge
|
|
treeMem ,
|
|
maxTreeNodes ,
|
|
true , // balance tree?
|
|
0 , //g_conf.m_tagdbMaxCacheMem ,
|
|
0 , //maxCacheNodes ,
|
|
false , // half keys?
|
|
false , //m_tagdbSaveCache
|
|
NULL ); //&m_pc
|
|
}
|
|
|
|
//
|
|
// end support for "cache recs"
|
|
//
|
|
|
|
/*
|
|
bool Catdb::addColl ( char *coll, bool doVerify ) {
|
|
if ( ! m_rdb.addColl ( coll ) ) return false;
|
|
// verify
|
|
return true;
|
|
if ( verify(coll) ) return true;
|
|
// if not allowing scale, return false
|
|
if ( ! g_conf.m_allowScale ) return false;
|
|
// otherwise let it go
|
|
log ( "db: Verify failed, but scaling is allowed, passing." );
|
|
return true;
|
|
}
|
|
*/
|
|
|
|
bool Catdb::verify ( char *coll ) {
|
|
char *rdbName = "Catdb";
|
|
log ( LOG_INFO, "db: Verifying %s for coll %s...", rdbName, coll );
|
|
g_threads.disableThreads();
|
|
|
|
Msg5 msg5;
|
|
//Msg5 msg5b;
|
|
RdbList list;
|
|
key_t startKey;
|
|
key_t endKey;
|
|
startKey.setMin();
|
|
endKey.setMax();
|
|
//int32_t minRecSizes = 64000;
|
|
|
|
if ( ! msg5.getList ( RDB_CATDB ,
|
|
0,//collnum ,
|
|
&list ,
|
|
startKey ,
|
|
endKey ,
|
|
64000 , // minRecSizes ,
|
|
true , // includeTree ,
|
|
false , // add to cache?
|
|
0 , // max cache age
|
|
0 , // startFileNum ,
|
|
-1 , // numFiles ,
|
|
NULL , // state
|
|
NULL , // callback
|
|
0 , // niceness
|
|
false , // err correction?
|
|
NULL ,
|
|
0 ,
|
|
-1 ,
|
|
true ,
|
|
-1LL ,
|
|
NULL,//&msg5b ,
|
|
true )) {
|
|
g_threads.enableThreads();
|
|
return log("db: HEY! it did not block");
|
|
}
|
|
|
|
int32_t count = 0;
|
|
int32_t got = 0;
|
|
for ( list.resetListPtr() ; ! list.isExhausted() ;
|
|
list.skipCurrentRecord() ) {
|
|
key_t k = list.getCurrentKey();
|
|
count++;
|
|
//uint32_t groupId = g_catdb.getGroupId ( &k );
|
|
//uint32_t shardNum = getShardNum ( RDB_CATDB , &k );
|
|
//if ( groupId == g_hostdb.m_groupId ) got++;
|
|
uint32_t shardNum = getShardNum( RDB_CATDB , &k );
|
|
if ( shardNum == getMyShardNum() ) got++;
|
|
}
|
|
if ( got != count ) {
|
|
log ("db: Out of first %"INT32" records in %s, only %"INT32" belong "
|
|
"to our group.",count,rdbName,got);
|
|
// exit if NONE, we probably got the wrong data
|
|
if ( got == 0 ) log("db: Are you sure you have the "
|
|
"right "
|
|
"data in the right directory? "
|
|
"Exiting.");
|
|
log ( "db: Exiting due to %s inconsistency.", rdbName );
|
|
g_threads.enableThreads();
|
|
return g_conf.m_bypassValidation;
|
|
}
|
|
log ( LOG_INFO, "db: %s passed verification successfully for %"INT32" recs.",
|
|
rdbName, count );
|
|
// DONE
|
|
g_threads.enableThreads();
|
|
return true;
|
|
}
|
|
|
|
void Catdb::normalizeUrl ( Url *srcUrl, Url *dstUrl ) {
|
|
char urlStr[MAX_URL_LEN];
|
|
int32_t urlStrLen = srcUrl->getUrlLen();
|
|
gbmemcpy(urlStr, srcUrl->getUrl(), urlStrLen);
|
|
// fix the url
|
|
urlStrLen = g_categories->fixUrl(urlStr, urlStrLen);
|
|
// create the normalized url
|
|
dstUrl->set(urlStr, urlStrLen, true, false, false, true);
|
|
}
|
|
|
|
// . dddddddd dddddddd dddddddd dddddddd d = domain hash w/o collection
|
|
// . uuuuuuuu uuuuuuuu uuuuuuuu uuuuuuuu u = url hash
|
|
// . uuuuuuuu uuuuuuuu uuuuuuuu uuuuuuuu
|
|
key_t Catdb::makeKey ( Url *site, bool isDelete ) {
|
|
|
|
key_t k;
|
|
// . get startKey based on "site"'s domain
|
|
// . if "site"'s domain is an ip address (non-canonical) then use ip
|
|
getKeyRange ( site->isIp() , site , &k , NULL);
|
|
// set lower 64 bits of key to hash of this url
|
|
k.n0 = hash64 ( site->getUrl() , site->getUrlLen() );
|
|
// clear low bit if we're a delete, otherwise set it
|
|
if ( isDelete ) k.n0 &= 0xfffffffffffffffeLL;
|
|
else k.n0 |= 0x0000000000000001LL;
|
|
return k;
|
|
}
|
|
|
|
// . get startKey,endKey for all SiteRecs from "url"'s domain
|
|
// . key has the following format:
|
|
// . dddddddd dddddddd dddddddd dddddddd d = domain hash w/ collection
|
|
// . uuuuuuuu uuuuuuuu uuuuuuuu uuuuuuuu u = url hash
|
|
// . uuuuuuuu uuuuuuuu uuuuuuuu uuuuuuuu
|
|
// . putting domain as first 32bits will cluster all SiteRecs from the
|
|
// same domain together on the same machine
|
|
void Catdb::getKeyRange ( bool useIp , Url *url,
|
|
key_t *startKey , key_t *endKey ) {
|
|
// log warning msg if we need to
|
|
if ( useIp && ! url->hasIp() )
|
|
log(LOG_LOGIC,"db: tagdb: getKeyRange: useIp is true, "
|
|
"but url has no ip");
|
|
// . the upper 32 bits of the key is basically hash of the domain
|
|
// . mask out the low-order byte (hi byte in little endian order)
|
|
uint32_t h;
|
|
// . make sure we use htonl() on ip domain so top byte is not zero!
|
|
// . this made all our ip-based sites stored in group #0 before
|
|
// if ( useIp ) h = htonl ( url->getIpDomain() ) ;
|
|
// . only hash first 3 bytes of ip domain to keep together w/ ip
|
|
// . if rdbid is tagdb then use hostname as key else use domain
|
|
if ( useIp ) {
|
|
// do htonl so most significant byte is first
|
|
int32_t ipdom = htonl(url->getIpDomain());
|
|
h = hash32 ( (char *)&ipdom , 3 ) ;
|
|
}
|
|
else
|
|
h = hash32 (url->getDomain(), url->getDomainLen());
|
|
|
|
// incorporate collection into "h"
|
|
//h = hash32 ( coll , collLen , h );
|
|
// now make the keys
|
|
key_t k;
|
|
// top 4 bytes is always the domain hash (ip or canonical domain)
|
|
k.n1 = h;
|
|
// don't set the low del bit for startKey
|
|
k.n0 = 0x0000000000000000LL;
|
|
// assign the startKey
|
|
if ( startKey ) *startKey = k;
|
|
// set the low del bit for startKey
|
|
k.n0 = 0xffffffffffffffffLL;
|
|
// endkey is just as simple
|
|
if ( endKey ) *endKey = k;
|
|
}
|
|
|
|
// move the current list pointer back until we hit the start of
|
|
// a valid key
|
|
char *Catdb::moveToCorrectKey ( char *listPtr,
|
|
RdbList *list,
|
|
uint32_t domainHash ) {
|
|
char *listEnd = list->getListEnd();
|
|
char *listStart = list->getList();
|
|
char *p = listPtr;
|
|
// move back from the end
|
|
if (listEnd - p < (int32_t)sizeof(key_t))
|
|
p -= sizeof(key_t);
|
|
// loop until we get it
|
|
for ( ; p > listStart; p--){
|
|
// check for the domain hash in the key
|
|
if ( ((key_t*)p)->n1 == domainHash ) {
|
|
// . verify the match
|
|
// get the current rec size and check
|
|
// the next rec for correct data
|
|
int32_t recSize = list->getRecSize(p);
|
|
char *checkp = p + recSize;
|
|
// step 1, verify the start of the next rec is good
|
|
if ( recSize >= 0 && ( checkp == listEnd ||
|
|
( checkp < listEnd &&
|
|
((key_t*)checkp)->n1 == domainHash ) ) ) {
|
|
// return if we're at end
|
|
if ( checkp == listEnd )
|
|
return p;
|
|
// step 2, verify good rec on next rec
|
|
recSize = list->getRecSize(checkp);
|
|
checkp = checkp + recSize;
|
|
if ( recSize >= 0 && ( checkp == listEnd ||
|
|
( checkp < listEnd &&
|
|
((key_t*)checkp)->n1 == domainHash )))
|
|
// good match, return it
|
|
return p;
|
|
}
|
|
}
|
|
// otherwise backup a byte
|
|
}
|
|
// we'll get here if p == listStart
|
|
return p;
|
|
}
|
|
|
|
// binary search on the given list for the given key
|
|
void Catdb::listSearch ( RdbList *list,
|
|
key_t exactKey,
|
|
char **data,
|
|
int32_t *dataSize ) {
|
|
// init the data
|
|
*data = NULL;
|
|
*dataSize = 0;
|
|
list->resetListPtr();
|
|
// for small lists, just loop through the list
|
|
if (list->getListSize() < 16*1024) {
|
|
while ( ! list->isExhausted() ) {
|
|
// for debug!
|
|
/*
|
|
CatRec crec;
|
|
crec.set ( NULL,
|
|
list->getCurrentData(),
|
|
list->getCurrentDataSize(),
|
|
false);
|
|
log("catdb: caturl=%s #catid=%"INT32" version=%"INT32""
|
|
,crec.m_url
|
|
,(int32_t)crec.m_numCatids
|
|
,(int32_t)crec.m_version
|
|
);
|
|
*/
|
|
// check the current key
|
|
if ( list->getCurrentKey() != exactKey ) {
|
|
// miss, next
|
|
list->skipCurrentRecord();
|
|
continue;
|
|
}
|
|
// get site from this rec
|
|
*data = list->getCurrentData();
|
|
*dataSize = list->getCurrentDataSize();
|
|
break;
|
|
}
|
|
}
|
|
// otherwise do a binary search on the large lists
|
|
else {
|
|
// init the low and high
|
|
char *low = list->getList();
|
|
char *high = list->getListEnd();
|
|
// move the high ptr to the start of last rec
|
|
high = moveToCorrectKey(high, list, exactKey.n1);
|
|
// binary search
|
|
char *currRec;
|
|
while ( low <= high ) {
|
|
// next check spot
|
|
int32_t delta = high - low;
|
|
currRec = low + (delta / 2);
|
|
//currRec = (char*)(((uint64_t)low +
|
|
// (uint64_t)high)/2);
|
|
// do correction
|
|
currRec = moveToCorrectKey( currRec,
|
|
list,
|
|
exactKey.n1 );
|
|
// check for hit
|
|
if (list->getKey(currRec) == exactKey) {
|
|
// hit, save it and get out
|
|
*data = list->getData(currRec);
|
|
*dataSize = list->getDataSize(currRec);
|
|
break;
|
|
}
|
|
else if (list->getKey(currRec) > exactKey) {
|
|
// move high to currRec - one rec
|
|
high = moveToCorrectKey ( currRec - 1,
|
|
list,
|
|
exactKey.n1 );
|
|
}
|
|
else {
|
|
// move low to currRec + one rec
|
|
low = currRec + list->getRecSize(currRec);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// now given an RdbList of SiteRecs can we find the best matching rec
|
|
// for our site?
|
|
char *Catdb::getRec ( RdbList *list , Url *url , int32_t *recSize,
|
|
char* coll, int32_t collLen ) {
|
|
key_t exactKey;
|
|
int64_t startTime = gettimeofdayInMilliseconds();
|
|
int64_t took;
|
|
char *data;
|
|
int32_t dataSize;
|
|
// for now, only get exact hits for catdb
|
|
// check for an exact key/url match
|
|
exactKey = makeKey(url, false);
|
|
// go throught the list looking for the exact key
|
|
//list->resetListPtr();
|
|
data = NULL;
|
|
dataSize = 0;
|
|
// call the search
|
|
listSearch ( list, exactKey, &data, &dataSize );
|
|
// make sure the url matches
|
|
if (data && dataSize > 0) {
|
|
// get the url
|
|
/*
|
|
char *x;
|
|
int32_t xlen;
|
|
// hit, check the url
|
|
// for catdb, skip over the catids
|
|
if (m_rdbid == RDB_CATDB) {
|
|
unsigned char numCatids = *data;
|
|
// . point to stored url/site
|
|
// . skip dataSize/fileNum
|
|
int32_t skip = 1 + (4 * numCatids) + 4;
|
|
x = data + skip;
|
|
xlen = dataSize - skip;
|
|
}
|
|
else {
|
|
// . point to stored url/site
|
|
// . skip dataSize/fileNum
|
|
x = data + 4;
|
|
xlen = dataSize - 4;
|
|
}
|
|
// set the site
|
|
Url site;
|
|
site.set ( x , xlen , false );
|
|
*/
|
|
CatRec site;
|
|
site.set (url, data, dataSize, false);
|
|
// check for an exact match against the full url
|
|
int32_t uflen = url->getUrlLen();
|
|
char *ufull = url->getUrl();
|
|
//int32_t sflen = site.getUrlLen();
|
|
//char *sfull = site.getUrl();
|
|
int32_t sflen = site.m_urlLen;
|
|
char *sfull = site.m_url;
|
|
// if we match, return this rec
|
|
if ( sflen == uflen &&
|
|
strncmp ( sfull, ufull, sflen ) == 0 ) {
|
|
*recSize = dataSize;
|
|
}
|
|
else {
|
|
*recSize = 0;
|
|
data = NULL;
|
|
}
|
|
}
|
|
else {
|
|
*recSize = 0;
|
|
data = NULL;
|
|
}
|
|
took = gettimeofdayInMilliseconds() - startTime;
|
|
if ( took > 10 )
|
|
log(LOG_INFO, "catdb: catdb lookup took %"INT64" ms, "
|
|
"listSize=%"INT32"", took, list->getListSize() );
|
|
return data;
|
|
}
|
|
|
|
// . find the indirect matches in the list which match a sub path
|
|
// of the url
|
|
int32_t Catdb::getIndirectMatches ( RdbList *list ,
|
|
Url *url ,
|
|
char **matchRecs ,
|
|
int32_t *matchRecSizes ,
|
|
int32_t maxMatches ,
|
|
char *coll,
|
|
int32_t collLen) {
|
|
char path[MAX_URL_LEN+1];
|
|
int32_t pathLen;
|
|
Url partialUrl;
|
|
key_t partialUrlKey;
|
|
// start with the whole url...include real catid in indirect
|
|
gbmemcpy(path, url->getUrl(), url->getUrlLen());
|
|
pathLen = url->getUrlLen();
|
|
// loop looking for partial matches
|
|
char *data = NULL;
|
|
int32_t dataSize = 0;
|
|
int32_t numMatches = 0;
|
|
while ( numMatches < maxMatches ) {
|
|
// make the partial url
|
|
partialUrl.set(path, pathLen, true);
|
|
normalizeUrl(&partialUrl, &partialUrl);
|
|
// make the next key
|
|
partialUrlKey = makeKey ( &partialUrl, false );
|
|
// search for it
|
|
listSearch ( list, partialUrlKey, &data, &dataSize );
|
|
// store a hit
|
|
if ( data && dataSize > 0 ) {
|
|
// get the url
|
|
char *x;
|
|
int32_t xlen;
|
|
// hit, check the url
|
|
// for catdb, skip over the catids
|
|
/*
|
|
unsigned char numCatids = *data;
|
|
// . point to stored url/site
|
|
// . skip dataSize/fileNum
|
|
int32_t skip = 1 + (4 * numCatids) + 4;
|
|
x = data + skip;
|
|
xlen = dataSize - skip;
|
|
*/
|
|
CatRec sr;
|
|
sr.set ( url, data, dataSize, false);
|
|
x = sr.m_url;
|
|
xlen = sr.m_urlLen;
|
|
// ensure it's a sub-path
|
|
if ( xlen <= url->getUrlLen() &&
|
|
strncasecmp(x, url->getUrl(), xlen) == 0 ) {
|
|
//char msg[4096];
|
|
//char *mp = msg;
|
|
//mp += sprintf(mp, "For ");
|
|
//gbmemcpy(mp, url->getUrl(), url->getUrlLen());
|
|
//mp += url->getUrlLen();
|
|
//mp += sprintf(mp, " , got Indirect: ");
|
|
//gbmemcpy(mp, x, xlen);
|
|
//mp += xlen;
|
|
//*mp = '\0';
|
|
//log ( LOG_INFO, "tagdb: %s", msg );
|
|
matchRecs [numMatches] = data;
|
|
matchRecSizes[numMatches] = dataSize;
|
|
numMatches++;
|
|
}
|
|
}
|
|
// make the next partial url
|
|
pathLen--;
|
|
while ( pathLen > 3 && path[pathLen-1] != '/' )
|
|
pathLen--;
|
|
// check for end
|
|
if ( pathLen <= 3 || strncmp(&path[pathLen-3], "://", 3) == 0 )
|
|
break;
|
|
// chop off the trailing /
|
|
pathLen--;
|
|
}
|
|
return numMatches;
|
|
}
|