409 lines
12 KiB
C++
409 lines
12 KiB
C++
// Matt Wells, copyright Feb 2001
|
|
|
|
// maintains a simple array of CollectionRecs
|
|
|
|
#ifndef GB_COLLECTIONDB_H
|
|
#define GB_COLLECTIONDB_H
|
|
|
|
#include <atomic>
|
|
#include "SafeBuf.h"
|
|
#include "rdbid_t.h"
|
|
#include "collnum_t.h"
|
|
#include "spider_status_t.h"
|
|
#include "GbMutex.h"
|
|
#include "WordVariationsConfig.h"
|
|
|
|
|
|
class Collectiondb {
|
|
|
|
public:
|
|
Collectiondb();
|
|
~Collectiondb();
|
|
|
|
void reset() ;
|
|
|
|
// called by main.cpp to fill in our m_recs[] array with
|
|
// all the coll.*.*/coll.conf info
|
|
bool loadAllCollRecs ( );
|
|
|
|
// . this will save all conf files back to disk that need it
|
|
// . returns false and sets g_errno on error, true on success
|
|
bool save ( );
|
|
|
|
bool isInitializing() const { return m_initializing; }
|
|
|
|
// returns i so that m_recs[i].m_coll = coll
|
|
collnum_t getCollnum(const char *coll, int32_t collLen) const;
|
|
collnum_t getCollnum(const char *coll) const; // coll is NULL terminated here
|
|
|
|
const char *getCollName(collnum_t collnum) const;
|
|
|
|
// get coll rec specified in the HTTP request
|
|
class CollectionRec *getRec ( class HttpRequest *r ,
|
|
bool useDefaultRec = true );
|
|
|
|
//Returns the specified collection name, or the default collection if no collection name was specified
|
|
const char *getDefaultColl(const char *collname_from_httprequest);
|
|
|
|
// . get collectionRec from name
|
|
// returns NULL if not available
|
|
class CollectionRec *getRec ( const char *coll );
|
|
|
|
class CollectionRec *getRec ( const char *coll , int32_t collLen );
|
|
|
|
class CollectionRec *getRec ( collnum_t collnum);
|
|
|
|
//class CollectionRec *getDefaultRec ( ) ;
|
|
|
|
class CollectionRec *getFirstRec ( ) ;
|
|
collnum_t getFirstCollnum() const ;
|
|
|
|
int32_t getNumRecs() const { return m_numRecs; }
|
|
int32_t getNumRecsUsed() const { return m_numRecsUsed; }
|
|
|
|
// what collnum will be used the next time a coll is added?
|
|
collnum_t reserveCollNum ( ) ;
|
|
|
|
bool addExistingColl ( const char *coll, collnum_t collnum );
|
|
|
|
bool addNewColl( const char *coll, collnum_t newCollnum ) ;
|
|
|
|
bool addRdbBaseToAllRdbsForEachCollRec ( ) ;
|
|
bool addRdbBasesForCollRec ( CollectionRec *cr ) ;
|
|
|
|
// returns false if blocked, true otherwise.
|
|
bool deleteRec2 ( collnum_t collnum );
|
|
|
|
//void deleteSpiderColl ( class SpiderColl *sc );
|
|
|
|
// returns false if blocked, true otherwise.
|
|
bool resetColl2(collnum_t oldCollnum, collnum_t newCollnum);
|
|
|
|
|
|
private:
|
|
// after main.cpp loads all rdb trees it calls this to remove
|
|
// bogus collnums from the trees i guess
|
|
bool cleanTrees();
|
|
|
|
bool registerCollRec(CollectionRec *cr);
|
|
|
|
bool growRecPtrBuf(collnum_t collnum);
|
|
bool setRecPtr(collnum_t collnum, CollectionRec *cr);
|
|
|
|
class CollectionRec **m_recs;
|
|
|
|
// m_recs[] points into a safebuf that is just an array
|
|
// of collectionrec ptrs. so we have to grow that safebuf possibly
|
|
// in order to add a new collection rec ptr to m_recs
|
|
SafeBuf m_recPtrBuf;
|
|
|
|
int32_t m_numRecs;
|
|
int32_t m_numRecsUsed;
|
|
|
|
int32_t m_wrapped;
|
|
|
|
bool m_initializing;
|
|
};
|
|
|
|
extern class Collectiondb g_collectiondb;
|
|
|
|
// Matt Wells, copyright Feb 2002
|
|
|
|
// . a collection record specifies the spider/index/search parms of a
|
|
// collection of web pages
|
|
// . there's a Msg class to send an update signal to all the hosts once
|
|
// we've used Msg1 to add a new rec or delete an old. The update signal
|
|
// will make the receiving hosts flush their CollectionRec buf so they
|
|
// have to send out a Msg0 to get it again
|
|
// . we have a default collection record, a main collection record and
|
|
// then other collection records
|
|
// . the default collection record values override all
|
|
// . but the collection record values can override SiteRec values
|
|
// . so if spider is disabled in default collection record, then nobody
|
|
// can spider!
|
|
// . override the g_conf.* vars where * is in this class to use
|
|
// Collection db's default values
|
|
// . then add in the values of the specialzed collection record
|
|
// . so change "if ( g_conf.m_spideringEnabled )" to something like
|
|
// Msg33 msg33;
|
|
// if ( ! msg33.getCollectionRec ( m_coll, m_collLen ) ) return false;
|
|
// CollectionRec *r = msg33.getRec();
|
|
// CollectoinRec *d = msg33.getDefaultRec();
|
|
// if ( ! r->m_spideringEnabled || ! d->m_spideringEnabled ) continue;
|
|
// ... otherwise, spider for the m_coll collection
|
|
// ... pass msg33 to Msg14::spiderDoc(), etc...
|
|
|
|
// how many url filtering patterns?
|
|
#define MAX_FILTERS 96 // up to 96 url regular expression patterns
|
|
|
|
#define SUMMARYHIGHLIGHTTAGMAXSIZE 128
|
|
|
|
#include "max_coll_len.h"
|
|
#include "HashTableX.h"
|
|
|
|
// fake this for now
|
|
#define RDB_END2 80
|
|
|
|
class CollectionRec {
|
|
|
|
public:
|
|
|
|
// active linked list of collectionrecs used by spider.cpp
|
|
class CollectionRec *m_nextActive;
|
|
|
|
// these just set m_xml to NULL
|
|
CollectionRec();
|
|
virtual ~CollectionRec();
|
|
|
|
int64_t getNumDocsIndexed();
|
|
|
|
// . stuff used by Collectiondb
|
|
// . do we need a save or not?
|
|
bool save();
|
|
void setNeedsSave() { m_needsSave = true; }
|
|
|
|
private:
|
|
std::atomic<bool> m_needsSave;
|
|
|
|
public:
|
|
bool load ( const char *coll , int32_t collNum ) ;
|
|
void reset();
|
|
|
|
// Clear memory structures used by URL filters
|
|
void clearUrlFilters();
|
|
|
|
// for customcrawls
|
|
bool rebuildUrlFilters();
|
|
|
|
// for regular crawls
|
|
bool rebuildUrlFilters2();
|
|
|
|
bool rebuildLangRules( const char *lang , const char *tld );
|
|
|
|
bool rebuildPrivacoreRules();
|
|
bool rebuildPrivacoreDKOnlyRules();
|
|
bool rebuildPrivacoreOldOnlyRules();
|
|
|
|
bool m_urlFiltersHavePageCounts;
|
|
|
|
// the all important collection name, NULL terminated
|
|
char m_coll [ MAX_COLL_LEN + 1 ] ;
|
|
int32_t m_collLen;
|
|
|
|
// used by SpiderCache.cpp. g_collectiondb.m_recs[m_collnum] = this
|
|
collnum_t m_collnum;
|
|
|
|
// for doing DailyMerge.cpp stuff
|
|
int32_t m_dailyMergeStarted; // time_t
|
|
int32_t m_dailyMergeTrigger;
|
|
|
|
char m_dailyMergeDOWList[48];
|
|
|
|
int64_t m_spiderCorruptCount;
|
|
|
|
// holds ips that have been detected as being throttled and we need
|
|
// to backoff and use proxies on
|
|
HashTableX m_twitchyTable;
|
|
|
|
// spider controls for this collection
|
|
bool m_spideringEnabled ;
|
|
int32_t m_spiderDelayInMilliseconds;
|
|
int32_t m_spiderReindexDelayMS;
|
|
|
|
// is in active list in spider.cpp?
|
|
bool m_isActive;
|
|
|
|
bool m_makeImageThumbnails;
|
|
|
|
int32_t m_thumbnailMaxWidthHeight ;
|
|
|
|
bool m_indexBody;
|
|
|
|
bool m_dedupingEnabled ; // dedup content on same hostname
|
|
bool m_dedupURLByDefault ;
|
|
bool m_dupCheckWWW ;
|
|
bool m_useSimplifiedRedirects ;
|
|
bool m_oneVotePerIpDom ;
|
|
bool m_doUrlSpamCheck ; //filter urls w/ naughty hostnames
|
|
bool m_doLinkSpamCheck ; //filters dynamically generated pages
|
|
bool m_siteClusterByDefault ;
|
|
bool m_useRobotsTxt ;
|
|
bool m_obeyRelNoFollowLinks ;
|
|
bool m_forceUseFloaters ;
|
|
bool m_automaticallyUseProxies ;
|
|
bool m_automaticallyBackOff ;
|
|
bool m_recycleContent ;
|
|
bool m_getLinkInfo ; // turn off to save seeks
|
|
bool m_computeSiteNumInlinks ;
|
|
|
|
int32_t m_percentSimilarSummary ; // Dedup by summary similiarity
|
|
int32_t m_summDedupNumLines ;
|
|
|
|
int32_t m_maxQueryTerms;
|
|
|
|
spider_status_t m_spiderStatus;
|
|
|
|
//ranking settings
|
|
float m_sameLangWeight;
|
|
float m_unknownLangWeight;
|
|
float m_siteRankMultiplier;
|
|
|
|
// Language stuff
|
|
char m_defaultSortLanguage2[6];
|
|
|
|
|
|
SafeBuf m_collectionPasswords;
|
|
SafeBuf m_collectionIps;
|
|
|
|
// from Conf.h
|
|
int32_t m_posdbMinFilesToMerge ;
|
|
int32_t m_titledbMinFilesToMerge ;
|
|
int32_t m_linkdbMinFilesToMerge ;
|
|
int32_t m_tagdbMinFilesToMerge ;
|
|
int32_t m_spiderdbMinFilesToMerge;
|
|
|
|
bool m_dedupResultsByDefault ;
|
|
bool m_doTagdbLookups ;
|
|
bool m_useCanonicalRedirects ;
|
|
|
|
int32_t m_maxNumSpiders ; // per local spider host
|
|
|
|
int32_t m_lastResetCount;
|
|
|
|
// controls for query-dependent summary/title generation
|
|
int32_t m_titleMaxLen;
|
|
int32_t m_summaryMaxLen;
|
|
int32_t m_summaryMaxNumLines;
|
|
int32_t m_summaryMaxNumCharsPerLine;
|
|
|
|
bool m_getDocIdScoringInfo;
|
|
|
|
// list of url patterns to be indexed.
|
|
SafeBuf m_siteListBuf;
|
|
|
|
// can be "web" "english" "romantic" "german" etc.
|
|
SafeBuf m_urlFiltersProfile;
|
|
|
|
// . now the url regular expressions
|
|
// . we chain down the regular expressions
|
|
// . if a url matches we use that tagdb rec #
|
|
// . if it doesn't match any of the patterns, we use the default site #
|
|
// . just one regexp per Pattern
|
|
// . all of these arrays should be the same size, but we need to
|
|
// include a count because Parms.cpp expects a count before each
|
|
// array since it handle them each individually
|
|
|
|
int32_t m_numRegExs;
|
|
// make this now use g_collectiondb.m_stringBuf safebuf and
|
|
// make Parms.cpp use that stringbuf rather than store into here...
|
|
SafeBuf m_regExs[ MAX_FILTERS ];
|
|
|
|
int32_t m_numSpiderFreqs; // useless, just for Parms::setParm()
|
|
float m_spiderFreqs[ MAX_FILTERS ];
|
|
|
|
int32_t m_numSpiderPriorities; // useless, just for Parms::setParm()
|
|
char m_spiderPriorities[ MAX_FILTERS ];
|
|
|
|
int32_t m_numMaxSpidersPerRule; // useless, just for Parms::setParm()
|
|
int32_t m_maxSpidersPerRule[ MAX_FILTERS ];
|
|
|
|
// same ip waits now here instead of "page priority"
|
|
int32_t m_numSpiderIpWaits; // useless, just for Parms::setParm()
|
|
int32_t m_spiderIpWaits[ MAX_FILTERS ];
|
|
|
|
// same goes for max spiders per ip
|
|
int32_t m_numSpiderIpMaxSpiders;
|
|
int32_t m_spiderIpMaxSpiders [ MAX_FILTERS ];
|
|
|
|
int32_t m_numHarvestLinks;
|
|
bool m_harvestLinks[ MAX_FILTERS ];
|
|
|
|
int32_t m_numForceDelete;
|
|
char m_forceDelete[ MAX_FILTERS ];
|
|
|
|
// dummy?
|
|
int32_t m_numRegExs9;
|
|
|
|
bool m_doQueryHighlighting;
|
|
|
|
char m_summaryFrontHighlightTag[SUMMARYHIGHLIGHTTAGMAXSIZE] ;
|
|
char m_summaryBackHighlightTag [SUMMARYHIGHLIGHTTAGMAXSIZE] ;
|
|
|
|
SafeBuf m_htmlRoot;
|
|
SafeBuf m_htmlHead;
|
|
SafeBuf m_htmlTail;
|
|
|
|
class SpiderColl *m_spiderColl;
|
|
GbMutex m_spiderCollMutex;
|
|
|
|
int32_t m_overflow;
|
|
int32_t m_overflow2;
|
|
|
|
int32_t m_maxAddUrlsPerIpDomPerDay;
|
|
|
|
// . max content length of text/html or text/plain document
|
|
// . we will not download, index or store more than this many bytes
|
|
int32_t m_maxTextDocLen;
|
|
|
|
// . max content length of other (pdf, word, xls, ppt, ps)
|
|
// . we will not index or store more than this many bytes
|
|
int32_t m_maxOtherDocLen;
|
|
|
|
// . max content length of other (pdf, word, xls, ppt, ps)
|
|
// . we will not download more than this many bytes
|
|
// . if content would be truncated, we will not even download at all
|
|
// because the html converter needs 100% of the doc otherwise it
|
|
// will have an error
|
|
int32_t m_maxOtherDocDownloadLen;
|
|
|
|
// . puts <br>s in the summary to keep its width below this
|
|
// . but we exceed this width before we would split a word
|
|
int32_t m_summaryMaxWidth;
|
|
|
|
// how long a robots.txt can be in the cache (Msg13.cpp/Robotdb.cpp)
|
|
int32_t m_maxRobotsCacheAge;
|
|
|
|
int32_t m_crawlDelayDefaultForNoRobotsTxtMS;
|
|
int32_t m_crawlDelayDefaultForRobotsTxtMS;
|
|
|
|
|
|
// check URL filters for manual ban and force delete?
|
|
bool m_checkURLFilters;
|
|
// rewrite domain-like queries for this collection?
|
|
bool m_modifyDomainLikeSearches;
|
|
bool m_domainLikeSearchDisablesSiteCluster;
|
|
// rewrite API-like queries?
|
|
bool m_modifyAPILikeSearches;
|
|
|
|
WordVariationsConfig m_word_variations_config;
|
|
|
|
// read from cache
|
|
bool m_rcache;
|
|
|
|
bool m_hideAllClustered;
|
|
|
|
// use this not m_bases to get the RdbBase
|
|
class RdbBase *getBase(rdbid_t rdbId);
|
|
|
|
// Rdb.cpp uses this after deleting an RdbBase and adding new one
|
|
void setBasePtr(rdbid_t rdbId, class RdbBase *base);
|
|
|
|
private:
|
|
// . now chuck this into CollectionRec instead of having a fixed
|
|
// array of them in Rdb.h called m_bases[]
|
|
// . leave this out of any copy of course
|
|
class RdbBase *m_bases[RDB_END2];
|
|
|
|
public:
|
|
// for poulating the sortbydate table
|
|
class Msg5 *m_msg5;
|
|
|
|
// each Rdb has a tree, so keep the pos/neg key count here so
|
|
// that RdbTree does not have to have its own array limited by
|
|
// MAX_COLLS which we did away with because we made this dynamic.
|
|
int32_t m_numPosKeysInTree[RDB_END2];
|
|
int32_t m_numNegKeysInTree[RDB_END2];
|
|
};
|
|
|
|
#endif // GB_COLLECTIONDB_H
|