327 lines
9.9 KiB
C++
327 lines
9.9 KiB
C++
// Matt Wells, copyright Sep 2000
|
|
|
|
// contains one RdbBase for each collection
|
|
|
|
#ifndef GB_RDB_H
|
|
#define GB_RDB_H
|
|
|
|
#include "RdbBase.h"
|
|
#include "RdbTree.h"
|
|
#include "RdbMem.h"
|
|
#include "RdbDump.h"
|
|
#include "RdbBuckets.h"
|
|
#include "RdbIndex.h"
|
|
#include "Hostdb.h"
|
|
#include "rdbid_t.h"
|
|
#include <atomic>
|
|
|
|
bool makeTrashDir() ;
|
|
|
|
// get the RdbBase class for an rdbId and collection name
|
|
class RdbBase *getRdbBase(rdbid_t rdbId, collnum_t collnum);
|
|
|
|
// maps an rdbId to an Rdb
|
|
class Rdb *getRdbFromId ( rdbid_t rdbId ) ;
|
|
|
|
// the reverse of the above
|
|
rdbid_t getIdFromRdb ( class Rdb *rdb ) ;
|
|
bool isSecondaryRdb ( rdbid_t rdbId ) ;
|
|
|
|
// get the dbname
|
|
const char *getDbnameFromId(rdbid_t rdbId);
|
|
|
|
//initialize all primary Rdbs
|
|
bool initialiseAllPrimaryRdbs();
|
|
|
|
// size of keys
|
|
char getKeySizeFromRdbId(rdbid_t rdbId);
|
|
|
|
// and this is -1 if dataSize is variable
|
|
int32_t getDataSizeFromRdbId ( rdbid_t rdbId );
|
|
void forceMergeAll(rdbid_t rdbId);
|
|
|
|
// main.cpp calls this
|
|
void attemptMergeAllCallback ( int fd , void *state ) ;
|
|
void attemptMergeAll ( );
|
|
|
|
class Rdb {
|
|
public:
|
|
|
|
Rdb ( );
|
|
~Rdb ( );
|
|
|
|
bool addRdbBase1 ( const char *coll );
|
|
|
|
bool delColl ( const char *coll );
|
|
|
|
bool resetBase ( collnum_t collnum );
|
|
bool deleteAllRecs ( collnum_t collnum ) ;
|
|
bool deleteColl ( collnum_t collnum , collnum_t newCollnum ) ;
|
|
|
|
bool init ( const char *dbname , // "indexdb","tagdb",...
|
|
int32_t fixedDataSize , //= -1 ,
|
|
int32_t minToMerge , //, //= 2 ,
|
|
int32_t maxTreeMem , //= 1024*1024*32 ,
|
|
int32_t maxTreeNodes ,
|
|
bool useHalfKeys ,
|
|
char keySize,
|
|
bool useIndexFile);
|
|
|
|
bool needsSave() const;
|
|
|
|
// . returns false and sets g_errno on error
|
|
// . caller should retry later on g_errno of ENOMEM or ETRYAGAIN
|
|
// . returns the node # in the tree it added the record to
|
|
// . key low bit must be set (otherwise it indicates a delete)
|
|
/**
|
|
* @note special behaviour when index is enabled. if corresponding opposite key is found in tree/bucket,
|
|
* it is assumed that we want the opposite key deleted.
|
|
* eg:
|
|
* positive key in tree, negative key passed to addRecord (positive key deleted)
|
|
* negative key in tree, positive key passed to addRecord (negative key deleted)
|
|
*
|
|
* the reason for this is so that we can eliminate negative key in tree
|
|
* (negative key should never be there when index is used, unless for special reasons. eg: posdb for deleted document)
|
|
*
|
|
* the scenario where we want to eliminate a negative key is when we deleted a document, and then we respider successfully
|
|
*/
|
|
bool addRecord(collnum_t collnum, const char *key, const char *data, int32_t dataSize);
|
|
|
|
// returns false if no room in tree or m_mem for a list to add
|
|
bool hasRoom(int32_t numRecs, int32_t dataSize) const;
|
|
|
|
// . returns false on error and sets errno
|
|
// . return true on success
|
|
// . if we can't handle all records in list we don't add any and
|
|
// set errno to ETRYAGAIN or ENOMEM
|
|
// . we copy all data so you can free your list when we're done
|
|
bool addList(collnum_t collnum, RdbList *list) {
|
|
return addList(collnum,list,true);
|
|
}
|
|
bool addListNoSpaceCheck(collnum_t collnum, RdbList *list) {
|
|
return addList(collnum,list,false);
|
|
}
|
|
|
|
bool deleteTreeNode(collnum_t collnum, const char *key);
|
|
|
|
void verifyTreeIntegrity();
|
|
|
|
bool isSecondaryRdb() const {
|
|
return ::isSecondaryRdb(m_rdbId);
|
|
}
|
|
|
|
bool isInitialized() const { return m_initialized; }
|
|
|
|
int32_t getFixedDataSize() const { return m_fixedDataSize; }
|
|
|
|
bool useHalfKeys() const { return m_useHalfKeys; }
|
|
char getKeySize() const { return m_ks; }
|
|
int32_t getPageSize() const { return m_pageSize; }
|
|
|
|
bool isTitledb() const { return m_rdbId==RDB_TITLEDB || m_rdbId==RDB2_TITLEDB2; }
|
|
|
|
RdbTree* getTree() { return (m_useTree ? &m_tree : NULL); }
|
|
RdbBuckets *getBuckets() { return (m_useTree ? NULL : &m_buckets); }
|
|
|
|
int32_t getAvailMem() const { return m_mem.getAvailMem(); }
|
|
int32_t getUsedMem() const { return m_mem.getUsedMem(); }
|
|
bool useTree() const { return m_useTree;}
|
|
|
|
int32_t getNumUsedNodes() const;
|
|
int32_t getMaxTreeMem() const;
|
|
int32_t getTreeMemOccupied() const;
|
|
int32_t getTreeMemAllocated() const;
|
|
int32_t getNumNegativeKeys() const;
|
|
|
|
void cleanTree();
|
|
|
|
RdbBase *getBase(collnum_t collnum );
|
|
const RdbBase *getBase(collnum_t collnum ) const { return const_cast<Rdb*>(this)->getBase(collnum); }
|
|
int32_t getNumBases() const;
|
|
|
|
|
|
// how much mem is allocated for our maps?
|
|
int64_t getMapMemAllocated() const;
|
|
|
|
int32_t getNumFiles() const;
|
|
|
|
// sum of all parts of all big files
|
|
int32_t getNumSmallFiles() const;
|
|
int64_t getDiskSpaceUsed() const;
|
|
|
|
// use the maps and tree to estimate the size of this list
|
|
int64_t estimateListSize(collnum_t collnum,
|
|
const char *startKey, const char *endKey, char *maxKey,
|
|
int64_t oldTruncationLimit) const;
|
|
|
|
//Get list from tree or buckets. Returns true on success
|
|
bool getTreeList(RdbList *result,
|
|
collnum_t collnum,
|
|
const void *startKey, const void *endKey,
|
|
int32_t minRecSizes,
|
|
int32_t *numPositiveRecs, int32_t *numNegativeRecs,
|
|
int32_t *memUsedByTree, int32_t *numUsedNodes);
|
|
|
|
// positive minus negative
|
|
int64_t getNumTotalRecs(bool useCache = false) const;
|
|
|
|
int64_t getCollNumTotalRecs(collnum_t collnum) const; //could technically be static
|
|
|
|
// used for keeping track of stats
|
|
void didSeek() { m_numSeeks++; }
|
|
void didRead(int64_t bytes) { m_numRead += bytes; }
|
|
void didReSeek() { m_numReSeeks++; }
|
|
int64_t getNumSeeks() const { return m_numSeeks; }
|
|
int64_t getNumReSeeks() const { return m_numReSeeks; }
|
|
int64_t getNumRead() const { return m_numRead ; }
|
|
|
|
// net stats for "get" requests
|
|
void readRequestGet(int32_t bytes) { m_numReqsGet++; m_numNetReadGet += bytes; }
|
|
void sentReplyGet(int32_t bytes) { m_numRepliesGet++; m_numNetSentGet += bytes; }
|
|
int64_t getNumRequestsGet() const { return m_numReqsGet; }
|
|
int64_t getNetReadGet() const { return m_numNetReadGet; }
|
|
int64_t getNumRepliesGet() const { return m_numRepliesGet; }
|
|
int64_t getNetSentGet() const { return m_numNetSentGet; }
|
|
|
|
// net stats for "add" requests
|
|
void readRequestAdd(int32_t bytes) { m_numReqsAdd++; m_numNetReadAdd += bytes; }
|
|
void sentReplyAdd(int32_t bytes) { m_numRepliesAdd++ ; m_numNetSentAdd += bytes; }
|
|
int64_t getNumRequestsAdd() const { return m_numReqsAdd; }
|
|
int64_t getNetReadAdd() const { return m_numNetReadAdd; }
|
|
int64_t getNumRepliesAdd() const { return m_numRepliesAdd; }
|
|
int64_t getNetSentAdd() const { return m_numNetSentAdd; }
|
|
|
|
rdbid_t getRdbId() const { return m_rdbId; }
|
|
const char* getDbname() const { return m_dbname; }
|
|
|
|
bool isDumping() const { return m_isDumping; }
|
|
|
|
bool isUseIndexFile() const { return m_useIndexFile; }
|
|
|
|
// . you'll lose your data in this class if you call this
|
|
void reset();
|
|
|
|
bool isSavingTree() const;
|
|
|
|
bool saveTree(bool useThread, void *state, void (*callback)(void *state));
|
|
bool saveIndexes();
|
|
bool saveMaps();
|
|
|
|
// . load the tree named "saved.dat", keys must be out of order because
|
|
// tree is not balanced
|
|
bool loadTree ( ) ;
|
|
|
|
static bool initializeRdbDumpThread();
|
|
static void finalizeRdbDumpThread();
|
|
static bool hasPendingRdbDumpJob();
|
|
|
|
void submitRdbDumpJob(bool forceDump);
|
|
|
|
bool needsDump() const;
|
|
|
|
// these are used for computing load on a machine
|
|
bool isMerging() const;
|
|
void incrementNumMerges() { ++m_numMergesOut; }
|
|
void decrementNumMerges() { --m_numMergesOut; }
|
|
|
|
// PageRepair.cpp calls this when it is done rebuilding an rdb
|
|
// and wants to tell the primary rdb to reload itself using the newly
|
|
// rebuilt files, pointed to by rdb2.
|
|
bool updateToRebuildFiles ( Rdb *rdb2 , char *coll ) ;
|
|
|
|
private:
|
|
bool addRdbBase2 ( collnum_t collnum );
|
|
void addBase(collnum_t collnum, RdbBase *base);
|
|
|
|
// returns false if no room in tree or m_mem for a list to add
|
|
bool hasRoom(RdbList *list);
|
|
|
|
static void dumpRdb(void *item);
|
|
|
|
bool getTreeCollExist(collnum_t collnum) const;
|
|
|
|
// . write out tree to a file with keys in order
|
|
bool dumpTree();
|
|
|
|
bool addList(collnum_t collnum, RdbList *list, bool checkForRoom);
|
|
// get the directory name where this rdb stores its files
|
|
const char *getDir() const { return g_hostdb.m_dir; }
|
|
|
|
bool dumpColl(RdbBase *base);
|
|
|
|
// . called when we've dumped the tree to disk w/ keys ordered
|
|
void doneDumping ( );
|
|
|
|
int32_t reclaimMemFromDeletedTreeNodes();
|
|
int32_t m_lastReclaim;
|
|
|
|
int32_t m_fixedDataSize;
|
|
|
|
char m_dbname [32];
|
|
int32_t m_dbnameLen;
|
|
|
|
bool m_useIndexFile;
|
|
|
|
// for storing records in memory
|
|
RdbTree m_tree;
|
|
RdbBuckets m_buckets;
|
|
|
|
bool m_useTree;
|
|
|
|
// for dumping a table to an rdb file
|
|
RdbDump m_dump;
|
|
|
|
// memory for us to use to avoid calling malloc()/mdup()/...
|
|
RdbMem m_mem;
|
|
|
|
mutable int32_t m_cacheLastTime;
|
|
mutable int64_t m_cacheLastTotal;
|
|
|
|
std::atomic<int32_t> m_numMergesOut;
|
|
|
|
int32_t m_minToMerge; // need at least this many files b4 merging
|
|
|
|
int32_t m_dumpErrno;
|
|
|
|
// a dummy data string for deleting records when m_fixedDataSize > 0
|
|
|
|
// for keeping stats
|
|
std::atomic<int64_t> m_numSeeks;
|
|
std::atomic<int64_t> m_numReSeeks;
|
|
std::atomic<int64_t> m_numRead;
|
|
|
|
// network request/reply info for get requests
|
|
std::atomic<int64_t> m_numReqsGet;
|
|
std::atomic<int64_t> m_numNetReadGet;
|
|
std::atomic<int64_t> m_numRepliesGet;
|
|
std::atomic<int64_t> m_numNetSentGet;
|
|
|
|
// network request/reply info for add requests
|
|
std::atomic<int64_t> m_numReqsAdd;
|
|
std::atomic<int64_t> m_numNetReadAdd;
|
|
std::atomic<int64_t> m_numRepliesAdd;
|
|
std::atomic<int64_t> m_numNetSentAdd;
|
|
|
|
// . when we dump list to an rdb file, can we use short keys?
|
|
// . currently exclusively used by indexdb
|
|
bool m_useHalfKeys;
|
|
|
|
bool m_niceness;
|
|
|
|
char m_treeAllocName[64]; //for memory used m_tree/m_buckets
|
|
char m_memAllocName[64]; //for memory used by m_mem
|
|
|
|
// set to true when dumping tree so RdbMem does not use the memory
|
|
// being dumped to hold newly added records
|
|
std::atomic<bool> m_isDumping;
|
|
|
|
rdbid_t m_rdbId;
|
|
|
|
char m_ks; // key size
|
|
int32_t m_pageSize;
|
|
|
|
bool m_initialized;
|
|
};
|
|
|
|
#endif // GB_RDB_H
|