Files
privacore-open-source-searc…/HashTableX.h

353 lines
10 KiB
C
Raw Normal View History

2013-08-02 13:12:24 -07:00
// Matt Wells, Copyright, Dec. 2002
// . generic hash table class
2016-03-08 22:14:30 +01:00
#ifndef GB_HASHTABLEX_H
#define GB_HASHTABLEX_H
2013-08-02 13:12:24 -07:00
#include "SafeBuf.h"
2016-06-28 11:01:47 +02:00
#include "Sanity.h"
2016-06-20 19:29:10 +02:00
2013-08-02 13:12:24 -07:00
class HashTableX {
public:
2014-11-10 14:45:11 -08:00
bool set ( int32_t keySize ,
int32_t dataSize ,
int32_t initialNumSlots , // = 0 ,
2013-08-02 13:12:24 -07:00
char *buf , // = NULL ,
2014-11-10 14:45:11 -08:00
int32_t bufSize , // = 0 ,
2013-08-02 13:12:24 -07:00
bool allowDups , // = false ,
2014-11-10 14:45:11 -08:00
int32_t niceness , // = MAX_NICENESS ,
2016-03-14 23:10:06 +01:00
const char *allocName ,
2013-09-25 11:58:03 -06:00
bool useKeyMagic = false );
2013-08-02 13:12:24 -07:00
// key size is 0 if UNinitialized
2016-05-19 18:37:26 +02:00
bool isInitialized ( ) const { return (m_ks != 0); }
2013-08-02 13:12:24 -07:00
HashTableX ( );
~HashTableX ( );
// . add key/value entry to hash table
// . will grow hash table if it needs to
// . returns false and sets g_errno on error, returns true otherwise
2016-02-05 10:59:55 +01:00
bool addKey ( const void *key , const void *value , int32_t *slot = NULL );
2013-08-02 13:12:24 -07:00
// for value-less hashtables
2016-02-05 10:59:55 +01:00
bool addKey ( const void *key );
2013-08-02 13:12:24 -07:00
// . remove key/value entry to hash table.
// . returns false and sets g_errno on error.
2016-02-05 10:59:55 +01:00
bool removeKey ( const void *key );
2013-08-02 13:12:24 -07:00
// same as remove
2016-05-19 18:37:26 +02:00
bool deleteSlot ( int32_t n ) { return removeSlot(n); }
2013-08-02 13:12:24 -07:00
// like removeKey. returns false and sets g_errno on error.
2014-11-10 14:45:11 -08:00
bool removeSlot ( int32_t n );
2013-08-02 13:12:24 -07:00
// a replacement for TermTable.cpp
2016-02-05 10:59:55 +01:00
bool addTerm ( const int64_t *wid , int32_t score = 1 ) {
2014-11-10 14:45:11 -08:00
int32_t slot = getSlot ( wid );
2013-08-02 13:12:24 -07:00
if ( slot<0 ) return addKey( wid ,&score,&slot);
uint32_t *val = (uint32_t *)getValueFromSlot ( slot );
// overflow check
if ( *val + (uint32_t)score < *val ) *val = 0xffffffff;
else *val = *val + score;
return true;
2016-05-19 18:37:26 +02:00
}
2013-08-02 13:12:24 -07:00
// a replacement for TermTable.cpp
2016-03-14 23:10:06 +01:00
uint32_t getScore ( const int64_t *wid ) const {
2014-11-10 14:45:11 -08:00
int32_t slot = getSlot ( wid );
2013-08-02 13:12:24 -07:00
if ( slot < 0 ) return 0;
2016-03-14 23:10:06 +01:00
return *(const uint32_t *)getValueFromSlot ( slot );
2016-05-19 18:37:26 +02:00
}
2013-08-02 13:12:24 -07:00
// a replacement for TermTable.cpp
2016-03-14 23:10:06 +01:00
uint64_t getScore64FromSlot ( int32_t slot ) const {
2016-05-19 18:37:26 +02:00
return *(const uint64_t *)getValueFromSlot ( slot ); }
2013-08-02 13:12:24 -07:00
2016-02-05 10:59:55 +01:00
bool addTerm32 ( const int32_t *wid , int32_t score = 1 ) {
2014-11-10 14:45:11 -08:00
int32_t slot = getSlot ( wid );
2013-08-02 13:12:24 -07:00
if ( slot<0 ) return addKey( wid ,&score,&slot);
uint32_t *val = (uint32_t *)getValueFromSlot ( slot );
// overflow check
if ( *val + (uint32_t)score < *val ) *val = 0xffffffff;
else *val = *val + score;
return true;
2016-05-19 18:37:26 +02:00
}
2016-02-05 10:59:55 +01:00
bool addTerm32 ( const uint32_t *wid , int32_t score = 1 ) {
2014-11-10 14:45:11 -08:00
int32_t slot = getSlot ( wid );
2013-08-02 13:12:24 -07:00
if ( slot<0 ) return addKey( wid ,&score,&slot);
uint32_t *val = (uint32_t *)getValueFromSlot ( slot );
// overflow check
if ( *val + (uint32_t)score < *val ) *val = 0xffffffff;
else *val = *val + score;
return true;
2016-05-19 18:37:26 +02:00
}
2016-02-05 10:59:55 +01:00
bool addScore ( const int32_t *key , int32_t score = 1 ) {
return addTerm32 ( key , score );
2016-05-19 18:37:26 +02:00
}
2016-03-14 23:10:06 +01:00
uint32_t getScore32 ( const int32_t *wid ) const {
2014-11-10 14:45:11 -08:00
int32_t slot = getSlot ( wid );
2013-08-02 13:12:24 -07:00
if ( slot < 0 ) return 0;
2016-03-14 23:10:06 +01:00
return *(const uint32_t *)getValueFromSlot ( slot );
2016-05-19 18:37:26 +02:00
}
2013-08-02 13:12:24 -07:00
2016-02-05 10:59:55 +01:00
bool addTerm144 ( const key144_t *kp , int32_t score = 1 ) {
/*
// debug XmlDoc.cpp's hash table
int64_t termId = ((key144_t *)kp)->n2 >> 16;
uint64_t d = 0LL;
d = ((unsigned char *)kp)[11];
d <<= 32;
d |= *(uint32_t *)(((unsigned char *)kp)+7);
d >>= 2;
if ( d==110324895284 && termId == 39206941907955LL ) {
log("got it");
2016-06-20 19:29:10 +02:00
gbshutdownAbort(true);
}
*/
2013-08-02 13:12:24 -07:00
// grow it!
if ( (m_numSlots < 20 || 4 * m_numSlotsUsed >= m_numSlots) &&
m_numSlots < m_maxSlots ) {
2014-10-30 13:36:39 -06:00
int64_t growTo ;
growTo = ((int64_t)m_numSlots * 150LL )/100LL+20LL;
2013-08-02 13:12:24 -07:00
if ( growTo > m_maxSlots ) growTo = m_maxSlots;
2014-11-10 14:45:11 -08:00
if ( ! setTableSize ( (int32_t)growTo , NULL , 0 ) )
2013-08-02 13:12:24 -07:00
return false;
}
// hash it up
2016-02-05 10:59:55 +01:00
int32_t n = hash32 ( (const char *)kp, 18 );
2013-08-02 13:12:24 -07:00
// then mask it
n &= m_mask;
2014-11-10 14:45:11 -08:00
int32_t count = 0;
2013-08-02 13:12:24 -07:00
while ( count++ < m_numSlots ) {
// this is set to 0x01 if non-empty
if ( m_flags [ n ] == 0 ) {
gbmemcpy( &((key144_t *)m_keys)[n] ,kp,18);
2013-08-02 13:12:24 -07:00
m_vals[n*m_ds] = score;
m_flags[n] = 1;
m_numSlotsUsed++;
return true;
}
// get the key there
if (((key144_t *)m_keys)[n] == *kp) {
uint32_t *val = (uint32_t *)&m_vals[n*m_ds];
// overflow check
if ( *val + (uint32_t)score < *val )
*val = 0xffffffff;
else
*val = *val + score;
return true;
}
// advance otherwise
if ( ++n == m_numSlots ) n = 0;
}
// crazy!
log("hash: table is full!");
2016-06-20 19:29:10 +02:00
gbshutdownAbort(true);
2016-08-21 17:18:11 +02:00
/*NOTREACHED*/
2013-08-02 13:12:24 -07:00
return true;
2016-05-19 18:37:26 +02:00
}
2013-08-02 13:12:24 -07:00
// return 32-bit checksum of keys in table
2016-03-14 23:10:06 +01:00
int32_t getKeyChecksum32 () const;
2013-08-02 13:12:24 -07:00
// . used by ../english/Bits.h to store stop words, abbr's, ...
// . returns the score for this termId (0 means empty usually)
// . return 0 if key not in hash table
2016-02-05 10:59:55 +01:00
void *getValue ( const void *key ) {
2013-08-02 13:12:24 -07:00
// make it fast
2016-02-05 10:59:55 +01:00
if ( m_ks == 4 ) return getValue32 ( *(const int32_t *)key );
if ( m_ks == 8 ) return getValue64 ( *(const int64_t *)key );
2013-08-02 13:12:24 -07:00
// returns -1 if key not in hash table
2014-11-10 14:45:11 -08:00
int32_t n = getOccupiedSlotNum ( key );
2013-08-02 13:12:24 -07:00
if ( n < 0 ) return NULL;
return &m_vals[n*m_ds];
2016-05-19 18:37:26 +02:00
}
2013-08-02 13:12:24 -07:00
// . specialized for 32-bit keys for speed
// . returns NULL if not in table
2014-11-10 14:45:11 -08:00
void *getValue32 ( int32_t key ) {
2013-08-02 13:12:24 -07:00
// return NULL if completely empty
if ( m_numSlots <= 0 ) return NULL;
// sanity check
2016-06-20 19:29:10 +02:00
if ( m_ks != 4 ) { gbshutdownAbort(true); }
2014-11-10 14:45:11 -08:00
int32_t n;
2013-09-25 11:58:03 -06:00
if ( ! m_useKeyMagic ) {
// mask on the lower 32 bits i guess
n = key & m_mask;
}
else {
// get lower 32 bits of key
2014-11-10 14:45:11 -08:00
//n = (uint32_t)key;
n =*(uint32_t *)(((char *)&key) +m_maskKeyOffset);
2013-09-25 11:58:03 -06:00
// use magic to "randomize" key a little
n^=g_hashtab[(unsigned char)((char *)&key)[m_maskKeyOffset]][0];
2013-09-25 11:58:03 -06:00
// mask on the lower 32 bits i guess
n &= m_mask;
}
2014-11-10 14:45:11 -08:00
int32_t count = 0;
2013-08-02 13:12:24 -07:00
while ( count++ < m_numSlots ) {
// this is set to 0x01 if non-empty
if ( m_flags [ n ] == 0 ) return NULL;
// get the key there
2014-11-10 14:45:11 -08:00
if (((int32_t *)m_keys)[n] == key)
2013-08-02 13:12:24 -07:00
return &m_vals[n*m_ds];
// advance otherwise
if ( ++n == m_numSlots ) n = 0;
}
return NULL;
2016-05-19 18:37:26 +02:00
}
2013-08-02 13:12:24 -07:00
// . specialized for 64-bit keys for speed
// . returns NULL if not in table
2014-10-30 13:36:39 -06:00
void *getValue64 ( int64_t key ) {
2013-08-02 13:12:24 -07:00
// return NULL if completely empty
if ( m_numSlots <= 0 ) return NULL;
// sanity check
2016-06-20 19:29:10 +02:00
if ( m_ks != 8 ) { gbshutdownAbort(true); }
2014-11-10 14:45:11 -08:00
int32_t n;
2013-09-25 11:58:03 -06:00
if ( ! m_useKeyMagic ) {
// mask on the lower 32 bits i guess
// get lower 32 bits of key
n = key & m_mask;
}
else {
// use magic to "randomize" key a little
2014-11-10 14:45:11 -08:00
n =*(uint32_t *)(((char *)&key) +m_maskKeyOffset);
n ^= g_hashtab[(unsigned char)((char *)&key)[m_maskKeyOffset]][0];
2013-09-25 11:58:03 -06:00
// mask on the lower 32 bits i guess
n &= m_mask;
}
2014-11-10 14:45:11 -08:00
int32_t count = 0;
2013-08-02 13:12:24 -07:00
while ( count++ < m_numSlots ) {
// this is set to 0x01 if non-empty
if ( m_flags [ n ] == 0 ) return NULL;
// get the key there
2014-10-30 13:36:39 -06:00
if (((int64_t *)m_keys)[n] == key)
2013-08-02 13:12:24 -07:00
return &m_vals[n*m_ds];
// advance otherwise
if ( ++n == m_numSlots ) n = 0;
}
return NULL;
2016-05-19 18:37:26 +02:00
}
2013-08-02 13:12:24 -07:00
// value of 0 means empty
2016-05-19 18:37:26 +02:00
bool isEmpty ( const void *key ) const { return (getSlot(key) < 0); }
2013-08-02 13:12:24 -07:00
2016-05-19 18:37:26 +02:00
bool isInTable ( const void *key ) const { return (getSlot(key) >= 0); }
2013-08-02 13:12:24 -07:00
2016-05-19 18:37:26 +02:00
bool isEmpty ( int32_t n ) const { return (m_flags[n] == 0); }
2013-08-02 13:12:24 -07:00
2016-05-19 18:37:26 +02:00
bool isTableEmpty ( ) const { return (m_numSlotsUsed == 0); }
2013-08-02 13:12:24 -07:00
2016-05-19 18:37:26 +02:00
void * getKey ( int32_t n ) { return m_keys + n * m_ks; }
const void *getKey ( int32_t n ) const { return m_keys + n * m_ks; }
void * getKeyFromSlot ( int32_t n ) { return m_keys + n * m_ks; }
const void *getKeyFromSlot ( int32_t n ) const { return m_keys + n * m_ks; }
2013-08-02 13:12:24 -07:00
2016-03-14 23:10:06 +01:00
int64_t getKey64FromSlot ( int32_t n ) const {
2014-10-30 13:36:39 -06:00
return *(int64_t *)(m_keys+n*m_ks); }
2013-08-02 13:12:24 -07:00
2016-05-19 18:37:26 +02:00
int32_t getSlot ( const void *key ) const { return getOccupiedSlotNum ( key ); }
2013-08-02 13:12:24 -07:00
2016-03-14 23:10:06 +01:00
int32_t getNextSlot ( int32_t slot, const void *key ) const;
2013-08-02 13:12:24 -07:00
// count how many slots have this key
2016-03-14 23:10:06 +01:00
int32_t getCount ( const void *key ) const;
2013-08-02 13:12:24 -07:00
2016-02-05 10:59:55 +01:00
void setValue ( int32_t n , const void *val ) {
if (m_ds == 4) ((int32_t *)m_vals)[n] = *(const int32_t *)val;
else if (m_ds == 8) ((int64_t *)m_vals)[n] = *(const int64_t *)val;
else gbmemcpy(m_vals+n*m_ds,val,m_ds);
2016-05-19 18:37:26 +02:00
}
2013-08-02 13:12:24 -07:00
2016-05-19 18:37:26 +02:00
void * getValueFromSlot ( int32_t n ) { return m_vals + n * m_ds; }
const void *getValueFromSlot ( int32_t n ) const { return m_vals + n * m_ds; }
void * getDataFromSlot ( int32_t n ) { return m_vals + n * m_ds; }
const void *getDataFromSlot ( int32_t n ) const { return m_vals + n * m_ds; }
2013-08-02 13:12:24 -07:00
// frees the used memory, etc.
void reset ( );
// removes all key/value pairs from hash table, vacates all slots
void clear ( );
// how many are occupied?
2016-05-19 18:37:26 +02:00
int32_t getNumSlotsUsed ( ) const { return m_numSlotsUsed; }
int32_t getNumUsedSlots ( ) const { return m_numSlotsUsed; }
2013-08-02 13:12:24 -07:00
2016-03-14 23:10:06 +01:00
bool isEmpty() const {
if ( m_numSlotsUsed == 0 ) return true;
2016-05-19 18:37:26 +02:00
return false; }
2013-08-02 13:12:24 -07:00
// how many are there total? used and unused.
2016-05-19 18:37:26 +02:00
int32_t getNumSlots ( ) const { return m_numSlots; }
2013-08-02 13:12:24 -07:00
// both return false and set g_errno on error, true otherwise
2016-02-05 10:59:55 +01:00
bool load ( const char *dir, const char *filename ,
2014-11-10 14:45:11 -08:00
char **tbuf = NULL , int32_t *tsize = NULL );
2013-08-02 13:12:24 -07:00
2016-02-05 10:59:55 +01:00
bool save ( const char *dir, const char *filename ,
const char *tbuf = NULL , int32_t tsize = 0);
2013-08-02 13:12:24 -07:00
2014-11-10 14:45:11 -08:00
bool setTableSize ( int32_t numSlots , char *buf , int32_t bufSize );
2013-08-02 13:12:24 -07:00
2016-05-19 18:37:26 +02:00
void disableWrites () { m_isWritable = false; }
void enableWrites () { m_isWritable = true ; }
2013-08-02 13:12:24 -07:00
bool m_isWritable;
private:
2016-03-14 23:10:06 +01:00
int32_t getOccupiedSlotNum ( const void *key ) const;
2013-08-02 13:12:24 -07:00
public:
// . the array of buckets in which we store the terms
// . scores are allowed to exceed 8 bits for weighting purposes
char *m_keys;
char *m_vals;
char *m_flags;
2014-11-10 14:45:11 -08:00
int32_t m_numSlots;
int32_t m_numSlotsUsed;
2013-08-02 13:12:24 -07:00
uint32_t m_mask;
char m_doFree;
char *m_buf;
2014-11-10 14:45:11 -08:00
int32_t m_bufSize;
2013-08-02 13:12:24 -07:00
char m_useKeyMagic;
2014-11-10 14:45:11 -08:00
int32_t m_ks;
int32_t m_ds;
2013-08-02 13:12:24 -07:00
char m_allowDups;
2014-11-10 14:45:11 -08:00
int32_t m_niceness;
2013-08-02 13:12:24 -07:00
// a flag used by XmlDoc.cpp
bool m_addIffNotUnique;
bool m_isSaving;
bool m_needsSave;
// limits growing to this # of slots total
2014-10-30 13:36:39 -06:00
int64_t m_maxSlots;
2013-08-02 13:12:24 -07:00
2016-03-14 23:10:06 +01:00
const char *m_allocName;
2013-08-02 13:12:24 -07:00
2014-11-10 14:45:11 -08:00
int32_t m_maskKeyOffset;
2013-08-02 13:12:24 -07:00
// the addon buf used by SOME hashtables. data that the ptrs
// in the table itself reference.
char *m_txtBuf;
2014-11-10 14:45:11 -08:00
int32_t m_txtBufSize;
2013-08-02 13:12:24 -07:00
};
2016-03-08 22:14:30 +01:00
#endif // GB_HASHTABLEX_H