label the bigger safebuf chunks of mem

so we can see a better breakdown of mem
on the stats page, not just a big "SafeBuf"
allocation.
This commit is contained in:
mwells 2013-11-19 23:53:40 -07:00
parent 9c62ab362c
commit 46a683a904
13 changed files with 58 additions and 17 deletions

@ -799,7 +799,7 @@ bool sendPageAutoban ( TcpSocket *s , HttpRequest *r ) {
}
bool AutoBan::printTable( TcpSocket *s , HttpRequest *r ) {
SafeBuf sb(512 * 512);
SafeBuf sb(512 * 512,"autobbuf");
//read in all of the possible cgi parms off the bat:
//long user = g_pages.getUserType( s , r );
char *username = g_users.getUsername(r);

@ -462,6 +462,10 @@ bool Mem::init ( long long maxMem ) {
// this is called by C++ classes' constructors to register mem
void Mem::addMem ( void *mem , long size , const char *note , char isnew ) {
// enforce safebuf::setLabel being called
//if ( size>=100000 && note && strcmp(note,"SafeBuf")==0 ) {
// char *xx=NULL;*xx=0; }
//validate();
// sanity check

@ -23,6 +23,8 @@ bool sendPagePerf ( TcpSocket *s , HttpRequest *r ) {
// don't allow pages bigger than 128k in cache
char buf [ 64*1024 ];
SafeBuf p(buf, 64*1024);
p.setLabel ( "perfgrph" );
// print standard header
g_pages.printAdminTop ( &p , s , r );

@ -672,7 +672,7 @@ bool gotResults ( void *state ) {
//SafeBuf sb(local, 128000);
SafeBuf sb;
// reserve 1.5MB now!
if ( ! sb.reserve(1500000) ) // 128000) )
if ( ! sb.reserve(1500000 ,"pgresbuf" ) ) // 128000) )
return true;
SearchInput *si = &st->m_si;

@ -152,8 +152,8 @@ void sendReply ( void *state ) {
TcpSocket *s = st->m_socket;
SafeBuf buf( 1024*32 );
SafeBuf tmpBuf( 1024 );
SafeBuf buf( 1024*32 , "tmpbuf0" );
SafeBuf tmpBuf( 1024 , "tmpbuf1" );
//
// take these out until we need them!

@ -710,6 +710,9 @@ bool PosdbTable::allocTopTree ( ) {
// return false;
if ( m_r->m_getDocIdScoringInfo ) {
m_scoreInfoBuf.setLabel ("scinfobuf" );
// . for holding the scoring info
// . add 1 for the \0 safeMemcpy() likes to put at the end so
// it will not realloc on us
@ -729,6 +732,10 @@ bool PosdbTable::allocTopTree ( ) {
// compute. so this could easily get into the megabytes, most
// of the time we will not need nearly that much however.
numPairs *= xx;
m_pairScoreBuf.setLabel ( "pairbuf" );
m_singleScoreBuf.setLabel ("snglbuf" );
// but alloc it just in case
if ( ! m_pairScoreBuf.reserve (numPairs * sizeof(PairScore) ) )
return false;
@ -784,7 +791,7 @@ bool PosdbTable::allocTopTree ( ) {
slots = 20000000;
}
// each site hash is 4 bytes
if ( ! m_siteHashList.reserve ( slots ) )
if ( ! m_siteHashList.reserve ( slots ,"shshbuf" ) )
return false;
// quad # of sites to have space in between
if ( ! m_dt.set(4,0,slots,NULL,0,false,0,"pdtdt"))

@ -22,11 +22,12 @@
// 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,0,0,0,0,0,0,0,0
// };
SafeBuf::SafeBuf(long initSize) {
SafeBuf::SafeBuf(long initSize, char *label ) {
if(initSize <= 0) initSize = 1;
m_capacity = initSize;
m_length = 0;
m_buf = (char*)mrealloc(NULL, 0, m_capacity, "SafeBuf");
m_label = label;
m_buf = (char*)mrealloc(NULL, 0, m_capacity, m_label );
if(!m_buf) m_capacity = 0;
m_usingStack = false;
m_encoding = csUTF8;
@ -39,6 +40,11 @@ SafeBuf::SafeBuf() {
m_buf = NULL;
m_usingStack = false;
m_encoding = csUTF8;
m_label = NULL;
}
void SafeBuf::setLabel ( char *label ) {
m_label = label;
}
SafeBuf::SafeBuf(char* stackBuf, long cap) {
@ -47,6 +53,7 @@ SafeBuf::SafeBuf(char* stackBuf, long cap) {
m_buf = stackBuf;
m_length = 0;
m_encoding = csUTF8;
m_label = NULL;
}
SafeBuf::SafeBuf(char *heapBuf, long bufMax, long bytesInUse, bool ownData) {
@ -292,8 +299,14 @@ bool SafeBuf::advance ( long i ) {
return true;
}
bool SafeBuf::reserve(long i, char *label) {
if ( ! label ) label = "SafeBuf";
bool SafeBuf::reserve(long i , char *label) {
// if we don't already have a label and they provided one, use it
if ( ! m_label ) {
if ( label ) m_label = label;
else m_label = "SafeBuf";
}
if(m_length + i > m_capacity) {
char *tmpBuf = m_buf;
long tmpCap = m_capacity;
@ -301,7 +314,7 @@ bool SafeBuf::reserve(long i, char *label) {
m_buf = NULL;
m_capacity += i;
//if(m_capacity < 8) m_capacity = 8;
m_buf = (char*)mrealloc(m_buf, 0, m_capacity, label);
m_buf = (char*)mrealloc(m_buf, 0, m_capacity,m_label);
if(!m_buf) {
m_buf = tmpBuf;
m_capacity = tmpCap;
@ -315,7 +328,7 @@ bool SafeBuf::reserve(long i, char *label) {
}
m_capacity += i;
//if(m_capacity < 8) m_capacity = 8;
m_buf = (char*)mrealloc(m_buf, tmpCap, m_capacity,label);
m_buf = (char*)mrealloc(m_buf, tmpCap, m_capacity,m_label);
if(!m_buf) {
m_buf = tmpBuf;
m_capacity = tmpCap;
@ -330,11 +343,11 @@ bool SafeBuf::reserve(long i, char *label) {
//reserve this many bytes, if we need to alloc, we double the
//buffer size.
bool SafeBuf::reserve2x(long i) {
bool SafeBuf::reserve2x(long i, char *label) {
//watch out for overflow!
if((m_capacity << 1) + i < 0) return false;
if(i + m_length >= m_capacity)
return reserve(m_capacity + i);
return reserve(m_capacity + i,label);
else return true;
}

@ -8,12 +8,14 @@
struct SafeBuf {
//*TRUCTORS
SafeBuf();
SafeBuf(long initSize);
SafeBuf(long initSize, char *label = NULL);
//be careful with passing in a stackBuf! it could go out
//of scope independently of the safebuf.
SafeBuf(char* stackBuf, long cap);
SafeBuf(char *heapBuf, long bufMax, long bytesInUse, bool ownData);
~SafeBuf();
void setLabel ( char *label );
// CAUTION: BE CAREFUL WHEN USING THE FOLLOWING TWO FUNCTIONS!!
// setBuf() allows you reset the contents of the SafeBuf to either
@ -99,8 +101,8 @@ struct SafeBuf {
void reset() { m_length = 0; }
void purge(); // Clear all data and free all allocated memory
bool advance ( long i ) ;
bool reserve(long i, char *label=NULL);
bool reserve2x(long i);
bool reserve(long i , char *label = NULL );
bool reserve2x(long i, char *label = NULL );
bool inlineStyleTags();
void incrementLength(long i) { m_length += i; }
void setLength(long i) { m_length = i; };
@ -290,6 +292,7 @@ struct SafeBuf {
long m_capacity;
long m_length;
char *m_buf;
char *m_label;
bool m_usingStack;
short m_encoding; // output charset

@ -252,6 +252,8 @@ bool Sections::set ( Words *w ,
// breathe
QUICKPOLL(m_niceness);
m_sectionPtrBuf.setLabel("psectbuf");
// separate buf now for section ptr for each word
if ( ! m_sectionPtrBuf.reserve ( nw *4 ) ) return true;
m_sectionPtrs = (Section **)m_sectionPtrBuf.getBufStart();
@ -260,6 +262,8 @@ bool Sections::set ( Words *w ,
// allocate m_sectionBuf
m_sections = NULL;
m_sectionBuf.setLabel ( "sectbuf" );
if ( ! m_sectionBuf.reserve ( need ) )
return true;
@ -15160,6 +15164,9 @@ bool Sections::print2 ( SafeBuf *sbuf ,
// save ptrs
m_sbuf = sbuf;
m_sbuf->setLabel ("sectprnt");
//m_pt = pt;
//m_et = et;
//m_at = at;

@ -1000,6 +1000,8 @@ bool Speller::loadUnifiedDict() {
bool needRebuild = false;
m_unifiedBuf.setLabel("unibuf");
// this MUST be there
if ( m_unifiedBuf.fillFromFile(g_hostdb.m_dir,
"unifiedDict-buf.txt" ) == 0 )

@ -1214,7 +1214,7 @@ StatState *Statsdb::getStatState ( long us ) {
return ss;
}
// reserve
if ( ! m_sb0.reserve2x ( sizeof(StatState ) ) ) return NULL;
if ( ! m_sb0.reserve2x ( sizeof(StatState),"statgph" ) ) return NULL;
// make it otherwise
StatState *ss = (StatState *)m_sb0.getBuf();
// store the offset

@ -22,6 +22,8 @@ Wiktionary::Wiktionary () {
// . now m_langTable just maps to langId, no POS bits...
//m_langTable.set ( 6 , 1,0,NULL,0,false,0 ,"wkt-lang");
m_synTable.set ( 6 , 4,0,NULL,0,true,0 ,"wkt-synt");
m_synBuf.setLabel("synbuf");
}
void Wiktionary::reset() {

@ -98,6 +98,7 @@ static bool getWordPosVec ( Words *words ,
static void getMetaListWrapper ( void *state ) ;
XmlDoc::XmlDoc() {
m_esbuf.setLabel("exputfbuf");
for ( long i = 0 ; i < MAX_XML_DOCS ; i++ ) m_xmlDocs[i] = NULL;
m_freed = false;
m_contentInjected = false;