Added logging and hard coded error checks to BigFile.cpp

This commit is contained in:
Brian Rasmusson
2015-12-18 12:05:23 +01:00
parent f5d4f0bf26
commit 784a38b198
3 changed files with 239 additions and 72 deletions

@ -214,7 +214,8 @@ bool BigFile::addParts ( char *dirname ) {
// set our directory class
if (!dir.open())
{
return log(LOG_ERROR, "disk: openDir (\"%s\") failed",dirname);
log(LOG_ERROR, "disk: openDir (\"%s\") failed",dirname);
return false;
}
// match files with this pattern in the directory
@ -273,7 +274,7 @@ bool BigFile::addParts ( char *dirname ) {
// continue;
// }
// make this part file
if ( ! addPart ( part ) )
if( !addPart(part) )
{
log(LOG_ERROR,"%s:%s: END. addPart failed, returning false.", __FILE__,__FUNCTION__);
return false;
@ -397,10 +398,12 @@ bool BigFile::addPart ( int32_t n )
return true;
}
bool BigFile::doesExist ( ) {
return m_numParts;
}
// if we can open it with a valid fd, then it exists
bool BigFile::doesPartExist ( int32_t n ) {
//if ( n >= MAX_PART_FILES ) return false;
@ -411,16 +414,9 @@ bool BigFile::doesPartExist ( int32_t n ) {
return false;
}
static int64_t s_vfd = 0;
// do not use part files for this open so we can open regular really >2GB
// sized files with it
// bool BigFile::open2 ( int flags ,
// void *pc ,
// int64_t maxFileSize ,
// int permissions ) {
// return open ( flags , pc , maxFileSize , permissions , false );
// }
// . overide File::open so we can set m_numParts
// . set maxFileSize when opening a new file for writing and using
@ -481,19 +477,17 @@ void BigFile::makeFilename_r ( char *baseFilename ,
char *xx=NULL; *xx=0;
}
//int BigFile::getfdByOffset ( int64_t offset ) {
// return getfd ( offset / MAX_PART_SIZE , true /*forReading?*/ );
//}
// . get the fd of the nth file
// . will try to open the file if it hasn't yet been opened
int BigFile::getfd ( int32_t n , bool forReading ) { // , int64_t *vfd ) {
int BigFile::getfd ( int32_t n , bool forReading ) {
// boundary check
if ( n >= m_maxParts && ! addPart ( n ) ) {
log("disk: Part number %"INT32" > %"INT32". fd "
"not available.",
n,m_maxParts);
if ( n >= m_maxParts && ! addPart ( n ) )
{
log(LOG_ERROR, "disk: Part number %"INT32" > %"INT32". fd not available.",
n, m_maxParts);
// return -1 to indicate can't do it
return -1;
}
@ -518,12 +512,14 @@ int BigFile::getfd ( int32_t n , bool forReading ) { // , int64_t *vfd ) {
// get it's file descriptor
int fd = f->getfd ( ) ;
if ( fd >= -1 ) return fd;
// otherwise, fd is -2 and it's never been opened?!?!
g_errno = EBADENGINEER;
log(LOG_LOGIC,"disk: fd is -2.");
return -1;
}
// . return -2 on error
// . return -1 if does not exist
// . otherwise return the big file's complete file size (can be well over 2gb)
@ -555,6 +551,7 @@ int64_t BigFile::getFileSize ( ) {
return totalSize;
}
// . return -2 on error
// . return -1 if does not exist
// . otherwise returns the oldest of the last mod dates of all the part files
@ -581,6 +578,7 @@ time_t BigFile::getLastModifiedTime ( ) {
return m_lastModified;
}
// . returns false if blocked, true otherwise
// . sets g_errno on error
// . we need a ptr to the ptr to this BigFile so if we get deleted and
@ -601,6 +599,7 @@ bool BigFile::read ( void *buf ,
hitDisk , allocOff );
}
// . returns false if blocked, true otherwise
// . sets g_errno on error
bool BigFile::write ( void *buf ,
@ -624,6 +623,7 @@ bool BigFile::write ( void *buf ,
true , 0 );
}
// . returns false if blocked, true otherwise
// . sets g_errno on error
// . we divide into 2 writes in case write spans 2 files
@ -1096,6 +1096,7 @@ bool BigFile::readwrite ( void *buf ,
return true;
}
// . this should be called from the main process after getting our call OUR callback here
void doneWrapper ( void *state , ThreadEntry *t ) {
@ -1398,6 +1399,7 @@ void *readwriteWrapper_r ( void *state , ThreadEntry *t ) {
return NULL;
}
// . returns false and sets errno on error, true on success
// . don't log shit when you're in a thread anymore
// . if we receive a cancel sig while in pread/pwrite it will return -1
@ -1589,45 +1591,220 @@ bool readwrite_r ( FileState *fstate , ThreadEntry *t ) {
goto loop;
}
////////////////////////////////////////
// non-blocking unlink/rename code
////////////////////////////////////////
bool BigFile::unlink ( ) {
return unlinkRename ( NULL , -1 , false, NULL, NULL );
bool BigFile::unlink ( )
{
bool rc;
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: BEGIN. filename [%s]", __FILE__,__FUNCTION__, getFilename());
//### BR 20151218: Hard coded check to help us debug file deletion problem
if( strstr(getFilename(),"posdb0001") )
{
log(LOG_ERROR,"%s:%s: TRYING TO unlink posdb0001!!", __FILE__,__FUNCTION__);
logAllData(LOG_ERROR);
g_process.abort(false);
return false;
}
rc=unlinkRename( NULL , -1 , false, NULL, NULL );
if( !rc )
{
log(LOG_ERROR,"%s:%s: unlink failed. filename [%s]", __FILE__,__FUNCTION__, getFilename());
logAllData(LOG_ERROR);
}
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: END. returning [%s]", __FILE__,__FUNCTION__, rc?"true":"false");
return rc;
}
bool BigFile::move ( char *newDir ) {
return rename ( m_baseFilename.getBufStart() , newDir );
bool BigFile::move ( char *newDir )
{
bool rc;
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: BEGIN. filename [%s] newDir [%s]", __FILE__,__FUNCTION__, getFilename(), newDir);
//### BR 20151218: Hard coded check to help us debug file deletion problem
if( strstr(getFilename(),"posdb0001") )
{
log(LOG_ERROR,"%s:%s: TRYING TO move posdb0001!!", __FILE__,__FUNCTION__);
logAllData(LOG_ERROR);
g_process.abort(false);
return false;
}
rc = rename( m_baseFilename.getBufStart() , newDir );
if( !rc )
{
log(LOG_ERROR,"%s:%s: move failed. filename [%s] newDir [%s]", __FILE__,__FUNCTION__, getFilename(), newDir);
logAllData(LOG_ERROR);
}
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: END. returning [%s]", __FILE__,__FUNCTION__, rc?"true":"false");
return rc;
}
bool BigFile::rename ( char *newBaseFilename , char *newBaseFilenameDir ) {
return unlinkRename ( newBaseFilename, -1, false, NULL, NULL ,
newBaseFilenameDir );
}
bool BigFile::chopHead ( int32_t part ) {
return unlinkRename ( NULL, part, false, NULL, NULL );
bool BigFile::rename(char *newBaseFilename , char *newBaseFilenameDir )
{
bool rc;
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: BEGIN. newBaseFilename [%s] newBaseFilenameDir [%s]", __FILE__,__FUNCTION__, newBaseFilename, newBaseFilenameDir);
//### BR 20151218: Hard coded check to help us debug file deletion problem
if( strstr(getFilename(),"posdb0001") )
{
log(LOG_ERROR,"%s:%s: TRYING TO rename posdb0001!!", __FILE__,__FUNCTION__);
logAllData(LOG_ERROR);
g_process.abort(false);
return false;
}
rc=unlinkRename ( newBaseFilename, -1, false, NULL, NULL, newBaseFilenameDir );
if( !rc )
{
log(LOG_ERROR,"%s:%s: rename failed. newBaseFilename [%s] newBaseFilenameDir [%s]", __FILE__,__FUNCTION__, newBaseFilename, newBaseFilenameDir);
logAllData(LOG_ERROR);
}
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: END. returning [%s]", __FILE__,__FUNCTION__, rc?"true":"false");
return rc;
}
bool BigFile::unlink ( void (* callback) ( void *state ) ,
void *state ) {
return unlinkRename ( NULL , -1 , true, callback , state );
bool BigFile::chopHead(int32_t part )
{
bool rc;
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: BEGIN. part %"INT32"", __FILE__,__FUNCTION__, part);
//### BR 20151218: Hard coded check to help us debug file deletion problem
if( strstr(getFilename(),"posdb0001") )
{
log(LOG_ERROR,"%s:%s: TRYING TO chopHead ON posdb0001!! part=%"INT32"", __FILE__,__FUNCTION__, part);
logAllData(LOG_ERROR);
g_process.abort(false);
return false;
}
rc=unlinkRename ( NULL, part, false, NULL, NULL );
if( !rc )
{
log(LOG_ERROR,"%s:%s: chopHead failed. part %"INT32"", __FILE__,__FUNCTION__, part);
logAllData(LOG_ERROR);
}
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: END. returning [%s]", __FILE__,__FUNCTION__, rc?"true":"false");
return rc;
}
bool BigFile::rename ( char *newBaseFilename ,
void (* callback) ( void *state ) ,
void *state ) {
return unlinkRename ( newBaseFilename, -1, true, callback, state);
bool BigFile::unlink(void (* callback) ( void *state ) , void *state )
{
bool rc;
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: BEGIN.", __FILE__,__FUNCTION__);
//### BR 20151218: Hard coded check to help us debug file deletion problem
if( strstr(getFilename(),"posdb0001") )
{
log(LOG_ERROR,"%s:%s: TRYING TO unlink posdb0001!! (callback)", __FILE__,__FUNCTION__);
logAllData(LOG_ERROR);
g_process.abort(false);
return false;
}
rc=unlinkRename ( NULL , -1 , true, callback , state );
if( !rc )
{
log(LOG_ERROR,"%s:%s: unlink (callback) failed. filename [%s]", __FILE__,__FUNCTION__, getFilename());
logAllData(LOG_ERROR);
}
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: END. returning [%s]", __FILE__,__FUNCTION__, rc?"true":"false");
return rc;
}
bool BigFile::chopHead ( int32_t part ,
void (* callback) ( void *state ) ,
void *state ) {
bool BigFile::rename(char *newBaseFilename, void (*callback)(void *state), void *state)
{
bool rc;
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: BEGIN. filename [%s] newBaseFilename [%s]", __FILE__,__FUNCTION__, getFilename(), newBaseFilename);
//### BR 20151218: Hard coded check to help us debug file deletion problem
if( strstr(getFilename(),"posdb0001") )
{
log(LOG_ERROR,"%s:%s: TRYING TO rename posdb0001!! (callback)", __FILE__,__FUNCTION__);
logAllData(LOG_ERROR);
g_process.abort(false);
return false;
}
rc=unlinkRename ( newBaseFilename, -1, true, callback, state);
if( !rc )
{
log(LOG_ERROR,"%s:%s: rename (callback) failed. filename [%s] newBaseFilename [%s]", __FILE__,__FUNCTION__, getFilename(), newBaseFilename);
logAllData(LOG_ERROR);
}
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: END. returning [%s]", __FILE__,__FUNCTION__, rc?"true":"false");
return rc;
}
bool BigFile::chopHead(int32_t part, void (*callback)(void *state), void *state)
{
bool rc;
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: BEGIN. part %"INT32"", __FILE__,__FUNCTION__, part);
//### BR 20151218: Hard coded check to help us debug file deletion problem
if( strstr(getFilename(),"posdb0001") )
{
log(LOG_ERROR,"%s:%s: TRYING TO chopHead ON posdb0001!! (callback) part %"INT32"", __FILE__,__FUNCTION__, part);
logAllData(LOG_ERROR);
g_process.abort(false);
return false;
}
//for ( int32_t i = 0 ; i < part ; i++ )
// set return value to false if we blocked somewhere
return unlinkRename ( NULL, part, true, callback, state );
rc=unlinkRename ( NULL, part, true, callback, state );
if( !rc )
{
log(LOG_ERROR,"%s:%s: chopHead (callback) failed. part %"INT32"", __FILE__,__FUNCTION__, part);
logAllData(LOG_ERROR);
}
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: END. returning [%s]", __FILE__,__FUNCTION__, rc?"true":"false");
return rc;
}
static void *renameWrapper_r ( void *state , ThreadEntry *t ) ;
static void *unlinkWrapper_r ( void *state , ThreadEntry *t ) ;
static void doneRenameWrapper ( void *state , ThreadEntry *t ) ;

@ -144,18 +144,8 @@ class BigFile {
int64_t maxFileSize = -1 ,
int permissions =
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH );
//bool usePartFiles = true );
// this will set usepartfiles to false! so use this to open large
// warc or arc files
//bool open2 ( int flags ,
// //class DiskPageCache *pc = NULL ,
// void *pc = NULL ,
// int64_t maxFileSize = -1 ,
// int permissions =
// S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH );
void logAllData(int32_t log_type);
int getFlags() { return m_flags; };
@ -306,6 +296,7 @@ class BigFile {
// . returns false if blocked, true otherwise
// . sets g_errno on error
private:
bool unlinkRename ( char *newBaseFilename ,
int32_t part ,
bool useThread ,
@ -313,33 +304,22 @@ class BigFile {
void *state ,
char *newBaseFilenameDir = NULL ) ;
// . add all parts from this directory
// . called by set() above for normal dir as well as stripe dir
bool addParts ( char *dirname ) ;
bool addPart ( int32_t n ) ;
void logAllData(int32_t log_type);
//bool unlinkPart ( int32_t n , bool block );
File *getFile2 ( int32_t n ) {
if ( n >= m_maxParts ) return NULL;
File **filePtrs = (File **)m_filePtrsBuf.getBufStart();
File *f = filePtrs[n];
//if ( ! f ->calledSet() ) return NULL;
// this will be NULL if addPart(n) never called
return f;
};
// if part file not created, will create it
//File *getPartFile2 ( int32_t n ) { return getFile2(n); }
bool reset ( );
// for basefilename to avoid an alloc
char m_tmpBaseBuf[32];
//int32_t m_permissions;
int32_t m_flags;
public:
// our most important the directory and filename
SafeBuf m_dir ;// [256];
SafeBuf m_baseFilename ;//[256];
@ -350,9 +330,16 @@ class BigFile {
// if first char in this dir is 0 then use m_dir
SafeBuf m_newBaseFilenameDir ;//[256];
File *getFile2 ( int32_t n ) {
if ( n >= m_maxParts ) return NULL;
File **filePtrs = (File **)m_filePtrsBuf.getBufStart();
File *f = filePtrs[n];
//if ( ! f ->calledSet() ) return NULL;
// this will be NULL if addPart(n) never called
return f;
};
//int32_t m_permissions;
int32_t m_flags;
bool reset ( );
// determined in open() override
int m_numParts;

@ -1,8 +1,10 @@
#include "gb-include.h"
#include "Mem.h"
#include <sys/types.h> // pid_t/getpid()
#if defined(__linux__)
#include <sys/types.h>
#include <sys/syscall.h>
#endif
#include "Loop.h"
#include "Conf.h"
#include "Process.h"
@ -311,9 +313,10 @@ bool Log::logR ( int64_t now , int32_t type , char *msg , bool asterisk ,
}
// Get thread id. pthread_self instead?
long tid=syscall(SYS_gettid);
p += sprintf(p, "%06ld ", tid);
unsigned tid=(unsigned)syscall(SYS_gettid);
p += sprintf(p, "%06u ", tid);
// Log level
p += sprintf(p, "%s ", getTypeString(type));