forked from Mirrors/privacore-open-source-search-engine
Added logging and hard coded error checks to BigFile.cpp
This commit is contained in:
255
BigFile.cpp
255
BigFile.cpp
@ -214,7 +214,8 @@ bool BigFile::addParts ( char *dirname ) {
|
||||
// set our directory class
|
||||
if (!dir.open())
|
||||
{
|
||||
return log(LOG_ERROR, "disk: openDir (\"%s\") failed",dirname);
|
||||
log(LOG_ERROR, "disk: openDir (\"%s\") failed",dirname);
|
||||
return false;
|
||||
}
|
||||
|
||||
// match files with this pattern in the directory
|
||||
@ -273,7 +274,7 @@ bool BigFile::addParts ( char *dirname ) {
|
||||
// continue;
|
||||
// }
|
||||
// make this part file
|
||||
if ( ! addPart ( part ) )
|
||||
if( !addPart(part) )
|
||||
{
|
||||
log(LOG_ERROR,"%s:%s: END. addPart failed, returning false.", __FILE__,__FUNCTION__);
|
||||
return false;
|
||||
@ -397,10 +398,12 @@ bool BigFile::addPart ( int32_t n )
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
bool BigFile::doesExist ( ) {
|
||||
return m_numParts;
|
||||
}
|
||||
|
||||
|
||||
// if we can open it with a valid fd, then it exists
|
||||
bool BigFile::doesPartExist ( int32_t n ) {
|
||||
//if ( n >= MAX_PART_FILES ) return false;
|
||||
@ -411,16 +414,9 @@ bool BigFile::doesPartExist ( int32_t n ) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
static int64_t s_vfd = 0;
|
||||
|
||||
// do not use part files for this open so we can open regular really >2GB
|
||||
// sized files with it
|
||||
// bool BigFile::open2 ( int flags ,
|
||||
// void *pc ,
|
||||
// int64_t maxFileSize ,
|
||||
// int permissions ) {
|
||||
// return open ( flags , pc , maxFileSize , permissions , false );
|
||||
// }
|
||||
|
||||
// . overide File::open so we can set m_numParts
|
||||
// . set maxFileSize when opening a new file for writing and using
|
||||
@ -481,19 +477,17 @@ void BigFile::makeFilename_r ( char *baseFilename ,
|
||||
char *xx=NULL; *xx=0;
|
||||
}
|
||||
|
||||
//int BigFile::getfdByOffset ( int64_t offset ) {
|
||||
// return getfd ( offset / MAX_PART_SIZE , true /*forReading?*/ );
|
||||
//}
|
||||
|
||||
// . get the fd of the nth file
|
||||
// . will try to open the file if it hasn't yet been opened
|
||||
int BigFile::getfd ( int32_t n , bool forReading ) { // , int64_t *vfd ) {
|
||||
int BigFile::getfd ( int32_t n , bool forReading ) {
|
||||
|
||||
// boundary check
|
||||
if ( n >= m_maxParts && ! addPart ( n ) ) {
|
||||
log("disk: Part number %"INT32" > %"INT32". fd "
|
||||
"not available.",
|
||||
n,m_maxParts);
|
||||
if ( n >= m_maxParts && ! addPart ( n ) )
|
||||
{
|
||||
log(LOG_ERROR, "disk: Part number %"INT32" > %"INT32". fd not available.",
|
||||
n, m_maxParts);
|
||||
|
||||
// return -1 to indicate can't do it
|
||||
return -1;
|
||||
}
|
||||
@ -518,12 +512,14 @@ int BigFile::getfd ( int32_t n , bool forReading ) { // , int64_t *vfd ) {
|
||||
// get it's file descriptor
|
||||
int fd = f->getfd ( ) ;
|
||||
if ( fd >= -1 ) return fd;
|
||||
|
||||
// otherwise, fd is -2 and it's never been opened?!?!
|
||||
g_errno = EBADENGINEER;
|
||||
log(LOG_LOGIC,"disk: fd is -2.");
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
// . return -2 on error
|
||||
// . return -1 if does not exist
|
||||
// . otherwise return the big file's complete file size (can be well over 2gb)
|
||||
@ -555,6 +551,7 @@ int64_t BigFile::getFileSize ( ) {
|
||||
return totalSize;
|
||||
}
|
||||
|
||||
|
||||
// . return -2 on error
|
||||
// . return -1 if does not exist
|
||||
// . otherwise returns the oldest of the last mod dates of all the part files
|
||||
@ -581,6 +578,7 @@ time_t BigFile::getLastModifiedTime ( ) {
|
||||
return m_lastModified;
|
||||
}
|
||||
|
||||
|
||||
// . returns false if blocked, true otherwise
|
||||
// . sets g_errno on error
|
||||
// . we need a ptr to the ptr to this BigFile so if we get deleted and
|
||||
@ -601,6 +599,7 @@ bool BigFile::read ( void *buf ,
|
||||
hitDisk , allocOff );
|
||||
}
|
||||
|
||||
|
||||
// . returns false if blocked, true otherwise
|
||||
// . sets g_errno on error
|
||||
bool BigFile::write ( void *buf ,
|
||||
@ -624,6 +623,7 @@ bool BigFile::write ( void *buf ,
|
||||
true , 0 );
|
||||
}
|
||||
|
||||
|
||||
// . returns false if blocked, true otherwise
|
||||
// . sets g_errno on error
|
||||
// . we divide into 2 writes in case write spans 2 files
|
||||
@ -1096,6 +1096,7 @@ bool BigFile::readwrite ( void *buf ,
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
// . this should be called from the main process after getting our call OUR callback here
|
||||
void doneWrapper ( void *state , ThreadEntry *t ) {
|
||||
|
||||
@ -1398,6 +1399,7 @@ void *readwriteWrapper_r ( void *state , ThreadEntry *t ) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
// . returns false and sets errno on error, true on success
|
||||
// . don't log shit when you're in a thread anymore
|
||||
// . if we receive a cancel sig while in pread/pwrite it will return -1
|
||||
@ -1589,45 +1591,220 @@ bool readwrite_r ( FileState *fstate , ThreadEntry *t ) {
|
||||
goto loop;
|
||||
}
|
||||
|
||||
|
||||
////////////////////////////////////////
|
||||
// non-blocking unlink/rename code
|
||||
////////////////////////////////////////
|
||||
|
||||
bool BigFile::unlink ( ) {
|
||||
return unlinkRename ( NULL , -1 , false, NULL, NULL );
|
||||
bool BigFile::unlink ( )
|
||||
{
|
||||
bool rc;
|
||||
|
||||
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: BEGIN. filename [%s]", __FILE__,__FUNCTION__, getFilename());
|
||||
|
||||
|
||||
//### BR 20151218: Hard coded check to help us debug file deletion problem
|
||||
if( strstr(getFilename(),"posdb0001") )
|
||||
{
|
||||
log(LOG_ERROR,"%s:%s: TRYING TO unlink posdb0001!!", __FILE__,__FUNCTION__);
|
||||
logAllData(LOG_ERROR);
|
||||
|
||||
g_process.abort(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
rc=unlinkRename( NULL , -1 , false, NULL, NULL );
|
||||
if( !rc )
|
||||
{
|
||||
log(LOG_ERROR,"%s:%s: unlink failed. filename [%s]", __FILE__,__FUNCTION__, getFilename());
|
||||
logAllData(LOG_ERROR);
|
||||
}
|
||||
|
||||
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: END. returning [%s]", __FILE__,__FUNCTION__, rc?"true":"false");
|
||||
return rc;
|
||||
}
|
||||
|
||||
bool BigFile::move ( char *newDir ) {
|
||||
return rename ( m_baseFilename.getBufStart() , newDir );
|
||||
|
||||
|
||||
bool BigFile::move ( char *newDir )
|
||||
{
|
||||
bool rc;
|
||||
|
||||
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: BEGIN. filename [%s] newDir [%s]", __FILE__,__FUNCTION__, getFilename(), newDir);
|
||||
|
||||
//### BR 20151218: Hard coded check to help us debug file deletion problem
|
||||
if( strstr(getFilename(),"posdb0001") )
|
||||
{
|
||||
log(LOG_ERROR,"%s:%s: TRYING TO move posdb0001!!", __FILE__,__FUNCTION__);
|
||||
logAllData(LOG_ERROR);
|
||||
|
||||
g_process.abort(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
rc = rename( m_baseFilename.getBufStart() , newDir );
|
||||
if( !rc )
|
||||
{
|
||||
log(LOG_ERROR,"%s:%s: move failed. filename [%s] newDir [%s]", __FILE__,__FUNCTION__, getFilename(), newDir);
|
||||
logAllData(LOG_ERROR);
|
||||
}
|
||||
|
||||
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: END. returning [%s]", __FILE__,__FUNCTION__, rc?"true":"false");
|
||||
return rc;
|
||||
}
|
||||
|
||||
bool BigFile::rename ( char *newBaseFilename , char *newBaseFilenameDir ) {
|
||||
return unlinkRename ( newBaseFilename, -1, false, NULL, NULL ,
|
||||
newBaseFilenameDir );
|
||||
}
|
||||
bool BigFile::chopHead ( int32_t part ) {
|
||||
return unlinkRename ( NULL, part, false, NULL, NULL );
|
||||
|
||||
bool BigFile::rename(char *newBaseFilename , char *newBaseFilenameDir )
|
||||
{
|
||||
bool rc;
|
||||
|
||||
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: BEGIN. newBaseFilename [%s] newBaseFilenameDir [%s]", __FILE__,__FUNCTION__, newBaseFilename, newBaseFilenameDir);
|
||||
|
||||
|
||||
//### BR 20151218: Hard coded check to help us debug file deletion problem
|
||||
if( strstr(getFilename(),"posdb0001") )
|
||||
{
|
||||
log(LOG_ERROR,"%s:%s: TRYING TO rename posdb0001!!", __FILE__,__FUNCTION__);
|
||||
logAllData(LOG_ERROR);
|
||||
|
||||
g_process.abort(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
rc=unlinkRename ( newBaseFilename, -1, false, NULL, NULL, newBaseFilenameDir );
|
||||
if( !rc )
|
||||
{
|
||||
log(LOG_ERROR,"%s:%s: rename failed. newBaseFilename [%s] newBaseFilenameDir [%s]", __FILE__,__FUNCTION__, newBaseFilename, newBaseFilenameDir);
|
||||
logAllData(LOG_ERROR);
|
||||
}
|
||||
|
||||
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: END. returning [%s]", __FILE__,__FUNCTION__, rc?"true":"false");
|
||||
return rc;
|
||||
}
|
||||
|
||||
bool BigFile::unlink ( void (* callback) ( void *state ) ,
|
||||
void *state ) {
|
||||
return unlinkRename ( NULL , -1 , true, callback , state );
|
||||
|
||||
bool BigFile::chopHead(int32_t part )
|
||||
{
|
||||
bool rc;
|
||||
|
||||
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: BEGIN. part %"INT32"", __FILE__,__FUNCTION__, part);
|
||||
|
||||
//### BR 20151218: Hard coded check to help us debug file deletion problem
|
||||
if( strstr(getFilename(),"posdb0001") )
|
||||
{
|
||||
log(LOG_ERROR,"%s:%s: TRYING TO chopHead ON posdb0001!! part=%"INT32"", __FILE__,__FUNCTION__, part);
|
||||
logAllData(LOG_ERROR);
|
||||
|
||||
g_process.abort(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
rc=unlinkRename ( NULL, part, false, NULL, NULL );
|
||||
if( !rc )
|
||||
{
|
||||
log(LOG_ERROR,"%s:%s: chopHead failed. part %"INT32"", __FILE__,__FUNCTION__, part);
|
||||
logAllData(LOG_ERROR);
|
||||
}
|
||||
|
||||
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: END. returning [%s]", __FILE__,__FUNCTION__, rc?"true":"false");
|
||||
return rc;
|
||||
}
|
||||
|
||||
bool BigFile::rename ( char *newBaseFilename ,
|
||||
void (* callback) ( void *state ) ,
|
||||
void *state ) {
|
||||
return unlinkRename ( newBaseFilename, -1, true, callback, state);
|
||||
|
||||
bool BigFile::unlink(void (* callback) ( void *state ) , void *state )
|
||||
{
|
||||
bool rc;
|
||||
|
||||
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: BEGIN.", __FILE__,__FUNCTION__);
|
||||
|
||||
|
||||
//### BR 20151218: Hard coded check to help us debug file deletion problem
|
||||
if( strstr(getFilename(),"posdb0001") )
|
||||
{
|
||||
log(LOG_ERROR,"%s:%s: TRYING TO unlink posdb0001!! (callback)", __FILE__,__FUNCTION__);
|
||||
logAllData(LOG_ERROR);
|
||||
|
||||
g_process.abort(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
rc=unlinkRename ( NULL , -1 , true, callback , state );
|
||||
if( !rc )
|
||||
{
|
||||
log(LOG_ERROR,"%s:%s: unlink (callback) failed. filename [%s]", __FILE__,__FUNCTION__, getFilename());
|
||||
logAllData(LOG_ERROR);
|
||||
}
|
||||
|
||||
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: END. returning [%s]", __FILE__,__FUNCTION__, rc?"true":"false");
|
||||
return rc;
|
||||
}
|
||||
|
||||
bool BigFile::chopHead ( int32_t part ,
|
||||
void (* callback) ( void *state ) ,
|
||||
void *state ) {
|
||||
|
||||
bool BigFile::rename(char *newBaseFilename, void (*callback)(void *state), void *state)
|
||||
{
|
||||
bool rc;
|
||||
|
||||
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: BEGIN. filename [%s] newBaseFilename [%s]", __FILE__,__FUNCTION__, getFilename(), newBaseFilename);
|
||||
|
||||
|
||||
//### BR 20151218: Hard coded check to help us debug file deletion problem
|
||||
if( strstr(getFilename(),"posdb0001") )
|
||||
{
|
||||
log(LOG_ERROR,"%s:%s: TRYING TO rename posdb0001!! (callback)", __FILE__,__FUNCTION__);
|
||||
logAllData(LOG_ERROR);
|
||||
|
||||
g_process.abort(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
rc=unlinkRename ( newBaseFilename, -1, true, callback, state);
|
||||
if( !rc )
|
||||
{
|
||||
log(LOG_ERROR,"%s:%s: rename (callback) failed. filename [%s] newBaseFilename [%s]", __FILE__,__FUNCTION__, getFilename(), newBaseFilename);
|
||||
logAllData(LOG_ERROR);
|
||||
}
|
||||
|
||||
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: END. returning [%s]", __FILE__,__FUNCTION__, rc?"true":"false");
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
bool BigFile::chopHead(int32_t part, void (*callback)(void *state), void *state)
|
||||
{
|
||||
bool rc;
|
||||
|
||||
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: BEGIN. part %"INT32"", __FILE__,__FUNCTION__, part);
|
||||
|
||||
|
||||
//### BR 20151218: Hard coded check to help us debug file deletion problem
|
||||
if( strstr(getFilename(),"posdb0001") )
|
||||
{
|
||||
log(LOG_ERROR,"%s:%s: TRYING TO chopHead ON posdb0001!! (callback) part %"INT32"", __FILE__,__FUNCTION__, part);
|
||||
logAllData(LOG_ERROR);
|
||||
g_process.abort(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
//for ( int32_t i = 0 ; i < part ; i++ )
|
||||
// set return value to false if we blocked somewhere
|
||||
return unlinkRename ( NULL, part, true, callback, state );
|
||||
rc=unlinkRename ( NULL, part, true, callback, state );
|
||||
if( !rc )
|
||||
{
|
||||
log(LOG_ERROR,"%s:%s: chopHead (callback) failed. part %"INT32"", __FILE__,__FUNCTION__, part);
|
||||
logAllData(LOG_ERROR);
|
||||
}
|
||||
|
||||
if( g_conf.m_logDebugDetailed ) log(LOG_DEBUG,"%s:%s: END. returning [%s]", __FILE__,__FUNCTION__, rc?"true":"false");
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
|
||||
static void *renameWrapper_r ( void *state , ThreadEntry *t ) ;
|
||||
static void *unlinkWrapper_r ( void *state , ThreadEntry *t ) ;
|
||||
static void doneRenameWrapper ( void *state , ThreadEntry *t ) ;
|
||||
|
47
BigFile.h
47
BigFile.h
@ -144,18 +144,8 @@ class BigFile {
|
||||
int64_t maxFileSize = -1 ,
|
||||
int permissions =
|
||||
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH );
|
||||
//bool usePartFiles = true );
|
||||
|
||||
// this will set usepartfiles to false! so use this to open large
|
||||
// warc or arc files
|
||||
//bool open2 ( int flags ,
|
||||
// //class DiskPageCache *pc = NULL ,
|
||||
// void *pc = NULL ,
|
||||
// int64_t maxFileSize = -1 ,
|
||||
// int permissions =
|
||||
// S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH );
|
||||
|
||||
|
||||
void logAllData(int32_t log_type);
|
||||
|
||||
int getFlags() { return m_flags; };
|
||||
|
||||
@ -306,6 +296,7 @@ class BigFile {
|
||||
|
||||
// . returns false if blocked, true otherwise
|
||||
// . sets g_errno on error
|
||||
private:
|
||||
bool unlinkRename ( char *newBaseFilename ,
|
||||
int32_t part ,
|
||||
bool useThread ,
|
||||
@ -313,33 +304,22 @@ class BigFile {
|
||||
void *state ,
|
||||
char *newBaseFilenameDir = NULL ) ;
|
||||
|
||||
|
||||
// . add all parts from this directory
|
||||
// . called by set() above for normal dir as well as stripe dir
|
||||
bool addParts ( char *dirname ) ;
|
||||
|
||||
bool addPart ( int32_t n ) ;
|
||||
|
||||
void logAllData(int32_t log_type);
|
||||
|
||||
//bool unlinkPart ( int32_t n , bool block );
|
||||
|
||||
File *getFile2 ( int32_t n ) {
|
||||
if ( n >= m_maxParts ) return NULL;
|
||||
File **filePtrs = (File **)m_filePtrsBuf.getBufStart();
|
||||
File *f = filePtrs[n];
|
||||
//if ( ! f ->calledSet() ) return NULL;
|
||||
// this will be NULL if addPart(n) never called
|
||||
return f;
|
||||
};
|
||||
|
||||
// if part file not created, will create it
|
||||
//File *getPartFile2 ( int32_t n ) { return getFile2(n); }
|
||||
|
||||
bool reset ( );
|
||||
|
||||
// for basefilename to avoid an alloc
|
||||
char m_tmpBaseBuf[32];
|
||||
|
||||
|
||||
//int32_t m_permissions;
|
||||
int32_t m_flags;
|
||||
|
||||
public:
|
||||
// our most important the directory and filename
|
||||
SafeBuf m_dir ;// [256];
|
||||
SafeBuf m_baseFilename ;//[256];
|
||||
@ -350,9 +330,16 @@ class BigFile {
|
||||
// if first char in this dir is 0 then use m_dir
|
||||
SafeBuf m_newBaseFilenameDir ;//[256];
|
||||
|
||||
File *getFile2 ( int32_t n ) {
|
||||
if ( n >= m_maxParts ) return NULL;
|
||||
File **filePtrs = (File **)m_filePtrsBuf.getBufStart();
|
||||
File *f = filePtrs[n];
|
||||
//if ( ! f ->calledSet() ) return NULL;
|
||||
// this will be NULL if addPart(n) never called
|
||||
return f;
|
||||
};
|
||||
|
||||
//int32_t m_permissions;
|
||||
int32_t m_flags;
|
||||
bool reset ( );
|
||||
|
||||
// determined in open() override
|
||||
int m_numParts;
|
||||
|
9
Log.cpp
9
Log.cpp
@ -1,8 +1,10 @@
|
||||
#include "gb-include.h"
|
||||
|
||||
#include "Mem.h"
|
||||
#include <sys/types.h> // pid_t/getpid()
|
||||
#if defined(__linux__)
|
||||
#include <sys/types.h>
|
||||
#include <sys/syscall.h>
|
||||
#endif
|
||||
#include "Loop.h"
|
||||
#include "Conf.h"
|
||||
#include "Process.h"
|
||||
@ -311,9 +313,10 @@ bool Log::logR ( int64_t now , int32_t type , char *msg , bool asterisk ,
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Get thread id. pthread_self instead?
|
||||
long tid=syscall(SYS_gettid);
|
||||
p += sprintf(p, "%06ld ", tid);
|
||||
unsigned tid=(unsigned)syscall(SYS_gettid);
|
||||
p += sprintf(p, "%06u ", tid);
|
||||
|
||||
// Log level
|
||||
p += sprintf(p, "%s ", getTypeString(type));
|
||||
|
Reference in New Issue
Block a user