Add force/unforced rename and fix bug where first file was renamed

BigFile.cpp

@@ -1411,13 +1411,13 @@ bool BigFile::move ( const char *newDir )
 }
 
 
-bool BigFile::rename(const char *newBaseFilename, const char *newBaseFilenameDir )
+bool BigFile::rename(const char *newBaseFilename, const char *newBaseFilenameDir, bool force )
 {
 	bool rc;
 
 	logTrace( g_conf.m_logTraceBigFile, "BEGIN. newBaseFilename [%s] newBaseFilenameDir [%s]", newBaseFilename, newBaseFilenameDir);
 	
-	rc=unlinkRename ( newBaseFilename, -1, false, NULL, NULL, newBaseFilenameDir );
+	rc=unlinkRename ( newBaseFilename, -1, false, NULL, NULL, newBaseFilenameDir, force );
 	// rc indicates blocked/unblocked
 	
 	logTrace( g_conf.m_logTraceBigFile, "END. returning [%s]", rc?"true":"false");
@@ -1453,13 +1453,13 @@ bool BigFile::unlink(void (* callback) ( void *state ) , void *state )
 }
 
 
-bool BigFile::rename(const char *newBaseFilename, void (*callback)(void *state), void *state) 
+bool BigFile::rename(const char *newBaseFilename, void (*callback)(void *state), void *state, bool force)
 {
 	bool rc;
 
 	logTrace( g_conf.m_logTraceBigFile, "BEGIN. filename [%s] newBaseFilename [%s]", getFilename(), newBaseFilename);
 
-	rc=unlinkRename ( newBaseFilename, -1, true, callback, state);
+	rc=unlinkRename ( newBaseFilename, -1, true, callback, state, NULL, force );
 	// rc indicates blocked/unblocked
 	
 	logTrace( g_conf.m_logTraceBigFile, "END. returning [%s]", rc?"true":"false");
@@ -1467,7 +1467,7 @@ bool BigFile::rename(const char *newBaseFilename, void (*callback)(void *state),
 }
 
 
-bool BigFile::chopHead(int32_t part, void (*callback)(void *state), void *state) 
+bool BigFile::chopHead(int32_t part, void (*callback)(void *state), void *state)
 {
 	bool rc;
 
@@ -1497,11 +1497,8 @@ static void doneUnlinkWrapper ( void *state, job_exit_t exit_type );
 bool BigFile::unlinkRename ( // non-NULL for renames, NULL for unlinks
 			     const char *newBaseFilename             ,
 			     // part num to unlink, -1 for all (or rename)
-			     int32_t  part                        , 
-			     bool  useThread                   ,
-			     void (* callback) ( void *state ) , 
-			     void *state                       ,
-			     const char *newBaseFilenameDir          ) {
+			     int32_t  part, bool  useThread, void (* callback) ( void *state ),
+			     void *state, const char *newBaseFilenameDir, bool force ) {
 	logTrace( g_conf.m_logTraceBigFile, "BEGIN" );
 
 	// fail in read only mode
@@ -1614,6 +1611,7 @@ bool BigFile::unlinkRename ( // non-NULL for renames, NULL for unlinks
 		// base in ptr to file, but set f->m_this and f->m_i 
 		f->m_bigfile = this;
 		f->m_i    = i;
+		f->setForceRename( force );
 
 		// assume thread launched, doneRoutine() will decrement these
 		m_numThreads++; 
@@ -1704,6 +1702,8 @@ static void renameWrapper_r ( void *state ) {
 	BigFile *THIS = f->m_bigfile;
 	// get the ith file we just unlinked
 	int32_t      i = f->m_i;
+	bool forceRename = f->getForceRename();
+
 	// . get the new full name for this file
 	// . based on m_dir/m_stripeDir and m_baseFilename
 	char newFilename [ 1024 ];
@@ -1724,7 +1724,7 @@ static void renameWrapper_r ( void *state ) {
 	log(LOG_TRACE,"%s:%s:%d: disk: rename [%s] to [%s]", 
 	   __FILE__, __func__, __LINE__,oldFilename,newFilename);
 
-	if ( ::access( newFilename, F_OK ) != 0 ) {
+	if ( forceRename || ::access( newFilename, F_OK ) != 0 ) {
 		// suppress error (we will catch it in rename anyway)
 		errno = 0;
 

BigFile.h

@@ -179,7 +179,8 @@ class BigFile {
 	// . renames ALL parts too
 	// . doesn't change directory, just the base filename
 	// . use m_dir if newBaseFilenameDir is NULL
-	bool rename ( const char *newBaseFilename, const char *newBaseFilenameDir=NULL) ;
+	// . force = rename even if newFile exist
+	bool rename ( const char *newBaseFilename, const char *newBaseFilenameDir=NULL, bool force=false ) ;
 
 	bool move ( const char *newDir );
 
@@ -192,9 +193,7 @@ class BigFile {
 	// . they set g_errno on error
 	bool unlink   ( void (* callback) ( void *state ) , 
 		        void *state ) ;
-	bool rename   ( const char *newBaseFilename,
-		        void (* callback) ( void *state ) , 
-		        void *state ) ;
+	bool rename   ( const char *newBaseFilename, void (* callback) ( void *state ) , void *state, bool force=false ) ;
 	bool chopHead ( int32_t part , 
 			void (* callback) ( void *state ) , 
 			void *state ) ;
@@ -272,7 +271,8 @@ private:
 			    bool  useThread                   ,
 			    void (* callback) ( void *state ) ,
 			    void *state                       ,
-			    const char *newBaseFilenameDir = NULL   );
+			    const char *newBaseFilenameDir = NULL,
+				bool force = false );
 
 
 	// . add all parts from this directory

File.h

@@ -65,6 +65,14 @@ class File {
 	// returns false and sets errno on error, returns true on success
 	bool rename ( const char *newFilename );
 
+	void setForceRename( bool forceRename ) {
+		m_forceRename = forceRename;
+	}
+
+	bool getForceRename() const {
+		return m_forceRename;
+	}
+
 	bool calledOpen () { return m_calledOpen; }
 	bool calledSet  () { return m_calledSet; }
 
@@ -175,7 +183,6 @@ class File {
 
 	// THIS file's VIRTUAL descriptor
 	//int m_vfd;
-
 private:
 	// now just the real fd. is -1 if not opened
 	int m_fd;
@@ -186,6 +193,8 @@ private:
 	
 	char m_calledOpen;
 	char m_calledSet;
+
+	bool m_forceRename;
 };
 
 #endif // GB_FILE_H

RdbBase.cpp

@@ -897,8 +897,9 @@ bool RdbBase::incorporateMerge ( ) {
 	for ( int32_t i = a ; i < b ; i++ ) {
 		// incase we are starting with just the
 		// linkdb0001.003.dat file and not the stuff we merged
-		if ( ! m_files[i] )
+		if ( ! m_files[i] ) {
 			continue;
+		}
 
 		// debug msg
 		log(LOG_INFO,"merge: Unlinking merged file %s/%s (#%" PRId32").",
@@ -908,7 +909,8 @@ bool RdbBase::incorporateMerge ( ) {
 		// . they will save the filename before spawning so we can
 		//   delete the m_files[i] now
 		if ( ! m_files[i]->unlink ( doneWrapper , this ) ) {
-			m_numThreads++; g_numThreads++;
+			m_numThreads++;
+			g_numThreads++;
 		} else {
 			// debug msg
 			// MDW this cores if file is bad... if collection got delete from under us i guess!!
@@ -919,7 +921,8 @@ bool RdbBase::incorporateMerge ( ) {
 		log(LOG_INFO,"merge: Unlinking map file %s (#%" PRId32").", m_maps[i]->getFilename(),i);
 
 		if ( ! m_maps[i]->unlink  ( doneWrapper , this ) ) {
-			m_numThreads++; g_numThreads++;
+			m_numThreads++;
+			g_numThreads++;
 		} else {
 			// debug msg
 			log(LOG_INFO,"merge: Unlinked %s (#%" PRId32").", m_maps[i]->getFilename(), i);
@@ -971,11 +974,16 @@ void RdbBase::doneWrapper2 ( ) {
 
 	int32_t x = m_x;
 	int32_t a = m_a;
+
 	// . the fileId of the merge file becomes that of a
 	// . but secondary id should remain the same
 	m_fileIds [ x ] = m_fileIds [ a ];
-	if ( ! m_maps [x]->rename(m_maps[a]->getFilename(),doneWrapper3,this)){
-		m_numThreads++; g_numThreads++; }
+
+	log(LOG_INFO,"db: Renaming %s to %s", m_files[x]->getFilename(), m_files[a]->getFilename());
+	if ( ! m_maps[x]->rename( m_maps[a]->getFilename(), doneWrapper3, this ) ) {
+		m_numThreads++;
+		g_numThreads++;
+	}
 
 	// sanity check
 	m_files[x]->m_fileSize = -1;
@@ -984,8 +992,7 @@ void RdbBase::doneWrapper2 ( ) {
 	int64_t fs2 = m_maps[x]->getFileSize();
 	// compare
 	if ( fs != fs2 ) {
-		log("build: Map file size does not agree with actual file "
-		    "size");
+		log("build: Map file size does not agree with actual file size");
 		g_process.shutdownAbort(true);
 	}
 
@@ -993,24 +1000,32 @@ void RdbBase::doneWrapper2 ( ) {
 		// debug statement
 		log(LOG_INFO,"db: Renaming %s of size %" PRId64" to %s",
 		    m_files[x]->getFilename(),fs , m_files[a]->getFilename());
-		// rename it, this may block
-		if ( ! m_files[x]->rename ( m_files[a]->getFilename() ,
-					    doneWrapper3 , this ) ) {
-			m_numThreads++; g_numThreads++; }
-	}
-	else {
-		// rename to this (titledb%04" PRId32"-%03" PRId32".dat)
-		char buf [ 1024 ];
-		// use m_dbname in case its titledbRebuild
-		sprintf ( buf , "%s%04" PRId32"-%03" PRId32".dat" ,
-			  m_dbname, m_fileIds[a], m_fileIds2[x] );
 
 		// rename it, this may block
-		if ( ! m_files   [ x ]->rename ( buf , doneWrapper3, this ) ) {
-			m_numThreads++; g_numThreads++; }
+		if ( ! m_files[x]->rename ( m_files[a]->getFilename(), doneWrapper3, this ) ) {
+			m_numThreads++;
+			g_numThreads++;
+		}
+	} else {
+		// rename to this (titledb%04" PRId32"-%03" PRId32".dat)
+		char buf [ 1024 ];
+
+		// use m_dbname in case its titledbRebuild
+		sprintf ( buf , "%s%04" PRId32"-%03" PRId32".dat", m_dbname, m_fileIds[a], m_fileIds2[x] );
+
+		// rename it, this may block
+		if ( ! m_files[ x ]->rename ( buf, doneWrapper3, this ) ) {
+			m_numThreads++;
+			g_numThreads++;
+		}
 	}
+
 	// if we blocked on all, keep going
-	if ( m_numThreads == 0 ) { doneWrapper4 ( ); return ; }
+	if ( m_numThreads == 0 ) {
+		doneWrapper4 ( );
+		return ;
+	}
+
 	// . otherwise we blocked
 	// . we are now unlinking
 	// . this is so Msg3.cpp can avoid reading the [a,b) files
@@ -1527,9 +1542,10 @@ bool RdbBase::attemptMerge ( int32_t niceness, bool forceMergeAll, bool doLog ,
 			}
 			return false;
 		}
+
 		// make a log note
-		log(LOG_INFO,"merge: Resuming killed merge for %s coll=%s.",
-		    m_dbname,m_coll);
+		log(LOG_INFO,"merge: Resuming killed merge for %s coll=%s.", m_dbname,m_coll);
+
 		// compute the total size of merged file
 		mint = 0;
 		int32_t mm = 0;
@@ -1552,21 +1568,19 @@ bool RdbBase::attemptMerge ( int32_t niceness, bool forceMergeAll, bool doLog ,
 			if ( i > j ) mm++;
 			mint += m_files[i]->getFileSize();
 		}
+
 		if ( mm != n ) {
-			log( LOG_INFO, "merge: Only merging %" PRId32" instead of the original %" PRId32" files.",mm,n);
-			// cause the "if (mm==0)" to kick in below
-			if ( mm == 1 || mm == 0 ) {
-				mm = 0;
-				// fix renaming final merged file tagdb-001.dat
-				mergeFileId = 2;
-				m_fileIds[j] = 1;
-			}
+			log( LOG_INFO, "merge: Only merging %" PRId32" instead of the original %" PRId32" files.", mm, n );
 		}
+
 		// how many files to merge?
 		n = mm;
+
 		// allow a single file to continue merging if the other
 		// file got merged out already
-		if ( mm > 0 ) overide = true;
+		if ( mm > 0 ) {
+			overide = true;
+		}
 
 		// if we've already merged and already unlinked, then the
 		// process exited, now we restart with just the final 
@@ -1574,20 +1588,25 @@ bool RdbBase::attemptMerge ( int32_t niceness, bool forceMergeAll, bool doLog ,
 		if ( mm == 0 && rdbId != RDB_TITLEDB ) {
 			m_isMerging = false;
 			// make a fake file before us that we were merging
-			// since it got nuked on disk
-			//incorporateMerge();
+			// since it got nuked on disk incorporateMerge();
 			char fbuf[256];
-			sprintf(fbuf,"%s%04" PRId32".dat",m_dbname,mergeFileId-1);
-			if ( m_isTitledb )
-				sprintf(fbuf,"%s%04" PRId32"-%03" PRId32".dat",
-					m_dbname,mergeFileId-1,id2);
+
+			if ( m_isTitledb ) {
+				sprintf( fbuf, "%s%04" PRId32"-%03" PRId32".dat", m_dbname, mergeFileId + 1, id2 );
+			} else {
+				sprintf( fbuf, "%s%04" PRId32".dat", m_dbname, mergeFileId + 1 );
+			}
 			log(LOG_INFO, "merge: renaming final merged file %s",fbuf);
-			// this does not use a thread...
 			m_files[j]->rename(fbuf);
-			sprintf(fbuf,"%s%04" PRId32".map",m_dbname,mergeFileId-1);
-			//File *mf = m_maps[j]->getFile();
-			m_maps[j]->rename(fbuf);
+
+			sprintf( fbuf, "%s%04" PRId32".map", m_dbname, mergeFileId + 1 );
+
+			// we could potentially have a 'regenerated' map file that has already been moved.
+			// eg: merge dies after moving map file, but before moving data files.
+			//     next start up, map file will be regenerated. means we now have both even & odd map files
 			log(LOG_INFO, "merge: renaming final merged file %s",fbuf);
+			m_maps[j]->rename(fbuf, true);
+
 			return false;
 		}
 

RdbMap.h

@@ -89,13 +89,12 @@ class RdbMap {
 		   int32_t fixedDataSize , bool useHalfKeys , char keySize ,
 		   int32_t pageSize );
 
-	bool rename ( const char *newMapFilename ) {
-		return m_file.rename ( newMapFilename );
+	bool rename ( const char *newMapFilename, bool force = false ) {
+		return m_file.rename ( newMapFilename, NULL, force );
 	}
 
-	bool rename ( const char *newMapFilename,
-		      void (* callback)(void *state) , void *state ) { 
-		return m_file.rename ( newMapFilename , callback , state );
+	bool rename ( const char *newMapFilename, void (* callback)(void *state) , void *state, bool force = false ) {
+		return m_file.rename ( newMapFilename , callback , state, force );
 	}
 
 	char *getFilename ( ) { return m_file.getFilename(); }