Fix some bugs with special posdb delete doc key.

Rdb.cpp

@@ -1945,10 +1945,22 @@ bool Rdb::addRecord(collnum_t collnum, char *key, char *data, int32_t dataSize)
 	KEYXOR(oppKey, 0x01);
 
 	if (m_useIndexFile) {
+		bool isSpecialKey;
+		if (m_rdbId == RDB_POSDB || m_rdbId == RDB2_POSDB2) {
+			isSpecialKey = (Posdb::getTermId(key) == POSDB_DELETEDOC_TERMID);
+		} else {
+			/// @todo ALC cater for other rdb types here
+			gbshutdownLogicError();
+		}
+
 		// there are no negative keys when we're using index (except special keys eg: posdb with termId 0)
 		// if we're adding key that have a corresponding opposite key, it means we want to remove the key from the tree
+		// even if it's a positive key (how else would we remove the special negative key?)
 		bool deleted = m_useTree ? m_tree.deleteNode(collnum, oppKey, true) : m_buckets.deleteNode(collnum, oppKey);
 		if (deleted) {
+			// assume that we don't need to delete from index even when we get positive special key
+			// since positive special key will only be inserted when a new document is added
+			// this means that other keys should overwrite the existing deleted docId
 			logTrace(g_conf.m_logTraceRdb, "END. %s: Key with corresponding opposite key deleted in tree. Returning true", m_dbname);
 			return true;
 		}
@@ -1965,14 +1977,15 @@ bool Rdb::addRecord(collnum_t collnum, char *key, char *data, int32_t dataSize)
 			// we will have non-special keys here to simplify logic in XmlDoc::getMetaList (and we can't really be sure
 			// if the key we're adding is in RdbTree/RdbBuckets at that point of time. It could potentially be dumped
 			// after the check.
-			if (m_rdbId == RDB_POSDB || m_rdbId == RDB2_POSDB2) {
-				if (Posdb::getTermId(key) != POSDB_DELETEDOC_TERMID) {
-					logTrace(g_conf.m_logTraceRdb, "END. %s: Negative key with non-zero termId found. Returning true", m_dbname);
-					return true;
-				}
-			} else {
-				/// @todo ALC cater for other rdb types here
-				gbshutdownLogicError();
+			if (!isSpecialKey) {
+				logTrace(g_conf.m_logTraceRdb, "END. %s: Negative key with non-zero termId found. Returning true", m_dbname);
+				return true;
+			}
+		} else {
+			// make sure that positive special key is not persisted (reasons as delete key above; the XmlDoc::getMetaList part)
+			if (isSpecialKey) {
+				logTrace(g_conf.m_logTraceRdb, "END. %s: Positive key with zero termId found. Returning true", m_dbname);
+				return true;
 			}
 		}
 	} else {

XmlDoc.cpp

@@ -13344,9 +13344,15 @@ char *XmlDoc::getMetaList(bool forDelete) {
 		need += spiderStatusDocMetaList->length();
 	}
 
+	/// @todo ALC verify that we actually need sizeof(key128_t)
 	// space for indexdb AND DATEDB! +2 for rdbids
-	int32_t needIndexdb = tt1.m_numSlotsUsed * (sizeof(key144_t) + 2 + sizeof(key128_t));
-	need += needIndexdb;
+	int32_t needPosdb = tt1.m_numSlotsUsed * (sizeof(posdbkey_t) + 2 + sizeof(key128_t));
+	if (!forDelete) {
+		// need 1 additional key for special key (with termid 0)
+		needPosdb += sizeof(posdbkey_t) + 1;
+	}
+
+	need += needPosdb;
 
 	// clusterdb keys. plus one for rdbId
 	int32_t needClusterdb = nd ? 13 : 0;
@@ -13589,13 +13595,46 @@ char *XmlDoc::getMetaList(bool forDelete) {
 	saved = m_p;
 
 	// store indexdb terms into m_metaList[]
-	if (m_usePosdb && !addTable144(&tt1, m_docId)) {
-		logTrace(g_conf.m_logTraceXmlDoc, "END, addTable144 failed");
-		return NULL;
+	if (m_usePosdb) {
+		if (!addTable144(&tt1, m_docId)) {
+			logTrace(g_conf.m_logTraceXmlDoc, "END, addTable144 failed");
+			return NULL;
+		}
+
+		/// @todo ALC we need to handle delete keys for other rdb types
+
+		// we need to add delete key per document when it's deleted (with term 0)
+		// we also need to add positive key per document when it's new
+		// in case there is already a delete key in the tree/bucket (this will not be persisted and will be removed in Rdb::addRecord)
+		if (g_conf.m_noInMemoryPosdbMerge) {
+			// we don't need to do this if getMetaList is called to get negative keys
+			if (!forDelete) {
+				if ((m_isInIndex && !m_wasInIndex) || (!m_isInIndex && m_wasInIndex)) {
+					char key[MAX_KEY_BYTES];
+
+					int64_t docId;
+					bool delKey = (!m_isInIndex);
+					if (!m_isInIndex) {
+						// deleted doc
+						docId = *od->getDocId();
+					} else {
+						// new doc
+						docId = *nd->getDocId();
+					}
+
+					// add posdb doc key
+					*m_p++ = RDB_POSDB;
+
+					Posdb::makeKey(&key, POSDB_DELETEDOC_TERMID, docId, 0, 0, 0, 0, 0, 0, 0, 0, 0, delKey, false);
+					memcpy(m_p, &key, sizeof(posdbkey_t));
+					m_p += sizeof(posdbkey_t);
+				}
+			}
+		}
 	}
 
 	// sanity check
-	if (m_p - saved > needIndexdb) {
+	if (m_p - saved > needPosdb) {
 		g_process.shutdownAbort(true);
 	}
 
@@ -14137,19 +14176,6 @@ skipNewAdd2:
 			}
 		}
 
-		/// @todo ALC we need to handle delete keys for other rdb types
-
-		// we need to add delete key per document when it's deleted (with term 0)
-		if (g_conf.m_noInMemoryPosdbMerge && !m_isInIndex) {
-			char key[MAX_KEY_BYTES];
-
-			// add posdb delete key
-			Posdb::makeStartKey(&key, POSDB_DELETEDOC_TERMID, *od->getDocId());
-			*nptr++ = RDB_POSDB;
-			memcpy(nptr, &key, sizeof(posdbkey_t));
-			nptr += sizeof(posdbkey_t);
-		}
-
 		// sanity. check for metalist breach
 		if (nptr > nmax) {
 			g_process.shutdownAbort(true);