if indexing diffbot url and it had

error, do NOT add a spider reply.
This commit is contained in:
Matt Wells
2014-01-25 07:01:26 -08:00
parent 308106673c
commit 29a574d85a

@ -2114,15 +2114,18 @@ bool XmlDoc::indexDoc ( ) {
// add a new request as well as a reply for this one
if ( m_indexCodeValid && m_indexCode == EFAKEFIRSTIP ) success = false;
// ignore failed child docs like diffbot pages
if ( m_isChildDoc ) {
log("build: child doc had internal error. not adding "
"spider reply for %s",m_firstUrl.m_url);
success = true;
}
if ( success ) return true;
// . ignore failed child docs like diffbot pages
// . they are getting EMALFORMEDSECTIONS
if ( m_isChildDoc ) {
log("build: done indexing child doc. error=%s. not adding "
"spider reply for %s",
mstrerror(g_errno),
m_firstUrl.m_url);
return true;
}
///
// otherwise, an internal error. we must add a SpiderReply
// to spiderdb to release the lock.