Merge branch 'diffbot' of github.com:gigablast/open-source-search-engine into diffbot

This commit is contained in:
Matt Wells
2014-01-06 15:56:18 -08:00

@ -9214,8 +9214,8 @@ Url **XmlDoc::getRedirUrl() {
// . many people use 301 on their root pages though, so treat
// it like a temporary redirect, like exclusivelyequine.com
if ( simplifiedRedir && ! m_allowSimplifiedRedirs &&
// for custom crawling clients don't like this i guess
! cr->m_isCustomCrawl ) {
// for custom BULK clients don't like this i guess
cr->m_isCustomCrawl != 2 ) {
// returns false if blocked, true otherwise
//return addSimplifiedRedirect();
m_redirError = EDOCSIMPLIFIEDREDIR;