mirror of
https://github.com/privacore/open-source-search-engine.git
synced 2025-07-15 02:36:08 -04:00
if a seed has a bad ip lookup one round make sure
we try it the next round!
This commit is contained in:
@ -11447,6 +11447,10 @@ int32_t getUrlFilterNum2 ( SpiderRequest *sreq ,
|
||||
if ( errCode != EDNSTIMEDOUT &&
|
||||
errCode != ETCPTIMEDOUT &&
|
||||
errCode != EDNSDEAD &&
|
||||
// add this here too now because we had some
|
||||
// seeds that failed one time and the crawl
|
||||
// never repeated after that!
|
||||
errCode != EBADIP &&
|
||||
// assume diffbot is temporarily experiencing errs
|
||||
errCode != EDIFFBOTINTERNALERROR &&
|
||||
// if diffbot received empty content when d'lding
|
||||
|
Reference in New Issue
Block a user