Forks/privacore-open-source-search-engine
1
0
Fork 0
forked from Mirrors/privacore-open-source-search-engine
Code Pull Requests Activity

update crawlstatus msgs.

Browse Source
This commit is contained in:
Matt Wells
2013-10-29 13:16:01 -07:00
parent 1a6c221d36
commit 1b79c5696e
2 changed files with 20 additions and 7 deletions

18
PageCrawlBot.cpp

@ -2726,7 +2726,7 @@ bool printCrawlBotPage2 ( TcpSocket *socket ,
//if ( cx->m_collectionNameAlias.length() > 0 )
// alias=cx->m_collectionNameAlias.getBufStart();
//long paused = 1;
char *ss = "In progress.";
char *ss = "Crawl in progress.";
if ( cx->m_spiderStatusMsg )
ss = cx->m_spiderStatusMsg;
// 0 means not to RE-crawl
@ -2735,7 +2735,7 @@ bool printCrawlBotPage2 ( TcpSocket *socket ,
if ( cx->m_collectiveRespiderFrequency > 0.0 &&
getTimeGlobal() < cx->m_spiderRoundStartTime ) {
long now = getTimeGlobal();
sprintf(tmp,"Spidering next round in %li "
sprintf(tmp,"Next crawl round to start in %li "
"seconds.",
cx->m_spiderRoundStartTime - now
);
@ -2745,11 +2745,21 @@ bool printCrawlBotPage2 ( TcpSocket *socket ,
// were left and we are not recrawling!
if ( cx->m_collectiveRespiderFrequency == 0.0 &&
! cx->m_globalCrawlInfo.m_hasUrlsReadyToSpider ) {
ss = "Crawl has exhausted all urls and "
"repeatCrawl is set to 0.0.";
ss = "Crawl has completed and no "
"repeatCrawl is scheduled.";
}
if ( ! cx->m_spideringEnabled )
ss = "Crawl paused.";
// if spiderdb is empty for this coll, then no url
// has been added to spiderdb yet.. either seed or spot
Rdb *rdb = g_spiderdb.getRdb();
RdbBase *base = rdb->getBase ( cx->m_collnum );
long recCount = 0;
if ( base ) recCount = base->getNumTotalRecs();
if ( recCount == 0 )
ss = "No urls are available to crawl.";
CrawlInfo *ci = &cx->m_localCrawlInfo;
long sentAlert = (long)ci->m_sentCrawlDoneAlert;
if ( sentAlert ) sentAlert = 1;

9
Spider.cpp

@ -4176,7 +4176,8 @@ void SpiderLoop::spiderDoledUrls ( ) {
if ( cr->m_maxCrawlRounds > 0 &&
cr->m_spiderRoundNum >= cr->m_maxCrawlRounds ) {
cr->m_spiderStatus = SP_MAXROUNDS;
cr->m_spiderStatusMsg = "Hit maxCrawlRounds limit.";
cr->m_spiderStatusMsg = "Crawl has reached "
"maxCrawlRounds limit.";
// it'll send a SP_ROUNDDONE email first
// so no need to repeat it, but we do want to
// update the status msg
@ -4188,7 +4189,8 @@ void SpiderLoop::spiderDoledUrls ( ) {
if ( cr->m_globalCrawlInfo.m_pageDownloadSuccesses >=
cr->m_maxToCrawl ) {
cr->m_spiderStatus = SP_MAXTOCRAWL;
cr->m_spiderStatusMsg = "Hit maxToCrawl limit.";
cr->m_spiderStatusMsg = "Crawl has reached maxToCrawl "
"limit.";
sendNotificationForCollRec ( cr );
continue;
}
@ -4197,7 +4199,8 @@ void SpiderLoop::spiderDoledUrls ( ) {
if ( cr->m_globalCrawlInfo.m_pageProcessSuccesses >=
cr->m_maxToProcess ) {
cr->m_spiderStatus = SP_MAXTOPROCESS;
cr->m_spiderStatusMsg = "Hit maxToProcess limit.";
cr->m_spiderStatusMsg = "Crawl has reached "
"maxToProcess limit.";
sendNotificationForCollRec ( cr );
continue;
}