Mirrors/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2025-07-18 08:36:07 -04:00
Code Issues Packages Projects Releases Wiki Activity

fixed bug with double-check in crawler

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4269 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
orbiter
2007-12-12 01:32:25 +00:00
parent b1e08d354c
commit 01554f4012
3 changed files with 10 additions and 4 deletions

3
source/de/anomic/index/indexRAMRI.java

@ -389,7 +389,8 @@ public final class indexRAMRI implements indexRI {
}
public int sizeContainer(String wordHash) {
return ((indexContainer) cache.get(wordHash)).size();
indexContainer c = (indexContainer) cache.get(wordHash);
return (c == null) ? 0 : c.size();
}
public synchronized indexContainer getContainer(String wordHash, Set urlselection) {

2
source/de/anomic/plasma/plasmaCrawlProfile.java

@ -354,7 +354,7 @@ public class plasmaCrawlProfile {
long l = Long.parseLong(r) * 60000L;
return (l < 0) ? Long.MAX_VALUE : l;
} catch (NumberFormatException e) {
return 0;
return Long.MAX_VALUE;
}
}
public int domFilterDepth() {

9
source/de/anomic/plasma/plasmaCrawlStacker.java

@ -457,12 +457,17 @@ public final class plasmaCrawlStacker extends Thread {
String dbocc = sb.crawlQueues.urlExists(entry.url().hash());
indexURLEntry oldEntry = this.sb.wordIndex.loadedURL.load(entry.url().hash(), null, 0);
boolean recrawl = (oldEntry != null) && ((System.currentTimeMillis() - oldEntry.loaddate().getTime()) > profile.recrawlIfOlder());
// apply recrawl rule
if ((dbocc != null) && (!(recrawl))) {
// do double-check
if ((dbocc != null) && (!recrawl)) {
reason = plasmaCrawlEURL.DOUBLE_REGISTERED + dbocc + ")";
this.log.logFine("URL '" + entry.url().toString() + "' is double registered in '" + dbocc + "'. " + "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms");
return reason;
}
if ((oldEntry != null) && (!recrawl)) {
reason = plasmaCrawlEURL.DOUBLE_REGISTERED + "LURL)";
this.log.logFine("URL '" + entry.url().toString() + "' is double registered in 'LURL'. " + "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms");
return reason;
}
// show potential re-crawl
if (recrawl) {