Mirrors/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2025-07-20 08:54:37 -04:00
Code Issues Packages Projects Releases Wiki Activity

minor fixes

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@659 6c8d7289-2bf4-0310-a012-ef5d649a1542
This commit is contained in:
allo
2005-09-05 10:34:26 +00:00
parent 4557966cdb
commit ebc39a7b9a
1 changed files with 2 additions and 4 deletions

6
source/de/anomic/data/robotsParser.java

@ -104,9 +104,7 @@ public class robotsParser{
line=lines[i];
if(line.startsWith("Disallow:")){
line=line.substring(9);
if(line.startsWith(" ")){
line=line.substring(1);
}
line=line.trim()
deny.add(line);
}
}
@ -120,7 +118,7 @@ public class robotsParser{
Iterator it=deny.iterator();
allowed=true;
while(it.hasNext()){
if(url.indexOf((String)it.next()) >= 0){
if(url.startsWith((String)it.next()){
allowed=false;
break;
}