Remove unused cacheStart & cacheLen from Robots::isAllowed

This commit is contained in:
Ai Lin Chia
2016-03-18 10:29:46 +01:00
parent ed840968ec
commit 02647dc862
3 changed files with 2 additions and 22 deletions

@ -6,12 +6,7 @@
// taken from Robotdb.cpp
bool Robots::isAllowed ( Url *url, char *userAgent, char *file, int32_t fileLen, bool *userAgentFound,
bool substringMatch, int32_t *crawlDelay, char **cacheStart, int32_t *cacheLen,
bool *hadAllowOrDisallow ) {
// assume nothing to cache yet
*cacheLen = 0;
*cacheStart = file;
bool substringMatch, int32_t *crawlDelay, bool *hadAllowOrDisallow ) {
// assume user agent is not in the file
*userAgentFound = false;
*hadAllowOrDisallow = false;
@ -91,19 +86,12 @@ bool Robots::isAllowed ( Url *url, char *userAgent, char *file, int32_t fileLen,
goto loop;
}
// we got it, if first instance start our cache here
if ( !*userAgentFound ) {
*cacheStart = f;
}
*userAgentFound = true;
flag = 0;
urlLoop:
// if p is NULL now there is no more lines
if ( ! p ) {
// set our cache stop to the end of the file
*cacheLen = (file + fileLen) - *cacheStart;
return allowed;
}
@ -113,8 +101,6 @@ bool Robots::isAllowed ( Url *url, char *userAgent, char *file, int32_t fileLen,
// if it's another user-agent line ... ignore it unless we already
// have seen another line (not user-agent), in which case we got another set of group
if ( flag && flen==10 && strncasecmp( f, "user-agent", 10 ) == 0 ) {
// set our cache stop here
*cacheLen = f - *cacheStart;
goto gotAgent;
}

@ -8,7 +8,7 @@ class Url;
class Robots {
public:
static bool isAllowed ( Url *url, char *userAgent, char *file, int32_t fileLen, bool *userAgentFound, bool substringMatch,
int32_t *crawlDelay, char **cacheStart, int32_t *cacheLen, bool *hadAllowOrDisallow );
int32_t *crawlDelay, bool *hadAllowOrDisallow );
};
#endif // GB_ROBOTS_H

@ -8506,8 +8506,6 @@ bool *XmlDoc::getIsAllowed ( ) {
// this is set to true if our userAgent was found explicitly
bool uaFound;
bool allowed;
char *cacheStart;
int32_t cacheLen;
bool hadAllowOrDisallow;
// now use left-anchored substring match so we can match Botname/1.0
allowed = Robots::isAllowed ( cu ,
@ -8517,8 +8515,6 @@ bool *XmlDoc::getIsAllowed ( ) {
&uaFound ,
true , // substrmatch?
&m_crawlDelay ,
&cacheStart ,
&cacheLen ,
&hadAllowOrDisallow );
@ -8539,8 +8535,6 @@ bool *XmlDoc::getIsAllowed ( ) {
&uaFound ,
false , // substrmatch?
&m_crawlDelay ,
&cacheStart ,
&cacheLen ,
&hadAllowOrDisallow );
if( g_conf.m_logTraceXmlDoc ) log("%s:%s:%d: Robots::isAllowed returned %s for '*' bot", __FILE__,__func__,__LINE__, (allowed?"true":"false"));