mirror of
https://github.com/privacore/open-source-search-engine.git
synced 2025-07-15 02:36:08 -04:00
Remove unused cacheStart & cacheLen from Robots::isAllowed
This commit is contained in:
16
Robots.cpp
16
Robots.cpp
@ -6,12 +6,7 @@
|
||||
|
||||
// taken from Robotdb.cpp
|
||||
bool Robots::isAllowed ( Url *url, char *userAgent, char *file, int32_t fileLen, bool *userAgentFound,
|
||||
bool substringMatch, int32_t *crawlDelay, char **cacheStart, int32_t *cacheLen,
|
||||
bool *hadAllowOrDisallow ) {
|
||||
// assume nothing to cache yet
|
||||
*cacheLen = 0;
|
||||
*cacheStart = file;
|
||||
|
||||
bool substringMatch, int32_t *crawlDelay, bool *hadAllowOrDisallow ) {
|
||||
// assume user agent is not in the file
|
||||
*userAgentFound = false;
|
||||
*hadAllowOrDisallow = false;
|
||||
@ -91,19 +86,12 @@ bool Robots::isAllowed ( Url *url, char *userAgent, char *file, int32_t fileLen,
|
||||
goto loop;
|
||||
}
|
||||
|
||||
// we got it, if first instance start our cache here
|
||||
if ( !*userAgentFound ) {
|
||||
*cacheStart = f;
|
||||
}
|
||||
|
||||
*userAgentFound = true;
|
||||
flag = 0;
|
||||
|
||||
urlLoop:
|
||||
// if p is NULL now there is no more lines
|
||||
if ( ! p ) {
|
||||
// set our cache stop to the end of the file
|
||||
*cacheLen = (file + fileLen) - *cacheStart;
|
||||
return allowed;
|
||||
}
|
||||
|
||||
@ -113,8 +101,6 @@ bool Robots::isAllowed ( Url *url, char *userAgent, char *file, int32_t fileLen,
|
||||
// if it's another user-agent line ... ignore it unless we already
|
||||
// have seen another line (not user-agent), in which case we got another set of group
|
||||
if ( flag && flen==10 && strncasecmp( f, "user-agent", 10 ) == 0 ) {
|
||||
// set our cache stop here
|
||||
*cacheLen = f - *cacheStart;
|
||||
goto gotAgent;
|
||||
}
|
||||
|
||||
|
2
Robots.h
2
Robots.h
@ -8,7 +8,7 @@ class Url;
|
||||
class Robots {
|
||||
public:
|
||||
static bool isAllowed ( Url *url, char *userAgent, char *file, int32_t fileLen, bool *userAgentFound, bool substringMatch,
|
||||
int32_t *crawlDelay, char **cacheStart, int32_t *cacheLen, bool *hadAllowOrDisallow );
|
||||
int32_t *crawlDelay, bool *hadAllowOrDisallow );
|
||||
};
|
||||
|
||||
#endif // GB_ROBOTS_H
|
||||
|
@ -8506,8 +8506,6 @@ bool *XmlDoc::getIsAllowed ( ) {
|
||||
// this is set to true if our userAgent was found explicitly
|
||||
bool uaFound;
|
||||
bool allowed;
|
||||
char *cacheStart;
|
||||
int32_t cacheLen;
|
||||
bool hadAllowOrDisallow;
|
||||
// now use left-anchored substring match so we can match Botname/1.0
|
||||
allowed = Robots::isAllowed ( cu ,
|
||||
@ -8517,8 +8515,6 @@ bool *XmlDoc::getIsAllowed ( ) {
|
||||
&uaFound ,
|
||||
true , // substrmatch?
|
||||
&m_crawlDelay ,
|
||||
&cacheStart ,
|
||||
&cacheLen ,
|
||||
&hadAllowOrDisallow );
|
||||
|
||||
|
||||
@ -8539,8 +8535,6 @@ bool *XmlDoc::getIsAllowed ( ) {
|
||||
&uaFound ,
|
||||
false , // substrmatch?
|
||||
&m_crawlDelay ,
|
||||
&cacheStart ,
|
||||
&cacheLen ,
|
||||
&hadAllowOrDisallow );
|
||||
|
||||
if( g_conf.m_logTraceXmlDoc ) log("%s:%s:%d: Robots::isAllowed returned %s for '*' bot", __FILE__,__func__,__LINE__, (allowed?"true":"false"));
|
||||
|
Reference in New Issue
Block a user