Forks/privacore-open-source-search-engine
1
0
Fork 0
forked from Mirrors/privacore-open-source-search-engine
Code Pull Requests Activity

Make sure we process all robots meta tags (if there are multiple)

Browse Source
This commit is contained in:
Ai Lin Chia
2017-11-24 12:26:46 +01:00
parent cf22910ce5
commit 1472ffe999
1 changed files with 8 additions and 2 deletions

10
XmlDoc.cpp

@ -16602,12 +16602,18 @@ bool *XmlDoc::parseRobotsMetaTag() {
const char *content = nullptr;
int32_t contentLen = 0;
if (xml->getTagValue("name", "robots", "content", &content, &contentLen, true, TAG_META)) {
int32_t startNode = 0;
while (startNode < xml->getNumNodes() &&
xml->getTagValue("name", "robots", "content", &content, &contentLen, true, TAG_META, &startNode)) {
parseRobotsMetaTagContent(content, contentLen);
++startNode;
}
if (xml->getTagValue("name", g_conf.m_spiderBotName, "content", &content, &contentLen, true, TAG_META)) {
startNode = 0;
while (startNode < xml->getNumNodes() &&
xml->getTagValue("name", g_conf.m_spiderBotName, "content", &content, &contentLen, true, TAG_META, &startNode)) {
parseRobotsMetaTagContent(content, contentLen);
++startNode;
}
m_parsedRobotsMetaTag = true;