mirror of
https://github.com/privacore/open-source-search-engine.git
synced 2025-07-16 02:46:08 -04:00
Add tools/dump_unwanted
This commit is contained in:
170
tools/dump_unwanted.cpp
Normal file
170
tools/dump_unwanted.cpp
Normal file
@ -0,0 +1,170 @@
|
||||
#include "XmlDoc.h"
|
||||
#include "Collectiondb.h"
|
||||
#include "SpiderCache.h"
|
||||
#include "Titledb.h"
|
||||
#include "Doledb.h"
|
||||
#include "CountryCode.h"
|
||||
#include "Log.h"
|
||||
#include "Conf.h"
|
||||
#include "Mem.h"
|
||||
#include "UrlBlockCheck.h"
|
||||
#include "UrlMatchList.h"
|
||||
#include "WantedChecker.h"
|
||||
#include <libgen.h>
|
||||
#include <algorithm>
|
||||
|
||||
static void print_usage(const char *argv0) {
|
||||
fprintf(stdout, "Usage: %s [-h] PATH\n", argv0);
|
||||
fprintf(stdout, "Dump unwanted titlerec\n");
|
||||
fprintf(stdout, "\n");
|
||||
fprintf(stdout, " -h, --help display this help and exit\n");
|
||||
}
|
||||
|
||||
static void cleanup() {
|
||||
g_log.m_disabled = true;
|
||||
|
||||
g_linkdb.reset();
|
||||
g_clusterdb.reset();
|
||||
g_spiderCache.reset();
|
||||
g_doledb.reset();
|
||||
g_spiderdb.reset();
|
||||
g_tagdb.reset();
|
||||
g_titledb.reset();
|
||||
g_posdb.reset();
|
||||
|
||||
g_collectiondb.reset();
|
||||
|
||||
g_loop.reset();
|
||||
|
||||
WantedChecker::finalize();
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc < 2) {
|
||||
print_usage(argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (strcmp(argv[1], "-h") == 0 || strcmp(argv[1], "--help") == 0 ) {
|
||||
print_usage(argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
g_log.m_disabled = true;
|
||||
|
||||
// initialize library
|
||||
g_mem.init();
|
||||
hashinit();
|
||||
|
||||
// current dir
|
||||
char path[PATH_MAX];
|
||||
realpath(argv[1], path);
|
||||
size_t pathLen = strlen(path);
|
||||
if (path[pathLen] != '/') {
|
||||
strcat(path, "/");
|
||||
}
|
||||
|
||||
g_hostdb.init(-1, false, false, true, path);
|
||||
g_conf.init(path);
|
||||
|
||||
ucInit();
|
||||
|
||||
// initialize rdbs
|
||||
g_loop.init();
|
||||
|
||||
g_collectiondb.loadAllCollRecs();
|
||||
|
||||
g_posdb.init();
|
||||
g_titledb.init();
|
||||
g_tagdb.init();
|
||||
g_spiderdb.init();
|
||||
g_doledb.init();
|
||||
g_spiderCache.init();
|
||||
g_clusterdb.init();
|
||||
g_linkdb.init();
|
||||
|
||||
g_collectiondb.addRdbBaseToAllRdbsForEachCollRec();
|
||||
|
||||
g_log.m_disabled = false;
|
||||
g_log.m_logPrefix = false;
|
||||
|
||||
CollectionRec *cr = g_collectiondb.getRec("main");
|
||||
if (!cr) {
|
||||
logf(LOG_TRACE, "No main collection found");
|
||||
return 1;
|
||||
}
|
||||
|
||||
// initialize shlib & blacklist
|
||||
if (!WantedChecker::initialize()) {
|
||||
fprintf(stderr, "Unable to initialize WantedChecker");
|
||||
return 1;
|
||||
}
|
||||
|
||||
g_urlBlackList.init();
|
||||
g_urlWhiteList.init();
|
||||
|
||||
Msg5 msg5;
|
||||
RdbList list;
|
||||
|
||||
key96_t startKey;
|
||||
startKey.setMin();
|
||||
|
||||
key96_t endKey;
|
||||
endKey.setMax();
|
||||
|
||||
while (msg5.getList(RDB_TITLEDB, cr->m_collnum, &list, &startKey, &endKey, 10485760, true, 0, -1, NULL, NULL, 0, true, -1, false)) {
|
||||
if (list.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
|
||||
for (list.resetListPtr(); !list.isExhausted(); list.skipCurrentRecord()) {
|
||||
key96_t key = list.getCurrentKey();
|
||||
int64_t docId = Titledb::getDocIdFromKey(&key);
|
||||
|
||||
XmlDoc xmlDoc;
|
||||
if (!xmlDoc.set2(list.getCurrentRec(), list.getCurrentRecSize(), "main", NULL, 0)) {
|
||||
logf(LOG_TRACE, "Unable to set XmlDoc for docId=%" PRIu64, docId);
|
||||
continue;
|
||||
}
|
||||
|
||||
// extract the url
|
||||
Url *url = xmlDoc.getFirstUrl();
|
||||
const char *reason = NULL;
|
||||
|
||||
if (isUrlUnwanted(*url, &reason)) {
|
||||
fprintf(stdout, "%" PRId64"|%s|%s\n", docId, reason, url->getUrl());
|
||||
continue;
|
||||
}
|
||||
|
||||
Url **redirUrlPtr = xmlDoc.getRedirUrl();
|
||||
if (redirUrlPtr && *redirUrlPtr) {
|
||||
Url *redirUrl = *redirUrlPtr;
|
||||
if (isUrlUnwanted(*redirUrl, &reason)) {
|
||||
fprintf(stdout, "%" PRId64"|redir %s|%s|%s\n", docId, reason, url->getUrl(), redirUrl->getUrl());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// check content
|
||||
int32_t contentLen = xmlDoc.size_utf8Content > 0 ? (xmlDoc.size_utf8Content - 1) : 0;
|
||||
if (contentLen > 0) {
|
||||
if (!WantedChecker::check_single_content(url->getUrl(), xmlDoc.ptr_utf8Content, contentLen).wanted) {
|
||||
fprintf(stdout, "%" PRId64"|blocked content|%s\n", docId, url->getUrl());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
startKey = *(key96_t *)list.getLastKey();
|
||||
startKey++;
|
||||
|
||||
// watch out for wrap around
|
||||
if ( startKey < *(key96_t *)list.getLastKey() ) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
cleanup();
|
||||
|
||||
return 0;
|
||||
}
|
Reference in New Issue
Block a user