mirror of
https://github.com/privacore/open-source-search-engine.git
synced 2025-07-14 02:36:06 -04:00
added version to SpiderRequest and SpiderReply.
Added constructor to SpiderRequest and SpiderReply that initializes all members
This commit is contained in:
@ -78,6 +78,8 @@ int32_t SpiderRequest::print ( SafeBuf *sbarg ) {
|
||||
|
||||
// indicate it's a request not a reply
|
||||
sb->safePrintf("REQ ");
|
||||
sb->safePrintf("ver=%d ", (int)m_version);
|
||||
|
||||
sb->safePrintf("uh48=%" PRIx64" ",getUrlHash48());
|
||||
// if negtaive bail early now
|
||||
if ( (m_key.n0 & 0x01) == 0x00 ) {
|
||||
@ -105,11 +107,8 @@ int32_t SpiderRequest::print ( SafeBuf *sbarg ) {
|
||||
timeStruct = gmtime_r(&ts,&tm_buf);
|
||||
strftime ( time , 256 , "%b %e %T %Y UTC", timeStruct );
|
||||
sb->safePrintf("addedTime=%s(%" PRIu32") ",time,(uint32_t)m_addedTime );
|
||||
|
||||
sb->safePrintf("pageNumInlinks=%i ",(int)m_pageNumInlinks);
|
||||
|
||||
sb->safePrintf("hopCount=%" PRId32" ",(int32_t)m_hopCount );
|
||||
|
||||
sb->safePrintf("ufn=%" PRId32" ", (int32_t)m_ufn);
|
||||
// why was this unsigned?
|
||||
sb->safePrintf("priority=%" PRId32" ", (int32_t)m_priority);
|
||||
@ -158,12 +157,12 @@ int32_t SpiderReply::print ( SafeBuf *sbarg ) {
|
||||
|
||||
// indicate it's a reply
|
||||
sb->safePrintf("REP ");
|
||||
sb->safePrintf("ver=%d ", (int)m_version);
|
||||
|
||||
sb->safePrintf("uh48=%" PRIx64" ",getUrlHash48());
|
||||
sb->safePrintf("parentDocId=%" PRIu64" ",getParentDocId());
|
||||
|
||||
|
||||
// if negtaive bail early now
|
||||
// if negative bail early now
|
||||
if ( (m_key.n0 & 0x01) == 0x00 ) {
|
||||
sb->safePrintf("[DELETE]");
|
||||
if ( ! sbarg ) printf("%s",sb->getBufStart() );
|
||||
|
23
Spider.h
23
Spider.h
@ -14,6 +14,11 @@ class RdbList;
|
||||
class HashTableX;
|
||||
class SpiderColl;
|
||||
|
||||
|
||||
#define SPIDERREQ_CURRENT_VERSION 1
|
||||
#define SPIDERREP_CURRENT_VERSION 1
|
||||
|
||||
|
||||
// lower from 1300 to 300
|
||||
#define MAXUDPSLOTS 300
|
||||
|
||||
@ -467,7 +472,7 @@ public:
|
||||
uint8_t m_sameErrCount;
|
||||
|
||||
|
||||
uint8_t m_reservedb3;
|
||||
uint8_t m_version;
|
||||
uint8_t m_reservedb4;
|
||||
|
||||
// info on the page we were harvest from
|
||||
@ -636,6 +641,7 @@ public:
|
||||
m_ufn = -1;
|
||||
// this too
|
||||
m_priority = -1;
|
||||
m_version = SPIDERREQ_CURRENT_VERSION;
|
||||
}
|
||||
|
||||
static int32_t getNeededSize ( int32_t urlLen ) {
|
||||
@ -697,6 +703,10 @@ public:
|
||||
bool setFromInject(const char *url);
|
||||
|
||||
bool isCorrupt() const;
|
||||
|
||||
SpiderRequest() {
|
||||
reset();
|
||||
}
|
||||
} __attribute__((packed, aligned(4)));
|
||||
|
||||
// . XmlDoc adds this record to spiderdb after attempting to spider a url
|
||||
@ -744,7 +754,7 @@ public:
|
||||
int32_t m_siteNumInlinks;
|
||||
|
||||
uint8_t m_sameErrCount;
|
||||
uint8_t m_reserved_u8a;
|
||||
uint8_t m_version;
|
||||
uint8_t m_reserved_u8b;
|
||||
uint8_t m_reserved_u8c;
|
||||
|
||||
@ -838,7 +848,10 @@ public:
|
||||
int32_t getRecSize () const { return m_dataSize + 4 + sizeof(key128_t); }
|
||||
|
||||
// clear all
|
||||
void reset() { memset ( this , 0 , sizeof(SpiderReply) ); }
|
||||
void reset() {
|
||||
memset(this, 0, sizeof(SpiderReply));
|
||||
m_version = SPIDERREP_CURRENT_VERSION;
|
||||
}
|
||||
|
||||
void setKey ( int32_t firstIp, int64_t parentDocId, int64_t uh48, bool isDel ) ;
|
||||
|
||||
@ -851,6 +864,10 @@ public:
|
||||
int64_t getParentDocId() const {
|
||||
return Spiderdb::getParentDocId(&m_key);
|
||||
}
|
||||
|
||||
SpiderReply() {
|
||||
reset();
|
||||
}
|
||||
} __attribute__((packed, aligned(4)));
|
||||
|
||||
// was 1000 but breached, now equals SR_READ_SIZE/sizeof(SpiderReply)
|
||||
|
Reference in New Issue
Block a user