added version to SpiderRequest and SpiderReply.

Added constructor to SpiderRequest and SpiderReply that initializes all members
This commit is contained in:
Brian Rasmusson
2017-07-04 12:51:18 +02:00
parent b378faef10
commit 27f614d917
2 changed files with 24 additions and 8 deletions

@ -78,6 +78,8 @@ int32_t SpiderRequest::print ( SafeBuf *sbarg ) {
// indicate it's a request not a reply
sb->safePrintf("REQ ");
sb->safePrintf("ver=%d ", (int)m_version);
sb->safePrintf("uh48=%" PRIx64" ",getUrlHash48());
// if negtaive bail early now
if ( (m_key.n0 & 0x01) == 0x00 ) {
@ -105,11 +107,8 @@ int32_t SpiderRequest::print ( SafeBuf *sbarg ) {
timeStruct = gmtime_r(&ts,&tm_buf);
strftime ( time , 256 , "%b %e %T %Y UTC", timeStruct );
sb->safePrintf("addedTime=%s(%" PRIu32") ",time,(uint32_t)m_addedTime );
sb->safePrintf("pageNumInlinks=%i ",(int)m_pageNumInlinks);
sb->safePrintf("hopCount=%" PRId32" ",(int32_t)m_hopCount );
sb->safePrintf("ufn=%" PRId32" ", (int32_t)m_ufn);
// why was this unsigned?
sb->safePrintf("priority=%" PRId32" ", (int32_t)m_priority);
@ -158,12 +157,12 @@ int32_t SpiderReply::print ( SafeBuf *sbarg ) {
// indicate it's a reply
sb->safePrintf("REP ");
sb->safePrintf("ver=%d ", (int)m_version);
sb->safePrintf("uh48=%" PRIx64" ",getUrlHash48());
sb->safePrintf("parentDocId=%" PRIu64" ",getParentDocId());
// if negtaive bail early now
// if negative bail early now
if ( (m_key.n0 & 0x01) == 0x00 ) {
sb->safePrintf("[DELETE]");
if ( ! sbarg ) printf("%s",sb->getBufStart() );

@ -14,6 +14,11 @@ class RdbList;
class HashTableX;
class SpiderColl;
#define SPIDERREQ_CURRENT_VERSION 1
#define SPIDERREP_CURRENT_VERSION 1
// lower from 1300 to 300
#define MAXUDPSLOTS 300
@ -467,7 +472,7 @@ public:
uint8_t m_sameErrCount;
uint8_t m_reservedb3;
uint8_t m_version;
uint8_t m_reservedb4;
// info on the page we were harvest from
@ -636,6 +641,7 @@ public:
m_ufn = -1;
// this too
m_priority = -1;
m_version = SPIDERREQ_CURRENT_VERSION;
}
static int32_t getNeededSize ( int32_t urlLen ) {
@ -697,6 +703,10 @@ public:
bool setFromInject(const char *url);
bool isCorrupt() const;
SpiderRequest() {
reset();
}
} __attribute__((packed, aligned(4)));
// . XmlDoc adds this record to spiderdb after attempting to spider a url
@ -744,7 +754,7 @@ public:
int32_t m_siteNumInlinks;
uint8_t m_sameErrCount;
uint8_t m_reserved_u8a;
uint8_t m_version;
uint8_t m_reserved_u8b;
uint8_t m_reserved_u8c;
@ -838,7 +848,10 @@ public:
int32_t getRecSize () const { return m_dataSize + 4 + sizeof(key128_t); }
// clear all
void reset() { memset ( this , 0 , sizeof(SpiderReply) ); }
void reset() {
memset(this, 0, sizeof(SpiderReply));
m_version = SPIDERREP_CURRENT_VERSION;
}
void setKey ( int32_t firstIp, int64_t parentDocId, int64_t uh48, bool isDel ) ;
@ -851,6 +864,10 @@ public:
int64_t getParentDocId() const {
return Spiderdb::getParentDocId(&m_key);
}
SpiderReply() {
reset();
}
} __attribute__((packed, aligned(4)));
// was 1000 but breached, now equals SR_READ_SIZE/sizeof(SpiderReply)