Mirrors/privacore-open-source-search-engine
1
0
Fork 1
mirror of https://github.com/privacore/open-source-search-engine.git synced 2025-01-22 02:18:42 -05:00
Code Issues Packages Projects Releases Wiki Activity
master
privacore-open-source-searc.../Images.cpp
Ivan Skytte Jørgensen 8b335971fa Removed non-const pointer-returning methods from Url class
2018-08-24 13:37:43 +02:00

1280 lines
36 KiB
C++
Raw Permalink Blame History

#include "Images.h"
#include "Conf.h"
#include "Query.h"
#include "Xml.h"
#include "tokenizer.h"
#include "Sections.h"
#include "XmlDoc.h"
#include "Collectiondb.h"
#include "JobScheduler.h"
#include "Hostdb.h"
#include "Process.h"
#include "Posdb.h"
#include "File.h"
#include "Errno.h"
#include <pthread.h>
#include <fcntl.h>
#include <arpa/inet.h>
#include "gbmemcpy.h"
#include <unistd.h>
#include "gbmemcpy.h"
// TODO: image is bad if repeated on same page, check for that
//static void gotTermFreqWrapper ( void *state ) ;
static void gotTermListWrapper ( void *state ) ;
static void thumbStartWrapper_r ( void *state );
static void getImageInfo(const char *buf, int32_t size, int32_t *dx, int32_t *dy, int32_t *it);
Images::Images ( ) {
reset();
}
void Images::reset() {
m_imgData = NULL;
m_imgDataSize = 0;
m_setCalled = false;
m_thumbnailValid = false;
m_imgReply = NULL;
m_imgReplyLen = 0;
m_imgReplyMaxLen = 0;
m_numImages = 0;
m_imageBufValid = false;
m_phase = 0;
// Coverity
m_i = 0;
m_j = 0;
m_xd = NULL;
m_state = NULL;
m_callback = NULL;
m_xysize = 0;
m_errno = 0;
m_hadError = 0;
m_stopDownloading = false;
memset(m_statusBuf, 0, sizeof(m_statusBuf));
m_collnum = 0;
m_docId = 0;
m_latestIp = 0;
memset(m_imageNodes, 0, sizeof(m_imageNodes));
memset(m_termIds, 0, sizeof(m_termIds));
memset(m_errors, 0, sizeof(m_errors));
m_pageUrl = NULL;
m_xml = NULL;
memset(&m_msg13Request, 0, sizeof(m_msg13Request));
m_httpStatus = 0;
m_imgType = 0;
m_dx = 0;
m_dy = 0;
m_thumbnailSize = 0;
m_tdx = 0;
m_tdy = 0;
}
void Images::setCandidates(Url *pageUrl, const TokenizerResult *tr, Xml *xml, Sections *sections) {
// not valid for now
m_thumbnailValid = false;
// reset our array of image node candidates
m_numImages = 0;
// flag it
m_setCalled = true;
// strange...
if ( m_imgReply ) { g_process.shutdownAbort(true); }
// save this
m_xml = xml;
m_pageUrl = pageUrl;
//
// first add any open graph candidate.
// basically they page telling us the best image straight up.
//
int32_t node2 = -1;
int32_t startNode = 0;
// . field can be stuff like "summary","description","keywords",...
// . if "convertHtmlEntites" is true we change < to &lt; and > to &gt;
// . <meta property="og:image" content="http://example.com/rock2.jpg"/>
// . <meta property="og:image" content="http://example.com/rock3.jpg"/>
ogimgloop:
char ubuf[2000];
int32_t ulen = xml->getMetaContent( ubuf, 1999, "og:image", 8, "property", startNode, &node2 );
// update this in case goto ogimgloop is called
startNode = node2 + 1;
// see section below for explanation of what we are storing here...
if ( node2 >= 0 ) {
// save it
m_imageNodes[m_numImages] = node2;
Query q;
if ( ulen > MAX_URL_LEN ) goto ogimgloop;
// set it to the full url
Url iu;
// use "pageUrl" as the baseUrl
iu.set( pageUrl, ubuf, ulen );
// skip if invalid domain or TLD
if ( iu.getDomainLen() <= 0 ) goto ogimgloop;
// for looking it up on disk to see if unique or not
char buf[2000];
// if we don't put in quotes it expands '|' into
// the "PiiPe" operator in Query.cpp
snprintf ( buf , 1999, "gbimage:\"%s\"",iu.getUrl());
// TODO: make sure this is a no-split termid storage thingy
// in Msg14.cpp
if ( !q.set(buf, langUnknown, 1.0, 1.0, NULL, false, true, ABS_MAX_QUERY_TERMS) ) return;
// sanity test
if(q.getNumTerms()!=1) { g_process.shutdownAbort(true); }
// store the termid
m_termIds[m_numImages] = q.getTermId(0);
// advance the counter
m_numImages++;
// try to get more graph images if we have some room
if ( m_numImages + 2 < MAX_IMAGES ) goto ogimgloop;
}
//m_pageSite = pageSite;
// scan the words
int32_t nw = tr->size();
//int32_t *scores = scoresArg->m_scores;
Section **sp = NULL;
if ( sections ) sp = sections->m_sectionPtrs;
// not if we don't have any identified sections
if ( sections && sections->m_numSections <= 0 ) sp = NULL;
// the positive scored window
int32_t firstPosScore = -1;
int32_t lastPosScore = -1;
int32_t badFlags = SEC_SCRIPT|SEC_STYLE|SEC_SELECT;
// find positive scoring window
for ( int32_t i = 0 ; i < nw ; i++ ) {
// skip if in bad section
if ( sp && (sp[i]->m_flags & badFlags) ) continue;
if ( (*tr)[i].is_alfanum ) continue;
// set first positive scoring guy
if ( firstPosScore == -1 ) firstPosScore = i;
// keep track of last guy
lastPosScore = i;
}
// sanity check
if ( getNumXmlNodes() > 512 ) { g_process.shutdownAbort(true); }
// . pedal firstPosScore back until we hit a section boundary
// . i.e. stop once we hit a front/back tag pair, like <div> and </div>
char tc[512];
memset ( tc , 0 , 512 );
int32_t a = firstPosScore;
for ( ; a >= 0 ; a-- ) {
const auto &token = (*tr)[a];
// get the tid
nodeid_t tid = token.token_hash;
// remove back bit, if any
tid &= BACKBITCOMP;
// skip if not a tag, or a generic xml tag
if ( tid <= 1 ) continue;
// mark it
if ( token.token_hash&BACKBIT ) tc[tid] |= 0x02;
else tc[tid] |= 0x01;
// continue if not a full front/back pair
if ( tc[tid] != 0x03 ) continue;
// continue if not a "section" type tag (see Scores.cpp)
if ( tid != TAG_DIV &&
tid != TAG_TEXTAREA &&
tid != TAG_TR &&
tid != TAG_TD &&
tid != TAG_TABLE )
continue;
// ok we should stop now
break;
}
// min is 0
if ( a < 0 ) a = 0;
// now look for the image urls within this window
for ( int32_t i = a ; i < lastPosScore ; i++ ) {
const auto &token = (*tr)[i];
// skip if not <img> tag
if (token.nodeid != TAG_IMG ) continue;
// get the node num into Xml.cpp::m_nodes[] array
int32_t nn = token.xml_node_index;
// check width to rule out small decorating imgs
int32_t width = xml->getLong(nn,nn+1,"width", -1 );
if ( width != -1 && width < 50 ) continue;
// same with height
int32_t height = xml->getLong(nn,nn+1, "height", -1 );
if ( height != -1 && height < 50 ) continue;
// get the url of the image
int32_t srcLen;
const char *src = xml->getString(nn,"src",&srcLen);
// skip if none
if ( srcLen <= 2 ) continue;
// set it to the full url
Url iu;
// use "pageUrl" as the baseUrl
iu.set( pageUrl, src, srcLen );
// skip if invalid domain or TLD
if ( iu.getDomainLen() <= 0 ) continue;
// skip if not from same domain as page url
//int32_t dlen = pageUrl->getDomainLen();
//if ( iu.getDomainLen() != dlen ) continue;
//if(strncmp(iu.getDomain(),pageUrl->getDomain(),dlen))continue
// get the full url
const char *u = iu.getUrl();
int32_t ulen = iu.getUrlLen();
// skip common crap
if ( strncasestr(u,ulen,"logo" ) ) continue;
if ( strncasestr(u,ulen,"comment" ) ) continue;
if ( strncasestr(u,ulen,"print" ) ) continue;
if ( strncasestr(u,ulen,"subscribe" ) ) continue;
if ( strncasestr(u,ulen,"header" ) ) continue;
if ( strncasestr(u,ulen,"footer" ) ) continue;
if ( strncasestr(u,ulen,"menu" ) ) continue;
if ( strncasestr(u,ulen,"button" ) ) continue;
if ( strncasestr(u,ulen,"banner" ) ) continue;
if ( strncasestr(u,ulen,"ad.doubleclick.") ) continue;
if ( strncasestr(u,ulen,"ads.webfeat." ) ) continue;
if ( strncasestr(u,ulen,"xads.zedo." ) ) continue;
// save it
m_imageNodes[m_numImages] = nn;
// before we lookup the image url to see if it is unique we
// must first make sure that we have an adequate number of
// permalinks from this same site
// we need at least 10 before we extract image thumbnails.
char buf[2000];
// set the query
Query q;
// if we do have 10 or more, then we lookup the image url to
// make sure it is indeed unique
sprintf ( buf , "gbimage:\"%s\"",u);
// TODO: make sure this is a no-split termid storage thingy
// in Msg14.cpp
if ( !q.set(buf, langUnknown, 1.0, 1.0, NULL, false, true, ABS_MAX_QUERY_TERMS) )
// return true with g_errno set on error
return;
if(q.getNumTerms()!= 1) { g_process.shutdownAbort(true); }
// store the termid
m_termIds[m_numImages] = q.getTermId(0);
// advance the counter
m_numImages++;
// break if full
if ( m_numImages >= MAX_IMAGES ) break;
}
}
// . returns false if blocked, returns true otherwise
// . sets g_errno on error
bool Images::getThumbnail ( const char *pageSite,
int32_t siteLen ,
int64_t docId ,
XmlDoc *xd ,
collnum_t collnum,//char *coll ,
//char **statusPtr ,
void *state ,
void (*callback)(void *state) ) {
// sanity check
if ( ! m_setCalled ) { g_process.shutdownAbort(true); }
// we haven't had any error
m_hadError = 0;
// no reason to stop yet
m_stopDownloading = false;
// reset here now
m_i = 0;
m_j = 0;
m_phase = 0;
// sanity check
if ( ! m_pageUrl ) { g_process.shutdownAbort(true); }
// sanity check
if ( ! pageSite ) { g_process.shutdownAbort(true); }
// we need to be a permalink
//if ( ! isPermalink ) return true;
// save these
//m_statusPtr = statusPtr;
// save this
m_collnum = collnum;
m_docId = docId;
m_callback = callback;
m_state = state;
// if no candidates, we are done, no error
if ( m_numImages == 0 ) return true;
//Vector *v = xd->getTagVector();
// this will at least have one component, the 0/NULL component
uint32_t *tph = xd->getTagPairHash32();
// must not block or error on us
if ( tph == (void *)-1 ) { g_process.shutdownAbort(true); }
// must not error on use?
if ( ! tph ) { g_process.shutdownAbort(true); }
// . see DupDetector.cpp, very similar to this
// . see how many pages we have from our same site with our same
// html template (and that are permalinks)
char buf[2000];
// site MUST NOT start with "http://"
if ( siteLen>=7 && memcmp(pageSite, "http://", 7)==0) { g_process.shutdownAbort(true); }
// this must match what we hash in XmlDoc::hashNoSplit()
sprintf ( buf , "gbsitetemplate:%" PRIu32"%*.*s", (uint32_t)*tph, (int)siteLen,(int)siteLen,pageSite);
// TODO: make sure this is a no-split termid storage thingy
// in Msg14.cpp
Query q;
if ( !q.set(buf, langUnknown, 1.0, 1.0, NULL, false, true, ABS_MAX_QUERY_TERMS) )
// return true with g_errno set on error
return true;
if(q.getNumTerms()!=1) { g_process.shutdownAbort(true); }
// store the termid
int64_t termId = q.getTermId(0);
key144_t startKey ;
key144_t endKey ;
Posdb::makeStartKey(&startKey,termId);
Posdb::makeEndKey (&endKey ,termId);
// get shard of that (this termlist is sharded by termid -
// see XmlDoc.cpp::hashNoSplit() where it hashes gbsitetemplate: term)
int32_t shardNum = g_hostdb.getShardNumByTermId ( &startKey );
logDebug(g_conf.m_logDebugImage, "image: image checking %s list on shard %" PRId32,buf,shardNum);
// just use msg0 and limit to like 1k or something
if ( ! m_msg0.getList ( -1 , // hostid
RDB_POSDB ,
m_collnum ,
&m_list , // RdbList ptr
(const char *)&startKey,
(const char *)&endKey,
1024 , // minRecSize
this ,
gotTermListWrapper ,
MAX_NICENESS ,
false , // err correction?
true , // inc tree?
-1 , // firstHostId
0 , // start filenum
-1 , // numFiles
30000 , // timeout
false , // isRealMerge?
false , // doIndexdbSplit?
shardNum ))// force paritysplit
return false;
// did not block
return gotTermFreq();
}
// returns false if blocked, true otherwise
bool Images::gotTermFreq ( ) {
// error?
if ( g_errno ) return true;
// bail if less than 10
//int64_t nt = m_msg36.getTermFreq();
// each key but the first is 12 bytes (compressed)
int64_t nt = (m_list.getListSize() - 6)/ 12;
// . return true, without g_errno set, we are done
// . if we do not have 10 or more webpages that share this same
// template then do not do image extraction at all, it is too risky
// that we get a bad image
// . MDW: for debugging, do not require 10 pages of same template
//if ( nt < 10 ) return true;
if ( nt < -2 ) return true;
// now see which of the image urls are unique
if ( ! launchRequests () ) return false;
// i guess we did not block
return true;
}
// . returns false if blocked, true otherwise
// . see if other pages we've indexed have this same image url
bool Images::launchRequests ( ) {
// loop over all images
for ( int32_t i = m_i ; i < m_numImages ; i++ ) {
// advance
m_i++;
// assume no error
m_errors[i] = 0;
// make the keys. each term is a gbimage:<imageUrl> term
// so we are searching for the image url to see how often
// it is repeated on other pages.
key144_t startKey ;
key144_t endKey ;
Posdb::makeStartKey(&startKey,m_termIds[i]);
Posdb::makeEndKey (&endKey ,m_termIds[i]);
uint32_t shardNum;
// assume to be for posdb here
shardNum = g_hostdb.getShardNumByTermId ( &startKey );
logDebug(g_conf.m_logDebugImage, "image: image checking shardnum %" PRId32" (termid0=%" PRIu64")for image url #%" PRId32,
shardNum ,m_termIds[i],i);
// get the termlist
if ( ! m_msg0.getList ( -1 , // hostid
RDB_POSDB,
m_collnum ,
&m_list , // RdbList ptr
(const char *)&startKey,
(const char *)&endKey,
1024 , // minRecSize
this ,
gotTermListWrapper ,
MAX_NICENESS ,
false , // err correction?
true , // inc tree?
-1 , // firstHostId
0 , // start filenum
-1 , // numFiles
30000 , // timeout
false , // isRealMerge?
false , // doIndexdbSplit?
shardNum ))// force paritysplit
return false;
// process the msg36 response
gotTermList ();
}
// i guess we didn't block
return downloadImages();
}
// we got a reply
void gotTermListWrapper ( void *state ) {
Images *THIS = (Images *)state;
// process/store the reply
THIS->gotTermList();
// try to launch more, returns false if it blocks
if ( ! THIS->launchRequests() ) return;
// all done
THIS->m_callback ( THIS->m_state );
}
void Images::gotTermList ( ) {
int32_t i = m_i - 1;
// i guess get errors too
if ( g_errno ) m_errors[i] = g_errno;
// set a globalish flag
if ( ! m_hadError ) m_hadError = g_errno;
// on error skip this
if ( g_errno ) return;
// check docids in termlist
m_list.resetListPtr();
// loop over it
for ( ; ! m_list.isExhausted() ; m_list.skipCurrentRecord() ) {
// is it us? if so ignore it
if ( Posdb::getDocId(m_list.getCurrentRec()) == m_docId ) continue;
// crap, i guess our image url is not unique. mark it off.
m_errors[i] = EDOCDUP;
// no need to go further
break;
}
}
bool Images::downloadImages() {
// all done if we got a valid thumbnail
//if ( m_thumbnailValid ) return true;
int32_t srcLen;
const char *src = NULL;
// . download each leftover image
// . stop as soon as we get one with good dimensions
// . make a thumbnail of that one
for ( ; m_j < m_numImages ; m_j++ , m_phase = 0 ) {
// did collection get nuked?
CollectionRec *cr = g_collectiondb.getRec(m_collnum);
if ( ! cr ) { g_errno = ENOCOLLREC; return true; }
// clear error
g_errno = 0;
if ( m_phase == 0 ) {
// advance
m_phase++;
// only if not diffbot, we set "src" above for it
// get the url of the image
src = getImageUrl ( m_j , &srcLen );
// use "pageUrl" as the baseUrl
m_imageUrl.set( m_pageUrl, src, srcLen );
// if we should stop, stop
if ( m_stopDownloading ) break;
// skip if bad or not unique
if ( m_errors[m_j] ) continue;
// set status msg
sprintf ( m_statusBuf ,"downloading image %" PRId32,m_j);
// point to it
if ( m_xd ) m_xd->setStatus ( m_statusBuf );
}
// get image ip
if ( m_phase == 1 ) {
// advance
m_phase++;
// this increments phase if it should
if ( ! getImageIp() ) return false;
// error?
if ( g_errno ) continue;
}
// download the actual image
if ( m_phase == 2 ) {
// advance
m_phase++;
// download image data
if ( ! downloadImage() ) return false;
// error downloading?
if ( g_errno ) continue;
}
// get thumbnail using threaded call to netpbm stuff
if ( m_phase == 3 ) {
// advance
m_phase++;
// call pnmscale etc. to make thumbnail
if ( ! makeThumb() ) return false;
// error downloading?
if ( g_errno ) continue;
}
// error making thumb or just not a good thumb size?
if ( ! m_thumbnailValid ) {
// free old image we downloaded, if any
m_msg13.reset();
// i guess do this too, it was pointing at it in msg13
m_imgReply = NULL;
// try the next image candidate
continue;
}
// it's a keeper
int32_t urlSize = m_imageUrl.getUrlLen() + 1; // include \0
// . make our ThumbnailArray out of it
int32_t need = 0;
// the array itself
need += sizeof(ThumbnailArray);
// and each thumbnail it contains
need += urlSize;
need += m_thumbnailSize;
need += sizeof(ThumbnailInfo);
// reserve it
if( !m_imageBuf.reserve ( need ) ) {
logError("Could not reserve needed %" PRId32 " bytes, bailing!", need);
return false;
}
// point to array
ThumbnailArray *ta =(ThumbnailArray *)m_imageBuf.getBufStart();
// set that as much as possible, version...
ta->m_version = 0;
// and thumb count
ta->m_numThumbnails = 1;
// now store the thumbnail info
ThumbnailInfo *ti = ta->getThumbnailInfo (0);
// and set our one thumbnail
ti->m_origDX = m_dx;
ti->m_origDY = m_dy;
ti->m_dx = m_tdx;
ti->m_dy = m_tdy;
ti->m_urlSize = urlSize;
ti->m_dataSize = m_thumbnailSize;
// now copy the data over sequentially
char *p = ti->m_buf;
// the image url
gbmemcpy(p,m_imageUrl.getUrl(),urlSize);
p += urlSize;
// the image thumbnail data
gbmemcpy(p,m_imgData,m_thumbnailSize);
p += m_thumbnailSize;
// update buf length of course
m_imageBuf.setLength ( p - m_imageBuf.getBufStart() );
// validate the buffer
m_imageBufValid = true;
// save mem. do this after because m_imgData uses m_msg13's
// reply buf to store the thumbnail for now...
m_msg13.reset();
m_imgReply = NULL;
g_errno = 0;
return true;
}
// don't tell caller EBADIMG it will make him fail to index doc
g_errno = 0;
return true;
}
static void gotImgIpWrapper ( void *state , int32_t ip ) {
Images *THIS = (Images *)state;
// control loop
if ( ! THIS->downloadImages() ) return;
// call callback at this point, we are done with the download loop
THIS->m_callback ( THIS->m_state );
}
bool Images::getImageIp ( ) {
if (!m_msgc.getIp(m_imageUrl.getHost(), m_imageUrl.getHostLen(), &m_latestIp, this, gotImgIpWrapper)) {
// we blocked
return false;
}
return true;
}
static void downloadImageWrapper ( void *state ) {
Images *THIS = (Images *)state;
// control loop
if ( ! THIS->downloadImages() ) return;
// all done
THIS->m_callback ( THIS->m_state );
}
bool Images::downloadImage ( ) {
// error?
if ( m_latestIp == 0 || m_latestIp == -1 ) {
log(LOG_DEBUG,"images: ip of %s is %" PRId32" (%s)",
m_imageUrl.getUrl(),m_latestIp,mstrerror(g_errno));
// ignore errors
g_errno = 0;
return true;
}
CollectionRec *cr = g_collectiondb.getRec(m_collnum);
if ( ! cr ) { g_errno = ENOCOLLREC; return true; }
// assume success
m_httpStatus = 200;
// set the request
Msg13Request *r = &m_msg13Request;
r->reset();
r->m_maxTextDocLen = 200000;
r->m_maxOtherDocLen = 500000;
r->m_urlIp = m_latestIp;
// url is the most important
r->ptr_url = const_cast<char*>(m_imageUrl.getUrl());
r->size_url = m_imageUrl.getUrlLen()+1; // include \0
// . try to download it
// . i guess we are ignoring hammers at this point
if ( ! m_msg13.getDoc(r,this,downloadImageWrapper))
return false;
return true;
}
// Use of ThreadEntry parameter is NOT thread safe
static void makeThumbWrapper ( void *state, job_exit_t exit_type ) {
Images *that = (Images *)state;
// store saved error into g_errno
g_errno = that->m_errno;
// control loop
if ( ! that->downloadImages() ) return;
// all done
that->m_callback ( that->m_state );
}
bool Images::makeThumb ( ) {
// did it have an error?
if ( g_errno ) {
// just give up on all of them if one has an error
log ( LOG_WARN, "image: had error downloading image on page %s: %s. "
"Not downloading any more.",
m_pageUrl->getUrl(),mstrerror(g_errno));
// stop it
m_stopDownloading = true;
return true;
}
char *buf;
int32_t bufLen, bufMaxLen;
HttpMime mime;
m_imgData = NULL;
m_imgDataSize = 0;
log( LOG_DEBUG, "image: gotImage() entered." );
// . if there was a problem, just ignore, don't let it stop getting
// the real page.
if ( g_errno ) {
log( "ERROR? g_errno puked: %s", mstrerror(g_errno) );
//g_errno = 0;
return true;
}
//if ( ! slot ) return true;
// extract image data from the socket
buf = m_msg13.m_replyBuf;
bufLen = m_msg13.m_replyBufSize;
bufMaxLen = m_msg13.m_replyBufAllocSize;
// no image?
if ( ! buf || bufLen <= 0 ) {
g_errno = EBADIMG;
return true;
}
// we are image candidate #i
//int32_t i = m_j - 1;
// get img tag node
// get the url of the image
int32_t srcLen;
const char *src = getImageUrl ( m_j , &srcLen );
// set it to the full url
Url iu;
// use "pageUrl" as the baseUrl
iu.set( m_pageUrl, src, srcLen );
// get the mime
if ( ! mime.set ( buf, bufLen, &iu ) ) {
log ( "image: MIME.set() failed in gotImage()" );
// give up on the remaining images then
m_stopDownloading = true;
g_errno = EBADIMG;
return true;
}
// set the status so caller can see
int32_t httpStatus = mime.getHttpStatus();
// check the status
if ( httpStatus != 200 ) {
log( LOG_DEBUG, "image: http status of img download is %" PRId32".",
m_httpStatus);
// give up on the remaining images then
m_stopDownloading = true;
g_errno = EBADIMG;
return true;
}
// make sure this is an image
m_imgType = mime.getContentType();
if ( m_imgType < CT_GIF || m_imgType > CT_TIFF ) {
log( LOG_DEBUG, "image: gotImage() states that this image is "
"not in a format we currently handle." );
// try the next image if any
g_errno = EBADIMG;
return true;
}
// get the content
m_imgData = buf + mime.getMimeLen();
m_imgDataSize = bufLen - mime.getMimeLen();
// Reset socket, so socket doesn't free the data, now we own
// We must free the buf after thumbnail is inserted in TitleRec
m_imgReply = buf;//slot->m_readBuf;
m_imgReplyLen = bufLen;//slot->m_readBufSize;
m_imgReplyMaxLen = bufMaxLen;//slot->m_readBufMaxSize;
// do not let UdpServer free the reply, we own it now
//slot->m_readBuf = NULL;
if ( m_imgReplyLen == 0 ) {
log( LOG_DEBUG, "image: Returned empty image reply!" );
g_errno = EBADIMG;
return true;
}
// get next if too small
if ( m_imgDataSize < 20 ) { g_errno = EBADIMG; return true; }
int32_t imageType;
getImageInfo ( m_imgData, m_imgDataSize, &m_dx, &m_dy, &imageType );
// log the image dimensions
log( LOG_DEBUG,"image: Image Link: %s", iu.getUrl() );
log( LOG_DEBUG,"image: Max Buffer Size: %" PRIu32" bytes.",m_imgReplyMaxLen);
log( LOG_DEBUG,"image: Image Original Size: %" PRIu32" bytes.",m_imgReplyLen);
log( LOG_DEBUG,"image: Image Buffer @ %p - %p",m_imgReply,
m_imgReply+m_imgReplyMaxLen );
log( LOG_DEBUG, "image: Size: %" PRIu32"px x %" PRIu32"px", m_dx, m_dy );
// what is this?
if ( m_dx <= 0 || m_dy <= 0 ) {
log(LOG_DEBUG, "image: Image has bad dimensions.");
g_errno = EBADIMG;
return true;
}
// skip if bad dimensions
if( ((m_dx < 50) || (m_dy < 50)) && ((m_dx > 0) && (m_dy > 0)) ) {
log(LOG_DEBUG,
"image: Image is too small to represent a news article." );
g_errno = EBADIMG;
return true;
}
// skip if bad aspect ratio. 5x1 or 1x5 is bad i guess
if ( m_dx > 0 && m_dy > 0 ) {
float aspect = (float)m_dx / (float)m_dy;
if ( aspect < .2 || aspect > 5.0 ) {
log(LOG_DEBUG,
"image: Image aspect ratio is worse that 5 to 1");
g_errno = EBADIMG;
return true;
}
}
CollectionRec *cr = g_collectiondb.getRec(m_collnum);
if ( ! cr ) { g_errno = ENOCOLLREC; return true; }
// save how big of thumbnails we should make. user can change
// this in the 'spider controls'
m_xysize = cr->m_thumbnailMaxWidthHeight ;
// make it 250 pixels if no decent value provided
if ( m_xysize <= 0 ) m_xysize = 250;
// and keep it sane
if ( m_xysize > 2048 ) m_xysize = 2048;
// update status
if ( m_xd ) m_xd->setStatus ( "making thumbnail" );
// log it
log ( LOG_DEBUG, "image: gotImage() thumbnailing image." );
// create the thumbnail...
// reset this... why?
g_errno = 0;
// reset this since filterStart_r() will set it on error
m_errno = 0;
// callThread returns true on success, in which case we block
if ( g_jobScheduler.submit(thumbStartWrapper_r, makeThumbWrapper, this, thread_type_generate_thumbnail, MAX_NICENESS) ) {
return false;
}
// threads might be off
logf ( LOG_DEBUG, "image: Calling thumbnail gen without thread.");
thumbStartWrapper_r ( this );
return true;
}
// Use of ThreadEntry parameter is NOT thread safe
void thumbStartWrapper_r ( void *state ) {
Images *that = (Images *)state;
// assume no error
that->m_errno = 0;
that->thumbStart_r ( true /* am thread?*/ );
if (g_errno && !that->m_errno) {
that->m_errno = g_errno;
}
}
void Images::thumbStart_r ( bool amThread ) {
int64_t start = gettimeofdayInMilliseconds();
//static char scmd[200] = "%stopnm %s | "
// "pnmscale -xysize 100 100 - | "
// "ppmtojpeg - > %s";
log( LOG_DEBUG, "image: thumbStart_r entered." );
//DIR *d;
//char cmd[2500];
//sprintf( cmd, "%strash", g_hostdb.m_dir );
makeTrashDir();
// get thread id. pthread_t is 64 bit and pid_t is 32 bit on
// 64 bit oses
pthread_t id = pthread_self();
// pass the input to the program through this file
// rather than a pipe, since popen() seems broken.
// m_dir ends in / so this should work.
char in[364];
snprintf ( in , 363,"%strash/in.%" PRId64
, g_hostdb.m_dir, (int64_t)id );
unlink ( in );
log( LOG_DEBUG, "image: thumbStart_r create in file." );
// collect the output from the filter from this file
// m_dir ends in / so this should work.
char out[364];
snprintf ( out , 363,"%strash/out.%" PRId64
, g_hostdb.m_dir, (int64_t)id );
unlink ( out );
log( LOG_DEBUG, "image: thumbStart_r create out file." );
// ignore errno from those unlinks
errno = 0;
// Open/Create temporary file to store image to
int fhndl;
if( (fhndl = open( in, O_RDWR+O_CREAT ,
getFileCreationFlags()
)) < 0 ) {
log(LOG_WARN, "image: Could not open file, %s, for writing: %s - %d.",
in, mstrerror( m_errno ), fhndl );
m_imgDataSize = 0;
return;
}
// Write image data into temporary file
if( write( fhndl, m_imgData, m_imgDataSize ) < 0 ) {
log(LOG_WARN, "image: Could not write to file, %s: %s.",
in, mstrerror( m_errno ) );
close( fhndl );
unlink( in );
m_imgDataSize = 0;
return;
}
// Close temporary image file now that we have finished writing
if( close( fhndl ) < 0 ) {
log(LOG_WARN, "image: Could not close file, %s, for writing: %s.",
in, mstrerror( m_errno ) );
unlink( in );
m_imgDataSize = 0;
return;
}
fhndl = 0;
// Grab content type from mime
//int32_t imgType = mime.getContentType();
char ext[5];
switch( m_imgType ) {
case CT_GIF:
strcpy( ext, "gif" );
break;
case CT_JPG:
strcpy( ext, "jpeg" );
break;
case CT_PNG:
strcpy( ext, "png" );
break;
case CT_TIFF:
strcpy( ext, "tiff" );
break;
case CT_BMP:
strcpy( ext, "bmp" );
break;
}
char cmd[2500];
//sprintf( cmd, scmd, ext, in, out);
const char *wdir = g_hostdb.m_dir;
// wdir ends in / so this should work.
snprintf( cmd, sizeof(cmd),
"LD_LIBRARY_PATH=%s %s%stopnm %s | "
"LD_LIBRARY_PATH=%s %spnmscale -xysize %" PRId32" %" PRId32" - | "
// put all its stderr msgs into /dev/null
// so "jpegtopnm: WRITING PPM FILE" doesn't clog console
"LD_LIBRARY_PATH=%s %sppmtojpeg - > %s"
, wdir, wdir, ext, in
, wdir, wdir, m_xysize, m_xysize
, wdir, wdir, out
);
cmd[sizeof(cmd)-1] = '\0';
// if they already have netpbm package installed use that then
static bool s_checked = false;
static bool s_hasNetpbm = false;
if ( ! s_checked ) {
s_checked = true;
File f;
f.set("/usr/bin/pnmscale");
s_hasNetpbm = f.doesExist() ;
}
if ( s_hasNetpbm )
snprintf( cmd, sizeof(cmd),
"%stopnm %s | "
"pnmscale -xysize %" PRId32" %" PRId32" - | "
"ppmtojpeg - > %s"
, ext, in
, m_xysize, m_xysize
, out
);
// Call clone function for the shell to execute command
int err = system( cmd ); // m_thmbconvTimeout );
//if( (m_dx != 0) && (m_dy != 0) )
// unlink( in );
unlink ( in );
if ( err == 127 ) {
m_errno = EBADENGINEER;
log("image: /bin/sh does not exist.");
unlink ( out );
m_stopDownloading = true;
return;
}
// this will happen if you don't upgrade glibc to 2.2.4-32 or above
if ( err != 0 ) {
m_errno = EBADENGINEER;
log(LOG_WARN, "image: Call to system(\"%s\") had error.",cmd);
unlink ( out );
m_stopDownloading = true;
return;
}
// Open new file with thumbnail image
if( (fhndl = open( out, O_RDONLY )) < 0 ) {
log(LOG_WARN, "image: Could not open file, %s, for reading: %s.",
out, mstrerror( m_errno ) );
unlink ( out );
m_stopDownloading = true;
return;
}
if( (m_thumbnailSize = lseek( fhndl, 0, SEEK_END )) < 0 ) {
log( LOG_WARN, "image: Seek of file, %s, returned invalid size: %" PRId32,
out, m_thumbnailSize );
m_stopDownloading = true;
close(fhndl);
unlink ( out );
return;
}
if( m_thumbnailSize > m_imgReplyMaxLen ) {
log(LOG_DEBUG,"image: Image thumbnail larger than buffer!" );
log(LOG_DEBUG,"image: File Read Bytes: %" PRId32, m_thumbnailSize);
log(LOG_DEBUG,"image: Buf Max Bytes : %" PRId32,m_imgReplyMaxLen );
log(LOG_DEBUG,"image: -----------------------" );
log(LOG_DEBUG,"image: Diff : %" PRId32,
m_imgReplyMaxLen-m_thumbnailSize );
close(fhndl);
unlink ( out );
return;
}
if( lseek( fhndl, 0, SEEK_SET ) < 0 ) {
log( "image: Seek couldn't rewind file, %s.", out );
m_stopDownloading = true;
close(fhndl);
unlink ( out );
return;
}
// . Read contents back into image ptr
// . this is somewhat of a hack since it overwrites the original img
if( (m_thumbnailSize = read( fhndl, m_imgData, m_imgDataSize )) < 0 ) {
log( LOG_WARN, "image: Could not read from file, %s: %s.",
out, mstrerror( m_errno ) );
close( fhndl );
m_stopDownloading = true;
unlink( out );
return;
}
if( close( fhndl ) < 0 ) {
log( LOG_WARN, "image: Could not close file, %s, for reading: %s.",
out, mstrerror( m_errno ) );
unlink( out );
m_stopDownloading = true;
unlink ( out );
return;
}
fhndl = 0;
unlink( out );
int64_t stop = gettimeofdayInMilliseconds();
// tell the loop above not to download anymore, we got one
m_thumbnailValid = true;
// MDW: this was m_imgReply
getImageInfo ( m_imgData , m_thumbnailSize , &m_tdx , &m_tdy , NULL );
// now make the meta data struct
// <imageUrl>\0<width><height><thumbnailData>
log( LOG_DEBUG, "image: Thumbnail size: %" PRId32" bytes.", m_imgDataSize );
log( LOG_DEBUG, "image: Thumbnail dx=%" PRId32" dy=%" PRId32".", m_tdx,m_tdy );
log( LOG_DEBUG, "image: Thumbnail generated in %" PRId64"ms.", stop-start );
}
// . *it is the image type
void getImageInfo(const char *buf, int32_t bufSize,
int32_t *dx, int32_t *dy, int32_t *it)
{
// default to zeroes
*dx = 0;
*dy = 0;
// get the dimensions of the image
if( strncasestr( buf, 20, "Exif" ) ) {
log(LOG_DEBUG, "image: Image Link: ");
log(LOG_DEBUG, "image: We currently do not handle EXIF image "
"types." );
// try the nextone
return;
}
else if( strncasestr( buf, 20, "GIF" ) ) {
if ( it ) *it = CT_GIF;
log( LOG_DEBUG, "image: GIF INFORMATION:" );
if( bufSize > 9 ) {
*dx = ((uint32_t)buf[7]) << 8;
*dx += (unsigned char)buf[6];
*dy = ((uint32_t)buf[9]) << 8;
*dy += (unsigned char)buf[8];
}
}
else if( strncasestr( buf, 20, "JFIF" ) ) {
if ( it ) *it = CT_JPG;
log( LOG_DEBUG, "image: JPEG INFORMATION:" );
int32_t i;
for( i = 0; i < bufSize; i++ ) {
if( bufSize < i+8 )
break;
if( (unsigned char)buf[i] != 0xFF ) continue;
if( (unsigned char)buf[i+1] == 0xC0 ){
*dy = ((uint32_t)buf[i+5]) << 8;
*dy += (unsigned char)buf[i+6];
*dx = ((uint32_t)buf[i+7]) << 8;
*dx += (unsigned char)buf[i+8];
break;
}
else if( (unsigned char) buf[i+1] == 0xC2 ) {
*dy = ((uint32_t)buf[i+5]) << 8;
*dy += (unsigned char)buf[i+6];
*dx = ((uint32_t)buf[i+7]) << 8;
*dx += (unsigned char)buf[i+8];
break;
}
}
}
else if( strncasestr( buf, 20, "PNG" ) ) {
if ( it ) *it = CT_PNG;
log( LOG_DEBUG, "image: PNG INFORMATION:" );
if( bufSize > 25 ) {
*dx=(uint32_t)(*(uint32_t *)&buf[16]);
*dy=(uint32_t)(*(uint32_t *)&buf[20]);
// these are in network order
*dx = ntohl(*dx);
*dy = ntohl(*dy);
}
}
else if( strncasestr( buf, 20, "MM" ) ) {
if ( it ) *it = CT_TIFF;
log( LOG_DEBUG, "image: TIFF INFORMATION:" );
int32_t startCnt = (uint32_t)buf[7]+4;
for( int32_t i = startCnt; i < bufSize; i += 12 ) {
if( bufSize < i+10 )
break;
if( buf[i] != 0x01 ) continue;
if( buf[i+1] == 0x01 )
*dy = (uint32_t)
(*(uint16_t *)&buf[i+8]);
else if( buf[i+1] == 0x00 )
*dx = (uint32_t)
(*(uint16_t *)&buf[i+8]);
}
}
else if( strncasestr( buf, 20, "II" ) ) {
if ( it ) *it = CT_TIFF;
log( LOG_DEBUG, "image: TIFF INFORMATION:" );
int32_t startCnt = (uint32_t)buf[7]+4;
for( int32_t i = startCnt; i < bufSize; i += 12 ) {
if( bufSize < i+10 )
break;
if( buf[i] == 0x01 && buf[i+1] == 0x01 )
*dy = (uint32_t)
(*(uint16_t *)&buf[i+8]);
if( buf[i] == 0x00 && buf[i+1] == 0x01 )
*dx = (uint32_t)
(*(uint16_t *)&buf[i+8]);
}
}
else if( strncasestr( buf, 20, "BM" ) ) {
if ( it ) *it = CT_BMP;
log( LOG_DEBUG, "image: BMP INFORMATION:" );
if( bufSize > 27 ) {
*dx=(uint32_t)(*(uint32_t *)&buf[18]);
*dy=(uint32_t)(*(uint32_t *)&buf[22]);
}
}
else
log( LOG_DEBUG, "image: Image Corrupted? No type found in "
"data." );
}
// container is maxWidth X maxHeight, so try to fix widget in there
bool ThumbnailInfo::printThumbnailInHtml ( SafeBuf *sb ,
int32_t maxWidth ,
int32_t maxHeight,
bool printLink ,
int32_t *retNewdx ,
const char *style ,
char format ) {
if ( ! style ) style = "";
// account for scrollbar on the right
//maxSide -= (int32_t)SCROLLBAR_WIDTH;
// avoid distortion.
// if image is wide, use that to scale
if ( m_dx <= 0 ) return true;
if ( m_dy <= 0 ) return true;
float xscale =
(float)maxWidth/
(float)m_dx;
float yscale =
(float)maxHeight/
(float)m_dy;
float min = xscale;
if ( yscale < min ) min = yscale;
int32_t newdx = (int32_t)((float)m_dx * min);
int32_t newdy = (int32_t)((float)m_dy * min);
// might be FORMAT_AJAX!
if ( printLink && format !=FORMAT_XML && format != FORMAT_JSON )
sb->safePrintf("<a href=%s>", getUrl() );
if ( format !=FORMAT_XML && format != FORMAT_JSON )
sb->safePrintf("<img width=%" PRId32" height=%" PRId32" align=left "
"%s"
"src=\"data:image/"
"jpg;base64,"
, newdx
, newdy
, style
);
if ( format == FORMAT_XML )
sb->safePrintf("\t<imageBase64>");
if ( format == FORMAT_JSON )
sb->safePrintf("\t\"imageBase64\":\"");
// encode image in base 64
sb->base64Encode ( getData(), m_dataSize );
if ( format !=FORMAT_XML && format != FORMAT_JSON ) {
sb->safePrintf("\">");
if ( printLink ) sb->safePrintf ("</a>");
}
if ( format == FORMAT_XML )
sb->safePrintf("</imageBase64>\n");
if ( format == FORMAT_JSON )
sb->safePrintf("\",\n");
// widget needs to know the width of the thumb for formatting
// the text either on top of the thumb or to the right of it
if ( retNewdx ) *retNewdx = newdx;
return true;
}
const char *Images::getImageUrl ( int32_t j , int32_t *urlLen ) {
int32_t node = m_imageNodes[j];
int32_t srcLen = 0;
char *src = m_xml->getString(node,"src",&srcLen);
// maybe it was an og:image meta tag
if ( ! src )
src = m_xml->getString(node,"content",&srcLen);
// wtf?
if ( ! src )
log("image: image bad/null src");
*urlLen = srcLen;
return src;
}
Reference in New Issue View Git Blame Copy Permalink