3554 lines
118 KiB
C++
3554 lines
118 KiB
C++
#include "Query.h"
|
|
#include "Bits.h"
|
|
#include "Phrases.h"
|
|
#include "Url.h"
|
|
#include "Domains.h"
|
|
#include "Clusterdb.h" // g_clusterdb.getNumGlobalRecs()
|
|
#include "StopWords.h" // isQueryStopWord()
|
|
#include "Sections.h"
|
|
#include "Speller.h"
|
|
#include "Mem.h"
|
|
#include "Msg3a.h"
|
|
#include "HashTableX.h"
|
|
#include "Synonyms.h"
|
|
#include "HighFrequencyTermShortcuts.h"
|
|
#include "Wiki.h"
|
|
#include "ScoringWeights.h"
|
|
#include "RdbList.h"
|
|
#include "Process.h"
|
|
#include "Conf.h"
|
|
#include "termid_mask.h"
|
|
#include "Collectiondb.h"
|
|
#include "GbUtil.h"
|
|
#include <set>
|
|
#include "Lemma.h"
|
|
#include "Errno.h"
|
|
|
|
|
|
#include "GbMutex.h"
|
|
#include "ScopedLock.h"
|
|
|
|
static int count_quotes(const char *s, size_t len);
|
|
|
|
|
|
Query::Query()
|
|
: m_queryWordBuf("Query4"),
|
|
m_tr(),
|
|
m_filteredQuery("qrystk"),
|
|
m_originalQuery("oqbuf"),
|
|
m_bigramWeight(1.0),
|
|
m_synonymWeight(1.0),
|
|
m_word_variations_config()
|
|
{
|
|
m_qwords = NULL;
|
|
m_numWords = 0;
|
|
m_qwords = NULL;
|
|
m_numTerms = 0;
|
|
|
|
// Coverity
|
|
m_langId = langUnknown;
|
|
m_useQueryStopWords = false;
|
|
m_allowHighFreqTermCache = false;
|
|
m_numTermsUntruncated = 0;
|
|
m_isBoolean = false;
|
|
m_maxQueryTerms = 0;
|
|
|
|
memset(m_expressions, 0, sizeof(m_expressions));
|
|
|
|
reset ( );
|
|
}
|
|
|
|
Query::~Query ( ) {
|
|
reset ( );
|
|
}
|
|
|
|
void Query::reset ( ) {
|
|
|
|
// if Query::constructor() was called explicitly then we have to
|
|
// call destructors explicitly as well...
|
|
// essentially call QueryTerm::reset() on each query term
|
|
for ( int32_t i = 0 ; i < m_numWords ; i++ ) {
|
|
QueryWord *qw = &m_qwords[i];
|
|
qw->destructor();
|
|
}
|
|
|
|
m_queryTermBuf.purge();
|
|
m_qterms = NULL;
|
|
m_tr.clear();
|
|
|
|
m_filteredQuery.purge();
|
|
m_originalQuery.purge();
|
|
m_docIdRestriction = 0LL;
|
|
m_numWords = 0;
|
|
m_numTerms = 0;
|
|
|
|
m_queryWordBuf.purge();
|
|
m_qwords = NULL;
|
|
m_numExpressions = 0;
|
|
// the site: and ip: query terms will disable site clustering & caching
|
|
m_hasPositiveSiteField = false;
|
|
m_hasIpField = false;
|
|
m_hasUrlField = false;
|
|
m_hasSubUrlField = false;
|
|
m_truncated = false;
|
|
}
|
|
|
|
// . returns false and sets g_errno on error
|
|
// . "query" must be NULL terminated
|
|
// . if boolFlag is 0 we ignore all boolean operators
|
|
// . if boolFlag is 1 we assume query is boolen
|
|
// . if boolFlag is 2 we attempt to detect if query is boolean or not
|
|
// . if "keepAllSingles" is true we do not ignore any single word UNLESS
|
|
// it is a boolean operator (IGNORE_BOOLOP), fieldname (IGNORE_FIELDNAME)
|
|
// a punct word (IGNORE_DEFAULT) or part of one field value (IGNORE_DEFAULT)
|
|
// This is used for term highlighting (Highlight.cpp and Summary.cpp)
|
|
bool Query::set(const char *query,
|
|
// need language for doing synonyms
|
|
lang_t langId,
|
|
float bigramWeight,
|
|
float synonymWeight,
|
|
const WordVariationsConfig *wordVariationsConfig,
|
|
bool useQueryStopWords,
|
|
bool allowHighFreqTermCache,
|
|
int32_t maxQueryTerms)
|
|
{
|
|
static const WordVariationsConfig defaultWordVariationsConfig;
|
|
if(!wordVariationsConfig)
|
|
wordVariationsConfig = &defaultWordVariationsConfig;
|
|
log(LOG_DEBUG,"query: set2(query='%s', langId=%d, wiktionaryWordVariations=%s, languageSpecificWordVariations=%s useQueryStopWords=%s maxQueryTerms=%d)",
|
|
query, (int)langId, wordVariationsConfig->m_wiktionaryWordVariations?"true":"false", wordVariationsConfig->m_languageSpecificWordVariations?"true":"false", useQueryStopWords?"true":"false", maxQueryTerms);
|
|
|
|
reset();
|
|
|
|
m_langId = langId;
|
|
m_useQueryStopWords = useQueryStopWords;
|
|
m_allowHighFreqTermCache = allowHighFreqTermCache;
|
|
|
|
// fix summary rerank and highlighting.
|
|
bool keepAllSingles = true;
|
|
|
|
m_maxQueryTerms = maxQueryTerms;
|
|
|
|
// assume boolean auto-detect.
|
|
char boolFlag = 2;
|
|
|
|
|
|
if ( ! query ) return true;
|
|
|
|
m_bigramWeight = bigramWeight;
|
|
m_synonymWeight = synonymWeight;
|
|
m_word_variations_config = *wordVariationsConfig;
|
|
|
|
int32_t queryLen = strlen(query);
|
|
|
|
// truncate query if too big
|
|
if ( queryLen >= ABS_MAX_QUERY_LEN ) {
|
|
log(LOG_WARN, "query: Query length of %" PRId32" must be less than %" PRId32". Truncating.",
|
|
queryLen,(int32_t)ABS_MAX_QUERY_LEN);
|
|
queryLen = ABS_MAX_QUERY_LEN - 1;
|
|
m_truncated = true;
|
|
}
|
|
// save original query
|
|
if( !m_originalQuery.reserve ( queryLen + 1 ) ) {
|
|
logError("Failed to reserve %" PRId32 " bytes, bailing", queryLen+1);
|
|
return true;
|
|
}
|
|
m_originalQuery.safeMemcpy(query, queryLen);
|
|
m_originalQuery.nullTerm();
|
|
|
|
const char *q = query;
|
|
// see if it should be boolean...
|
|
for ( int32_t i = 0 ; i < queryLen ; i++ ) {
|
|
// but if bool flag is 0 that means it is NOT boolean!
|
|
// it must be one for autodetection. so do not autodetect
|
|
// unless this is 2.
|
|
if ( boolFlag != 2 ) break;
|
|
if ( q[i]=='A' && q[i+1]=='N' && q[i+2]=='D' &&
|
|
(q[i+3]==' ' || q[i+3]=='(') )
|
|
boolFlag = 1;
|
|
if ( q[i]=='O' && q[i+1]=='R' &&
|
|
(q[i+2]==' ' || q[i+2]=='(') )
|
|
boolFlag = 1;
|
|
if ( q[i]=='N' && q[i+1]=='O' && q[i+2]=='T' &&
|
|
(q[i+3]==' ' || q[i+3]=='(') )
|
|
boolFlag = 1;
|
|
}
|
|
|
|
// if we did not set the flag to 1 set it to 0. force to non-bool
|
|
if ( boolFlag == 2 ) boolFlag = 0;
|
|
|
|
// reserve some space, guessing how much we'd need
|
|
int32_t need = queryLen * 2 + 32;
|
|
if ( ! m_filteredQuery.reserve ( need ) )
|
|
return false;
|
|
|
|
bool inQuotesFlag = false;
|
|
// . copy query into m_buf
|
|
// . translate ( and ) to special query operators so Words class
|
|
// can parse them as their own word to make parsing bool queries ez
|
|
// for parsing out the boolean operators in setBitScoresBoolean()
|
|
for ( int32_t i = 0 ; i < queryLen ; i++ ) {
|
|
|
|
// gotta count quotes! we ignore operators in quotes
|
|
// so you can search for diffbotUri:"article|0|123456"
|
|
if ( query[i] == '\"' ) inQuotesFlag = !inQuotesFlag;
|
|
|
|
if ( inQuotesFlag ) {
|
|
//*p = query [i];
|
|
//p++;
|
|
m_filteredQuery.pushChar(query[i]);
|
|
continue;
|
|
}
|
|
|
|
// translate ( and )
|
|
if ( boolFlag == 1 && query[i] == '(' ) {
|
|
m_filteredQuery.safeMemcpy ( " LeFtP " , 7 );
|
|
continue;
|
|
}
|
|
if ( boolFlag == 1 && query[i] == ')' ) {
|
|
m_filteredQuery.safeMemcpy ( " RiGhP " , 7 );
|
|
continue;
|
|
}
|
|
if ( query[i] == '|' ) {
|
|
m_filteredQuery.safeMemcpy ( " PiiPE " , 7 );
|
|
continue;
|
|
}
|
|
|
|
if(query[i] == '[') {
|
|
// translate [#w] [#p] [#s] [w] [p] [s] [nrw] to operators
|
|
char *endptr=NULL;
|
|
double val;
|
|
if(is_digit(query[i+1]))
|
|
val=strtod(query+i+1,&endptr);
|
|
if(endptr!=NULL && endptr!=query+1) {
|
|
size_t j = (size_t)(endptr-query);
|
|
if(query[j]=='w' && query[j+1]==']') {
|
|
m_filteredQuery.safePrintf(" LeFtB %f w RiGhB ", val);
|
|
i = j + 1;
|
|
continue;
|
|
} else if(query[j]=='p' && query[j+1]==']') {
|
|
m_filteredQuery.safePrintf(" LeFtB %f p RiGhB ", val);
|
|
i = j + 1;
|
|
continue;
|
|
} else if(query[j]=='s' && query[j+1]==']') {
|
|
m_filteredQuery.safePrintf(" LeFtB %f s RiGhB ", val);
|
|
i = j + 1;
|
|
continue;
|
|
}
|
|
} else if(query[i+1] == 'w' && query[i+2]==']') {
|
|
m_filteredQuery.safePrintf(" LeFtB w RiGhB ");
|
|
i = i + 2;
|
|
continue;
|
|
} else if(query[i+1] == 'p' && query[i+2]==']') {
|
|
m_filteredQuery.safePrintf(" LeFtB p RiGhB ");
|
|
i = i + 2;
|
|
continue;
|
|
} else if(query[i+1] == 's' && query[i+2]==']') {
|
|
m_filteredQuery.safePrintf(" LeFtB s RiGhB ");
|
|
i = i + 2;
|
|
continue;
|
|
} else if( i+4 < queryLen && query[i+1] == 'n' && query[i+2] == 'r' && query[i+3] == 'w' && query[i+4]==']') {
|
|
// user specified [nrw] before word, meaning treat it as not required
|
|
m_filteredQuery.safePrintf(" LeFtB nrw RiGhB ");
|
|
i = i + 4;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// TODO: copy altavista's operators here? & | !
|
|
// otherwise, just a plain copy
|
|
m_filteredQuery.pushChar ( query[i] );
|
|
}
|
|
// NULL terminate
|
|
m_filteredQuery.nullTerm();
|
|
if(m_filteredQuery.length() != queryLen || memcmp(m_filteredQuery.getBufStart(),query,queryLen)!=0)
|
|
log(LOG_INFO,"query: m_filteredQuery=%*.*s", m_filteredQuery.length(),m_filteredQuery.length(),m_filteredQuery.getBufStart());
|
|
|
|
Phrases phrases;
|
|
|
|
// set m_qwords[] array from m_buf
|
|
if ( ! setQWords(boolFlag, keepAllSingles, phrases) )
|
|
return false;
|
|
|
|
// set m_qterms from m_qwords, always succeeds
|
|
setQTerms();
|
|
|
|
// disable stuff for site:, ip: and url: queries
|
|
for ( int32_t i = 0 ; i < m_numWords ; i++ ) {
|
|
const QueryWord *qw = &m_qwords[i];
|
|
if ( qw->m_ignoreWord ) continue;
|
|
if ( qw->m_fieldCode == FIELD_SITE &&
|
|
qw->m_wordSign != '-' )
|
|
m_hasPositiveSiteField = true;
|
|
else if ( qw->m_fieldCode == FIELD_IP )
|
|
m_hasIpField = true;
|
|
else if ( qw->m_fieldCode == FIELD_URL )
|
|
m_hasUrlField = true;
|
|
else if ( qw->m_fieldCode == FIELD_SUBURL )
|
|
m_hasSubUrlField = true;
|
|
}
|
|
|
|
// set m_docIdRestriction if a term is gbdocid:
|
|
for ( int32_t i = 0 ; i < m_numTerms && ! m_isBoolean ; i++ ) {
|
|
// get it
|
|
QueryTerm *qt = &m_qterms[i];
|
|
|
|
if( qt->m_fieldCode == FIELD_GBTERMID ) {
|
|
const char *ds = m_qterms[i].m_term + 9; // strlen("gbtermid:")
|
|
qt->m_termId = atoll(ds);
|
|
}
|
|
|
|
// gbdocid:?
|
|
if ( qt->m_fieldCode != FIELD_GBDOCID ) continue;
|
|
// get docid
|
|
const char *ds = m_qterms[i].m_term + 8;
|
|
m_docIdRestriction = atoll(ds);
|
|
break;
|
|
}
|
|
|
|
// . keep it simple for now
|
|
// . we limit to MAX_EXRESSIONS to like 10 now i guess
|
|
if ( m_isBoolean ) {
|
|
m_numExpressions = 1;
|
|
if ( ! m_expressions[0].addExpression ( 0 ,
|
|
m_numWords ,
|
|
this , // Query
|
|
0 ) ) // level
|
|
// return false with g_errno set on error
|
|
return false;
|
|
}
|
|
|
|
|
|
log(LOG_DEBUG,"query: m_numWords=%d, m_numTerms=%d", m_numWords, m_numTerms);
|
|
|
|
// . if it is not truncated, no need to use hard counts
|
|
// . comment this line and the next one out for testing hard counts
|
|
if ( ! m_truncated ) return true;
|
|
// if they just hit the admin's ceiling, there's nothing we can do
|
|
if ( m_numTerms >= m_maxQueryTerms ) return true;
|
|
// a temp log message
|
|
log(LOG_DEBUG,"query: Encountered %" PRId32" query terms.",m_numTerms);
|
|
|
|
// otherwise, we're below m_maxQueryTerms BUT above MAX_QUERY_TERMS
|
|
// so we can use hard counts to get more power...
|
|
|
|
// . use the hard count for excessive query terms to save explicit bits
|
|
// . just look for operands on the first level that are not OR'ed
|
|
char redo = 0;
|
|
for ( int32_t i = 0 ; i < m_numWords ; i++ ) {
|
|
// get the ith word
|
|
QueryWord *qw = &m_qwords[i];
|
|
// stop at first OR on this level
|
|
if ( qw->m_opcode == opcode_t::OP_OR ) break;
|
|
// skip all punct
|
|
if ( qw->m_isPunct ) continue;
|
|
// if we are a boolean query,the next operator can NOT be OP_OR
|
|
// because we can not used terms that are involved in an OR
|
|
// as a hard count term, because they are not required terms
|
|
for ( int32_t j=i+1 ; m_isBoolean && j<m_numWords; j++ ) {
|
|
// stop at previous operator
|
|
opcode_t opcode = m_qwords[j].m_opcode;
|
|
if ( opcode == opcode_t::OP_NONE ) continue;
|
|
if ( opcode != opcode_t::OP_OR ) break;
|
|
// otherwise, the next operator is an OR, so do not
|
|
// use a hard count for this term
|
|
goto stop;
|
|
}
|
|
// mark it so we can reduce our number of explicit bits used
|
|
redo = 1;
|
|
}
|
|
|
|
stop:
|
|
// if nothing changed, return now
|
|
if ( ! redo ) return true;
|
|
|
|
// . set the query terms again if we have a long query
|
|
if ( ! setQTerms() )
|
|
return false;
|
|
|
|
return true;
|
|
}
|
|
|
|
// returns false and sets g_errno on error
|
|
bool Query::setQTerms() {
|
|
if(g_conf.m_logTraceQuery) {
|
|
logTrace(g_conf.m_logTraceQuery, "Query::setQTerms(words:%zu)", m_tr.size());
|
|
for(unsigned i=0; i<m_tr.size(); i++) {
|
|
logTrace(g_conf.m_logTraceQuery, " word #%u: '%*.*s'", i, (int)m_tr[i].token_len, (int)m_tr[i].token_len, m_tr[i].token_start);
|
|
int64_t phraseTermId = m_qwords[i].m_bigramId&TERMID_MASK;
|
|
int64_t wordTermId = m_qwords[i].m_wordId&TERMID_MASK;
|
|
logTrace(g_conf.m_logTraceQuery, " m_bigramId=%20" PRId64" (%15" PRId64"), m_ignorePhrase=%d m_bigramLen=%d", m_qwords[i].m_bigramId, phraseTermId, m_qwords[i].m_ignorePhrase, m_qwords[i].m_bigramLen);
|
|
logTrace(g_conf.m_logTraceQuery, " m_wordId =%20" PRId64" (%15" PRId64"), m_ignoreWord=%d, m_quoteStart=%d, m_quoteEnd=%d, fieldCode=%s, m_prefixHash=0x%lx", m_qwords[i].m_wordId, wordTermId, m_qwords[i].m_ignoreWord, m_qwords[i].m_quoteStart, m_qwords[i].m_quoteEnd, m_qwords[i].m_fieldCode?getFieldCodeName(m_qwords[i].m_fieldCode):"",m_qwords[i].m_prefixHash);
|
|
|
|
}
|
|
}
|
|
// . set m_qptrs/m_qtermIds/m_qbits
|
|
// . use one bit position for each phraseId and wordId
|
|
|
|
// count phrases first for allocating
|
|
//Removed: elaborate counting of possible bigrams. Done instead: this:
|
|
int numCandidatePhrases = m_numWords-1;
|
|
|
|
// count single terms
|
|
int numCandidateSingles = 0;
|
|
for ( int32_t i = 0 ; i < m_numWords; i++ ) {
|
|
const QueryWord *qw = &m_qwords[i];
|
|
if ( qw->m_ignoreWord &&
|
|
qw->m_ignoreWord != IGNORE_QSTOP) continue;
|
|
// ignore if in quotes and part of phrase, watch out
|
|
// for things like "word", a single word in quotes.
|
|
if ( qw->m_quoteStart >= 0 && qw->m_bigramId ) continue;
|
|
// if we are not start of quote and NOT in a phrase we
|
|
// must be the tailing word i guess.
|
|
// fixes '"john smith" -"bob dole"' from having
|
|
// smith and dole as query terms.
|
|
if ( qw->m_quoteStart >= 0 && qw->m_quoteStart != i )
|
|
continue;
|
|
// ignore if weight is absolute zero
|
|
if ( almostEqualFloat(qw->m_userWeightForWord,0) )
|
|
continue;
|
|
numCandidateSingles++;
|
|
}
|
|
// thirdly, count synonyms
|
|
int numCandidateSynonyms = 0;
|
|
Synonyms syn;
|
|
if(m_word_variations_config.m_wiktionaryWordVariations) {
|
|
int64_t to = hash64n("to");
|
|
for ( int32_t i = 0 ; i < m_numWords ; i++ ) {
|
|
// get query word
|
|
const QueryWord *qw = &m_qwords[i];
|
|
// skip if in quotes, we will not get synonyms for it
|
|
if ( qw->m_inQuotes ) continue;
|
|
// skip if has plus sign in front
|
|
if ( qw->m_wordSign == '+' ) continue;
|
|
// not '-' either i guess
|
|
if ( qw->m_wordSign == '-' ) continue;
|
|
// no url: stuff, maybe only title
|
|
if ( qw->m_fieldCode && qw->m_fieldCode != FIELD_TITLE )
|
|
continue;
|
|
// ignore title: etc. words, they are field names
|
|
if ( qw->m_ignoreWord == IGNORE_FIELDNAME ) continue;
|
|
// ignore boolean operators
|
|
if ( qw->m_ignoreWord ) continue;// IGNORE_BOOLOP
|
|
// ignore if word weight is zero or synonym weight is zero
|
|
if(almostEqualFloat(qw->m_userWeightForWord,0))
|
|
continue;
|
|
if(almostEqualFloat(qw->m_userWeightForSynonym,0))
|
|
continue;
|
|
// no, hurts 'Greencastle IN economic development'
|
|
if ( qw->m_wordId == to ) continue;
|
|
// single letters...
|
|
if ( qw->m_wordLen == 1 ) continue;
|
|
// set the synonyms for this word
|
|
char tmpBuf [ TMPSYNBUFSIZE ];
|
|
int32_t naids = syn.getSynonyms ( &m_tr,
|
|
i ,
|
|
// language of the query.
|
|
// 0 means unknown. if this
|
|
// is 0 we sample synonyms
|
|
// from all languages.
|
|
m_langId ,
|
|
tmpBuf );
|
|
// if no synonyms, all done
|
|
if ( naids <= 0 ) continue;
|
|
numCandidateSynonyms += naids;
|
|
}
|
|
}
|
|
|
|
std::vector<std::string> wvg_source_words;
|
|
std::vector<int> wvg_source_word_index; //idx in wvg_source_words -> idx of queryword
|
|
if(m_word_variations_config.m_languageSpecificWordVariations) {
|
|
for(int i=0; i<m_numWords; i++) {
|
|
const QueryWord *qw = &m_qwords[i];
|
|
if(qw->m_inQuotes) continue;
|
|
if(qw->m_wordSign == '+') continue;
|
|
if(qw->m_wordSign == '-') continue;
|
|
if(qw->m_fieldCode && qw->m_fieldCode != FIELD_TITLE )
|
|
continue;
|
|
if(qw->m_ignoreWord == IGNORE_FIELDNAME) continue;
|
|
// ignore if word weight is zero or synonym weight is zero
|
|
if(almostEqualFloat(qw->m_userWeightForWord,0))
|
|
continue;
|
|
if(almostEqualFloat(qw->m_userWeightForSynonym,0))
|
|
continue;
|
|
wvg_source_words.emplace_back(qw->m_word,qw->m_wordLen);
|
|
wvg_source_word_index.emplace_back(i);
|
|
}
|
|
auto wvg(WordVariationGenerator::get_generator(m_langId));
|
|
m_wordVariations = wvg->query_variations(wvg_source_words, m_word_variations_config.m_word_variations_weights, m_word_variations_config.m_word_variations_threshold);
|
|
numCandidateSynonyms += m_wordVariations.size();
|
|
if(!m_wordVariations.empty())
|
|
logTrace(g_conf.m_logTraceQuery, "word variations produced %d variants", (int)m_wordVariations.size());
|
|
else
|
|
logTrace(g_conf.m_logTraceQuery, "word variations didn't produce any");
|
|
} else
|
|
m_wordVariations.clear();
|
|
if(g_conf.m_logTraceQuery) {
|
|
logTrace(g_conf.m_logTraceQuery, "m_wordVariations.size()=%zu", m_wordVariations.size());
|
|
for(unsigned i=0; i<m_wordVariations.size(); i++)
|
|
logTrace(g_conf.m_logTraceQuery, " variation #%u: %s weight=%f src=[%d..%d)", i, m_wordVariations[i].word.c_str(), m_wordVariations[i].weight, m_wordVariations[i].source_word_start, m_wordVariations[i].source_word_end);
|
|
}
|
|
|
|
if(m_word_variations_config.m_lemmaWordVariations)
|
|
numCandidateSynonyms += 10;
|
|
|
|
m_numTermsUntruncated = numCandidatePhrases+numCandidateSingles+numCandidateSynonyms;
|
|
logTrace(g_conf.m_logTraceQuery, "m_numTermsUntruncated=%d (%d phrases, %d singles, %d synonyms)", m_numTermsUntruncated, numCandidatePhrases, numCandidateSingles, numCandidateSynonyms);
|
|
const int numQueryTerms = std::min(std::min(m_numTermsUntruncated,m_maxQueryTerms),ABS_MAX_QUERY_TERMS);
|
|
if(numQueryTerms!=m_numTermsUntruncated)
|
|
log(LOG_DEBUG, "m_numTermsUntruncated=%d (%d phrases, %d singles, %d synonyms), will be truncated to %d terms for query '%s'",
|
|
m_numTermsUntruncated, numCandidatePhrases, numCandidateSingles, numCandidateSynonyms, numQueryTerms,
|
|
m_filteredQuery.getBufStart());
|
|
|
|
// allocate the term buffer
|
|
if(numQueryTerms) {
|
|
int32_t need = numQueryTerms * sizeof(QueryTerm);
|
|
if ( ! m_queryTermBuf.reserve ( need ) )
|
|
return false;
|
|
m_queryTermBuf.setLabel("stkbuf3");
|
|
const char *pp = m_queryTermBuf.getBufStart();
|
|
m_qterms = (QueryTerm *)pp;
|
|
}
|
|
|
|
// call constructor on each one here
|
|
for(int32_t i = 0; i < numQueryTerms; i++) {
|
|
QueryTerm *qt = &m_qterms[i];
|
|
qt->constructor();
|
|
}
|
|
|
|
|
|
int32_t n = 0;
|
|
|
|
// do phrase terms
|
|
for(int32_t i = 0; i<m_numWords && n<numQueryTerms; i++) {
|
|
if(numCandidateSingles+numCandidatePhrases > m_maxQueryTerms) {
|
|
//we won't have room for both phrases and singles. Put in as many singles as possible. But phrases
|
|
//must come first in the list due to bad assumptions elsewhere in the code.
|
|
if(numQueryTerms - n - 1 < numCandidateSingles)
|
|
break;
|
|
}
|
|
|
|
QueryWord *qw = &m_qwords[i];
|
|
// skip if ignored... mdw...
|
|
if ( ! qw->m_bigramId ) continue;
|
|
if ( qw->m_ignorePhrase ) continue; // could be a repeat
|
|
// none if weight is absolute zero
|
|
if ( almostEqualFloat(qw->m_userWeightForPhrase, 0) )
|
|
continue;
|
|
|
|
QueryTerm *qt = &m_qterms[n];
|
|
qt->m_qword = qw ;
|
|
qt->m_piped = qw->m_piped;
|
|
qt->m_isPhrase = true ;
|
|
qt->m_synonymOf = NULL;
|
|
qt->m_ignored = false;
|
|
qt->m_term = NULL;
|
|
qt->m_termLen = 0;
|
|
qt->m_langIdBitsValid = false;
|
|
qt->m_langIdBits = 0;
|
|
// stop word? no, we're a phrase term
|
|
qt->m_isQueryStopWord = false;
|
|
// change in both places
|
|
qt->m_termId = qw->m_bigramId & TERMID_MASK;
|
|
qt->m_rawTermId = qw->m_rawPhraseId;
|
|
// boolean queries are not allowed term signs for phrases
|
|
// UNLESS it is a '*' soft require sign which we need for
|
|
// phrases like: "cat dog" AND pig
|
|
if ( m_isBoolean && qw->m_phraseSign != '*' ) {
|
|
qt->m_termSign = '\0';
|
|
}
|
|
// if not boolean, ensure to change signs in both places
|
|
else {
|
|
qt->m_termSign = qw->m_phraseSign;
|
|
}
|
|
|
|
qw->m_queryWordTerm = NULL;
|
|
// IndexTable.cpp uses this one
|
|
qt->m_inQuotes = qw->m_inQuotes;
|
|
// point to the string itself that is the phrase
|
|
qt->m_term = qw->m_word;
|
|
qt->m_termLen = qw->m_bigramLen;
|
|
|
|
// the QueryWord should have a direct link to the QueryTerm,
|
|
// at least for phrase, so we can OR in the bits of its
|
|
// constituents in the for loop below
|
|
qw->m_queryPhraseTerm = qt ;
|
|
// assign score weight, we're a phrase here
|
|
qt->m_termWeight = m_bigramWeight;
|
|
qt->m_userWeight = qw->m_userWeightForPhrase ;
|
|
qt->m_fieldCode = qw->m_fieldCode;
|
|
|
|
// stuff before a pipe always has a weight of 1
|
|
if ( qt->m_piped ) {
|
|
qt->m_userWeight = 1;
|
|
}
|
|
n++;
|
|
}
|
|
|
|
// now if we have enough room, do the singles
|
|
for(int32_t i = 0; i < m_numWords && n<numQueryTerms; i++) {
|
|
QueryWord *qw = &m_qwords[i];
|
|
|
|
if ( qw->m_ignoreWord &&
|
|
qw->m_ignoreWord != IGNORE_QSTOP) continue;
|
|
|
|
// ignore if in quotes and part of phrase, watch out
|
|
// for things like "word", a single word in quotes.
|
|
if ( qw->m_quoteStart >= 0 && qw->m_bigramId ) continue;
|
|
|
|
// if we are not start of quote and NOT in a phrase we
|
|
// must be the tailing word i guess.
|
|
// fixes '"john smith" -"bob dole"' from having
|
|
// smith and dole as query terms.
|
|
if ( qw->m_quoteStart >= 0 && qw->m_quoteStart != i )
|
|
continue;
|
|
|
|
// ignore if weight is absolute zero
|
|
if ( almostEqualFloat(qw->m_userWeightForWord,0) )
|
|
continue;
|
|
|
|
QueryTerm *qt = &m_qterms[n];
|
|
qt->m_qword = qw ;
|
|
qt->m_piped = qw->m_piped;
|
|
qt->m_isPhrase = false ;
|
|
qt->m_synonymOf = NULL;
|
|
// ignore some synonym terms if tf is too low
|
|
qt->m_ignored = qw->m_ignoreWord;
|
|
// stop word? no, we're a phrase term
|
|
qt->m_isQueryStopWord = qw->m_isQueryStopWord;
|
|
// change in both places
|
|
qt->m_termId = qw->m_wordId & TERMID_MASK;
|
|
qt->m_rawTermId = qw->m_rawWordId;
|
|
// boolean queries are not allowed term signs
|
|
if ( m_isBoolean ) {
|
|
qt->m_termSign = '\0';
|
|
// boolean fix for "health OR +sports" because
|
|
// the + there means exact word match, no synonyms.
|
|
if ( qw->m_wordSign == '+' ) {
|
|
qt->m_termSign = qw->m_wordSign;
|
|
}
|
|
}
|
|
// if not boolean, ensure to change signs in both places
|
|
else {
|
|
qt->m_termSign = qw->m_wordSign;
|
|
}
|
|
int32_t pw = i-1;
|
|
// . back up until word that contains quote if in a quoted
|
|
// phrase
|
|
// . UOR can only support two word phrases really...
|
|
if (m_qwords[i].m_quoteStart >= 0)
|
|
pw = m_qwords[i].m_quoteStart ;
|
|
if ( pw > 0 ) pw--;
|
|
|
|
// back two more if field
|
|
int32_t fieldStart=-1;
|
|
int32_t fieldLen=0;
|
|
|
|
if(pw == 0 && m_qwords[pw].m_ignoreWord==IGNORE_FIELDNAME)
|
|
fieldStart = pw;
|
|
|
|
if(pw > 0 && m_qwords[pw-1].m_ignoreWord==IGNORE_FIELDNAME) {
|
|
pw -= 1;
|
|
fieldStart = pw;
|
|
}
|
|
while(pw > 0 && m_qwords[pw].m_ignoreWord == IGNORE_FIELDNAME) {
|
|
pw--;
|
|
fieldStart = pw;
|
|
}
|
|
|
|
|
|
// skip if it is punct. fixes queries like
|
|
// "(this OR that)" from including '(' or from including
|
|
// a space.
|
|
if ( fieldStart >-1 &&
|
|
m_qwords[fieldStart].m_isPunct &&
|
|
fieldStart+1<m_numWords )
|
|
fieldStart++;
|
|
|
|
if (fieldStart > -1) {
|
|
pw = i;
|
|
while (pw < m_numWords && m_qwords[pw].m_fieldCode)
|
|
pw++;
|
|
|
|
fieldLen = m_qwords[pw-1].m_word +
|
|
m_qwords[pw-1].m_wordLen -
|
|
m_qwords[fieldStart].m_word;
|
|
}
|
|
qw->m_queryWordTerm = qt;
|
|
// IndexTable.cpp uses this one
|
|
qt->m_inQuotes = qw->m_inQuotes;
|
|
// point to the string itself that is the word
|
|
|
|
if (fieldLen > 0) {
|
|
qt->m_term = m_qwords[fieldStart].m_word;
|
|
qt->m_termLen = fieldLen;
|
|
// fix for query
|
|
// text:"" foo bar ""
|
|
if ( pw-1 < i ) {
|
|
log("query: bad query %s",m_originalQuery.getBufStart());
|
|
g_errno = EMALFORMEDQUERY;
|
|
return false;
|
|
}
|
|
// skip past the end of the field value
|
|
i = pw-1;
|
|
}
|
|
else {
|
|
qt->m_termLen = qw->m_wordLen;
|
|
qt->m_term = qw->m_word;
|
|
}
|
|
|
|
// assign score weight, we're a single-term here
|
|
qt->m_termWeight = 1.0;
|
|
qt->m_userWeight = qw->m_userWeightForWord;
|
|
qt->m_fieldCode = qw->m_fieldCode;
|
|
qt->m_userNotRequired = qw->m_userNotRequiredForWord;
|
|
|
|
// stuff before a pipe always has a weight of 1
|
|
if ( qt->m_piped ) {
|
|
qt->m_userWeight = 1;
|
|
}
|
|
n++;
|
|
}
|
|
|
|
// Handle shared explicit bits
|
|
for ( int32_t i = 0; i < n ; i++ ){
|
|
QueryTerm *qt = &m_qterms[i];
|
|
// assume not in a phrase
|
|
qt->m_rightPhraseTermNum = -1;
|
|
qt->m_leftPhraseTermNum = -1;
|
|
qt->m_rightPhraseTerm = NULL;
|
|
qt->m_leftPhraseTerm = NULL;
|
|
}
|
|
|
|
// . set m_inPhrase
|
|
for (int32_t i = 0; i < m_numWords ; i++ ) {
|
|
const QueryWord *qw = &m_qwords[i];
|
|
QueryTerm *qt = qw->m_queryWordTerm;
|
|
if (!qt) continue;
|
|
// set flag if in a a phrase, and set phrase term num
|
|
if ( qw->m_queryPhraseTerm ) {
|
|
QueryTerm *pt = qw->m_queryPhraseTerm;
|
|
qt->m_rightPhraseTermNum = pt - m_qterms;
|
|
qt->m_rightPhraseTerm = pt;
|
|
}
|
|
// if we're in the middle of the phrase
|
|
int32_t pn = qw->m_leftPhraseStart;
|
|
// convert word to its phrase QueryTerm ptr, if any
|
|
QueryTerm *tt = NULL;
|
|
if ( pn >= 0 ) tt = m_qwords[pn].m_queryPhraseTerm;
|
|
if ( tt ) {
|
|
qt->m_leftPhraseTermNum = tt - m_qterms;
|
|
qt->m_leftPhraseTerm = tt;
|
|
}
|
|
// . there might be some phrase term that actually contains
|
|
// the same word as we are, but a different occurence
|
|
// . like '"knowledge management" AND NOT management' query
|
|
// . made it from "j < i" into "j < m_numWords" because
|
|
// 'test "test bed"' was not working but '"test bed" test'
|
|
// was working.
|
|
for ( int32_t j = 0 ; j < m_numWords ; j++ ) {
|
|
// must be our same wordId (same word, different occ.)
|
|
const QueryWord *qw2 = &m_qwords[j];
|
|
if ( qw2->m_wordId != qw->m_wordId ) continue;
|
|
// get first word in the phrase that jth word is in
|
|
int32_t pn2 = qw2->m_leftPhraseStart;
|
|
// we might be the guy that starts it!
|
|
if ( pn2 < 0 && qw2->m_quoteStart != -1 ) pn2 = j;
|
|
// if neither is the case, skip this query word
|
|
if ( pn2 < 0 ) continue;
|
|
// he implies us!
|
|
QueryTerm *tt2 = m_qwords[pn2].m_queryPhraseTerm;
|
|
if ( tt2 ) {
|
|
qt->m_leftPhraseTermNum = tt2 - m_qterms;
|
|
qt->m_leftPhraseTerm = tt2;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
if(g_conf.m_logTraceQuery) {
|
|
logTrace(g_conf.m_logTraceQuery, "query-terms before word variations:");
|
|
for(int i=0; i<n; i++)
|
|
logTrace(g_conf.m_logTraceQuery, " query-term #%d: termid=%15" PRId64" '%*.*s'", i, m_qterms[i].m_termId, m_qterms[i].m_termLen,m_qterms[i].m_termLen,m_qterms[i].m_term);
|
|
}
|
|
|
|
////////////
|
|
//
|
|
// . add synonym query terms now
|
|
// . skip this part if language is unknown i guess
|
|
//
|
|
////////////
|
|
|
|
if(m_word_variations_config.m_wiktionaryWordVariations) {
|
|
int64_t to = hash64n("to");
|
|
for(int32_t i = 0; i<m_numWords && n<numQueryTerms; i++) {
|
|
// get query word
|
|
QueryWord *qw = &m_qwords[i];
|
|
// skip if in quotes, we will not get synonyms for it
|
|
if ( qw->m_inQuotes ) continue;
|
|
// skip if has plus sign in front
|
|
if ( qw->m_wordSign == '+' ) continue;
|
|
// not '-' either i guess
|
|
if ( qw->m_wordSign == '-' ) continue;
|
|
// no url: stuff, maybe only title
|
|
if ( qw->m_fieldCode && qw->m_fieldCode != FIELD_TITLE )
|
|
continue;
|
|
// skip if ignored like a stopword (stop to->too)
|
|
//if ( qw->m_ignoreWord ) continue;
|
|
// ignore title: etc. words, they are field names
|
|
if ( qw->m_ignoreWord == IGNORE_FIELDNAME ) continue;
|
|
// ignore boolean operators
|
|
if ( qw->m_ignoreWord ) continue;// IGNORE_BOOLOP
|
|
// ignore if word weight is zero or synonym weight is zero
|
|
if(almostEqualFloat(qw->m_userWeightForWord,0))
|
|
continue;
|
|
if(almostEqualFloat(qw->m_userWeightForSynonym,0))
|
|
continue;
|
|
// no, hurts 'Greencastle IN economic development'
|
|
if ( qw->m_wordId == to ) continue;
|
|
// single letters...
|
|
if ( qw->m_wordLen == 1 ) continue;
|
|
// set the synonyms for this word
|
|
char tmpBuf [ TMPSYNBUFSIZE ];
|
|
int32_t naids = syn.getSynonyms ( &m_tr,
|
|
i ,
|
|
// language of the query.
|
|
// 0 means unknown. if this
|
|
// is 0 we sample synonyms
|
|
// from all languages.
|
|
m_langId ,
|
|
tmpBuf );
|
|
// if no synonyms, all done
|
|
if ( naids <= 0 ) continue;
|
|
// sanity
|
|
if ( naids > MAX_SYNS ) { g_process.shutdownAbort(true); }
|
|
// now make the buffer to hold them for us
|
|
qw->m_synWordBuf.setLabel("qswbuf");
|
|
qw->m_synWordBuf.safeMemcpy ( &syn.m_synWordBuf );
|
|
// get the term for this word
|
|
QueryTerm *origTerm = qw->m_queryWordTerm;
|
|
// loop over synonyms for word #i now
|
|
for(int32_t j = 0; j < naids && n<numQueryTerms; j++) {
|
|
// this happens for 'da da da'
|
|
if ( ! origTerm ) continue;
|
|
|
|
// add that query term
|
|
QueryTerm *qt = &m_qterms[n];
|
|
qt->m_qword = qw; // NULL;
|
|
qt->m_piped = qw->m_piped;
|
|
qt->m_isPhrase = false ;
|
|
qt->m_langIdBits = 0;
|
|
// synonym of this term...
|
|
qt->m_synonymOf = origTerm;
|
|
// nuke this crap since it was done above and we
|
|
// missed out!
|
|
qt->m_rightPhraseTermNum = -1;
|
|
qt->m_leftPhraseTermNum = -1;
|
|
qt->m_rightPhraseTerm = NULL;
|
|
qt->m_leftPhraseTerm = NULL;
|
|
// need this for displaying language of syn in
|
|
// the json/xml feed in PageResults.cpp
|
|
qt->m_langIdBitsValid = true;
|
|
int langId = syn.m_langIds[j];
|
|
uint64_t langBit = (uint64_t)1 << langId;
|
|
if ( langId >= 64 ) langBit = 0;
|
|
qt->m_langIdBits |= langBit;
|
|
// need this for Matches.cpp
|
|
qt->m_synWids0 = syn.m_wids0[j];
|
|
qt->m_synWids1 = syn.m_wids1[j];
|
|
int32_t na = syn.m_numAlnumWords[j];
|
|
// how many words were in the base we used to
|
|
// get the synonym. i.e. if the base is "new jersey"
|
|
// then it's 2! and the synonym "nj" has one alnum
|
|
// word.
|
|
int32_t ba = syn.m_numAlnumWordsInBase[j];
|
|
qt->m_numAlnumWordsInSynonym = na;
|
|
|
|
// crap, "nj" is a synonym of the PHRASE TERM
|
|
// bigram "new jersey" not of the single word term
|
|
// "new" so fix that.
|
|
if ( ba == 2 && origTerm->m_rightPhraseTerm )
|
|
qt->m_synonymOf = origTerm->m_rightPhraseTerm;
|
|
|
|
// ignore some synonym terms if tf is too low
|
|
qt->m_ignored = qw->m_ignoreWord;
|
|
// stop word? no, we're a phrase term
|
|
qt->m_isQueryStopWord = qw->m_isQueryStopWord;
|
|
// change in both places
|
|
int64_t wid = syn.m_aids[j];
|
|
// might be in a title: field or something
|
|
if ( qw->m_prefixHash ) {
|
|
int64_t ph = qw->m_prefixHash;
|
|
wid= hash64h(wid,ph);
|
|
}
|
|
qt->m_termId = wid & TERMID_MASK;
|
|
qt->m_rawTermId = syn.m_aids[j];
|
|
// boolean queries are not allowed term signs
|
|
if ( m_isBoolean ) {
|
|
qt->m_termSign = '\0';
|
|
// boolean fix for "health OR +sports" because
|
|
// the + there means exact word match, no syns
|
|
if ( qw->m_wordSign == '+' ) {
|
|
qt->m_termSign = qw->m_wordSign;
|
|
}
|
|
}
|
|
// if not bool, ensure to change signs in both places
|
|
else {
|
|
qt->m_termSign = qw->m_wordSign;
|
|
}
|
|
// IndexTable.cpp uses this one
|
|
qt->m_inQuotes = qw->m_inQuotes;
|
|
// usually this is right
|
|
const char *ptr = syn.m_termPtrs[j];
|
|
// buf if it is NULL that means we transformed the
|
|
// word by like removing accent marks and stored
|
|
// it in m_synWordBuf, as opposed to just pointing
|
|
// to a line in memory of wiktionary-buf.txt.
|
|
if ( ! ptr ) {
|
|
int32_t off = syn.m_termOffs[j];
|
|
if ( off < 0 ) {
|
|
g_process.shutdownAbort(true); }
|
|
if ( off > qw->m_synWordBuf.length() ) {
|
|
g_process.shutdownAbort(true); }
|
|
// use QueryWord::m_synWordBuf which should
|
|
// be persistent and not disappear like
|
|
// syn.m_synWordBuf.
|
|
ptr = qw->m_synWordBuf.getBufStart() + off;
|
|
}
|
|
// point to the string itself that is the word
|
|
qt->m_term = ptr;
|
|
qt->m_termLen = syn.m_termLens[j];
|
|
// assign score weight, we're a synonym here
|
|
qt->m_termWeight = m_synonymWeight;
|
|
qt->m_userWeight = qw->m_userWeightForSynonym;
|
|
qt->m_fieldCode = qw->m_fieldCode;
|
|
|
|
// stuff before a pipe always has a weight of 1
|
|
if ( qt->m_piped ) {
|
|
qt->m_userWeight = 1;
|
|
}
|
|
// otherwise, add it
|
|
n++;
|
|
}
|
|
}
|
|
}
|
|
|
|
if(m_word_variations_config.m_languageSpecificWordVariations) {
|
|
logTrace(g_conf.m_logTraceQuery, "Word variations: %zu", m_wordVariations.size());
|
|
for(unsigned i=0; i<m_wordVariations.size() && n<numQueryTerms; i++) {
|
|
auto const &word_variation(m_wordVariations[i]);
|
|
int wordStartIdx = wvg_source_word_index[word_variation.source_word_start];
|
|
int wordEndIdx = wvg_source_word_index[word_variation.source_word_end-1];
|
|
logTrace(g_conf.m_logTraceQuery, " Word variation #%u: '%s' weight=%f src=[%u..%u]", i, word_variation.word.c_str(), word_variation.weight, wordStartIdx, wordEndIdx);
|
|
QueryWord *qw = &m_qwords[wordStartIdx];
|
|
if((unsigned)qw->m_wordLen==word_variation.word.length() &&
|
|
memcmp(qw->m_word, word_variation.word.data(), word_variation.word.length())==0)
|
|
{
|
|
//Variation is the same as the base word. The word-variation-plugin is allowed to produce that.
|
|
continue; //skip
|
|
}
|
|
QueryTerm *origTerm = qw->m_queryWordTerm;
|
|
|
|
//handle if the word variant is a bigram/phrase
|
|
bool isPhrase = false;
|
|
if(wordEndIdx-wordStartIdx>1) {
|
|
logTrace(g_conf.m_logTraceQuery, "Word variation '%s' spans more than 1 word", word_variation.word.c_str());
|
|
if(wordEndIdx-wordStartIdx==2) {
|
|
//find bigram pointing to first word
|
|
QueryTerm *bigramQueryTerm = NULL;
|
|
for(int j=0; j<n && !bigramQueryTerm; j++) {
|
|
if(m_qterms[j].m_qword==qw && m_qterms[j].m_isPhrase)
|
|
bigramQueryTerm = &m_qterms[j];
|
|
}
|
|
if(bigramQueryTerm) {
|
|
logTrace(g_conf.m_logTraceQuery, "Word variation covers '%.*s'", bigramQueryTerm->m_termLen, bigramQueryTerm->m_term);
|
|
origTerm = bigramQueryTerm;
|
|
isPhrase = true;
|
|
} else
|
|
log(LOG_LOGIC,"Word variation '%s' bigram/phrase didn't find base bigram", word_variation.word.c_str());
|
|
} else {
|
|
log(LOG_LOGIC,"Word variation '%s' spans more than 2 words. This is not supported (yet)", word_variation.word.c_str());
|
|
}
|
|
}
|
|
|
|
// add that query term
|
|
QueryTerm *qt = &m_qterms[n];
|
|
qt->m_qword = qw; // NULL;
|
|
qt->m_piped = qw->m_piped;
|
|
qt->m_isPhrase = isPhrase;
|
|
qt->m_langIdBits = 0;
|
|
// synonym of this term...
|
|
qt->m_synonymOf = origTerm;
|
|
// nuke this crap since it was done above and we
|
|
// missed out!
|
|
qt->m_rightPhraseTermNum = -1;
|
|
qt->m_leftPhraseTermNum = -1;
|
|
qt->m_rightPhraseTerm = NULL;
|
|
qt->m_leftPhraseTerm = NULL;
|
|
// need this for displaying language of syn in
|
|
// the json/xml feed in PageResults.cpp
|
|
qt->m_langIdBitsValid = true;
|
|
//int langId = syn.m_langIds[j]; //syn-todo?
|
|
//uint64_t langBit = (uint64_t)1 << langId; //syn-todo?
|
|
//if(langId >= 64) langBit = 0; //syn-todo?
|
|
//qt->m_langIdBits |= langBit; //syn-todo?
|
|
// need this for Matches.cpp
|
|
qt->m_synWids0 = 0;
|
|
qt->m_synWids1 = 0;
|
|
qt->m_numAlnumWordsInSynonym = 0;
|
|
|
|
// ignore some synonym terms if tf is too low
|
|
qt->m_ignored = qw->m_ignoreWord;
|
|
// stop word? no, we're a phrase term
|
|
qt->m_isQueryStopWord = qw->m_isQueryStopWord;
|
|
// change in both places
|
|
//int64_t wid = syn.m_aids[j];
|
|
int64_t wid = hash64Lower_utf8_nospaces(word_variation.word.data(), word_variation.word.length());
|
|
// might be in a title: field or something
|
|
if(qw->m_prefixHash) {
|
|
int64_t ph = qw->m_prefixHash;
|
|
wid= hash64h(wid,ph);
|
|
}
|
|
qt->m_termId = wid & TERMID_MASK;
|
|
//qt->m_rawTermId = syn.m_aids[j]; //syn-todo?
|
|
// boolean queries are not allowed term signs
|
|
if(m_isBoolean) {
|
|
qt->m_termSign = '\0';
|
|
// boolean fix for "health OR +sports" because
|
|
// the + there means exact word match, no syns
|
|
if(qw->m_wordSign == '+') {
|
|
qt->m_termSign = qw->m_wordSign;
|
|
}
|
|
}
|
|
// if not bool, ensure to change signs in both places
|
|
else {
|
|
qt->m_termSign = qw->m_wordSign;
|
|
}
|
|
// IndexTable.cpp uses this one
|
|
qt->m_inQuotes = qw->m_inQuotes;
|
|
// point to the string itself that is the word
|
|
qt->m_term = word_variation.word.data();
|
|
qt->m_termLen = word_variation.word.length();
|
|
// assign score weight
|
|
qt->m_termWeight = word_variation.weight;
|
|
qt->m_userWeight = qw->m_userWeightForSynonym;
|
|
qt->m_fieldCode = qw->m_fieldCode ;
|
|
// stuff before a pipe always has a weight of 1
|
|
if(qt->m_piped) {
|
|
qt->m_userWeight = 1;
|
|
}
|
|
// otherwise, add it
|
|
n++;
|
|
}
|
|
}
|
|
|
|
if(m_word_variations_config.m_lemmaWordVariations && m_langId==langDanish) {
|
|
logTrace(g_conf.m_logTraceQuery, "Lexicon-based lemma synonyms");
|
|
for(int32_t i = 0; i<m_numWords && n<numQueryTerms; i++) {
|
|
if(!m_tr[i].is_alfanum)
|
|
continue;
|
|
std::string w(m_tr[i].token_start,m_tr[i].token_len);
|
|
logTrace(g_conf.m_logTraceQuery, "Checking lemma for '%s'", w.c_str());
|
|
auto le = lemma_lexicon->lookup(w);
|
|
if(!le) {
|
|
//Not found as-is in lexicon. Try lowercase in case it is a capitalized word
|
|
char lowercase_word[128];
|
|
if(w.size()<sizeof(lowercase_word)) {
|
|
size_t sz = to_lower_utf8(lowercase_word,lowercase_word+sizeof(lowercase_word), w.data(), w.data()+w.size());
|
|
lowercase_word[sz] = '\0';
|
|
if(sz!=w.size() || memcmp(w.data(),lowercase_word,w.size())!=0) {
|
|
le = lemma_lexicon->lookup(lowercase_word);
|
|
}
|
|
}
|
|
}
|
|
if(!le) {
|
|
//Not found as-is in lexicon. Try capitalized in case it is a lowercase or uppercase word
|
|
char capitalized_word[128];
|
|
if(w.size()<sizeof(capitalized_word)) {
|
|
size_t sz = to_capitalized_utf8(capitalized_word,capitalized_word+sizeof(capitalized_word), w.data(), w.data()+w.size());
|
|
capitalized_word[sz] = '\0';
|
|
if(sz!=w.size() || memcmp(w.data(),capitalized_word,w.size())!=0) {
|
|
w = capitalized_word;
|
|
le = lemma_lexicon->lookup(w);
|
|
}
|
|
}
|
|
}
|
|
if(!le) {
|
|
//Not found as-is in lexicon. Try uppercasing it
|
|
char uppercase_word[128];
|
|
if(w.size()<sizeof(uppercase_word)) {
|
|
size_t sz = to_upper_utf8(uppercase_word,uppercase_word+sizeof(uppercase_word), w.data(), w.data()+w.size());
|
|
uppercase_word[sz] = '\0';
|
|
if(sz!=w.size() || memcmp(w.data(),uppercase_word,w.size())!=0) {
|
|
w = uppercase_word;
|
|
le = lemma_lexicon->lookup(w);
|
|
}
|
|
}
|
|
}
|
|
if(!le)
|
|
continue; //unknown word
|
|
auto wf = le->find_base_wordform();
|
|
if(!wf)
|
|
continue;
|
|
if(wf->written_form_length==w.size() && memcmp(wf->written_form,w.data(),w.size())==0)
|
|
continue; //already base form)
|
|
logTrace(g_conf.m_logTraceQuery, "Generating synonym from lemma: %s -> %.*s", w.c_str(), wf->written_form_length,wf->written_form);
|
|
|
|
QueryWord *qw = &m_qwords[i];
|
|
QueryTerm *origTerm = qw->m_queryWordTerm;
|
|
|
|
// add that query term
|
|
QueryTerm *qt = &m_qterms[n];
|
|
qt->m_qword = qw; // NULL;
|
|
qt->m_piped = qw->m_piped;
|
|
qt->m_isPhrase = false;
|
|
qt->m_langIdBits = 0;
|
|
// synonym of this term...
|
|
qt->m_synonymOf = origTerm;
|
|
// nuke this crap since it was done above and we
|
|
// missed out!
|
|
qt->m_rightPhraseTermNum = -1;
|
|
qt->m_leftPhraseTermNum = -1;
|
|
qt->m_rightPhraseTerm = NULL;
|
|
qt->m_leftPhraseTerm = NULL;
|
|
// need this for displaying language of syn in
|
|
// the json/xml feed in PageResults.cpp
|
|
qt->m_langIdBitsValid = true;
|
|
//int langId = syn.m_langIds[j]; //syn-todo?
|
|
//uint64_t langBit = (uint64_t)1 << langId; //syn-todo?
|
|
//if(langId >= 64) langBit = 0; //syn-todo?
|
|
//qt->m_langIdBits |= langBit; //syn-todo?
|
|
// need this for Matches.cpp
|
|
qt->m_synWids0 = 0;
|
|
qt->m_synWids1 = 0;
|
|
qt->m_numAlnumWordsInSynonym = 0;
|
|
|
|
// ignore some synonym terms if tf is too low
|
|
qt->m_ignored = qw->m_ignoreWord;
|
|
// stop word? no, we're a phrase term
|
|
qt->m_isQueryStopWord = qw->m_isQueryStopWord;
|
|
// change in both places
|
|
//int64_t wid = syn.m_aids[j];
|
|
int64_t wid = hash64Lower_utf8_nospaces(wf->written_form,wf->written_form_length);
|
|
// might be in a title: field or something
|
|
if(qw->m_prefixHash) {
|
|
int64_t ph = qw->m_prefixHash;
|
|
wid= hash64h(wid,ph);
|
|
}
|
|
qt->m_termId = wid & TERMID_MASK;
|
|
//qt->m_rawTermId = syn.m_aids[j]; //syn-todo?
|
|
// boolean queries are not allowed term signs
|
|
if(m_isBoolean) {
|
|
qt->m_termSign = '\0';
|
|
// boolean fix for "health OR +sports" because
|
|
// the + there means exact word match, no syns
|
|
if(qw->m_wordSign == '+') {
|
|
qt->m_termSign = qw->m_wordSign;
|
|
}
|
|
}
|
|
// if not bool, ensure to change signs in both places
|
|
else {
|
|
qt->m_termSign = qw->m_wordSign;
|
|
}
|
|
// IndexTable.cpp uses this one
|
|
qt->m_inQuotes = qw->m_inQuotes;
|
|
// point to the string itself that is the word
|
|
qt->m_term = wf->written_form;
|
|
qt->m_termLen = wf->written_form_length;
|
|
// assign score weight
|
|
qt->m_termWeight = m_synonymWeight;
|
|
qt->m_userWeight = qw->m_userWeightForSynonym;
|
|
qt->m_fieldCode = qw->m_fieldCode ;
|
|
// stuff before a pipe always has a weight of 1
|
|
if(qt->m_piped) {
|
|
qt->m_userWeight = 1;
|
|
}
|
|
// otherwise, add it
|
|
n++;
|
|
}
|
|
}
|
|
|
|
//Merge duplicated synonyms.
|
|
//If one of the above synonym-generations produced the same word (eg. from wiktionary, word-variations and as lemma) then we want to use
|
|
//the one with highest weight
|
|
for(int i=0; i<n; i++) {
|
|
if(m_qterms[i].m_synonymOf) {
|
|
//it's a synonym. Are there other synonyms on the same base word with the same form? If so then merge/delete
|
|
for(int j=i+1; j<n; ) {
|
|
if(m_qterms[j].m_synonymOf == m_qterms[i].m_synonymOf &&
|
|
m_qterms[j].m_termLen == m_qterms[i].m_termLen &&
|
|
memcmp(m_qterms[j].m_term,m_qterms[i].m_term,m_qterms[j].m_termLen)==0)
|
|
{
|
|
//Identical synonyms of same base word
|
|
//note: direct memcmp() test. Downside is that we don't eliminate uppercase/lowercase duplicates, but neither
|
|
//do we get into trouble with German eszet, Lithuanian i, ligatures, titlecase, etc.
|
|
logTrace(g_conf.m_logTraceQuery, "merging identical synonyms '%.*s' for word '%.*s'", m_qterms[i].m_termLen,m_qterms[i].m_term, m_qterms[i].m_synonymOf->m_termLen,m_qterms[i].m_synonymOf->m_term);
|
|
m_qterms[i].m_termWeight = std::max(m_qterms[i].m_termWeight,m_qterms[j].m_termWeight);
|
|
m_qterms[i].m_userWeight = std::max(m_qterms[i].m_userWeight,m_qterms[j].m_userWeight);
|
|
memmove(m_qterms+j, m_qterms+j+1, sizeof(m_qterms[0])*(n-j-1));
|
|
n--;
|
|
} else
|
|
j++;
|
|
}
|
|
}
|
|
}
|
|
|
|
m_numTerms = n;
|
|
|
|
if ( n > ABS_MAX_QUERY_TERMS ) { g_process.shutdownAbort(true); }
|
|
|
|
// . if only have one term and it is a signless phrase, make it signed
|
|
// . don't forget to set m_termSigns too!
|
|
if ( n == 1 && m_qterms[0].m_isPhrase && ! m_qterms[0].m_termSign ) {
|
|
m_qterms[0].m_termSign = '*';
|
|
}
|
|
|
|
// . now set m_phrasePart for Summary.cpp's hackfix filter
|
|
// . only set this for the non-phrase terms, since keepAllSingles is
|
|
// set to true when setting the Query for Summary.cpp::set in order
|
|
// to match the singles
|
|
for ( int32_t i = 0 ; i < m_numTerms ; i++ ) {
|
|
// skip cd-rom too, if not in quotes
|
|
if ( ! m_qterms[i].m_inQuotes ) continue;
|
|
// is next term also in a quoted phrase?
|
|
if ( i - 1 < 0 ) continue;
|
|
//if ( ! m_qterms[i+1].m_isPhrase ) continue;
|
|
if ( ! m_qterms[i-1].m_inQuotes ) continue;
|
|
// are we in the same quoted phrase?
|
|
if ( m_qterms[i+0].m_qword->m_quoteStart !=
|
|
m_qterms[i-1].m_qword->m_quoteStart ) continue;
|
|
}
|
|
|
|
// if we have '+test -test':
|
|
//if ( negativeBits & requiredBits )
|
|
// m_numTerms = 0;
|
|
|
|
for ( int32_t i = 0 ; i < m_numTerms ; i++ ) {
|
|
QueryTerm *qt = &m_qterms[i];
|
|
// assume not required
|
|
qt->m_isRequired = false;
|
|
// skip signless phrases
|
|
if ( qt->m_isPhrase && qt->m_termSign == '\0' ) continue;
|
|
if ( qt->m_isPhrase && qt->m_termSign == '*' ) continue;
|
|
if ( qt->m_synonymOf ) continue;
|
|
// IGNORE_QSTOP?
|
|
if ( qt->m_ignored ) continue;
|
|
|
|
// user specified "[nrw]" before word
|
|
if( qt->m_userNotRequired) continue;
|
|
|
|
// mark it
|
|
qt->m_isRequired = true;
|
|
}
|
|
|
|
|
|
//If there are two highfreqterms in a row then PosdbTable will ignore the bigram of them because it can't tie the bigram to any required term.
|
|
//Example: "key west mystery writers fest" where "key" and "west" are highfreqterms, and therefore postdbtable will ignore the bigram "key+west".
|
|
//Options:
|
|
// 1: do nothing
|
|
// 2: mark the bigram as ignored
|
|
// 3: mark the bigram as required
|
|
// 4: rewrite Query and the queryterminfo handling in posdbtable so the bigram can be optional.
|
|
//We do (3) because it will likely filter out more bad results than good results. This is a hack because marking a bigram as required normally
|
|
//requires us to to be sure those two words are connected, but for "key west" we are guessing.
|
|
//TODO: reqwrite query+postdbtable so qword/qterm can be optional, etc.
|
|
//The structure of qwords+qterms make sthis code unnecessarily clumsy
|
|
for(int i=0; i+2<m_numWords; i++) {
|
|
if(m_qwords[i ].m_ignoreWord==IGNORE_HIGHFREMTERM &&
|
|
m_qwords[i+2].m_ignoreWord==IGNORE_HIGHFREMTERM)
|
|
{
|
|
if(m_qwords[i].m_queryPhraseTerm && m_qwords[i].m_queryPhraseTerm->m_isPhrase) {
|
|
logTrace(g_conf.m_logTraceQuery, "query-words #%d (%.*s) and #%d (%.*s) are both high-freq-terms. Marking bigram as required",
|
|
i, m_qwords[i].m_wordLen, m_qwords[i].m_word,
|
|
i+2, m_qwords[i+2].m_wordLen, m_qwords[i+2].m_word);
|
|
m_qwords[i].m_queryPhraseTerm->m_isRequired = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
//workaround/hack for double-highfreqterm searchs, such as "of a" or "the the" or "the who"
|
|
if(m_numWords==3 &&
|
|
m_qwords[0].m_ignoreWord==IGNORE_HIGHFREMTERM &&
|
|
m_qwords[2].m_ignoreWord==IGNORE_HIGHFREMTERM &&
|
|
m_numTerms==1 &&
|
|
!m_qterms[0].m_isRequired)
|
|
{
|
|
log(LOG_DEBUG, "query: Looks like a highfreqterm-highfreqterm query type. Requiring one-and-only QueryTerm/bigram");
|
|
m_qterms[0].m_isRequired = true;
|
|
//todo: we should investigate if QueryTerm::m_isRequired actually has any effect. It is used
|
|
//in a single place in PosdbTable for not generating a QueryTermInfo, but it appears it works
|
|
//fine even with the QTI.
|
|
}
|
|
|
|
//if all words are high-freq-terms then we have to mark the generated bigrams as required, otherwise PosdbTable.cpp gets unhappy and
|
|
//logs "no required terms in query!"
|
|
bool allAlnumWordsAreIgnored = true;
|
|
for(int i=0; i<m_numWords; i++) {
|
|
if(is_alnum_utf8_string(m_qwords[i].m_word,m_qwords[i].m_word+m_qwords[i].m_wordLen) &&
|
|
(m_qwords[i].m_ignoreWord!=IGNORE_HIGHFREMTERM && m_qwords[i].m_ignoreWord!=IGNORE_QSTOP))
|
|
allAlnumWordsAreIgnored = false;
|
|
}
|
|
if(allAlnumWordsAreIgnored) {
|
|
log(LOG_DEBUG, "query: all alfanum-terms are ignored (highfreq/qstop). Marking bigrams as required");
|
|
for(int i=0; i<m_numTerms; i++) {
|
|
if(m_qterms[i].m_isPhrase)
|
|
m_qterms[i].m_isRequired = true;
|
|
}
|
|
}
|
|
|
|
// required quoted phrase terms
|
|
for ( int32_t i = 0 ; i < m_numTerms ; i++ ) {
|
|
QueryTerm *qt = &m_qterms[i];
|
|
// quoted phrase?
|
|
if ( ! qt->m_isPhrase ) continue;
|
|
if ( ! qt->m_inQuotes ) continue;
|
|
// mark it
|
|
qt->m_isRequired = true;
|
|
}
|
|
|
|
|
|
// . for query 'to be or not to be shakespeare'
|
|
// require 'tobe' 'beor' 'tobe' because
|
|
// they are bigrams in the wikipedia phrase 'to be or not to be'
|
|
// and they all consist solely of query stop words. as of
|
|
// 8/20/2012 i took 'not' off the query stop word list.
|
|
// . require bigrams that consist of 2 query stop words and
|
|
// are in a wikipedia phrase. set termSign to '+' i guess?
|
|
// . for 'in the nick' , a wiki phrase, make "in the" required
|
|
// and give a big bonus for "the nick" below.
|
|
for ( int32_t i = 0 ; i < m_numTerms ; i++ ) {
|
|
QueryTerm *qt = &m_qterms[i];
|
|
// don't require if negative
|
|
if ( qt->m_termSign == '-' ) continue;
|
|
// only check bigrams here
|
|
if ( ! qt->m_isPhrase ) continue;
|
|
// get the query word that starts this phrase
|
|
const QueryWord *qw1 = qt->m_qword;
|
|
// must be in a wikiphrase
|
|
if ( qw1->m_wikiPhraseId <= 0 ) continue;
|
|
// what query word # is that?
|
|
int32_t qwn = qw1 - m_qwords;
|
|
// get the next alnum word after that
|
|
// assume its the last word in our bigram phrase
|
|
const QueryWord *qw2 = &m_qwords[qwn+2];
|
|
// must be in same wikiphrase
|
|
if ( qw2->m_wikiPhraseId != qw1->m_wikiPhraseId ) continue;
|
|
// must be two stop words
|
|
if ( ! qw1->m_isQueryStopWord ) continue;
|
|
if ( ! qw2->m_isQueryStopWord ) continue;
|
|
// mark it
|
|
qt->m_isRequired = true;
|
|
}
|
|
|
|
// . for query 'to be or not to be shakespeare'
|
|
// give big bonus for 'ornot' and 'notto' bigram terms because
|
|
// the single terms 'or' and 'to' are ignored and because
|
|
// 'to be or not to be' is a wikipedia phrase
|
|
// . on 8/20/2012 i took 'not' off the query stop word list.
|
|
// . now give a big bonus for bigrams whose two terms are in the
|
|
// same wikipedia phrase and one and only one of the terms in
|
|
// the bigram is a query stop word
|
|
// . in general 'ornot' is considered a "synonym" of 'not' and
|
|
// gets hit with a .90 score factor, but that should never
|
|
// happen, it should be 1.00 and in this special case it should
|
|
// be 1.20
|
|
// . so for 'time enough for love' the phrase term "enough for"
|
|
// gets its m_isWikiHalfStopBigram set AND that phrase term
|
|
// is a synonym term of the single word term "enough" and is treated
|
|
// as such in the Posdb.cpp logic.
|
|
for ( int32_t i = 0 ; i < m_numTerms ; i++ ) {
|
|
QueryTerm *qt = &m_qterms[i];
|
|
// assume not!
|
|
qt->m_isWikiHalfStopBigram = false;
|
|
// don't require if negative
|
|
if ( qt->m_termSign == '-' ) continue;
|
|
// only check bigrams here
|
|
if ( ! qt->m_isPhrase ) continue;
|
|
// get the query word that starts this phrase
|
|
const QueryWord *qw1 = qt->m_qword;
|
|
// must be in a wikiphrase
|
|
if ( qw1->m_wikiPhraseId <= 0 ) continue;
|
|
// what query word # is that?
|
|
int32_t qwn = qw1 - m_qwords;
|
|
// get the next alnum word after that
|
|
// assume its the last word in our bigram phrase
|
|
const QueryWord *qw2 = &m_qwords[qwn+2];
|
|
// must be in same wikiphrase
|
|
if ( qw2->m_wikiPhraseId != qw1->m_wikiPhraseId ) continue;
|
|
// if both query stop words, should have been handled above
|
|
// we need one to be a query stop word and the other not
|
|
// for this algo
|
|
if ( qw1->m_isQueryStopWord && qw2->m_isQueryStopWord )
|
|
continue;
|
|
// skip if neither is a query stop word
|
|
if ( ! qw1->m_isQueryStopWord&& ! qw2->m_isQueryStopWord )
|
|
continue;
|
|
// one must be a stop word i guess
|
|
// so for 'the time machine' we do not count 'time machine'
|
|
// as a halfstopwikibigram
|
|
if ( ! qw1->m_isQueryStopWord && ! qw2->m_isQueryStopWord )
|
|
continue;
|
|
|
|
// special flag
|
|
qt->m_isWikiHalfStopBigram = true;
|
|
}
|
|
|
|
if(g_conf.m_logTraceQuery)
|
|
traceTermsToLog("final query-terms");
|
|
|
|
return true;
|
|
}
|
|
|
|
bool Query::setQWords ( char boolFlag ,
|
|
bool keepAllSingles ,
|
|
Phrases &phrases ) {
|
|
|
|
// . break query up into Words and phrases
|
|
// . because we now deal with boolean queries, we make parentheses
|
|
// their own separate Word, so tell "words" we're setting a query
|
|
plain_tokenizer_phase_1(m_filteredQuery.getBufStart(), m_filteredQuery.length(), &m_tr);
|
|
calculate_tokens_hashes(&m_tr);
|
|
|
|
//hackety-hack...
|
|
//The tokenizer phase 2 also recognizes "C++" and "john's", but we cannot use phase 2 because Phrases and Query are
|
|
//incompatible with phase-2 tokens (too many assumptions about strictly increasing positions and contiguous memory layout)
|
|
//So instead we implement special cases here, until we have time to fix the whole Query class.
|
|
for(size_t i=0; i+1<m_tr.size(); i++) {
|
|
//Hack for C++
|
|
if(m_tr[i].is_alfanum && !m_tr[i+1].is_alfanum &&
|
|
m_tr[i].token_len==1 && (m_tr[i].token_start[0]=='c' || m_tr[i].token_start[0]=='C') &&
|
|
m_tr[i+1].token_len>=2 && memcmp(m_tr[i+1].token_start,"++",2)==0)
|
|
{
|
|
m_tr[i].token_len += 2;
|
|
m_tr[i].end_pos += 2;
|
|
m_tr[i+1].start_pos += 2;
|
|
m_tr[i+1].token_start += 2;
|
|
m_tr[i+1].token_len -= 2;
|
|
if(m_tr[i+1].token_len==0)
|
|
m_tr.tokens.erase(m_tr.tokens.begin()+i+1);
|
|
continue;
|
|
}
|
|
//Hack for F#
|
|
if(m_tr[i].is_alfanum && !m_tr[i+1].is_alfanum &&
|
|
m_tr[i].token_len==1 && (m_tr[i].token_start[0]=='f' || m_tr[i].token_start[0]=='F') &&
|
|
m_tr[i+1].token_len>=1 && memcmp(m_tr[i+1].token_start,"#",1)==0)
|
|
{
|
|
m_tr[i].token_len += 1;
|
|
m_tr[i].end_pos += 1;
|
|
m_tr[i+1].start_pos += 1;
|
|
m_tr[i+1].token_start += 1;
|
|
m_tr[i+1].token_len -= 1;
|
|
if(m_tr[i+1].token_len==0)
|
|
m_tr.tokens.erase(m_tr.tokens.begin()+i+1);
|
|
continue;
|
|
}
|
|
//Hack for C#
|
|
if(m_tr[i].is_alfanum && !m_tr[i+1].is_alfanum &&
|
|
m_tr[i].token_len==1 && (m_tr[i].token_start[0]=='c' || m_tr[i].token_start[0]=='C') &&
|
|
m_tr[i+1].token_len>=1 && memcmp(m_tr[i+1].token_start,"#",1)==0)
|
|
{
|
|
m_tr[i].token_len += 1;
|
|
m_tr[i].end_pos += 1;
|
|
m_tr[i+1].start_pos += 1;
|
|
m_tr[i+1].token_start += 1;
|
|
m_tr[i+1].token_len -= 1;
|
|
if(m_tr[i+1].token_len==0)
|
|
m_tr.tokens.erase(m_tr.tokens.begin()+i+1);
|
|
continue;
|
|
}
|
|
//Hack for C#
|
|
if(m_tr[i].is_alfanum && !m_tr[i+1].is_alfanum &&
|
|
m_tr[i].token_len==1 && m_tr[i].token_start[0]=='A' &&
|
|
m_tr[i+1].token_len>=1 && memcmp(m_tr[i+1].token_start,"*",1)==0)
|
|
{
|
|
m_tr[i].token_len += 1;
|
|
m_tr[i].end_pos += 1;
|
|
m_tr[i+1].start_pos += 1;
|
|
m_tr[i+1].token_start += 1;
|
|
m_tr[i+1].token_len -= 1;
|
|
if(m_tr[i+1].token_len==0)
|
|
m_tr.tokens.erase(m_tr.tokens.begin()+i+1);
|
|
continue;
|
|
}
|
|
//Hack for possessive-apostrophe (no need for extra codepoint checks - people usually don't type them in a search field)
|
|
if(i+2<m_tr.size() &&
|
|
m_tr[i].is_alfanum && !m_tr[i+1].is_alfanum && m_tr[i+2].is_alfanum &&
|
|
m_tr[i+1].token_len==1 && (m_tr[i+1].token_start[0]=='\'' || m_tr[i+1].token_start[0]=='`') &&
|
|
m_tr[i+2].token_len==1 && (m_tr[i+2].token_start[0]=='s' || m_tr[i+2].token_start[0]=='S'))
|
|
{
|
|
m_tr[i].end_pos = m_tr[i+2].end_pos;
|
|
m_tr[i].token_len += m_tr[i+1].token_len + m_tr[i+2].token_len;
|
|
m_tr.tokens.erase(m_tr.tokens.begin()+i+1,m_tr.tokens.begin()+i+3);
|
|
continue;
|
|
}
|
|
}
|
|
for(size_t i=0; i+2<m_tr.size(); ) {
|
|
const auto &t0 = m_tr[i+0];
|
|
const auto &t1 = m_tr[i+1];
|
|
const auto &t2 = m_tr[i+2];
|
|
if(t0.token_end()==t1.token_start && t1.token_end()==t2.token_start &&
|
|
is_slash_abbreviation(t0.token_start, t0.token_len+t1.token_len+t2.token_len))
|
|
{
|
|
size_t sl = t0.token_len+t2.token_len;
|
|
char *s = (char*)m_tr.egstack.alloc(sl);
|
|
memcpy(s, t0.token_start, t0.token_len);
|
|
memcpy(s+t0.token_len, t2.token_start, t2.token_len);
|
|
m_tr.tokens.emplace_back(t0.start_pos, t2.end_pos, s,sl, false, true);
|
|
m_tr.tokens.erase(m_tr.tokens.begin()+i, m_tr.tokens.begin()+i+3);
|
|
} else
|
|
i++;
|
|
}
|
|
|
|
int32_t numWords = m_tr.size();
|
|
// truncate it
|
|
if ( numWords > ABS_MAX_QUERY_WORDS ) {
|
|
log("query: Had %" PRId32" words. Max is %" PRId32". Truncating.",
|
|
numWords,(int32_t)ABS_MAX_QUERY_WORDS);
|
|
numWords = ABS_MAX_QUERY_WORDS;
|
|
m_truncated = true;
|
|
}
|
|
m_numWords = numWords;
|
|
// alloc the mem if we need to (mdw left off here)
|
|
int32_t need = m_numWords * sizeof(QueryWord);
|
|
// sanity check
|
|
if ( m_qwords ) { g_process.shutdownAbort(true); }
|
|
// point m_qwords to our generic buffer if it will fit
|
|
if(!m_queryWordBuf.reserve(need)) {
|
|
log(LOG_WARN, "query: Could not allocate mem for query.");
|
|
return false;
|
|
}
|
|
m_qwords = (QueryWord *)m_queryWordBuf.getBufStart();
|
|
// reset safebuf in there
|
|
for ( int32_t i = 0 ; i < m_numWords ; i++ )
|
|
m_qwords[i].constructor();
|
|
|
|
// is all alpha chars in query in upper case? caps lock on?
|
|
bool allUpper = true;
|
|
const char *p = m_filteredQuery.getBufStart();
|
|
const char *pend = m_filteredQuery.getBufPtr();
|
|
for ( ; p < pend ; p += getUtf8CharSize(p) )
|
|
if ( is_alpha_utf8 ( p ) && ! is_upper_utf8 ( p ) ) {
|
|
allUpper = false; break; }
|
|
|
|
// . come back here from below when we detect dat query is not boolean
|
|
// . we need to redo the bits cuz they may have been messed with below
|
|
// redo:
|
|
// field code we are in
|
|
field_code_t fieldCode = FIELD_UNSET;
|
|
char fieldSign = 0;
|
|
const char *field = NULL;
|
|
int32_t fieldLen = 0;
|
|
// keep track of the start of different chunks of quotes
|
|
int32_t quoteStart = -1;
|
|
bool inQuotes = false;
|
|
//bool inVQuotes = false;
|
|
char quoteSign = 0;
|
|
// the current little sign
|
|
char wordSign = 0;
|
|
// when reading first word in link: ... field we skip the following
|
|
// words until we hit a space because we hash them all together
|
|
bool ignoreTilSpace = false;
|
|
// assume we're NOT a boolean query
|
|
m_isBoolean = false;
|
|
// used to not respect the bool operator if it is the first word
|
|
bool firstWord = true;
|
|
|
|
// the query processing is broken into 3 stages.
|
|
|
|
// . STAGE #1
|
|
// . reset all query words to default
|
|
// set all m_ignoreWord and m_ignorePhrase to IGNORE_DEFAULT
|
|
// . set m_isFieldName, m_fieldCode and m_quoteStart for query words.
|
|
// no field names in quotes. +title:"hey there".
|
|
// set m_quoteStart to -1 if not in quotes.
|
|
// . if quotes immediately follow field code's ':' then distribute
|
|
// the field code to all words in the quotes
|
|
// . distribute +/- signs across quotes and fields to m_wordSigns.
|
|
// support -title:"hey there".
|
|
// . set m_quoteStart to -1 if only one alnum word is
|
|
// in quotes, what's the point of that?
|
|
// . set boolean op codes (m_opcode). cannot be in quotes.
|
|
// cannot have a field code. cannot have a word sign (+/-).
|
|
// . set m_wordId of FIELD_LINK, _URL, _SITE, _IP fields.
|
|
// m_wordId of first should be hash of the whole field value.
|
|
// only set its m_ignoreWord to 0, keep it's m_ignorePhrase to DEF.
|
|
// . set m_ignore of non-op codes, non-fieldname, alnum words to 0.
|
|
// . set m_wordId of each non-ignored alnum word.
|
|
|
|
// . STAGE #2
|
|
// . customize Bits class:
|
|
// first alnum word can start phrase.
|
|
// first alnum word in quotes (m_quoteStart >= 0 ) can start phrase.
|
|
// connected on the right but not on the left.. can start phrase.
|
|
// no pair across any double quote
|
|
// no pair across ".." --- UNLESS in quotes!
|
|
// no pair across any change of field code.
|
|
// field names may not be part of any phrase or paired across.
|
|
// boolean ops may not be part of any phrase or paired across.
|
|
// ignored words may not be part of any phrase or paired across.
|
|
|
|
// . STAGE #3
|
|
// . set phrases class w/ custom Bits class mods.
|
|
// . set m_bigramId and m_rawPhraseId of all QueryWords. if phraseId
|
|
// is not 0 (phrase exists) then set m_ignorePhrase to 0.
|
|
// . set m_leftConnected, m_rightConnected. word you are connecting
|
|
// to must not be ignored. (no field names or op codes).
|
|
// ensure you are in a phrase with the connected word, too, to
|
|
// really be connected.
|
|
// . set m_leftPhraseStart and m_rightPhraseEnd for all
|
|
// m_inQuotePhrase is not needed since if m_quoteStart is >= 0
|
|
// we MUST be in a quoted phrase!
|
|
// . if word is Connected then set m_ignoreWord to IGNORE_CONNECTED.
|
|
// set his m_phraseSign to m_wordSign (if not 0) or '*' (if it is 0).
|
|
// m_wordSign may have inherited quote or field sign.
|
|
// . if word's m_quoteStart is >= 0 set m_ignoreWord to IGNORE_QUOTED
|
|
// set his m_phraseSign to m_wordSign (if not 0) or '*' (if it is 0)
|
|
// m_wordSign may have inherited quote or field sign.
|
|
// . if one word in a phrase is negative, then set m_phraseSign to '-'
|
|
|
|
// set the Bits used for making phrases from the Words class
|
|
Bits bits;
|
|
if ( !bits.set(&m_tr)) {
|
|
log(LOG_WARN, "query: Had error processing query: %s.", mstrerror(g_errno));
|
|
return false;
|
|
}
|
|
|
|
float userWeightForWord = 1;
|
|
float userWeightForPhrase = 1;
|
|
float userWeightForSynonym = 1;
|
|
bool userNotRequiredForWord = false;
|
|
int32_t ignorei = -1;
|
|
|
|
// assume we contain no pipe operator
|
|
int32_t pi = -1;
|
|
|
|
int32_t posNum = 0;
|
|
const char *ignoreTill = NULL;
|
|
|
|
// loop over all words, these QueryWords are 1-1 with "words"
|
|
for ( int32_t i = 0 ; i < numWords && i < ABS_MAX_QUERY_WORDS ; i++ ) {
|
|
// convenience var, these are 1-1 with "words"
|
|
QueryWord *qw = &m_qwords[i];
|
|
// set to defaults?
|
|
memset ( qw , 0 , sizeof(QueryWord) );
|
|
// but quotestart should be -1
|
|
qw->m_quoteStart = -1;
|
|
qw->m_leftPhraseStart = -1;
|
|
// assume QueryWord is ignored by default
|
|
qw->m_ignoreWord = IGNORE_DEFAULT;
|
|
qw->m_ignorePhrase = IGNORE_DEFAULT;
|
|
qw->m_ignoreWordInBoolQuery = false;
|
|
qw->m_word = m_tr[i].token_start;
|
|
qw->m_wordLen = m_tr[i].token_len;
|
|
qw->m_isPunct = !m_tr[i].is_alfanum;
|
|
|
|
qw->m_posNum = posNum;
|
|
|
|
// count 1 unit for it
|
|
posNum++;
|
|
|
|
// we ignore the facet value range list...
|
|
if ( ignoreTill && qw->m_word < ignoreTill )
|
|
continue;
|
|
|
|
// . we duplicated this code from XmlDoc.cpp's
|
|
// getWordPosVec() function
|
|
if ( qw->m_isPunct ) { // ! wids[i] ) {
|
|
const char *wp = qw->m_word;
|
|
int32_t wplen = qw->m_wordLen;
|
|
// simple space or sequence of just white space
|
|
if ( is_wspace_utf8_string(m_tr[i].token_start, m_tr[i].token_end()))
|
|
posNum += 0;
|
|
// 'cd-rom'
|
|
else if ( wp[0]=='-' && wplen==1 )
|
|
posNum += 0;
|
|
// 'mr. x'
|
|
else if ( wp[0]=='.' && is_wspace_utf8_string(m_tr[i].token_start+1, m_tr[i].token_end()))
|
|
posNum += 0;
|
|
// animal (dog)
|
|
else
|
|
posNum++;
|
|
}
|
|
|
|
const char *w = m_tr[i].token_start;
|
|
int32_t wlen = m_tr[i].token_len;
|
|
// assume it is a query weight operator
|
|
qw->m_queryOp = true;
|
|
// ignore it? (this is for query weight operators)
|
|
if ( i <= ignorei ) continue;
|
|
// deal with pipe operators
|
|
if ( wlen == 5 &&
|
|
w[0]=='P'&&w[1]=='i'&&w[2]=='i'&&w[3]=='P'&&w[4]=='E') {
|
|
pi = i;
|
|
qw->m_opcode = opcode_t::OP_PIPE;
|
|
continue;
|
|
}
|
|
// [133.0r]
|
|
// is it the bracket operator?
|
|
// " LeFtB 113 rp RiGhB "
|
|
if ( wlen == 5 &&
|
|
w[0]=='L'&&w[1]=='e'&&w[2]=='F'&&w[3]=='t'&&w[4]=='B'&&
|
|
i+4 < numWords ) {
|
|
// s MUST point to a number
|
|
const char *s = m_tr[i+2].token_start;
|
|
int32_t slen = m_tr[i+2].token_len;
|
|
|
|
// if no number, it must be
|
|
// " leFtB w RiGhB " or " leFtB p RiGhB "
|
|
if ( ! is_digit(s[0]) ) {
|
|
if(s[0] == 'w') {
|
|
// word weight reset
|
|
userWeightForWord = 1;
|
|
ignorei = i + 4;
|
|
} else if(s[0] == 'p') {
|
|
// phrase weight reset
|
|
userWeightForPhrase = 1;
|
|
} else if(s[0] == 's') {
|
|
// phrase weight reset
|
|
userWeightForSynonym = 1;
|
|
} else if(s[0] == 'n' && s[1] == 'r' && s[2] == 'w') {
|
|
// set word as not required
|
|
userNotRequiredForWord = true;
|
|
}
|
|
ignorei = i + 4;
|
|
} else {
|
|
// get the number
|
|
float fval = atof2 (s, slen);
|
|
// s2 MUST point to the a,r,ap,rp string
|
|
const char *s2 = m_tr[i+4].token_start;
|
|
// is it a phrase?
|
|
if(s2[0] == 'w') {
|
|
userWeightForWord = fval;
|
|
} else if(s2[0] == 'p') {
|
|
userWeightForPhrase = fval;
|
|
} else if(s2[0] == 's') {
|
|
userWeightForSynonym = fval;
|
|
}
|
|
// ignore all following words up and inc. i+6
|
|
ignorei = i + 6;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
// assign score weight, if any for this guy
|
|
qw->m_userWeightForWord = userWeightForWord;
|
|
qw->m_userWeightForPhrase = userWeightForPhrase;
|
|
qw->m_userWeightForSynonym = userWeightForSynonym;
|
|
// Set required state based on user input
|
|
qw->m_userNotRequiredForWord = userNotRequiredForWord;
|
|
qw->m_queryOp = false;
|
|
|
|
|
|
// does word #i have a space in it? that will cancel fieldCode
|
|
// if we were in a field
|
|
bool endField = false;
|
|
if(has_space(m_tr[i].token_start, m_tr[i].token_end()) && ! inQuotes)
|
|
endField = true;
|
|
// TODO: fix title:" hey there" (space in quotes is ok)
|
|
// if there's a quote before the first space then
|
|
// it's ok!!!
|
|
if ( endField ) {
|
|
const char *s = m_tr[i].token_start;
|
|
const char *send = s + m_tr[i].token_len;
|
|
for ( ; s < send ; s++ ) {
|
|
// if the space is inside the quotes then it
|
|
// doesn't count!
|
|
if(*s == '\"') {
|
|
endField = false;
|
|
break;
|
|
}
|
|
if(is_wspace_a(*s))
|
|
break;
|
|
}
|
|
}
|
|
// cancel the field if we hit a space (not in quotes)
|
|
if ( endField ) {
|
|
// cancel the field
|
|
fieldCode = FIELD_UNSET;
|
|
fieldLen = 0;
|
|
field = NULL;
|
|
// we no longer have to ignore for link: et al
|
|
ignoreTilSpace = false;
|
|
}
|
|
// . maintain inQuotes and quoteStart
|
|
// . quoteStart is the word # that starts the current quote
|
|
int32_t nq = count_quotes(m_tr[i].token_start, m_tr[i].token_len);
|
|
|
|
if ( nq > 0 ) { // && ! ignoreQuotes ) {
|
|
// toggle quotes if we need to
|
|
if ( nq & 0x01 ) inQuotes = ! inQuotes;
|
|
// set quote sign to sign before the quote
|
|
if ( inQuotes ) {
|
|
quoteSign = '\0';
|
|
for ( const char *p = w + wlen - 1 ; p > w ; p--){
|
|
if ( *p != '\"' ) continue;
|
|
if ( *(p-1) == '-' ) quoteSign = '-';
|
|
if ( *(p-1) == '+' ) quoteSign = '+';
|
|
break;
|
|
}
|
|
}
|
|
// . quoteStart is the word # the quotes started at
|
|
// . it is -1 if not in quotes
|
|
// . now we set it to the alnum word AFTER us!!
|
|
if ( inQuotes && i+1< numWords ) quoteStart = i+1;
|
|
else quoteStart = -1;
|
|
}
|
|
//log(LOG_DEBUG, "Query: nq: %" PRId32" inQuotes: %d,quoteStart: %" PRId32,
|
|
// nq, inQuotes, quoteStart);
|
|
// does word #i have a space in it? that will cancel fieldCode
|
|
// if we were in a field
|
|
// TODO: fix title:" hey there" (space in quotes is ok)
|
|
bool cancelField = false;
|
|
if ( has_space(m_tr[i].token_start, m_tr[i].token_end()) && ! inQuotes )
|
|
cancelField = true;
|
|
// fix title:"foo bar" "another quote" so "another quote"
|
|
// is not in the title: field
|
|
if ( has_space(m_tr[i].token_start, m_tr[i].token_end()) && inQuotes && nq>= 2 )
|
|
cancelField = true;
|
|
|
|
// BUT if we have a quote, and they just got turned off,
|
|
// and the space is not after the quote, do not cancel field!
|
|
if ( nq == 1 && cancelField ) {
|
|
// if we hit the space BEFORE the quote, do NOT cancel
|
|
// the field
|
|
for ( const char *p = w + wlen - 1 ; p > w ; p--) {
|
|
// hey, we got the quote first, keep field
|
|
if ( *p == '\"' ) {cancelField = false; break;}
|
|
// otherwise, we got space first? cancel it!
|
|
if ( is_wspace_a(*p) ) break;
|
|
}
|
|
}
|
|
if ( cancelField ) {
|
|
// cancel the field
|
|
fieldCode = FIELD_UNSET;
|
|
fieldLen = 0;
|
|
field = NULL;
|
|
// we no longer have to ignore for link: et al
|
|
ignoreTilSpace = false;
|
|
}
|
|
// skip if we should
|
|
if ( ignoreTilSpace ){
|
|
if (m_qwords[i-1].m_fieldCode){
|
|
qw->m_fieldCode = m_qwords[i-1].m_fieldCode;
|
|
}
|
|
continue;
|
|
}
|
|
// . is this word potentially a field?
|
|
// . it cannot be another field name in a field
|
|
if(i < m_numWords-2 &&
|
|
m_tr[i+1].token_len==1 && m_tr[i+1].token_start[0]==':' &&
|
|
!is_wspace_utf8_string(m_tr[i+2].token_start,m_tr[i+2].token_end()) &&
|
|
(!is_punct_utf8(m_tr[i+2].token_start) || m_tr[i+2].token_start[0]=='\"' || m_tr[i+2].token_start[0]=='-') &&
|
|
! fieldCode && ! inQuotes)
|
|
{
|
|
// field name may have started before though if it
|
|
// was a compound field name containing hyphens,
|
|
// underscores or periods
|
|
int32_t j = i-1 ;
|
|
while ( j > 0 &&
|
|
((m_qwords[j].m_rawWordId != 0) ||
|
|
( m_qwords[j].m_wordLen ==1 &&
|
|
((m_qwords[j].m_word)[0]=='-' ||
|
|
(m_qwords[j].m_word)[0]=='_' ||
|
|
(m_qwords[j].m_word)[0]=='.')))) {
|
|
j--;
|
|
}
|
|
|
|
if ( j < 0 ) {
|
|
j = 0;
|
|
}
|
|
|
|
// advance j to a non-punct word
|
|
while (!m_tr[j].is_alfanum)
|
|
j++;
|
|
|
|
// ignore all of these words then,
|
|
// they're part of field name
|
|
int32_t tlen = 0;
|
|
for ( int32_t k = j ; k <= i ; k++ )
|
|
tlen += m_tr[k].token_len;
|
|
|
|
//is it recognized field name,like "title" or "url"?
|
|
fieldCode = getFieldCode (m_tr[j].token_start, tlen);
|
|
if(fieldCode) {
|
|
//Previously this was done in all cases to support searching for sub-sub-sub...fields in json/xml
|
|
//The downside was that copy-paste of colon-separated words or artist names like "L:Ron:Harald" didn't work.
|
|
|
|
// set field name to the compound name if it is
|
|
field = m_tr[j].token_start;
|
|
fieldLen = tlen;
|
|
if(j == i)
|
|
fieldSign = wordSign;
|
|
else
|
|
fieldSign = m_qwords[j].m_wordSign;
|
|
//FIXME: TokenizerResult does not promise that tokens that are adjacent in the source string also are adjacent in memory
|
|
// (but since Query only does phase-1 tokenization and the tokenizer currently only does tricky things in phase 2 it currently holds)
|
|
|
|
// if so, it does NOT get its own QueryWord,
|
|
// but its sign can be inherited by its members
|
|
for ( int32_t k = j ; k <= i ; k++ )
|
|
m_qwords[k].m_ignoreWord = IGNORE_FIELDNAME;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// what quote chunk are we in? this is 0 if we're not in quotes
|
|
if ( inQuotes ) qw->m_quoteStart = quoteStart ;
|
|
else qw->m_quoteStart = -1;
|
|
qw->m_inQuotes = inQuotes;
|
|
|
|
// ptr to field, if any
|
|
qw->m_fieldCode = fieldCode;
|
|
// if we are a punct word, see if we end in a sign that can
|
|
// be applied to the next word, a non-punct word
|
|
if ( !m_tr[i].is_alfanum ) {
|
|
wordSign = w[wlen-1];
|
|
if ( wordSign != '-' && wordSign != '+') wordSign = 0;
|
|
if ( wlen>1 &&!is_wspace_a (w[wlen-2]) ) wordSign = 0;
|
|
if ( i > 0 && wlen == 1 ) wordSign = 0;
|
|
|
|
// don't add any QueryWord for a punctuation word
|
|
continue;
|
|
}
|
|
|
|
// what is the sign of our term? +, -, *, ...
|
|
char mysign;
|
|
if ( fieldCode ) mysign = fieldSign;
|
|
else if ( inQuotes ) mysign = quoteSign;
|
|
else mysign = wordSign;
|
|
// are we doing default AND?
|
|
//if ( forcePlus && ! *mysign ) mysign = '+';
|
|
// store the sign
|
|
qw->m_wordSign = mysign;
|
|
// what quote chunk are we in? this is 0 if we're not in quotes
|
|
if ( inQuotes ) qw->m_quoteStart = quoteStart ;
|
|
else qw->m_quoteStart = -1;
|
|
|
|
// . get prefix hash of collection name and field
|
|
// . but first convert field to lower case
|
|
uint64_t ph;
|
|
int32_t fflen = fieldLen;
|
|
if ( fflen > 62 ) fflen = 62;
|
|
char ff[64];
|
|
to_lower3_a ( field , fflen , ff );
|
|
|
|
ph = hash64 ( ff , fflen );
|
|
// map "intitle" map to "title"
|
|
if ( fieldCode == FIELD_TITLE )
|
|
ph = hash64 ( "title", 5 );
|
|
// make "suburl" map to "inurl"
|
|
if ( fieldCode == FIELD_SUBURL )
|
|
ph = hash64 ( "inurl", 5 );
|
|
|
|
// fix for filetype:pdf queries
|
|
if ( fieldCode == FIELD_TYPE )
|
|
ph = hash64 ("type",4);
|
|
|
|
// ptr to field, if any
|
|
qw->m_fieldCode = fieldCode;
|
|
|
|
// prefix hash
|
|
qw->m_prefixHash = ph;
|
|
|
|
// if we're hashing a url:, link:, site: or ip: term,
|
|
// then we need to hash ALL up to the first space
|
|
if ( fieldCode == FIELD_URL ||
|
|
fieldCode == FIELD_EXT ||
|
|
fieldCode == FIELD_LINK ||
|
|
fieldCode == FIELD_SITELINK||
|
|
fieldCode == FIELD_LINKS||
|
|
fieldCode == FIELD_SITE ||
|
|
fieldCode == FIELD_IP ||
|
|
fieldCode == FIELD_GBFIELDMATCH ) {
|
|
// . find 1st space -- that terminates the field value
|
|
// . make "end" point to the end of the entire query
|
|
const char *end = m_tr[m_tr.size()-1].token_end();
|
|
// use this for gbmin:price:1.99 etc.
|
|
int32_t firstColonLen = -1;
|
|
int32_t lastColonLen = -1;
|
|
int32_t colonCount = 0;
|
|
|
|
// "w" points to the first alnumword after the field,
|
|
// so for site:xyz.com "w" points to the 'x' and wlen
|
|
// would be 3 in that case sinze xyz is a word of 3
|
|
// chars. so advance
|
|
// wlen until we hit a space.
|
|
while (w + wlen < end) {
|
|
// stop at first white space
|
|
if (is_wspace_utf8(w + wlen)) break;
|
|
// in case of gbmin:price:1.99 record first ':'
|
|
if (w[wlen] == ':') {
|
|
lastColonLen = wlen;
|
|
if (firstColonLen == -1)
|
|
firstColonLen = wlen;
|
|
colonCount++;
|
|
}
|
|
// fix "gbsortbyint:date)"
|
|
// these are used as boolean operators
|
|
// so do not include them in the value.
|
|
// we also did this above to set cancelField
|
|
// to true.
|
|
if (w[wlen] == '(' || w[wlen] == ')')
|
|
break;
|
|
|
|
wlen++;
|
|
}
|
|
// ignore following words until we hit a space
|
|
ignoreTilSpace = true;
|
|
// the hash. keep it case insensitive. only
|
|
// the fieldmatch stuff should be case-sensitive.
|
|
// this may change later.
|
|
uint64_t wid = hash64Lower_utf8(w, wlen, 0LL);
|
|
|
|
if (fieldCode == FIELD_GBFIELDMATCH) {
|
|
// hash the json field name. (i.e. tag.uri)
|
|
// make it case sensitive as
|
|
// seen in XmlDoc.cpp::hashFacet2().
|
|
// the other fields are hashed in
|
|
// XmlDoc.cpp::hashNumber3().
|
|
// CASE SENSITIVE!!!!
|
|
wid = hash64(w, firstColonLen, 0LL);
|
|
// if it is like
|
|
// gbfieldmatch:tag.uri:"http://xyz.com/poo"
|
|
// then we should hash the string into
|
|
// an int just like how the field value would
|
|
// be hashed when adding gbfacetstr: terms
|
|
// in XmlDoc.cpp:hashFacet2(). the hash of
|
|
// the tag.uri field, for example, is set
|
|
// in hashFacet1() and set to "val32". so
|
|
// hash it just like that does here.
|
|
const char *a = w + firstColonLen + 1;
|
|
// . skip over colon at start
|
|
if (a[0] == ':') a++;
|
|
// . skip over quotes at start/end
|
|
bool inQuotes = false;
|
|
if (a[0] == '\"') {
|
|
inQuotes = true;
|
|
a++;
|
|
}
|
|
// end of field
|
|
const char *b = a;
|
|
// if not in quotes advance until
|
|
// we hit whitespace
|
|
char cs;
|
|
for (; !inQuotes && *b; b += cs) {
|
|
cs = getUtf8CharSize(b);
|
|
if (is_wspace_utf8(b)) break;
|
|
}
|
|
// if in quotes, go until we hit quote
|
|
for (; inQuotes && *b != '\"'; b++)
|
|
;
|
|
// now hash that up. this must be 64 bit
|
|
// to match in XmlDoc.cpp::hashFieldMatch()
|
|
uint64_t val64 = hash64(a, b - a);
|
|
// make a composite of tag.uri and http://...
|
|
// just like XmlDoc.cpp::hashFacet2() does
|
|
wid = hash64(val64, wid);
|
|
}
|
|
|
|
// should we have normalized before hashing?
|
|
if (fieldCode == FIELD_URL ||
|
|
fieldCode == FIELD_LINK ||
|
|
fieldCode == FIELD_SITELINK ||
|
|
fieldCode == FIELD_LINKS ||
|
|
fieldCode == FIELD_SITE) {
|
|
Url url;
|
|
url.set( w, wlen, ( fieldCode != FIELD_SITE ), false );
|
|
|
|
if (fieldCode == FIELD_SITELINK) {
|
|
wid = hash64(url.getHost(), url.getHostLen());
|
|
} else {
|
|
wid = hash64(url.getUrl(), url.getUrlLen());
|
|
}
|
|
}
|
|
|
|
// like we do it in XmlDoc.cpp's hashString()
|
|
if (ph) {
|
|
qw->m_wordId = hash64h(wid, ph);
|
|
} else {
|
|
qw->m_wordId = wid;
|
|
}
|
|
|
|
qw->m_rawWordId = 0LL; // only for highlighting?
|
|
qw->m_bigramId = 0LL;
|
|
qw->m_rawPhraseId = 0LL;
|
|
qw->m_opcode = opcode_t::OP_NONE;
|
|
|
|
// definitely not a query stop word
|
|
qw->m_isQueryStopWord = false;
|
|
|
|
// do not ignore the wordId
|
|
qw->m_ignoreWord = IGNORE_NO_IGNORE;
|
|
|
|
// we are the first word?
|
|
firstWord = false;
|
|
|
|
// we're done with this one
|
|
continue;
|
|
}
|
|
|
|
|
|
opcode_t opcode = opcode_t::OP_NONE;
|
|
// if query is all in upper case and we're doing boolean
|
|
// DETECT, then assume not boolean
|
|
if ( allUpper && boolFlag == 2 ) boolFlag = 0;
|
|
// . is this word a boolean operator?
|
|
// . cannot be in quotes or field
|
|
if ( boolFlag >= 1 && ! inQuotes && ! fieldCode ) {
|
|
// are we an operator?
|
|
if ( ! firstWord && wlen==2 &&
|
|
w[0]=='O' && w[1]=='R')
|
|
opcode = opcode_t::OP_OR;
|
|
else if ( ! firstWord && wlen==3 &&
|
|
w[0]=='A' && w[1]=='N' && w[2]=='D')
|
|
opcode = opcode_t::OP_AND;
|
|
else if ( ! firstWord && wlen==3 &&
|
|
w[0]=='N' && w[1]=='O' && w[2]=='T')
|
|
opcode = opcode_t::OP_NOT;
|
|
else if ( wlen==5 && w[0]=='L' && w[1]=='e' &&
|
|
w[2]=='F' && w[3]=='t' && w[4]=='P' )
|
|
opcode = opcode_t::OP_LEFTPAREN;
|
|
else if ( wlen==5 && w[0]=='R' && w[1]=='i' &&
|
|
w[2]=='G' && w[3]=='h' && w[4]=='P' )
|
|
opcode = opcode_t::OP_RIGHTPAREN;
|
|
// no pair across or even include any boolean op phrs
|
|
if ( opcode != opcode_t::OP_NONE ) {
|
|
bits.clearBits(i,D_CAN_PAIR_ACROSS);
|
|
bits.clearBits(i,D_CAN_BE_IN_PHRASE);
|
|
qw->m_ignoreWord = IGNORE_BOOLOP;
|
|
qw->m_opcode = opcode;
|
|
if ( opcode == opcode_t::OP_LEFTPAREN ) continue;
|
|
if ( opcode == opcode_t::OP_RIGHTPAREN ) continue;
|
|
// if this is uncommented all of our operators
|
|
// become actual query terms (mdw)
|
|
if ( opcode == opcode_t::OP_UOR ) continue;
|
|
// if you just have ANDs and ()'s that does
|
|
// not make you a boolean query! we are bool
|
|
// by default!!
|
|
if ( opcode == opcode_t::OP_AND ) continue;
|
|
m_isBoolean = true;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// . add single-word term id
|
|
// . this is computed by hash64AsciiLower()
|
|
// . but only hash64Lower_a if _HASHWITHACCENTS_ is true
|
|
uint64_t wid = m_tr[i].token_hash;
|
|
qw->m_rawWordId = wid;
|
|
// we now have a first word already set
|
|
firstWord = false;
|
|
// . are we a QUERY stop word?
|
|
// . NEVER count as stop word if it's in all CAPS and
|
|
// not all letters in the whole query is NOT in all CAPS
|
|
// . It's probably an acronym
|
|
if ( m_tr[i].token_len>1 &&
|
|
is_upper_utf8_string(m_tr[i].token_start, m_tr[i].token_end()) &&
|
|
! allUpper )
|
|
{
|
|
qw->m_isQueryStopWord = false;
|
|
qw->m_isStopWord = false;
|
|
} else {
|
|
qw->m_isQueryStopWord =::isQueryStopWord (w,wlen,wid,
|
|
m_langId);
|
|
// . BUT, if it is a single letter contraction thing
|
|
// . ninad: make this == 1 if in utf8! TODO!! it is!
|
|
if ( i>0 && wlen == 1 && w[-1] == '\'' )
|
|
qw->m_isQueryStopWord = true;
|
|
qw->m_isStopWord =::isStopWord (w,wlen,wid);
|
|
}
|
|
// . do not count as query stop word if it is the last in query
|
|
// . like the query: 'baby names that start with j'
|
|
if ( i + 2 > numWords ) {
|
|
qw->m_isQueryStopWord = false;
|
|
}
|
|
|
|
// like we do it in XmlDoc.cpp's hashString()
|
|
if ( ph ) {
|
|
qw->m_wordId = hash64(wid, ph);
|
|
} else {
|
|
qw->m_wordId = wid;
|
|
}
|
|
|
|
// do not ignore the word
|
|
qw->m_ignoreWord = IGNORE_NO_IGNORE;
|
|
|
|
//except if it is a high-frequency-term and expensive to look up. In that case ignore the word but keep the phrases/bigrams thereof
|
|
uint64_t termId = (qw->m_wordId & TERMID_MASK);
|
|
if(g_conf.m_useHighFrequencyTermCache &&
|
|
m_allowHighFreqTermCache && g_hfts.is_registered_term(termId)) {
|
|
log(LOG_DEBUG, "query: term='%.*s' with termId %lu is a highfreq term. Marking it for ignoring", wlen, w, termId);
|
|
qw->m_ignoreWord = IGNORE_HIGHFREMTERM;
|
|
}
|
|
|
|
// reset for next word
|
|
userNotRequiredForWord = false;
|
|
}
|
|
|
|
//If there's only one alphanumerical word and it was ignored due to high-freq-term then the query is treated as 0 terms and will return an empty
|
|
//result. Therefore un-ignore the single word and let it fetch (best-efort) results from the high-freq-term-cache
|
|
int numAlfanumWords = 0;
|
|
int numAlfanumWordsHighFreqTerms = 0;
|
|
int alfanumWordIndex = -1;
|
|
for(int i=0; i<numWords; i++) {
|
|
if(m_tr[i].is_alfanum) {
|
|
alfanumWordIndex = i;
|
|
numAlfanumWords++;
|
|
if(m_qwords[i].m_ignoreWord==IGNORE_HIGHFREMTERM)
|
|
numAlfanumWordsHighFreqTerms++;
|
|
|
|
}
|
|
}
|
|
if(numAlfanumWords == 1 && numAlfanumWordsHighFreqTerms==1)
|
|
m_qwords[alfanumWordIndex].m_ignoreWord = IGNORE_NO_IGNORE;
|
|
|
|
// pipe those that should be piped
|
|
for ( int32_t i = 0 ; i < pi ; i++ ) m_qwords[i].m_piped = true;
|
|
|
|
// . set m_leftConnected and m_rightConnected
|
|
// . we are connected to the first non-punct word on our left
|
|
// if we are separated by a small $ of defined punctuation
|
|
// . see getIsConnection() for that definition
|
|
// . this allows us to just lookup the phrase for things like
|
|
// "cd-rom" rather than lookup "cd" , "rom" and "cd-rom"
|
|
// . skip if prev word is IGNORE_BOOLOP, IGNORE_FIELDNAME or
|
|
// IGNORE_DEFAULT
|
|
// . we have to set outside the main loop above since we check
|
|
// the m_ignoreWord member of the i+2nd word
|
|
for ( int32_t i = 0 ; i < numWords ; i++ ) {
|
|
QueryWord *qw = &m_qwords[i];
|
|
if ( qw->m_ignoreWord ) continue;
|
|
if ( i + 2 < numWords && ! m_qwords[i+2].m_ignoreWord&&
|
|
isConnection(i+1) )
|
|
qw->m_rightConnected = true;
|
|
if ( i - 2 >= 0 && ! m_qwords[i-2].m_ignoreWord &&
|
|
isConnection(i-1) )
|
|
qw->m_leftConnected = true;
|
|
}
|
|
|
|
// now modify the Bits class before generating phrases
|
|
for ( int32_t i = 0 ; i < numWords ; i++ ) {
|
|
// get default bits
|
|
unsigned char b = bits.queryBits(i);
|
|
// allow pairing across anything by default
|
|
b |= D_CAN_PAIR_ACROSS;
|
|
// get Query Word
|
|
QueryWord *qw = &m_qwords[i];
|
|
// . skip if part of a query weight operator
|
|
// . cannot be in a phrase, or anything
|
|
if ( qw->m_queryOp && qw->m_opcode==opcode_t::OP_NONE) {
|
|
b = D_CAN_PAIR_ACROSS;
|
|
}
|
|
// is this word a sequence of punctuation and spaces?
|
|
else if ( !m_tr[i].is_alfanum ) {
|
|
// pair across ANY punct, even double spaces by default
|
|
b |= D_CAN_PAIR_ACROSS;
|
|
// but do not pair across anything with a quote in it
|
|
if ( count_quotes(m_tr[i].token_start, m_tr[i].token_len) > 0 )
|
|
b &= ~D_CAN_PAIR_ACROSS;
|
|
// continue if we're in quotes
|
|
else if ( qw->m_quoteStart >= 0 ) goto next;
|
|
// continue if we're in a field
|
|
else if ( qw->m_fieldCode > 0 ) goto next;
|
|
// if guy on left is in field, do not pair across
|
|
if ( i > 0 && m_qwords[i-1].m_fieldCode > 0 )
|
|
b &= ~D_CAN_PAIR_ACROSS;
|
|
// or if guy on right in field
|
|
if ( i +1 < numWords && m_qwords[i+1].m_fieldCode > 0 )
|
|
b &= ~D_CAN_PAIR_ACROSS;
|
|
// do not pair across ".." when not in quotes/field
|
|
const char *w = m_tr[i].token_start;
|
|
int32_t wlen = m_tr[i].token_len;
|
|
for ( int32_t j = 0 ; j < wlen-1 ; j++ ) {
|
|
if ( w[j ]!='.' ) continue;
|
|
if ( w[j+1]!='.' ) continue;
|
|
b &= ~D_CAN_PAIR_ACROSS;
|
|
break;
|
|
}
|
|
}
|
|
else {
|
|
// . no field names, bool operators, cruft in fields
|
|
// can be any part of a phrase
|
|
// . no pair across any change of field code
|
|
// . 'girl title:boy' --> no "girl title" phrase!
|
|
if ( qw->m_ignoreWord && qw->m_ignoreWord!=IGNORE_HIGHFREMTERM ) {
|
|
b &= ~D_CAN_PAIR_ACROSS;
|
|
b &= ~D_CAN_BE_IN_PHRASE;
|
|
}
|
|
// . no boolean ops
|
|
// . 'this OR that' --> no "this OR that" phrase
|
|
if ( qw->m_opcode != opcode_t::OP_NONE ) {
|
|
b &= ~D_CAN_PAIR_ACROSS;
|
|
b &= ~D_CAN_BE_IN_PHRASE;
|
|
}
|
|
if ( qw->m_wordSign == '-' && qw->m_quoteStart < 0) {
|
|
b &= ~D_CAN_PAIR_ACROSS;
|
|
b &= ~D_CAN_BE_IN_PHRASE;
|
|
}
|
|
|
|
}
|
|
next:
|
|
// set it back all tweaked
|
|
bits.assignBits(i,b);
|
|
}
|
|
|
|
// treat strongly connected phrases like cd-rom and 3.2.0.3 as being
|
|
// in quotes for the most part, therefore, set m_quoteStart for them
|
|
int32_t j;
|
|
int32_t qs = -1;
|
|
for ( j = 0 ; j < numWords ; j++ ) {
|
|
// skip all but strongly connected words
|
|
if ( m_qwords[j].m_ignoreWord != IGNORE_CONNECTED &&
|
|
// must also be non punct word OR a space
|
|
( m_tr[j].is_alfanum || *m_tr[j].token_start==' ' ) ) {
|
|
// break the "quote", if any
|
|
qs = -1; continue; }
|
|
// if he is punctuation and qs is -1, skip him,
|
|
// punctuation words can no longer start a quote
|
|
if ( !m_tr[j].is_alfanum && qs == -1 ) continue;
|
|
// uningore him if we should
|
|
if ( keepAllSingles ) m_qwords[j].m_ignoreWord = IGNORE_NO_IGNORE;
|
|
// if already in quotes, don't bother!
|
|
if ( m_qwords[j].m_quoteStart >= 0 ) continue;
|
|
// remember him
|
|
if ( qs == -1 ) qs = j;
|
|
// he starts the phrase
|
|
m_qwords[j].m_quoteStart = qs;
|
|
// force him into a quoted phrase
|
|
m_qwords[j].m_inQuotes = true;
|
|
//m_qwords[j].m_inQuotedPhrase = true;
|
|
}
|
|
|
|
// fix for tags.uri:http://foo.com/bar so it works like
|
|
// tags.uri:"http://foo.com/bar" like it should
|
|
int32_t first = -1;
|
|
for ( j = 0 ; j < numWords ; j++ ) {
|
|
// stop when we hit spaces
|
|
if ( has_wspace_utf8_string(m_tr[j].token_start, m_tr[j].token_end()) ) {
|
|
first = -1;
|
|
continue;
|
|
}
|
|
// skip if not in field
|
|
if ( ! m_qwords[j].m_fieldCode ) continue;
|
|
// must be in a generic field, the other fields like site:
|
|
// will be messed up by this logic
|
|
if ( m_qwords[j].m_fieldCode != FIELD_GENERIC ) continue;
|
|
// first alnumword in field?
|
|
if ( first == -1 ) {
|
|
// must be alnum
|
|
if ( m_qwords[j].m_isPunct ) continue;
|
|
// must have punct then another alnum word
|
|
if ( j+2 >= numWords ) break;
|
|
// spaces screw it up
|
|
if ( has_wspace_utf8_string(m_tr[j+1].token_start, m_tr[j+1].token_end()) ) continue;
|
|
// then an alnum word after
|
|
first = j;
|
|
}
|
|
// we are in fake quoted phrase
|
|
m_qwords[j].m_inQuotes = true;
|
|
m_qwords[j].m_quoteStart = first;
|
|
}
|
|
|
|
// make the phrases from the words and the tweaked Bits class
|
|
if ( !phrases.set(m_tr,bits) )
|
|
return false;
|
|
|
|
// do phrases stuff
|
|
for ( int32_t i = 0 ; i < numWords ; i++ ) {
|
|
// get the ith QueryWord
|
|
QueryWord *qw = &m_qwords[i];
|
|
|
|
//if word is ignored (and it is not due to high-freq-term) then don't generate a phrase/bigram query term
|
|
if(qw->m_ignoreWord && qw->m_ignoreWord!=IGNORE_HIGHFREMTERM)
|
|
continue;
|
|
if ( qw->m_fieldCode && qw->m_quoteStart < 0) continue;
|
|
// get the first word # to our left that starts a phrase
|
|
// of which we are a member
|
|
qw->m_leftPhraseStart = -1;
|
|
for ( int32_t j = i - 1 ; j >= 0 ; j-- ) {
|
|
if ( ! bits.canPairAcross(j+1) ) break;
|
|
if ( !m_tr[j].is_alfanum ) continue;
|
|
|
|
qw->m_leftPhraseStart = j;
|
|
// we can't pair across alnum words now, we just want bigrams
|
|
if ( m_tr[j].is_alfanum ) break;
|
|
// now we do bigrams so only allow two words even
|
|
// if they are stop words
|
|
break;
|
|
}
|
|
// . is this word in a quoted phrase?
|
|
// . the whole phrase must be in the same set of quotes
|
|
// . if we're in a left phrase, he must be in our quotes
|
|
if ( qw->m_leftPhraseStart >= 0 &&
|
|
qw->m_quoteStart >= 0 &&
|
|
qw->m_leftPhraseStart >= qw->m_quoteStart )
|
|
qw->m_inQuotedPhrase = true;
|
|
// if we start a phrase, ensure next guy is in our quote
|
|
if ( ! qw->m_ignorePhrase && i+1 < numWords &&
|
|
m_qwords[i+1].m_quoteStart >= 0 &&
|
|
m_qwords[i+1].m_quoteStart <= i )
|
|
qw->m_inQuotedPhrase = true;
|
|
// are we the first word in the quote?
|
|
if ( i-1>=0 && qw->m_quoteStart == i )
|
|
qw->m_inQuotedPhrase = true;
|
|
// ignore single words that are in a quoted phrase
|
|
if ( ! keepAllSingles && qw->m_inQuotedPhrase )
|
|
qw->m_ignoreWord = IGNORE_QUOTED;
|
|
|
|
// . get phrase info for this term
|
|
// . a pid (phraseId)of 0 indicates it does not start a phrase
|
|
// . raw phrase termId
|
|
uint64_t pid = 0LL;
|
|
|
|
phrases.getMinWordsInPhrase(i,(int64_t *)&pid);;
|
|
|
|
// store it
|
|
qw->m_rawPhraseId = pid;
|
|
|
|
// does word #i start a phrase?
|
|
if ( pid != 0 ) {
|
|
uint64_t ph = qw->m_prefixHash ;
|
|
|
|
// like we do it in XmlDoc.cpp's hashString()
|
|
if ( ph ) qw->m_bigramId = hash64 ( pid , ph );
|
|
else qw->m_bigramId = pid;
|
|
|
|
//calculate length of phrase(bigram) in bytes
|
|
int32_t numWordsInPhrase = phrases.getNumWordsInPhrase2(i);
|
|
int phraseLen = 0;
|
|
for(int j=i; j<i+numWordsInPhrase; j++)
|
|
phraseLen += m_qwords[j].m_wordLen;
|
|
qw->m_bigramLen = phraseLen;
|
|
|
|
// do not ignore the phrase, it's valid
|
|
qw->m_ignorePhrase = IGNORE_NO_IGNORE;
|
|
}
|
|
|
|
|
|
// . phrase sign is inherited from word's sign if it's a minus
|
|
// . word sign is inherited from field, quote or right before
|
|
// the word
|
|
// . that is, all words in -"to be or not" will have a '-' sign
|
|
// . phraseId may or may not be 0 at this point
|
|
if ( qw->m_wordSign == '-' ) qw->m_phraseSign = '-';
|
|
|
|
// . dist word signs to others in the same connected string
|
|
// . use "-cd-rom x-box" w/ no connector in between
|
|
// . test queries:
|
|
// . +cd-rom +x-box
|
|
// . -cd-rom +x-box
|
|
// . -m-o-n
|
|
// . who was the first (was is a query stop word)
|
|
// . www.xxx.com
|
|
// . welcome to har.com
|
|
// . hezekiah walker the love family affair ii live at radio
|
|
// city music hall
|
|
// . fotostudio +m-o-n-a-r-t
|
|
// . fotostudio -m-o-n-a-r-t
|
|
// . i'm home
|
|
if ( qw->m_leftConnected && qw->m_leftPhraseStart >= 0 )
|
|
qw->m_wordSign = m_qwords[i-2].m_wordSign;
|
|
|
|
// . if we connected to the alnum word on our right then
|
|
// soft require the phrase (i.e. treat like a single term)
|
|
// . example: cd-rom or www.xxx.com
|
|
// . 'welcome to har.com' should get a '*' for "har.com" sign
|
|
if ( qw->m_rightConnected ) {
|
|
if ( qw->m_wordSign) qw->m_phraseSign = qw->m_wordSign;
|
|
else qw->m_phraseSign = '*';
|
|
}
|
|
|
|
// . if we're in quotes then any phrase we have should be
|
|
// soft required (i.e. treated like a single term)
|
|
// . we do not allow phrases in queries to pair across
|
|
// quotes. See where we tweak the Bits class above.
|
|
if ( qw->m_quoteStart >= 0 ) {
|
|
qw->m_phraseSign = '*';
|
|
}
|
|
|
|
// . if we are the last word in a phrase that consists of all
|
|
// PLAIN stop words then make the phrase have a '*'
|
|
// . 'to be or not to be .. test' (cannot pair across "..")
|
|
// . don't use QUERY stop words cuz of "who was the first?" qry
|
|
if ( pid ) {
|
|
int32_t nw = phrases.getNumWordsInPhrase2(i);
|
|
int32_t j;
|
|
// search up to this far
|
|
int32_t maxj = i + nw;
|
|
// but not past our truncated limit
|
|
if ( maxj > ABS_MAX_QUERY_WORDS )
|
|
maxj = ABS_MAX_QUERY_WORDS;
|
|
|
|
for ( j = i ; j < maxj ; j++ ) {
|
|
// skip punct
|
|
if ( !m_tr[j].is_alfanum ) continue;
|
|
// break out if not a stop word
|
|
if ( ! bits.isStopWord(j) ) break;
|
|
// break out if has a term sign
|
|
if ( m_qwords[j].m_wordSign ) break;
|
|
}
|
|
// if everybody in phrase #i was a signless stopword
|
|
// and the phrase was signless, make it have a '*' sign
|
|
if ( j >= maxj && m_qwords[i].m_phraseSign == '\0' )
|
|
m_qwords[i].m_phraseSign = '*';
|
|
// . if a constituent has a - sign, then the whole
|
|
// phrase becomes negative, too
|
|
// . fixes 'apple -computer' truncation problem
|
|
for ( int32_t j = i ; j < maxj ; j++ )
|
|
if ( m_qwords[j].m_wordSign == '-' )
|
|
qw->m_phraseSign = '-';
|
|
}
|
|
|
|
// . ignore unsigned QUERY stop words that are not yet ignored
|
|
// and are in unignored phrases
|
|
// . 'who was the first taiwanese president' should not get
|
|
// "who was" term sign changed to '*' because "was" is a
|
|
// QUERY stop word. So ignore singles query stop words
|
|
// in phrases now
|
|
if ( //! keepAllSingles &&
|
|
(qw->m_isQueryStopWord && !m_isBoolean) &&
|
|
m_useQueryStopWords &&
|
|
! qw->m_fieldCode &&
|
|
// fix 'the tigers'
|
|
//(qw->m_leftPhraseStart >= 0 || qw->m_bigramId > 0 ) &&
|
|
! qw->m_wordSign &&
|
|
! qw->m_ignoreWord )
|
|
qw->m_ignoreWord = IGNORE_QSTOP;
|
|
|
|
// . ignore and/or between quoted phrases, save user from
|
|
// themselves (they meant AND/OR)
|
|
if ( ! keepAllSingles && qw->m_isQueryStopWord &&
|
|
! qw->m_fieldCode &&
|
|
m_useQueryStopWords &&
|
|
! qw->m_bigramId && ! qw->m_inQuotes &&
|
|
((qw->m_wordId == 255176654160863LL) ||
|
|
(qw->m_wordId == 46196171999655LL)) )
|
|
qw->m_ignoreWord = IGNORE_QSTOP;
|
|
// . ignore repeated single words and phrases
|
|
// . look at the old termIds for this, too
|
|
// . should ignore 2nd 'time' in 'time after time' then
|
|
// . but boolean queries often need to repeat terms
|
|
|
|
// . NEW - words much be same sign and not in different
|
|
// . quoted phrases to be ignored -partap
|
|
if ( ! m_isBoolean && !qw->m_ignoreWord ) {
|
|
for ( int32_t j = 0 ; j < i ; j++ ) {
|
|
if ( m_qwords[j].m_ignoreWord ) continue;
|
|
if ( m_qwords[j].m_wordId == qw->m_wordId &&
|
|
m_qwords[j].m_wordSign ==qw->m_wordSign &&
|
|
(!keepAllSingles ||
|
|
(m_qwords[j].m_quoteStart
|
|
== qw->m_quoteStart))){
|
|
qw->m_ignoreWord = IGNORE_REPEAT;
|
|
}
|
|
}
|
|
}
|
|
if ( ! m_isBoolean && !qw->m_ignorePhrase ) {
|
|
// ignore repeated phrases too!
|
|
for ( int32_t j = 0 ; j < i ; j++ ) {
|
|
if ( m_qwords[j].m_ignorePhrase ) continue;
|
|
if ( m_qwords[j].m_bigramId == qw->m_bigramId &&
|
|
m_qwords[j].m_phraseSign
|
|
== qw->m_phraseSign)
|
|
qw->m_ignorePhrase = IGNORE_REPEAT;
|
|
}
|
|
}
|
|
}
|
|
|
|
// . if we only have one quoted query then force its sign to be '+'
|
|
// . '"get the phrase" the' --> +"get the phrase" (last the is ignored)
|
|
// . "time enough for love" --> +"time enough" +"enough for love"
|
|
// . if all unignored words are in the same set of quotes then change
|
|
// all '*' (soft-required) phrase signs to '+'
|
|
for ( j= 0 ; j < numWords ; j++ ) {
|
|
if ( !m_tr[j].is_alfanum) continue;
|
|
if ( m_qwords[j].m_quoteStart < 0 ) break;
|
|
if ( m_qwords[j].m_ignoreWord ) continue;
|
|
if ( j < 2 ) continue;
|
|
if ( m_qwords[j-2].m_quoteStart != m_qwords[j].m_quoteStart )
|
|
break;
|
|
}
|
|
if ( j >= numWords ) {
|
|
for ( j= 0 ; j < numWords ; j++ ) {
|
|
if ( m_qwords[j].m_phraseSign == '*' )
|
|
m_qwords[j].m_phraseSign = '+';
|
|
}
|
|
}
|
|
|
|
// . force a plus on any site: or ip: query terms
|
|
// . also disable site clustering if we have either of these terms
|
|
for ( int32_t i = 0 ; i < m_numWords ; i++ ) {
|
|
QueryWord *qw = &m_qwords[i];
|
|
if ( qw->m_ignoreWord ) continue;
|
|
if ( qw->m_wordSign ) continue;
|
|
if ( qw->m_fieldCode != FIELD_SITE &&
|
|
qw->m_fieldCode != FIELD_IP ) continue;
|
|
qw->m_wordSign = '+';
|
|
}
|
|
|
|
// . if one or more of a phrase's constituent terms exceeded
|
|
// term #MAX_QUERY_TERMS then we should also soft require that phrase
|
|
// . fixes 'hezekiah walker the love family affair ii live at
|
|
// radio city music hall'
|
|
// . how many non-ignored phrases?
|
|
int32_t count = 0;
|
|
for ( int32_t i = 0 ; i < m_numWords ; i++ ) {
|
|
QueryWord *qw = &m_qwords[i];
|
|
if ( qw->m_ignorePhrase ) continue;
|
|
if ( ! qw->m_bigramId ) continue;
|
|
count++;
|
|
}
|
|
for ( int32_t i = 0 ; i < numWords ; i++ ) {
|
|
QueryWord *qw = &m_qwords[i];
|
|
// count non-ignored words
|
|
if ( qw->m_ignoreWord ) continue;
|
|
// if under limit, continue
|
|
if ( count++ < ABS_MAX_QUERY_TERMS ) continue;
|
|
// . otherwise, ignore
|
|
// . if we set this for our UOR'ed terms from SearchInput.cpp's
|
|
// UOR'ed facebook interests then it causes us to get no results!
|
|
// so make sure that MAX_QUERY_TERMS is big enough with respect to
|
|
// the opCount in SearchInput.cpp
|
|
qw->m_ignoreWord = IGNORE_BREECH;
|
|
// left phrase should get a '*'
|
|
int32_t left = qw->m_leftPhraseStart;
|
|
if ( left >= 0 && ! m_qwords[left].m_phraseSign )
|
|
m_qwords[left].m_phraseSign = '*';
|
|
// our phrase should get a '*'
|
|
if ( qw->m_bigramId && ! qw->m_phraseSign )
|
|
qw->m_phraseSign = '*';
|
|
}
|
|
|
|
// . fix the 'x -50a' query so it returns results
|
|
// . how many non-negative, non-ignored words/phrases do we have?
|
|
count = 0;
|
|
for ( int32_t i = 0 ; i < m_numWords ; i++ ) {
|
|
const QueryWord *qw = &m_qwords[i];
|
|
if ( qw->m_ignoreWord ) continue;
|
|
if ( qw->m_wordSign == '-' ) continue;
|
|
count++;
|
|
}
|
|
for ( int32_t i = 0 ; i < m_numWords ; i++ ) {
|
|
const QueryWord *qw = &m_qwords[i];
|
|
if ( qw->m_ignorePhrase ) continue;
|
|
if ( qw->m_phraseSign == '-' ) continue;
|
|
if ( qw->m_bigramId == 0LL ) continue;
|
|
count++;
|
|
}
|
|
// if everybody is ignored or negative UNignore first query stop word
|
|
if ( count == 0 ) {
|
|
for ( int32_t i = 0 ; i < m_numWords ; i++ ) {
|
|
QueryWord *qw = &m_qwords[i];
|
|
if ( qw->m_ignoreWord != IGNORE_QSTOP ) continue;
|
|
qw->m_ignoreWord = IGNORE_NO_IGNORE;
|
|
count++;
|
|
break;
|
|
}
|
|
}
|
|
|
|
quoteStart = -1;
|
|
int32_t quoteEnd = -1;
|
|
// set m_quoteENd
|
|
for ( int32_t i = m_numWords - 1 ; i >= 0 ; i-- ) {
|
|
// get ith word
|
|
QueryWord *qw = &m_qwords[i];
|
|
// skip if ignored
|
|
if ( qw->m_ignoreWord ) continue;
|
|
// skip if not in quotes
|
|
if ( qw->m_quoteStart < 0 ) continue;
|
|
// if match previous guy...
|
|
if ( qw->m_quoteStart == quoteStart ) {
|
|
// inherit the end
|
|
qw->m_quoteEnd = quoteEnd;
|
|
// all done
|
|
continue;
|
|
}
|
|
// ok, we are the end then
|
|
quoteEnd = i;
|
|
quoteStart = qw->m_quoteStart;
|
|
}
|
|
|
|
|
|
int32_t wkid = 0;
|
|
int32_t upTo = -1;
|
|
|
|
//
|
|
// set the wiki phrase ids
|
|
//
|
|
for ( int32_t i = 0 ; i < m_numWords ; i++ ) {
|
|
// get ith word
|
|
QueryWord *qw = &m_qwords[i];
|
|
// in a phrase from before?
|
|
if ( i < upTo ) {
|
|
qw->m_wikiPhraseId = wkid;
|
|
continue;
|
|
}
|
|
// assume none
|
|
qw->m_wikiPhraseId = 0;
|
|
// skip if punct
|
|
if ( !m_tr[i].is_alfanum ) continue;
|
|
// get word
|
|
int32_t nwk ;
|
|
nwk = g_wiki.getNumWordsInWikiPhrase ( i , &m_tr );
|
|
// bail if none
|
|
if ( nwk <= 1 ) continue;
|
|
|
|
// inc it
|
|
wkid++;
|
|
// store it
|
|
qw->m_wikiPhraseId = wkid;
|
|
// set loop parm
|
|
upTo = i + nwk;
|
|
}
|
|
|
|
// consider terms strongly connected like wikipedia title phrases
|
|
for ( int32_t i = 0 ; i + 2 < m_numWords ; i++ ) {
|
|
// get ith word
|
|
QueryWord *qw1 = &m_qwords[i];
|
|
// must not already be in a wikiphrase
|
|
//if ( qw1->m_wikiPhraseId > 0 ) continue;
|
|
// what query word # is that?
|
|
int32_t qwn = qw1 - m_qwords;
|
|
// get the next alnum word after that
|
|
// assume its the last word in our bigram phrase
|
|
QueryWord *qw2 = &m_qwords[qwn+2];
|
|
// must be in same wikiphrase
|
|
if ( qw2->m_wikiPhraseId > 0 ) continue;
|
|
|
|
// if there is a strong connector like the . in 'dmoz.org'
|
|
// then consider it a wiki bigram too
|
|
if ( ! qw1->m_rightConnected ) continue;
|
|
if ( ! qw2->m_leftConnected ) continue;
|
|
|
|
// fix 'rdf.org.dumps' so org.dumps gets same
|
|
// wikiphraseid as rdf.org
|
|
int id;
|
|
if ( qw1->m_wikiPhraseId ) id = qw1->m_wikiPhraseId;
|
|
else id = ++wkid;
|
|
|
|
// store it
|
|
qw1->m_wikiPhraseId = id;
|
|
|
|
qw2->m_wikiPhraseId = id;
|
|
}
|
|
|
|
// all done
|
|
return true;
|
|
}
|
|
|
|
|
|
void Query::modifyQuery(DerivedScoringWeights *scoringWeights, const CollectionRec& cr, bool *doSiteClustering) {
|
|
logTrace(g_conf.m_logTraceQuery, "Query::modifyQuery: q='%s', modifyDomainLikeSearches=%s, modifyAPILikeSearches=%s", originalQuery(),cr.m_modifyDomainLikeSearches?"true":"false", cr.m_modifyAPILikeSearches?"true":"false");
|
|
logTrace(g_conf.m_logTraceQuery, " m_numWords = %d", m_numWords);
|
|
logTrace(g_conf.m_logTraceQuery, " m_numTerms = %d", m_numTerms);
|
|
if(cr.m_modifyDomainLikeSearches) {
|
|
bool looksLikeADomain = false;
|
|
// is it a domain in the form of domain.tld ?
|
|
if(m_numWords==3 &&
|
|
is_alnum_utf8_string(m_qwords[0].m_word,m_qwords[0].m_word+m_qwords[0].m_wordLen) &&
|
|
m_qwords[1].m_wordLen==1 && m_qwords[1].m_word[0]=='.' &&
|
|
is_alnum_utf8_string(m_qwords[2].m_word,m_qwords[2].m_word+m_qwords[2].m_wordLen))
|
|
looksLikeADomain = true;
|
|
// is it a domain in the form of host.domain.tld ?
|
|
if(m_numWords==5 &&
|
|
is_alnum_utf8_string(m_qwords[0].m_word,m_qwords[0].m_word+m_qwords[0].m_wordLen) &&
|
|
m_qwords[1].m_wordLen==1 && m_qwords[1].m_word[0]=='.' &&
|
|
is_alnum_utf8_string(m_qwords[2].m_word,m_qwords[2].m_word+m_qwords[2].m_wordLen) &&
|
|
m_qwords[3].m_wordLen==1 && m_qwords[3].m_word[0]=='.' &&
|
|
is_alnum_utf8_string(m_qwords[4].m_word,m_qwords[4].m_word+m_qwords[4].m_wordLen))
|
|
looksLikeADomain = true;
|
|
// is it a domain in the form of host.domain.tld1.tld2 ? (eg www.example.co.uk)
|
|
if(m_numWords==7 &&
|
|
is_alnum_utf8_string(m_qwords[0].m_word,m_qwords[0].m_word+m_qwords[0].m_wordLen) &&
|
|
m_qwords[1].m_wordLen==1 && m_qwords[1].m_word[0]=='.' &&
|
|
is_alnum_utf8_string(m_qwords[2].m_word,m_qwords[2].m_word+m_qwords[2].m_wordLen) &&
|
|
m_qwords[3].m_wordLen==1 && m_qwords[3].m_word[0]=='.' &&
|
|
is_alnum_utf8_string(m_qwords[4].m_word,m_qwords[4].m_word+m_qwords[4].m_wordLen) &&
|
|
m_qwords[5].m_wordLen==1 && m_qwords[5].m_word[0]=='.' &&
|
|
is_alnum_utf8_string(m_qwords[6].m_word,m_qwords[6].m_word+m_qwords[6].m_wordLen))
|
|
looksLikeADomain = true;
|
|
if(looksLikeADomain) {
|
|
if(!isTLD(m_qwords[m_numWords-1].m_word,m_qwords[m_numWords-1].m_wordLen) &&
|
|
!isTLD(m_qwords[m_numWords-3].m_word,m_qwords[m_numWords-3].m_wordLen+m_qwords[m_numWords-2].m_wordLen+m_qwords[m_numWords-1].m_wordLen))
|
|
looksLikeADomain = false; //nope - last component(s) isn't a known tld
|
|
}
|
|
if(looksLikeADomain) {
|
|
log(LOG_DEBUG, "query:Query '%s' looks like a domain", originalQuery());
|
|
//set all non-synonym terms as required and boost inUrl weight.
|
|
for(int i=0; i<m_numTerms; i++) {
|
|
if(!m_qterms[i].m_synonymOf && !m_qterms[i].m_ignored) {
|
|
m_qterms[i].m_isRequired = true;
|
|
m_qterms[i].m_rightPhraseTermNum = -1;
|
|
m_qterms[i].m_leftPhraseTermNum = -1;
|
|
m_qterms[i].m_rightPhraseTerm = NULL;
|
|
m_qterms[i].m_leftPhraseTerm = NULL;
|
|
}
|
|
}
|
|
if(isTLD(m_qwords[m_numWords-1].m_word,m_qwords[m_numWords-1].m_wordLen)) {
|
|
//The last term is marked non-required because the tld terms are normally not indexed (see XmlDoc::hashUrl() -> hashString() -> hashString3())
|
|
//high-freq-terms and stopwords means that the term may not have been generated, so look for it
|
|
for(int i=0; i<m_numTerms; i++) {
|
|
if(m_qterms[i].m_qword == &(m_qwords[m_numWords-1]) && !m_qterms[i].m_isPhrase)
|
|
m_qterms[i].m_isRequired = false;
|
|
}
|
|
}
|
|
scoringWeights->m_hashGroupWeights[HASHGROUP_INURL] *= 10; //factor 10 seems to work fine
|
|
if(cr.m_domainLikeSearchDisablesSiteCluster)
|
|
*doSiteClustering = false;
|
|
log(LOG_DEBUG, "query:Query modified");
|
|
traceTermsToLog("domain-like search terms");
|
|
return;
|
|
}
|
|
}
|
|
|
|
if(cr.m_modifyAPILikeSearches) {
|
|
bool looksLikeAnAPI = false;
|
|
//is it something like "file.open" or "file.open()" ?
|
|
//todo: detect java packages like java.util.HashSet (but most java programmers probably has built-in help in their IDE so they would rarely use this)
|
|
if(m_numWords==3 &&
|
|
is_alnum_api_utf8_string(m_qwords[0].m_word,m_qwords[0].m_word+m_qwords[0].m_wordLen) &&
|
|
m_qwords[1].m_wordLen==1 && m_qwords[1].m_word[0]=='.' &&
|
|
is_alnum_api_utf8_string(m_qwords[2].m_word,m_qwords[2].m_word+m_qwords[2].m_wordLen))
|
|
looksLikeAnAPI = true;
|
|
if(m_numWords==4 &&
|
|
is_alnum_api_utf8_string(m_qwords[0].m_word,m_qwords[0].m_word+m_qwords[0].m_wordLen) &&
|
|
m_qwords[1].m_wordLen==1 && m_qwords[1].m_word[0]=='.' &&
|
|
is_alnum_api_utf8_string(m_qwords[2].m_word,m_qwords[2].m_word+m_qwords[2].m_wordLen) &&
|
|
m_qwords[3].m_wordLen==2 && m_qwords[3].m_word[0]=='(' && m_qwords[3].m_word[1]==')')
|
|
looksLikeAnAPI = true;
|
|
//or "file::open()"
|
|
if(m_numWords==3 &&
|
|
is_alnum_api_utf8_string(m_qwords[0].m_word,m_qwords[0].m_word+m_qwords[0].m_wordLen) &&
|
|
m_qwords[1].m_wordLen==2 && m_qwords[1].m_word[0]==':' && m_qwords[1].m_word[1]==':' &&
|
|
is_alnum_api_utf8_string(m_qwords[2].m_word,m_qwords[2].m_word+m_qwords[2].m_wordLen))
|
|
looksLikeAnAPI = true;
|
|
if(m_numWords==4 &&
|
|
is_alnum_api_utf8_string(m_qwords[0].m_word,m_qwords[0].m_word+m_qwords[0].m_wordLen) &&
|
|
m_qwords[1].m_wordLen==2 && m_qwords[1].m_word[0]==':' && m_qwords[1].m_word[1]==':' &&
|
|
is_alnum_api_utf8_string(m_qwords[2].m_word,m_qwords[2].m_word+m_qwords[2].m_wordLen) &&
|
|
m_qwords[3].m_wordLen==2 && m_qwords[3].m_word[0]=='(' && m_qwords[3].m_word[1]==')')
|
|
looksLikeAnAPI = true;
|
|
if(looksLikeAnAPI) {
|
|
log(LOG_DEBUG, "query:Query '%s' looks like an API or function call", originalQuery());
|
|
//set all non-synonym terms as required
|
|
for(int i=0; i<m_numTerms; i++) {
|
|
if(!m_qterms[i].m_synonymOf && !m_qterms[i].m_ignored) {
|
|
m_qterms[i].m_isRequired = true;
|
|
m_qterms[i].m_rightPhraseTermNum = -1;
|
|
m_qterms[i].m_leftPhraseTermNum = -1;
|
|
m_qterms[i].m_rightPhraseTerm = NULL;
|
|
m_qterms[i].m_leftPhraseTerm = NULL;
|
|
}
|
|
}
|
|
log(LOG_DEBUG, "query:Query modified");
|
|
traceTermsToLog("api-like search terms");
|
|
return;
|
|
}
|
|
}
|
|
log(LOG_DEBUG, "query: Query not modified");
|
|
}
|
|
|
|
|
|
|
|
// return -1 if does not exist in query, otherwise return the query word num
|
|
int32_t Query::getWordNum(int64_t wordId) const {
|
|
// skip if punct or whatever
|
|
if ( wordId == 0LL || wordId == -1LL ) return -1;
|
|
for ( int32_t i = 0 ; i < m_numWords ; i++ ) {
|
|
const QueryWord *qw = &m_qwords[i];
|
|
// the non-raw word id includes a hash with "0", which
|
|
// signifies an empty field term
|
|
if ( qw->m_rawWordId == wordId ) return i;
|
|
}
|
|
// otherwise, not found
|
|
return -1;
|
|
}
|
|
|
|
static HashTableX s_table;
|
|
static bool s_isInitialized = false;
|
|
static GbMutex s_tableMutex;
|
|
|
|
// 3rd field = m_hasColon
|
|
const struct QueryField g_fields[] = {
|
|
{"url",
|
|
FIELD_URL,
|
|
true,
|
|
"url:www.example.com/page.html",
|
|
"Matches the page with that exact url. Uses the first url, not "
|
|
"the url it redirects to, if any." ,
|
|
NULL,
|
|
0 },
|
|
|
|
{"ext",
|
|
FIELD_EXT,
|
|
true,
|
|
"ext:doc",
|
|
"Match documents whose url ends in the <i>.doc</i> file extension.",
|
|
NULL,
|
|
0 },
|
|
|
|
|
|
{"link",
|
|
FIELD_LINK,
|
|
true,
|
|
"link:www.example.com/foo.html",
|
|
"Matches all the documents that have a link to "
|
|
"http://www.example.com/foobar.html",
|
|
NULL,
|
|
0 },
|
|
|
|
{"sitelink",
|
|
FIELD_SITELINK,
|
|
true,
|
|
"sitelink:abc.foobar.com",
|
|
"Matches all documents that link to any page on the "
|
|
"<i>abc.foobar.com</i> site.",
|
|
NULL,
|
|
0 },
|
|
|
|
{"site",
|
|
FIELD_SITE,
|
|
true,
|
|
"site:example.com",
|
|
"Matches all documents on the example.com domain.",
|
|
NULL,
|
|
0 },
|
|
|
|
{"site",
|
|
FIELD_SITE,
|
|
true,
|
|
"site:www.example.com/dir1/dir2/",
|
|
"Matches all documents whose url starts with "
|
|
"www.example.com/dir1/dir2/",
|
|
NULL,
|
|
QTF_DUP },
|
|
|
|
{"sitenoindex",
|
|
FIELD_SITE,
|
|
true,
|
|
"sitenoindex:example.com",
|
|
"Matches all documents on the example.com domain that in not indexed.",
|
|
NULL,
|
|
0 },
|
|
|
|
{"ip",
|
|
FIELD_IP,
|
|
true,
|
|
"ip:192.0.2.1",
|
|
"Matches all documents whose IP is 192.0.2.1.",
|
|
NULL,
|
|
0 },
|
|
|
|
|
|
{"ip",
|
|
FIELD_IP,
|
|
true,
|
|
"ip:192.0.2",
|
|
"Matches all documents whose IP STARTS with 192.0.2.",
|
|
NULL,
|
|
QTF_DUP },
|
|
|
|
|
|
{"inurl",
|
|
FIELD_SUBURL,
|
|
true,
|
|
"inurl:dog",
|
|
"Matches all documents that have the word dog in their url, like "
|
|
"http://www.example.com/dog/food.html. However will not match "
|
|
"http://www.example.com/dogfood.html because it is not an "
|
|
"individual word. It must be delineated by punctuation.",
|
|
NULL,
|
|
0 },
|
|
|
|
|
|
{"suburl",
|
|
FIELD_SUBURL,
|
|
true,
|
|
"suburl:dog",
|
|
"Same as inurl.",
|
|
NULL,
|
|
0},
|
|
|
|
{"intitle",
|
|
FIELD_TITLE,
|
|
false,
|
|
"title:cat",
|
|
"Matches all the documents that have the word cat in their "
|
|
"title.",
|
|
NULL,
|
|
0 },
|
|
|
|
|
|
{"intitle",
|
|
FIELD_TITLE,
|
|
false,
|
|
"title:\"cat food\"",
|
|
"Matches all the documents that have the phrase \"cat food\" "
|
|
"in their title.",
|
|
NULL,
|
|
QTF_DUP },
|
|
|
|
|
|
{"title",
|
|
FIELD_TITLE,
|
|
false,
|
|
"title:cat",
|
|
"Same as intitle:",
|
|
NULL,
|
|
0},
|
|
|
|
{"type",
|
|
FIELD_TYPE,
|
|
false,
|
|
"type:json",
|
|
"Matches all documents that are in JSON format. "
|
|
"Other possible types include "
|
|
"<i>html, text, xml, pdf, doc, xls, ppt, ps, css, json, status.</i> "
|
|
"<i>status</i> matches special documents that are stored every time "
|
|
"a url is spidered so you can see all the spider attempts and when "
|
|
"they occurred as well as the outcome.",
|
|
NULL,
|
|
0},
|
|
|
|
{"filetype",
|
|
FIELD_TYPE,
|
|
false,
|
|
"filetype:json",
|
|
"Same as type: above.",
|
|
NULL,
|
|
0},
|
|
|
|
{"gblang",
|
|
FIELD_GBLANG,
|
|
false,
|
|
"gblang:de",
|
|
"Matches all documents in german. "
|
|
"The supported language abbreviations "
|
|
"are at the bottom of the <a href=\"/admin/filters\">url filters</a> "
|
|
"page. Some more "
|
|
"common ones are <i>gblang:en, gblang:es, gblang:fr, "
|
|
// need quotes for this one!!
|
|
"gblang:\"zh_cn\"</i> (note the quotes for zh_cn!).",
|
|
NULL,
|
|
0},
|
|
|
|
{"gbcountry",
|
|
FIELD_GBCOUNTRY,
|
|
false,
|
|
"gbcountry:us",
|
|
"Matches documents determined by Gigablast to be from the United "
|
|
"States. See the country abbreviations in the CountryCode.cpp "
|
|
"open source distribution. Some more popular examples include: "
|
|
"de, fr, uk, ca, cn.",
|
|
NULL,
|
|
0} ,
|
|
|
|
// mdw
|
|
|
|
{"gbdocid",
|
|
FIELD_GBDOCID,
|
|
false,
|
|
"gbdocid:123456",
|
|
"Matches the document with the docid 123456",
|
|
NULL,
|
|
0},
|
|
|
|
{"gbtermid",
|
|
FIELD_GBTERMID,
|
|
false,
|
|
"gbtermid:123456",
|
|
"Matches the documents for the term with termid 123456",
|
|
NULL,
|
|
0},
|
|
|
|
//
|
|
// for content type CT_STATUS documents (Spider status docs)
|
|
//
|
|
|
|
{"gbdocspiderdate",
|
|
FIELD_GENERIC,
|
|
false,
|
|
"gbdocspiderdate:1400081479",
|
|
"Matches documents that have "
|
|
"that spider date timestamp (UTC). "
|
|
//"Does not include the "
|
|
//"special spider status documents. "
|
|
"This is the time the document "
|
|
"completed downloading.",
|
|
"Date Related Query Operators",
|
|
QTF_BEGINNEWTABLE},
|
|
|
|
|
|
{"gbspiderdate",
|
|
FIELD_GENERIC,
|
|
false,
|
|
"gbspiderdate:1400081479",
|
|
"Like above.",
|
|
//, but DOES include the special spider status documents.",
|
|
NULL,
|
|
0},
|
|
|
|
{"gbdocindexdate",
|
|
FIELD_GENERIC,
|
|
false,
|
|
"gbdocindexdate:1400081479",
|
|
"Like above, but is the time the document was last indexed. "
|
|
"This time is "
|
|
"slightly greater than or equal to the spider date.",//Does not "
|
|
//"include the special spider status documents.",
|
|
NULL,
|
|
0},
|
|
|
|
|
|
{"gbindexdate",
|
|
FIELD_GENERIC,
|
|
false,
|
|
"gbindexdate:1400081479",
|
|
"Like above.",//, but it does include the special spider status "
|
|
//"documents.",
|
|
NULL,
|
|
0},
|
|
|
|
// they don't need to know about this
|
|
{"gbcontenthash", FIELD_GBCONTENTHASH, false,"","",NULL,QTF_HIDE}
|
|
};
|
|
|
|
void resetQuery ( ) {
|
|
s_table.reset();
|
|
}
|
|
|
|
|
|
|
|
int32_t getNumFieldCodes ( ) {
|
|
return (int32_t)sizeof(g_fields) / (int32_t)sizeof(QueryField);
|
|
}
|
|
|
|
static bool initFieldTable(){
|
|
|
|
ScopedLock sl(s_tableMutex);
|
|
if ( ! s_isInitialized ) {
|
|
// set up the hash table
|
|
if ( ! s_table.set ( 8 , 4 , 255,NULL,0,false,"qryfldtbl" ) ) {
|
|
log(LOG_WARN, "build: Could not init table of query fields.");
|
|
return false;
|
|
}
|
|
// now add in all the stop words
|
|
int32_t n = getNumFieldCodes();
|
|
for ( int32_t i = 0 ; i < n ; i++ ) {
|
|
// skip if dup
|
|
int64_t h = hash64b ( g_fields[i].text );
|
|
|
|
// if already in there it is a dup
|
|
if ( s_table.isInTable ( &h ) ) continue;
|
|
|
|
// store the entity index in the hash table as score
|
|
if ( ! s_table.addTerm(h, i+1) ) return false;
|
|
}
|
|
s_isInitialized = true;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
|
|
field_code_t getFieldCode(const char *s, int32_t len) {
|
|
if ( !initFieldTable() ) {
|
|
return FIELD_UNSET;
|
|
}
|
|
|
|
int64_t h = hash64Lower_a( s, len );
|
|
int32_t i = (int32_t) s_table.getScore(h);
|
|
|
|
if ( i == 0 ) {
|
|
return FIELD_UNSET;
|
|
}
|
|
|
|
return g_fields[i-1].field;
|
|
}
|
|
|
|
const char *getFieldCodeName(field_code_t fc) {
|
|
switch(fc) {
|
|
case FIELD_UNSET: return "unset";
|
|
case FIELD_URL: return "url";
|
|
case FIELD_LINK: return "link";
|
|
case FIELD_SITE: return "site";
|
|
case FIELD_IP: return "ip";
|
|
case FIELD_SUBURL: return "suburl";
|
|
case FIELD_TITLE: return "title";
|
|
case FIELD_TYPE: return "type";
|
|
case FIELD_EXT: return "ext";
|
|
case FIELD_LINKS: return "links";
|
|
case FIELD_SITELINK: return "sitelink";
|
|
case FIELD_GENERIC: return "generic";
|
|
case FIELD_GBLANG: return "gblang";
|
|
case FIELD_GBCOUNTRY: return "gbcountry";
|
|
case FIELD_GBTERMID: return "gbtermid";
|
|
case FIELD_GBDOCID: return "gbdocid";
|
|
case FIELD_GBCONTENTHASH: return "gbcontenthash";
|
|
case FIELD_GBFIELDMATCH: return "gbfieldmatch";
|
|
default: return NULL;
|
|
}
|
|
}
|
|
|
|
|
|
// guaranteed to be punctuation
|
|
bool Query::isConnection(unsigned i) const {
|
|
auto const &token = m_tr[i];
|
|
if(token.token_len==1) {
|
|
switch(*token.token_start) {
|
|
// . only allow apostrophe if it's NOT a 's
|
|
// . so contractions are ok, and names too
|
|
case '\'':
|
|
// no, i think we should require it. google seems to,
|
|
// and msn and yahoo do. 'john's room -"john's" gives
|
|
// no result son yahoo and msn.
|
|
return true;
|
|
case ':': return true;
|
|
case '-': return true;
|
|
case '.': return true;
|
|
case '@': return true;
|
|
case '#': return true;
|
|
case '/': return true;
|
|
case '_': return true;
|
|
case '&': return true;
|
|
case '=': return true;
|
|
case '\\': return true;
|
|
default: return false;
|
|
}
|
|
}
|
|
//if ( len == 3 && s[0]==' ' && s[1]=='&' && s[2]==' ' ) return true;
|
|
if(token.token_len==3 &&
|
|
token.token_start[0]==':' && token.token_start[1]=='/' && token.token_start[2]=='/' )
|
|
return true;
|
|
return false;
|
|
}
|
|
|
|
|
|
void Query::dumpToLog() const
|
|
{
|
|
log(LOG_DEBUG, "Query:setQTerms: dumping %d query-words:", m_numWords);
|
|
for(int i=0; i<m_numWords; i++) {
|
|
const QueryWord &qw = m_qwords[i];
|
|
log(" qword #%d:",i);
|
|
log(" word='%*.*s'", (int)qw.m_wordLen, (int)qw.m_wordLen, qw.m_word);
|
|
log(" phrase='%*.*s'", (int)qw.m_bigramLen, (int)qw.m_bigramLen, qw.m_word);
|
|
log(" m_wordId=%" PRId64, qw.m_wordId);
|
|
log(" m_bigramId=%" PRId64, qw.m_bigramId);
|
|
if(qw.m_queryWordTerm)
|
|
log(" m_queryWordTerm= #%d", (int)(qw.m_queryWordTerm-m_qterms));
|
|
}
|
|
log("Query:setQTerms: dumping %d query-terms:", m_numTerms);
|
|
for(int i=0; i<m_numTerms; i++) {
|
|
const QueryTerm &qt = m_qterms[i];
|
|
log(" term #%d:",i);
|
|
log(" m_term='%*.*s'", (int)qt.m_termLen, (int)qt.m_termLen, qt.m_term);
|
|
log(" m_isPhrase=%s synonym=%s", qt.m_isPhrase?"true":"false", qt.m_synonymOf?"true":"false");
|
|
log(" m_termId=%" PRId64, qt.m_termId);
|
|
log(" m_rawTermId=%" PRId64, qt.m_rawTermId);
|
|
log(" m_isWikiHalfStopBigram=%s", qt.m_isWikiHalfStopBigram?"true":"false");
|
|
log(" m_leftPhraseTermNum=%d, m_leftPhraseTerm=%p", qt.m_leftPhraseTermNum, (void*)qt.m_leftPhraseTerm);
|
|
log(" m_rightPhraseTermNum=%d, m_rightPhraseTerm=%p", qt.m_rightPhraseTermNum, (void*)qt.m_rightPhraseTerm);
|
|
log(" m_rightPhraseTermNum=%d, m_rightPhraseTerm=%p", qt.m_rightPhraseTermNum, (void*)qt.m_rightPhraseTerm);
|
|
log(" m_rightPhraseTermNum=%d, m_rightPhraseTerm=%p", qt.m_rightPhraseTermNum, (void*)qt.m_rightPhraseTerm);
|
|
log(" m_termFreqWeight=%f m_termWeight=%f m_userWeight=%f", qt.m_termFreqWeight, qt.m_termWeight, qt.m_userWeight);
|
|
if(qt.m_synonymOf)
|
|
log(" m_synonymOf=#%d '%.*s'", (int)(qt.m_synonymOf-m_qterms), qt.m_synonymOf->m_termLen, qt.m_synonymOf->m_term);
|
|
}
|
|
}
|
|
|
|
void Query::traceTermsToLog(const char *header) {
|
|
logTrace(g_conf.m_logTraceQuery, "%s: %d queryterms:", header, m_numTerms);
|
|
for(int i=0; i<m_numTerms; i++) {
|
|
logTrace(g_conf.m_logTraceQuery, " query-term #%d: termid=%15" PRId64" '%*.*s', t-weight=%f u-weight=%f %s", i, m_qterms[i].m_termId, m_qterms[i].m_termLen,m_qterms[i].m_termLen,m_qterms[i].m_term, m_qterms[i].m_termWeight,m_qterms[i].m_userWeight, m_qterms[i].m_ignored?"ignored":"");
|
|
logTrace(g_conf.m_logTraceQuery, " qstopw=%s req=%s", m_qterms[i].m_isQueryStopWord?"true":"false", m_qterms[i].m_isRequired?"yes":"no");
|
|
}
|
|
}
|
|
|
|
|
|
|
|
////////////////////////////////////////////////////////
|
|
////////////////////////////////////////////////////////
|
|
////////// ONLY BOOLEAN STUFF BELOW HERE /////////////
|
|
////////////////////////////////////////////////////////
|
|
////////////////////////////////////////////////////////
|
|
|
|
// return false and set g_errno on error
|
|
// returns how many words expression was
|
|
bool Expression::addExpression (int32_t start,
|
|
int32_t end,
|
|
Query *q,
|
|
int32_t level
|
|
) {
|
|
|
|
if ( level >= MAX_EXPRESSIONS ) {
|
|
g_errno = ETOOMANYPARENS;
|
|
return false;
|
|
}
|
|
|
|
// the # of the first alnumpunct word in the expression
|
|
m_expressionStartWord = start;
|
|
m_q = q;
|
|
|
|
int32_t i = m_expressionStartWord;
|
|
|
|
// "start" is the current alnumpunct word we are parsing out
|
|
for ( ; i<end ; i++ ) {
|
|
|
|
QueryWord *qwords = q->m_qwords;
|
|
|
|
QueryWord * qw = &qwords[i];
|
|
|
|
// set leaf node if not an opcode like "AND" and not punct.
|
|
if ( qw->m_opcode==opcode_t::OP_NONE && qw->isAlphaWord()){
|
|
continue;
|
|
}
|
|
if (qw->m_opcode == opcode_t::OP_NOT) {
|
|
continue;
|
|
}
|
|
else if (qw->m_opcode == opcode_t::OP_LEFTPAREN ) {
|
|
// this is expression
|
|
// . it should advance "i" to end of expression
|
|
// point to next...
|
|
q->m_numExpressions++;
|
|
// make a new one:
|
|
Expression *e=&q->m_expressions[q->m_numExpressions-1];
|
|
// now set it
|
|
if ( ! e->addExpression ( i+1, // skip over (
|
|
end ,
|
|
q ,
|
|
level + 1) )
|
|
return false;
|
|
// skip over it. pt to ')'
|
|
i += e->m_numWordsInExpression;
|
|
qw->m_expressionPtr = e;
|
|
}
|
|
else if (qw->m_opcode == opcode_t::OP_RIGHTPAREN ) {
|
|
// return size i guess, include )
|
|
m_numWordsInExpression = i - m_expressionStartWord+1;
|
|
return true;
|
|
}
|
|
else if (qw->m_opcode!=opcode_t::OP_NONE) {
|
|
continue;
|
|
}
|
|
// white space?
|
|
}
|
|
|
|
m_numWordsInExpression = i - m_expressionStartWord;
|
|
|
|
return true;
|
|
}
|
|
|
|
// each bit is 1-1 with the explicit terms in the boolean query
|
|
bool Query::matchesBoolQuery(const unsigned char *bitVec, int32_t vecSize) const {
|
|
return m_expressions[0].isTruth ( bitVec , vecSize );
|
|
}
|
|
|
|
|
|
static bool isBitNumSet(int32_t opBitNum, const unsigned char *bitVec, int32_t vecSize) {
|
|
int32_t byte = opBitNum / 8;
|
|
int32_t mask = 1<<(opBitNum % 8);
|
|
if ( byte >= vecSize ) { g_process.shutdownAbort(true); }
|
|
return bitVec[byte] & mask;
|
|
}
|
|
|
|
// . "bits" are 1-1 with the query words in Query::m_qwords[] array
|
|
// including ignored words and spaces i guess since Expression::add()
|
|
// seems to do that.
|
|
bool Expression::isTruth(const unsigned char *bitVec, int32_t vecSize) const {
|
|
|
|
//
|
|
// operand1 operand2 operator1 operand3 operator2 ....
|
|
//
|
|
|
|
// result: -1 means unknown at this point
|
|
int32_t result = -1;
|
|
|
|
opcode_t prevOpCode = opcode_t::OP_NONE;
|
|
int32_t prevResult ;
|
|
// result of current operand
|
|
int32_t opResult = -1;
|
|
|
|
int32_t i = m_expressionStartWord;
|
|
int32_t iend = i + m_numWordsInExpression;
|
|
|
|
bool hasNot = false;
|
|
|
|
for ( ; i < iend ; i++ ) {
|
|
|
|
const QueryWord *qw = &m_q->m_qwords[i];
|
|
|
|
// ignore parentheses, aren't real opcodes.
|
|
// we just want OP_AND/OP_OR/OP_NOT
|
|
opcode_t opcode = qw->m_opcode;
|
|
if ( opcode != opcode_t::OP_AND &&
|
|
opcode != opcode_t::OP_OR &&
|
|
opcode != opcode_t::OP_NOT )
|
|
opcode = opcode_t::OP_NONE;
|
|
|
|
if ( opcode == opcode_t::OP_NOT ) {
|
|
hasNot = true;
|
|
continue;
|
|
}
|
|
|
|
|
|
// so operands are expressions as well
|
|
const Expression *e = (const Expression *)qw->m_expressionPtr;
|
|
if ( e ) {
|
|
// save prev one. -1 means no prev.
|
|
prevResult = opResult;
|
|
// set new onw
|
|
opResult = e->isTruth ( bitVec , vecSize );
|
|
// skip over that expression. point to ')'
|
|
i += e->m_numWordsInExpression;
|
|
// flip?
|
|
if ( hasNot ) {
|
|
if ( opResult == 1 ) opResult = 0;
|
|
else opResult = 1;
|
|
hasNot = false;
|
|
}
|
|
}
|
|
|
|
if ( opcode!=opcode_t::OP_NONE && ! e ) {
|
|
prevOpCode = opcode;//m_opSlots[i];
|
|
continue;
|
|
}
|
|
|
|
// simple operand
|
|
if ( opcode==opcode_t::OP_NONE && ! e ) {
|
|
// for regular word operands
|
|
// ignore it like a space?
|
|
if ( qw->m_ignoreWord ) continue;
|
|
// ignore gbsortby:offerprice in bool queries
|
|
// at least for evaluating them
|
|
if ( qw->m_ignoreWordInBoolQuery ) continue;
|
|
// save old one
|
|
prevResult = opResult;
|
|
// convert word to term #
|
|
const QueryTerm *qt = qw->m_queryWordTerm;
|
|
// fix title:"notre dame" AND NOT irish
|
|
if ( ! qt ) qt = qw->m_queryPhraseTerm;
|
|
if ( ! qt ) continue;
|
|
// phrase terms are not required and therefore
|
|
// do not have a v alid qt->m_bitNum set, so dont core
|
|
if ( ! qt->m_isRequired ) continue;
|
|
// . m_bitNum is set in Posdb.cpp when it sets its
|
|
// QueryTermInfo array
|
|
// . it is basically the query term #
|
|
// . see iff that bit is set in this docid's vec
|
|
opResult = isBitNumSet ( qt->m_bitNum,bitVec,vecSize );
|
|
// flip?
|
|
if ( hasNot ) {
|
|
if ( opResult == 1 ) opResult = 0;
|
|
else opResult = 1;
|
|
hasNot = false;
|
|
}
|
|
}
|
|
|
|
// need two to tango. i.e. (true OR false)
|
|
if ( prevResult == -1 ) continue;
|
|
|
|
// if this is not the first time... we got two
|
|
if ( prevOpCode == opcode_t::OP_AND ) {
|
|
// if first operation we encount is A AND B then
|
|
// default result to on. only allow an AND operation
|
|
// to turn if off.
|
|
if ( result == -1 ) result = 1;
|
|
if ( ! prevResult ) result = 0;
|
|
if ( ! opResult ) result = 0;
|
|
}
|
|
else if ( prevOpCode == opcode_t::OP_OR ) {
|
|
// if first operation we encount is A OR B then
|
|
// default result to off
|
|
if ( result == -1 ) result = 0;
|
|
if ( prevResult ) result = 1;
|
|
if ( opResult ) result = 1;
|
|
}
|
|
}
|
|
|
|
// if we never set result, then it was probably a single
|
|
// argument expression like something in double parens like
|
|
// ((site:xyz.com OR site:abc.com)). so set it to value of
|
|
// first operand, opResult.
|
|
if ( prevOpCode == opcode_t::OP_NONE && result == -1 ) result = opResult;
|
|
|
|
if ( result == -1 ) return true;
|
|
if ( result == 0 ) return false;
|
|
return true;
|
|
}
|
|
|
|
// if any one query term is split, msg3a has to split the query
|
|
bool Query::isSplit() const {
|
|
for(int32_t i = 0; i < m_numTerms; i++)
|
|
if(m_qterms[i].isSplit()) return true;
|
|
return false;
|
|
}
|
|
|
|
void QueryTerm::constructor ( ) {
|
|
m_qword = NULL;
|
|
m_isPhrase = false;
|
|
m_termId = 0;
|
|
m_rawTermId = 0;
|
|
m_termSign = 0;
|
|
m_bitNum = 0;
|
|
m_term = NULL;
|
|
m_termLen = 0;
|
|
m_posdbListPtr = NULL;
|
|
m_langIdBits = 0;
|
|
m_langIdBitsValid = false;
|
|
m_termFreq = 0;
|
|
m_termFreqWeight = 0.0;
|
|
m_isQueryStopWord = false;
|
|
m_inQuotes = false;
|
|
m_termWeight = 0;
|
|
m_userWeight = 0;
|
|
m_userNotRequired = false;
|
|
m_piped = false;
|
|
m_ignored = false;
|
|
m_synonymOf = NULL;
|
|
m_synWids0 = 0;
|
|
m_synWids1 = 0;
|
|
m_numAlnumWordsInSynonym = 1;
|
|
m_fieldCode = FIELD_UNSET;
|
|
m_isRequired = false;
|
|
m_isWikiHalfStopBigram = false;
|
|
m_leftPhraseTermNum = 0;
|
|
m_rightPhraseTermNum = 0;
|
|
m_leftPhraseTerm = NULL;
|
|
m_rightPhraseTerm = NULL;
|
|
memset(m_startKey,0,sizeof(m_startKey));
|
|
memset(m_endKey,0,sizeof(m_endKey));
|
|
}
|
|
|
|
bool QueryTerm::isSplit() const {
|
|
if(!m_fieldCode) return true;
|
|
if(m_fieldCode == FIELD_GBCONTENTHASH) return false;
|
|
return true;
|
|
}
|
|
|
|
// hash of all the query terms
|
|
int64_t Query::getQueryHash() const {
|
|
int64_t qh = 0LL;
|
|
for ( int32_t i = 0 ; i < m_numTerms ; i++ ) {
|
|
const QueryTerm *qt = &m_qterms[i];
|
|
qh = hash64 ( qt->m_termId , qh );
|
|
}
|
|
return qh;
|
|
}
|
|
|
|
void QueryWord::constructor () {
|
|
m_synWordBuf.constructor();
|
|
}
|
|
|
|
void QueryWord::destructor () {
|
|
m_synWordBuf.purge();
|
|
}
|
|
|
|
|
|
static int count_quotes(const char *s, size_t len) {
|
|
int count = 0;
|
|
while(len--)
|
|
if(*s++ == '\"')
|
|
count++;
|
|
return count;
|
|
}
|