Mirrors/privacore-open-source-search-engine
1
0
Fork 1
mirror of https://github.com/privacore/open-source-search-engine.git synced 2025-07-14 02:36:06 -04:00
Code Issues Packages Projects Releases Wiki Activity

unicode: bugfix Words::addWords()

Browse Source
This commit is contained in:
Ivan Skytte Jørgensen
2018-02-08 16:12:27 +01:00
parent 673c9c107f
commit 3f43604cd1
1 changed files with 2 additions and 1 deletions

3
Words.cpp

@ -301,7 +301,8 @@ bool Words::addWords( char *s, int32_t nodeLen, bool computeWordIds ) {
// get props
uint32_t props = UnicodeMaps::query_properties(c);
// good stuff?
if(props&(Unicode::White_Space|Unicode::Extender)) continue;
if(props&(Unicode::Extender)) continue;
//(todo): props&ignorable (which is quite complicated)
//something abotu ignorable
// stop? if UC_WORCHAR is set, that means its an alnum
if(!UnicodeMaps::is_wordchar(c)) {