mirror of
https://github.com/privacore/open-source-search-engine.git
synced 2025-10-30 16:36:11 -04:00
9 lines
224 B
C++
9 lines
224 B
C++
#include"tokenizer.h"
|
|
#include "hash.h"
|
|
|
|
void calculate_tokens_hashes(TokenizerResult *tr) {
|
|
for(auto &token : tr->tokens)
|
|
if(token.is_alfanum)
|
|
token.token_hash = hash64Lower_utf8(token.token_start,token.token_len);
|
|
}
|