privacore-open-source-search-engine/tokenizer/tokenizer3.cpp

#include "tokenizer.h"
#include "Xml.h"

void xml_tokenizer_phase_1(const Xml *xml, TokenizerResult *tr) {
	if(xml->getNumNodes()==0)
		return;
	const char *first_pos = xml->getNode(0);
	
	for(int i=0; i<xml->getNumNodes(); i++) {
		const char *node = xml->getNode(i);
		int node_len = xml->getNodeLen(i);
		
		if(!xml->isTag(i)) {
			//Not a tag. Just run a plain tokenizer on the contained text
			plain_tokenizer_phase_1_downcall(node,node_len, node-first_pos, tr);
		} else {
			//tag
			nodeid_t node_id = xml->getNodeId(i);
			if(xml->isBackTag(i))
				node_id |= BACKBIT;
			tr->tokens.emplace_back(node-first_pos, node-first_pos+node_len, node,node_len, node_id,i);
		}
	}
}


void xml_tokenizer_phase_1_subset(const Xml *xml, unsigned start_node, unsigned end_node, TokenizerResult *tr) {
	if(end_node<=start_node)
		return;
	if(xml->getNumNodes()==0)
		return;
	const char *first_pos = xml->getNode(0);
	
	for(unsigned i=start_node; i<(unsigned)xml->getNumNodes() && i<end_node; i++) {
		const char *node = xml->getNode(i);
		int node_len = xml->getNodeLen(i);
		
		if(!xml->isTag(i)) {
			//Not a tag. Just run a plain tokenizer on the contained text
			plain_tokenizer_phase_1_downcall(node,node_len, node-first_pos, tr);
		} else {
			//tag
			nodeid_t node_id = xml->getNodeId(i);
			if(xml->isBackTag(i))
				node_id |= BACKBIT;
			tr->tokens.emplace_back(node-first_pos, node-first_pos+node_len, node,node_len, node_id,i);
		}
	}
}