diff options
Diffstat (limited to 'src/mongo/db/fts/fts_matcher.cpp')
| -rw-r--r-- | src/mongo/db/fts/fts_matcher.cpp | 31 |
1 files changed, 14 insertions, 17 deletions
diff --git a/src/mongo/db/fts/fts_matcher.cpp b/src/mongo/db/fts/fts_matcher.cpp index 492dbdf7b7b..634bcf345cd 100644 --- a/src/mongo/db/fts/fts_matcher.cpp +++ b/src/mongo/db/fts/fts_matcher.cpp @@ -31,6 +31,7 @@ #include "mongo/platform/basic.h" #include "mongo/db/fts/fts_matcher.h" +#include "mongo/db/fts/fts_tokenizer.h" #include "mongo/db/fts/fts_element_iterator.h" #include "mongo/platform/strcasestr.h" @@ -96,15 +97,13 @@ namespace mongo { bool FTSMatcher::_hasPositiveTerm_string( const FTSLanguage* language, const string& raw ) const { - Tokenizer i( *language, raw ); - Stemmer stemmer( *language ); - while ( i.more() ) { - Token t = i.next(); - if ( t.type != Token::TEXT ) { - continue; - } - string word = stemmer.stem( _query.normalizeString( t.data ) ); - if ( _query.getPositiveTerms().count( word ) > 0 ) { + std::unique_ptr<FTSTokenizer> tokenizer(language->createTokenizer()); + + tokenizer->reset(raw.c_str(), _query.getCaseSensitive()); + + while (tokenizer->moveNext()) { + string word = tokenizer->get().toString(); + if (_query.getPositiveTerms().count(word) > 0) { return true; } } @@ -130,14 +129,12 @@ namespace mongo { bool FTSMatcher::_hasNegativeTerm_string( const FTSLanguage* language, const string& raw ) const { - Tokenizer i( *language, raw ); - Stemmer stemmer( *language ); - while ( i.more() ) { - Token t = i.next(); - if ( t.type != Token::TEXT ) { - continue; - } - string word = stemmer.stem( _query.normalizeString( t.data ) ); + std::unique_ptr<FTSTokenizer> tokenizer(language->createTokenizer()); + + tokenizer->reset(raw.c_str(), _query.getCaseSensitive()); + + while (tokenizer->moveNext()) { + string word = tokenizer->get().toString(); if ( _query.getNegatedTerms().count( word ) > 0 ) { return true; } |