diff options
Diffstat (limited to 'src/mongo/db/fts/fts_tokenizer.h')
-rw-r--r-- | src/mongo/db/fts/fts_tokenizer.h | 84 |
1 files changed, 42 insertions, 42 deletions
diff --git a/src/mongo/db/fts/fts_tokenizer.h b/src/mongo/db/fts/fts_tokenizer.h index 2b345d89266..40cdbde2cb8 100644 --- a/src/mongo/db/fts/fts_tokenizer.h +++ b/src/mongo/db/fts/fts_tokenizer.h @@ -35,58 +35,58 @@ namespace mongo { namespace fts { - class FTSLanguage; - class StopWords; +class FTSLanguage; +class StopWords; + +/** + * FTSTokenizer + * A iterator of "documents" where a document contains space delimited words. + * For each word returns a stem or lemma version of a word optimized for full text indexing. + * Supports various options to control how tokens are generated. + */ +class FTSTokenizer { +public: + virtual ~FTSTokenizer() = default; /** - * FTSTokenizer - * A iterator of "documents" where a document contains space delimited words. - * For each word returns a stem or lemma version of a word optimized for full text indexing. - * Supports various options to control how tokens are generated. + * Options for generating tokens */ - class FTSTokenizer { - public: - virtual ~FTSTokenizer() = default; - - /** - * Options for generating tokens - */ - enum Options { - /** - * Default means lower cased, and stop words are not filtered. - */ - None = 0, - - /** - * Do not lower case terms. - */ - GenerateCaseSensitiveTokens = 1 << 0, - - /** - * Filter out stop words from return tokens. - */ - FilterStopWords = 1 << 1, - }; - + enum Options { /** - * Process a new document, and discards any previous results. - * May be called multiple times on an instance of an iterator. + * Default means lower cased, and stop words are not filtered. */ - virtual void reset(StringData document, Options options) = 0; + None = 0, /** - * Moves to the next token in the iterator. - * Returns false when the iterator reaches end of the document. + * Do not lower case terms. */ - virtual bool moveNext() = 0; + GenerateCaseSensitiveTokens = 1 << 0, /** - * Returns stemmed form, normalized, and lowercased depending on the parameter - * to the reset method. - * Returned StringData is valid until next call to moveNext(). + * Filter out stop words from return tokens. */ - virtual StringData get() const = 0; + FilterStopWords = 1 << 1, }; -} // namespace fts -} // namespace mongo + /** + * Process a new document, and discards any previous results. + * May be called multiple times on an instance of an iterator. + */ + virtual void reset(StringData document, Options options) = 0; + + /** + * Moves to the next token in the iterator. + * Returns false when the iterator reaches end of the document. + */ + virtual bool moveNext() = 0; + + /** + * Returns stemmed form, normalized, and lowercased depending on the parameter + * to the reset method. + * Returned StringData is valid until next call to moveNext(). + */ + virtual StringData get() const = 0; +}; + +} // namespace fts +} // namespace mongo |