summaryrefslogtreecommitdiff
path: root/src/mongo/db/fts/fts_tokenizer.h
diff options
context:
space:
mode:
Diffstat (limited to 'src/mongo/db/fts/fts_tokenizer.h')
-rw-r--r--src/mongo/db/fts/fts_tokenizer.h84
1 files changed, 42 insertions, 42 deletions
diff --git a/src/mongo/db/fts/fts_tokenizer.h b/src/mongo/db/fts/fts_tokenizer.h
index 2b345d89266..40cdbde2cb8 100644
--- a/src/mongo/db/fts/fts_tokenizer.h
+++ b/src/mongo/db/fts/fts_tokenizer.h
@@ -35,58 +35,58 @@
namespace mongo {
namespace fts {
- class FTSLanguage;
- class StopWords;
+class FTSLanguage;
+class StopWords;
+
+/**
+ * FTSTokenizer
+ * A iterator of "documents" where a document contains space delimited words.
+ * For each word returns a stem or lemma version of a word optimized for full text indexing.
+ * Supports various options to control how tokens are generated.
+ */
+class FTSTokenizer {
+public:
+ virtual ~FTSTokenizer() = default;
/**
- * FTSTokenizer
- * A iterator of "documents" where a document contains space delimited words.
- * For each word returns a stem or lemma version of a word optimized for full text indexing.
- * Supports various options to control how tokens are generated.
+ * Options for generating tokens
*/
- class FTSTokenizer {
- public:
- virtual ~FTSTokenizer() = default;
-
- /**
- * Options for generating tokens
- */
- enum Options {
- /**
- * Default means lower cased, and stop words are not filtered.
- */
- None = 0,
-
- /**
- * Do not lower case terms.
- */
- GenerateCaseSensitiveTokens = 1 << 0,
-
- /**
- * Filter out stop words from return tokens.
- */
- FilterStopWords = 1 << 1,
- };
-
+ enum Options {
/**
- * Process a new document, and discards any previous results.
- * May be called multiple times on an instance of an iterator.
+ * Default means lower cased, and stop words are not filtered.
*/
- virtual void reset(StringData document, Options options) = 0;
+ None = 0,
/**
- * Moves to the next token in the iterator.
- * Returns false when the iterator reaches end of the document.
+ * Do not lower case terms.
*/
- virtual bool moveNext() = 0;
+ GenerateCaseSensitiveTokens = 1 << 0,
/**
- * Returns stemmed form, normalized, and lowercased depending on the parameter
- * to the reset method.
- * Returned StringData is valid until next call to moveNext().
+ * Filter out stop words from return tokens.
*/
- virtual StringData get() const = 0;
+ FilterStopWords = 1 << 1,
};
-} // namespace fts
-} // namespace mongo
+ /**
+ * Process a new document, and discards any previous results.
+ * May be called multiple times on an instance of an iterator.
+ */
+ virtual void reset(StringData document, Options options) = 0;
+
+ /**
+ * Moves to the next token in the iterator.
+ * Returns false when the iterator reaches end of the document.
+ */
+ virtual bool moveNext() = 0;
+
+ /**
+ * Returns stemmed form, normalized, and lowercased depending on the parameter
+ * to the reset method.
+ * Returned StringData is valid until next call to moveNext().
+ */
+ virtual StringData get() const = 0;
+};
+
+} // namespace fts
+} // namespace mongo