SERVER-17620 RLP Tokenizer

3 files changed, 8 insertions, 6 deletions

diff --git a/src/mongo/db/fts/fts_basic_tokenizer.cpp b/src/mongo/db/fts/fts_basic_tokenizer.cpp
index b0b1448ab60..80509b68147 100644
--- a/src/mongo/db/fts/fts_basic_tokenizer.cpp
+++ b/src/mongo/db/fts/fts_basic_tokenizer.cpp
@@ -48,9 +48,10 @@ namespace fts {
         : _language(language), _stemmer(language), _stopWords(StopWords::getStopWords(language)) {
     }
 
-    void BasicFTSTokenizer::reset(const char* document, Options options) {
+    void BasicFTSTokenizer::reset(StringData document, Options options) {
         _options = options;
-        _tokenizer = stdx::make_unique<Tokenizer>(_language, document);
+        _document = document.toString();
+        _tokenizer = stdx::make_unique<Tokenizer>(_language, _document);
     }
 
     bool BasicFTSTokenizer::moveNext() {
diff --git a/src/mongo/db/fts/fts_basic_tokenizer.h b/src/mongo/db/fts/fts_basic_tokenizer.h
index 7e89289488f..45b3ad8e074 100644
--- a/src/mongo/db/fts/fts_basic_tokenizer.h
+++ b/src/mongo/db/fts/fts_basic_tokenizer.h
@@ -58,17 +58,18 @@ namespace fts {
     public:
         BasicFTSTokenizer(const FTSLanguage* language);
 
-        void reset(const char* document, Options options) override;
+        void reset(StringData document, Options options) final;
 
-        bool moveNext() override;
+        bool moveNext() final;
 
-        StringData get() const override;
+        StringData get() const final;
 
     private:
         const FTSLanguage* const _language;
         const Stemmer _stemmer;
         const StopWords* const _stopWords;
 
+        std::string _document;
         std::unique_ptr<Tokenizer> _tokenizer;
         Options _options;
 
diff --git a/src/mongo/db/fts/fts_tokenizer.h b/src/mongo/db/fts/fts_tokenizer.h
index 6db2014b5fb..2b345d89266 100644
--- a/src/mongo/db/fts/fts_tokenizer.h
+++ b/src/mongo/db/fts/fts_tokenizer.h
@@ -72,7 +72,7 @@ namespace fts {
          * Process a new document, and discards any previous results.
          * May be called multiple times on an instance of an iterator.
          */
-        virtual void reset(const char* document, Options options) = 0;
+        virtual void reset(StringData document, Options options) = 0;
 
         /**
          * Moves to the next token in the iterator.