SERVER-17520: Add support for pluggable FTS tokenizers

1 files changed, 20 insertions, 4 deletions

diff --git a/src/mongo/db/fts/fts_language.h b/src/mongo/db/fts/fts_language.h
index 3a9acbbdd94..ce45e0b812a 100644
--- a/src/mongo/db/fts/fts_language.h
+++ b/src/mongo/db/fts/fts_language.h
@@ -39,8 +39,10 @@ namespace mongo {
 
     namespace fts {
 
+        class FTSTokenizer;
+
         #define MONGO_FTS_LANGUAGE_DECLARE( language, name, version ) \
-            FTSLanguage language; \
+            BasicFTSLanguage language; \
             MONGO_INITIALIZER_GENERAL( language, MONGO_NO_PREREQUISITES, \
                                        ( "FTSAllLanguagesRegistered" ) ) \
                                      ( ::mongo::InitializerContext* context ) { \
@@ -70,6 +72,8 @@ namespace mongo {
             /** Create an uninitialized language. */
             FTSLanguage();
 
+            virtual ~FTSLanguage() {}
+
             /**
              * Returns the language as a std::string in canonical form (lowercased English name).  It is
              * an error to call str() on an uninitialized language.
@@ -77,6 +81,12 @@ namespace mongo {
             const std::string& str() const;
 
             /**
+             * Returns a new FTSTokenizer instance for this language.
+             * Lifetime is scoped to FTSLanguage (which are currently all process lifetime)
+             */
+            virtual std::unique_ptr<FTSTokenizer> createTokenizer() const = 0;
+
+            /**
              * Register std::string 'languageName' as a new language with text index version
              * 'textIndexVersion'.  Saves the resulting language to out-argument 'languageOut'.
              * Subsequent calls to FTSLanguage::make() will recognize the newly-registered language
@@ -120,9 +130,15 @@ namespace mongo {
 
         typedef StatusWith<const FTSLanguage*> StatusWithFTSLanguage;
 
-        extern FTSLanguage languagePorterV1;
-        extern FTSLanguage languageEnglishV2;
-        extern FTSLanguage languageFrenchV2;
+
+        class BasicFTSLanguage : public FTSLanguage {
+        public:
+            std::unique_ptr<FTSTokenizer> createTokenizer() const override;
+        };
+
+        extern BasicFTSLanguage languagePorterV1;
+        extern BasicFTSLanguage languageEnglishV2;
+        extern BasicFTSLanguage languageFrenchV2;
 
     }
 }