diff options
Diffstat (limited to 'src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardAnalyzer.cpp')
-rw-r--r-- | src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardAnalyzer.cpp | 46 |
1 files changed, 46 insertions, 0 deletions
diff --git a/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardAnalyzer.cpp b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardAnalyzer.cpp new file mode 100644 index 000000000..e0994c41a --- /dev/null +++ b/src/assistant/3rdparty/clucene/src/CLucene/analysis/standard/StandardAnalyzer.cpp @@ -0,0 +1,46 @@ +/*------------------------------------------------------------------------------ +* Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team +* +* Distributable under the terms of either the Apache License (Version 2.0) or +* the GNU Lesser General Public License, as specified in the COPYING file. +------------------------------------------------------------------------------*/ +#include "CLucene/StdHeader.h" +#include "StandardAnalyzer.h" + +#include "CLucene/util/VoidMap.h" +#include "CLucene/util/Reader.h" +#include "CLucene/analysis/AnalysisHeader.h" +#include "CLucene/analysis/Analyzers.h" +#include "StandardFilter.h" +#include "StandardTokenizer.h" + +CL_NS_USE(util) +CL_NS_USE(analysis) + +CL_NS_DEF2(analysis,standard) + + StandardAnalyzer::StandardAnalyzer(): + stopSet(false) + { + StopFilter::fillStopTable( &stopSet,CL_NS(analysis)::StopAnalyzer::ENGLISH_STOP_WORDS); + } + + StandardAnalyzer::StandardAnalyzer( const TCHAR** stopWords): + stopSet(false) + { + StopFilter::fillStopTable( &stopSet,stopWords ); + } + + StandardAnalyzer::~StandardAnalyzer(){ + } + + + TokenStream* StandardAnalyzer::tokenStream(const TCHAR* fieldName, Reader* reader) + { + TokenStream* ret = _CLNEW StandardTokenizer(reader); + ret = _CLNEW StandardFilter(ret,true); + ret = _CLNEW LowerCaseFilter(ret,true); + ret = _CLNEW StopFilter(ret,true, &stopSet); + return ret; + } +CL_NS_END2 |