ngram_word_analyzer.h

Go to the documentation of this file.

#ifndef META_NGRAM_WORD_ANALYZER_H_
#define META_NGRAM_WORD_ANALYZER_H_

#include "analyzers/analyzer_factory.h"
#include "analyzers/ngram/ngram_analyzer.h"
#include "util/clonable.h"

namespace meta
{
namespace analyzers
{

class ngram_word_analyzer
    : public util::multilevel_clonable<analyzer, ngram_analyzer,
                                       ngram_word_analyzer>
{
    using base = util::multilevel_clonable<analyzer, ngram_analyzer,
                                           ngram_word_analyzer>;

  public:
    ngram_word_analyzer(uint16_t n, std::unique_ptr<token_stream> stream);

    ngram_word_analyzer(const ngram_word_analyzer& other);

    virtual void tokenize(corpus::document& doc) override;

    const static std::string id;

  private:
    std::unique_ptr<token_stream> stream_;
};

template <>
std::unique_ptr<analyzer>
    make_analyzer<ngram_word_analyzer>(const cpptoml::table&,
                                       const cpptoml::table&);
}
}
#endif