icu_tokenizer.h

Go to the documentation of this file.

#ifndef META_ICU_TOKENIZER_H_
#define META_ICU_TOKENIZER_H_

#include "analyzers/token_stream.h"
#include "util/clonable.h"
#include "util/pimpl.h"

namespace meta
{
namespace corpus
{
class document;
}
}

namespace meta
{
namespace analyzers
{
namespace tokenizers
{

class icu_tokenizer : public util::clonable<token_stream, icu_tokenizer>
{
  public:
    icu_tokenizer();

    icu_tokenizer(const icu_tokenizer& other);

    icu_tokenizer(icu_tokenizer&& other);

    ~icu_tokenizer();

    void set_content(const std::string& content) override;

    std::string next() override;

    operator bool() const override;

    const static std::string id;

  private:
    class impl;

    util::pimpl<impl> impl_;
};
}
}
}
#endif