character_tokenizer.h

Go to the documentation of this file.

#ifndef META_CHARACTER_TOKENIZER_H_
#define META_CHARACTER_TOKENIZER_H_

#include "analyzers/token_stream.h"
#include "util/clonable.h"

namespace meta
{
namespace corpus
{
class document;
}
}

namespace meta
{
namespace analyzers
{
namespace tokenizers
{

class character_tokenizer
    : public util::clonable<token_stream, character_tokenizer>
{
  public:
    character_tokenizer();

    void set_content(const std::string& content) override;

    std::string next() override;

    operator bool() const override;

    const static std::string id;

  private:
    std::string content_;

    uint64_t idx_;
};
}
}
}
#endif