ptb_normalizer.h

Go to the documentation of this file.

#ifndef META_PTB_NORMALIZER_H_
#define META_PTB_NORMALIZER_H_

#include <deque>
#include <memory>
#include "analyzers/token_stream.h"
#include "util/clonable.h"

namespace meta
{
namespace analyzers
{
namespace filters
{

class ptb_normalizer : public util::clonable<token_stream, ptb_normalizer>
{
  public:
    ptb_normalizer(std::unique_ptr<token_stream> source);

    ptb_normalizer(const ptb_normalizer& other);

    void set_content(const std::string& content) override;

    std::string next() override;

    operator bool() const override;

    const static std::string id;

  private:
    std::string current_token();

    void parse_token(const std::string& token);

    std::unique_ptr<token_stream> source_;

    std::deque<std::string> tokens_;
};
}
}
}
#endif