english_normalizer.h

Go to the documentation of this file.

#ifndef META_ENGLISH_NORMALIZER_H_
#define META_ENGLISH_NORMALIZER_H_

#include <deque>
#include <memory>
#include "analyzers/token_stream.h"
#include "util/clonable.h"
#include "util/optional.h"

namespace meta
{
namespace analyzers
{
namespace filters
{

class english_normalizer
    : public util::clonable<token_stream, english_normalizer>
{
  public:
    english_normalizer(std::unique_ptr<token_stream> source);

    english_normalizer(const english_normalizer& other);

    void set_content(const std::string& content) override;

    std::string next() override;

    operator bool() const override;

    const static std::string id;

  private:
    bool is_whitespace(const std::string& token) const;

    void parse_token(const std::string& token);

    uint64_t starting_quotes(uint64_t start, const std::string& token);

    bool is_quote(char c);

    uint64_t strip_dashes(uint64_t start, const std::string& token);

    uint64_t word(uint64_t start, const std::string& token);

    std::string current_token();

    std::unique_ptr<token_stream> source_;

    std::deque<std::string> tokens_;
};
}
}
}
#endif