sentence_boundary.h

Go to the documentation of this file.

#ifndef META_SENTENCE_BOUNDARY_H_
#define META_SENTENCE_BOUNDARY_H_

#include <deque>
#include <memory>
#include <unordered_set>

#include "analyzers/filter_factory.h"
#include "util/clonable.h"
#include "util/optional.h"

namespace cpptoml
{
class table;
}

namespace meta
{
namespace analyzers
{
namespace filters
{

class sentence_boundary : public util::clonable<token_stream, sentence_boundary>
{
  public:
    static void load_heuristics(const cpptoml::table& config);

    sentence_boundary(std::unique_ptr<token_stream> source);

    sentence_boundary(const sentence_boundary& other);

    void set_content(const std::string& content) override;

    std::string next() override;

    operator bool() const override;

    const static std::string id;

  private:
    std::string current_token();

    static bool possible_punc(const std::string& token);

    static bool possible_end(const std::string& token);

    static bool possible_start(const std::string& token);

    std::unique_ptr<token_stream> source_;

    std::deque<std::string> tokens_;

    util::optional<std::string> prev_;

    static std::unordered_set<std::string> punc_set;

    static std::unordered_set<std::string> start_exception_set;

    static std::unordered_set<std::string> end_exception_set;

    static bool heuristics_loaded;
};

template <>
std::unique_ptr<token_stream>
    make_filter<sentence_boundary>(std::unique_ptr<token_stream>,
                                   const cpptoml::table&);
}
}
}
#endif