language_model.h

Go to the documentation of this file.

#ifndef META_LANGUAGE_MODEL_H_
#define META_LANGUAGE_MODEL_H_

#include <deque>
#include <memory>
#include <string>
#include <unordered_map>

namespace meta
{
namespace lm
{
class language_model
{
  public:
    language_model(const std::string& config_file);

    language_model(const std::string& config_file, size_t n);

    std::string generate(unsigned int seed) const;

    std::string next_token(const std::deque<std::string>& tokens,
                           double random) const;

    double perplexity(const std::string& tokens) const;

    double perplexity_per_word(const std::string& tokens) const;

    double prob(std::deque<std::string> tokens) const;

  private:

    void learn_model(const std::string& config_file);

    std::string make_string(const std::deque<std::string>& tokens) const;

    std::deque<std::string> make_deque(const std::string& tokens) const;

    std::unique_ptr<language_model> interp_;

    std::unordered_map
        <std::string, std::unordered_map<std::string, double>> dist_;

    size_t N_;

    constexpr static double lambda_ = 0.7;
};
}
}

#endif