postings_data.h

Go to the documentation of this file.

#ifndef META_POSTINGS_DATA_
#define META_POSTINGS_DATA_

#include <fstream>
#include <limits>
#include <utility>
#include <vector>

#include "meta.h"
#include "io/compressed_file_reader.h"
#include "io/compressed_file_writer.h"
#include "util/sparse_vector.h"

namespace meta
{
namespace index
{

template <class, class>
class postings_data;

template <class PrimaryKey, class SecondaryKey>
io::compressed_file_reader& operator>>(io::compressed_file_reader&,
                                       postings_data<PrimaryKey,
                                                     SecondaryKey>&);

template <class PrimaryKey, class SecondaryKey>
class postings_data
{
  public:
    using primary_key_type = PrimaryKey;
    using secondary_key_type = SecondaryKey;
    using pair_t = std::pair<SecondaryKey, double>;
    using count_t = std::vector<pair_t>;

    static_assert(
        (std::is_integral<PrimaryKey>::value
         || std::is_base_of<util::numeric, PrimaryKey>::value
         || std::is_same<PrimaryKey, std::string>::value)
        &&
        (std::is_integral<SecondaryKey>::value
         || std::is_base_of<util::numeric, SecondaryKey>::value),
        "primary and secondary keys in postings data must be numeric types");

    static_assert(sizeof(uint64_t) == sizeof(double),
                  "sizeof(uint64_t) must equal sizeof(double) since "
                  "reinterpret_cast is used in postings_data");

    postings_data() = default;

    postings_data(PrimaryKey p_id);

    void merge_with(postings_data& other);

    void increase_count(SecondaryKey s_id, double amount);

    double count(SecondaryKey s_id) const;

    const count_t& counts() const;

    void set_counts(const count_t& counts);

    bool operator<(const postings_data& other) const;

    friend void stream_helper(io::compressed_file_reader& in,
                              postings_data<PrimaryKey, SecondaryKey>& pd)
    {
        pd.counts_.clear();
        uint32_t num_pairs = in.next();
        for (uint32_t i = 0; i < num_pairs; ++i)
        {
            SecondaryKey s_id = SecondaryKey{in.next()};
            uint64_t count = in.next();
            pd.counts_.emplace_back(s_id, static_cast<double>(count));
        }
    }

    friend io::compressed_file_reader& operator>>
        <>(io::compressed_file_reader& in,
           postings_data<PrimaryKey, SecondaryKey>& pd);

    friend io::compressed_file_writer& operator<<(
        io::compressed_file_writer& out,
        const postings_data<PrimaryKey, SecondaryKey>& pd)
    {
        if (pd.counts_.empty())
            return out;

        out.write(pd.p_id_);
        uint32_t size = pd.counts_.size();
        out.write(size);
        for (auto& p : pd.counts_)
        {
            out.write(p.first);
            out.write(p.second);
        }

        return out;
    }

    void write_compressed(io::compressed_file_writer& writer) const;

    void read_compressed(io::compressed_file_reader& reader);

    void write_libsvm(std::ofstream& out) const
    {
        out << p_id_;
        for (auto& c : counts_)
            out << ' ' << (c.first + 1) << ':' << c.second;
        out << '\n';
    }

    PrimaryKey primary_key() const;

    void set_primary_key(PrimaryKey new_key);

    uint64_t inverse_frequency() const;

    uint64_t bytes_used() const;

  private:
    PrimaryKey p_id_;

    util::sparse_vector<SecondaryKey, double> counts_;

    const static uint64_t delimiter_ = std::numeric_limits<uint64_t>::max();
};

template <class PrimaryKey, class SecondaryKey>
io::compressed_file_reader& operator>>(io::compressed_file_reader& in,
                                       postings_data<PrimaryKey,
                                                     SecondaryKey>& pd)
{
    pd.p_id_ = in.next();
    stream_helper(in, pd);
    return in;
}

template <>
inline io::compressed_file_reader& operator>>
    <>(io::compressed_file_reader& in, postings_data<std::string, doc_id>& pd)
{
    pd.p_id_ = in.next_string();
    stream_helper(in, pd);
    return in;
}

template <class PrimaryKey, class SecondaryKey>
bool operator==(const postings_data<PrimaryKey, SecondaryKey>& lhs,
                const postings_data<PrimaryKey, SecondaryKey>& rhs);
}
}

namespace std
{
template <class PrimaryKey, class SecondaryKey>
struct hash<meta::index::postings_data<PrimaryKey, SecondaryKey>>
{
    using pdata_t = meta::index::postings_data<PrimaryKey, SecondaryKey>;
    size_t operator()(const pdata_t& pd) const
    {
        return std::hash<PrimaryKey>{}(pd.primary_key());
    }
};
}

#include "index/postings_data.tcc"
#endif