chunk_handler.h

Go to the documentation of this file.

#ifndef META_INDEX_CHUNK_HANDLER_H_
#define META_INDEX_CHUNK_HANDLER_H_

#include <atomic>
#include <functional>
#include <mutex>
#include <queue>
#include <stdexcept>
#include <unordered_set>
#include <utility>
#include <vector>

#include "index/chunk.h"
#include "util/optional.h"

namespace meta
{
namespace index
{

template <class Index>
class chunk_handler
{
  public:
    using index_pdata_type = typename Index::index_pdata_type;
    using primary_key_type = typename index_pdata_type::primary_key_type;
    using secondary_key_type = typename index_pdata_type::secondary_key_type;
    using chunk_t = chunk<primary_key_type, secondary_key_type>;

    class producer
    {
      public:
        producer(chunk_handler* parent);

        template <class Container>
        void operator()(const secondary_key_type& key, const Container& counts);

        ~producer();

      private:
        void flush_chunk();

        std::unordered_set<index_pdata_type> pdata_;

        uint64_t chunk_size_;

        const static uint64_t constexpr max_size = 1024 * 1024 * 128; // 128 MB

        chunk_handler* parent_;
    };

    chunk_handler(const std::string& prefix);

    producer make_producer();

    uint32_t size() const;

    uint64_t final_size() const;

    void merge_chunks();

    uint64_t unique_primary_keys() const;

    class chunk_handler_exception : public std::runtime_error
    {
        using std::runtime_error::runtime_error;
    };

  private:
    void write_chunk(std::vector<index_pdata_type>& pdata);

    std::string prefix_;

    std::atomic<uint32_t> chunk_num_{0};

    std::priority_queue<chunk_t> chunks_;

    mutable std::mutex mutables_;

    util::optional<uint64_t> unique_primary_keys_;
};
}
}

#include "index/chunk_handler.tcc"
#endif