2013-11-26 11:55:11 +00:00
|
|
|
#pragma once
|
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Storages/MarkCache.h>
|
|
|
|
#include <Storages/MergeTree/MarkRange.h>
|
|
|
|
#include <Storages/MergeTree/MergeTreeData.h>
|
2017-06-14 10:50:22 +00:00
|
|
|
#include <Storages/MergeTree/MergeTreeRangeReader.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Core/NamesAndTypes.h>
|
2013-11-26 11:55:11 +00:00
|
|
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2016-11-20 12:43:20 +00:00
|
|
|
class IDataType;
|
|
|
|
class CachedCompressedReadBuffer;
|
|
|
|
class CompressedReadBufferFromFile;
|
2016-01-11 21:46:36 +00:00
|
|
|
|
2017-01-24 17:25:47 +00:00
|
|
|
|
|
|
|
/// Reads the data between pairs of marks in the same part. When reading consecutive ranges, avoids unnecessary seeks.
|
|
|
|
/// When ranges are almost consecutive, seeks are fast because they are performed inside the buffer.
|
|
|
|
/// Avoids loading the marks file if it is not needed (e.g. when reading the whole part).
|
2016-12-10 04:51:36 +00:00
|
|
|
class MergeTreeReader : private boost::noncopyable
|
2013-11-26 11:55:11 +00:00
|
|
|
{
|
|
|
|
public:
|
2017-04-01 07:20:54 +00:00
|
|
|
using ValueSizeMap = std::map<std::string, double>;
|
2015-12-13 04:52:13 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
MergeTreeReader(const String & path, /// Path to the directory containing the part
|
2017-12-25 21:57:29 +00:00
|
|
|
const MergeTreeData::DataPartPtr & data_part, const NamesAndTypesList & columns,
|
2017-04-01 07:20:54 +00:00
|
|
|
UncompressedCache * uncompressed_cache,
|
|
|
|
MarkCache * mark_cache,
|
|
|
|
bool save_marks_in_cache,
|
|
|
|
MergeTreeData & storage, const MarkRanges & all_mark_ranges,
|
|
|
|
size_t aio_threshold, size_t max_read_buffer_size,
|
|
|
|
const ValueSizeMap & avg_value_size_hints = ValueSizeMap{},
|
|
|
|
const ReadBufferFromFileBase::ProfileCallback & profile_callback = ReadBufferFromFileBase::ProfileCallback{},
|
|
|
|
clockid_t clock_type = CLOCK_MONOTONIC_COARSE);
|
2014-07-23 15:24:45 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
~MergeTreeReader();
|
2013-11-26 11:55:11 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
const ValueSizeMap & getAvgValueSizeHints() const;
|
2015-09-16 17:49:08 +00:00
|
|
|
|
2017-07-11 09:32:39 +00:00
|
|
|
/// Create MergeTreeRangeReader iterator, which allows reading arbitrary number of rows from range.
|
2017-06-14 10:50:22 +00:00
|
|
|
MergeTreeRangeReader readRange(size_t from_mark, size_t to_mark);
|
2013-11-26 11:55:11 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
/// Add columns from ordered_names that are not present in the block.
|
|
|
|
/// Missing columns are added in the order specified by ordered_names.
|
|
|
|
/// If at least one column was added, reorders all columns in the block according to ordered_names.
|
|
|
|
void fillMissingColumns(Block & res, const Names & ordered_names, const bool always_reorder = false);
|
2014-12-04 15:50:48 +00:00
|
|
|
|
2016-07-19 10:57:57 +00:00
|
|
|
private:
|
2017-04-01 07:20:54 +00:00
|
|
|
class Stream
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
Stream(
|
|
|
|
const String & path_prefix_, const String & extension_, size_t marks_count_,
|
|
|
|
const MarkRanges & all_mark_ranges,
|
|
|
|
MarkCache * mark_cache, bool save_marks_in_cache,
|
|
|
|
UncompressedCache * uncompressed_cache,
|
|
|
|
size_t aio_threshold, size_t max_read_buffer_size,
|
|
|
|
const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type);
|
2013-11-26 11:55:11 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void seekToMark(size_t index);
|
2013-11-26 11:55:11 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
ReadBuffer * data_buffer;
|
2016-05-04 18:04:36 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
private:
|
|
|
|
Stream() = default;
|
2016-12-10 04:51:36 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
/// NOTE: lazily loads marks from the marks cache.
|
|
|
|
const MarkInCompressedFile & getMark(size_t index);
|
2017-01-19 11:40:06 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void loadMarks();
|
2017-01-19 11:40:06 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
std::string path_prefix;
|
|
|
|
std::string extension;
|
2017-01-19 11:40:06 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
size_t marks_count;
|
2017-01-19 11:40:06 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
MarkCache * mark_cache;
|
|
|
|
bool save_marks_in_cache;
|
|
|
|
MarkCache::MappedPtr marks;
|
2017-01-19 11:40:06 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
std::unique_ptr<CachedCompressedReadBuffer> cached_buffer;
|
|
|
|
std::unique_ptr<CompressedReadBufferFromFile> non_cached_buffer;
|
|
|
|
};
|
2013-11-26 11:55:11 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
using FileStreams = std::map<std::string, std::unique_ptr<Stream>>;
|
2013-11-26 11:55:11 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
/// avg_value_size_hints are used to reduce the number of reallocations when creating columns of variable size.
|
|
|
|
ValueSizeMap avg_value_size_hints;
|
|
|
|
String path;
|
|
|
|
MergeTreeData::DataPartPtr data_part;
|
2017-01-24 20:44:12 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
FileStreams streams;
|
2015-04-09 00:37:08 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
/// Columns that are read.
|
2017-12-25 21:57:29 +00:00
|
|
|
NamesAndTypesList columns;
|
2015-04-09 00:37:08 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
UncompressedCache * uncompressed_cache;
|
|
|
|
MarkCache * mark_cache;
|
|
|
|
/// If save_marks_in_cache is false, then, if marks are not in cache, we will load them but won't save in the cache, to avoid evicting other data.
|
|
|
|
bool save_marks_in_cache;
|
2015-04-16 06:12:35 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
MergeTreeData & storage;
|
|
|
|
MarkRanges all_mark_ranges;
|
|
|
|
size_t aio_threshold;
|
|
|
|
size_t max_read_buffer_size;
|
2013-11-26 11:55:11 +00:00
|
|
|
|
2017-11-20 02:15:15 +00:00
|
|
|
void addStreams(const String & name, const IDataType & type, const MarkRanges & all_mark_ranges,
|
2017-08-07 07:31:16 +00:00
|
|
|
const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type);
|
2015-04-09 00:37:08 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void readData(
|
|
|
|
const String & name, const IDataType & type, IColumn & column,
|
2017-07-11 09:32:39 +00:00
|
|
|
size_t from_mark, bool continue_reading, size_t max_rows_to_read,
|
2017-08-07 07:31:16 +00:00
|
|
|
bool read_offsets = true);
|
2015-04-02 03:08:43 +00:00
|
|
|
|
2017-07-11 09:32:39 +00:00
|
|
|
/// Return the number of rows has been read or zero if there is no columns to read.
|
|
|
|
/// If continue_reading is true, continue reading from last state, otherwise seek to from_mark
|
|
|
|
size_t readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res);
|
2017-06-14 10:50:22 +00:00
|
|
|
|
|
|
|
friend class MergeTreeRangeReader;
|
2013-11-26 11:55:11 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
}
|