2015-07-13 16:18:28 +00:00
|
|
|
#pragma once
|
|
|
|
|
2021-01-23 13:18:24 +00:00
|
|
|
#include <atomic>
|
|
|
|
#include <memory>
|
|
|
|
#include <variant>
|
|
|
|
#include <optional>
|
|
|
|
|
2018-10-08 19:45:17 +00:00
|
|
|
#include <Columns/ColumnDecimal.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Columns/ColumnString.h>
|
2018-12-10 15:25:45 +00:00
|
|
|
#include <Common/HashTable/HashMap.h>
|
2021-01-04 21:07:45 +00:00
|
|
|
#include <Common/HashTable/HashSet.h>
|
2018-12-10 15:25:45 +00:00
|
|
|
#include "DictionaryStructure.h"
|
|
|
|
#include "IDictionary.h"
|
|
|
|
#include "IDictionarySource.h"
|
2021-01-23 13:18:24 +00:00
|
|
|
#include "DictionaryHelpers.h"
|
2015-07-13 16:18:28 +00:00
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
2015-07-13 17:10:16 +00:00
|
|
|
class RangeHashedDictionary final : public IDictionaryBase
|
2015-07-13 16:18:28 +00:00
|
|
|
{
|
|
|
|
public:
|
2017-04-01 07:20:54 +00:00
|
|
|
RangeHashedDictionary(
|
2020-07-14 18:46:29 +00:00
|
|
|
const StorageID & dict_id_,
|
2019-08-03 11:02:40 +00:00
|
|
|
const DictionaryStructure & dict_struct_,
|
|
|
|
DictionarySourcePtr source_ptr_,
|
|
|
|
const DictionaryLifetime dict_lifetime_,
|
|
|
|
bool require_nonempty_);
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
std::string getTypeName() const override { return "RangeHashed"; }
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-07-21 06:35:58 +00:00
|
|
|
size_t getBytesAllocated() const override { return bytes_allocated; }
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-07-21 06:35:58 +00:00
|
|
|
size_t getQueryCount() const override { return query_count.load(std::memory_order_relaxed); }
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
double getHitRate() const override { return 1.0; }
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-07-21 06:35:58 +00:00
|
|
|
size_t getElementCount() const override { return element_count; }
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
double getLoadFactor() const override { return static_cast<double>(element_count) / bucket_count; }
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2019-06-02 12:11:01 +00:00
|
|
|
std::shared_ptr<const IExternalLoadable> clone() const override
|
2019-01-19 23:27:52 +00:00
|
|
|
{
|
2020-07-16 14:25:39 +00:00
|
|
|
return std::make_shared<RangeHashedDictionary>(getDictionaryID(), dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty);
|
2019-01-19 23:27:52 +00:00
|
|
|
}
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
const IDictionarySource * getSource() const override { return source_ptr.get(); }
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
const DictionaryLifetime & getLifetime() const override { return dict_lifetime; }
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
const DictionaryStructure & getStructure() const override { return dict_struct; }
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
bool isInjective(const std::string & attribute_name) const override
|
|
|
|
{
|
|
|
|
return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective;
|
|
|
|
}
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2021-01-23 13:18:24 +00:00
|
|
|
DictionaryKeyType getKeyType() const override { return DictionaryKeyType::range; }
|
2018-09-13 13:33:44 +00:00
|
|
|
|
2020-12-20 20:11:28 +00:00
|
|
|
ColumnPtr getColumn(
|
|
|
|
const std::string& attribute_name,
|
|
|
|
const DataTypePtr & result_type,
|
|
|
|
const Columns & key_columns,
|
|
|
|
const DataTypes & key_types,
|
2021-02-16 21:33:02 +00:00
|
|
|
const ColumnPtr & default_values_column) const override;
|
2020-12-20 20:11:28 +00:00
|
|
|
|
2021-01-23 13:18:24 +00:00
|
|
|
ColumnUInt8::Ptr hasKeys(const Columns & key_columns, const DataTypes & key_types) const override;
|
2020-12-20 20:11:28 +00:00
|
|
|
|
|
|
|
using RangeStorageType = Int64;
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2019-02-18 18:51:46 +00:00
|
|
|
BlockInputStreamPtr getBlockInputStream(const Names & column_names, size_t max_block_size) const override;
|
2017-05-04 18:14:23 +00:00
|
|
|
|
2018-09-13 13:33:44 +00:00
|
|
|
struct Range
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2018-09-13 13:33:44 +00:00
|
|
|
RangeStorageType left;
|
|
|
|
RangeStorageType right;
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2018-09-13 13:33:44 +00:00
|
|
|
static bool isCorrectDate(const RangeStorageType & date);
|
2018-12-10 15:25:45 +00:00
|
|
|
bool contains(const RangeStorageType & value) const;
|
2017-04-01 07:20:54 +00:00
|
|
|
};
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-12-25 19:00:48 +00:00
|
|
|
private:
|
2017-04-01 07:20:54 +00:00
|
|
|
template <typename T>
|
|
|
|
struct Value final
|
|
|
|
{
|
|
|
|
Range range;
|
2021-01-04 21:07:45 +00:00
|
|
|
std::optional<T> value;
|
2017-04-01 07:20:54 +00:00
|
|
|
};
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2018-12-10 15:25:45 +00:00
|
|
|
template <typename T>
|
|
|
|
using Values = std::vector<Value<T>>;
|
|
|
|
template <typename T>
|
|
|
|
using Collection = HashMap<UInt64, Values<T>>;
|
|
|
|
template <typename T>
|
|
|
|
using Ptr = std::unique_ptr<Collection<T>>;
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2021-01-04 21:07:45 +00:00
|
|
|
using NullableSet = HashSet<Key, DefaultHash<Key>>;
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
struct Attribute final
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
AttributeUnderlyingType type;
|
2021-01-04 21:07:45 +00:00
|
|
|
bool is_nullable;
|
|
|
|
|
2018-12-10 15:25:45 +00:00
|
|
|
std::variant<
|
|
|
|
UInt8,
|
|
|
|
UInt16,
|
|
|
|
UInt32,
|
|
|
|
UInt64,
|
|
|
|
UInt128,
|
|
|
|
Int8,
|
|
|
|
Int16,
|
|
|
|
Int32,
|
|
|
|
Int64,
|
|
|
|
Decimal32,
|
|
|
|
Decimal64,
|
|
|
|
Decimal128,
|
|
|
|
Float32,
|
|
|
|
Float64,
|
2020-12-20 20:11:28 +00:00
|
|
|
StringRef>
|
2018-12-10 15:25:45 +00:00
|
|
|
null_values;
|
|
|
|
std::variant<
|
|
|
|
Ptr<UInt8>,
|
|
|
|
Ptr<UInt16>,
|
|
|
|
Ptr<UInt32>,
|
|
|
|
Ptr<UInt64>,
|
|
|
|
Ptr<UInt128>,
|
|
|
|
Ptr<Int8>,
|
|
|
|
Ptr<Int16>,
|
|
|
|
Ptr<Int32>,
|
|
|
|
Ptr<Int64>,
|
|
|
|
Ptr<Decimal32>,
|
|
|
|
Ptr<Decimal64>,
|
|
|
|
Ptr<Decimal128>,
|
|
|
|
Ptr<Float32>,
|
|
|
|
Ptr<Float64>,
|
|
|
|
Ptr<StringRef>>
|
|
|
|
maps;
|
2017-04-01 07:20:54 +00:00
|
|
|
std::unique_ptr<Arena> string_arena;
|
|
|
|
};
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void createAttributes();
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void loadData();
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
template <typename T>
|
|
|
|
void addAttributeSize(const Attribute & attribute);
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void calculateBytesAllocated();
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
template <typename T>
|
2021-01-26 10:47:12 +00:00
|
|
|
static void createAttributeImpl(Attribute & attribute, const Field & null_value);
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2021-01-26 10:47:12 +00:00
|
|
|
static Attribute createAttribute(const DictionaryAttribute& attribute, const Field & null_value);
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2021-01-23 16:47:33 +00:00
|
|
|
template <typename AttributeType, typename OutputType, typename ValueSetter, typename DefaultValueExtractor>
|
2017-04-01 07:20:54 +00:00
|
|
|
void getItemsImpl(
|
|
|
|
const Attribute & attribute,
|
2020-12-20 20:11:28 +00:00
|
|
|
const Columns & key_columns,
|
|
|
|
ValueSetter && set_value,
|
2021-01-23 16:47:33 +00:00
|
|
|
DefaultValueExtractor & default_value_extractor) const;
|
2016-06-07 19:11:04 +00:00
|
|
|
|
2021-01-29 14:48:13 +00:00
|
|
|
template <typename AttributeType>
|
|
|
|
ColumnUInt8::Ptr hasKeysImpl(
|
|
|
|
const Attribute & attribute,
|
|
|
|
const PaddedPODArray<Key> & ids,
|
|
|
|
const PaddedPODArray<RangeStorageType> & dates) const;
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
template <typename T>
|
2021-01-26 10:47:12 +00:00
|
|
|
static void setAttributeValueImpl(Attribute & attribute, const Key id, const Range & range, const Field & value);
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2021-01-26 10:47:12 +00:00
|
|
|
static void setAttributeValue(Attribute & attribute, const Key id, const Range & range, const Field & value);
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
const Attribute & getAttribute(const std::string & attribute_name) const;
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
const Attribute & getAttributeWithType(const std::string & name, const AttributeUnderlyingType type) const;
|
2015-07-14 13:06:25 +00:00
|
|
|
|
2018-09-17 15:04:57 +00:00
|
|
|
template <typename RangeType>
|
2018-12-10 15:25:45 +00:00
|
|
|
void getIdsAndDates(PaddedPODArray<Key> & ids, PaddedPODArray<RangeType> & start_dates, PaddedPODArray<RangeType> & end_dates) const;
|
2017-05-04 18:14:23 +00:00
|
|
|
|
2018-09-17 15:04:57 +00:00
|
|
|
template <typename T, typename RangeType>
|
2018-12-10 15:25:45 +00:00
|
|
|
void getIdsAndDates(
|
|
|
|
const Attribute & attribute,
|
|
|
|
PaddedPODArray<Key> & ids,
|
|
|
|
PaddedPODArray<RangeType> & start_dates,
|
|
|
|
PaddedPODArray<RangeType> & end_dates) const;
|
2018-09-17 15:04:57 +00:00
|
|
|
|
|
|
|
template <typename RangeType>
|
2019-02-18 18:51:46 +00:00
|
|
|
BlockInputStreamPtr getBlockInputStreamImpl(const Names & column_names, size_t max_block_size) const;
|
2018-09-17 15:04:57 +00:00
|
|
|
|
|
|
|
friend struct RangeHashedDIctionaryCallGetBlockInputStreamImpl;
|
2017-05-04 18:14:23 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
const DictionaryStructure dict_struct;
|
|
|
|
const DictionarySourcePtr source_ptr;
|
|
|
|
const DictionaryLifetime dict_lifetime;
|
|
|
|
const bool require_nonempty;
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-07-21 06:35:58 +00:00
|
|
|
std::map<std::string, size_t> attribute_index_by_name;
|
2017-04-01 07:20:54 +00:00
|
|
|
std::vector<Attribute> attributes;
|
2015-07-13 16:18:28 +00:00
|
|
|
|
2017-07-21 06:35:58 +00:00
|
|
|
size_t bytes_allocated = 0;
|
|
|
|
size_t element_count = 0;
|
|
|
|
size_t bucket_count = 0;
|
|
|
|
mutable std::atomic<size_t> query_count{0};
|
2015-07-13 16:18:28 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
}
|