#pragma once #include #include #include #include #include #include #include #include namespace DB { class HashedDictionary final : public IDictionary { public: HashedDictionary(const std::string & name, const DictionaryStructure & dict_struct, DictionarySourcePtr source_ptr, const DictionaryLifetime dict_lifetime) : name{name}, dict_struct(dict_struct), source_ptr{std::move(source_ptr)}, dict_lifetime(dict_lifetime) { createAttributes(); loadData(); calculateBytesAllocated(); } HashedDictionary(const HashedDictionary & other) : HashedDictionary{other.name, other.dict_struct, other.source_ptr->clone(), other.dict_lifetime} {} std::string getName() const override { return name; } std::string getTypeName() const override { return "HashedDictionary"; } std::size_t getBytesAllocated() const override { return bytes_allocated; } bool isCached() const override { return false; } DictionaryPtr clone() const override { return std::make_unique(*this); } const IDictionarySource * getSource() const override { return source_ptr.get(); } const DictionaryLifetime & getLifetime() const override { return dict_lifetime; } bool hasHierarchy() const override { return hierarchical_attribute; } id_t toParent(const id_t id) const override { const auto attr = hierarchical_attribute; const auto & map = *std::get>>(attr->maps); const auto it = map.find(id); return it != map.end() ? it->second : std::get(attr->null_values); } void toParent(const PODArray & ids, PODArray & out) const override { getItems(*hierarchical_attribute, ids, out); } #define DECLARE_INDIVIDUAL_GETTER(TYPE) \ TYPE get##TYPE(const std::string & attribute_name, const id_t id) const override\ {\ const auto & attribute = getAttribute(attribute_name);\ if (attribute.type != AttributeUnderlyingType::TYPE)\ throw Exception{\ "Type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\ ErrorCodes::TYPE_MISMATCH\ };\ \ const auto & map = *std::get>>(attribute.maps);\ const auto it = map.find(id);\ \ return it != map.end() ? TYPE{it->second} : std::get(attribute.null_values);\ } DECLARE_INDIVIDUAL_GETTER(UInt8) DECLARE_INDIVIDUAL_GETTER(UInt16) DECLARE_INDIVIDUAL_GETTER(UInt32) DECLARE_INDIVIDUAL_GETTER(UInt64) DECLARE_INDIVIDUAL_GETTER(Int8) DECLARE_INDIVIDUAL_GETTER(Int16) DECLARE_INDIVIDUAL_GETTER(Int32) DECLARE_INDIVIDUAL_GETTER(Int64) DECLARE_INDIVIDUAL_GETTER(Float32) DECLARE_INDIVIDUAL_GETTER(Float64) #undef DECLARE_INDIVIDUAL_GETTER String getString(const std::string & attribute_name, const id_t id) const override { const auto & attribute = getAttribute(attribute_name); if (attribute.type != AttributeUnderlyingType::String) throw Exception{ "Type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH }; const auto & map = *std::get>>(attribute.maps); const auto it = map.find(id); return it != map.end() ? String{it->second} : std::get(attribute.null_values); } #define DECLARE_MULTIPLE_GETTER(TYPE)\ void get##TYPE(const std::string & attribute_name, const PODArray & ids, PODArray & out) const override\ {\ const auto & attribute = getAttribute(attribute_name);\ if (attribute.type != AttributeUnderlyingType::TYPE)\ throw Exception{\ "Type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\ ErrorCodes::TYPE_MISMATCH\ };\ \ getItems(attribute, ids, out);\ } DECLARE_MULTIPLE_GETTER(UInt8) DECLARE_MULTIPLE_GETTER(UInt16) DECLARE_MULTIPLE_GETTER(UInt32) DECLARE_MULTIPLE_GETTER(UInt64) DECLARE_MULTIPLE_GETTER(Int8) DECLARE_MULTIPLE_GETTER(Int16) DECLARE_MULTIPLE_GETTER(Int32) DECLARE_MULTIPLE_GETTER(Int64) DECLARE_MULTIPLE_GETTER(Float32) DECLARE_MULTIPLE_GETTER(Float64) #undef DECLARE_MULTIPLE_GETTER void getString(const std::string & attribute_name, const PODArray & ids, ColumnString * out) const override { const auto & attribute = getAttribute(attribute_name); if (attribute.type != AttributeUnderlyingType::String) throw Exception{ "Type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH }; const auto & attr = *std::get>>(attribute.maps); const auto & null_value = std::get(attribute.null_values); for (const auto i : ext::range(0, ids.size())) { const auto it = attr.find(ids[i]); const auto string_ref = it != attr.end() ? it->second : StringRef{null_value}; out->insertData(string_ref.data, string_ref.size); } } private: struct attribute_t final { AttributeUnderlyingType type; std::tuple null_values; std::tuple>, std::unique_ptr>, std::unique_ptr>, std::unique_ptr>, std::unique_ptr>, std::unique_ptr>, std::unique_ptr>, std::unique_ptr>, std::unique_ptr>, std::unique_ptr>, std::unique_ptr>> maps; std::unique_ptr string_arena; }; void createAttributes() { const auto size = dict_struct.attributes.size(); attributes.reserve(size); for (const auto & attribute : dict_struct.attributes) { attribute_index_by_name.emplace(attribute.name, attributes.size()); attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value)); if (attribute.hierarchical) { hierarchical_attribute = &attributes.back(); if (hierarchical_attribute->type != AttributeUnderlyingType::UInt64) throw Exception{ "Hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH }; } } } void loadData() { auto stream = source_ptr->loadAll(); stream->readPrefix(); while (const auto block = stream->read()) { const auto & id_column = *block.getByPosition(0).column; for (const auto attribute_idx : ext::range(0, attributes.size())) { const auto & attribute_column = *block.getByPosition(attribute_idx + 1).column; auto & attribute = attributes[attribute_idx]; for (const auto row_idx : ext::range(0, id_column.size())) setAttributeValue(attribute, id_column[row_idx].get(), attribute_column[row_idx]); } } stream->readSuffix(); } template void addAttributeSize(const attribute_t & attribute) { const auto & map_ref = std::get>>(attribute.maps); bytes_allocated += sizeof(HashMap) + map_ref->getBufferSizeInBytes(); } void calculateBytesAllocated() { bytes_allocated += attributes.size() * sizeof(attributes.front()); for (const auto & attribute : attributes) { switch (attribute.type) { case AttributeUnderlyingType::UInt8: addAttributeSize(attribute); break; case AttributeUnderlyingType::UInt16: addAttributeSize(attribute); break; case AttributeUnderlyingType::UInt32: addAttributeSize(attribute); break; case AttributeUnderlyingType::UInt64: addAttributeSize(attribute); break; case AttributeUnderlyingType::Int8: addAttributeSize(attribute); break; case AttributeUnderlyingType::Int16: addAttributeSize(attribute); break; case AttributeUnderlyingType::Int32: addAttributeSize(attribute); break; case AttributeUnderlyingType::Int64: addAttributeSize(attribute); break; case AttributeUnderlyingType::Float32: addAttributeSize(attribute); break; case AttributeUnderlyingType::Float64: addAttributeSize(attribute); break; case AttributeUnderlyingType::String: { addAttributeSize(attribute); bytes_allocated += sizeof(Arena) + attribute.string_arena->size(); break; } } } } template void createAttributeImpl(attribute_t & attribute, const Field & null_value) { std::get(attribute.null_values) = null_value.get::Type>(); std::get>>(attribute.maps) = std::make_unique>(); } attribute_t createAttributeWithType(const AttributeUnderlyingType type, const Field & null_value) { attribute_t attr{type}; switch (type) { case AttributeUnderlyingType::UInt8: createAttributeImpl(attr, null_value); break; case AttributeUnderlyingType::UInt16: createAttributeImpl(attr, null_value); break; case AttributeUnderlyingType::UInt32: createAttributeImpl(attr, null_value); break; case AttributeUnderlyingType::UInt64: createAttributeImpl(attr, null_value); break; case AttributeUnderlyingType::Int8: createAttributeImpl(attr, null_value); break; case AttributeUnderlyingType::Int16: createAttributeImpl(attr, null_value); break; case AttributeUnderlyingType::Int32: createAttributeImpl(attr, null_value); break; case AttributeUnderlyingType::Int64: createAttributeImpl(attr, null_value); break; case AttributeUnderlyingType::Float32: createAttributeImpl(attr, null_value); break; case AttributeUnderlyingType::Float64: createAttributeImpl(attr, null_value); break; case AttributeUnderlyingType::String: { const auto & null_value_ref = std::get(attr.null_values) = null_value.get(); std::get>>(attr.maps) = std::make_unique>(); attr.string_arena = std::make_unique(); break; } } return attr; } template void getItems(const attribute_t & attribute, const PODArray & ids, PODArray & out) const { const auto & attr = *std::get>>(attribute.maps); const auto null_value = std::get(attribute.null_values); for (const auto i : ext::range(0, ids.size())) { const auto it = attr.find(ids[i]); out[i] = it != attr.end() ? it->second : null_value; } } template void setAttributeValueImpl(attribute_t & attribute, const id_t id, const T value) { auto & map = *std::get>>(attribute.maps); map.insert({ id, value }); } void setAttributeValue(attribute_t & attribute, const id_t id, const Field & value) { switch (attribute.type) { case AttributeUnderlyingType::UInt8: setAttributeValueImpl(attribute, id, value.get()); break; case AttributeUnderlyingType::UInt16: setAttributeValueImpl(attribute, id, value.get()); break; case AttributeUnderlyingType::UInt32: setAttributeValueImpl(attribute, id, value.get()); break; case AttributeUnderlyingType::UInt64: setAttributeValueImpl(attribute, id, value.get()); break; case AttributeUnderlyingType::Int8: setAttributeValueImpl(attribute, id, value.get()); break; case AttributeUnderlyingType::Int16: setAttributeValueImpl(attribute, id, value.get()); break; case AttributeUnderlyingType::Int32: setAttributeValueImpl(attribute, id, value.get()); break; case AttributeUnderlyingType::Int64: setAttributeValueImpl(attribute, id, value.get()); break; case AttributeUnderlyingType::Float32: setAttributeValueImpl(attribute, id, value.get()); break; case AttributeUnderlyingType::Float64: setAttributeValueImpl(attribute, id, value.get()); break; case AttributeUnderlyingType::String: { auto & map = *std::get>>(attribute.maps); const auto & string = value.get(); const auto string_in_arena = attribute.string_arena->insert(string.data(), string.size()); map.insert({ id, StringRef{string_in_arena, string.size()} }); break; } } } const attribute_t & getAttribute(const std::string & attribute_name) const { const auto it = attribute_index_by_name.find(attribute_name); if (it == std::end(attribute_index_by_name)) throw Exception{ "No such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS }; return attributes[it->second]; } const std::string name; const DictionaryStructure dict_struct; const DictionarySourcePtr source_ptr; const DictionaryLifetime dict_lifetime; std::map attribute_index_by_name; std::vector attributes; const attribute_t * hierarchical_attribute = nullptr; std::size_t bytes_allocated = 0; }; }