From 8ea2990e7deb5d819ebdb4b3da3b419b3abcbcc1 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Fri, 10 Jul 2015 17:43:49 +0300 Subject: [PATCH] dbms: ExternalDictionaries: remove most invididual getters --- .../include/DB/Dictionaries/CacheDictionary.h | 50 ---------------- dbms/include/DB/Dictionaries/FlatDictionary.h | 53 ----------------- .../DB/Dictionaries/HashedDictionary.h | 57 +------------------ dbms/include/DB/Dictionaries/IDictionary.h | 26 ++++----- .../DB/Functions/FunctionsDictionaries.h | 55 +++++++++--------- 5 files changed, 39 insertions(+), 202 deletions(-) diff --git a/dbms/include/DB/Dictionaries/CacheDictionary.h b/dbms/include/DB/Dictionaries/CacheDictionary.h index 766b4b4fd5a..07db7c0dda2 100644 --- a/dbms/include/DB/Dictionaries/CacheDictionary.h +++ b/dbms/include/DB/Dictionaries/CacheDictionary.h @@ -87,61 +87,11 @@ public: bool hasHierarchy() const override { return hierarchical_attribute; } - id_t toParent(const id_t id) const override - { - PODArray ids{1, id}; - PODArray out{1}; - getItems(*hierarchical_attribute, ids, out); - return out.front(); - } - void toParent(const PODArray & ids, PODArray & out) const override { getItems(*hierarchical_attribute, ids, out); } -#define DECLARE_INDIVIDUAL_GETTER(TYPE) \ - TYPE get##TYPE(const std::string & attribute_name, const id_t id) const override\ - {\ - auto & attribute = getAttribute(attribute_name);\ - if (attribute.type != AttributeUnderlyingType::TYPE)\ - throw Exception{\ - "Type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\ - ErrorCodes::TYPE_MISMATCH\ - };\ - \ - PODArray ids{1, id};\ - PODArray out{1};\ - getItems(attribute, ids, out);\ - return out.front();\ - } - DECLARE_INDIVIDUAL_GETTER(UInt8) - DECLARE_INDIVIDUAL_GETTER(UInt16) - DECLARE_INDIVIDUAL_GETTER(UInt32) - DECLARE_INDIVIDUAL_GETTER(UInt64) - DECLARE_INDIVIDUAL_GETTER(Int8) - DECLARE_INDIVIDUAL_GETTER(Int16) - DECLARE_INDIVIDUAL_GETTER(Int32) - DECLARE_INDIVIDUAL_GETTER(Int64) - DECLARE_INDIVIDUAL_GETTER(Float32) - DECLARE_INDIVIDUAL_GETTER(Float64) -#undef DECLARE_INDIVIDUAL_GETTER - String getString(const std::string & attribute_name, const id_t id) const override - { - auto & attribute = getAttribute(attribute_name); - if (attribute.type != AttributeUnderlyingType::String) - throw Exception{ - "Type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), - ErrorCodes::TYPE_MISMATCH - }; - - PODArray ids{1, id}; - ColumnString out; - getItems(attribute, ids, &out); - - return String{out.getDataAt(0)}; - }; - #define DECLARE_MULTIPLE_GETTER(TYPE)\ void get##TYPE(const std::string & attribute_name, const PODArray & ids, PODArray & out) const override\ {\ diff --git a/dbms/include/DB/Dictionaries/FlatDictionary.h b/dbms/include/DB/Dictionaries/FlatDictionary.h index 629fe2ddd2e..1ae5b976fde 100644 --- a/dbms/include/DB/Dictionaries/FlatDictionary.h +++ b/dbms/include/DB/Dictionaries/FlatDictionary.h @@ -81,64 +81,11 @@ public: bool hasHierarchy() const override { return hierarchical_attribute; } - id_t toParent(const id_t id) const override - { - const auto attr = hierarchical_attribute; - const auto & array = *std::get>>(attr->arrays); - - query_count.fetch_add(1, std::memory_order_relaxed); - - return id < array.size() ? array[id] : std::get(attr->null_values); - } - void toParent(const PODArray & ids, PODArray & out) const override { getItems(*hierarchical_attribute, ids, out); } -#define DECLARE_INDIVIDUAL_GETTER(TYPE) \ - TYPE get##TYPE(const std::string & attribute_name, const id_t id) const override\ - {\ - const auto & attribute = getAttribute(attribute_name);\ - if (attribute.type != AttributeUnderlyingType::TYPE)\ - throw Exception{\ - "Type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\ - ErrorCodes::TYPE_MISMATCH\ - };\ - \ - const auto & array = *std::get>>(attribute.arrays);\ - \ - query_count.fetch_add(1, std::memory_order_relaxed);\ - \ - return id < array.size() ? array[id] : std::get(attribute.null_values);\ - } - DECLARE_INDIVIDUAL_GETTER(UInt8) - DECLARE_INDIVIDUAL_GETTER(UInt16) - DECLARE_INDIVIDUAL_GETTER(UInt32) - DECLARE_INDIVIDUAL_GETTER(UInt64) - DECLARE_INDIVIDUAL_GETTER(Int8) - DECLARE_INDIVIDUAL_GETTER(Int16) - DECLARE_INDIVIDUAL_GETTER(Int32) - DECLARE_INDIVIDUAL_GETTER(Int64) - DECLARE_INDIVIDUAL_GETTER(Float32) - DECLARE_INDIVIDUAL_GETTER(Float64) -#undef DECLARE_INDIVIDUAL_GETTER - String getString(const std::string & attribute_name, const id_t id) const override - { - const auto & attribute = getAttribute(attribute_name); - if (attribute.type != AttributeUnderlyingType::String) - throw Exception{ - "Type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), - ErrorCodes::TYPE_MISMATCH - }; - - const auto & array = *std::get>>(attribute.arrays); - - query_count.fetch_add(1, std::memory_order_relaxed); - - return id < array.size() ? String{array[id]} : std::get(attribute.null_values); - } - #define DECLARE_MULTIPLE_GETTER(TYPE)\ void get##TYPE(const std::string & attribute_name, const PODArray & ids, PODArray & out) const override\ {\ diff --git a/dbms/include/DB/Dictionaries/HashedDictionary.h b/dbms/include/DB/Dictionaries/HashedDictionary.h index d07a67ae7ed..08aad57d63a 100644 --- a/dbms/include/DB/Dictionaries/HashedDictionary.h +++ b/dbms/include/DB/Dictionaries/HashedDictionary.h @@ -10,6 +10,7 @@ #include #include + namespace DB { @@ -78,67 +79,11 @@ public: bool hasHierarchy() const override { return hierarchical_attribute; } - id_t toParent(const id_t id) const override - { - const auto attr = hierarchical_attribute; - const auto & map = *std::get>>(attr->maps); - const auto it = map.find(id); - - query_count.fetch_add(1, std::memory_order_relaxed); - - return it != map.end() ? it->second : std::get(attr->null_values); - } - void toParent(const PODArray & ids, PODArray & out) const override { getItems(*hierarchical_attribute, ids, out); } -#define DECLARE_INDIVIDUAL_GETTER(TYPE) \ - TYPE get##TYPE(const std::string & attribute_name, const id_t id) const override\ - {\ - const auto & attribute = getAttribute(attribute_name);\ - if (attribute.type != AttributeUnderlyingType::TYPE)\ - throw Exception{\ - "Type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\ - ErrorCodes::TYPE_MISMATCH\ - };\ - \ - const auto & map = *std::get>>(attribute.maps);\ - const auto it = map.find(id);\ - \ - query_count.fetch_add(1, std::memory_order_relaxed);\ - \ - return it != map.end() ? TYPE{it->second} : std::get(attribute.null_values);\ - } - DECLARE_INDIVIDUAL_GETTER(UInt8) - DECLARE_INDIVIDUAL_GETTER(UInt16) - DECLARE_INDIVIDUAL_GETTER(UInt32) - DECLARE_INDIVIDUAL_GETTER(UInt64) - DECLARE_INDIVIDUAL_GETTER(Int8) - DECLARE_INDIVIDUAL_GETTER(Int16) - DECLARE_INDIVIDUAL_GETTER(Int32) - DECLARE_INDIVIDUAL_GETTER(Int64) - DECLARE_INDIVIDUAL_GETTER(Float32) - DECLARE_INDIVIDUAL_GETTER(Float64) -#undef DECLARE_INDIVIDUAL_GETTER - String getString(const std::string & attribute_name, const id_t id) const override - { - const auto & attribute = getAttribute(attribute_name); - if (attribute.type != AttributeUnderlyingType::String) - throw Exception{ - "Type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), - ErrorCodes::TYPE_MISMATCH - }; - - const auto & map = *std::get>>(attribute.maps); - const auto it = map.find(id); - - query_count.fetch_add(1, std::memory_order_relaxed); - - return it != map.end() ? String{it->second} : std::get(attribute.null_values); - } - #define DECLARE_MULTIPLE_GETTER(TYPE)\ void get##TYPE(const std::string & attribute_name, const PODArray & ids, PODArray & out) const override\ {\ diff --git a/dbms/include/DB/Dictionaries/IDictionary.h b/dbms/include/DB/Dictionaries/IDictionary.h index 446e4fe593f..48fb75f531d 100644 --- a/dbms/include/DB/Dictionaries/IDictionary.h +++ b/dbms/include/DB/Dictionaries/IDictionary.h @@ -56,7 +56,16 @@ public: virtual bool hasHierarchy() const = 0; /// do not call unless you ensure that hasHierarchy() returns true - virtual id_t toParent(id_t id) const = 0; + id_t toParent(id_t id) const + { + const PODArray ids(1, id); + PODArray out(1); + + toParent(ids, out); + + return out.front(); + } + virtual void toParent(const PODArray & ids, PODArray & out) const = 0; bool in(id_t child_id, const id_t ancestor_id) const @@ -67,20 +76,7 @@ public: return child_id != 0; } - /// functions for individual access - virtual UInt8 getUInt8(const std::string & attribute_name, id_t id) const = 0; - virtual UInt16 getUInt16(const std::string & attribute_name, id_t id) const = 0; - virtual UInt32 getUInt32(const std::string & attribute_name, id_t id) const = 0; - virtual UInt64 getUInt64(const std::string & attribute_name, id_t id) const = 0; - virtual Int8 getInt8(const std::string & attribute_name, id_t id) const = 0; - virtual Int16 getInt16(const std::string & attribute_name, id_t id) const = 0; - virtual Int32 getInt32(const std::string & attribute_name, id_t id) const = 0; - virtual Int64 getInt64(const std::string & attribute_name, id_t id) const = 0; - virtual Float32 getFloat32(const std::string & attribute_name, id_t id) const = 0; - virtual Float64 getFloat64(const std::string & attribute_name, id_t id) const = 0; - virtual String getString(const std::string & attribute_name, id_t id) const = 0; - - /// functions for multiple access + /// return mapped values for a collection of identifiers virtual void getUInt8(const std::string & attr_name, const PODArray & ids, PODArray & out) const = 0; virtual void getUInt16(const std::string & attr_name, const PODArray & ids, PODArray & out) const = 0; virtual void getUInt32(const std::string & attr_name, const PODArray & ids, PODArray & out) const = 0; diff --git a/dbms/include/DB/Functions/FunctionsDictionaries.h b/dbms/include/DB/Functions/FunctionsDictionaries.h index 8fefd8e91a4..d7361e23d54 100644 --- a/dbms/include/DB/Functions/FunctionsDictionaries.h +++ b/dbms/include/DB/Functions/FunctionsDictionaries.h @@ -839,9 +839,12 @@ private: } else if (const auto id_col = typeid_cast *>(id_col_untyped)) { + const PODArray ids{1, id_col->getData()}; + auto out = std::make_unique(); + dictionary->getString(attr_name, ids, out.get()); + block.getByPosition(result).column = new ColumnConst{ - id_col->size(), - dictionary->getString(attr_name, id_col->getData()) + id_col->size(), out->getDataAtWithTerminatingZero(0).toString() }; } else @@ -863,10 +866,6 @@ template struct DictGetTraits; #define DECLARE_DICT_GET_TRAITS(TYPE, DATA_TYPE) \ template <> struct DictGetTraits\ {\ - static TYPE get(const IDictionary * const dict, const std::string & name, const IDictionary::id_t id)\ - {\ - return dict->get##TYPE(name, id);\ - }\ static void get(const IDictionary * const dict, const std::string & name, const PODArray & ids, PODArray & out)\ {\ dict->get##TYPE(name, ids, out);\ @@ -996,10 +995,11 @@ private: } else if (const auto id_col = typeid_cast *>(id_col_untyped)) { - block.getByPosition(result).column = new ColumnConst{ - id_col->size(), - DictGetTraits::get(dictionary, attr_name, id_col->getData()) - }; + const PODArray ids{1, id_col->getData()}; + PODArray data(1); + DictGetTraits::get(dictionary, attr_name, ids, data); + + block.getByPosition(result).column = new ColumnConst{id_col->size(), data.front()}; } else { @@ -1113,10 +1113,7 @@ private: if (!dict) return false; - const auto id_col_untyped = block.getByPosition(arguments[1]).column.get(); - if (const auto id_col = typeid_cast *>(id_col_untyped)) - { - const auto & in = id_col->getData(); + const auto get_hierarchies = [&] (const PODArray & in, PODArray & out, PODArray & offsets) { const auto size = in.size(); /// copy of `in` array @@ -1157,12 +1154,6 @@ private: std::swap(in_array, out_array); } - const auto backend = new ColumnVector; - const auto array = new ColumnArray{backend}; - block.getByPosition(result).column = array; - - auto & out = backend->getData(); - auto & offsets = array->getOffsets(); out.reserve(total_count); offsets.resize(size); @@ -1172,21 +1163,29 @@ private: out.insert_assume_reserved(std::begin(ids), std::end(ids)); offsets[i] = out.size(); } + }; + + const auto id_col_untyped = block.getByPosition(arguments[1]).column.get(); + if (const auto id_col = typeid_cast *>(id_col_untyped)) + { + const auto & in = id_col->getData(); + const auto backend = new ColumnVector; + const auto array = new ColumnArray{backend}; + block.getByPosition(result).column = array; + + get_hierarchies(in, backend->getData(), array->getOffsets()); } else if (const auto id_col = typeid_cast *>(id_col_untyped)) { - Array res; + const PODArray in(1, id_col->getData()); + const auto backend = new ColumnVector; + const auto array = new ColumnArray{backend}; - IDictionary::id_t cur = id_col->getData(); - while (cur) - { - res.push_back(cur); - cur = dictionary->toParent(cur); - } + get_hierarchies(in, backend->getData(), array->getOffsets()); block.getByPosition(result).column = new ColumnConstArray{ id_col->size(), - res, + (*array)[0].get(), new DataTypeArray{new DataTypeUInt64} }; }