thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-27 10:02:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

changed ComplexKeyCacheDictionary key serialization

Browse Source
This commit is contained in:
Nikolai Kochetov 2017-06-08 20:51:46 +03:00
parent 679cccc52d
commit 2f7f9a328e
9 changed files with 34 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dbms/src/Columns/ColumnFixedString.cpp
View File

@ -76,16 +76,6 @@ void ColumnFixedString::insertData(const char * pos, size_t length)
memcpy(&chars[old_size], pos, length);
}
size_t ColumnFixedString::getSerializedSize(size_t index) const
{
return n;
}
void ColumnFixedString::serializeValue(size_t index, char * buffer) const
{
memcpy(buffer, &chars[n * index], n);
}
StringRef ColumnFixedString::serializeValueIntoArena(size_t index, Arena & arena, char const *& begin) const
{
auto pos = arena.allocContinue(n, begin);

4
dbms/src/Columns/ColumnFixedString.h
View File

@ -93,10 +93,6 @@ public:
chars.resize_assume_reserved(chars.size() - n * elems);
}
size_t getSerializedSize(size_t n) const override;
void serializeValue(size_t n, char * buffer) const override;
StringRef serializeValueIntoArena(size_t index, Arena & arena, char const *& begin) const override;
const char * deserializeAndInsertFromArena(const char * pos) override;

14
dbms/src/Columns/ColumnString.h
View File

@ -158,20 +158,6 @@ public:
offsets.resize_assume_reserved(offsets.size() - n);
}
size_t getSerializedSize(size_t n) const override
{
return sizeof(size_t) + sizeAt(n);
}
void serializeValue(size_t n, char * buffer) const override
{
size_t string_size = sizeAt(n);
size_t offset = offsetAt(n);
memcpy(buffer, &string_size, sizeof(string_size));
memcpy(buffer + sizeof(string_size), &chars[offset], string_size);
}
StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const override
{
size_t string_size = sizeAt(n);

11
dbms/src/Columns/ColumnVector.cpp
View File

@ -27,17 +27,6 @@ namespace ErrorCodes
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
}
template <typename T>
size_t ColumnVector<T>::getSerializedSize(size_t n) const
{
return sizeof(T);
}
template <typename T>
void ColumnVector<T>::serializeValue(size_t n, char * buffer) const
{
memcpy(buffer, &data[n], sizeof(T));
}
template <typename T>
StringRef ColumnVector<T>::serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const

3
dbms/src/Columns/ColumnVector.h
View File

@ -171,9 +171,6 @@ public:
data.resize_assume_reserved(data.size() - n);
}
size_t getSerializedSize(size_t n) const override;
void serializeValue(size_t n, char * buffer) const override;
StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const override;
const char * deserializeAndInsertFromArena(const char * pos) override;

5
dbms/src/Columns/IColumn.h
View File

@ -157,11 +157,6 @@ public:
*/
virtual void popBack(size_t n) = 0;
// Returns number of bytes needed to serializes n-th element.
virtual size_t getSerializedSize(size_t n) const { throw Exception("Cannot getSerializedSize() for column " + getName(), ErrorCodes::NOT_IMPLEMENTED); }
// Serializes n-th element. Writes getSerializedSize(n) bytes into buffer
virtual void serializeValue(size_t n, char * buffer) const { throw Exception("Cannot serializeValue() for column " + getName(), ErrorCodes::NOT_IMPLEMENTED); }
/** Serializes n-th element. Serialized element should be placed continuously inside Arena's memory.
* Serialized value can be deserialized to reconstruct original object. Is used in aggregation.
* The method is similar to getDataAt(), but can work when element's value cannot be mapped to existing continuous memory chunk,

2
dbms/src/Dictionaries/CacheDictionary.cpp
View File

@ -977,7 +977,7 @@ PaddedPODArray<CacheDictionary::Key> CacheDictionary::getCachedIds() const
for (size_t idx = 0; idx < cells.size(); ++idx)
{
auto & cell = cells[idx];
if (!isEmptyCell(idx))
if (!isEmptyCell(idx) && !cells[idx].isDefault())
{
array.push_back(cell.id);
}

43
dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
View File

@ -266,7 +266,7 @@ void ComplexKeyCacheDictionary::has(const Columns & key_columns, const DataTypes
/// fetch up-to-date values, decide which ones require update
for (const auto row : ext::range(0, rows_num))
{
const StringRef key = placeKeysInPool(row, key_columns, keys, temporary_keys_pool);
const StringRef key = placeKeysInPool(row, key_columns, keys, *dict_struct.key, temporary_keys_pool);
keys_array[row] = key;
const auto find_result = findCellIdx(key, now);
const auto & cell_idx = find_result.cell_idx;
@ -458,7 +458,7 @@ void ComplexKeyCacheDictionary::getItemsNumberImpl(
/// fetch up-to-date values, decide which ones require update
for (const auto row : ext::range(0, rows_num))
{
const StringRef key = placeKeysInPool(row, key_columns, keys, temporary_keys_pool);
const StringRef key = placeKeysInPool(row, key_columns, keys, *dict_struct.key, temporary_keys_pool);
keys_array[row] = key;
const auto find_result = findCellIdx(key, now);
@ -537,7 +537,7 @@ void ComplexKeyCacheDictionary::getItemsString(
/// fetch up-to-date values, discard on fail
for (const auto row : ext::range(0, rows_num))
{
const StringRef key = placeKeysInPool(row, key_columns, keys, temporary_keys_pool);
const StringRef key = placeKeysInPool(row, key_columns, keys, *dict_struct.key, temporary_keys_pool);
SCOPE_EXIT(temporary_keys_pool.rollback(key.size));
const auto find_result = findCellIdx(key, now);
@ -582,7 +582,7 @@ void ComplexKeyCacheDictionary::getItemsString(
const auto now = std::chrono::system_clock::now();
for (const auto row : ext::range(0, rows_num))
{
const StringRef key = placeKeysInPool(row, key_columns, keys, temporary_keys_pool);
const StringRef key = placeKeysInPool(row, key_columns, keys, *dict_struct.key, temporary_keys_pool);
keys_array[row] = key;
const auto find_result = findCellIdx(key, now);
@ -900,7 +900,7 @@ StringRef ComplexKeyCacheDictionary::allocKey(const size_t row, const Columns &
if (key_size_is_fixed)
return placeKeysInFixedSizePool(row, key_columns);
return placeKeysInPool(row, key_columns, keys, *keys_pool);
return placeKeysInPool(row, key_columns, keys, *dict_struct.key, *keys_pool);
}
void ComplexKeyCacheDictionary::freeKey(const StringRef key) const
@ -913,15 +913,18 @@ void ComplexKeyCacheDictionary::freeKey(const StringRef key) const
template <typename Pool>
StringRef ComplexKeyCacheDictionary::placeKeysInPool(
const size_t row, const Columns & key_columns, StringRefs & keys, Pool & pool)
const size_t row, const Columns & key_columns, StringRefs & keys,
const std::vector<DictionaryAttribute> & key_attributes, Pool & pool)
{
const auto keys_size = key_columns.size();
size_t sum_keys_size{};
for (size_t j = 0; j < keys_size; ++j)
{
keys[j].size = key_columns[j]->getSerializedSize(row);
keys[j] = key_columns[j]->getDataAt(row);
sum_keys_size += keys[j].size;
if (key_attributes[j].underlying_type == AttributeUnderlyingType::String)
sum_keys_size += sizeof(size_t) + 1;
}
auto place = pool.alloc(sum_keys_size);
@ -929,9 +932,25 @@ StringRef ComplexKeyCacheDictionary::placeKeysInPool(
auto key_start = place;
for (size_t j = 0; j < keys_size; ++j)
{
key_columns[j]->serializeValue(row, key_start);
keys[j].data = key_start;
key_start += keys[j].size;
if (key_attributes[j].underlying_type == AttributeUnderlyingType::String)
{
auto start = key_start;
auto key_size = keys[j].size + 1;
memcpy(key_start, &key_size, sizeof(size_t));
key_start += sizeof(size_t);
memcpy(key_start, keys[j].data, keys[j].size);
key_start += keys[j].size;
*key_start = '\0';
++key_start;
keys[j].data = start;
keys[j].size += sizeof(size_t) + 1;
}
else
{
memcpy(key_start, keys[j].data, keys[j].size);
keys[j].data = key_start;
key_start += keys[j].size;
}
}
return { place, sum_keys_size };
@ -981,7 +1000,8 @@ BlockInputStreamPtr ComplexKeyCacheDictionary::getBlockInputStream(const Names &
const ProfilingScopedReadRWLock read_lock{rw_lock, ProfileEvents::DictCacheLockReadNs};
for (auto idx : ext::range(0, cells.size()))
if (!isEmptyCell(idx))
if (!isEmptyCell(idx)
&& !cells[idx].isDefault())
keys.push_back(cells[idx].key);
}
@ -989,5 +1009,4 @@ BlockInputStreamPtr ComplexKeyCacheDictionary::getBlockInputStream(const Names &
return std::make_shared<BlockInputStreamType>(shared_from_this(), max_block_size, keys, column_names);
}
}

3
dbms/src/Dictionaries/ComplexKeyCacheDictionary.h
View File

@ -235,7 +235,8 @@ private:
template <typename Arena>
static StringRef placeKeysInPool(
const std::size_t row, const Columns & key_columns, StringRefs & keys, Arena & pool);
const std::size_t row, const Columns & key_columns, StringRefs & keys,
const std::vector<DictionaryAttribute> & key_attributes, Arena & pool);
StringRef placeKeysInFixedSizePool(
const std::size_t row, const Columns & key_columns) const;