thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

dbms: better [#METR-2944].

Browse Source
This commit is contained in:
Alexey Milovidov 2015-11-29 20:06:30 +03:00
parent cc75d4603b
commit eb4034f7f2
12 changed files with 115 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
dbms/include/DB/Columns/ColumnAggregateFunction.h
View File

@ -194,22 +194,21 @@ public:
return getData().size() * sizeof(getData()[0]);
}
ColumnPtr cut(size_t start, size_t length) const override
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override
{
if (start + length > getData().size())
const ColumnAggregateFunction & src_concrete = static_cast<const ColumnAggregateFunction &>(src);
if (start + length > src_concrete.getData().size())
throw Exception("Parameters start = "
+ toString(start) + ", length = "
+ toString(length) + " are out of bound in ColumnAggregateFunction::cut() method"
" (data.size() = " + toString(getData().size()) + ").",
+ toString(length) + " are out of bound in ColumnAggregateFunction::insertRangeFrom method"
" (data.size() = " + toString(src_concrete.getData().size()) + ").",
ErrorCodes::PARAMETER_OUT_OF_BOUND);
ColumnAggregateFunction * res_ = new ColumnAggregateFunction(*this);
ColumnPtr res = res_;
res_->getData().resize(length);
for (size_t i = 0; i < length; ++i)
res_->getData()[i] = getData()[start + i];
return res;
auto & data = getData();
size_t old_size = data.size();
data.resize(old_size + length);
memcpy(&data[old_size], &src_concrete.getData()[start], length * sizeof(data[0]));
}
ColumnPtr filter(const Filter & filter) const override

2
dbms/include/DB/Columns/ColumnArray.h
View File

@ -147,7 +147,7 @@ public:
return pos;
}
ColumnPtr cut(size_t start, size_t length) const override;
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
void insert(const Field & x) override
{

8
dbms/include/DB/Columns/ColumnConst.h
View File

@ -63,9 +63,13 @@ public:
Field operator[](size_t n) const override { return FieldType(getDataFromHolder()); }
void get(size_t n, Field & res) const override { res = FieldType(getDataFromHolder()); }
ColumnPtr cut(size_t start, size_t length) const override
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override
{
return new Derived(length, data, data_type);
if (getDataFromHolder() != static_cast<const Derived &>(src).getDataFromHolder())
throw Exception("Cannot insert different element into constant column " + getName(),
ErrorCodes::CANNOT_INSERT_ELEMENT_INTO_CONSTANT_COLUMN);
s += length;
}
void insert(const Field & x) override

20
dbms/include/DB/Columns/ColumnFixedString.h
View File

@ -5,6 +5,7 @@
#include <DB/Common/PODArray.h>
#include <DB/Common/Arena.h>
#include <DB/Columns/IColumn.h>
#include <DB/IO/ReadHelpers.h>
namespace DB
@ -172,13 +173,20 @@ public:
}
}
ColumnPtr cut(size_t start, size_t length) const override
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override
{
ColumnFixedString * res_ = new ColumnFixedString(n);
ColumnPtr res = res_;
res_->chars.resize(n * length);
memcpy(&res_->chars[0], &chars[n * start], n * length);
return res;
const ColumnFixedString & src_concrete = static_cast<const ColumnFixedString &>(src);
if (start + length > src_concrete.size())
throw Exception("Parameters start = "
+ toString(start) + ", length = "
+ toString(length) + " are out of bound in ColumnFixedString::insertRangeFrom method"
" (size() = " + toString(src_concrete.size()) + ").",
ErrorCodes::PARAMETER_OUT_OF_BOUND);
size_t old_size = chars.size();
chars.resize(old_size + length * n);
memcpy(&chars[old_size], &src_concrete.chars[start * n], length * n);
}
ColumnPtr filter(const IColumn::Filter & filt) const override

36
dbms/include/DB/Columns/ColumnString.h
View File

@ -145,39 +145,37 @@ public:
return pos + string_size;
}
ColumnPtr cut(size_t start, size_t length) const override
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override
{
if (length == 0)
return new ColumnString;
return;
if (start + length > offsets.size())
throw Exception("Parameter out of bound in IColumnString::cut() method.",
const ColumnString & src_concrete = static_cast<const ColumnString &>(src);
if (start + length > src_concrete.offsets.size())
throw Exception("Parameter out of bound in IColumnString::insertRangeFrom method.",
ErrorCodes::PARAMETER_OUT_OF_BOUND);
size_t nested_offset = offsetAt(start);
size_t nested_length = offsets[start + length - 1] - nested_offset;
size_t nested_offset = src_concrete.offsetAt(start);
size_t nested_length = src_concrete.offsets[start + length - 1] - nested_offset;
ColumnString * res_ = new ColumnString;
ColumnPtr res = res_;
size_t old_chars_size = chars.size();
chars.resize(old_chars_size + nested_length);
memcpy(&chars[old_chars_size], &src_concrete.chars[nested_offset], nested_length);
res_->chars.resize(nested_length);
memcpy(&res_->chars[0], &chars[nested_offset], nested_length);
Offsets_t & res_offsets = res_->offsets;
if (start == 0)
if (start == 0 && offsets.empty())
{
res_offsets.assign(offsets.begin(), offsets.begin() + length);
offsets.assign(src_concrete.offsets.begin(), src_concrete.offsets.begin() + length);
}
else
{
res_offsets.resize(length);
size_t old_size = offsets.size();
size_t prev_max_offset = old_size ? offsets.back() : 0;
offsets.resize(old_size + length);
for (size_t i = 0; i < length; ++i)
res_offsets[i] = offsets[start + i] - nested_offset;
offsets[old_size + i] = src_concrete.offsets[start + i] - nested_offset + prev_max_offset;
}
return res;
}
ColumnPtr filter(const Filter & filt) const override

19
dbms/include/DB/Columns/ColumnTuple.h
View File

@ -23,7 +23,7 @@ public:
size_t size = data.columns();
columns.resize(size);
for (size_t i = 0; i < size; ++i)
columns[i] = data.getByPosition(i).column;
columns[i] = data.unsafeGetByPosition(i).column;
}
std::string getName() const override { return "Tuple"; }
@ -115,15 +115,12 @@ public:
return pos;
}
ColumnPtr cut(size_t start, size_t length) const override
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override
{
Block res_block = data.cloneEmpty();
for (size_t i = 0; i < columns.size(); ++i)
res_block.getByPosition(i).column = data.getByPosition(i).column->cut(start, length);
return new ColumnTuple(res_block);
data.unsafeGetByPosition(i).column->insertRangeFrom(
*static_cast<const ColumnTuple &>(src).data.unsafeGetByPosition(i).column.get(),
start, length);
}
ColumnPtr filter(const Filter & filt) const override
@ -131,7 +128,7 @@ public:
Block res_block = data.cloneEmpty();
for (size_t i = 0; i < columns.size(); ++i)
res_block.getByPosition(i).column = data.getByPosition(i).column->filter(filt);
res_block.unsafeGetByPosition(i).column = data.unsafeGetByPosition(i).column->filter(filt);
return new ColumnTuple(res_block);
}
@ -141,7 +138,7 @@ public:
Block res_block = data.cloneEmpty();
for (size_t i = 0; i < columns.size(); ++i)
res_block.getByPosition(i).column = data.getByPosition(i).column->permute(perm, limit);
res_block.unsafeGetByPosition(i).column = data.unsafeGetByPosition(i).column->permute(perm, limit);
return new ColumnTuple(res_block);
}
@ -151,7 +148,7 @@ public:
Block res_block = data.cloneEmpty();
for (size_t i = 0; i < columns.size(); ++i)
res_block.getByPosition(i).column = data.getByPosition(i).column->replicate(offsets);
res_block.unsafeGetByPosition(i).column = data.unsafeGetByPosition(i).column->replicate(offsets);
return new ColumnTuple(res_block);
}

16
dbms/include/DB/Columns/ColumnVector.h
View File

@ -254,18 +254,20 @@ public:
data.push_back(DB::get<typename NearestFieldType<T>::Type>(x));
}
ColumnPtr cut(size_t start, size_t length) const override
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override
{
if (start + length > data.size())
const ColumnVector & src_vec = static_cast<const ColumnVector &>(src);
if (start + length > src_vec.data.size())
throw Exception("Parameters start = "
+ toString(start) + ", length = "
+ toString(length) + " are out of bound in IColumnVector<T>::cut() method"
" (data.size() = " + toString(data.size()) + ").",
+ toString(length) + " are out of bound in ColumnVector::insertRangeFrom method"
" (data.size() = " + toString(src_vec.data.size()) + ").",
ErrorCodes::PARAMETER_OUT_OF_BOUND);
Self * res = new Self(length);
memcpy(&res->getData()[0], &data[start], length * sizeof(data[0]));
return res;
size_t old_size = data.size();
data.resize(old_size + length);
memcpy(&data[old_size], &src_vec.data[start], length * sizeof(data[0]));
}
ColumnPtr filter(const IColumn::Filter & filt) const override

12
dbms/include/DB/Columns/IColumn.h
View File

@ -111,7 +111,12 @@ public:
/** Удалить всё кроме диапазона элементов.
* Используется, например, для операции LIMIT.
*/
virtual SharedPtr<IColumn> cut(size_t start, size_t length) const = 0;
virtual SharedPtr<IColumn> cut(size_t start, size_t length) const
{
SharedPtr<IColumn> res = cloneEmpty();
res.get()->insertRangeFrom(*this, start, length);
return res;
}
/** Вставить значение в конец столбца (количество значений увеличится на 1).
* Используется для преобразования из строк в блоки (например, при чтении значений из текстового дампа)
@ -123,6 +128,11 @@ public:
*/
virtual void insertFrom(const IColumn & src, size_t n) { insert(src[n]); }
/** Вставить в конец столбца диапазон элементов из другого столбца.
* Может использоваться для склейки столбцов.
*/
virtual void insertRangeFrom(const IColumn & src, size_t start, size_t length) = 0;
/** Вставить данные, расположенные в указанном куске памяти, если возможно.
* (если не реализуемо - кидает исключение)
* Используется для оптимизации некоторых вычислений (например, агрегации).

4
dbms/include/DB/Columns/IColumnDummy.h
View File

@ -45,9 +45,9 @@ public:
throw Exception("Method getExtremes is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
ColumnPtr cut(size_t start, size_t length) const override
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override
{
return cloneDummy(length);
s += length;
}
ColumnPtr filter(const Filter & filt) const override

33
dbms/src/Columns/ColumnArray.cpp
View File

@ -6,37 +6,38 @@ namespace DB
{
ColumnPtr ColumnArray::cut(size_t start, size_t length) const
void ColumnArray::insertRangeFrom(const IColumn & src, size_t start, size_t length)
{
if (length == 0)
return new ColumnArray(data);
return;
if (start + length > getOffsets().size())
throw Exception("Parameter out of bound in IColumnArray::cut() method.",
const ColumnArray & src_concrete = static_cast<const ColumnArray &>(src);
if (start + length > src_concrete.getOffsets().size())
throw Exception("Parameter out of bound in ColumnArray::insertRangeFrom method.",
ErrorCodes::PARAMETER_OUT_OF_BOUND);
size_t nested_offset = offsetAt(start);
size_t nested_length = getOffsets()[start + length - 1] - nested_offset;
size_t nested_offset = src_concrete.offsetAt(start);
size_t nested_length = src_concrete.getOffsets()[start + length - 1] - nested_offset;
ColumnArray * res_ = new ColumnArray(data);
ColumnPtr res = res_;
data->insertRangeFrom(src_concrete.getData(), nested_offset, nested_length);
res_->data = data->cut(nested_offset, nested_length);
Offsets_t & res_offsets = res_->getOffsets();
Offsets_t & cur_offsets = getOffsets();
const Offsets_t & src_offsets = src_concrete.getOffsets();
if (start == 0)
if (start == 0 && cur_offsets.empty())
{
res_offsets.assign(getOffsets().begin(), getOffsets().begin() + length);
cur_offsets.assign(src_offsets.begin(), src_offsets.begin() + length);
}
else
{
res_offsets.resize(length);
size_t old_size = cur_offsets.size();
size_t prev_max_offset = old_size ? cur_offsets.back() : 0;
cur_offsets.resize(old_size + length);
for (size_t i = 0; i < length; ++i)
res_offsets[i] = getOffsets()[start + i] - nested_offset;
cur_offsets[old_size + i] = src_offsets[start + i] - nested_offset + prev_max_offset;
}
return res;
}

10
dbms/tests/queries/0_stateless/00283_column_cut.reference Normal file
View File

@ -0,0 +1,10 @@
5 5 [0,1,2,3,4] ['0','1','2','3','4'] [[],[0],[0,1],[0,1,2],[0,1,2,3]] [[],['0'],['0','1'],['0','1','2'],['0','1','2','3']] ['0\0\0','1\0\0','2\0\0','3\0\0','4\0\0']
6 6 [0,1,2,3,4,5] ['0','1','2','3','4','5'] [[],[0],[0,1],[0,1,2],[0,1,2,3],[0,1,2,3,4]] [[],['0'],['0','1'],['0','1','2'],['0','1','2','3'],['0','1','2','3','4']] ['0\0\0','1\0\0','2\0\0','3\0\0','4\0\0','5\0\0']
7 7 [0,1,2,3,4,5,6] ['0','1','2','3','4','5','6'] [[],[0],[0,1],[0,1,2],[0,1,2,3],[0,1,2,3,4],[0,1,2,3,4,5]] [[],['0'],['0','1'],['0','1','2'],['0','1','2','3'],['0','1','2','3','4'],['0','1','2','3','4','5']] ['0\0\0','1\0\0','2\0\0','3\0\0','4\0\0','5\0\0','6\0\0']
8 8 [0,1,2,3,4,5,6,7] ['0','1','2','3','4','5','6','7'] [[],[0],[0,1],[0,1,2],[0,1,2,3],[0,1,2,3,4],[0,1,2,3,4,5],[0,1,2,3,4,5,6]] [[],['0'],['0','1'],['0','1','2'],['0','1','2','3'],['0','1','2','3','4'],['0','1','2','3','4','5'],['0','1','2','3','4','5','6']] ['0\0\0','1\0\0','2\0\0','3\0\0','4\0\0','5\0\0','6\0\0','7\0\0']
9 9 [0,1,2,3,4,5,6,7,8] ['0','1','2','3','4','5','6','7','8'] [[],[0],[0,1],[0,1,2],[0,1,2,3],[0,1,2,3,4],[0,1,2,3,4,5],[0,1,2,3,4,5,6],[0,1,2,3,4,5,6,7]] [[],['0'],['0','1'],['0','1','2'],['0','1','2','3'],['0','1','2','3','4'],['0','1','2','3','4','5'],['0','1','2','3','4','5','6'],['0','1','2','3','4','5','6','7']] ['0\0\0','1\0\0','2\0\0','3\0\0','4\0\0','5\0\0','6\0\0','7\0\0','8\0\0']
10 10 [0,1,2,3,4,5,6,7,8,9] ['0','1','2','3','4','5','6','7','8','9'] [[],[0],[0,1],[0,1,2],[0,1,2,3],[0,1,2,3,4],[0,1,2,3,4,5],[0,1,2,3,4,5,6],[0,1,2,3,4,5,6,7],[0,1,2,3,4,5,6,7,8]] [[],['0'],['0','1'],['0','1','2'],['0','1','2','3'],['0','1','2','3','4'],['0','1','2','3','4','5'],['0','1','2','3','4','5','6'],['0','1','2','3','4','5','6','7'],['0','1','2','3','4','5','6','7','8']] ['0\0\0','1\0\0','2\0\0','3\0\0','4\0\0','5\0\0','6\0\0','7\0\0','8\0\0','9\0\0']
11 11 [0,1,2,3,4,5,6,7,8,9,10] ['0','1','2','3','4','5','6','7','8','9','10'] [[],[0],[0,1],[0,1,2],[0,1,2,3],[0,1,2,3,4],[0,1,2,3,4,5],[0,1,2,3,4,5,6],[0,1,2,3,4,5,6,7],[0,1,2,3,4,5,6,7,8],[0,1,2,3,4,5,6,7,8,9]] [[],['0'],['0','1'],['0','1','2'],['0','1','2','3'],['0','1','2','3','4'],['0','1','2','3','4','5'],['0','1','2','3','4','5','6'],['0','1','2','3','4','5','6','7'],['0','1','2','3','4','5','6','7','8'],['0','1','2','3','4','5','6','7','8','9']] ['0\0\0','1\0\0','2\0\0','3\0\0','4\0\0','5\0\0','6\0\0','7\0\0','8\0\0','9\0\0','10\0']
12 12 [0,1,2,3,4,5,6,7,8,9,10,11] ['0','1','2','3','4','5','6','7','8','9','10','11'] [[],[0],[0,1],[0,1,2],[0,1,2,3],[0,1,2,3,4],[0,1,2,3,4,5],[0,1,2,3,4,5,6],[0,1,2,3,4,5,6,7],[0,1,2,3,4,5,6,7,8],[0,1,2,3,4,5,6,7,8,9],[0,1,2,3,4,5,6,7,8,9,10]] [[],['0'],['0','1'],['0','1','2'],['0','1','2','3'],['0','1','2','3','4'],['0','1','2','3','4','5'],['0','1','2','3','4','5','6'],['0','1','2','3','4','5','6','7'],['0','1','2','3','4','5','6','7','8'],['0','1','2','3','4','5','6','7','8','9'],['0','1','2','3','4','5','6','7','8','9','10']] ['0\0\0','1\0\0','2\0\0','3\0\0','4\0\0','5\0\0','6\0\0','7\0\0','8\0\0','9\0\0','10\0','11\0']
13 13 [0,1,2,3,4,5,6,7,8,9,10,11,12] ['0','1','2','3','4','5','6','7','8','9','10','11','12'] [[],[0],[0,1],[0,1,2],[0,1,2,3],[0,1,2,3,4],[0,1,2,3,4,5],[0,1,2,3,4,5,6],[0,1,2,3,4,5,6,7],[0,1,2,3,4,5,6,7,8],[0,1,2,3,4,5,6,7,8,9],[0,1,2,3,4,5,6,7,8,9,10],[0,1,2,3,4,5,6,7,8,9,10,11]] [[],['0'],['0','1'],['0','1','2'],['0','1','2','3'],['0','1','2','3','4'],['0','1','2','3','4','5'],['0','1','2','3','4','5','6'],['0','1','2','3','4','5','6','7'],['0','1','2','3','4','5','6','7','8'],['0','1','2','3','4','5','6','7','8','9'],['0','1','2','3','4','5','6','7','8','9','10'],['0','1','2','3','4','5','6','7','8','9','10','11']] ['0\0\0','1\0\0','2\0\0','3\0\0','4\0\0','5\0\0','6\0\0','7\0\0','8\0\0','9\0\0','10\0','11\0','12\0']
14 14 [0,1,2,3,4,5,6,7,8,9,10,11,12,13] ['0','1','2','3','4','5','6','7','8','9','10','11','12','13'] [[],[0],[0,1],[0,1,2],[0,1,2,3],[0,1,2,3,4],[0,1,2,3,4,5],[0,1,2,3,4,5,6],[0,1,2,3,4,5,6,7],[0,1,2,3,4,5,6,7,8],[0,1,2,3,4,5,6,7,8,9],[0,1,2,3,4,5,6,7,8,9,10],[0,1,2,3,4,5,6,7,8,9,10,11],[0,1,2,3,4,5,6,7,8,9,10,11,12]] [[],['0'],['0','1'],['0','1','2'],['0','1','2','3'],['0','1','2','3','4'],['0','1','2','3','4','5'],['0','1','2','3','4','5','6'],['0','1','2','3','4','5','6','7'],['0','1','2','3','4','5','6','7','8'],['0','1','2','3','4','5','6','7','8','9'],['0','1','2','3','4','5','6','7','8','9','10'],['0','1','2','3','4','5','6','7','8','9','10','11'],['0','1','2','3','4','5','6','7','8','9','10','11','12']] ['0\0\0','1\0\0','2\0\0','3\0\0','4\0\0','5\0\0','6\0\0','7\0\0','8\0\0','9\0\0','10\0','11\0','12\0','13\0']

10
dbms/tests/queries/0_stateless/00283_column_cut.sql Normal file
View File

@ -0,0 +1,10 @@
SELECT
number,
toString(number),
range(number) AS arr,
arrayMap(x -> toString(x), arr) AS arr_s,
arrayMap(x -> range(x), arr) AS arr_arr,
arrayMap(x -> arrayMap(y -> toString(y), x), arr_arr) AS arr_arr_s,
arrayMap(x -> toFixedString(x, 3), arr_s) AS arr_fs
FROM system.numbers
LIMIT 5, 10;