From 665fabde9e60eea5f6364200cdad56d00ed39298 Mon Sep 17 00:00:00 2001 From: antikvist Date: Sat, 18 Apr 2020 19:20:34 +0300 Subject: [PATCH 001/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- .../AggregateFunctionWelchTTest.cpp | 28 ++ .../AggregateFunctionWelchTTest.h | 251 ++++++++++++++++++ .../registerAggregateFunctions.cpp | 1 + .../registerAggregateFunctions.h | 1 + 4 files changed, 281 insertions(+) create mode 100644 src/AggregateFunctions/AggregateFunctionWelchTTest.cpp create mode 100644 src/AggregateFunctions/AggregateFunctionWelchTTest.h diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp new file mode 100644 index 00000000000..46b533a2c0b --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -0,0 +1,28 @@ +#include +#include +#include + + +namespace DB +{ + +namespace +{ + +AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, const DataTypes & argument_types, const Array & parameters) +{ + + + return std::make_shared(argument_types, parameters); + +} + +} + +void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) +{ + + factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest, AggregateFunctionFactory::CaseInsensitive); +} + +} diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h new file mode 100644 index 00000000000..a52528df431 --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -0,0 +1,251 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +// hard-codded values - part of the algorithm + +#define SIGN_LVL_CNT 6 + +Float64 CriticalValuesTable[SIGN_LVL_CNT][102] = { + // for significance level = 0.2 + {0.2, 3.078, 1.886, 1.638, 1.533, 1.476, 1.44, 1.415, 1.397, 1.383, 1.372, 1.363, 1.356, 1.35, 1.345, 1.341, 1.337, 1.333, 1.33, 1.328, 1.325, 1.323, 1.321, 1.319, 1.318, 1.316, 1.315, 1.314, 1.313, 1.311, 1.31, 1.309, 1.309, 1.308, 1.307, 1.306, 1.306, 1.305, 1.304, 1.304, 1.303, 1.303, 1.302, 1.302, 1.301, 1.301, 1.3, 1.3, 1.299, 1.299, 1.299, 1.298, 1.298, 1.298, 1.297, 1.297, 1.297, 1.297, 1.296, 1.296, 1.296, 1.296, 1.295, 1.295, 1.295, 1.295, 1.295, 1.294, 1.294, 1.294, 1.294, 1.294, 1.293, 1.293, 1.293, 1.293, 1.293, 1.293, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.29, 1.29, 1.29, 1.29, 1.29, 1.282} + + // for significance level = 0.1 + {0.1, 6.314, 2.92, 2.353, 2.132, 2.015, 1.943, 1.895, 1.86, 1.833, 1.812, 1.796, 1.782, 1.771, 1.761, 1.753, 1.746, 1.74, 1.734, 1.729, 1.725, 1.721, 1.717, 1.714, 1.711, 1.708, 1.706, 1.703, 1.701, 1.699, 1.697, 1.696, 1.694, 1.692, 1.691, 1.69, 1.688, 1.687, 1.686, 1.685, 1.684, 1.683, 1.682, 1.681, 1.68, 1.679, 1.679, 1.678, 1.677, 1.677, 1.676, 1.675, 1.675, 1.674, 1.674, 1.673, 1.673, 1.672, 1.672, 1.671, 1.671, 1.67, 1.67, 1.669, 1.669, 1.669, 1.668, 1.668, 1.668, 1.667, 1.667, 1.667, 1.666, 1.666, 1.666, 1.665, 1.665, 1.665, 1.665, 1.664, 1.664, 1.664, 1.664, 1.663, 1.663, 1.663, 1.663, 1.663, 1.662, 1.662, 1.662, 1.662, 1.662, 1.661, 1.661, 1.661, 1.661, 1.661, 1.661, 1.66, 1.66, 1.645} + + // for significance level = 0.05 + {0.05, 12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145, 2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, 2.045, 2.042, 2.04, 2.037, 2.035, 2.032, 2.03, 2.028, 2.026, 2.024, 2.023, 2.021, 2.02, 2.018, 2.017, 2.015, 2.014, 2.013, 2.012, 2.011, 2.01, 2.009, 2.008, 2.007, 2.006, 2.005, 2.004, 2.003, 2.002, 2.002, 2.001, 2.0, 2.0, 1.999, 1.998, 1.998, 1.997, 1.997, 1.996, 1.995, 1.995, 1.994, 1.994, 1.993, 1.993, 1.993, 1.992, 1.992, 1.991, 1.991, 1.99, 1.99, 1.99, 1.989, 1.989, 1.989, 1.988, 1.988, 1.988, 1.987, 1.987, 1.987, 1.986, 1.986, 1.986, 1.986, 1.985, 1.985, 1.985, 1.984, 1.984, 1.984, 1.96} + + // for significance level = 0.02 + {0.02, 31.821, 6.965, 4.541, 3.747, 3.365, 3.143, 2.998, 2.896, 2.821, 2.764, 2.718, 2.681, 2.65, 2.624, 2.602, 2.583, 2.567, 2.552, 2.539, 2.528, 2.518, 2.508, 2.5, 2.492, 2.485, 2.479, 2.473, 2.467, 2.462, 2.457, 2.453, 2.449, 2.445, 2.441, 2.438, 2.434, 2.431, 2.429, 2.426, 2.423, 2.421, 2.418, 2.416, 2.414, 2.412, 2.41, 2.408, 2.407, 2.405, 2.403, 2.402, 2.4, 2.399, 2.397, 2.396, 2.395, 2.394, 2.392, 2.391, 2.39, 2.389, 2.388, 2.387, 2.386, 2.385, 2.384, 2.383, 2.382, 2.382, 2.381, 2.38, 2.379, 2.379, 2.378, 2.377, 2.376, 2.376, 2.375, 2.374, 2.374, 2.373, 2.373, 2.372, 2.372, 2.371, 2.37, 2.37, 2.369, 2.369, 2.368, 2.368, 2.368, 2.367, 2.367, 2.366, 2.366, 2.365, 2.365, 2.365, 2.364, 2.326} + + // for significance level = 0.01 + {0.01, 63.657, 9.925, 5.841, 4.604, 4.032, 3.707, 3.499, 3.355, 3.25, 3.169, 3.106, 3.055, 3.012, 2.977, 2.947, 2.921, 2.898, 2.878, 2.861, 2.845, 2.831, 2.819, 2.807, 2.797, 2.787, 2.779, 2.771, 2.763, 2.756, 2.75, 2.744, 2.738, 2.733, 2.728, 2.724, 2.719, 2.715, 2.712, 2.708, 2.704, 2.701, 2.698, 2.695, 2.692, 2.69, 2.687, 2.685, 2.682, 2.68, 2.678, 2.676, 2.674, 2.672, 2.67, 2.668, 2.667, 2.665, 2.663, 2.662, 2.66, 2.659, 2.657, 2.656, 2.655, 2.654, 2.652, 2.651, 2.65, 2.649, 2.648, 2.647, 2.646, 2.645, 2.644, 2.643, 2.642, 2.641, 2.64, 2.64, 2.639, 2.638, 2.637, 2.636, 2.636, 2.635, 2.634, 2.634, 2.633, 2.632, 2.632, 2.631, 2.63, 2.63, 2.629, 2.629, 2.628, 2.627, 2.627, 2.626, 2.626, 2.576} + + // for significance level = 0.002 + {0.002, 318.313, 22.327, 10.215, 7.173, 5.893, 5.208, 4.782, 4.499, 4.296, 4.143, 4.024, 3.929, 3.852, 3.787, 3.733, 3.686, 3.646, 3.61, 3.579, 3.552, 3.527, 3.505, 3.485, 3.467, 3.45, 3.435, 3.421, 3.408, 3.396, 3.385, 3.375, 3.365, 3.356, 3.348, 3.34, 3.333, 3.326, 3.319, 3.313, 3.307, 3.301, 3.296, 3.291, 3.286, 3.281, 3.277, 3.273, 3.269, 3.265, 3.261, 3.258, 3.255, 3.251, 3.248, 3.245, 3.242, 3.239, 3.237, 3.234, 3.232, 3.229, 3.227, 3.225, 3.223, 3.22, 3.218, 3.216, 3.214, 3.213, 3.211, 3.209, 3.207, 3.206, 3.204, 3.202, 3.201, 3.199, 3.198, 3.197, 3.195, 3.194, 3.193, 3.191, 3.19, 3.189, 3.188, 3.187, 3.185, 3.184, 3.183, 3.182, 3.181, 3.18, 3.179, 3.178, 3.177, 3.176, 3.175, 3.175, 3.174, 3.09} +} + +// our algorithm implementation via vectors: +// https://gist.github.com/ltybc-coder/792748cfdb2f7cadef424ffb7b011c71 +// col, col, bool +template +struct AggregateFunctionWelchTTestData final { + + size_t size_x = 0; + size_t size_y = 0; + X sum_x = 0; + Y sum_y = 0; + X square_sum_x = 0; + Y square_sum_y = 0; + Float64 mean_x = 0; + Float64 mean_y = 0; + + /* + not yet sure how to use them + void add_x(X x) { + mean_x = (Float64)(sum_x + x) / (size_x + 1); + size_x ++; + sum_x += x; + square_sum_x += x * x; + } + + void add_y(Y y) { + mean_y = (sum_y + y) / (size_y + 1); + size_y ++; + sum_y += y; + square_sum_y += y * y; + } + */ + + void add(X x, Y y) { + sum_x += x; + sum_y += y; + size_x++; + size_y++; + mean_x = (Float64) sum_x / size_x; + mean_y = (Float64) sum_y / size_y; + square_sum_x += x * x; + square_sum_y += y * y; + } + + void merge(const AggregateFunctionWelchTTestData &other) { + sum_x += other.sum_x; + sum_y += other.sum_y; + size_x += other.size_x; + size_y += other.size_y; + mean_x = (Float64) sum_x / size_x; + mean_y = (Float64) sum_y / size_y; + square_sum_x += other.square_sum_x; + square_sum_y += other.square_sum_y; + } + + void serialize(WriteBuffer &buf) const { + writeBinary(mean_x, buf); + writeBinary(mean_y, buf); + writeBinary(sum_x, buf); + writeBinary(sum_y, buf); + writeBinary(square_sum_x, buf); + writeBinary(square_sum_y, buf); + writeBinary(size_x, buf); + writeBinary(size_y, buf); + } + + void deserialize(ReadBuffer &buf) { + readBinary(mean_x, buf); + readBinary(mean_y, buf); + readBinary(sum_x, buf); + readBinary(sum_y, buf); + readBinary(square_sum_x, buf); + readBinary(square_sum_y, buf); + readBinary(size_x, buf); + readBinary(size_y, buf); + } + + Float64 get_sx() const { + return (Float64)(square_sum_x + size_x * mean_x * mean_x - 2 * mean_x * sum_x) / (size_x - 1); + } + + Float64 get_sy() const { + return (Float64)(square_sum_y + size_y * mean_y * mean_y - 2 * mean_y * sum_y) / (size_y - 1); + } + + Float64 get_T(Float64 sx, Float64 sy) const { + return (Float64)(mean_x - mean_y) / std::sqrt(sx / size_x + sy / size_y); + } + + Float64 get_degrees_of_freed(Float64 sx, Float64 sy) const { + return (Float64)(sx / size_x + sy / size_y) * (sx / size_x + sy / size_y) / + ((sx * sx / (size_x * size_x * (size_x - 1))) + (sy * sy / (size_y * size_y * (size_y - 1)))); + } + + Ret get_result(Float64 t, Float64 dof, Float64 parametr) const + { + //find our table + int table; + for (int i = 0; i < SIGN_LVL_CNT; ++i) + { + if (CriticalValuesTable[i][0] == parametr) + { + table = i; + } + } + + //round or make infinity dof + dof = (int) dof; + if (dof > 100) + { + dof = 101; + } + //check if abs of t is greater than table[dof] + t = abs(t) + if(t > CriticalValuesTable[table][dof]) { + return (UInt8) 1; + //in this case we reject the null hypothesis + } + else { + return (UInt8) 0; + } + } +}; + +template +class AggregateFunctionWelchTTest final : public + IAggregateFunctionDataHelper< + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest + > +{ +public: + AggregateFunctionWelchTTest( + const DataTypes & arguments, + const Array & params + ): + IAggregateFunctionDataHelper< + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest + > {arguments, params} + { + // notice: arguments has been in factory + } + + String getName() const override + { + return "WelchTTest"; + } + + void add( + AggregateDataPtr place, + const IColumn ** columns, + size_t row_num, + Arena * + ) const override + { + auto col_x = assert_cast *>(columns[0]); + auto col_y = assert_cast *>(columns[1]); + + X x = col_x->getData()[row_num]; + Y y = col_y->getData()[row_num]; + + this->data(place).add(x, y); + } + + void merge( + AggregateDataPtr place, + ConstAggregateDataPtr rhs, Arena * + ) const override + { + this->data(place).merge(this->data(rhs)); + } + + void serialize( + ConstAggregateDataPtr place, + WriteBuffer & buf + ) const override + { + this->data(place).serialize(buf); + } + + void deserialize( + AggregateDataPtr place, + ReadBuffer & buf, Arena * + ) const override + { + this->data(place).deserialize(buf); + } + + void insertResultInto( + ConstAggregateDataPtr place, + IColumn & to + ) const override + { + Float64 significance_level = applyVisitor(FieldVisitorConvertToNumber(), params[0]); + + Float64 sx = this->data(place).get_sx(); + Float64 sy = this->data(place).get_sy(); + Float64 t_value = this->data(place).get_T(sx, sy); + Float64 dof = this->data(place).get_degrees_of_freed(sx, sy); + Ret result = this->data(place).get_result(t_value, dof, significance_level); + + auto & column = static_cast(to); + column.getData().push_back(result); + } + + +} +}; + +}; \ No newline at end of file diff --git a/src/AggregateFunctions/registerAggregateFunctions.cpp b/src/AggregateFunctions/registerAggregateFunctions.cpp index a9ab1d4f8ea..adc72ec9169 100644 --- a/src/AggregateFunctions/registerAggregateFunctions.cpp +++ b/src/AggregateFunctions/registerAggregateFunctions.cpp @@ -45,6 +45,7 @@ void registerAggregateFunctions() registerAggregateFunctionMoving(factory); registerAggregateFunctionCategoricalIV(factory); registerAggregateFunctionAggThrow(factory); + registerAggregateFunctionWelchTTest(factory); } { diff --git a/src/AggregateFunctions/registerAggregateFunctions.h b/src/AggregateFunctions/registerAggregateFunctions.h index 88cdf4a504d..046b125dec5 100644 --- a/src/AggregateFunctions/registerAggregateFunctions.h +++ b/src/AggregateFunctions/registerAggregateFunctions.h @@ -35,6 +35,7 @@ void registerAggregateFunctionSimpleLinearRegression(AggregateFunctionFactory &) void registerAggregateFunctionMoving(AggregateFunctionFactory &); void registerAggregateFunctionCategoricalIV(AggregateFunctionFactory &); void registerAggregateFunctionAggThrow(AggregateFunctionFactory &); +void registerAggregateFunctionWelchTTest(AggregateFunctionFactory &); class AggregateFunctionCombinatorFactory; void registerAggregateFunctionCombinatorIf(AggregateFunctionCombinatorFactory &); From 69c1f33b0b22bd103cbce8f0ce747dc12937d1a1 Mon Sep 17 00:00:00 2001 From: antikvist Date: Mon, 27 Apr 2020 00:09:56 +0300 Subject: [PATCH 002/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- .../AggregateFunctionWelchTTest.h | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index a52528df431..be1e176d540 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -4,13 +4,17 @@ #include #include #include +#include #include #include #include #include #include #include -#include + +#include +#include +#include namespace DB @@ -21,19 +25,19 @@ namespace DB Float64 CriticalValuesTable[SIGN_LVL_CNT][102] = { // for significance level = 0.2 - {0.2, 3.078, 1.886, 1.638, 1.533, 1.476, 1.44, 1.415, 1.397, 1.383, 1.372, 1.363, 1.356, 1.35, 1.345, 1.341, 1.337, 1.333, 1.33, 1.328, 1.325, 1.323, 1.321, 1.319, 1.318, 1.316, 1.315, 1.314, 1.313, 1.311, 1.31, 1.309, 1.309, 1.308, 1.307, 1.306, 1.306, 1.305, 1.304, 1.304, 1.303, 1.303, 1.302, 1.302, 1.301, 1.301, 1.3, 1.3, 1.299, 1.299, 1.299, 1.298, 1.298, 1.298, 1.297, 1.297, 1.297, 1.297, 1.296, 1.296, 1.296, 1.296, 1.295, 1.295, 1.295, 1.295, 1.295, 1.294, 1.294, 1.294, 1.294, 1.294, 1.293, 1.293, 1.293, 1.293, 1.293, 1.293, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.29, 1.29, 1.29, 1.29, 1.29, 1.282} + {0.2, 3.078, 1.886, 1.638, 1.533, 1.476, 1.44, 1.415, 1.397, 1.383, 1.372, 1.363, 1.356, 1.35, 1.345, 1.341, 1.337, 1.333, 1.33, 1.328, 1.325, 1.323, 1.321, 1.319, 1.318, 1.316, 1.315, 1.314, 1.313, 1.311, 1.31, 1.309, 1.309, 1.308, 1.307, 1.306, 1.306, 1.305, 1.304, 1.304, 1.303, 1.303, 1.302, 1.302, 1.301, 1.301, 1.3, 1.3, 1.299, 1.299, 1.299, 1.298, 1.298, 1.298, 1.297, 1.297, 1.297, 1.297, 1.296, 1.296, 1.296, 1.296, 1.295, 1.295, 1.295, 1.295, 1.295, 1.294, 1.294, 1.294, 1.294, 1.294, 1.293, 1.293, 1.293, 1.293, 1.293, 1.293, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.29, 1.29, 1.29, 1.29, 1.29, 1.282}, // for significance level = 0.1 - {0.1, 6.314, 2.92, 2.353, 2.132, 2.015, 1.943, 1.895, 1.86, 1.833, 1.812, 1.796, 1.782, 1.771, 1.761, 1.753, 1.746, 1.74, 1.734, 1.729, 1.725, 1.721, 1.717, 1.714, 1.711, 1.708, 1.706, 1.703, 1.701, 1.699, 1.697, 1.696, 1.694, 1.692, 1.691, 1.69, 1.688, 1.687, 1.686, 1.685, 1.684, 1.683, 1.682, 1.681, 1.68, 1.679, 1.679, 1.678, 1.677, 1.677, 1.676, 1.675, 1.675, 1.674, 1.674, 1.673, 1.673, 1.672, 1.672, 1.671, 1.671, 1.67, 1.67, 1.669, 1.669, 1.669, 1.668, 1.668, 1.668, 1.667, 1.667, 1.667, 1.666, 1.666, 1.666, 1.665, 1.665, 1.665, 1.665, 1.664, 1.664, 1.664, 1.664, 1.663, 1.663, 1.663, 1.663, 1.663, 1.662, 1.662, 1.662, 1.662, 1.662, 1.661, 1.661, 1.661, 1.661, 1.661, 1.661, 1.66, 1.66, 1.645} + {0.1, 6.314, 2.92, 2.353, 2.132, 2.015, 1.943, 1.895, 1.86, 1.833, 1.812, 1.796, 1.782, 1.771, 1.761, 1.753, 1.746, 1.74, 1.734, 1.729, 1.725, 1.721, 1.717, 1.714, 1.711, 1.708, 1.706, 1.703, 1.701, 1.699, 1.697, 1.696, 1.694, 1.692, 1.691, 1.69, 1.688, 1.687, 1.686, 1.685, 1.684, 1.683, 1.682, 1.681, 1.68, 1.679, 1.679, 1.678, 1.677, 1.677, 1.676, 1.675, 1.675, 1.674, 1.674, 1.673, 1.673, 1.672, 1.672, 1.671, 1.671, 1.67, 1.67, 1.669, 1.669, 1.669, 1.668, 1.668, 1.668, 1.667, 1.667, 1.667, 1.666, 1.666, 1.666, 1.665, 1.665, 1.665, 1.665, 1.664, 1.664, 1.664, 1.664, 1.663, 1.663, 1.663, 1.663, 1.663, 1.662, 1.662, 1.662, 1.662, 1.662, 1.661, 1.661, 1.661, 1.661, 1.661, 1.661, 1.66, 1.66, 1.645}, // for significance level = 0.05 - {0.05, 12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145, 2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, 2.045, 2.042, 2.04, 2.037, 2.035, 2.032, 2.03, 2.028, 2.026, 2.024, 2.023, 2.021, 2.02, 2.018, 2.017, 2.015, 2.014, 2.013, 2.012, 2.011, 2.01, 2.009, 2.008, 2.007, 2.006, 2.005, 2.004, 2.003, 2.002, 2.002, 2.001, 2.0, 2.0, 1.999, 1.998, 1.998, 1.997, 1.997, 1.996, 1.995, 1.995, 1.994, 1.994, 1.993, 1.993, 1.993, 1.992, 1.992, 1.991, 1.991, 1.99, 1.99, 1.99, 1.989, 1.989, 1.989, 1.988, 1.988, 1.988, 1.987, 1.987, 1.987, 1.986, 1.986, 1.986, 1.986, 1.985, 1.985, 1.985, 1.984, 1.984, 1.984, 1.96} + {0.05, 12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145, 2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, 2.045, 2.042, 2.04, 2.037, 2.035, 2.032, 2.03, 2.028, 2.026, 2.024, 2.023, 2.021, 2.02, 2.018, 2.017, 2.015, 2.014, 2.013, 2.012, 2.011, 2.01, 2.009, 2.008, 2.007, 2.006, 2.005, 2.004, 2.003, 2.002, 2.002, 2.001, 2.0, 2.0, 1.999, 1.998, 1.998, 1.997, 1.997, 1.996, 1.995, 1.995, 1.994, 1.994, 1.993, 1.993, 1.993, 1.992, 1.992, 1.991, 1.991, 1.99, 1.99, 1.99, 1.989, 1.989, 1.989, 1.988, 1.988, 1.988, 1.987, 1.987, 1.987, 1.986, 1.986, 1.986, 1.986, 1.985, 1.985, 1.985, 1.984, 1.984, 1.984, 1.96}, // for significance level = 0.02 - {0.02, 31.821, 6.965, 4.541, 3.747, 3.365, 3.143, 2.998, 2.896, 2.821, 2.764, 2.718, 2.681, 2.65, 2.624, 2.602, 2.583, 2.567, 2.552, 2.539, 2.528, 2.518, 2.508, 2.5, 2.492, 2.485, 2.479, 2.473, 2.467, 2.462, 2.457, 2.453, 2.449, 2.445, 2.441, 2.438, 2.434, 2.431, 2.429, 2.426, 2.423, 2.421, 2.418, 2.416, 2.414, 2.412, 2.41, 2.408, 2.407, 2.405, 2.403, 2.402, 2.4, 2.399, 2.397, 2.396, 2.395, 2.394, 2.392, 2.391, 2.39, 2.389, 2.388, 2.387, 2.386, 2.385, 2.384, 2.383, 2.382, 2.382, 2.381, 2.38, 2.379, 2.379, 2.378, 2.377, 2.376, 2.376, 2.375, 2.374, 2.374, 2.373, 2.373, 2.372, 2.372, 2.371, 2.37, 2.37, 2.369, 2.369, 2.368, 2.368, 2.368, 2.367, 2.367, 2.366, 2.366, 2.365, 2.365, 2.365, 2.364, 2.326} + {0.02, 31.821, 6.965, 4.541, 3.747, 3.365, 3.143, 2.998, 2.896, 2.821, 2.764, 2.718, 2.681, 2.65, 2.624, 2.602, 2.583, 2.567, 2.552, 2.539, 2.528, 2.518, 2.508, 2.5, 2.492, 2.485, 2.479, 2.473, 2.467, 2.462, 2.457, 2.453, 2.449, 2.445, 2.441, 2.438, 2.434, 2.431, 2.429, 2.426, 2.423, 2.421, 2.418, 2.416, 2.414, 2.412, 2.41, 2.408, 2.407, 2.405, 2.403, 2.402, 2.4, 2.399, 2.397, 2.396, 2.395, 2.394, 2.392, 2.391, 2.39, 2.389, 2.388, 2.387, 2.386, 2.385, 2.384, 2.383, 2.382, 2.382, 2.381, 2.38, 2.379, 2.379, 2.378, 2.377, 2.376, 2.376, 2.375, 2.374, 2.374, 2.373, 2.373, 2.372, 2.372, 2.371, 2.37, 2.37, 2.369, 2.369, 2.368, 2.368, 2.368, 2.367, 2.367, 2.366, 2.366, 2.365, 2.365, 2.365, 2.364, 2.326}, // for significance level = 0.01 - {0.01, 63.657, 9.925, 5.841, 4.604, 4.032, 3.707, 3.499, 3.355, 3.25, 3.169, 3.106, 3.055, 3.012, 2.977, 2.947, 2.921, 2.898, 2.878, 2.861, 2.845, 2.831, 2.819, 2.807, 2.797, 2.787, 2.779, 2.771, 2.763, 2.756, 2.75, 2.744, 2.738, 2.733, 2.728, 2.724, 2.719, 2.715, 2.712, 2.708, 2.704, 2.701, 2.698, 2.695, 2.692, 2.69, 2.687, 2.685, 2.682, 2.68, 2.678, 2.676, 2.674, 2.672, 2.67, 2.668, 2.667, 2.665, 2.663, 2.662, 2.66, 2.659, 2.657, 2.656, 2.655, 2.654, 2.652, 2.651, 2.65, 2.649, 2.648, 2.647, 2.646, 2.645, 2.644, 2.643, 2.642, 2.641, 2.64, 2.64, 2.639, 2.638, 2.637, 2.636, 2.636, 2.635, 2.634, 2.634, 2.633, 2.632, 2.632, 2.631, 2.63, 2.63, 2.629, 2.629, 2.628, 2.627, 2.627, 2.626, 2.626, 2.576} + {0.01, 63.657, 9.925, 5.841, 4.604, 4.032, 3.707, 3.499, 3.355, 3.25, 3.169, 3.106, 3.055, 3.012, 2.977, 2.947, 2.921, 2.898, 2.878, 2.861, 2.845, 2.831, 2.819, 2.807, 2.797, 2.787, 2.779, 2.771, 2.763, 2.756, 2.75, 2.744, 2.738, 2.733, 2.728, 2.724, 2.719, 2.715, 2.712, 2.708, 2.704, 2.701, 2.698, 2.695, 2.692, 2.69, 2.687, 2.685, 2.682, 2.68, 2.678, 2.676, 2.674, 2.672, 2.67, 2.668, 2.667, 2.665, 2.663, 2.662, 2.66, 2.659, 2.657, 2.656, 2.655, 2.654, 2.652, 2.651, 2.65, 2.649, 2.648, 2.647, 2.646, 2.645, 2.644, 2.643, 2.642, 2.641, 2.64, 2.64, 2.639, 2.638, 2.637, 2.636, 2.636, 2.635, 2.634, 2.634, 2.633, 2.632, 2.632, 2.631, 2.63, 2.63, 2.629, 2.629, 2.628, 2.627, 2.627, 2.626, 2.626, 2.576}, // for significance level = 0.002 {0.002, 318.313, 22.327, 10.215, 7.173, 5.893, 5.208, 4.782, 4.499, 4.296, 4.143, 4.024, 3.929, 3.852, 3.787, 3.733, 3.686, 3.646, 3.61, 3.579, 3.552, 3.527, 3.505, 3.485, 3.467, 3.45, 3.435, 3.421, 3.408, 3.396, 3.385, 3.375, 3.365, 3.356, 3.348, 3.34, 3.333, 3.326, 3.319, 3.313, 3.307, 3.301, 3.296, 3.291, 3.286, 3.281, 3.277, 3.273, 3.269, 3.265, 3.261, 3.258, 3.255, 3.251, 3.248, 3.245, 3.242, 3.239, 3.237, 3.234, 3.232, 3.229, 3.227, 3.225, 3.223, 3.22, 3.218, 3.216, 3.214, 3.213, 3.211, 3.209, 3.207, 3.206, 3.204, 3.202, 3.201, 3.199, 3.198, 3.197, 3.195, 3.194, 3.193, 3.191, 3.19, 3.189, 3.188, 3.187, 3.185, 3.184, 3.183, 3.182, 3.181, 3.18, 3.179, 3.178, 3.177, 3.176, 3.175, 3.175, 3.174, 3.09} @@ -145,19 +149,19 @@ struct AggregateFunctionWelchTTestData final { } //round or make infinity dof - dof = (int) dof; + dof = static_cast(dof); if (dof > 100) { dof = 101; } //check if abs of t is greater than table[dof] - t = abs(t) + t = abs(t); if(t > CriticalValuesTable[table][dof]) { - return (UInt8) 1; + return static_cast(1); //in this case we reject the null hypothesis } else { - return (UInt8) 0; + return static_cast(0); } } }; From 62460faf97e2a10d3245d7b11d139240fcafa84b Mon Sep 17 00:00:00 2001 From: antikvist Date: Mon, 27 Apr 2020 01:59:41 +0300 Subject: [PATCH 003/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 2 +- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 46b533a2c0b..a4e0a54775b 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -9,7 +9,7 @@ namespace DB namespace { -AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, const DataTypes & argument_types, const Array & parameters) +AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argument_types, const Array & parameters) { diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index be1e176d540..0279849be2c 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -41,7 +41,7 @@ Float64 CriticalValuesTable[SIGN_LVL_CNT][102] = { // for significance level = 0.002 {0.002, 318.313, 22.327, 10.215, 7.173, 5.893, 5.208, 4.782, 4.499, 4.296, 4.143, 4.024, 3.929, 3.852, 3.787, 3.733, 3.686, 3.646, 3.61, 3.579, 3.552, 3.527, 3.505, 3.485, 3.467, 3.45, 3.435, 3.421, 3.408, 3.396, 3.385, 3.375, 3.365, 3.356, 3.348, 3.34, 3.333, 3.326, 3.319, 3.313, 3.307, 3.301, 3.296, 3.291, 3.286, 3.281, 3.277, 3.273, 3.269, 3.265, 3.261, 3.258, 3.255, 3.251, 3.248, 3.245, 3.242, 3.239, 3.237, 3.234, 3.232, 3.229, 3.227, 3.225, 3.223, 3.22, 3.218, 3.216, 3.214, 3.213, 3.211, 3.209, 3.207, 3.206, 3.204, 3.202, 3.201, 3.199, 3.198, 3.197, 3.195, 3.194, 3.193, 3.191, 3.19, 3.189, 3.188, 3.187, 3.185, 3.184, 3.183, 3.182, 3.181, 3.18, 3.179, 3.178, 3.177, 3.176, 3.175, 3.175, 3.174, 3.09} -} +}; // our algorithm implementation via vectors: // https://gist.github.com/ltybc-coder/792748cfdb2f7cadef424ffb7b011c71 From dcfb99b9877528fe09db02a1afff6346de2c8e57 Mon Sep 17 00:00:00 2001 From: antikvist Date: Mon, 27 Apr 2020 19:16:02 +0300 Subject: [PATCH 004/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- .../AggregateFunctionWelchTTest.h | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 0279849be2c..ecc397cf731 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -17,6 +17,11 @@ #include +#include + +#include + + namespace DB { // hard-codded values - part of the algorithm @@ -80,8 +85,8 @@ struct AggregateFunctionWelchTTestData final { sum_y += y; size_x++; size_y++; - mean_x = (Float64) sum_x / size_x; - mean_y = (Float64) sum_y / size_y; + mean_x = static_cast(sum_x) / size_x; + mean_y = static_cast(sum_y) / size_y; square_sum_x += x * x; square_sum_y += y * y; } @@ -91,8 +96,8 @@ struct AggregateFunctionWelchTTestData final { sum_y += other.sum_y; size_x += other.size_x; size_y += other.size_y; - mean_x = (Float64) sum_x / size_x; - mean_y = (Float64) sum_y / size_y; + mean_x = static_cast(sum_x) / size_x; + mean_y = static_cast(sum_y) / size_y; square_sum_x += other.square_sum_x; square_sum_y += other.square_sum_y; } @@ -120,19 +125,19 @@ struct AggregateFunctionWelchTTestData final { } Float64 get_sx() const { - return (Float64)(square_sum_x + size_x * mean_x * mean_x - 2 * mean_x * sum_x) / (size_x - 1); + return static_cast(square_sum_x + size_x * mean_x * mean_x - 2 * mean_x * sum_x) / (size_x - 1); } Float64 get_sy() const { - return (Float64)(square_sum_y + size_y * mean_y * mean_y - 2 * mean_y * sum_y) / (size_y - 1); + return static_cast(square_sum_y + size_y * mean_y * mean_y - 2 * mean_y * sum_y) / (size_y - 1); } Float64 get_T(Float64 sx, Float64 sy) const { - return (Float64)(mean_x - mean_y) / std::sqrt(sx / size_x + sy / size_y); + return static_cast(mean_x - mean_y) / std::sqrt(sx / size_x + sy / size_y); } Float64 get_degrees_of_freed(Float64 sx, Float64 sy) const { - return (Float64)(sx / size_x + sy / size_y) * (sx / size_x + sy / size_y) / + return static_cast(sx / size_x + sy / size_y) * (sx / size_x + sy / size_y) / ((sx * sx / (size_x * size_x * (size_x - 1))) + (sy * sy / (size_y * size_y * (size_y - 1)))); } From 7cfe5ef42b80a45243e70d3b73cf459054e46890 Mon Sep 17 00:00:00 2001 From: antikvist Date: Fri, 1 May 2020 00:36:37 +0300 Subject: [PATCH 005/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index ecc397cf731..e716553e065 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -144,7 +144,7 @@ struct AggregateFunctionWelchTTestData final { Ret get_result(Float64 t, Float64 dof, Float64 parametr) const { //find our table - int table; + int table = 0; for (int i = 0; i < SIGN_LVL_CNT; ++i) { if (CriticalValuesTable[i][0] == parametr) @@ -249,12 +249,11 @@ public: Float64 dof = this->data(place).get_degrees_of_freed(sx, sy); Ret result = this->data(place).get_result(t_value, dof, significance_level); - auto & column = static_cast(to); + //check the type + auto & column = static_cast &>(to); column.getData().push_back(result); } -} }; - -}; \ No newline at end of file +}; From 4f56cc32ae388c5dc002afe071b0b24a9d1adafb Mon Sep 17 00:00:00 2001 From: antikvist Date: Wed, 6 May 2020 00:48:05 +0300 Subject: [PATCH 006/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 6 +++--- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index a4e0a54775b..c4a0c6d4e2b 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -9,11 +9,11 @@ namespace DB namespace { +template AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argument_types, const Array & parameters) { - - - return std::make_shared(argument_types, parameters); + Float64 significance_level = applyVisitor(FieldVisitorConvertToNumber(), params[0]); + return std::make_shared>(argument_types, parameters); } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index e716553e065..a1416f9ab41 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -154,14 +154,14 @@ struct AggregateFunctionWelchTTestData final { } //round or make infinity dof - dof = static_cast(dof); - if (dof > 100) + i_dof = static_cast(dof); + if (i_dof > 100) { - dof = 101; + i_dof = 101; } //check if abs of t is greater than table[dof] t = abs(t); - if(t > CriticalValuesTable[table][dof]) { + if(t > CriticalValuesTable[table][i_dof]) { return static_cast(1); //in this case we reject the null hypothesis } From f0ac5b441f7dccbdfe31512c90ae02da1d9921fa Mon Sep 17 00:00:00 2001 From: antikvist Date: Wed, 6 May 2020 02:07:51 +0300 Subject: [PATCH 007/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 1 - src/AggregateFunctions/AggregateFunctionWelchTTest.h | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index c4a0c6d4e2b..2c8d1e0aed8 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -12,7 +12,6 @@ namespace template AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argument_types, const Array & parameters) { - Float64 significance_level = applyVisitor(FieldVisitorConvertToNumber(), params[0]); return std::make_shared>(argument_types, parameters); } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index a1416f9ab41..9445ccc506d 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -154,7 +154,7 @@ struct AggregateFunctionWelchTTestData final { } //round or make infinity dof - i_dof = static_cast(dof); + int i_dof = static_cast(dof); if (i_dof > 100) { i_dof = 101; From 3dde788146bcef2697cd2701b60d82f603d9c887 Mon Sep 17 00:00:00 2001 From: antikvist Date: Thu, 7 May 2020 14:17:58 +0300 Subject: [PATCH 008/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 9445ccc506d..5203ba1f988 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -186,7 +186,7 @@ public: IAggregateFunctionDataHelper< AggregateFunctionWelchTTestData, AggregateFunctionWelchTTest - > {arguments, params} + > ({argument}, params) { // notice: arguments has been in factory } From 92afa6c0decda463505263f2956005c58e831862 Mon Sep 17 00:00:00 2001 From: antikvist Date: Thu, 7 May 2020 17:14:29 +0300 Subject: [PATCH 009/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 5203ba1f988..22dda8ea244 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -186,7 +186,7 @@ public: IAggregateFunctionDataHelper< AggregateFunctionWelchTTestData, AggregateFunctionWelchTTest - > ({argument}, params) + > ({arguments}, params) { // notice: arguments has been in factory } From 7975d8d5b0c9c4534f108344deb1fdd279cf1eaf Mon Sep 17 00:00:00 2001 From: antikvist Date: Thu, 7 May 2020 23:11:25 +0300 Subject: [PATCH 010/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- .../AggregateFunctionCount.cpp | 2 +- .../AggregateFunctionWelchTTest.h | 47 +++++++++++-------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionCount.cpp b/src/AggregateFunctions/AggregateFunctionCount.cpp index 6c22fec87a2..7ede78e720f 100644 --- a/src/AggregateFunctions/AggregateFunctionCount.cpp +++ b/src/AggregateFunctions/AggregateFunctionCount.cpp @@ -22,7 +22,7 @@ AggregateFunctionPtr createAggregateFunctionCount(const std::string & name, cons void registerAggregateFunctionCount(AggregateFunctionFactory & factory) { - factory.registerFunction("count", createAggregateFunctionCount, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("count", createAggregateFunctionCount); } } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 22dda8ea244..e2e720a12ef 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include #include @@ -12,14 +14,8 @@ #include #include -#include -#include -#include - - #include -#include namespace DB @@ -52,7 +48,8 @@ Float64 CriticalValuesTable[SIGN_LVL_CNT][102] = { // https://gist.github.com/ltybc-coder/792748cfdb2f7cadef424ffb7b011c71 // col, col, bool template -struct AggregateFunctionWelchTTestData final { +struct AggregateFunctionWelchTTestData final +{ size_t size_x = 0; size_t size_y = 0; @@ -65,14 +62,16 @@ struct AggregateFunctionWelchTTestData final { /* not yet sure how to use them - void add_x(X x) { + void add_x(X x) + { mean_x = (Float64)(sum_x + x) / (size_x + 1); size_x ++; sum_x += x; square_sum_x += x * x; } - void add_y(Y y) { + void add_y(Y y) + { mean_y = (sum_y + y) / (size_y + 1); size_y ++; sum_y += y; @@ -80,7 +79,8 @@ struct AggregateFunctionWelchTTestData final { } */ - void add(X x, Y y) { + void add(X x, Y y) + { sum_x += x; sum_y += y; size_x++; @@ -91,7 +91,8 @@ struct AggregateFunctionWelchTTestData final { square_sum_y += y * y; } - void merge(const AggregateFunctionWelchTTestData &other) { + void merge(const AggregateFunctionWelchTTestData &other) + { sum_x += other.sum_x; sum_y += other.sum_y; size_x += other.size_x; @@ -102,7 +103,8 @@ struct AggregateFunctionWelchTTestData final { square_sum_y += other.square_sum_y; } - void serialize(WriteBuffer &buf) const { + void serialize(WriteBuffer &buf) const + { writeBinary(mean_x, buf); writeBinary(mean_y, buf); writeBinary(sum_x, buf); @@ -113,7 +115,8 @@ struct AggregateFunctionWelchTTestData final { writeBinary(size_y, buf); } - void deserialize(ReadBuffer &buf) { + void deserialize(ReadBuffer &buf) + { readBinary(mean_x, buf); readBinary(mean_y, buf); readBinary(sum_x, buf); @@ -124,19 +127,23 @@ struct AggregateFunctionWelchTTestData final { readBinary(size_y, buf); } - Float64 get_sx() const { + Float64 get_sx() const + { return static_cast(square_sum_x + size_x * mean_x * mean_x - 2 * mean_x * sum_x) / (size_x - 1); } - Float64 get_sy() const { + Float64 get_sy() const + { return static_cast(square_sum_y + size_y * mean_y * mean_y - 2 * mean_y * sum_y) / (size_y - 1); } - Float64 get_T(Float64 sx, Float64 sy) const { + Float64 get_T(Float64 sx, Float64 sy) const + { return static_cast(mean_x - mean_y) / std::sqrt(sx / size_x + sy / size_y); } - Float64 get_degrees_of_freed(Float64 sx, Float64 sy) const { + Float64 get_degrees_of_freed(Float64 sx, Float64 sy) const + { return static_cast(sx / size_x + sy / size_y) * (sx / size_x + sy / size_y) / ((sx * sx / (size_x * size_x * (size_x - 1))) + (sy * sy / (size_y * size_y * (size_y - 1)))); } @@ -161,11 +168,13 @@ struct AggregateFunctionWelchTTestData final { } //check if abs of t is greater than table[dof] t = abs(t); - if(t > CriticalValuesTable[table][i_dof]) { + if(t > CriticalValuesTable[table][i_dof]) + { return static_cast(1); //in this case we reject the null hypothesis } - else { + else + { return static_cast(0); } } From b390043f31fe9a4023e9a00b248560155f114b72 Mon Sep 17 00:00:00 2001 From: antikvist Date: Fri, 8 May 2020 00:39:51 +0300 Subject: [PATCH 011/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- .../AggregateFunctionWelchTTest.cpp | 12 ++++++++++-- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 12 +++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 2c8d1e0aed8..c8f2a46b1e4 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -12,7 +12,15 @@ namespace template AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argument_types, const Array & parameters) { - return std::make_shared>(argument_types, parameters); + // default value + Float64 significance_level = 0.1; + if (!params.empty()) + { + significance_level = applyVisitor(FieldVisitorConvertToNumber(), params[0]); + } + + + return std::make_shared>(significance_level, argument_types, parameters); } @@ -21,7 +29,7 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argumen void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); } } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index e2e720a12ef..367970fa4e0 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -168,7 +168,7 @@ struct AggregateFunctionWelchTTestData final } //check if abs of t is greater than table[dof] t = abs(t); - if(t > CriticalValuesTable[table][i_dof]) + if (t > CriticalValuesTable[table][i_dof]) { return static_cast(1); //in this case we reject the null hypothesis @@ -187,15 +187,22 @@ class AggregateFunctionWelchTTest final : public AggregateFunctionWelchTTest > { + + +private: + Float64 significance_level; + + public: AggregateFunctionWelchTTest( + Float64 sglvl_, const DataTypes & arguments, const Array & params ): IAggregateFunctionDataHelper< AggregateFunctionWelchTTestData, AggregateFunctionWelchTTest - > ({arguments}, params) + > ({arguments}, params), significance_level(sglvl_) { // notice: arguments has been in factory } @@ -250,7 +257,6 @@ public: IColumn & to ) const override { - Float64 significance_level = applyVisitor(FieldVisitorConvertToNumber(), params[0]); Float64 sx = this->data(place).get_sx(); Float64 sy = this->data(place).get_sy(); From ab7d1fb86fe2645bb7d3ed43cb79b41517f64b08 Mon Sep 17 00:00:00 2001 From: antikvist Date: Fri, 8 May 2020 00:44:31 +0300 Subject: [PATCH 012/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 367970fa4e0..4b199e799f5 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -264,7 +264,7 @@ public: Float64 dof = this->data(place).get_degrees_of_freed(sx, sy); Ret result = this->data(place).get_result(t_value, dof, significance_level); - //check the type + auto & column = static_cast &>(to); column.getData().push_back(result); } From 4b4ff06cab3642cd061f80a8d0eddb74ff0d79db Mon Sep 17 00:00:00 2001 From: antikvist Date: Fri, 8 May 2020 02:22:12 +0300 Subject: [PATCH 013/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 5 +++-- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index c8f2a46b1e4..151071091e4 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -1,6 +1,7 @@ #include #include #include +#include "registerAggregateFunctions.h" namespace DB @@ -14,9 +15,9 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argumen { // default value Float64 significance_level = 0.1; - if (!params.empty()) + if (!parameters.empty()) { - significance_level = applyVisitor(FieldVisitorConvertToNumber(), params[0]); + significance_level = applyVisitor(FieldVisitorConvertToNumber(), parameters[0]); } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 4b199e799f5..855b3fcd917 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -17,7 +17,6 @@ #include - namespace DB { // hard-codded values - part of the algorithm @@ -269,6 +268,5 @@ public: column.getData().push_back(result); } - }; }; From df4c312e141f938099ab5e9d00d17f40c442389d Mon Sep 17 00:00:00 2001 From: antikvist Date: Fri, 8 May 2020 13:17:59 +0300 Subject: [PATCH 014/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 151071091e4..bc759b9f8f1 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -30,7 +30,7 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argumen void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); + factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); } } From 6d6f0b00ba532c3bb637d5a8f6d2dcaf845f257e Mon Sep 17 00:00:00 2001 From: antikvist Date: Sat, 9 May 2020 01:55:09 +0300 Subject: [PATCH 015/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- .../AggregateFunctionWelchTTest.cpp | 25 ++++++++--- .../AggregateFunctionWelchTTest.h | 43 +++++++++++-------- 2 files changed, 43 insertions(+), 25 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index bc759b9f8f1..1634d0149da 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -3,26 +3,39 @@ #include #include "registerAggregateFunctions.h" +#include +#include +#include + namespace DB { namespace { +//template +static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) +{ + return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); +} -template -AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argument_types, const Array & parameters) +//template +AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, + const DataTypes & argument_types, + const Array & parameters) { // default value Float64 significance_level = 0.1; + if (parameters.size() > 1) + throw Exception("Aggregate function " + name + " requires two parameters or less.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); if (!parameters.empty()) { significance_level = applyVisitor(FieldVisitorConvertToNumber(), parameters[0]); } - - return std::make_shared>(significance_level, argument_types, parameters); - + AggregateFunctionPtr res (createWithExtraTypes(significance_level, argument_types, parameters)); + return res; } } @@ -30,7 +43,7 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argumen void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); + factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); } } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 855b3fcd917..bbf02200745 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -46,16 +46,16 @@ Float64 CriticalValuesTable[SIGN_LVL_CNT][102] = { // our algorithm implementation via vectors: // https://gist.github.com/ltybc-coder/792748cfdb2f7cadef424ffb7b011c71 // col, col, bool -template +//template struct AggregateFunctionWelchTTestData final { size_t size_x = 0; size_t size_y = 0; - X sum_x = 0; - Y sum_y = 0; - X square_sum_x = 0; - Y square_sum_y = 0; + Float64 sum_x = static_cast(0); + Float64 sum_y = static_cast(0); + Float64 square_sum_x = static_cast(0); + Float64 square_sum_y = static_cast(0); Float64 mean_x = 0; Float64 mean_y = 0; @@ -78,7 +78,7 @@ struct AggregateFunctionWelchTTestData final } */ - void add(X x, Y y) + void add(Float64 x, Float64 y) { sum_x += x; sum_y += y; @@ -147,7 +147,7 @@ struct AggregateFunctionWelchTTestData final ((sx * sx / (size_x * size_x * (size_x - 1))) + (sy * sy / (size_y * size_y * (size_y - 1)))); } - Ret get_result(Float64 t, Float64 dof, Float64 parametr) const + UInt8 get_result(Float64 t, Float64 dof, Float64 parametr) const { //find our table int table = 0; @@ -179,11 +179,11 @@ struct AggregateFunctionWelchTTestData final } }; -template -class AggregateFunctionWelchTTest final : public +//template +class AggregateFunctionWelchTTest : public IAggregateFunctionDataHelper< - AggregateFunctionWelchTTestData, - AggregateFunctionWelchTTest + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest > { @@ -199,8 +199,8 @@ public: const Array & params ): IAggregateFunctionDataHelper< - AggregateFunctionWelchTTestData, - AggregateFunctionWelchTTest + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest > ({arguments}, params), significance_level(sglvl_) { // notice: arguments has been in factory @@ -211,6 +211,11 @@ public: return "WelchTTest"; } + DataTypePtr getReturnType() const override + { + return std::make_shared(); + } + void add( AggregateDataPtr place, const IColumn ** columns, @@ -218,11 +223,11 @@ public: Arena * ) const override { - auto col_x = assert_cast *>(columns[0]); - auto col_y = assert_cast *>(columns[1]); + auto col_x = assert_cast *>(columns[0]); + auto col_y = assert_cast *>(columns[1]); - X x = col_x->getData()[row_num]; - Y y = col_y->getData()[row_num]; + Float64 x = col_x->getData()[row_num]; + Float64 y = col_y->getData()[row_num]; this->data(place).add(x, y); } @@ -261,10 +266,10 @@ public: Float64 sy = this->data(place).get_sy(); Float64 t_value = this->data(place).get_T(sx, sy); Float64 dof = this->data(place).get_degrees_of_freed(sx, sy); - Ret result = this->data(place).get_result(t_value, dof, significance_level); + UInt8 result = this->data(place).get_result(t_value, dof, significance_level); - auto & column = static_cast &>(to); + auto & column = static_cast &>(to); column.getData().push_back(result); } From f6e6c48d9c477f92f7669996e4742d1125061ea5 Mon Sep 17 00:00:00 2001 From: antikvist Date: Sun, 17 May 2020 12:25:41 +0300 Subject: [PATCH 016/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionCount.cpp | 2 +- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionCount.cpp b/src/AggregateFunctions/AggregateFunctionCount.cpp index 7ede78e720f..6c22fec87a2 100644 --- a/src/AggregateFunctions/AggregateFunctionCount.cpp +++ b/src/AggregateFunctions/AggregateFunctionCount.cpp @@ -22,7 +22,7 @@ AggregateFunctionPtr createAggregateFunctionCount(const std::string & name, cons void registerAggregateFunctionCount(AggregateFunctionFactory & factory) { - factory.registerFunction("count", createAggregateFunctionCount); + factory.registerFunction("count", createAggregateFunctionCount, AggregateFunctionFactory::CaseInsensitive); } } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 1634d0149da..90b1c445a14 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -14,7 +14,7 @@ namespace DB namespace { //template -static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) +IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) { return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); } From 6069750b0e6bf95ab180d033c0fe376102d540fd Mon Sep 17 00:00:00 2001 From: antikvist Date: Sun, 17 May 2020 14:15:49 +0300 Subject: [PATCH 017/325] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 90b1c445a14..1634d0149da 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -14,7 +14,7 @@ namespace DB namespace { //template -IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) +static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) { return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); } From 1903e6cec227f216159b23656213587beaea4981 Mon Sep 17 00:00:00 2001 From: antikvist Date: Fri, 12 Jun 2020 02:43:55 +0300 Subject: [PATCH 018/325] pray to ClickHouse gods --- .../AggregateFunctionWelchTTest.cpp | 21 +++++--- .../AggregateFunctionWelchTTest.h | 50 +++++++++---------- 2 files changed, 37 insertions(+), 34 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 1634d0149da..3a2bc831cc1 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -4,30 +4,34 @@ #include "registerAggregateFunctions.h" #include -#include #include +namespace ErrorCodes +{ +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + namespace DB { namespace { -//template +template static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) { return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); } -//template +template AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, - const DataTypes & argument_types, - const Array & parameters) + const DataTypes & argument_types, + const Array & parameters) { // default value Float64 significance_level = 0.1; if (parameters.size() > 1) - throw Exception("Aggregate function " + name + " requires two parameters or less.", + throw Exception("Aggregate function " + name + " requires one parameter or less.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); if (!parameters.empty()) { @@ -40,10 +44,11 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, } +template void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); + factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); } -} +} \ No newline at end of file diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index bbf02200745..d625cc908ec 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -46,16 +46,17 @@ Float64 CriticalValuesTable[SIGN_LVL_CNT][102] = { // our algorithm implementation via vectors: // https://gist.github.com/ltybc-coder/792748cfdb2f7cadef424ffb7b011c71 // col, col, bool -//template +template +//template struct AggregateFunctionWelchTTestData final { size_t size_x = 0; size_t size_y = 0; - Float64 sum_x = static_cast(0); - Float64 sum_y = static_cast(0); - Float64 square_sum_x = static_cast(0); - Float64 square_sum_y = static_cast(0); + X sum_x = static_cast(0); + Y sum_y = static_cast(0); + X square_sum_x = static_cast(0); + Y square_sum_y = static_cast(0); Float64 mean_x = 0; Float64 mean_y = 0; @@ -78,7 +79,7 @@ struct AggregateFunctionWelchTTestData final } */ - void add(Float64 x, Float64 y) + void add(X x, Y y) { sum_x += x; sum_y += y; @@ -147,7 +148,7 @@ struct AggregateFunctionWelchTTestData final ((sx * sx / (size_x * size_x * (size_x - 1))) + (sy * sy / (size_y * size_y * (size_y - 1)))); } - UInt8 get_result(Float64 t, Float64 dof, Float64 parametr) const + Ret get_result(Float64 t, Float64 dof, Float64 parametr) const { //find our table int table = 0; @@ -169,29 +170,27 @@ struct AggregateFunctionWelchTTestData final t = abs(t); if (t > CriticalValuesTable[table][i_dof]) { - return static_cast(1); + return static_cast(1); //in this case we reject the null hypothesis } else { - return static_cast(0); + return static_cast(0); } } }; -//template +template class AggregateFunctionWelchTTest : public - IAggregateFunctionDataHelper< - AggregateFunctionWelchTTestData, - AggregateFunctionWelchTTest - > + IAggregateFunctionDataHelper< + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest + > { - private: Float64 significance_level; - public: AggregateFunctionWelchTTest( Float64 sglvl_, @@ -213,7 +212,7 @@ public: DataTypePtr getReturnType() const override { - return std::make_shared(); + return std::make_shared>(); } void add( @@ -223,11 +222,11 @@ public: Arena * ) const override { - auto col_x = assert_cast *>(columns[0]); - auto col_y = assert_cast *>(columns[1]); + auto col_x = assert_cast *>(columns[0]); + auto col_y = assert_cast *>(columns[1]); - Float64 x = col_x->getData()[row_num]; - Float64 y = col_y->getData()[row_num]; + X x = col_x->getData()[row_num]; + Y y = col_y->getData()[row_num]; this->data(place).add(x, y); } @@ -257,7 +256,7 @@ public: } void insertResultInto( - ConstAggregateDataPtr place, + AggregateDataPtr place, IColumn & to ) const override { @@ -266,12 +265,11 @@ public: Float64 sy = this->data(place).get_sy(); Float64 t_value = this->data(place).get_T(sx, sy); Float64 dof = this->data(place).get_degrees_of_freed(sx, sy); - UInt8 result = this->data(place).get_result(t_value, dof, significance_level); + Ret result = this->data(place).get_result(t_value, dof, significance_level); - - auto & column = static_cast &>(to); + auto & column = static_cast &>(to); column.getData().push_back(result); } }; -}; +}; \ No newline at end of file From bbfccd491ece148818b5d0b09ffdd6abaedae908 Mon Sep 17 00:00:00 2001 From: antikvist Date: Fri, 12 Jun 2020 16:51:33 +0300 Subject: [PATCH 019/325] welch t-test --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 2 +- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 3a2bc831cc1..853a1182340 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -20,7 +20,7 @@ namespace template static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) { - return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); + return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); } template diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index d625cc908ec..bbd02d844c4 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -198,8 +198,8 @@ public: const Array & params ): IAggregateFunctionDataHelper< - AggregateFunctionWelchTTestData, - AggregateFunctionWelchTTest + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest > ({arguments}, params), significance_level(sglvl_) { // notice: arguments has been in factory From 13faa3d83f8e50a2e1ea1c23b0026a11c9d72163 Mon Sep 17 00:00:00 2001 From: antikvist Date: Sat, 13 Jun 2020 19:23:17 +0300 Subject: [PATCH 020/325] welch t-test --- .../AggregateFunctionWelchTTest.cpp | 26 ++++++++++------- .../AggregateFunctionWelchTTest.h | 28 +++++++++---------- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 853a1182340..9f451fd5d88 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -7,6 +7,8 @@ #include +// the return type is boolean (we use UInt8 as we do not have boolean in clickhouse) + namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -17,13 +19,11 @@ namespace DB namespace { -template -static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) -{ - return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); -} +//static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) +//{ +// return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); +//} -template AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, const DataTypes & argument_types, const Array & parameters) @@ -38,17 +38,23 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, significance_level = applyVisitor(FieldVisitorConvertToNumber(), parameters[0]); } - AggregateFunctionPtr res (createWithExtraTypes(significance_level, argument_types, parameters)); + AggregateFunctionPtr res; + DataTypePtr data_type = argument_types[0]; +// if (isDecimal(data_type)) +// res.reset(createWithDecimalType(*data_type, significance_level, argument_types, parameters)); +// else + res.reset(createWithNumericType(*data_type, significance_level, argument_types, parameters)); + + //AggregateFunctionPtr res (createWithExtraTypes(significance_level, argument_types, parameters)); return res; } } -template + void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - - factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); + factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); } } \ No newline at end of file diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index bbd02d844c4..13b9c992162 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -46,8 +46,8 @@ Float64 CriticalValuesTable[SIGN_LVL_CNT][102] = { // our algorithm implementation via vectors: // https://gist.github.com/ltybc-coder/792748cfdb2f7cadef424ffb7b011c71 // col, col, bool -template -//template +template +//template struct AggregateFunctionWelchTTestData final { @@ -148,7 +148,7 @@ struct AggregateFunctionWelchTTestData final ((sx * sx / (size_x * size_x * (size_x - 1))) + (sy * sy / (size_y * size_y * (size_y - 1)))); } - Ret get_result(Float64 t, Float64 dof, Float64 parametr) const + UInt8 get_result(Float64 t, Float64 dof, Float64 parametr) const { //find our table int table = 0; @@ -170,21 +170,21 @@ struct AggregateFunctionWelchTTestData final t = abs(t); if (t > CriticalValuesTable[table][i_dof]) { - return static_cast(1); + return static_cast(1); //in this case we reject the null hypothesis } else { - return static_cast(0); + return static_cast(0); } } }; -template +template class AggregateFunctionWelchTTest : public IAggregateFunctionDataHelper< - AggregateFunctionWelchTTestData, - AggregateFunctionWelchTTest + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest > { @@ -198,8 +198,8 @@ public: const Array & params ): IAggregateFunctionDataHelper< - AggregateFunctionWelchTTestData, - AggregateFunctionWelchTTest + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest > ({arguments}, params), significance_level(sglvl_) { // notice: arguments has been in factory @@ -212,7 +212,7 @@ public: DataTypePtr getReturnType() const override { - return std::make_shared>(); + return std::make_shared>(); } void add( @@ -265,11 +265,11 @@ public: Float64 sy = this->data(place).get_sy(); Float64 t_value = this->data(place).get_T(sx, sy); Float64 dof = this->data(place).get_degrees_of_freed(sx, sy); - Ret result = this->data(place).get_result(t_value, dof, significance_level); + UInt8 result = this->data(place).get_result(t_value, dof, significance_level); - auto & column = static_cast &>(to); + auto & column = static_cast &>(to); column.getData().push_back(result); } }; -}; \ No newline at end of file +}; \ No newline at end of file From cb8eec8def65f817773f623e57d8e02518d3c2bd Mon Sep 17 00:00:00 2001 From: antikvist Date: Sun, 14 Jun 2020 00:55:01 +0300 Subject: [PATCH 021/325] welch t-test --- .../AggregateFunctionWelchTTest.cpp | 24 +++++++++++-------- .../AggregateFunctionWelchTTest.h | 2 +- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 9f451fd5d88..8d2963aba74 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -12,6 +12,7 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int NOT_IMPLEMENTED; } namespace DB @@ -31,21 +32,24 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, // default value Float64 significance_level = 0.1; if (parameters.size() > 1) - throw Exception("Aggregate function " + name + " requires one parameter or less.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + { + throw Exception("Aggregate function " + name + " requires one parameter or less.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + } + if (!parameters.empty()) { significance_level = applyVisitor(FieldVisitorConvertToNumber(), parameters[0]); } AggregateFunctionPtr res; - DataTypePtr data_type = argument_types[0]; -// if (isDecimal(data_type)) -// res.reset(createWithDecimalType(*data_type, significance_level, argument_types, parameters)); -// else - res.reset(createWithNumericType(*data_type, significance_level, argument_types, parameters)); - //AggregateFunctionPtr res (createWithExtraTypes(significance_level, argument_types, parameters)); + if (isDecimal(argument_types[0]) || isDecimal(argument_types[1])) + { + throw Exception("Aggregate function " + name + " does not support decimal types.", ErrorCodes::NOT_IMPLEMENTED); + } + + res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, argument_types, parameters)); + return res; } @@ -54,7 +58,7 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); + factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest, AggregateFunctionFactory::CaseInsensitive); } -} \ No newline at end of file +} diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 13b9c992162..29f8e17b6be 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -272,4 +272,4 @@ public: } }; -}; \ No newline at end of file +}; From 9638eb7490bd8cfad234353d791115cd9988cbf6 Mon Sep 17 00:00:00 2001 From: antikvist Date: Sun, 14 Jun 2020 19:18:04 +0300 Subject: [PATCH 022/325] welch t-test --- .../AggregateFunctionWelchTTest.cpp | 15 ++++++--------- .../AggregateFunctionWelchTTest.h | 12 ++++++------ 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 8d2963aba74..c7349f28d90 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -20,10 +20,6 @@ namespace DB namespace { -//static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) -//{ -// return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); -//} AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, const DataTypes & argument_types, @@ -43,12 +39,13 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, AggregateFunctionPtr res; - if (isDecimal(argument_types[0]) || isDecimal(argument_types[1])) - { - throw Exception("Aggregate function " + name + " does not support decimal types.", ErrorCodes::NOT_IMPLEMENTED); - } +// if (isDecimal(argument_types[0]) || isDecimal(argument_types[1])) +// { +// throw Exception("Aggregate function " + name + " does not support decimal types.", ErrorCodes::NOT_IMPLEMENTED); +// } - res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, argument_types, parameters)); + res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, + argument_types, parameters)); return res; } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 29f8e17b6be..210b8990693 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -53,12 +53,12 @@ struct AggregateFunctionWelchTTestData final size_t size_x = 0; size_t size_y = 0; - X sum_x = static_cast(0); - Y sum_y = static_cast(0); - X square_sum_x = static_cast(0); - Y square_sum_y = static_cast(0); - Float64 mean_x = 0; - Float64 mean_y = 0; + X sum_x = static_cast(0); + Y sum_y = static_cast(0); + X square_sum_x = static_cast(0); + Y square_sum_y = static_cast(0); + Float64 mean_x = static_cast(0); + Float64 mean_y = static_cast(0); /* not yet sure how to use them From 3b30ea2f373d841f936b6b2bd5be25e335b82d70 Mon Sep 17 00:00:00 2001 From: antikvist Date: Sun, 14 Jun 2020 23:00:02 +0300 Subject: [PATCH 023/325] welch t-test --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index c7349f28d90..e7ba204046e 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -25,6 +25,8 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, const DataTypes & argument_types, const Array & parameters) { + assertBinary(name, argument_types); + // default value Float64 significance_level = 0.1; if (parameters.size() > 1) @@ -39,10 +41,10 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, AggregateFunctionPtr res; -// if (isDecimal(argument_types[0]) || isDecimal(argument_types[1])) -// { -// throw Exception("Aggregate function " + name + " does not support decimal types.", ErrorCodes::NOT_IMPLEMENTED); -// } + if (isDecimal(argument_types[0]) || isDecimal(argument_types[1])) + { + throw Exception("Aggregate function " + name + " does not support decimal types.", ErrorCodes::NOT_IMPLEMENTED); + } res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, argument_types, parameters)); From 2f073ab3f785fc267715940d685a1b7743c588f4 Mon Sep 17 00:00:00 2001 From: antikvist Date: Mon, 15 Jun 2020 21:18:22 +0300 Subject: [PATCH 024/325] welch t-test --- .../AggregateFunctionWelchTTest.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index e7ba204046e..b1c8f73d2e9 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -43,11 +43,17 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, if (isDecimal(argument_types[0]) || isDecimal(argument_types[1])) { - throw Exception("Aggregate function " + name + " does not support decimal types.", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Aggregate function " + name + " only supports numerical types.", ErrorCodes::NOT_IMPLEMENTED); + } + + else{ + res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, + argument_types, parameters)); } - res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, - argument_types, parameters)); + if(!res){ + throw Exception("Aggregate function " + name + " only supports numerical types.", ErrorCodes::NOT_IMPLEMENTED); + } return res; } From ccce2537f686152e09c7143517707e5dcec8fe95 Mon Sep 17 00:00:00 2001 From: antikvist Date: Mon, 15 Jun 2020 21:21:48 +0300 Subject: [PATCH 025/325] welch t-test --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index b1c8f73d2e9..28ad414146e 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -45,12 +45,13 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, { throw Exception("Aggregate function " + name + " only supports numerical types.", ErrorCodes::NOT_IMPLEMENTED); } - + else{ res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, argument_types, parameters)); } + if(!res){ throw Exception("Aggregate function " + name + " only supports numerical types.", ErrorCodes::NOT_IMPLEMENTED); } From 965bf4bd6535676f99b667b8630fcdce7f4f9e5b Mon Sep 17 00:00:00 2001 From: antikvist Date: Tue, 16 Jun 2020 01:38:35 +0300 Subject: [PATCH 026/325] welch t-test --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 28ad414146e..3d2e98e2a0e 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -46,13 +46,15 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, throw Exception("Aggregate function " + name + " only supports numerical types.", ErrorCodes::NOT_IMPLEMENTED); } - else{ + else + { res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, argument_types, parameters)); } - if(!res){ + if (!res) + { throw Exception("Aggregate function " + name + " only supports numerical types.", ErrorCodes::NOT_IMPLEMENTED); } From 51ff2f4e92d163593c192437e98a5a03bf67011e Mon Sep 17 00:00:00 2001 From: antikvist Date: Tue, 16 Jun 2020 09:14:54 +0300 Subject: [PATCH 027/325] welch t-test --- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 210b8990693..897c583d913 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -272,4 +272,5 @@ public: } }; + }; From a7f8d6b380f290f25725fef6ab97c10dba0c37b1 Mon Sep 17 00:00:00 2001 From: antikvist Date: Tue, 16 Jun 2020 12:45:46 +0300 Subject: [PATCH 028/325] welch t-test --- src/AggregateFunctions/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/src/AggregateFunctions/ya.make b/src/AggregateFunctions/ya.make index bfa32b6dd78..edae91cc745 100644 --- a/src/AggregateFunctions/ya.make +++ b/src/AggregateFunctions/ya.make @@ -49,6 +49,7 @@ SRCS( registerAggregateFunctions.cpp UniqCombinedBiasData.cpp UniqVariadicHash.cpp + AggregateFunctionWelchTTest.cpp ) END() From 21c5ecb597f3c3e4e65cbf4d48b3e604f9428931 Mon Sep 17 00:00:00 2001 From: antikvist Date: Tue, 16 Jun 2020 22:58:06 +0300 Subject: [PATCH 029/325] welch t-test --- .../AggregateFunctionWelchTTest.h | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 897c583d913..266da9fde5b 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -16,6 +16,10 @@ #include +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} namespace DB { @@ -127,6 +131,16 @@ struct AggregateFunctionWelchTTestData final readBinary(size_y, buf); } + size_t get_size_y() const + { + return size_y; + } + + size_t get_size_x() const + { + return size_x; + } + Float64 get_sx() const { return static_cast(square_sum_x + size_x * mean_x * mean_x - 2 * mean_x * sum_x) / (size_x - 1); @@ -162,10 +176,17 @@ struct AggregateFunctionWelchTTestData final //round or make infinity dof int i_dof = static_cast(dof); + if (i_dof > 100) { i_dof = 101; } + + if(i_dof < 100) + { + i_dof = 1; + } + //check if abs of t is greater than table[dof] t = abs(t); if (t > CriticalValuesTable[table][i_dof]) @@ -260,6 +281,13 @@ public: IColumn & to ) const override { + size_t size_x = this->data(place).get_size_x(); + size_t size_y = this->data(place).get_size_y(); + + if(size_x < 2 || size_y < 2) + { + throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS); + } Float64 sx = this->data(place).get_sx(); Float64 sy = this->data(place).get_sy(); From d0f92b5492b635881b72f32d2b4462b4f44b907c Mon Sep 17 00:00:00 2001 From: antikvist Date: Wed, 17 Jun 2020 18:43:22 +0300 Subject: [PATCH 030/325] welch t-test --- .../AggregateFunctionWelchTTest.h | 5 +++-- .../0_stateless/01319_welch_ttest.reference | 3 +++ tests/queries/0_stateless/01319_welch_ttest.sql | 17 +++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/01319_welch_ttest.reference create mode 100644 tests/queries/0_stateless/01319_welch_ttest.sql diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 266da9fde5b..e2ae1761c51 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -13,6 +13,7 @@ #include #include #include +#include #include @@ -189,7 +190,7 @@ struct AggregateFunctionWelchTTestData final //check if abs of t is greater than table[dof] t = abs(t); - if (t > CriticalValuesTable[table][i_dof]) + if (t >= CriticalValuesTable[table][i_dof]) { return static_cast(1); //in this case we reject the null hypothesis @@ -284,7 +285,7 @@ public: size_t size_x = this->data(place).get_size_x(); size_t size_y = this->data(place).get_size_y(); - if(size_x < 2 || size_y < 2) + if( size_x < 2 || size_y < 2) { throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS); } diff --git a/tests/queries/0_stateless/01319_welch_ttest.reference b/tests/queries/0_stateless/01319_welch_ttest.reference new file mode 100644 index 00000000000..e22493782f0 --- /dev/null +++ b/tests/queries/0_stateless/01319_welch_ttest.reference @@ -0,0 +1,3 @@ +1 +0 +0 diff --git a/tests/queries/0_stateless/01319_welch_ttest.sql b/tests/queries/0_stateless/01319_welch_ttest.sql new file mode 100644 index 00000000000..ea103cc433b --- /dev/null +++ b/tests/queries/0_stateless/01319_welch_ttest.sql @@ -0,0 +1,17 @@ +DROP TABLE IF EXISTS welch_ttest; +CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; + +INSERT INTO welch_ttest VALUES (2224.779, 2465.0984), (2588.11, 1909.0328), (1979.625, 1175.8747), (2137.442, 2171.378), (2565.818, 2193.2821), (1754.023, 2854.9475), (1654.947, 2060.1777), (1789.256, 2258.2366), (2320.659, 1856.0535), (2039.532, 1501.8126), (1983.497, 2987.6542), (2232.903, 1681.9778), (2513.93, 2479.6776), (2066.382, 1259.8584), (2492.715, 1120.9043), (1988.287, 1982.1213), (1840.036, 3012.3949), (2249.749, 2252.373), (1766.982, 2591.3122), (1724.84, 1940.589), (0, 1995.185), (0, 2535.1344), (0, 597.3155), (0, 2343.2192), (0, 3154.84), (0, 1125.1966), (0, 1227.8842), (0, 1692.805), (0, 2539.6772), (0, 1936.1927), (0, 1783.7795), (0, 1703.4384), (0, 2077.194), (0, 1614.4071), (0, 2360.0365), (0, 1619.2781), (0, 2033.5109), (0, 2333.7834), (0, 2144.0485), (0, 2583.8709), (0, 1116.7213), (0, 1601.9383), (0, 1570.0431), (0, 1963.0777), (0, 1639.2533), (0, 2277.5223), (0, 1991.9286), (0, 2044.3338), (0, 1794.4781), (0, 1597.9119) +SELECT WelchTTest(0.1)(left, right) from welch_ttest; + +DROP TABLE IF EXISTS welch_ttest; +CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; + +INSERT INTO welch_ttest VALUES (2224.779, 2465.0984), (2588.11, 1909.0328), (1979.625, 1175.8747), (2137.442, 2171.378), (2565.818, 2193.2821), (1754.023, 2854.9475), (1654.947, 2060.1777), (1789.256, 2258.2366), (2320.659, 1856.0535), (2039.532, 1501.8126), (1983.497, 2987.6542), (2232.903, 1681.9778), (2513.93, 2479.6776), (2066.382, 1259.8584), (2492.715, 1120.9043), (1988.287, 1982.1213), (1840.036, 3012.3949), (2249.749, 2252.373), (1766.982, 2591.3122), (1724.84, 1940.589), (0, 1995.185), (0, 2535.1344), (0, 597.3155), (0, 2343.2192), (0, 3154.84), (0, 1125.1966), (0, 1227.8842), (0, 1692.805), (0, 2539.6772), (0, 1936.1927), (0, 1783.7795), (0, 1703.4384), (0, 2077.194), (0, 1614.4071), (0, 2360.0365), (0, 1619.2781), (0, 2033.5109), (0, 2333.7834), (0, 2144.0485), (0, 2583.8709), (0, 1116.7213), (0, 1601.9383), (0, 1570.0431), (0, 1963.0777), (0, 1639.2533), (0, 2277.5223), (0, 1991.9286), (0, 2044.3338), (0, 1794.4781), (0, 1597.9119) +SELECT WelchTTest(0.02)(left, right) from welch_ttest; + +DROP TABLE IF EXISTS welch_ttest; +CREATE TABLE welch_ttest (left Int64, right Int64) ENGINE = Memory; + +INSERT INTO welch_ttest VALUES (1, 1), (1, 1), (1, 1); +SELECT WelchTTest(0.1)(left, right) from welch_ttest; \ No newline at end of file From d92160e734fe71b03e66b330512f2f261274becf Mon Sep 17 00:00:00 2001 From: antikvist Date: Wed, 17 Jun 2020 18:59:58 +0300 Subject: [PATCH 031/325] welch t-test --- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index e2ae1761c51..456effc53b8 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -183,7 +183,7 @@ struct AggregateFunctionWelchTTestData final i_dof = 101; } - if(i_dof < 100) + if (i_dof < 100) { i_dof = 1; } @@ -285,7 +285,7 @@ public: size_t size_x = this->data(place).get_size_x(); size_t size_y = this->data(place).get_size_y(); - if( size_x < 2 || size_y < 2) + if (size_x < 2 || size_y < 2) { throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS); } From e4792df9a96be8a99a68292affb7c406e40d15a5 Mon Sep 17 00:00:00 2001 From: antikvist Date: Wed, 17 Jun 2020 21:56:39 +0300 Subject: [PATCH 032/325] welch t-test --- tests/queries/0_stateless/01319_welch_ttest.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01319_welch_ttest.sql b/tests/queries/0_stateless/01319_welch_ttest.sql index ea103cc433b..26bd686efab 100644 --- a/tests/queries/0_stateless/01319_welch_ttest.sql +++ b/tests/queries/0_stateless/01319_welch_ttest.sql @@ -14,4 +14,5 @@ DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Int64, right Int64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (1, 1), (1, 1), (1, 1); -SELECT WelchTTest(0.1)(left, right) from welch_ttest; \ No newline at end of file +SELECT WelchTTest(0.1)(left, right) from welch_ttest; +DROP TABLE IF EXISTS welch_ttest; \ No newline at end of file From ae8ee1cbfaba57dcee2210d98ccf2ac7f054e6a4 Mon Sep 17 00:00:00 2001 From: antikvist Date: Wed, 17 Jun 2020 22:22:26 +0300 Subject: [PATCH 033/325] welch t-test --- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 456effc53b8..cc1417e6659 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -154,6 +154,16 @@ struct AggregateFunctionWelchTTestData final Float64 get_T(Float64 sx, Float64 sy) const { + if (sx == 0 && sy == 0) + { + throw Exception("division by zero encountered in Aggregate function WelchTTest", ErrorCodes::BAD_ARGUMENTS); + } + + if (sx == -sy && size_x == size_y) + { + throw Exception("division by zero encountered in Aggregate function WelchTTest", ErrorCodes::BAD_ARGUMENTS); + } + return static_cast(mean_x - mean_y) / std::sqrt(sx / size_x + sy / size_y); } From 08f9444842dfa6c0782237b839d491a54200c21e Mon Sep 17 00:00:00 2001 From: antikvist Date: Thu, 18 Jun 2020 00:02:50 +0300 Subject: [PATCH 034/325] welch t-test --- tests/queries/0_stateless/01319_welch_ttest.reference | 1 - tests/queries/0_stateless/01319_welch_ttest.sql | 5 ----- 2 files changed, 6 deletions(-) diff --git a/tests/queries/0_stateless/01319_welch_ttest.reference b/tests/queries/0_stateless/01319_welch_ttest.reference index e22493782f0..b261da18d51 100644 --- a/tests/queries/0_stateless/01319_welch_ttest.reference +++ b/tests/queries/0_stateless/01319_welch_ttest.reference @@ -1,3 +1,2 @@ 1 0 -0 diff --git a/tests/queries/0_stateless/01319_welch_ttest.sql b/tests/queries/0_stateless/01319_welch_ttest.sql index 26bd686efab..b8e881a069b 100644 --- a/tests/queries/0_stateless/01319_welch_ttest.sql +++ b/tests/queries/0_stateless/01319_welch_ttest.sql @@ -10,9 +10,4 @@ CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (2224.779, 2465.0984), (2588.11, 1909.0328), (1979.625, 1175.8747), (2137.442, 2171.378), (2565.818, 2193.2821), (1754.023, 2854.9475), (1654.947, 2060.1777), (1789.256, 2258.2366), (2320.659, 1856.0535), (2039.532, 1501.8126), (1983.497, 2987.6542), (2232.903, 1681.9778), (2513.93, 2479.6776), (2066.382, 1259.8584), (2492.715, 1120.9043), (1988.287, 1982.1213), (1840.036, 3012.3949), (2249.749, 2252.373), (1766.982, 2591.3122), (1724.84, 1940.589), (0, 1995.185), (0, 2535.1344), (0, 597.3155), (0, 2343.2192), (0, 3154.84), (0, 1125.1966), (0, 1227.8842), (0, 1692.805), (0, 2539.6772), (0, 1936.1927), (0, 1783.7795), (0, 1703.4384), (0, 2077.194), (0, 1614.4071), (0, 2360.0365), (0, 1619.2781), (0, 2033.5109), (0, 2333.7834), (0, 2144.0485), (0, 2583.8709), (0, 1116.7213), (0, 1601.9383), (0, 1570.0431), (0, 1963.0777), (0, 1639.2533), (0, 2277.5223), (0, 1991.9286), (0, 2044.3338), (0, 1794.4781), (0, 1597.9119) SELECT WelchTTest(0.02)(left, right) from welch_ttest; -DROP TABLE IF EXISTS welch_ttest; -CREATE TABLE welch_ttest (left Int64, right Int64) ENGINE = Memory; - -INSERT INTO welch_ttest VALUES (1, 1), (1, 1), (1, 1); -SELECT WelchTTest(0.1)(left, right) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; \ No newline at end of file From 4660da3e5e509bd2abd8674fc2babb4dcbeee1c4 Mon Sep 17 00:00:00 2001 From: antikvist Date: Thu, 18 Jun 2020 22:18:52 +0300 Subject: [PATCH 035/325] welch t-test --- .../AggregateFunctionWelchTTest.h | 34 +++++++++---------- .../0_stateless/01319_welch_ttest.reference | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index cc1417e6659..2f56e5e6b6c 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -132,27 +132,27 @@ struct AggregateFunctionWelchTTestData final readBinary(size_y, buf); } - size_t get_size_y() const + size_t getSizeY() const { return size_y; } - size_t get_size_x() const + size_t getSizeX() const { return size_x; } - Float64 get_sx() const + Float64 getSx() const { return static_cast(square_sum_x + size_x * mean_x * mean_x - 2 * mean_x * sum_x) / (size_x - 1); } - Float64 get_sy() const + Float64 getSy() const { return static_cast(square_sum_y + size_y * mean_y * mean_y - 2 * mean_y * sum_y) / (size_y - 1); } - Float64 get_T(Float64 sx, Float64 sy) const + Float64 getT(Float64 sx, Float64 sy) const { if (sx == 0 && sy == 0) { @@ -167,13 +167,13 @@ struct AggregateFunctionWelchTTestData final return static_cast(mean_x - mean_y) / std::sqrt(sx / size_x + sy / size_y); } - Float64 get_degrees_of_freed(Float64 sx, Float64 sy) const + Float64 getDegreesOfFreedom(Float64 sx, Float64 sy) const { return static_cast(sx / size_x + sy / size_y) * (sx / size_x + sy / size_y) / ((sx * sx / (size_x * size_x * (size_x - 1))) + (sy * sy / (size_y * size_y * (size_y - 1)))); } - UInt8 get_result(Float64 t, Float64 dof, Float64 parametr) const + UInt8 getResult(Float64 t, Float64 dof, Float64 parametr) const { //find our table int table = 0; @@ -193,7 +193,7 @@ struct AggregateFunctionWelchTTestData final i_dof = 101; } - if (i_dof < 100) + if (i_dof < 1) { i_dof = 1; } @@ -202,12 +202,12 @@ struct AggregateFunctionWelchTTestData final t = abs(t); if (t >= CriticalValuesTable[table][i_dof]) { - return static_cast(1); + return static_cast(0); //in this case we reject the null hypothesis } else { - return static_cast(0); + return static_cast(1); } } }; @@ -292,19 +292,19 @@ public: IColumn & to ) const override { - size_t size_x = this->data(place).get_size_x(); - size_t size_y = this->data(place).get_size_y(); + size_t size_x = this->data(place).getSizeX(); + size_t size_y = this->data(place).getSizeY(); if (size_x < 2 || size_y < 2) { throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS); } - Float64 sx = this->data(place).get_sx(); - Float64 sy = this->data(place).get_sy(); - Float64 t_value = this->data(place).get_T(sx, sy); - Float64 dof = this->data(place).get_degrees_of_freed(sx, sy); - UInt8 result = this->data(place).get_result(t_value, dof, significance_level); + Float64 sx = this->data(place).getSx(); + Float64 sy = this->data(place).getSy(); + Float64 t_value = this->data(place).getT(sx, sy); + Float64 dof = this->data(place).getDegreesOfFreedom(sx, sy); + UInt8 result = this->data(place).getResult(t_value, dof, significance_level); auto & column = static_cast &>(to); column.getData().push_back(result); diff --git a/tests/queries/0_stateless/01319_welch_ttest.reference b/tests/queries/0_stateless/01319_welch_ttest.reference index b261da18d51..aa47d0d46d4 100644 --- a/tests/queries/0_stateless/01319_welch_ttest.reference +++ b/tests/queries/0_stateless/01319_welch_ttest.reference @@ -1,2 +1,2 @@ -1 +0 0 From 839ee63294515a2899d5a5855bfe824231d17319 Mon Sep 17 00:00:00 2001 From: antikvist Date: Sat, 20 Jun 2020 19:31:00 +0300 Subject: [PATCH 036/325] welch --- .../{01319_welch_ttest.reference => 01322_welch_ttest.reference} | 0 .../0_stateless/{01319_welch_ttest.sql => 01322_welch_ttest.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{01319_welch_ttest.reference => 01322_welch_ttest.reference} (100%) rename tests/queries/0_stateless/{01319_welch_ttest.sql => 01322_welch_ttest.sql} (100%) diff --git a/tests/queries/0_stateless/01319_welch_ttest.reference b/tests/queries/0_stateless/01322_welch_ttest.reference similarity index 100% rename from tests/queries/0_stateless/01319_welch_ttest.reference rename to tests/queries/0_stateless/01322_welch_ttest.reference diff --git a/tests/queries/0_stateless/01319_welch_ttest.sql b/tests/queries/0_stateless/01322_welch_ttest.sql similarity index 100% rename from tests/queries/0_stateless/01319_welch_ttest.sql rename to tests/queries/0_stateless/01322_welch_ttest.sql From 05ad9b9fffe1a49cf91eaf3e5d0df89e42d55124 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 20 Aug 2020 23:59:40 +0300 Subject: [PATCH 037/325] opentelemetry wip --- base/daemon/BaseDaemon.cpp | 2 +- programs/server/config.xml | 10 ++++ src/Common/Exception.h | 7 ++- src/Interpreters/ClientInfo.h | 5 ++ src/Interpreters/Context.cpp | 53 +++++++++++------- src/Interpreters/Context.h | 2 + src/Interpreters/InterpreterSystemQuery.cpp | 4 +- src/Interpreters/OpenTelemetryLog.cpp | 46 ++++++++++++++++ src/Interpreters/OpenTelemetryLog.h | 57 ++++++++++++++++++++ src/Interpreters/SystemLog.cpp | 6 +++ src/Interpreters/SystemLog.h | 3 ++ src/Interpreters/executeQuery.cpp | 4 ++ src/Storages/StorageFile.cpp | 59 ++++++++++++++++++++- 13 files changed, 233 insertions(+), 25 deletions(-) create mode 100644 src/Interpreters/OpenTelemetryLog.cpp create mode 100644 src/Interpreters/OpenTelemetryLog.h diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 78801e71a6f..cc5a765a52d 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -153,7 +153,7 @@ static void signalHandler(int sig, siginfo_t * info, void * context) if (sig != SIGTSTP) /// This signal is used for debugging. { /// The time that is usually enough for separate thread to print info into log. - sleepForSeconds(10); + sleepForSeconds(1); call_default_signal_handler(sig); } diff --git a/programs/server/config.xml b/programs/server/config.xml index af01e880dc2..7e88150b95f 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -563,6 +563,16 @@ 60000 + + + system + opentelemetry_log
+ 7500 + + + diff --git a/src/Common/Exception.h b/src/Common/Exception.h index 763b90048bb..d0de8d6a3f2 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -39,8 +39,11 @@ public: const char * name() const throw() override { return "DB::Exception"; } const char * what() const throw() override { return message().data(); } - /// Add something to the existing message. - void addMessage(const std::string & arg) { extendedMessage(arg); } + template + void addMessage(Fmt&&... fmt) + { + extendedMessage(fmt::format(std::forward(fmt)...)); + } std::string getStackTraceString() const; diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index f3a99112170..575d1521b6f 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -57,6 +57,11 @@ public: String initial_user; String initial_query_id; Poco::Net::SocketAddress initial_address; + + + UInt128 trace_id; + UInt64 span_id; + UInt64 parent_span_id; /// All below are parameters related to initial query. diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 70cf41a679c..93e8ac49114 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1075,35 +1075,37 @@ void Context::setCurrentQueryId(const String & query_id) if (!client_info.current_query_id.empty()) throw Exception("Logical error: attempt to set query_id twice", ErrorCodes::LOGICAL_ERROR); - String query_id_to_set = query_id; + /// Generate random UUID, but using lower quality RNG, + /// because Poco::UUIDGenerator::generateRandom method is using /dev/random, that is very expensive. + /// NOTE: Actually we don't need to use UUIDs for query identifiers. + /// We could use any suitable string instead. + union + { + char bytes[16]; + struct + { + UInt64 a; + UInt64 b; + } words; + UInt128 uuid; + } random; + random.words.a = thread_local_rng(); //-V656 + random.words.b = thread_local_rng(); //-V656 + + trace_id = random.uuid; + + + String query_id_to_set = query_id; if (query_id_to_set.empty()) /// If the user did not submit his query_id, then we generate it ourselves. { - /// Generate random UUID, but using lower quality RNG, - /// because Poco::UUIDGenerator::generateRandom method is using /dev/random, that is very expensive. - /// NOTE: Actually we don't need to use UUIDs for query identifiers. - /// We could use any suitable string instead. - - union - { - char bytes[16]; - struct - { - UInt64 a; - UInt64 b; - } words; - } random; - - random.words.a = thread_local_rng(); //-V656 - random.words.b = thread_local_rng(); //-V656 - /// Use protected constructor. struct QueryUUID : Poco::UUID { QueryUUID(const char * bytes, Poco::UUID::Version version) : Poco::UUID(bytes, version) {} }; - + query_id_to_set = QueryUUID(random.bytes, Poco::UUID::UUID_RANDOM).toString(); } @@ -1705,6 +1707,17 @@ std::shared_ptr Context::getAsynchronousMetricLog() } +std::shared_ptr Context::getOpenTelemetryLog() +{ + auto lock = getLock(); + + if (!shared->system_logs) + return {}; + + return shared->system_logs->opentelemetry_log; +} + + CompressionCodecPtr Context::chooseCompressionCodec(size_t part_size, double part_size_ratio) const { auto lock = getLock(); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index c8d13baa9ae..6cabac3db0d 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -81,6 +81,7 @@ class TextLog; class TraceLog; class MetricLog; class AsynchronousMetricLog; +class OpenTelemetrySpanLog; struct MergeTreeSettings; class StorageS3Settings; class IDatabase; @@ -531,6 +532,7 @@ public: std::shared_ptr getTextLog(); std::shared_ptr getMetricLog(); std::shared_ptr getAsynchronousMetricLog(); + std::shared_ptr getOpenTelemetryLog(); /// Returns an object used to log operations with parts if it possible. /// Provide table name to make required checks. diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 4bfa84090c2..24fb0637b3d 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -321,7 +322,8 @@ BlockIO InterpreterSystemQuery::execute() [&] () { if (auto trace_log = context.getTraceLog()) trace_log->flush(true); }, [&] () { if (auto text_log = context.getTextLog()) text_log->flush(true); }, [&] () { if (auto metric_log = context.getMetricLog()) metric_log->flush(true); }, - [&] () { if (auto asynchronous_metric_log = context.getAsynchronousMetricLog()) asynchronous_metric_log->flush(true); } + [&] () { if (auto asynchronous_metric_log = context.getAsynchronousMetricLog()) asynchronous_metric_log->flush(true); }, + [&] () { if (auto opentelemetry_log = context.getOpenTelemetryLog()) opentelemetry_log->flush(true); } ); break; case Type::STOP_LISTEN_QUERIES: diff --git a/src/Interpreters/OpenTelemetryLog.cpp b/src/Interpreters/OpenTelemetryLog.cpp new file mode 100644 index 00000000000..511c820797a --- /dev/null +++ b/src/Interpreters/OpenTelemetryLog.cpp @@ -0,0 +1,46 @@ +#include "OpenTelemetryLog.h" + +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +Block OpenTelemetrySpanLogElement::createBlock() +{ + return { + {std::make_shared(), "trace_id"}, + {std::make_shared(), "span_id"}, + {std::make_shared(), "parent_span_id"}, + {std::make_shared(), "operation_name"}, + {std::make_shared(), "start_date"}, + {std::make_shared(), "start_time"}, + {std::make_shared(), "finish_time"}, + {std::make_shared(std::make_shared()), + "attribute.names"}, + {std::make_shared(std::make_shared()), + "attribute.values"} + }; +} + +void OpenTelemetrySpanLogElement::appendToBlock(MutableColumns & columns) const +{ + size_t i = 0; + + columns[i++]->insert(trace_id); + columns[i++]->insert(span_id); + columns[i++]->insert(parent_span_id); + columns[i++]->insert(operation_name); + columns[i++]->insert(DateLUT::instance().toDayNum(start_time)); + columns[i++]->insert(start_time); + columns[i++]->insert(finish_time); + columns[i++]->insert(attribute_names); + columns[i++]->insert(attribute_values); +} + +} + diff --git a/src/Interpreters/OpenTelemetryLog.h b/src/Interpreters/OpenTelemetryLog.h new file mode 100644 index 00000000000..723a4201783 --- /dev/null +++ b/src/Interpreters/OpenTelemetryLog.h @@ -0,0 +1,57 @@ +#pragma once + +#include + +namespace DB +{ + +/* +struct OpenTelemetrySpanContext +{ + UInt128 trace_id; + UInt64 span_id; + UInt8 trace_flags; + String trace_state; +}; +*/ + +// TODO figure out precisely which part of this is run time, and which part we +// must log. +struct OpenTelemetrySpan +{ + UInt128 trace_id; + UInt64 span_id; + UInt64 parent_span_id; + std::string operation_name; + time_t start_time{}; + time_t finish_time{}; + Array attribute_names; + Array attribute_values; + // I don't understand how Links work, namely, which direction should they + // point to, and how they are related with parent_span_id, so no Links for + // now. + + // The following fields look like something that is runtime only and doesn't + // require logging. + UInt8 trace_flags; + // Vendor-specific info, key-value pairs. Keep it as a string as described + // here: https://w3c.github.io/trace-context/#tracestate-header + String trace_state; +}; + +struct OpenTelemetrySpanLogElement : public OpenTelemetrySpan +{ + static std::string name() { return "OpenTelemetrySpanLog"; } + static Block createBlock(); + void appendToBlock(MutableColumns & columns) const; +}; + +// OpenTelemetry standartizes some Log data as well, so it's not just +// OpenTelemetryLog to avoid confusion. +class OpenTelemetrySpanLog : public SystemLog +{ +public: + using SystemLog::SystemLog; +}; + +} diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index e64444edd67..27236c99a23 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -88,6 +89,9 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi asynchronous_metric_log = createSystemLog( global_context, "system", "asynchronous_metric_log", config, "asynchronous_metric_log"); + opentelemetry_log = createSystemLog( + global_context, "system", "opentelemetry_log", config, + "opentelemetry_log"); if (query_log) logs.emplace_back(query_log.get()); @@ -105,6 +109,8 @@ SystemLogs::SystemLogs(Context & global_context, const Poco::Util::AbstractConfi logs.emplace_back(metric_log.get()); if (asynchronous_metric_log) logs.emplace_back(asynchronous_metric_log.get()); + if (opentelemetry_log) + logs.emplace_back(opentelemetry_log.get()); try { diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index a2e97747d12..15ebf04c6ce 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -71,6 +71,7 @@ class TraceLog; class CrashLog; class MetricLog; class AsynchronousMetricLog; +class OpenTelemetrySpanLog; class ISystemLog @@ -105,6 +106,8 @@ struct SystemLogs std::shared_ptr metric_log; /// Used to log all metrics. /// Metrics from system.asynchronous_metrics. std::shared_ptr asynchronous_metric_log; + /// OpenTelemetry trace spans + std::shared_ptr opentelemetry_log; std::vector logs; }; diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index fa5b3e0bcc2..27ac5fda56f 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -584,6 +584,10 @@ static std::tuple executeQueryImpl( if (auto query_log = context.getQueryLog()) query_log->add(elem); } + + if (auto opentelemetry_log = context.getOpenTelemetryLog()) + { + } }; auto exception_callback = [elem, &context, ast, log_queries, log_queries_min_type = settings.log_queries_min_type, quota(quota), diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 558216a6216..d22a0625a67 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -199,6 +199,45 @@ StorageFile::StorageFile(CommonArguments args) setInMemoryMetadata(storage_metadata); } +struct ScopedExceptionHint +{ + std::string hint; + + // Format message with fmt::format + template + ScopedExceptionHint(Fmt&&... fmt) + : hint(fmt::format(std::forward(fmt)...)) + { + fprintf(stderr, "hint %s created\n", hint.c_str()); + } + + ~ScopedExceptionHint() + { + fprintf(stderr, "hint %s is being destroyed\n", hint.c_str()); + std::exception_ptr exception = std::current_exception(); + if (exception) + { + try { + std::rethrow_exception(exception); + } + catch (Exception & e) + { + e.addMessage(hint); + fprintf(stderr, "added hint %s\n", hint.c_str()); + } + catch (...) + { + fprintf(stderr, "unknown exception\n"); + // pass? + } + } + else + { + fprintf(stderr, "no exception\n"); + } + } +}; + class StorageFileSource : public SourceWithProgress { public: @@ -317,10 +356,28 @@ public: if (!column_defaults.empty()) reader = std::make_shared(reader, column_defaults, context); + ScopedExceptionHint("Reading prefix of file '{}'", current_path); reader->readPrefix(); } - if (auto res = reader->read()) + ScopedExceptionHint("Reader->read() {}", reader->getName()); + + Block res; + try + { + res = reader->read(); + } + catch (Exception & e) + { + e.addMessage("Error while reading from '{}'", current_path); + throw; + } + catch (...) + { + throw; + } + + if (res) { Columns columns = res.getColumns(); UInt64 num_rows = res.rows(); From 54b3047d19d5aed63c59fe46215c70138052386d Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 27 Aug 2020 21:44:20 +0300 Subject: [PATCH 038/325] fixup --- src/Interpreters/ClientInfo.h | 3 +- src/Interpreters/Context.cpp | 7 ++-- src/Interpreters/OpenTelemetryLog.cpp | 13 +++++--- src/Interpreters/OpenTelemetryLog.h | 2 +- src/Interpreters/executeQuery.cpp | 48 +++++++++++++++++++++++++++ 5 files changed, 63 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index 575d1521b6f..bc6afec36f4 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -2,6 +2,7 @@ #include #include +#include namespace DB @@ -59,7 +60,7 @@ public: Poco::Net::SocketAddress initial_address; - UInt128 trace_id; + __uint128_t trace_id; UInt64 span_id; UInt64 parent_span_id; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 93e8ac49114..c225c332248 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1087,14 +1087,15 @@ void Context::setCurrentQueryId(const String & query_id) UInt64 a; UInt64 b; } words; - UInt128 uuid; + __uint128_t uuid; } random; random.words.a = thread_local_rng(); //-V656 random.words.b = thread_local_rng(); //-V656 - trace_id = random.uuid; - + client_info.trace_id = random.uuid; + client_info.span_id = 1; + client_info.parent_span_id = 0; String query_id_to_set = query_id; if (query_id_to_set.empty()) /// If the user did not submit his query_id, then we generate it ourselves. diff --git a/src/Interpreters/OpenTelemetryLog.cpp b/src/Interpreters/OpenTelemetryLog.cpp index 511c820797a..f8d7d684478 100644 --- a/src/Interpreters/OpenTelemetryLog.cpp +++ b/src/Interpreters/OpenTelemetryLog.cpp @@ -13,12 +13,15 @@ namespace DB Block OpenTelemetrySpanLogElement::createBlock() { return { + // event_date is the date part of event_time, used for indexing. + {std::make_shared(), "event_date"}, + // event_time is the span start time, named so to be compatible with + // the standard ClickHouse system log column names. + {std::make_shared(), "event_time"}, {std::make_shared(), "trace_id"}, {std::make_shared(), "span_id"}, {std::make_shared(), "parent_span_id"}, {std::make_shared(), "operation_name"}, - {std::make_shared(), "start_date"}, - {std::make_shared(), "start_time"}, {std::make_shared(), "finish_time"}, {std::make_shared(std::make_shared()), "attribute.names"}, @@ -31,12 +34,12 @@ void OpenTelemetrySpanLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; - columns[i++]->insert(trace_id); + columns[i++]->insert(DateLUT::instance().toDayNum(start_time)); + columns[i++]->insert(start_time); + columns[i++]->insert(UInt128(Int128(trace_id))); columns[i++]->insert(span_id); columns[i++]->insert(parent_span_id); columns[i++]->insert(operation_name); - columns[i++]->insert(DateLUT::instance().toDayNum(start_time)); - columns[i++]->insert(start_time); columns[i++]->insert(finish_time); columns[i++]->insert(attribute_names); columns[i++]->insert(attribute_values); diff --git a/src/Interpreters/OpenTelemetryLog.h b/src/Interpreters/OpenTelemetryLog.h index 723a4201783..73ad5382c95 100644 --- a/src/Interpreters/OpenTelemetryLog.h +++ b/src/Interpreters/OpenTelemetryLog.h @@ -19,7 +19,7 @@ struct OpenTelemetrySpanContext // must log. struct OpenTelemetrySpan { - UInt128 trace_id; + __uint128_t trace_id; UInt64 span_id; UInt64 parent_span_id; std::string operation_name; diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 27ac5fda56f..e74b24c4aa7 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -146,6 +147,11 @@ static void logQuery(const String & query, const Context & context, bool interna (current_user != "default" ? ", user: " + context.getClientInfo().current_user : ""), (!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()), joinLines(query)); + + LOG_TRACE(&Poco::Logger::get("executeQuery"), + "OpenTelemetry trace id {:x}, span id {:x}, parent span id {:x}", + context.getClientInfo().trace_id, context.getClientInfo().span_id, + context.getClientInfo().parent_span_id); } } @@ -216,6 +222,29 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c if (auto query_log = context.getQueryLog()) query_log->add(elem); + if (auto opentelemetry_log = context.getOpenTelemetryLog()) + { + OpenTelemetrySpanLogElement span; + span.trace_id = context.getClientInfo().trace_id; + span.span_id = context.getClientInfo().span_id; + span.parent_span_id = context.getClientInfo().parent_span_id; + span.operation_name = "query"; + span.start_time = current_time; + span.finish_time = current_time; + + // keep values synchonized to type enum in QueryLogElement::createBlock + span.attribute_names.push_back("status"); + span.attribute_values.push_back("ExceptionBeforeStart"); + + span.attribute_names.push_back("query"); + span.attribute_values.push_back(elem.query); + + span.attribute_names.push_back("query_id"); + span.attribute_values.push_back(elem.client_info.current_query_id); + + opentelemetry_log->add(span); + } + ProfileEvents::increment(ProfileEvents::FailedQuery); if (ast) @@ -587,6 +616,25 @@ static std::tuple executeQueryImpl( if (auto opentelemetry_log = context.getOpenTelemetryLog()) { + OpenTelemetrySpanLogElement span; + span.trace_id = context.getClientInfo().trace_id; + span.span_id = context.getClientInfo().span_id; + span.parent_span_id = context.getClientInfo().parent_span_id; + span.operation_name = "query"; + span.start_time = elem.query_start_time; + span.finish_time = time(nullptr); // current time + + // keep values synchonized to type enum in QueryLogElement::createBlock + span.attribute_names.push_back("status"); + span.attribute_values.push_back("QueryFinish"); + + span.attribute_names.push_back("query"); + span.attribute_values.push_back(elem.query); + + span.attribute_names.push_back("query_id"); + span.attribute_values.push_back(elem.client_info.current_query_id); + + opentelemetry_log->add(span); } }; From 2bef406200c9e097cfc42910b9b055e1f350b840 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 27 Aug 2020 21:49:06 +0300 Subject: [PATCH 039/325] remove accidental changes --- base/daemon/BaseDaemon.cpp | 2 +- src/Interpreters/ClientInfo.h | 1 - src/Storages/StorageFile.cpp | 59 +---------------------------------- 3 files changed, 2 insertions(+), 60 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index cc5a765a52d..78801e71a6f 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -153,7 +153,7 @@ static void signalHandler(int sig, siginfo_t * info, void * context) if (sig != SIGTSTP) /// This signal is used for debugging. { /// The time that is usually enough for separate thread to print info into log. - sleepForSeconds(1); + sleepForSeconds(10); call_default_signal_handler(sig); } diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index bc6afec36f4..52391d6cf73 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -59,7 +59,6 @@ public: String initial_query_id; Poco::Net::SocketAddress initial_address; - __uint128_t trace_id; UInt64 span_id; UInt64 parent_span_id; diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index d22a0625a67..558216a6216 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -199,45 +199,6 @@ StorageFile::StorageFile(CommonArguments args) setInMemoryMetadata(storage_metadata); } -struct ScopedExceptionHint -{ - std::string hint; - - // Format message with fmt::format - template - ScopedExceptionHint(Fmt&&... fmt) - : hint(fmt::format(std::forward(fmt)...)) - { - fprintf(stderr, "hint %s created\n", hint.c_str()); - } - - ~ScopedExceptionHint() - { - fprintf(stderr, "hint %s is being destroyed\n", hint.c_str()); - std::exception_ptr exception = std::current_exception(); - if (exception) - { - try { - std::rethrow_exception(exception); - } - catch (Exception & e) - { - e.addMessage(hint); - fprintf(stderr, "added hint %s\n", hint.c_str()); - } - catch (...) - { - fprintf(stderr, "unknown exception\n"); - // pass? - } - } - else - { - fprintf(stderr, "no exception\n"); - } - } -}; - class StorageFileSource : public SourceWithProgress { public: @@ -356,28 +317,10 @@ public: if (!column_defaults.empty()) reader = std::make_shared(reader, column_defaults, context); - ScopedExceptionHint("Reading prefix of file '{}'", current_path); reader->readPrefix(); } - ScopedExceptionHint("Reader->read() {}", reader->getName()); - - Block res; - try - { - res = reader->read(); - } - catch (Exception & e) - { - e.addMessage("Error while reading from '{}'", current_path); - throw; - } - catch (...) - { - throw; - } - - if (res) + if (auto res = reader->read()) { Columns columns = res.getColumns(); UInt64 num_rows = res.rows(); From 46127946788d0ae89aa8c895b8d31fbb453e04c1 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 28 Aug 2020 04:21:08 +0300 Subject: [PATCH 040/325] opentelemetry context propagation --- src/Core/Defines.h | 5 +- src/Interpreters/ClientInfo.cpp | 140 ++++++++++++++++++++++++++++++ src/Interpreters/ClientInfo.h | 17 +++- src/Interpreters/Context.cpp | 20 ++++- src/Interpreters/executeQuery.cpp | 33 +++++-- src/Interpreters/ya.make | 1 + src/Server/HTTPHandler.cpp | 23 ++++- src/Server/TCPHandler.cpp | 8 +- src/Storages/StorageURL.cpp | 18 +++- 9 files changed, 244 insertions(+), 21 deletions(-) diff --git a/src/Core/Defines.h b/src/Core/Defines.h index e244581c339..d19513d1434 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -67,8 +67,11 @@ /// Minimum revision supporting SettingsBinaryFormat::STRINGS. #define DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS 54429 +/// Minimum revision supporting OpenTelemetry +#define DBMS_MIN_REVISION_WITH_OPENTELEMETRY 54227 + /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change. -#define DBMS_TCP_PROTOCOL_VERSION 54226 +#define DBMS_TCP_PROTOCOL_VERSION 54227 /// The boundary on which the blocks for asynchronous file operations should be aligned. #define DEFAULT_AIO_FILE_BLOCK_SIZE 4096 diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp index 378375dcc18..9e501ca5c11 100644 --- a/src/Interpreters/ClientInfo.cpp +++ b/src/Interpreters/ClientInfo.cpp @@ -60,6 +60,18 @@ void ClientInfo::write(WriteBuffer & out, const UInt64 server_protocol_revision) if (server_protocol_revision >= DBMS_MIN_REVISION_WITH_VERSION_PATCH) writeVarUInt(client_version_patch, out); } + + if (server_protocol_revision >= DBMS_MIN_REVISION_WITH_OPENTELEMETRY) + { + // No point writing these numbers with variable length, because they + // are random and will probably require the full length anyway. + writeBinary(opentelemetry_trace_id, out); + writeBinary(opentelemetry_span_id, out); + writeBinary(opentelemetry_parent_span_id, out); + writeBinary(opentelemetry_tracestate, out); + writeBinary(opentelemetry_trace_flags, out); + std::cerr << fmt::format("wrote {:x}, {}, {}\n", opentelemetry_trace_id, opentelemetry_span_id, opentelemetry_parent_span_id) << StackTrace().toString() << std::endl; + } } @@ -113,6 +125,17 @@ void ClientInfo::read(ReadBuffer & in, const UInt64 client_protocol_revision) else client_version_patch = client_revision; } + + if (client_protocol_revision >= DBMS_MIN_REVISION_WITH_OPENTELEMETRY) + { + readBinary(opentelemetry_trace_id, in); + readBinary(opentelemetry_span_id, in); + readBinary(opentelemetry_parent_span_id, in); + readBinary(opentelemetry_tracestate, in); + readBinary(opentelemetry_trace_flags, in); + + std::cerr << fmt::format("read {:x}, {}, {}\n", opentelemetry_trace_id, opentelemetry_span_id, opentelemetry_parent_span_id) << StackTrace().toString() << std::endl; + } } @@ -123,6 +146,123 @@ void ClientInfo::setInitialQuery() client_name = (DBMS_NAME " ") + client_name; } +template +bool readLowercaseHexDigits(const char *& begin, const char * end, T & dest_value, std::string & error) +{ + char * dest_begin = reinterpret_cast(&dest_value); + char * dest_end = dest_begin + sizeof(dest_value); + bool odd_character = true; + for (;;) + { + if (begin == end) + { + if (dest_begin == dest_end) + { + return true; + } + error = fmt::format("Not enough charaters in the input, got {}, need {} more", end - begin, dest_end - dest_begin); + return false; + } + + if (dest_begin == dest_end) + { + return true; + } + + int cur = 0; + if (*begin >= '0' && *begin <= '9') + { + cur = *begin - '0'; + } + else if (*begin >= 'a' && *begin <= 'f') + { + cur = 10 + *begin - 'a'; + } + else + { + error = fmt::format("Encountered '{}' which is not a lowercase hexadecimal digit", *begin); + return false; + } + + // Two characters per byte, little-endian. + if (odd_character) + { + *(dest_end - 1) = cur; + } + else + { + *(dest_end - 1) = *(dest_end - 1) << 8 | cur; + --dest_end; + } + + begin++; + odd_character = !odd_character; + } +} + +bool ClientInfo::setOpenTelemetryTraceparent(const std::string & traceparent, + std::string & error) +{ + uint8_t version = -1; + __uint128_t trace_id = 0; + uint64_t trace_parent = 0; + uint8_t trace_flags = 0; + + const char * begin = &traceparent[0]; + const char * end = begin + traceparent.length(); + +#define CHECK_CONDITION(condition, ...) \ + ((condition) || (error = fmt::format(__VA_ARGS__), false)) + +#define CHECK_DELIMITER \ + (begin >= end \ + ? (error = fmt::format( \ + "Expected '-' delimiter, got EOL at position {}", \ + begin - &traceparent[0]), \ + false) \ + : *begin != '-' \ + ? (error = fmt::format( \ + "Expected '-' delimiter, got '{}' at position {}", \ + *begin, begin - &traceparent[0]), \ + false) \ + : (++begin, true)) + + bool result = readLowercaseHexDigits(begin, end, version, error) + && CHECK_CONDITION(version == 0, "Expected version 00, got {}", version) + && CHECK_DELIMITER + && readLowercaseHexDigits(begin, end, trace_id, error) + && CHECK_DELIMITER + && readLowercaseHexDigits(begin, end, trace_parent, error) + && CHECK_DELIMITER + && readLowercaseHexDigits(begin, end, trace_flags, error) + && CHECK_CONDITION(begin == end, + "Expected end of string, got {} at position {}", *begin, end - begin); + +#undef CHECK +#undef CHECK_DELIMITER + + if (!result) + { + return false; + } + + opentelemetry_trace_id = trace_id; + opentelemetry_parent_span_id = trace_parent; + opentelemetry_trace_flags = trace_flags; + return true; +} + + +std::string ClientInfo::getOpenTelemetryTraceparentForChild() const +{ + // This span is a parent for its children (so deep...), so we specify + // this span_id as a parent id. + return fmt::format("00-{:032x}-{:016x}-{:02x}", opentelemetry_trace_id, + opentelemetry_span_id, + // This cast is because fmt is being weird and complaining that + // "mixing character types is not allowed". + static_cast(opentelemetry_trace_flags)); +} void ClientInfo::fillOSUserHostNameAndVersionInfo() { diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index 52391d6cf73..413e1c42bf7 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -59,9 +59,17 @@ public: String initial_query_id; Poco::Net::SocketAddress initial_address; - __uint128_t trace_id; - UInt64 span_id; - UInt64 parent_span_id; + // OpenTelemetry things + __uint128_t opentelemetry_trace_id = 0; + // Span ID is not strictly the client info, but convenient to keep here. + // The span id we get the in the incoming client info becomes our parent span + // id, and the span id we send becomes downstream parent span id. + UInt64 opentelemetry_span_id = 0; + UInt64 opentelemetry_parent_span_id = 0; + // the incoming tracestate header, we just pass it downstream. + // https://www.w3.org/TR/trace-context/ + String opentelemetry_tracestate; + UInt8 opentelemetry_trace_flags; /// All below are parameters related to initial query. @@ -95,6 +103,9 @@ public: /// Initialize parameters on client initiating query. void setInitialQuery(); + bool setOpenTelemetryTraceparent(const std::string & traceparent, std::string & error); + std::string getOpenTelemetryTraceparentForChild() const; + private: void fillOSUserHostNameAndVersionInfo(); }; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index c225c332248..c5fd2d585e1 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1093,9 +1093,23 @@ void Context::setCurrentQueryId(const String & query_id) random.words.a = thread_local_rng(); //-V656 random.words.b = thread_local_rng(); //-V656 - client_info.trace_id = random.uuid; - client_info.span_id = 1; - client_info.parent_span_id = 0; + fmt::print(stderr, "traceid {}, ==0 {}\n", client_info.opentelemetry_trace_id, client_info.opentelemetry_trace_id == 0); + if (client_info.opentelemetry_trace_id == 0) + { + // If trace_id is not initialized, it means that this is an initial query + // without any parent OpenTelemetry trace. Use the randomly generated + // default query id as the new trace id. + client_info.opentelemetry_trace_id = random.uuid; + client_info.opentelemetry_parent_span_id = 0; + client_info.opentelemetry_span_id = thread_local_rng(); + } + else + { + // The incoming span id becomes our parent span id. + client_info.opentelemetry_parent_span_id = client_info.opentelemetry_span_id; + client_info.opentelemetry_span_id = thread_local_rng(); + } + fmt::print(stderr, "traceid {}, ==0 {}\n{}\n", client_info.opentelemetry_trace_id, client_info.opentelemetry_trace_id == 0, StackTrace().toString()); String query_id_to_set = query_id; if (query_id_to_set.empty()) /// If the user did not submit his query_id, then we generate it ourselves. diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index e74b24c4aa7..2a35bc205fa 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -149,9 +149,11 @@ static void logQuery(const String & query, const Context & context, bool interna joinLines(query)); LOG_TRACE(&Poco::Logger::get("executeQuery"), - "OpenTelemetry trace id {:x}, span id {:x}, parent span id {:x}", - context.getClientInfo().trace_id, context.getClientInfo().span_id, - context.getClientInfo().parent_span_id); + "OpenTelemetry trace id {:x}, span id {}, parent span id {}", + context.getClientInfo().opentelemetry_trace_id, context.getClientInfo().opentelemetry_span_id, + context.getClientInfo().opentelemetry_parent_span_id); + + std::cerr << StackTrace().toString() << std::endl; } } @@ -225,9 +227,9 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c if (auto opentelemetry_log = context.getOpenTelemetryLog()) { OpenTelemetrySpanLogElement span; - span.trace_id = context.getClientInfo().trace_id; - span.span_id = context.getClientInfo().span_id; - span.parent_span_id = context.getClientInfo().parent_span_id; + span.trace_id = context.getClientInfo().opentelemetry_trace_id; + span.span_id = context.getClientInfo().opentelemetry_span_id; + span.parent_span_id = context.getClientInfo().opentelemetry_parent_span_id; span.operation_name = "query"; span.start_time = current_time; span.finish_time = current_time; @@ -242,6 +244,13 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c span.attribute_names.push_back("query_id"); span.attribute_values.push_back(elem.client_info.current_query_id); + if (!context.getClientInfo().opentelemetry_tracestate.empty()) + { + span.attribute_names.push_back("tracestate"); + span.attribute_values.push_back( + context.getClientInfo().opentelemetry_tracestate); + } + opentelemetry_log->add(span); } @@ -617,9 +626,9 @@ static std::tuple executeQueryImpl( if (auto opentelemetry_log = context.getOpenTelemetryLog()) { OpenTelemetrySpanLogElement span; - span.trace_id = context.getClientInfo().trace_id; - span.span_id = context.getClientInfo().span_id; - span.parent_span_id = context.getClientInfo().parent_span_id; + span.trace_id = context.getClientInfo().opentelemetry_trace_id; + span.span_id = context.getClientInfo().opentelemetry_span_id; + span.parent_span_id = context.getClientInfo().opentelemetry_parent_span_id; span.operation_name = "query"; span.start_time = elem.query_start_time; span.finish_time = time(nullptr); // current time @@ -633,6 +642,12 @@ static std::tuple executeQueryImpl( span.attribute_names.push_back("query_id"); span.attribute_values.push_back(elem.client_info.current_query_id); + if (!context.getClientInfo().opentelemetry_tracestate.empty()) + { + span.attribute_names.push_back("tracestate"); + span.attribute_values.push_back( + context.getClientInfo().opentelemetry_tracestate); + } opentelemetry_log->add(span); } diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index 23cde61a744..8c4086722c8 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -113,6 +113,7 @@ SRCS( MutationsInterpreter.cpp MySQL/InterpretersMySQLDDLQuery.cpp NullableUtils.cpp + OpenTelemetryLog.cpp OptimizeIfChains.cpp OptimizeIfWithConstantConditionVisitor.cpp PartLog.cpp diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 95f56b715b8..fb630010198 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -96,6 +96,7 @@ namespace ErrorCodes extern const int WRONG_PASSWORD; extern const int REQUIRED_PASSWORD; + extern const int BAD_REQUEST_PARAMETER; extern const int INVALID_SESSION_TIMEOUT; extern const int HTTP_LENGTH_REQUIRED; } @@ -279,9 +280,7 @@ void HTTPHandler::processQuery( } } - std::string query_id = params.get("query_id", ""); context.setUser(user, password, request.clientAddress()); - context.setCurrentQueryId(query_id); if (!quota_key.empty()) context.setQuotaKey(quota_key); @@ -311,6 +310,26 @@ void HTTPHandler::processQuery( session->release(); }); + std::string query_id = params.get("query_id", ""); + context.setCurrentQueryId(query_id); + + if (request.has("traceparent")) + { + std::string opentelemetry_traceparent = request.get("traceparent"); + std::string error; + if (!context.getClientInfo().setOpenTelemetryTraceparent( + opentelemetry_traceparent, error)) + { + throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, + "Failed to parse OpenTelemetry traceparent header '{}': {}", + opentelemetry_traceparent, error); + } + + context.getClientInfo().opentelemetry_tracestate = request.get("tracestate", ""); + + + } + /// The client can pass a HTTP header indicating supported compression method (gzip or deflate). String http_response_compression_methods = request.get("Accept-Encoding", ""); CompressionMethod http_response_compression_method = CompressionMethod::None; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index ab4ce820666..e83bbb02cad 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -845,13 +845,17 @@ void TCPHandler::receiveQuery() state.is_empty = false; readStringBinary(state.query_id, *in); - query_context->setCurrentQueryId(state.query_id); - /// Client info ClientInfo & client_info = query_context->getClientInfo(); if (client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) client_info.read(*in, client_revision); + // It is convenient to generate default OpenTelemetry trace id and default + // query id together. ClientInfo might contain upstream trace id, so we + // decide whether to use the default ids after we have received the ClientInfo. + // We also set up the parent span id while we're at it. + query_context->setCurrentQueryId(state.query_id); + /// For better support of old clients, that does not send ClientInfo. if (client_info.query_kind == ClientInfo::QueryKind::NO_QUERY) { diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index c2f7bfd18d2..d5e86c08a8b 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -67,6 +67,22 @@ namespace const CompressionMethod compression_method) : SourceWithProgress(sample_block), name(std::move(name_)) { + ReadWriteBufferFromHTTP::HTTPHeaderEntries header; + + // Propagate OpenTelemetry trace context, if any, downstream. + auto & client_info = context.getClientInfo(); + if (client_info.opentelemetry_trace_id) + { + header.emplace_back("traceparent", + client_info.getOpenTelemetryTraceparentForChild()); + + if (!client_info.opentelemetry_tracestate.empty()) + { + header.emplace_back("tracestate", + client_info.opentelemetry_tracestate); + } + } + read_buf = wrapReadBufferWithCompressionMethod( std::make_unique( uri, @@ -76,7 +92,7 @@ namespace context.getSettingsRef().max_http_get_redirects, Poco::Net::HTTPBasicCredentials{}, DBMS_DEFAULT_BUFFER_SIZE, - ReadWriteBufferFromHTTP::HTTPHeaderEntries{}, + header, context.getRemoteHostFilter()), compression_method); From f87db0fbe3a9a32fdb2041ecb822cf304edbcf5e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 28 Aug 2020 04:34:06 +0300 Subject: [PATCH 041/325] less crazy parsing --- src/Interpreters/ClientInfo.cpp | 113 +++++++------------------------- 1 file changed, 24 insertions(+), 89 deletions(-) diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp index 9e501ca5c11..0e8da221ca9 100644 --- a/src/Interpreters/ClientInfo.cpp +++ b/src/Interpreters/ClientInfo.cpp @@ -146,107 +146,42 @@ void ClientInfo::setInitialQuery() client_name = (DBMS_NAME " ") + client_name; } -template -bool readLowercaseHexDigits(const char *& begin, const char * end, T & dest_value, std::string & error) -{ - char * dest_begin = reinterpret_cast(&dest_value); - char * dest_end = dest_begin + sizeof(dest_value); - bool odd_character = true; - for (;;) - { - if (begin == end) - { - if (dest_begin == dest_end) - { - return true; - } - error = fmt::format("Not enough charaters in the input, got {}, need {} more", end - begin, dest_end - dest_begin); - return false; - } - - if (dest_begin == dest_end) - { - return true; - } - - int cur = 0; - if (*begin >= '0' && *begin <= '9') - { - cur = *begin - '0'; - } - else if (*begin >= 'a' && *begin <= 'f') - { - cur = 10 + *begin - 'a'; - } - else - { - error = fmt::format("Encountered '{}' which is not a lowercase hexadecimal digit", *begin); - return false; - } - - // Two characters per byte, little-endian. - if (odd_character) - { - *(dest_end - 1) = cur; - } - else - { - *(dest_end - 1) = *(dest_end - 1) << 8 | cur; - --dest_end; - } - - begin++; - odd_character = !odd_character; - } -} - bool ClientInfo::setOpenTelemetryTraceparent(const std::string & traceparent, std::string & error) { uint8_t version = -1; - __uint128_t trace_id = 0; + __uint64_t trace_id_high = 0; + __uint64_t trace_id_low = 0; uint64_t trace_parent = 0; uint8_t trace_flags = 0; - const char * begin = &traceparent[0]; - const char * end = begin + traceparent.length(); + int result = sscanf(&traceparent[0], + "%2" SCNx8 "-%16" SCNx64 "%16" SCNx64 "-%16" SCNx64 "-%2" SCNx8, + &version, &trace_id_high, &trace_id_low, &trace_parent, &trace_flags); -#define CHECK_CONDITION(condition, ...) \ - ((condition) || (error = fmt::format(__VA_ARGS__), false)) - -#define CHECK_DELIMITER \ - (begin >= end \ - ? (error = fmt::format( \ - "Expected '-' delimiter, got EOL at position {}", \ - begin - &traceparent[0]), \ - false) \ - : *begin != '-' \ - ? (error = fmt::format( \ - "Expected '-' delimiter, got '{}' at position {}", \ - *begin, begin - &traceparent[0]), \ - false) \ - : (++begin, true)) - - bool result = readLowercaseHexDigits(begin, end, version, error) - && CHECK_CONDITION(version == 0, "Expected version 00, got {}", version) - && CHECK_DELIMITER - && readLowercaseHexDigits(begin, end, trace_id, error) - && CHECK_DELIMITER - && readLowercaseHexDigits(begin, end, trace_parent, error) - && CHECK_DELIMITER - && readLowercaseHexDigits(begin, end, trace_flags, error) - && CHECK_CONDITION(begin == end, - "Expected end of string, got {} at position {}", *begin, end - begin); - -#undef CHECK -#undef CHECK_DELIMITER - - if (!result) + if (result == EOF) { + error = "Failed to parse traceparent header (EOF)"; return false; } - opentelemetry_trace_id = trace_id; + if (result != 5) + { + error = fmt::format("Failed to parse traceparent header" + "(could only read {} parts instead of the expected 5)", + result); + return false; + } + + if (version != 0) + { + error = fmt::format("Unexpected version {} of traceparent header:" + "expected 00", version); + return false; + } + + opentelemetry_trace_id = static_cast<__uint128_t>(trace_id_high) << 64 + | trace_id_low; opentelemetry_parent_span_id = trace_parent; opentelemetry_trace_flags = trace_flags; return true; From 063cc386a3508cb8ec857373bfe10ca83f2a4c58 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 28 Aug 2020 22:02:50 +0300 Subject: [PATCH 042/325] cosmetic fixes --- src/Interpreters/ClientInfo.cpp | 27 +++++++++++++++++++++------ src/Interpreters/ClientInfo.h | 2 +- src/Interpreters/Context.cpp | 4 ++-- src/Interpreters/executeQuery.cpp | 2 +- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp index 0e8da221ca9..f7ed8eafa46 100644 --- a/src/Interpreters/ClientInfo.cpp +++ b/src/Interpreters/ClientInfo.cpp @@ -70,7 +70,6 @@ void ClientInfo::write(WriteBuffer & out, const UInt64 server_protocol_revision) writeBinary(opentelemetry_parent_span_id, out); writeBinary(opentelemetry_tracestate, out); writeBinary(opentelemetry_trace_flags, out); - std::cerr << fmt::format("wrote {:x}, {}, {}\n", opentelemetry_trace_id, opentelemetry_span_id, opentelemetry_parent_span_id) << StackTrace().toString() << std::endl; } } @@ -155,28 +154,44 @@ bool ClientInfo::setOpenTelemetryTraceparent(const std::string & traceparent, uint64_t trace_parent = 0; uint8_t trace_flags = 0; + // Version 00, which is the only one we can parse, is fixed width. Use this + // fact for an additional sanity check. + const int expected_length = 2 + 1 + 32 + 1 + 16 + 1 + 2; + if (traceparent.length() != expected_length) + { + error = fmt::format("unexpected length {}, expected {}", + traceparent.length(), expected_length); + return false; + } + + // clang-tidy doesn't like sscanf: + // error: 'sscanf' used to convert a string to an unsigned integer value, + // but function will not report conversion errors; consider using 'strtoul' + // instead [cert-err34-c,-warnings-as-errors] + // There is no other ready solution, and hand-rolling a more complicated + // parser for an HTTP header in C++ sounds like RCE. + // NOLINTNEXTLINE(cert-err34-c) int result = sscanf(&traceparent[0], "%2" SCNx8 "-%16" SCNx64 "%16" SCNx64 "-%16" SCNx64 "-%2" SCNx8, &version, &trace_id_high, &trace_id_low, &trace_parent, &trace_flags); if (result == EOF) { - error = "Failed to parse traceparent header (EOF)"; + error = "EOF"; return false; } + // We read uint128 as two uint64, so 5 parts and not 4. if (result != 5) { - error = fmt::format("Failed to parse traceparent header" - "(could only read {} parts instead of the expected 5)", + error = fmt::format("could only read {} parts instead of the expected 5", result); return false; } if (version != 0) { - error = fmt::format("Unexpected version {} of traceparent header:" - "expected 00", version); + error = fmt::format("unexpected version {}, expected 00", version); return false; } diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index 413e1c42bf7..604a9771a52 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -58,7 +58,7 @@ public: String initial_user; String initial_query_id; Poco::Net::SocketAddress initial_address; - + // OpenTelemetry things __uint128_t opentelemetry_trace_id = 0; // Span ID is not strictly the client info, but convenient to keep here. diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index c5fd2d585e1..362e762df6d 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1092,7 +1092,7 @@ void Context::setCurrentQueryId(const String & query_id) random.words.a = thread_local_rng(); //-V656 random.words.b = thread_local_rng(); //-V656 - + fmt::print(stderr, "traceid {}, ==0 {}\n", client_info.opentelemetry_trace_id, client_info.opentelemetry_trace_id == 0); if (client_info.opentelemetry_trace_id == 0) { @@ -1120,7 +1120,7 @@ void Context::setCurrentQueryId(const String & query_id) QueryUUID(const char * bytes, Poco::UUID::Version version) : Poco::UUID(bytes, version) {} }; - + query_id_to_set = QueryUUID(random.bytes, Poco::UUID::UUID_RANDOM).toString(); } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 2a35bc205fa..05a0211adce 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -147,7 +147,7 @@ static void logQuery(const String & query, const Context & context, bool interna (current_user != "default" ? ", user: " + context.getClientInfo().current_user : ""), (!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()), joinLines(query)); - + LOG_TRACE(&Poco::Logger::get("executeQuery"), "OpenTelemetry trace id {:x}, span id {}, parent span id {}", context.getClientInfo().opentelemetry_trace_id, context.getClientInfo().opentelemetry_span_id, From 79e0b184ac16675322f25d1d840c50201a2ab822 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 28 Aug 2020 23:40:23 +0300 Subject: [PATCH 043/325] A test + reinterpretAsUUID --- src/Functions/reinterpretStringAs.cpp | 8 +++-- src/Storages/StorageURL.cpp | 2 +- .../0_stateless/01455_opentelemetry.reference | 3 ++ .../0_stateless/01455_opentelemetry.sh | 31 +++++++++++++++++++ tests/queries/shell_config.sh | 20 ++++++------ 5 files changed, 52 insertions(+), 12 deletions(-) create mode 100644 tests/queries/0_stateless/01455_opentelemetry.reference create mode 100755 tests/queries/0_stateless/01455_opentelemetry.sh diff --git a/src/Functions/reinterpretStringAs.cpp b/src/Functions/reinterpretStringAs.cpp index bb290b33b6d..34e53f32dd2 100644 --- a/src/Functions/reinterpretStringAs.cpp +++ b/src/Functions/reinterpretStringAs.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -66,7 +67,7 @@ public: size_t offset = 0; for (size_t i = 0; i < size; ++i) { - ToFieldType value = 0; + ToFieldType value{}; memcpy(&value, &data_from[offset], std::min(static_cast(sizeof(ToFieldType)), offsets_from[i] - offset - 1)); vec_res[i] = value; offset = offsets_from[i]; @@ -88,7 +89,7 @@ public: size_t copy_size = std::min(step, sizeof(ToFieldType)); for (size_t i = 0; i < size; ++i) { - ToFieldType value = 0; + ToFieldType value{}; memcpy(&value, &data_from[offset], copy_size); vec_res[i] = value; offset += step; @@ -118,6 +119,7 @@ struct NameReinterpretAsFloat32 { static constexpr auto name = "reinterpretA struct NameReinterpretAsFloat64 { static constexpr auto name = "reinterpretAsFloat64"; }; struct NameReinterpretAsDate { static constexpr auto name = "reinterpretAsDate"; }; struct NameReinterpretAsDateTime { static constexpr auto name = "reinterpretAsDateTime"; }; +struct NameReinterpretAsUUID { static constexpr auto name = "reinterpretAsUUID"; }; using FunctionReinterpretAsUInt8 = FunctionReinterpretStringAs; using FunctionReinterpretAsUInt16 = FunctionReinterpretStringAs; @@ -131,6 +133,7 @@ using FunctionReinterpretAsFloat32 = FunctionReinterpretStringAs; using FunctionReinterpretAsDate = FunctionReinterpretStringAs; using FunctionReinterpretAsDateTime = FunctionReinterpretStringAs; +using FunctionReinterpretAsUUID = FunctionReinterpretStringAs; void registerFunctionsReinterpretStringAs(FunctionFactory & factory) @@ -147,6 +150,7 @@ void registerFunctionsReinterpretStringAs(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); } } diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index d5e86c08a8b..cd1b5dd882a 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -70,7 +70,7 @@ namespace ReadWriteBufferFromHTTP::HTTPHeaderEntries header; // Propagate OpenTelemetry trace context, if any, downstream. - auto & client_info = context.getClientInfo(); + const auto & client_info = context.getClientInfo(); if (client_info.opentelemetry_trace_id) { header.emplace_back("traceparent", diff --git a/tests/queries/0_stateless/01455_opentelemetry.reference b/tests/queries/0_stateless/01455_opentelemetry.reference new file mode 100644 index 00000000000..b0484d7df0b --- /dev/null +++ b/tests/queries/0_stateless/01455_opentelemetry.reference @@ -0,0 +1,3 @@ +1 +4 +1 diff --git a/tests/queries/0_stateless/01455_opentelemetry.sh b/tests/queries/0_stateless/01455_opentelemetry.sh new file mode 100755 index 00000000000..f537377dfe6 --- /dev/null +++ b/tests/queries/0_stateless/01455_opentelemetry.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -ue + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +# Generate some random trace id so that the prevous runs of the test do not interfere. +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") + +${CLICKHOUSE_CURL} --header "traceparent: 00-$trace_id-0000000000000010-01" --header "tracestate: some custom state" "http://localhost:8123/" --get --data-urlencode "query=select 1 from remote('127.0.0.2', system, one)" + +${CLICKHOUSE_CLIENT} -q "system flush logs" + +# Check that the HTTP traceparent was read, and then passed to `remote` instance. +# We expect 4 queries, because there are two DESC TABLE queries for the shard. +# This is bug-ish, see https://github.com/ClickHouse/ClickHouse/issues/14228 +${CLICKHOUSE_CLIENT} -q "select count(*) from system.opentelemetry_log where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id')))" + +# Check that the tracestate header was read and passed. Must have +# exactly the same value for all "query" spans in this trace. +${CLICKHOUSE_CLIENT} -q " + select count(distinct attribute.values) + from system.opentelemetry_log + array join attribute.names, attribute.values + where + trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + and operation_name = 'query' + and attribute.names = 'tracestate' +" + + diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index 8c66d79b5b1..cd4fd418c81 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -1,16 +1,18 @@ +#!/usr/bin/env bash + export CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE:="test"} export CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL:="warning"} -[ -n "$CLICKHOUSE_CONFIG_CLIENT" ] && CLICKHOUSE_CLIENT_OPT0+=" --config-file=${CLICKHOUSE_CONFIG_CLIENT} " -[ -n "${CLICKHOUSE_HOST}" ] && CLICKHOUSE_CLIENT_OPT0+=" --host=${CLICKHOUSE_HOST} " -[ -n "${CLICKHOUSE_PORT_TCP}" ] && CLICKHOUSE_CLIENT_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} " -[ -n "${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}" ] && CLICKHOUSE_CLIENT_OPT0+=" --send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL} " -[ -n "${CLICKHOUSE_DATABASE}" ] && CLICKHOUSE_CLIENT_OPT0+=" --database=${CLICKHOUSE_DATABASE} " +[ -v CLICKHOUSE_CONFIG_CLIENT ] && CLICKHOUSE_CLIENT_OPT0+=" --config-file=${CLICKHOUSE_CONFIG_CLIENT} " +[ -v CLICKHOUSE_HOST ] && CLICKHOUSE_CLIENT_OPT0+=" --host=${CLICKHOUSE_HOST} " +[ -v CLICKHOUSE_PORT_TCP ] && CLICKHOUSE_CLIENT_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} " +[ -v CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL ] && CLICKHOUSE_CLIENT_OPT0+=" --send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL} " +[ -v CLICKHOUSE_DATABASE ] && CLICKHOUSE_CLIENT_OPT0+=" --database=${CLICKHOUSE_DATABASE} " export CLICKHOUSE_BINARY=${CLICKHOUSE_BINARY:="clickhouse"} [ -x "$CLICKHOUSE_BINARY-client" ] && CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY-client} [ -x "$CLICKHOUSE_BINARY" ] && CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY client} export CLICKHOUSE_CLIENT_BINARY=${CLICKHOUSE_CLIENT_BINARY:=$CLICKHOUSE_BINARY-client} -export CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT:="$CLICKHOUSE_CLIENT_BINARY ${CLICKHOUSE_CLIENT_OPT0} ${CLICKHOUSE_CLIENT_OPT}"} +export CLICKHOUSE_CLIENT=${CLICKHOUSE_CLIENT:="$CLICKHOUSE_CLIENT_BINARY ${CLICKHOUSE_CLIENT_OPT0:-} ${CLICKHOUSE_CLIENT_OPT:-}"} [ -x "${CLICKHOUSE_BINARY}-local" ] && CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY}-local"} [ -x "${CLICKHOUSE_BINARY}" ] && CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY} local"} export CLICKHOUSE_LOCAL=${CLICKHOUSE_LOCAL:="${CLICKHOUSE_BINARY}-local"} @@ -42,7 +44,7 @@ export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:="8443"} export CLICKHOUSE_PORT_HTTP_PROTO=${CLICKHOUSE_PORT_HTTP_PROTO:="http"} # Add database to url params -if [ -n "${CLICKHOUSE_URL_PARAMS}" ] +if [ -v CLICKHOUSE_URL_PARAMS ] then export CLICKHOUSE_URL_PARAMS="${CLICKHOUSE_URL_PARAMS}&database=${CLICKHOUSE_DATABASE}" else @@ -53,7 +55,7 @@ export CLICKHOUSE_URL=${CLICKHOUSE_URL:="${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICK export CLICKHOUSE_URL_HTTPS=${CLICKHOUSE_URL_HTTPS:="https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTPS}/"} # Add url params to url -if [ -n "${CLICKHOUSE_URL_PARAMS}" ] +if [ -v CLICKHOUSE_URL_PARAMS ] then export CLICKHOUSE_URL="${CLICKHOUSE_URL}?${CLICKHOUSE_URL_PARAMS}" export CLICKHOUSE_URL_HTTPS="${CLICKHOUSE_URL_HTTPS}?${CLICKHOUSE_URL_PARAMS}" @@ -65,7 +67,7 @@ export CLICKHOUSE_URL_INTERSERVER=${CLICKHOUSE_URL_INTERSERVER:="${CLICKHOUSE_PO export CLICKHOUSE_CURL_COMMAND=${CLICKHOUSE_CURL_COMMAND:="curl"} export CLICKHOUSE_CURL_TIMEOUT=${CLICKHOUSE_CURL_TIMEOUT:="10"} -export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} -q --max-time ${CLICKHOUSE_CURL_TIMEOUT}"} +export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} -q -s --max-time ${CLICKHOUSE_CURL_TIMEOUT}"} export CLICKHOUSE_TMP=${CLICKHOUSE_TMP:="."} mkdir -p ${CLICKHOUSE_TMP} From c02f9e1dd1ca825a6dba653b2fb6cf3722ea195d Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Sat, 29 Aug 2020 01:48:48 +0300 Subject: [PATCH 044/325] remove some debug output to pass the tests --- src/Interpreters/Context.cpp | 2 -- src/Interpreters/executeQuery.cpp | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 362e762df6d..4bc66173a9c 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1093,7 +1093,6 @@ void Context::setCurrentQueryId(const String & query_id) random.words.a = thread_local_rng(); //-V656 random.words.b = thread_local_rng(); //-V656 - fmt::print(stderr, "traceid {}, ==0 {}\n", client_info.opentelemetry_trace_id, client_info.opentelemetry_trace_id == 0); if (client_info.opentelemetry_trace_id == 0) { // If trace_id is not initialized, it means that this is an initial query @@ -1109,7 +1108,6 @@ void Context::setCurrentQueryId(const String & query_id) client_info.opentelemetry_parent_span_id = client_info.opentelemetry_span_id; client_info.opentelemetry_span_id = thread_local_rng(); } - fmt::print(stderr, "traceid {}, ==0 {}\n{}\n", client_info.opentelemetry_trace_id, client_info.opentelemetry_trace_id == 0, StackTrace().toString()); String query_id_to_set = query_id; if (query_id_to_set.empty()) /// If the user did not submit his query_id, then we generate it ourselves. diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 05a0211adce..1da12d993ea 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -152,8 +152,6 @@ static void logQuery(const String & query, const Context & context, bool interna "OpenTelemetry trace id {:x}, span id {}, parent span id {}", context.getClientInfo().opentelemetry_trace_id, context.getClientInfo().opentelemetry_span_id, context.getClientInfo().opentelemetry_parent_span_id); - - std::cerr << StackTrace().toString() << std::endl; } } From fa8eebed780de4b79c000613341da25d4efe6f4a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Sat, 29 Aug 2020 02:25:30 +0300 Subject: [PATCH 045/325] more tests + clickhouse-client options --- programs/client/Client.cpp | 21 ++++++++ .../0_stateless/01455_opentelemetry.sh | 31 ----------- ...01455_opentelemetry_distributed.reference} | 3 ++ .../01455_opentelemetry_distributed.sh | 52 +++++++++++++++++++ 4 files changed, 76 insertions(+), 31 deletions(-) delete mode 100755 tests/queries/0_stateless/01455_opentelemetry.sh rename tests/queries/0_stateless/{01455_opentelemetry.reference => 01455_opentelemetry_distributed.reference} (50%) create mode 100755 tests/queries/0_stateless/01455_opentelemetry_distributed.sh diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index e3a9b68dc47..fc2adff337e 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -2180,6 +2180,8 @@ public: ("log-level", po::value(), "client log level") ("server_logs_file", po::value(), "put server logs into specified file") ("query-fuzzer-runs", po::value()->default_value(0), "query fuzzer runs") + ("opentelemetry-traceparent", po::value(), "OpenTelemetry traceparent header as described by W3C Trace Context recommendation") + ("opentelemetry-tracestate", po::value(), "OpenTelemetry tracestate header as described by W3C Trace Context recommendation") ; Settings cmd_settings; @@ -2348,6 +2350,25 @@ public: ignore_error = true; } + if (options.count("opentelemetry-traceparent")) + { + std::string traceparent = options["opentelemetry-traceparent"].as(); + std::string error; + if (!context.getClientInfo().setOpenTelemetryTraceparent( + traceparent, error)) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Cannot parse OpenTelemetry traceparent '{}': {}", + traceparent, error); + } + } + + if (options.count("opentelemetry-tracestate")) + { + context.getClientInfo().opentelemetry_tracestate = + options["opentelemetry-tracestate"].as(); + } + argsToConfig(common_arguments, config(), 100); clearPasswordFromCommandLine(argc, argv); diff --git a/tests/queries/0_stateless/01455_opentelemetry.sh b/tests/queries/0_stateless/01455_opentelemetry.sh deleted file mode 100755 index f537377dfe6..00000000000 --- a/tests/queries/0_stateless/01455_opentelemetry.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -set -ue - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. "$CURDIR"/../shell_config.sh - -# Generate some random trace id so that the prevous runs of the test do not interfere. -trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") - -${CLICKHOUSE_CURL} --header "traceparent: 00-$trace_id-0000000000000010-01" --header "tracestate: some custom state" "http://localhost:8123/" --get --data-urlencode "query=select 1 from remote('127.0.0.2', system, one)" - -${CLICKHOUSE_CLIENT} -q "system flush logs" - -# Check that the HTTP traceparent was read, and then passed to `remote` instance. -# We expect 4 queries, because there are two DESC TABLE queries for the shard. -# This is bug-ish, see https://github.com/ClickHouse/ClickHouse/issues/14228 -${CLICKHOUSE_CLIENT} -q "select count(*) from system.opentelemetry_log where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id')))" - -# Check that the tracestate header was read and passed. Must have -# exactly the same value for all "query" spans in this trace. -${CLICKHOUSE_CLIENT} -q " - select count(distinct attribute.values) - from system.opentelemetry_log - array join attribute.names, attribute.values - where - trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) - and operation_name = 'query' - and attribute.names = 'tracestate' -" - - diff --git a/tests/queries/0_stateless/01455_opentelemetry.reference b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference similarity index 50% rename from tests/queries/0_stateless/01455_opentelemetry.reference rename to tests/queries/0_stateless/01455_opentelemetry_distributed.reference index b0484d7df0b..5993b628ad4 100644 --- a/tests/queries/0_stateless/01455_opentelemetry.reference +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference @@ -1,3 +1,6 @@ 1 4 1 +1 +2 +1 diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh new file mode 100755 index 00000000000..df5c194b2be --- /dev/null +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -ue + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +function check_log +{ +${CLICKHOUSE_CLIENT} -nq " +system flush logs; + +-- Check the number of spans with given trace id, to verify it was propagated. +select count(*) + from system.opentelemetry_log + where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + and operation_name = 'query' + ; + +-- Check that the tracestate header was propagated. It must have exactly the +-- same non-empty value for all 'query' spans in this trace. +select count(distinct value) + from system.opentelemetry_log + array join attribute.names as name, attribute.values as value + where + trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + and operation_name = 'query' + and name = 'tracestate' + and length(value) > 0 + ; +" +} + +# Generate some random trace id so that the prevous runs of the test do not interfere. +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") + +# Check that the HTTP traceparent is read, and then passed through `remote` table function. +# We expect 4 queries, because there are two DESC TABLE queries for the shard. +# This is bug-ish, see https://github.com/ClickHouse/ClickHouse/issues/14228 +${CLICKHOUSE_CURL} --header "traceparent: 00-$trace_id-0000000000000010-01" --header "tracestate: some custom state" "http://localhost:8123/" --get --data-urlencode "query=select 1 from remote('127.0.0.2', system, one)" + +check_log + +# With another trace id, check that clickhouse-client accepts traceparent, and +# that it is passed through URL table function. We expect two query spans, one +# for the initial query, and one for the HTTP query. +trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") + +${CLICKHOUSE_CLIENT} --opentelemetry-traceparent "00-$trace_id-0000000000000020-02" --opentelemetry-tracestate "another custom state" --query " + select * from url('http://127.0.0.2:8123/?query=select%201', CSV, 'a int') +" + +check_log From d0a9926e7dba352adfe6e89c3fa8a3bd8f940377 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 8 Sep 2020 16:19:27 +0300 Subject: [PATCH 046/325] fixes and some docs --- docs/en/operations/opentelemetry-draft.md | 61 +++++++++++++++++++ programs/server/config.xml | 15 +++++ src/Core/Settings.h | 1 + src/Interpreters/ClientInfo.cpp | 49 ++++++++++----- src/Interpreters/ClientInfo.h | 2 +- src/Interpreters/Context.cpp | 26 +++++--- src/Interpreters/OpenTelemetryLog.cpp | 11 ++-- src/Interpreters/OpenTelemetryLog.h | 5 +- src/Interpreters/executeQuery.cpp | 54 +++++++++------- src/Server/TCPHandler.cpp | 26 +++++--- .../01455_opentelemetry_distributed.reference | 4 ++ .../01455_opentelemetry_distributed.sh | 41 +++++++++++-- 12 files changed, 226 insertions(+), 69 deletions(-) create mode 100644 docs/en/operations/opentelemetry-draft.md diff --git a/docs/en/operations/opentelemetry-draft.md b/docs/en/operations/opentelemetry-draft.md new file mode 100644 index 00000000000..3363b37b6d6 --- /dev/null +++ b/docs/en/operations/opentelemetry-draft.md @@ -0,0 +1,61 @@ +# [draft] OpenTelemetry support + +[OpenTelemetry](https://opentelemetry.io/) is an open standard for collecting +traces and metrics from distributed application. ClickHouse has some support +for OpenTelemetry. + + +## Supplying Trace Context to ClickHouse + +ClickHouse accepts trace context HTTP headers, as described by +the [W3C recommendation](https://www.w3.org/TR/trace-context/). +It also accepts trace context over native protocol that is used for +communication between ClickHouse servers or between the client and server. +For manual testing, trace context headers conforming to the Trace Context +recommendation can be supplied to `clickhouse-client` using +`--opentelemetry-traceparent` and `--opentelemetry-tracestate` flags. + +If no parent trace context is supplied, ClickHouse can start a new trace, with +probability controlled by the `opentelemetry_start_trace_probability` setting. + + +## Propagating the Trace Context + +The trace context is propagated to downstream services in the following cases: + +* Queries to remote ClickHouse servers, such as when using `Distributed` table + engine. + +* `URL` table function. Trace context information is sent in HTTP headers. + + +## Tracing the ClickHouse Itself + +ClickHouse creates _trace spans_ for each query and some of the query execution +stages, such as query planning or distributed queries. + +To be useful, the tracing information has to be exported to a monitoring system +that supports OpenTelemetry, such as Jaeger or Prometheus. ClickHouse avoids +a dependency on a particular monitoring system, instead only +providing the tracing data conforming to the standard. A natural way to do so +in an SQL RDBMS is a system table. OpenTelemetry trace span information +[required by the standard](https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/overview.md#span) +is stored in the system table called `system.opentelemetry_log`. + +The table must be enabled in the server configuration, see the `opentelemetry_log` +element in the default config file `config.xml`. It is enabled by default. + +The table has the following columns: + +- `trace_id` +- `span_id` +- `parent_span_id` +- `operation_name` +- `start_time` +- `finish_time` +- `finish_date` +- `attribute.name` +- `attribute.values` + +The tags or attributes are saved as two parallel arrays, containing the keys +and values. Use `ARRAY JOIN` to work with them. diff --git a/programs/server/config.xml b/programs/server/config.xml index 7e88150b95f..913f43fc435 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -567,6 +567,21 @@ OpenTelemetry log contains OpenTelemetry trace spans. --> + + + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time, trace_id) + system opentelemetry_log
7500 diff --git a/src/Core/Settings.h b/src/Core/Settings.h index f7438b356d5..bb3baae6ba4 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -218,6 +218,7 @@ class IColumn; M(UInt64, query_profiler_cpu_time_period_ns, 1000000000, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \ M(String, metrics_perf_events_list, "", "Comma separated list of perf metrics that will be measured throughout queries' execution. Empty means all events. See PerfEventInfo in sources for the available events.", 0) \ + M(Float, opentelemetry_start_trace_probability, 0., "Probability to start an OpenTelemetry trace for an incoming query.", 0) \ \ \ /** Limits during query execution are part of the settings. \ diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp index f7ed8eafa46..da9f14ec879 100644 --- a/src/Interpreters/ClientInfo.cpp +++ b/src/Interpreters/ClientInfo.cpp @@ -65,11 +65,19 @@ void ClientInfo::write(WriteBuffer & out, const UInt64 server_protocol_revision) { // No point writing these numbers with variable length, because they // are random and will probably require the full length anyway. - writeBinary(opentelemetry_trace_id, out); - writeBinary(opentelemetry_span_id, out); - writeBinary(opentelemetry_parent_span_id, out); - writeBinary(opentelemetry_tracestate, out); - writeBinary(opentelemetry_trace_flags, out); + if (opentelemetry_trace_id) + { + writeBinary(uint8_t(1), out); + writeBinary(opentelemetry_trace_id, out); + writeBinary(opentelemetry_span_id, out); + writeBinary(opentelemetry_parent_span_id, out); + writeBinary(opentelemetry_tracestate, out); + writeBinary(opentelemetry_trace_flags, out); + } + else + { + writeBinary(uint8_t(0), out); + } } } @@ -125,15 +133,24 @@ void ClientInfo::read(ReadBuffer & in, const UInt64 client_protocol_revision) client_version_patch = client_revision; } + // TODO what does it even mean to read this structure over HTTP? I thought + // this was for native protocol? See interface == Interface::HTTP. if (client_protocol_revision >= DBMS_MIN_REVISION_WITH_OPENTELEMETRY) { - readBinary(opentelemetry_trace_id, in); - readBinary(opentelemetry_span_id, in); - readBinary(opentelemetry_parent_span_id, in); - readBinary(opentelemetry_tracestate, in); - readBinary(opentelemetry_trace_flags, in); + uint8_t have_trace_id = 0; + readBinary(have_trace_id, in); + if (have_trace_id) + { + readBinary(opentelemetry_trace_id, in); + readBinary(opentelemetry_span_id, in); + readBinary(opentelemetry_parent_span_id, in); + readBinary(opentelemetry_tracestate, in); + readBinary(opentelemetry_trace_flags, in); - std::cerr << fmt::format("read {:x}, {}, {}\n", opentelemetry_trace_id, opentelemetry_span_id, opentelemetry_parent_span_id) << StackTrace().toString() << std::endl; + fmt::print(stderr, "read {:x}, {}, {} at\n{}\n", + opentelemetry_trace_id, opentelemetry_span_id, + opentelemetry_parent_span_id, StackTrace().toString()); + } } } @@ -149,8 +166,8 @@ bool ClientInfo::setOpenTelemetryTraceparent(const std::string & traceparent, std::string & error) { uint8_t version = -1; - __uint64_t trace_id_high = 0; - __uint64_t trace_id_low = 0; + uint64_t trace_id_high = 0; + uint64_t trace_id_low = 0; uint64_t trace_parent = 0; uint8_t trace_flags = 0; @@ -205,11 +222,11 @@ bool ClientInfo::setOpenTelemetryTraceparent(const std::string & traceparent, std::string ClientInfo::getOpenTelemetryTraceparentForChild() const { - // This span is a parent for its children (so deep...), so we specify - // this span_id as a parent id. + // This span is a parent for its children, so we specify this span_id as a + // parent id. return fmt::format("00-{:032x}-{:016x}-{:02x}", opentelemetry_trace_id, opentelemetry_span_id, - // This cast is because fmt is being weird and complaining that + // This cast is needed because fmt is being weird and complaining that // "mixing character types is not allowed". static_cast(opentelemetry_trace_flags)); } diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index 604a9771a52..f57d1853b76 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -69,7 +69,7 @@ public: // the incoming tracestate header, we just pass it downstream. // https://www.w3.org/TR/trace-context/ String opentelemetry_tracestate; - UInt8 opentelemetry_trace_flags; + UInt8 opentelemetry_trace_flags = 0; /// All below are parameters related to initial query. diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 4bc66173a9c..e12747a7f95 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1093,18 +1093,28 @@ void Context::setCurrentQueryId(const String & query_id) random.words.a = thread_local_rng(); //-V656 random.words.b = thread_local_rng(); //-V656 - if (client_info.opentelemetry_trace_id == 0) + if (client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY + && client_info.opentelemetry_trace_id == 0) { - // If trace_id is not initialized, it means that this is an initial query - // without any parent OpenTelemetry trace. Use the randomly generated - // default query id as the new trace id. - client_info.opentelemetry_trace_id = random.uuid; - client_info.opentelemetry_parent_span_id = 0; - client_info.opentelemetry_span_id = thread_local_rng(); + // If this is an initial query without any parent OpenTelemetry trace, we + // might start the trace ourselves, with some configurable probability. + std::bernoulli_distribution should_start_trace{ + settings.opentelemetry_start_trace_probability}; + + if (should_start_trace(thread_local_rng)) + { + // Use the randomly generated default query id as the new trace id. + client_info.opentelemetry_trace_id = random.uuid; + client_info.opentelemetry_parent_span_id = 0; + client_info.opentelemetry_span_id = thread_local_rng(); + // Mark this trace as sampled in the flags. + client_info.opentelemetry_trace_flags = 1; + } } else { - // The incoming span id becomes our parent span id. + // The incoming request has an OpenTelemtry trace context. Its span id + // becomes our parent span id. client_info.opentelemetry_parent_span_id = client_info.opentelemetry_span_id; client_info.opentelemetry_span_id = thread_local_rng(); } diff --git a/src/Interpreters/OpenTelemetryLog.cpp b/src/Interpreters/OpenTelemetryLog.cpp index f8d7d684478..161af01107f 100644 --- a/src/Interpreters/OpenTelemetryLog.cpp +++ b/src/Interpreters/OpenTelemetryLog.cpp @@ -13,16 +13,13 @@ namespace DB Block OpenTelemetrySpanLogElement::createBlock() { return { - // event_date is the date part of event_time, used for indexing. - {std::make_shared(), "event_date"}, - // event_time is the span start time, named so to be compatible with - // the standard ClickHouse system log column names. - {std::make_shared(), "event_time"}, {std::make_shared(), "trace_id"}, {std::make_shared(), "span_id"}, {std::make_shared(), "parent_span_id"}, {std::make_shared(), "operation_name"}, + {std::make_shared(), "start_time"}, {std::make_shared(), "finish_time"}, + {std::make_shared(), "finish_date"}, {std::make_shared(std::make_shared()), "attribute.names"}, {std::make_shared(std::make_shared()), @@ -34,13 +31,13 @@ void OpenTelemetrySpanLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; - columns[i++]->insert(DateLUT::instance().toDayNum(start_time)); - columns[i++]->insert(start_time); columns[i++]->insert(UInt128(Int128(trace_id))); columns[i++]->insert(span_id); columns[i++]->insert(parent_span_id); columns[i++]->insert(operation_name); + columns[i++]->insert(start_time); columns[i++]->insert(finish_time); + columns[i++]->insert(DateLUT::instance().toDayNum(finish_time)); columns[i++]->insert(attribute_names); columns[i++]->insert(attribute_values); } diff --git a/src/Interpreters/OpenTelemetryLog.h b/src/Interpreters/OpenTelemetryLog.h index 73ad5382c95..e2906a99a02 100644 --- a/src/Interpreters/OpenTelemetryLog.h +++ b/src/Interpreters/OpenTelemetryLog.h @@ -23,8 +23,9 @@ struct OpenTelemetrySpan UInt64 span_id; UInt64 parent_span_id; std::string operation_name; - time_t start_time{}; - time_t finish_time{}; + time_t start_time; + time_t finish_time; + UInt64 duration_ns; Array attribute_names; Array attribute_values; // I don't understand how Links work, namely, which direction should they diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 1da12d993ea..141a9ade57f 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -138,20 +138,26 @@ static void logQuery(const String & query, const Context & context, bool interna } else { - const auto & current_query_id = context.getClientInfo().current_query_id; - const auto & initial_query_id = context.getClientInfo().initial_query_id; - const auto & current_user = context.getClientInfo().current_user; + const auto & client_info = context.getClientInfo(); + + const auto & current_query_id = client_info.current_query_id; + const auto & initial_query_id = client_info.initial_query_id; + const auto & current_user = client_info.current_user; LOG_DEBUG(&Poco::Logger::get("executeQuery"), "(from {}{}{}) {}", - context.getClientInfo().current_address.toString(), - (current_user != "default" ? ", user: " + context.getClientInfo().current_user : ""), + client_info.current_address.toString(), + (current_user != "default" ? ", user: " + current_user : ""), (!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()), joinLines(query)); - LOG_TRACE(&Poco::Logger::get("executeQuery"), - "OpenTelemetry trace id {:x}, span id {}, parent span id {}", - context.getClientInfo().opentelemetry_trace_id, context.getClientInfo().opentelemetry_span_id, - context.getClientInfo().opentelemetry_parent_span_id); + if (client_info.opentelemetry_trace_id) + { + LOG_TRACE(&Poco::Logger::get("executeQuery"), + "OpenTelemetry trace id {:x}, span id {}, parent span id {}", + client_info.opentelemetry_trace_id, + client_info.opentelemetry_span_id, + client_info.opentelemetry_parent_span_id); + } } } @@ -222,7 +228,9 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c if (auto query_log = context.getQueryLog()) query_log->add(elem); - if (auto opentelemetry_log = context.getOpenTelemetryLog()) + if (auto opentelemetry_log = context.getOpenTelemetryLog(); + context.getClientInfo().opentelemetry_trace_id + && opentelemetry_log) { OpenTelemetrySpanLogElement span; span.trace_id = context.getClientInfo().opentelemetry_trace_id; @@ -231,20 +239,21 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c span.operation_name = "query"; span.start_time = current_time; span.finish_time = current_time; + span.duration_ns = 0; // keep values synchonized to type enum in QueryLogElement::createBlock - span.attribute_names.push_back("status"); + span.attribute_names.push_back("clickhouse.query_status"); span.attribute_values.push_back("ExceptionBeforeStart"); - span.attribute_names.push_back("query"); + span.attribute_names.push_back("db.statement"); span.attribute_values.push_back(elem.query); - span.attribute_names.push_back("query_id"); + span.attribute_names.push_back("clickhouse.query_id"); span.attribute_values.push_back(elem.client_info.current_query_id); if (!context.getClientInfo().opentelemetry_tracestate.empty()) { - span.attribute_names.push_back("tracestate"); + span.attribute_names.push_back("clickhouse.tracestate"); span.attribute_values.push_back( context.getClientInfo().opentelemetry_tracestate); } @@ -285,7 +294,7 @@ static std::tuple executeQueryImpl( bool has_query_tail, ReadBuffer * istr) { - time_t current_time = time(nullptr); + const time_t current_time = time(nullptr); /// If we already executing query and it requires to execute internal query, than /// don't replace thread context with given (it can be temporary). Otherwise, attach context to thread. @@ -621,7 +630,9 @@ static std::tuple executeQueryImpl( query_log->add(elem); } - if (auto opentelemetry_log = context.getOpenTelemetryLog()) + if (auto opentelemetry_log = context.getOpenTelemetryLog(); + context.getClientInfo().opentelemetry_trace_id + && opentelemetry_log) { OpenTelemetrySpanLogElement span; span.trace_id = context.getClientInfo().opentelemetry_trace_id; @@ -629,20 +640,21 @@ static std::tuple executeQueryImpl( span.parent_span_id = context.getClientInfo().opentelemetry_parent_span_id; span.operation_name = "query"; span.start_time = elem.query_start_time; - span.finish_time = time(nullptr); // current time + span.finish_time = elem.event_time; + span.duration_ns = elapsed_seconds * 1000000000; // keep values synchonized to type enum in QueryLogElement::createBlock - span.attribute_names.push_back("status"); + span.attribute_names.push_back("clickhouse.query_status"); span.attribute_values.push_back("QueryFinish"); - span.attribute_names.push_back("query"); + span.attribute_names.push_back("db.statement"); span.attribute_values.push_back(elem.query); - span.attribute_names.push_back("query_id"); + span.attribute_names.push_back("clickhouse.query_id"); span.attribute_values.push_back(elem.client_info.current_query_id); if (!context.getClientInfo().opentelemetry_tracestate.empty()) { - span.attribute_names.push_back("tracestate"); + span.attribute_names.push_back("clickhouse.tracestate"); span.attribute_values.push_back( context.getClientInfo().opentelemetry_tracestate); } diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index e83bbb02cad..49f737c9550 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -850,12 +850,6 @@ void TCPHandler::receiveQuery() if (client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) client_info.read(*in, client_revision); - // It is convenient to generate default OpenTelemetry trace id and default - // query id together. ClientInfo might contain upstream trace id, so we - // decide whether to use the default ids after we have received the ClientInfo. - // We also set up the parent span id while we're at it. - query_context->setCurrentQueryId(state.query_id); - /// For better support of old clients, that does not send ClientInfo. if (client_info.query_kind == ClientInfo::QueryKind::NO_QUERY) { @@ -884,8 +878,11 @@ void TCPHandler::receiveQuery() /// Per query settings are also passed via TCP. /// We need to check them before applying due to they can violate the settings constraints. - auto settings_format = (client_revision >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) ? SettingsWriteFormat::STRINGS_WITH_FLAGS - : SettingsWriteFormat::BINARY; + auto settings_format = + (client_revision >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) + ? SettingsWriteFormat::STRINGS_WITH_FLAGS + : SettingsWriteFormat::BINARY; + Settings passed_settings; passed_settings.read(*in, settings_format); auto settings_changes = passed_settings.changes(); @@ -900,12 +897,23 @@ void TCPHandler::receiveQuery() query_context->clampToSettingsConstraints(settings_changes); } query_context->applySettingsChanges(settings_changes); - const Settings & settings = query_context->getSettingsRef(); + + // Use the received query id, or generate a random default. It is convenient + // to also generate the default OpenTelemetry trace id at the same time, and + // and and set the trace parent. + // Why is this done here and not earlier: + // 1) ClientInfo might contain upstream trace id, so we decide whether to use + // the default ids after we have received the ClientInfo. + // 2) There is the opentelemetry_start_trace_probability setting that + // controls when we start a new trace. It can be changed via Native protocol, + // so we have to apply the changes first. + query_context->setCurrentQueryId(state.query_id); /// Sync timeouts on client and server during current query to avoid dangling queries on server /// NOTE: We use settings.send_timeout for the receive timeout and vice versa (change arguments ordering in TimeoutSetter), /// because settings.send_timeout is client-side setting which has opposite meaning on the server side. /// NOTE: these settings are applied only for current connection (not for distributed tables' connections) + const Settings & settings = query_context->getSettingsRef(); state.timeout_setter = std::make_unique(socket(), settings.receive_timeout, settings.send_timeout); readVarUInt(stage, *in); diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference index 5993b628ad4..176626befa7 100644 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference @@ -1,6 +1,10 @@ +===http=== 1 4 1 +===native=== 1 2 1 +===sampled=== +1 1 diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh index df5c194b2be..1cb65113179 100755 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh @@ -24,29 +24,60 @@ select count(distinct value) where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) and operation_name = 'query' - and name = 'tracestate' + and name = 'clickhouse.tracestate' and length(value) > 0 ; " } # Generate some random trace id so that the prevous runs of the test do not interfere. +echo "===http===" trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") # Check that the HTTP traceparent is read, and then passed through `remote` table function. # We expect 4 queries, because there are two DESC TABLE queries for the shard. # This is bug-ish, see https://github.com/ClickHouse/ClickHouse/issues/14228 -${CLICKHOUSE_CURL} --header "traceparent: 00-$trace_id-0000000000000010-01" --header "tracestate: some custom state" "http://localhost:8123/" --get --data-urlencode "query=select 1 from remote('127.0.0.2', system, one)" +${CLICKHOUSE_CURL} \ + --header "traceparent: 00-$trace_id-0000000000000010-01" \ + --header "tracestate: some custom state" "http://localhost:8123/" \ + --get \ + --data-urlencode "query=select 1 from remote('127.0.0.2', system, one)" check_log # With another trace id, check that clickhouse-client accepts traceparent, and # that it is passed through URL table function. We expect two query spans, one # for the initial query, and one for the HTTP query. +echo "===native===" trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") -${CLICKHOUSE_CLIENT} --opentelemetry-traceparent "00-$trace_id-0000000000000020-02" --opentelemetry-tracestate "another custom state" --query " - select * from url('http://127.0.0.2:8123/?query=select%201', CSV, 'a int') -" +${CLICKHOUSE_CLIENT} \ + --opentelemetry-traceparent "00-$trace_id-0000000000000020-02" \ + --opentelemetry-tracestate "another custom state" \ + --query "select * from url('http://127.0.0.2:8123/?query=select%201', CSV, 'a int')" check_log + +# Test sampled tracing. The traces should be started with the specified probability, +# only for initial queries. +echo "===sampled===" +query_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") + +for _ in {1..200} +do + ${CLICKHOUSE_CLIENT} \ + --opentelemetry_start_trace_probability=0.1 \ + --query_id "$query_id" \ + --query "select 1 from remote('127.0.0.2', system, one) format Null" +done + +${CLICKHOUSE_CLIENT} -q " + with count(*) as c + -- expect 200 * 0.1 = 20 sampled events on average + select c > 10, c < 30 + from system.opentelemetry_log + array join attribute.names as name, attribute.values as value + where name = 'clickhouse.query_id' + and value = '$query_id' + ; +" From f0c5459807baa594c5138a26a0fdd6b3714f6c9c Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 9 Sep 2020 14:05:49 +0300 Subject: [PATCH 047/325] tests --- src/Core/BaseSettings.h | 22 +++++++++++++------ .../queries/0_stateless/00396_uuid.reference | 1 + tests/queries/0_stateless/00396_uuid.sql | 6 +++++ .../01455_opentelemetry_distributed.reference | 2 +- .../01455_opentelemetry_distributed.sh | 2 +- 5 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h index 7de87b345c1..b193fdd4c93 100644 --- a/src/Core/BaseSettings.h +++ b/src/Core/BaseSettings.h @@ -390,13 +390,21 @@ String BaseSettings::valueToStringUtil(const std::string_view & name, c template Field BaseSettings::stringToValueUtil(const std::string_view & name, const String & str) { - const auto & accessor = Traits::Accessor::instance(); - if (size_t index = accessor.find(name); index != static_cast(-1)) - return accessor.stringToValueUtil(index, str); - if constexpr (Traits::allow_custom_settings) - return Field::restoreFromDump(str); - else - BaseSettingsHelpers::throwSettingNotFound(name); + try + { + const auto & accessor = Traits::Accessor::instance(); + if (size_t index = accessor.find(name); index != static_cast(-1)) + return accessor.stringToValueUtil(index, str); + if constexpr (Traits::allow_custom_settings) + return Field::restoreFromDump(str); + else + BaseSettingsHelpers::throwSettingNotFound(name); + } + catch (Exception & e) + { + e.addMessage("while parsing value '{}' for setting '{}'", str, name); + throw; + } } template diff --git a/tests/queries/0_stateless/00396_uuid.reference b/tests/queries/0_stateless/00396_uuid.reference index fe92b3684a6..d70322ec4c1 100644 --- a/tests/queries/0_stateless/00396_uuid.reference +++ b/tests/queries/0_stateless/00396_uuid.reference @@ -5,3 +5,4 @@ 01234567-89ab-cdef-0123-456789abcdef 01234567-89ab-cdef-0123-456789abcdef 01234567-89ab-cdef-0123-456789abcdef 01234567-89ab-cdef-0123-456789abcdef 01234567-89ab-cdef-0123-456789abcdef 01234567-89ab-cdef-0123-456789abcdef 3f1ed72e-f7fe-4459-9cbe-95fe9298f845 +1 diff --git a/tests/queries/0_stateless/00396_uuid.sql b/tests/queries/0_stateless/00396_uuid.sql index d671ce844e2..863ff13f5c2 100644 --- a/tests/queries/0_stateless/00396_uuid.sql +++ b/tests/queries/0_stateless/00396_uuid.sql @@ -5,3 +5,9 @@ SELECT hex(UUIDStringToNum(materialize('01234567-89ab-cdef-0123-456789abcdef'))) SELECT '01234567-89ab-cdef-0123-456789abcdef' AS str, UUIDNumToString(UUIDStringToNum(str)), UUIDNumToString(UUIDStringToNum(toFixedString(str, 36))); SELECT materialize('01234567-89ab-cdef-0123-456789abcdef') AS str, UUIDNumToString(UUIDStringToNum(str)), UUIDNumToString(UUIDStringToNum(toFixedString(str, 36))); SELECT toString(toUUID('3f1ed72e-f7fe-4459-9cbe-95fe9298f845')); + +-- conversion back and forth to big-endian hex string +with generateUUIDv4() as uuid, + identity(lower(hex(reverse(reinterpretAsString(uuid))))) as str, + reinterpretAsUUID(reverse(unhex(str))) uuid2 +select uuid = uuid2; diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference index 176626befa7..e0eadd91a97 100644 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference @@ -7,4 +7,4 @@ 2 1 ===sampled=== -1 1 +OK diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh index 1cb65113179..d3114ab66ff 100755 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh @@ -74,7 +74,7 @@ done ${CLICKHOUSE_CLIENT} -q " with count(*) as c -- expect 200 * 0.1 = 20 sampled events on average - select c > 10, c < 30 + select if(c > 10 and c < 30, 'OK', 'fail: ' || toString(c)) from system.opentelemetry_log array join attribute.names as name, attribute.values as value where name = 'clickhouse.query_id' From 8212d7fa2838c3f8c5251cc9070ecda6f50b29cb Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 17 Sep 2020 15:14:30 +0300 Subject: [PATCH 048/325] fix the uuid test --- tests/queries/0_stateless/00396_uuid.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00396_uuid.sql b/tests/queries/0_stateless/00396_uuid.sql index 863ff13f5c2..9d8b48bddb0 100644 --- a/tests/queries/0_stateless/00396_uuid.sql +++ b/tests/queries/0_stateless/00396_uuid.sql @@ -9,5 +9,5 @@ SELECT toString(toUUID('3f1ed72e-f7fe-4459-9cbe-95fe9298f845')); -- conversion back and forth to big-endian hex string with generateUUIDv4() as uuid, identity(lower(hex(reverse(reinterpretAsString(uuid))))) as str, - reinterpretAsUUID(reverse(unhex(str))) uuid2 + reinterpretAsUUID(reverse(unhex(str))) as uuid2 select uuid = uuid2; From 7b8ad02a124cad4cb90280cac856b60d54551fda Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 17 Sep 2020 15:14:49 +0300 Subject: [PATCH 049/325] cleanup --- src/Server/HTTPHandler.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index fb630010198..2f1e978d48f 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -326,8 +326,6 @@ void HTTPHandler::processQuery( } context.getClientInfo().opentelemetry_tracestate = request.get("tracestate", ""); - - } /// The client can pass a HTTP header indicating supported compression method (gzip or deflate). From a374541214c7a4bece9bf77df132eda23e4c6e85 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 17 Sep 2020 15:15:05 +0300 Subject: [PATCH 050/325] straighten the protocol version --- base/daemon/BaseDaemon.cpp | 2 +- programs/copier/ClusterCopierApp.cpp | 2 +- programs/server/Server.cpp | 2 +- src/Client/Connection.cpp | 4 +- src/Common/ClickHouseRevision.cpp | 2 +- src/Common/ClickHouseRevision.h | 2 +- src/Common/StatusFile.cpp | 2 +- src/Common/config_version.h.in | 13 +---- src/Core/Defines.h | 4 +- src/DataStreams/TemporaryFileStream.h | 3 +- src/Interpreters/Aggregator.cpp | 2 +- src/Interpreters/ClientInfo.cpp | 9 ++-- src/Interpreters/ClientInfo.h | 2 +- src/Interpreters/CrashLog.cpp | 2 +- src/Interpreters/QueryLog.cpp | 4 +- src/Interpreters/QueryThreadLog.cpp | 2 +- src/Interpreters/TextLog.cpp | 2 +- src/Interpreters/TraceLog.cpp | 2 +- .../Transforms/AggregatingTransform.cpp | 3 +- src/Server/TCPHandler.cpp | 53 +++++++++---------- src/Server/TCPHandler.h | 2 +- src/Storages/Distributed/DirectoryMonitor.cpp | 7 ++- .../DistributedBlockOutputStream.cpp | 7 ++- .../System/StorageSystemProcesses.cpp | 2 +- 24 files changed, 58 insertions(+), 77 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 78801e71a6f..22455d09cf2 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -781,7 +781,7 @@ void BaseDaemon::initializeTerminationAndSignalProcessing() void BaseDaemon::logRevision() const { Poco::Logger::root().information("Starting " + std::string{VERSION_FULL} - + " with revision " + std::to_string(ClickHouseRevision::get()) + + " with revision " + std::to_string(ClickHouseRevision::getVersionRevision()) + ", " + build_id_info + ", PID " + std::to_string(getpid())); } diff --git a/programs/copier/ClusterCopierApp.cpp b/programs/copier/ClusterCopierApp.cpp index ec64e118f45..08a7e50a9d7 100644 --- a/programs/copier/ClusterCopierApp.cpp +++ b/programs/copier/ClusterCopierApp.cpp @@ -105,7 +105,7 @@ void ClusterCopierApp::mainImpl() ThreadStatus thread_status; auto * log = &logger(); - LOG_INFO(log, "Starting clickhouse-copier (id {}, host_id {}, path {}, revision {})", process_id, host_id, process_path, ClickHouseRevision::get()); + LOG_INFO(log, "Starting clickhouse-copier (id {}, host_id {}, path {}, revision {})", process_id, host_id, process_path, ClickHouseRevision::getVersionRevision()); SharedContextHolder shared_context = Context::createShared(); auto context = std::make_unique(Context::createGlobal(shared_context.get())); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 56778b8dd69..3f3f8ccef3a 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -256,7 +256,7 @@ int Server::main(const std::vector & /*args*/) #endif #endif - CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get()); + CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::getVersionRevision()); CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger()); if (ThreadFuzzer::instance().isEffective()) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index ed27a878b5a..0adfd6a0103 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -162,14 +162,12 @@ void Connection::sendHello() || has_control_character(password)) throw Exception("Parameters 'default_database', 'user' and 'password' must not contain ASCII control characters", ErrorCodes::BAD_ARGUMENTS); - auto client_revision = ClickHouseRevision::get(); - writeVarUInt(Protocol::Client::Hello, *out); writeStringBinary((DBMS_NAME " ") + client_name, *out); writeVarUInt(DBMS_VERSION_MAJOR, *out); writeVarUInt(DBMS_VERSION_MINOR, *out); // NOTE For backward compatibility of the protocol, client cannot send its version_patch. - writeVarUInt(client_revision, *out); + writeVarUInt(DBMS_TCP_PROTOCOL_VERSION, *out); writeStringBinary(default_database, *out); writeStringBinary(user, *out); writeStringBinary(password, *out); diff --git a/src/Common/ClickHouseRevision.cpp b/src/Common/ClickHouseRevision.cpp index 0b81026adca..2c52ebb064a 100644 --- a/src/Common/ClickHouseRevision.cpp +++ b/src/Common/ClickHouseRevision.cpp @@ -6,6 +6,6 @@ namespace ClickHouseRevision { - unsigned get() { return VERSION_REVISION; } + unsigned getVersionRevision() { return VERSION_REVISION; } unsigned getVersionInteger() { return VERSION_INTEGER; } } diff --git a/src/Common/ClickHouseRevision.h b/src/Common/ClickHouseRevision.h index 1d097a5bf89..86d1e3db334 100644 --- a/src/Common/ClickHouseRevision.h +++ b/src/Common/ClickHouseRevision.h @@ -2,6 +2,6 @@ namespace ClickHouseRevision { - unsigned get(); + unsigned getVersionRevision(); unsigned getVersionInteger(); } diff --git a/src/Common/StatusFile.cpp b/src/Common/StatusFile.cpp index 7c6bbf814a0..b21454c9ed8 100644 --- a/src/Common/StatusFile.cpp +++ b/src/Common/StatusFile.cpp @@ -37,7 +37,7 @@ StatusFile::FillFunction StatusFile::write_full_info = [](WriteBuffer & out) { out << "PID: " << getpid() << "\n" << "Started at: " << LocalDateTime(time(nullptr)) << "\n" - << "Revision: " << ClickHouseRevision::get() << "\n"; + << "Revision: " << ClickHouseRevision::getVersionRevision() << "\n"; }; diff --git a/src/Common/config_version.h.in b/src/Common/config_version.h.in index c3c0c6df87b..880824f8ad0 100644 --- a/src/Common/config_version.h.in +++ b/src/Common/config_version.h.in @@ -2,18 +2,7 @@ // .h autogenerated by cmake! -#cmakedefine01 USE_DBMS_TCP_PROTOCOL_VERSION - -#if USE_DBMS_TCP_PROTOCOL_VERSION - #include "Core/Defines.h" - #ifndef VERSION_REVISION - #define VERSION_REVISION DBMS_TCP_PROTOCOL_VERSION - #endif -#else - #cmakedefine VERSION_REVISION @VERSION_REVISION@ -#endif - - +#cmakedefine VERSION_REVISION @VERSION_REVISION@ #cmakedefine VERSION_NAME "@VERSION_NAME@" #define DBMS_NAME VERSION_NAME #cmakedefine VERSION_MAJOR @VERSION_MAJOR@ diff --git a/src/Core/Defines.h b/src/Core/Defines.h index d19513d1434..0b8e0b8ae27 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -68,10 +68,10 @@ #define DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS 54429 /// Minimum revision supporting OpenTelemetry -#define DBMS_MIN_REVISION_WITH_OPENTELEMETRY 54227 +#define DBMS_MIN_REVISION_WITH_OPENTELEMETRY 54441 /// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change. -#define DBMS_TCP_PROTOCOL_VERSION 54227 +#define DBMS_TCP_PROTOCOL_VERSION 54441 /// The boundary on which the blocks for asynchronous file operations should be aligned. #define DEFAULT_AIO_FILE_BLOCK_SIZE 4096 diff --git a/src/DataStreams/TemporaryFileStream.h b/src/DataStreams/TemporaryFileStream.h index 6871800a540..b481cef1bb2 100644 --- a/src/DataStreams/TemporaryFileStream.h +++ b/src/DataStreams/TemporaryFileStream.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -23,7 +22,7 @@ struct TemporaryFileStream TemporaryFileStream(const std::string & path) : file_in(path) , compressed_in(file_in) - , block_in(std::make_shared(compressed_in, ClickHouseRevision::get())) + , block_in(std::make_shared(compressed_in, DBMS_TCP_PROTOCOL_VERSION)) {} TemporaryFileStream(const std::string & path, const Block & header_) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 86a33dccb53..466370a22a2 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -844,7 +844,7 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, co const std::string & path = file->path(); WriteBufferFromFile file_buf(path); CompressedWriteBuffer compressed_buf(file_buf); - NativeBlockOutputStream block_out(compressed_buf, ClickHouseRevision::get(), getHeader(false)); + NativeBlockOutputStream block_out(compressed_buf, DBMS_TCP_PROTOCOL_VERSION, getHeader(false)); LOG_DEBUG(log, "Writing part of aggregation data into temporary file {}.", path); ProfileEvents::increment(ProfileEvents::ExternalAggregationWritePart); diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp index da9f14ec879..25cf9b8294e 100644 --- a/src/Interpreters/ClientInfo.cpp +++ b/src/Interpreters/ClientInfo.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #if !defined(ARCADIA_BUILD) @@ -44,7 +43,7 @@ void ClientInfo::write(WriteBuffer & out, const UInt64 server_protocol_revision) writeBinary(client_name, out); writeVarUInt(client_version_major, out); writeVarUInt(client_version_minor, out); - writeVarUInt(client_revision, out); + writeVarUInt(client_tcp_protocol_version, out); } else if (interface == Interface::HTTP) { @@ -111,7 +110,7 @@ void ClientInfo::read(ReadBuffer & in, const UInt64 client_protocol_revision) readBinary(client_name, in); readVarUInt(client_version_major, in); readVarUInt(client_version_minor, in); - readVarUInt(client_revision, in); + readVarUInt(client_tcp_protocol_version, in); } else if (interface == Interface::HTTP) { @@ -130,7 +129,7 @@ void ClientInfo::read(ReadBuffer & in, const UInt64 client_protocol_revision) if (client_protocol_revision >= DBMS_MIN_REVISION_WITH_VERSION_PATCH) readVarUInt(client_version_patch, in); else - client_version_patch = client_revision; + client_version_patch = client_tcp_protocol_version; } // TODO what does it even mean to read this structure over HTTP? I thought @@ -244,7 +243,7 @@ void ClientInfo::fillOSUserHostNameAndVersionInfo() client_version_major = DBMS_VERSION_MAJOR; client_version_minor = DBMS_VERSION_MINOR; client_version_patch = DBMS_VERSION_PATCH; - client_revision = ClickHouseRevision::get(); + client_tcp_protocol_version = DBMS_TCP_PROTOCOL_VERSION; } diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h index e41956eaee0..5b5c3b400eb 100644 --- a/src/Interpreters/ClientInfo.h +++ b/src/Interpreters/ClientInfo.h @@ -82,7 +82,7 @@ public: UInt64 client_version_major = 0; UInt64 client_version_minor = 0; UInt64 client_version_patch = 0; - unsigned client_revision = 0; + unsigned client_tcp_protocol_version = 0; /// For http HTTPMethod http_method = HTTPMethod::UNKNOWN; diff --git a/src/Interpreters/CrashLog.cpp b/src/Interpreters/CrashLog.cpp index 12fd57c33dc..9d84d5a18e9 100644 --- a/src/Interpreters/CrashLog.cpp +++ b/src/Interpreters/CrashLog.cpp @@ -49,7 +49,7 @@ void CrashLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(trace); columns[i++]->insert(trace_full); columns[i++]->insert(VERSION_FULL); - columns[i++]->insert(ClickHouseRevision::get()); + columns[i++]->insert(ClickHouseRevision::getVersionRevision()); String build_id_hex; #if defined(__ELF__) && !defined(__FreeBSD__) diff --git a/src/Interpreters/QueryLog.cpp b/src/Interpreters/QueryLog.cpp index 62dbc114633..75e0fae615a 100644 --- a/src/Interpreters/QueryLog.cpp +++ b/src/Interpreters/QueryLog.cpp @@ -118,7 +118,7 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const appendClientInfo(client_info, columns, i); - columns[i++]->insert(ClickHouseRevision::get()); + columns[i++]->insert(ClickHouseRevision::getVersionRevision()); { Array threads_array; @@ -172,7 +172,7 @@ void QueryLogElement::appendClientInfo(const ClientInfo & client_info, MutableCo columns[i++]->insert(client_info.os_user); columns[i++]->insert(client_info.client_hostname); columns[i++]->insert(client_info.client_name); - columns[i++]->insert(client_info.client_revision); + columns[i++]->insert(client_info.client_tcp_protocol_version); columns[i++]->insert(client_info.client_version_major); columns[i++]->insert(client_info.client_version_minor); columns[i++]->insert(client_info.client_version_patch); diff --git a/src/Interpreters/QueryThreadLog.cpp b/src/Interpreters/QueryThreadLog.cpp index 22ad60d96b4..e5a8cf7c5cf 100644 --- a/src/Interpreters/QueryThreadLog.cpp +++ b/src/Interpreters/QueryThreadLog.cpp @@ -93,7 +93,7 @@ void QueryThreadLogElement::appendToBlock(MutableColumns & columns) const QueryLogElement::appendClientInfo(client_info, columns, i); - columns[i++]->insert(ClickHouseRevision::get()); + columns[i++]->insert(ClickHouseRevision::getVersionRevision()); if (profile_counters) { diff --git a/src/Interpreters/TextLog.cpp b/src/Interpreters/TextLog.cpp index d166b24ef4f..243bf6d299a 100644 --- a/src/Interpreters/TextLog.cpp +++ b/src/Interpreters/TextLog.cpp @@ -62,7 +62,7 @@ void TextLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(logger_name); columns[i++]->insert(message); - columns[i++]->insert(ClickHouseRevision::get()); + columns[i++]->insert(ClickHouseRevision::getVersionRevision()); columns[i++]->insert(source_file); columns[i++]->insert(source_line); diff --git a/src/Interpreters/TraceLog.cpp b/src/Interpreters/TraceLog.cpp index c4fa7307b1a..f7e82032f49 100644 --- a/src/Interpreters/TraceLog.cpp +++ b/src/Interpreters/TraceLog.cpp @@ -43,7 +43,7 @@ void TraceLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); columns[i++]->insert(event_time); columns[i++]->insert(timestamp_ns); - columns[i++]->insert(ClickHouseRevision::get()); + columns[i++]->insert(ClickHouseRevision::getVersionRevision()); columns[i++]->insert(static_cast(trace_type)); columns[i++]->insert(thread_id); columns[i++]->insertData(query_id.data(), query_id.size()); diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index 42caf4b3446..0a97cc3d4cb 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include @@ -56,7 +55,7 @@ namespace public: SourceFromNativeStream(const Block & header, const std::string & path) : ISource(header), file_in(path), compressed_in(file_in), - block_in(std::make_shared(compressed_in, ClickHouseRevision::get())) + block_in(std::make_shared(compressed_in, DBMS_TCP_PROTOCOL_VERSION)) { block_in->readPrefix(); } diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 2c7ca3ac6a4..ca8774c23ca 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include @@ -183,7 +182,7 @@ void TCPHandler::runImpl() /// Should we send internal logs to client? const auto client_logs_level = query_context->getSettingsRef().send_logs_level; - if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_LOGS + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SERVER_LOGS && client_logs_level != LogsLevel::none) { state.logs_queue = std::make_shared(); @@ -218,7 +217,7 @@ void TCPHandler::runImpl() state.need_receive_data_for_input = true; /// Send ColumnsDescription for input storage. - if (client_revision >= DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA && query_context->getSettingsRef().input_format_defaults_for_omitted_fields) { sendTableColumns(metadata_snapshot->getColumns()); @@ -248,7 +247,7 @@ void TCPHandler::runImpl() customizeContext(*query_context); - bool may_have_embedded_data = client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_SUPPORT_EMBEDDED_DATA; + bool may_have_embedded_data = client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_CLIENT_SUPPORT_EMBEDDED_DATA; /// Processing Query state.io = executeQuery(state.query, *query_context, false, state.stage, may_have_embedded_data); @@ -482,7 +481,7 @@ void TCPHandler::processInsertQuery(const Settings & connection_settings) state.io.out->writePrefix(); /// Send ColumnsDescription for insertion table - if (client_revision >= DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA) + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA) { const auto & table_id = query_context->getInsertionTable(); if (query_context->getSettingsRef().input_format_defaults_for_omitted_fields) @@ -638,7 +637,7 @@ void TCPHandler::processOrdinaryQueryWithProcessors() void TCPHandler::processTablesStatusRequest() { TablesStatusRequest request; - request.read(*in, client_revision); + request.read(*in, client_tcp_protocol_version); TablesStatusResponse response; for (const QualifiedTableName & table_name: request.tables) @@ -661,13 +660,13 @@ void TCPHandler::processTablesStatusRequest() } writeVarUInt(Protocol::Server::TablesStatusResponse, *out); - response.write(*out, client_revision); + response.write(*out, client_tcp_protocol_version); } void TCPHandler::receiveUnexpectedTablesStatusRequest() { TablesStatusRequest skip_request; - skip_request.read(*in, client_revision); + skip_request.read(*in, client_tcp_protocol_version); throw NetException("Unexpected packet TablesStatusRequest received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); } @@ -742,7 +741,7 @@ void TCPHandler::receiveHello() readVarUInt(client_version_major, *in); readVarUInt(client_version_minor, *in); // NOTE For backward compatibility of the protocol, client cannot send its version_patch. - readVarUInt(client_revision, *in); + readVarUInt(client_tcp_protocol_version, *in); readStringBinary(default_database, *in); readStringBinary(user, *in); readStringBinary(password, *in); @@ -750,7 +749,7 @@ void TCPHandler::receiveHello() LOG_DEBUG(log, "Connected {} version {}.{}.{}, revision: {}{}{}.", client_name, client_version_major, client_version_minor, client_version_patch, - client_revision, + client_tcp_protocol_version, (!default_database.empty() ? ", database: " + default_database : ""), (!user.empty() ? ", user: " + user : "")); @@ -781,12 +780,12 @@ void TCPHandler::sendHello() writeStringBinary(DBMS_NAME, *out); writeVarUInt(DBMS_VERSION_MAJOR, *out); writeVarUInt(DBMS_VERSION_MINOR, *out); - writeVarUInt(ClickHouseRevision::get(), *out); - if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE) + writeVarUInt(DBMS_TCP_PROTOCOL_VERSION, *out); + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE) writeStringBinary(DateLUT::instance().getTimeZone(), *out); - if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME) + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME) writeStringBinary(server_display_name, *out); - if (client_revision >= DBMS_MIN_REVISION_WITH_VERSION_PATCH) + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_VERSION_PATCH) writeVarUInt(DBMS_VERSION_PATCH, *out); out->next(); } @@ -847,8 +846,8 @@ void TCPHandler::receiveQuery() /// Client info ClientInfo & client_info = query_context->getClientInfo(); - if (client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) - client_info.read(*in, client_revision); + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) + client_info.read(*in, client_tcp_protocol_version); /// For better support of old clients, that does not send ClientInfo. if (client_info.query_kind == ClientInfo::QueryKind::NO_QUERY) @@ -858,7 +857,7 @@ void TCPHandler::receiveQuery() client_info.client_version_major = client_version_major; client_info.client_version_minor = client_version_minor; client_info.client_version_patch = client_version_patch; - client_info.client_revision = client_revision; + client_info.client_tcp_protocol_version = client_tcp_protocol_version; } /// Set fields, that are known apriori. @@ -879,7 +878,7 @@ void TCPHandler::receiveQuery() /// Per query settings are also passed via TCP. /// We need to check them before applying due to they can violate the settings constraints. auto settings_format = - (client_revision >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) + (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) ? SettingsWriteFormat::STRINGS_WITH_FLAGS : SettingsWriteFormat::BINARY; @@ -900,7 +899,7 @@ void TCPHandler::receiveQuery() // Use the received query id, or generate a random default. It is convenient // to also generate the default OpenTelemetry trace id at the same time, and - // and and set the trace parent. + // set the trace parent. // Why is this done here and not earlier: // 1) ClientInfo might contain upstream trace id, so we decide whether to use // the default ids after we have received the ClientInfo. @@ -933,11 +932,11 @@ void TCPHandler::receiveUnexpectedQuery() readStringBinary(skip_string, *in); ClientInfo skip_client_info; - if (client_revision >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) - skip_client_info.read(*in, client_revision); + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) + skip_client_info.read(*in, client_tcp_protocol_version); Settings skip_settings; - auto settings_format = (client_revision >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) ? SettingsWriteFormat::STRINGS_WITH_FLAGS + auto settings_format = (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) ? SettingsWriteFormat::STRINGS_WITH_FLAGS : SettingsWriteFormat::BINARY; skip_settings.read(*in, settings_format); @@ -1011,7 +1010,7 @@ void TCPHandler::receiveUnexpectedData() auto skip_block_in = std::make_shared( *maybe_compressed_in, last_block_in.header, - client_revision); + client_tcp_protocol_version); skip_block_in->read(); throw NetException("Unexpected packet Data received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); @@ -1038,7 +1037,7 @@ void TCPHandler::initBlockInput() state.block_in = std::make_shared( *state.maybe_compressed_in, header, - client_revision); + client_tcp_protocol_version); } } @@ -1069,7 +1068,7 @@ void TCPHandler::initBlockOutput(const Block & block) state.block_out = std::make_shared( *state.maybe_compressed_out, - client_revision, + client_tcp_protocol_version, block.cloneEmpty(), !connection_context.getSettingsRef().low_cardinality_allow_in_native_format); } @@ -1082,7 +1081,7 @@ void TCPHandler::initLogsBlockOutput(const Block & block) /// Use uncompressed stream since log blocks usually contain only one row state.logs_block_out = std::make_shared( *out, - client_revision, + client_tcp_protocol_version, block.cloneEmpty(), !connection_context.getSettingsRef().low_cardinality_allow_in_native_format); } @@ -1186,7 +1185,7 @@ void TCPHandler::sendProgress() { writeVarUInt(Protocol::Server::Progress, *out); auto increment = state.progress.fetchAndResetPiecewiseAtomically(); - increment.write(*out, client_revision); + increment.write(*out, client_tcp_protocol_version); out->next(); } diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 3fec89264be..3c7747481f0 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -124,7 +124,7 @@ private: UInt64 client_version_major = 0; UInt64 client_version_minor = 0; UInt64 client_version_patch = 0; - UInt64 client_revision = 0; + UInt64 client_tcp_protocol_version = 0; Context connection_context; std::optional query_context; diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index b67d3283ac9..61708dce325 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -357,7 +356,7 @@ void StorageDistributedDirectoryMonitor::readHeader( UInt64 initiator_revision; readVarUInt(initiator_revision, header_buf); - if (ClickHouseRevision::get() < initiator_revision) + if (DBMS_TCP_PROTOCOL_VERSION < initiator_revision) { LOG_WARNING(log, "ClickHouse shard version is older than ClickHouse initiator version. It may lack support for new features."); } @@ -576,7 +575,7 @@ public: explicit DirectoryMonitorBlockInputStream(const String & file_name) : in(file_name) , decompressing_in(in) - , block_in(decompressing_in, ClickHouseRevision::get()) + , block_in(decompressing_in, DBMS_TCP_PROTOCOL_VERSION) , log{&Poco::Logger::get("DirectoryMonitorBlockInputStream")} { Settings insert_settings; @@ -681,7 +680,7 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map readHeader(in, insert_settings, insert_query, client_info, log); CompressedReadBuffer decompressing_in(in); - NativeBlockInputStream block_in(decompressing_in, ClickHouseRevision::get()); + NativeBlockInputStream block_in(decompressing_in, DBMS_TCP_PROTOCOL_VERSION); block_in.readPrefix(); while (Block block = block_in.read()) diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 172a398258f..f08cdf76cbf 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -583,16 +582,16 @@ void DistributedBlockOutputStream::writeToShard(const Block & block, const std:: { WriteBufferFromFile out{first_file_tmp_path}; CompressedWriteBuffer compress{out}; - NativeBlockOutputStream stream{compress, ClickHouseRevision::get(), block.cloneEmpty()}; + NativeBlockOutputStream stream{compress, DBMS_TCP_PROTOCOL_VERSION, block.cloneEmpty()}; /// Prepare the header. /// We wrap the header into a string for compatibility with older versions: /// a shard will able to read the header partly and ignore other parts based on its version. WriteBufferFromOwnString header_buf; - writeVarUInt(ClickHouseRevision::get(), header_buf); + writeVarUInt(DBMS_TCP_PROTOCOL_VERSION, header_buf); writeStringBinary(query_string, header_buf); context.getSettingsRef().write(header_buf); - context.getClientInfo().write(header_buf, ClickHouseRevision::get()); + context.getClientInfo().write(header_buf, DBMS_TCP_PROTOCOL_VERSION); /// Add new fields here, for example: /// writeVarUInt(my_new_data, header_buf); diff --git a/src/Storages/System/StorageSystemProcesses.cpp b/src/Storages/System/StorageSystemProcesses.cpp index c65a6b78e41..d899a1708bf 100644 --- a/src/Storages/System/StorageSystemProcesses.cpp +++ b/src/Storages/System/StorageSystemProcesses.cpp @@ -91,7 +91,7 @@ void StorageSystemProcesses::fillData(MutableColumns & res_columns, const Contex res_columns[i++]->insert(process.client_info.os_user); res_columns[i++]->insert(process.client_info.client_hostname); res_columns[i++]->insert(process.client_info.client_name); - res_columns[i++]->insert(process.client_info.client_revision); + res_columns[i++]->insert(process.client_info.client_tcp_protocol_version); res_columns[i++]->insert(process.client_info.client_version_major); res_columns[i++]->insert(process.client_info.client_version_minor); res_columns[i++]->insert(process.client_info.client_version_patch); From 4c6da6b70cb4105ab3383c342e4bcb4523215ae0 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 29 Sep 2020 21:01:49 +0300 Subject: [PATCH 051/325] allow fast test to run locally --- docker/test/fasttest/run.sh | 165 +++++++++++++++++++++++++----------- 1 file changed, 114 insertions(+), 51 deletions(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index a277ddf9d36..b93aa0ce5cd 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -16,28 +16,55 @@ stage=${stage:-} read -ra FASTTEST_CMAKE_FLAGS <<< "${FASTTEST_CMAKE_FLAGS:-}" -function kill_clickhouse +FASTTEST_WORKSPACE=$(readlink -f "${FASTTEST_WORKSPACE:-.}") +FASTTEST_SOURCE=$(readlink -f "${FASTTEST_SOURCE:-$FASTTEST_WORKSPACE/ch}") +FASTTEST_BUILD=$(readlink -f "${FASTTEST_BUILD:-${BUILD:-$FASTTEST_WORKSPACE/build}}") +FASTTEST_DATA=$(readlink -f "${FASTTEST_DATA:-$FASTTEST_WORKSPACE/db-fasttest}") +FASTTEST_OUTPUT=$(readlink -f "${FASTTEST_OUTPUT:-$FASTTEST_WORKSPACE}") + +server_pid=none + +function stop_server { + if ! kill -- "$server_pid" + then + echo "The server we started ($server_pid) is not present, won't do anything" + return 0 + fi + for _ in {1..60} do - if ! pkill -f clickhouse-server ; then break ; fi + if ! kill -- "$server_pid" ; then break ; fi sleep 1 done - if pgrep -f clickhouse-server + if kill -0 -- "$server_pid" then pstree -apgT jobs - echo "Failed to kill the ClickHouse server $(pgrep -f clickhouse-server)" + echo "Failed to kill the ClickHouse server pid '$server_pid'" return 1 fi } -function wait_for_server_start +function start_server { + set -m # Spawn server in its own process groups + clickhouse-server --config-file="$FASTTEST_DATA/config.xml" -- --path "$FASTTEST_DATA" --user_files_path "$FASTTEST_DATA/user_files" &>> "$FASTTEST_OUTPUT/server.log" & + server_pid=$! + set +m + + if [ "$server_pid" == "0" ] + then + echo "Failed to start ClickHouse server" + # Avoid zero PID because `kill` treats it as our process group PID. + server_pid="none" + return 1 + fi + for _ in {1..60} do - if clickhouse-client --query "select 1" || ! pgrep -f clickhouse-server + if clickhouse-client --query "select 1" || ! kill -0 -- "$server_pid" then break fi @@ -47,20 +74,26 @@ function wait_for_server_start if ! clickhouse-client --query "select 1" then echo "Failed to wait until ClickHouse server starts." + server_pid="none" return 1 fi - echo "ClickHouse server pid '$(pgrep -f clickhouse-server)' started and responded" + if ! kill -0 -- "$server_pid" + then + echo "Wrong clickhouse server started: PID '$server_pid' we started is not running, but '$(pgrep -f clickhouse-server)' is running" + server_pid="none" + return 1 + fi + + echo "ClickHouse server pid '$server_pid' started and responded" } function clone_root { -git clone https://github.com/ClickHouse/ClickHouse.git | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/clone_log.txt -cd ClickHouse -CLICKHOUSE_DIR=$(pwd) -export CLICKHOUSE_DIR - +git clone https://github.com/ClickHouse/ClickHouse.git -- "$FASTTEST_SOURCE" | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" +( +cd "$FASTTEST_SOURCE" if [ "$PULL_REQUEST_NUMBER" != "0" ]; then if git fetch origin "+refs/pull/$PULL_REQUEST_NUMBER/merge"; then git checkout FETCH_HEAD @@ -71,22 +104,37 @@ if [ "$PULL_REQUEST_NUMBER" != "0" ]; then echo 'Checked out to commit' fi else - if [ "$COMMIT_SHA" != "" ]; then + if [ -v COMMIT_SHA ]; then git checkout "$COMMIT_SHA" fi fi +) } -function run +function clone_submodules { +( +cd "$FASTTEST_SOURCE" + SUBMODULES_TO_UPDATE=(contrib/boost contrib/zlib-ng contrib/libxml2 contrib/poco contrib/libunwind contrib/ryu contrib/fmtlib contrib/base64 contrib/cctz contrib/libcpuid contrib/double-conversion contrib/libcxx contrib/libcxxabi contrib/libc-headers contrib/lz4 contrib/zstd contrib/fastops contrib/rapidjson contrib/re2 contrib/sparsehash-c11) -git submodule update --init --recursive "${SUBMODULES_TO_UPDATE[@]}" | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/submodule_log.txt +git submodule sync +git submodule update --init --recursive "${SUBMODULES_TO_UPDATE[@]}" +git submodule foreach git reset --hard +git submodule foreach git checkout @ -f +git submodule foreach git clean -xfd +) +} + +function build +{ CMAKE_LIBS_CONFIG=(-DENABLE_LIBRARIES=0 -DENABLE_TESTS=0 -DENABLE_UTILS=0 -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_THINLTO=0 -DUSE_UNWIND=1) -export CCACHE_DIR=/ccache -export CCACHE_BASEDIR=/ClickHouse +# TODO remove this? we don't use ccache anyway. An option would be to download it +# from S3 simultaneously with cloning. +export CCACHE_DIR="$FASTTEST_WORKSPACE/ccache" +export CCACHE_BASEDIR="$FASTTEST_SOURCE" export CCACHE_NOHASHDIR=true export CCACHE_COMPILERCHECK=content export CCACHE_MAXSIZE=15G @@ -94,34 +142,40 @@ export CCACHE_MAXSIZE=15G ccache --show-stats ||: ccache --zero-stats ||: -mkdir build -cd build -cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_C_COMPILER=clang-10 "${CMAKE_LIBS_CONFIG[@]}" "${FASTTEST_CMAKE_FLAGS[@]}" | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/cmake_log.txt -time ninja clickhouse-bundle | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/build_log.txt -ninja install | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/install_log.txt - +mkdir "$FASTTEST_BUILD" ||: +( +cd "$FASTTEST_BUILD" +cmake "$FASTTEST_SOURCE" -DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_C_COMPILER=clang-10 "${CMAKE_LIBS_CONFIG[@]}" "${FASTTEST_CMAKE_FLAGS[@]}" | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/cmake_log.txt" +time ninja clickhouse-bundle | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/build_log.txt" +) ccache --show-stats ||: +} -mkdir -p /etc/clickhouse-server -mkdir -p /etc/clickhouse-client -mkdir -p /etc/clickhouse-server/config.d -mkdir -p /etc/clickhouse-server/users.d -ln -s /test_output /var/log/clickhouse-server -cp "$CLICKHOUSE_DIR/programs/server/config.xml" /etc/clickhouse-server/ -cp "$CLICKHOUSE_DIR/programs/server/users.xml" /etc/clickhouse-server/ +function configure +{ +export PATH="$FASTTEST_BUILD/programs:$FASTTEST_SOURCE/tests:$PATH" -# install tests config -$CLICKHOUSE_DIR/tests/config/install.sh +clickhouse-client --version +clickhouse-test --help + +mkdir -p "$FASTTEST_DATA"{,/client-config} +cp -a "$FASTTEST_SOURCE/programs/server/"{config,users}.xml "$FASTTEST_DATA" +cp -a "$FASTTEST_SOURCE/programs/server/"{config,users}.xml "$FASTTEST_DATA" +"$FASTTEST_SOURCE/tests/config/install.sh" "$FASTTEST_DATA" "$FASTTEST_DATA/client-config" # doesn't support SSL -rm -f /etc/clickhouse-server/config.d/secure_ports.xml +rm -f "$FASTTEST_DATA/config.d/secure_ports.xml" +} + +function run_tests +{ +clickhouse-server --version +clickhouse-test --help # Kill the server in case we are running locally and not in docker -kill_clickhouse +stop_server ||: -clickhouse-server --config /etc/clickhouse-server/config.xml --daemon - -wait_for_server_start +start_server TESTS_TO_SKIP=( parquet @@ -191,11 +245,10 @@ TESTS_TO_SKIP=( 01460_DistributedFilesToInsert ) -time clickhouse-test -j 8 --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/test_log.txt - +time clickhouse-test -j 8 --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt" # substr is to remove semicolon after test name -readarray -t FAILED_TESTS < <(awk '/FAIL|TIMEOUT|ERROR/ { print substr($3, 1, length($3)-1) }' /test_output/test_log.txt | tee /test_output/failed-parallel-tests.txt) +readarray -t FAILED_TESTS < <(awk '/FAIL|TIMEOUT|ERROR/ { print substr($3, 1, length($3)-1) }' "$FASTTEST_OUTPUT/test_log.txt" | tee "$FASTTEST_OUTPUT/failed-parallel-tests.txt") # We will rerun sequentially any tests that have failed during parallel run. # They might have failed because there was some interference from other tests @@ -206,19 +259,16 @@ readarray -t FAILED_TESTS < <(awk '/FAIL|TIMEOUT|ERROR/ { print substr($3, 1, le # explicit instead of guessing. if [[ -n "${FAILED_TESTS[*]}" ]] then - kill_clickhouse + stop_server ||: # Clean the data so that there is no interference from the previous test run. - rm -rf /var/lib/clickhouse ||: - mkdir /var/lib/clickhouse + rm -rf "$FASTTEST_DATA"/{meta,}data ||: - clickhouse-server --config /etc/clickhouse-server/config.xml --daemon - - wait_for_server_start + start_server echo "Going to run again: ${FAILED_TESTS[*]}" - clickhouse-test --no-long --testname --shard --zookeeper "${FAILED_TESTS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a /test_output/test_log.txt + clickhouse-test --no-long --testname --shard --zookeeper "${FAILED_TESTS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a "$FASTTEST_OUTPUT/test_log.txt" else echo "No failed tests" fi @@ -228,20 +278,33 @@ case "$stage" in "") ls -la ;& - "clone_root") clone_root # Pass control to the script from cloned sources, unless asked otherwise. if ! [ -v FASTTEST_LOCAL_SCRIPT ] then - stage=run "$CLICKHOUSE_DIR/docker/test/fasttest/run.sh" + # 'run' is deprecated, used for compatibility with old scripts. + # Replace with 'clone_submodules' after Nov 1, 2020. + stage=run "$FASTTEST_SOURCE/docker/test/fasttest/run.sh" exit $? fi ;& - "run") - run + # A deprecated stage that is called by old script and equivalent to everything + # after cloning root, starting with cloning submodules. + ;& +"clone_submodules") + clone_submodules | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/submodule_log.txt" + ;& +"build") + build + ;& +"configure") + configure + ;& +"run_tests") + run_tests ;& esac From 00f9888c5d3f631f0ac2b9f9ba06098d56bc33a6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 29 Sep 2020 21:55:06 +0300 Subject: [PATCH 052/325] fasttest fixup --- docker/test/fasttest/run.sh | 27 ++++++++++++-------- tests/config/install.sh | 50 ++++++++++++++++++------------------- 2 files changed, 41 insertions(+), 36 deletions(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index b93aa0ce5cd..6a3f16d52f6 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -26,15 +26,9 @@ server_pid=none function stop_server { - if ! kill -- "$server_pid" - then - echo "The server we started ($server_pid) is not present, won't do anything" - return 0 - fi - for _ in {1..60} do - if ! kill -- "$server_pid" ; then break ; fi + if ! pkill -f "clickhouse-server" && ! kill -- "$server_pid" ; then break ; fi sleep 1 done @@ -45,6 +39,8 @@ function stop_server echo "Failed to kill the ClickHouse server pid '$server_pid'" return 1 fi + + server_pid=none } function start_server @@ -126,10 +122,9 @@ git submodule foreach git clean -xfd ) } - -function build +function run_cmake { -CMAKE_LIBS_CONFIG=(-DENABLE_LIBRARIES=0 -DENABLE_TESTS=0 -DENABLE_UTILS=0 -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_THINLTO=0 -DUSE_UNWIND=1) +CMAKE_LIBS_CONFIG=("-DENABLE_LIBRARIES=0" "-DENABLE_TESTS=0" "-DENABLE_UTILS=0" "-DENABLE_EMBEDDED_COMPILER=0" "-DENABLE_THINLTO=0" "-DUSE_UNWIND=1") # TODO remove this? we don't use ccache anyway. An option would be to download it # from S3 simultaneously with cloning. @@ -143,13 +138,20 @@ ccache --show-stats ||: ccache --zero-stats ||: mkdir "$FASTTEST_BUILD" ||: + ( cd "$FASTTEST_BUILD" cmake "$FASTTEST_SOURCE" -DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_C_COMPILER=clang-10 "${CMAKE_LIBS_CONFIG[@]}" "${FASTTEST_CMAKE_FLAGS[@]}" | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/cmake_log.txt" -time ninja clickhouse-bundle | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/build_log.txt" ) +} +function build +{ +( +cd "$FASTTEST_BUILD" +time ninja clickhouse-bundle | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/build_log.txt" ccache --show-stats ||: +) } function configure @@ -297,6 +299,9 @@ case "$stage" in "clone_submodules") clone_submodules | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/submodule_log.txt" ;& +"run_cmake") + run_cmake + ;& "build") build ;& diff --git a/tests/config/install.sh b/tests/config/install.sh index 0f33854ef95..ef9604904e7 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -15,40 +15,40 @@ mkdir -p $DEST_SERVER_PATH/config.d/ mkdir -p $DEST_SERVER_PATH/users.d/ mkdir -p $DEST_CLIENT_PATH -ln -s $SRC_PATH/config.d/zookeeper.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/config.d/listen.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/config.d/part_log.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/config.d/text_log.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/config.d/metric_log.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/config.d/custom_settings_prefixes.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/config.d/macros.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/config.d/disks.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/config.d/secure_ports.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/config.d/clusters.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/config.d/graphite.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/config.d/database_atomic.xml $DEST_SERVER_PATH/config.d/ -ln -s $SRC_PATH/users.d/log_queries.xml $DEST_SERVER_PATH/users.d/ -ln -s $SRC_PATH/users.d/readonly.xml $DEST_SERVER_PATH/users.d/ -ln -s $SRC_PATH/users.d/access_management.xml $DEST_SERVER_PATH/users.d/ +ln -sf $SRC_PATH/config.d/zookeeper.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/listen.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/part_log.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/text_log.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/metric_log.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/custom_settings_prefixes.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/macros.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/disks.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/secure_ports.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/clusters.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/graphite.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/config.d/database_atomic.xml $DEST_SERVER_PATH/config.d/ +ln -sf $SRC_PATH/users.d/log_queries.xml $DEST_SERVER_PATH/users.d/ +ln -sf $SRC_PATH/users.d/readonly.xml $DEST_SERVER_PATH/users.d/ +ln -sf $SRC_PATH/users.d/access_management.xml $DEST_SERVER_PATH/users.d/ -ln -s $SRC_PATH/ints_dictionary.xml $DEST_SERVER_PATH/ -ln -s $SRC_PATH/strings_dictionary.xml $DEST_SERVER_PATH/ -ln -s $SRC_PATH/decimals_dictionary.xml $DEST_SERVER_PATH/ -ln -s $SRC_PATH/executable_dictionary.xml $DEST_SERVER_PATH/ +ln -sf $SRC_PATH/ints_dictionary.xml $DEST_SERVER_PATH/ +ln -sf $SRC_PATH/strings_dictionary.xml $DEST_SERVER_PATH/ +ln -sf $SRC_PATH/decimals_dictionary.xml $DEST_SERVER_PATH/ +ln -sf $SRC_PATH/executable_dictionary.xml $DEST_SERVER_PATH/ -ln -s $SRC_PATH/server.key $DEST_SERVER_PATH/ -ln -s $SRC_PATH/server.crt $DEST_SERVER_PATH/ -ln -s $SRC_PATH/dhparam.pem $DEST_SERVER_PATH/ +ln -sf $SRC_PATH/server.key $DEST_SERVER_PATH/ +ln -sf $SRC_PATH/server.crt $DEST_SERVER_PATH/ +ln -sf $SRC_PATH/dhparam.pem $DEST_SERVER_PATH/ # Retain any pre-existing config and allow ClickHouse to load it if required -ln -s --backup=simple --suffix=_original.xml \ +ln -sf --backup=simple --suffix=_original.xml \ $SRC_PATH/config.d/query_masking_rules.xml $DEST_SERVER_PATH/config.d/ if [[ -n "$USE_POLYMORPHIC_PARTS" ]] && [[ "$USE_POLYMORPHIC_PARTS" -eq 1 ]]; then - ln -s $SRC_PATH/config.d/polymorphic_parts.xml $DEST_SERVER_PATH/config.d/ + ln -sf $SRC_PATH/config.d/polymorphic_parts.xml $DEST_SERVER_PATH/config.d/ fi if [[ -n "$USE_DATABASE_ORDINARY" ]] && [[ "$USE_DATABASE_ORDINARY" -eq 1 ]]; then - ln -s $SRC_PATH/users.d/database_ordinary.xml $DEST_SERVER_PATH/users.d/ + ln -sf $SRC_PATH/users.d/database_ordinary.xml $DEST_SERVER_PATH/users.d/ fi ln -sf $SRC_PATH/client_config.xml $DEST_CLIENT_PATH/config.xml From 85ec02dcedcadea5cacd18913722a421100ccade Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 29 Sep 2020 23:09:52 +0300 Subject: [PATCH 053/325] fasttest fixup 2 --- docker/test/fasttest/run.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 6a3f16d52f6..6898000185d 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -306,7 +306,9 @@ case "$stage" in build ;& "configure") - configure + # The `install_log.txt` is also needed for compatibility with old CI task -- + # if there is no log, it will decide that build failed. + configure | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/install_log.txt" ;& "run_tests") run_tests From 13c6fd48296594f8dc705110fd142349a491a624 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 30 Sep 2020 00:05:05 +0300 Subject: [PATCH 054/325] fasttest fixup 3 --- docker/test/fasttest/run.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 6898000185d..a9cbe87ecf9 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -156,7 +156,8 @@ ccache --show-stats ||: function configure { -export PATH="$FASTTEST_BUILD/programs:$FASTTEST_SOURCE/tests:$PATH" +PATH="$FASTTEST_BUILD/programs:$FASTTEST_SOURCE/tests:$PATH" +export PATH clickhouse-client --version clickhouse-test --help @@ -171,6 +172,7 @@ rm -f "$FASTTEST_DATA/config.d/secure_ports.xml" function run_tests { +echo "$PATH" clickhouse-server --version clickhouse-test --help From edf5797d117467e879783a50a05b840740fd19ee Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 30 Sep 2020 17:21:06 +0300 Subject: [PATCH 055/325] fasttest fixup 3 --- docker/test/fasttest/run.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index a9cbe87ecf9..4a47fcfe4dc 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -156,9 +156,6 @@ ccache --show-stats ||: function configure { -PATH="$FASTTEST_BUILD/programs:$FASTTEST_SOURCE/tests:$PATH" -export PATH - clickhouse-client --version clickhouse-test --help @@ -172,7 +169,6 @@ rm -f "$FASTTEST_DATA/config.d/secure_ports.xml" function run_tests { -echo "$PATH" clickhouse-server --version clickhouse-test --help @@ -306,6 +302,8 @@ case "$stage" in ;& "build") build + PATH="$FASTTEST_BUILD/programs:$FASTTEST_SOURCE/tests:$PATH" + export PATH ;& "configure") # The `install_log.txt` is also needed for compatibility with old CI task -- From 222b6555175019c487d9f71196ea462231fa1530 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 30 Sep 2020 18:12:53 +0300 Subject: [PATCH 056/325] fasttest compat with old script --- docker/test/fasttest/run.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 4a47fcfe4dc..c4a9a0ea5f3 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -284,8 +284,14 @@ case "$stage" in # Pass control to the script from cloned sources, unless asked otherwise. if ! [ -v FASTTEST_LOCAL_SCRIPT ] then - # 'run' is deprecated, used for compatibility with old scripts. + # 'run' stage is deprecated, used for compatibility with old scripts. # Replace with 'clone_submodules' after Nov 1, 2020. + # cd and CLICKHOUSE_DIR are also a setup for old scripts, remove as well. + # In modern script we undo it by changing back into workspace dir right + # away, see below. Remove that as well. + cd "$FASTTEST_SOURCE" + CLICKHOUSE_DIR=$(pwd) + export CLICKHOUSE_DIR stage=run "$FASTTEST_SOURCE/docker/test/fasttest/run.sh" exit $? fi @@ -295,6 +301,10 @@ case "$stage" in # after cloning root, starting with cloning submodules. ;& "clone_submodules") + # Recover after being called from the old script that changes into source directory. + # See the compatibility hacks in `clone_root` stage above. Remove at the same time, + # after Nov 1, 2020. + cd "$FASTTEST_WORKSPACE" clone_submodules | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/submodule_log.txt" ;& "run_cmake") From 626c2a3e28e2b179aa78ad363866bc1a53dd90d6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 30 Sep 2020 19:16:33 +0300 Subject: [PATCH 057/325] microsecond precision for start/finish time --- programs/server/config.xml | 2 +- src/Interpreters/ClientInfo.cpp | 6 +++-- src/Interpreters/OpenTelemetryLog.cpp | 11 ++++---- src/Interpreters/OpenTelemetryLog.h | 8 ++++-- src/Interpreters/executeQuery.cpp | 37 ++++++++++++--------------- 5 files changed, 34 insertions(+), 30 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index b3269f2e842..9a1b626b26a 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -611,7 +611,7 @@ engine MergeTree partition by toYYYYMM(finish_date) - order by (finish_date, finish_time, trace_id) + order by (finish_date, finish_time_us, trace_id) system opentelemetry_log
diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp index 25cf9b8294e..5570a1bc88d 100644 --- a/src/Interpreters/ClientInfo.cpp +++ b/src/Interpreters/ClientInfo.cpp @@ -62,11 +62,12 @@ void ClientInfo::write(WriteBuffer & out, const UInt64 server_protocol_revision) if (server_protocol_revision >= DBMS_MIN_REVISION_WITH_OPENTELEMETRY) { - // No point writing these numbers with variable length, because they - // are random and will probably require the full length anyway. if (opentelemetry_trace_id) { + // Have OpenTelemetry header. writeBinary(uint8_t(1), out); + // No point writing these numbers with variable length, because they + // are random and will probably require the full length anyway. writeBinary(opentelemetry_trace_id, out); writeBinary(opentelemetry_span_id, out); writeBinary(opentelemetry_parent_span_id, out); @@ -75,6 +76,7 @@ void ClientInfo::write(WriteBuffer & out, const UInt64 server_protocol_revision) } else { + // Don't have OpenTelemetry header. writeBinary(uint8_t(0), out); } } diff --git a/src/Interpreters/OpenTelemetryLog.cpp b/src/Interpreters/OpenTelemetryLog.cpp index 161af01107f..a247e557fcc 100644 --- a/src/Interpreters/OpenTelemetryLog.cpp +++ b/src/Interpreters/OpenTelemetryLog.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -17,8 +18,8 @@ Block OpenTelemetrySpanLogElement::createBlock() {std::make_shared(), "span_id"}, {std::make_shared(), "parent_span_id"}, {std::make_shared(), "operation_name"}, - {std::make_shared(), "start_time"}, - {std::make_shared(), "finish_time"}, + {std::make_shared(6), "start_time_us"}, + {std::make_shared(6), "finish_time_us"}, {std::make_shared(), "finish_date"}, {std::make_shared(std::make_shared()), "attribute.names"}, @@ -35,9 +36,9 @@ void OpenTelemetrySpanLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(span_id); columns[i++]->insert(parent_span_id); columns[i++]->insert(operation_name); - columns[i++]->insert(start_time); - columns[i++]->insert(finish_time); - columns[i++]->insert(DateLUT::instance().toDayNum(finish_time)); + columns[i++]->insert(start_time_us); + columns[i++]->insert(finish_time_us); + columns[i++]->insert(DateLUT::instance().toDayNum(finish_time_us / 1000000)); columns[i++]->insert(attribute_names); columns[i++]->insert(attribute_values); } diff --git a/src/Interpreters/OpenTelemetryLog.h b/src/Interpreters/OpenTelemetryLog.h index e2906a99a02..2d8536ff47e 100644 --- a/src/Interpreters/OpenTelemetryLog.h +++ b/src/Interpreters/OpenTelemetryLog.h @@ -15,6 +15,10 @@ struct OpenTelemetrySpanContext }; */ +// using TimeMicroseconds = std::chrono::time_point< +// std::chrono::local_t, +// std::chrono::duration>; + // TODO figure out precisely which part of this is run time, and which part we // must log. struct OpenTelemetrySpan @@ -23,8 +27,8 @@ struct OpenTelemetrySpan UInt64 span_id; UInt64 parent_span_id; std::string operation_name; - time_t start_time; - time_t finish_time; + UInt64 start_time_us; + UInt64 finish_time_us; UInt64 duration_ns; Array attribute_names; Array attribute_values; diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 30f3dcd87b1..d89725b30f4 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -205,7 +205,7 @@ inline UInt64 time_in_seconds(std::chrono::time_point return std::chrono::duration_cast(timepoint.time_since_epoch()).count(); } -static void onExceptionBeforeStart(const String & query_for_logging, Context & context, time_t current_time, UInt64 current_time_microseconds, ASTPtr ast) +static void onExceptionBeforeStart(const String & query_for_logging, Context & context, UInt64 current_time_us, ASTPtr ast) { /// Exception before the query execution. if (auto quota = context.getQuota()) @@ -221,9 +221,9 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c // all callers to onExceptionBeforeStart upstream construct the timespec for event_time and // event_time_microseconds from the same timespec. So it can be assumed that both of these // times are equal upto the precision of a second. - elem.event_time = current_time; - elem.query_start_time = current_time; - elem.query_start_time_microseconds = current_time_microseconds; + elem.event_time = current_time_us / 1000000; + elem.query_start_time = current_time_us / 1000000; + elem.query_start_time_microseconds = current_time_us; elem.current_database = context.getCurrentDatabase(); elem.query = query_for_logging; @@ -252,8 +252,8 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c span.span_id = context.getClientInfo().opentelemetry_span_id; span.parent_span_id = context.getClientInfo().opentelemetry_parent_span_id; span.operation_name = "query"; - span.start_time = current_time; - span.finish_time = current_time; + span.start_time_us = current_time_us; + span.finish_time_us = current_time_us; span.duration_ns = 0; // keep values synchonized to type enum in QueryLogElement::createBlock @@ -309,12 +309,7 @@ static std::tuple executeQueryImpl( bool has_query_tail, ReadBuffer * istr) { - // current_time and current_time_microseconds are both constructed from the same time point - // to ensure that both the times are equal upto the precision of a second. - const auto now = std::chrono::system_clock::now(); - - auto current_time = time_in_seconds(now); - auto current_time_microseconds = time_in_microseconds(now); + const auto current_time = std::chrono::system_clock::now(); /// If we already executing query and it requires to execute internal query, than /// don't replace thread context with given (it can be temporary). Otherwise, attach context to thread. @@ -364,7 +359,7 @@ static std::tuple executeQueryImpl( logQuery(query_for_logging, context, internal); if (!internal) - onExceptionBeforeStart(query_for_logging, context, current_time, current_time_microseconds, ast); + onExceptionBeforeStart(query_for_logging, context, time_in_microseconds(current_time), ast); throw; } @@ -528,9 +523,9 @@ static std::tuple executeQueryImpl( elem.type = QueryLogElementType::QUERY_START; - elem.event_time = current_time; - elem.query_start_time = current_time; - elem.query_start_time_microseconds = current_time_microseconds; + elem.event_time = time_in_seconds(current_time); + elem.query_start_time = time_in_seconds(current_time); + elem.query_start_time_microseconds = time_in_microseconds(current_time); elem.current_database = context.getCurrentDatabase(); elem.query = query_for_logging; @@ -599,7 +594,9 @@ static std::tuple executeQueryImpl( elem.type = QueryLogElementType::QUERY_FINISH; - elem.event_time = time(nullptr); + const auto current_time = std::chrono::system_clock::now(); + + elem.event_time = time_in_seconds(current_time); status_info_to_query_log(elem, info, ast); @@ -660,8 +657,8 @@ static std::tuple executeQueryImpl( span.span_id = context.getClientInfo().opentelemetry_span_id; span.parent_span_id = context.getClientInfo().opentelemetry_parent_span_id; span.operation_name = "query"; - span.start_time = elem.query_start_time; - span.finish_time = elem.event_time; + span.start_time_us = elem.query_start_time_microseconds; + span.finish_time_us = time_in_microseconds(current_time); span.duration_ns = elapsed_seconds * 1000000000; // keep values synchonized to type enum in QueryLogElement::createBlock @@ -751,7 +748,7 @@ static std::tuple executeQueryImpl( if (query_for_logging.empty()) query_for_logging = prepareQueryForLogging(query, context); - onExceptionBeforeStart(query_for_logging, context, current_time, current_time_microseconds, ast); + onExceptionBeforeStart(query_for_logging, context, time_in_microseconds(current_time), ast); } throw; From a2cd346303d89043855bd6bf0aa385386d72e2fd Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 30 Sep 2020 20:35:46 +0300 Subject: [PATCH 058/325] fasttest fixup 5 --- docker/test/fasttest/run.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index c4a9a0ea5f3..5e586402bd7 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -15,13 +15,20 @@ stage=${stage:-} # empty parameter. read -ra FASTTEST_CMAKE_FLAGS <<< "${FASTTEST_CMAKE_FLAGS:-}" - FASTTEST_WORKSPACE=$(readlink -f "${FASTTEST_WORKSPACE:-.}") FASTTEST_SOURCE=$(readlink -f "${FASTTEST_SOURCE:-$FASTTEST_WORKSPACE/ch}") FASTTEST_BUILD=$(readlink -f "${FASTTEST_BUILD:-${BUILD:-$FASTTEST_WORKSPACE/build}}") FASTTEST_DATA=$(readlink -f "${FASTTEST_DATA:-$FASTTEST_WORKSPACE/db-fasttest}") FASTTEST_OUTPUT=$(readlink -f "${FASTTEST_OUTPUT:-$FASTTEST_WORKSPACE}") +# Export these variables, so that all subsequent invocations of the script +# use them, and not try to guess them anew, which leads to weird effects. +export FASTTEST_WORKSPACE +export FASTTEST_SOURCE +export FASTTEST_BUILD +export FASTTEST_DATA +export FASTTEST_OUT + server_pid=none function stop_server From 16ab4d59e6a90360064c765c6c390bb49ffbb764 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 30 Sep 2020 20:36:02 +0300 Subject: [PATCH 059/325] don't shadow local var --- src/Interpreters/executeQuery.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index d89725b30f4..94a58a1bda8 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -594,9 +594,9 @@ static std::tuple executeQueryImpl( elem.type = QueryLogElementType::QUERY_FINISH; - const auto current_time = std::chrono::system_clock::now(); + const auto finish_time = std::chrono::system_clock::now(); - elem.event_time = time_in_seconds(current_time); + elem.event_time = time_in_seconds(finish_time); status_info_to_query_log(elem, info, ast); @@ -658,7 +658,7 @@ static std::tuple executeQueryImpl( span.parent_span_id = context.getClientInfo().opentelemetry_parent_span_id; span.operation_name = "query"; span.start_time_us = elem.query_start_time_microseconds; - span.finish_time_us = time_in_microseconds(current_time); + span.finish_time_us = time_in_microseconds(finish_time); span.duration_ns = elapsed_seconds * 1000000000; // keep values synchonized to type enum in QueryLogElement::createBlock From 3a963c988d613d316bfaf9e0d129452586033173 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 2 Oct 2020 14:38:32 +0300 Subject: [PATCH 060/325] fixup --- docker/test/fasttest/run.sh | 5 ++++- src/Common/Exception.h | 22 ++++++++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 5e586402bd7..a0830ba5f12 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -329,7 +329,10 @@ case "$stage" in ;& "run_tests") run_tests - ;& + ;; +*) + echo "Unknown test stage '$stage'" + exit 1 esac pstree -apgT diff --git a/src/Common/Exception.h b/src/Common/Exception.h index d0de8d6a3f2..314c59cbf51 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -22,10 +22,14 @@ public: Exception() = default; Exception(const std::string & msg, int code); + Exception(int code, const std::string & message) + : Exception(message, code) + {} + // Format message with fmt::format, like the logging functions. - template - Exception(int code, Fmt&&... fmt) - : Exception(fmt::format(std::forward(fmt)...), code) + template + Exception(int code, const std::string & fmt, Args&&... args) + : Exception(fmt::format(fmt, std::forward(args)...), code) {} struct CreateFromPocoTag {}; @@ -39,10 +43,16 @@ public: const char * name() const throw() override { return "DB::Exception"; } const char * what() const throw() override { return message().data(); } - template - void addMessage(Fmt&&... fmt) + /// Add something to the existing message. + template + void addMessage(const std::string& format, Args&&... args) { - extendedMessage(fmt::format(std::forward(fmt)...)); + extendedMessage(fmt::format(format, std::forward(args)...)); + } + + void addMessage(const std::string& message) + { + extendedMessage(message); } std::string getStackTraceString() const; From 6802db6954e1ce36a6c1eea997f47f7331c139a9 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Mon, 12 Oct 2020 09:59:35 +0300 Subject: [PATCH 061/325] Database or Table Engine descrition template upd --- docs/README.md | 2 +- ...ate-table-engine.md => template-engine.md} | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) rename docs/_description_templates/{template-table-engine.md => template-engine.md} (59%) diff --git a/docs/README.md b/docs/README.md index c7fa0221726..8b3066501bf 100644 --- a/docs/README.md +++ b/docs/README.md @@ -195,7 +195,7 @@ Templates: - [Function](_description_templates/template-function.md) - [Setting](_description_templates/template-setting.md) -- [Table engine](_description_templates/template-table-engine.md) +- [Database or Table engine](_description_templates/template-engine.md) - [System table](_description_templates/template-system-table.md) diff --git a/docs/_description_templates/template-table-engine.md b/docs/_description_templates/template-engine.md similarity index 59% rename from docs/_description_templates/template-table-engine.md rename to docs/_description_templates/template-engine.md index c1bfcb3ec86..35181881134 100644 --- a/docs/_description_templates/template-table-engine.md +++ b/docs/_description_templates/template-engine.md @@ -1,8 +1,14 @@ # EngineName {#enginename} -- What the engine does. +- What the Database/Table engine does. - Relations with other engines if they exist. +## Creating a Database {#creating-a-database} +``` sql + CREATE DATABASE ... +``` +or + ## Creating a Table {#creating-a-table} ``` sql CREATE TABLE ... @@ -10,12 +16,19 @@ **Engine Parameters** -**Query Clauses** +**Query Clauses** (for Table engines only) -## Virtual columns {#virtual-columns} +## Virtual columns {#virtual-columns} (for Table engines only) List and virtual columns with description, if they exist. +## Data Types Support {#data_types-support} (for Database engines only) + +| EngineName | ClickHouse | +|-----------------------|------------------------------------| +| NativeDataTypeName | [ClickHouseDataTypeName](link#) | + + ## Specifics and recommendations {#specifics-and-recommendations} Algorithms From 20ebd4fd5bde96ff52ba2fa680d1b6a12b08191b Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Mon, 12 Oct 2020 19:37:04 +0300 Subject: [PATCH 062/325] better --- .../AggregateFunctionWelchTTest.cpp | 26 +-- .../AggregateFunctionWelchTTest.h | 205 +++++------------- .../0_stateless/01322_welch_ttest.reference | 8 +- .../queries/0_stateless/01322_welch_ttest.sql | 19 +- 4 files changed, 81 insertions(+), 177 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 3d2e98e2a0e..d9fce97680c 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -21,41 +21,25 @@ namespace DB namespace { -AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, - const DataTypes & argument_types, - const Array & parameters) +AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, const DataTypes & argument_types, const Array & parameters) { assertBinary(name, argument_types); - - // default value - Float64 significance_level = 0.1; - if (parameters.size() > 1) - { - throw Exception("Aggregate function " + name + " requires one parameter or less.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - } - - if (!parameters.empty()) - { - significance_level = applyVisitor(FieldVisitorConvertToNumber(), parameters[0]); - } + assertNoParameters(name, parameters); AggregateFunctionPtr res; if (isDecimal(argument_types[0]) || isDecimal(argument_types[1])) { - throw Exception("Aggregate function " + name + " only supports numerical types.", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::NOT_IMPLEMENTED); } - else { - res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, - argument_types, parameters)); + res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], argument_types)); } - if (!res) { - throw Exception("Aggregate function " + name + " only supports numerical types.", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::NOT_IMPLEMENTED); } return res; diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 2f56e5e6b6c..e445278e9e7 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -14,6 +14,7 @@ #include #include #include +#include #include @@ -24,38 +25,10 @@ extern const int BAD_ARGUMENTS; namespace DB { -// hard-codded values - part of the algorithm -#define SIGN_LVL_CNT 6 - -Float64 CriticalValuesTable[SIGN_LVL_CNT][102] = { - // for significance level = 0.2 - {0.2, 3.078, 1.886, 1.638, 1.533, 1.476, 1.44, 1.415, 1.397, 1.383, 1.372, 1.363, 1.356, 1.35, 1.345, 1.341, 1.337, 1.333, 1.33, 1.328, 1.325, 1.323, 1.321, 1.319, 1.318, 1.316, 1.315, 1.314, 1.313, 1.311, 1.31, 1.309, 1.309, 1.308, 1.307, 1.306, 1.306, 1.305, 1.304, 1.304, 1.303, 1.303, 1.302, 1.302, 1.301, 1.301, 1.3, 1.3, 1.299, 1.299, 1.299, 1.298, 1.298, 1.298, 1.297, 1.297, 1.297, 1.297, 1.296, 1.296, 1.296, 1.296, 1.295, 1.295, 1.295, 1.295, 1.295, 1.294, 1.294, 1.294, 1.294, 1.294, 1.293, 1.293, 1.293, 1.293, 1.293, 1.293, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.29, 1.29, 1.29, 1.29, 1.29, 1.282}, - - // for significance level = 0.1 - {0.1, 6.314, 2.92, 2.353, 2.132, 2.015, 1.943, 1.895, 1.86, 1.833, 1.812, 1.796, 1.782, 1.771, 1.761, 1.753, 1.746, 1.74, 1.734, 1.729, 1.725, 1.721, 1.717, 1.714, 1.711, 1.708, 1.706, 1.703, 1.701, 1.699, 1.697, 1.696, 1.694, 1.692, 1.691, 1.69, 1.688, 1.687, 1.686, 1.685, 1.684, 1.683, 1.682, 1.681, 1.68, 1.679, 1.679, 1.678, 1.677, 1.677, 1.676, 1.675, 1.675, 1.674, 1.674, 1.673, 1.673, 1.672, 1.672, 1.671, 1.671, 1.67, 1.67, 1.669, 1.669, 1.669, 1.668, 1.668, 1.668, 1.667, 1.667, 1.667, 1.666, 1.666, 1.666, 1.665, 1.665, 1.665, 1.665, 1.664, 1.664, 1.664, 1.664, 1.663, 1.663, 1.663, 1.663, 1.663, 1.662, 1.662, 1.662, 1.662, 1.662, 1.661, 1.661, 1.661, 1.661, 1.661, 1.661, 1.66, 1.66, 1.645}, - - // for significance level = 0.05 - {0.05, 12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145, 2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, 2.045, 2.042, 2.04, 2.037, 2.035, 2.032, 2.03, 2.028, 2.026, 2.024, 2.023, 2.021, 2.02, 2.018, 2.017, 2.015, 2.014, 2.013, 2.012, 2.011, 2.01, 2.009, 2.008, 2.007, 2.006, 2.005, 2.004, 2.003, 2.002, 2.002, 2.001, 2.0, 2.0, 1.999, 1.998, 1.998, 1.997, 1.997, 1.996, 1.995, 1.995, 1.994, 1.994, 1.993, 1.993, 1.993, 1.992, 1.992, 1.991, 1.991, 1.99, 1.99, 1.99, 1.989, 1.989, 1.989, 1.988, 1.988, 1.988, 1.987, 1.987, 1.987, 1.986, 1.986, 1.986, 1.986, 1.985, 1.985, 1.985, 1.984, 1.984, 1.984, 1.96}, - - // for significance level = 0.02 - {0.02, 31.821, 6.965, 4.541, 3.747, 3.365, 3.143, 2.998, 2.896, 2.821, 2.764, 2.718, 2.681, 2.65, 2.624, 2.602, 2.583, 2.567, 2.552, 2.539, 2.528, 2.518, 2.508, 2.5, 2.492, 2.485, 2.479, 2.473, 2.467, 2.462, 2.457, 2.453, 2.449, 2.445, 2.441, 2.438, 2.434, 2.431, 2.429, 2.426, 2.423, 2.421, 2.418, 2.416, 2.414, 2.412, 2.41, 2.408, 2.407, 2.405, 2.403, 2.402, 2.4, 2.399, 2.397, 2.396, 2.395, 2.394, 2.392, 2.391, 2.39, 2.389, 2.388, 2.387, 2.386, 2.385, 2.384, 2.383, 2.382, 2.382, 2.381, 2.38, 2.379, 2.379, 2.378, 2.377, 2.376, 2.376, 2.375, 2.374, 2.374, 2.373, 2.373, 2.372, 2.372, 2.371, 2.37, 2.37, 2.369, 2.369, 2.368, 2.368, 2.368, 2.367, 2.367, 2.366, 2.366, 2.365, 2.365, 2.365, 2.364, 2.326}, - - // for significance level = 0.01 - {0.01, 63.657, 9.925, 5.841, 4.604, 4.032, 3.707, 3.499, 3.355, 3.25, 3.169, 3.106, 3.055, 3.012, 2.977, 2.947, 2.921, 2.898, 2.878, 2.861, 2.845, 2.831, 2.819, 2.807, 2.797, 2.787, 2.779, 2.771, 2.763, 2.756, 2.75, 2.744, 2.738, 2.733, 2.728, 2.724, 2.719, 2.715, 2.712, 2.708, 2.704, 2.701, 2.698, 2.695, 2.692, 2.69, 2.687, 2.685, 2.682, 2.68, 2.678, 2.676, 2.674, 2.672, 2.67, 2.668, 2.667, 2.665, 2.663, 2.662, 2.66, 2.659, 2.657, 2.656, 2.655, 2.654, 2.652, 2.651, 2.65, 2.649, 2.648, 2.647, 2.646, 2.645, 2.644, 2.643, 2.642, 2.641, 2.64, 2.64, 2.639, 2.638, 2.637, 2.636, 2.636, 2.635, 2.634, 2.634, 2.633, 2.632, 2.632, 2.631, 2.63, 2.63, 2.629, 2.629, 2.628, 2.627, 2.627, 2.626, 2.626, 2.576}, - - // for significance level = 0.002 - {0.002, 318.313, 22.327, 10.215, 7.173, 5.893, 5.208, 4.782, 4.499, 4.296, 4.143, 4.024, 3.929, 3.852, 3.787, 3.733, 3.686, 3.646, 3.61, 3.579, 3.552, 3.527, 3.505, 3.485, 3.467, 3.45, 3.435, 3.421, 3.408, 3.396, 3.385, 3.375, 3.365, 3.356, 3.348, 3.34, 3.333, 3.326, 3.319, 3.313, 3.307, 3.301, 3.296, 3.291, 3.286, 3.281, 3.277, 3.273, 3.269, 3.265, 3.261, 3.258, 3.255, 3.251, 3.248, 3.245, 3.242, 3.239, 3.237, 3.234, 3.232, 3.229, 3.227, 3.225, 3.223, 3.22, 3.218, 3.216, 3.214, 3.213, 3.211, 3.209, 3.207, 3.206, 3.204, 3.202, 3.201, 3.199, 3.198, 3.197, 3.195, 3.194, 3.193, 3.191, 3.19, 3.189, 3.188, 3.187, 3.185, 3.184, 3.183, 3.182, 3.181, 3.18, 3.179, 3.178, 3.177, 3.176, 3.175, 3.175, 3.174, 3.09} -}; - -// our algorithm implementation via vectors: -// https://gist.github.com/ltybc-coder/792748cfdb2f7cadef424ffb7b011c71 -// col, col, bool template -//template struct AggregateFunctionWelchTTestData final { - size_t size_x = 0; size_t size_y = 0; X sum_x = static_cast(0); @@ -65,25 +38,6 @@ struct AggregateFunctionWelchTTestData final Float64 mean_x = static_cast(0); Float64 mean_y = static_cast(0); - /* - not yet sure how to use them - void add_x(X x) - { - mean_x = (Float64)(sum_x + x) / (size_x + 1); - size_x ++; - sum_x += x; - square_sum_x += x * x; - } - - void add_y(Y y) - { - mean_y = (sum_y + y) / (size_y + 1); - size_y ++; - sum_y += y; - square_sum_y += y * y; - } - */ - void add(X x, Y y) { sum_x += x; @@ -142,100 +96,80 @@ struct AggregateFunctionWelchTTestData final return size_x; } - Float64 getSx() const + Float64 getSxSquared() const { - return static_cast(square_sum_x + size_x * mean_x * mean_x - 2 * mean_x * sum_x) / (size_x - 1); + /// The original formulae looks like \frac{1}{size_x - 1} \sum_{i = 1}^{size_x}{(x_i - \bar{x}) ^ 2} + /// But we made some mathematical transformations not to store original sequences. + /// Also we dropped sqrt, because later it will be squared later. + return static_cast(square_sum_x + size_x * std::pow(mean_x, 2) - 2 * mean_x * sum_x) / (size_x - 1); } - Float64 getSy() const + Float64 getSySquared() const { - return static_cast(square_sum_y + size_y * mean_y * mean_y - 2 * mean_y * sum_y) / (size_y - 1); + /// The original formulae looks like \frac{1}{size_y - 1} \sum_{i = 1}^{size_y}{(y_i - \bar{y}) ^ 2} + /// But we made some mathematical transformations not to store original sequences. + /// Also we dropped sqrt, because later it will be squared later. + return static_cast(square_sum_y + size_y * std::pow(mean_y, 2) - 2 * mean_y * sum_y) / (size_y - 1); } - Float64 getT(Float64 sx, Float64 sy) const + Float64 getTStatisticSquared() const { - if (sx == 0 && sy == 0) + if (size_x == 0 || size_y == 0) { - throw Exception("division by zero encountered in Aggregate function WelchTTest", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Division by zero encountered in Aggregate function WelchTTest", ErrorCodes::BAD_ARGUMENTS); } - if (sx == -sy && size_x == size_y) - { - throw Exception("division by zero encountered in Aggregate function WelchTTest", ErrorCodes::BAD_ARGUMENTS); - } - - return static_cast(mean_x - mean_y) / std::sqrt(sx / size_x + sy / size_y); + return std::pow(mean_x - mean_y, 2) / (getSxSquared() / size_x + getSySquared() / size_y); } - Float64 getDegreesOfFreedom(Float64 sx, Float64 sy) const + Float64 getDegreesOfFreedom() const { - return static_cast(sx / size_x + sy / size_y) * (sx / size_x + sy / size_y) / - ((sx * sx / (size_x * size_x * (size_x - 1))) + (sy * sy / (size_y * size_y * (size_y - 1)))); + auto sx = getSxSquared(); + auto sy = getSySquared(); + Float64 numerator = std::pow(sx / size_x + sy / size_y, 2); + Float64 denominator_first = std::pow(sx, 2) / (std::pow(size_x, 2) * (size_x - 1)); + Float64 denominator_second = std::pow(sy, 2) / (std::pow(size_y, 2) * (size_y - 1)); + return numerator / (denominator_first + denominator_second); } - UInt8 getResult(Float64 t, Float64 dof, Float64 parametr) const + static Float64 integrateSimpson(Float64 a, Float64 b, std::function func, size_t iterations = 1e6) { - //find our table - int table = 0; - for (int i = 0; i < SIGN_LVL_CNT; ++i) - { - if (CriticalValuesTable[i][0] == parametr) - { - table = i; - } - } + double h = (b - a) / iterations; + Float64 sum_odds = 0.0; + for (size_t i = 1; i < iterations; i += 2) + sum_odds += func(a + i * h); + Float64 sum_evens = 0.0; + for (size_t i = 2; i < iterations; i += 2) + sum_evens += func(a + i * h); + return (func(a) + func(b) + 2 * sum_evens + 4 * sum_odds) * h / 3; + } - //round or make infinity dof - int i_dof = static_cast(dof); + Float64 getPValue() const + { + const Float64 v = getDegreesOfFreedom(); + const Float64 t = getTStatisticSquared(); + auto f = [&v] (double x) { return std::pow(x, v/2 - 1) / std::sqrt(1 - x); }; + Float64 numenator = integrateSimpson(0, v / (t + v), f); + Float64 denominator = std::exp(std::lgammal(v/2) + std::lgammal(0.5) - std::lgammal(v/2 + 0.5)); + return numenator / denominator; + } - if (i_dof > 100) - { - i_dof = 101; - } - - if (i_dof < 1) - { - i_dof = 1; - } - - //check if abs of t is greater than table[dof] - t = abs(t); - if (t >= CriticalValuesTable[table][i_dof]) - { - return static_cast(0); - //in this case we reject the null hypothesis - } - else - { - return static_cast(1); - } + Float64 getResult() const + { + return getPValue(); } }; +/// Returns p-value template -class AggregateFunctionWelchTTest : public - IAggregateFunctionDataHelper< - AggregateFunctionWelchTTestData, - AggregateFunctionWelchTTest - > +class AggregateFunctionWelchTTest : + public IAggregateFunctionDataHelper,AggregateFunctionWelchTTest> { -private: - Float64 significance_level; - public: - AggregateFunctionWelchTTest( - Float64 sglvl_, - const DataTypes & arguments, - const Array & params - ): - IAggregateFunctionDataHelper< - AggregateFunctionWelchTTestData, - AggregateFunctionWelchTTest - > ({arguments}, params), significance_level(sglvl_) - { - // notice: arguments has been in factory - } + AggregateFunctionWelchTTest(const DataTypes & arguments) + : IAggregateFunctionDataHelper, AggregateFunctionWelchTTest> ({arguments}, {}) + {} String getName() const override { @@ -244,15 +178,10 @@ public: DataTypePtr getReturnType() const override { - return std::make_shared>(); + return std::make_shared>(); } - void add( - AggregateDataPtr place, - const IColumn ** columns, - size_t row_num, - Arena * - ) const override + void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override { auto col_x = assert_cast *>(columns[0]); auto col_y = assert_cast *>(columns[1]); @@ -263,34 +192,22 @@ public: this->data(place).add(x, y); } - void merge( - AggregateDataPtr place, - ConstAggregateDataPtr rhs, Arena * - ) const override + void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override { this->data(place).merge(this->data(rhs)); } - void serialize( - ConstAggregateDataPtr place, - WriteBuffer & buf - ) const override + void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override { this->data(place).serialize(buf); } - void deserialize( - AggregateDataPtr place, - ReadBuffer & buf, Arena * - ) const override + void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override { this->data(place).deserialize(buf); } - void insertResultInto( - AggregateDataPtr place, - IColumn & to - ) const override + void insertResultInto(AggregateDataPtr place, IColumn & to, Arena * /*arena*/) const override { size_t size_x = this->data(place).getSizeX(); size_t size_y = this->data(place).getSizeY(); @@ -300,14 +217,8 @@ public: throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS); } - Float64 sx = this->data(place).getSx(); - Float64 sy = this->data(place).getSy(); - Float64 t_value = this->data(place).getT(sx, sy); - Float64 dof = this->data(place).getDegreesOfFreedom(sx, sy); - UInt8 result = this->data(place).getResult(t_value, dof, significance_level); - - auto & column = static_cast &>(to); - column.getData().push_back(result); + auto & column = static_cast &>(to); + column.getData().push_back(this->data(place).getResult()); } }; diff --git a/tests/queries/0_stateless/01322_welch_ttest.reference b/tests/queries/0_stateless/01322_welch_ttest.reference index aa47d0d46d4..015dd503b7e 100644 --- a/tests/queries/0_stateless/01322_welch_ttest.reference +++ b/tests/queries/0_stateless/01322_welch_ttest.reference @@ -1,2 +1,6 @@ -0 -0 +0.021378001462867 +0.021378 +0.090773324285671 +0.09077332 +0.00339907162713746 +0.00339907 diff --git a/tests/queries/0_stateless/01322_welch_ttest.sql b/tests/queries/0_stateless/01322_welch_ttest.sql index b8e881a069b..073e71f69fe 100644 --- a/tests/queries/0_stateless/01322_welch_ttest.sql +++ b/tests/queries/0_stateless/01322_welch_ttest.sql @@ -1,13 +1,18 @@ DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; - -INSERT INTO welch_ttest VALUES (2224.779, 2465.0984), (2588.11, 1909.0328), (1979.625, 1175.8747), (2137.442, 2171.378), (2565.818, 2193.2821), (1754.023, 2854.9475), (1654.947, 2060.1777), (1789.256, 2258.2366), (2320.659, 1856.0535), (2039.532, 1501.8126), (1983.497, 2987.6542), (2232.903, 1681.9778), (2513.93, 2479.6776), (2066.382, 1259.8584), (2492.715, 1120.9043), (1988.287, 1982.1213), (1840.036, 3012.3949), (2249.749, 2252.373), (1766.982, 2591.3122), (1724.84, 1940.589), (0, 1995.185), (0, 2535.1344), (0, 597.3155), (0, 2343.2192), (0, 3154.84), (0, 1125.1966), (0, 1227.8842), (0, 1692.805), (0, 2539.6772), (0, 1936.1927), (0, 1783.7795), (0, 1703.4384), (0, 2077.194), (0, 1614.4071), (0, 2360.0365), (0, 1619.2781), (0, 2033.5109), (0, 2333.7834), (0, 2144.0485), (0, 2583.8709), (0, 1116.7213), (0, 1601.9383), (0, 1570.0431), (0, 1963.0777), (0, 1639.2533), (0, 2277.5223), (0, 1991.9286), (0, 2044.3338), (0, 1794.4781), (0, 1597.9119) -SELECT WelchTTest(0.1)(left, right) from welch_ttest; - +INSERT INTO welch_ttest VALUES (27.5,27.1), (21.0,22.0), (19.0,20.8), (23.6,23.4), (17.0,23.4), (17.9,23.5), (16.9,25.8), (20.1,22.0), (21.9,24.8), (22.6,20.2), (23.1,21.9), (19.6,22.1), (19.0,22.9), (21.7,20.5), (21.4,24.4); +SELECT '0.021378001462867'; +SELECT roundBankers(WelchTTest(left, right), 8) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; + CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; +INSERT INTO welch_ttest VALUES (30.02,29.89), (29.99,29.93), (30.11,29.72), (29.97,29.98), (30.01,30.02), (29.99,29.98); +SELECT '0.090773324285671'; +SELECT roundBankers(WelchTTest(left, right), 8) from welch_ttest; +DROP TABLE IF EXISTS welch_ttest; -INSERT INTO welch_ttest VALUES (2224.779, 2465.0984), (2588.11, 1909.0328), (1979.625, 1175.8747), (2137.442, 2171.378), (2565.818, 2193.2821), (1754.023, 2854.9475), (1654.947, 2060.1777), (1789.256, 2258.2366), (2320.659, 1856.0535), (2039.532, 1501.8126), (1983.497, 2987.6542), (2232.903, 1681.9778), (2513.93, 2479.6776), (2066.382, 1259.8584), (2492.715, 1120.9043), (1988.287, 1982.1213), (1840.036, 3012.3949), (2249.749, 2252.373), (1766.982, 2591.3122), (1724.84, 1940.589), (0, 1995.185), (0, 2535.1344), (0, 597.3155), (0, 2343.2192), (0, 3154.84), (0, 1125.1966), (0, 1227.8842), (0, 1692.805), (0, 2539.6772), (0, 1936.1927), (0, 1783.7795), (0, 1703.4384), (0, 2077.194), (0, 1614.4071), (0, 2360.0365), (0, 1619.2781), (0, 2033.5109), (0, 2333.7834), (0, 2144.0485), (0, 2583.8709), (0, 1116.7213), (0, 1601.9383), (0, 1570.0431), (0, 1963.0777), (0, 1639.2533), (0, 2277.5223), (0, 1991.9286), (0, 2044.3338), (0, 1794.4781), (0, 1597.9119) -SELECT WelchTTest(0.02)(left, right) from welch_ttest; - +CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; +INSERT INTO welch_ttest VALUES (0.010268,0.159258), (0.000167,0.136278), (0.000167,0.122389); +SELECT '0.00339907162713746'; +SELECT roundBankers(WelchTTest(left, right), 8) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; \ No newline at end of file From e65a2a1cbd6d784e8a0307f9d84e0dde9c241e16 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Mon, 12 Oct 2020 21:10:01 +0300 Subject: [PATCH 063/325] add student t test --- .../AggregateFunctionStudentTTest.cpp | 56 +++++ .../AggregateFunctionStudentTTest.h | 230 ++++++++++++++++++ .../registerAggregateFunctions.cpp | 1 + .../registerAggregateFunctions.h | 1 + 4 files changed, 288 insertions(+) create mode 100644 src/AggregateFunctions/AggregateFunctionStudentTTest.cpp create mode 100644 src/AggregateFunctions/AggregateFunctionStudentTTest.h diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp new file mode 100644 index 00000000000..b6f32409946 --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp @@ -0,0 +1,56 @@ +#include +#include +#include +#include "registerAggregateFunctions.h" + +#include +#include + + +// the return type is boolean (we use UInt8 as we do not have boolean in clickhouse) + +namespace ErrorCodes +{ +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int NOT_IMPLEMENTED; +} + +namespace DB +{ + +namespace +{ + +AggregateFunctionPtr createAggregateFunctionStudentTTest(const std::string & name, const DataTypes & argument_types, const Array & parameters) +{ + assertBinary(name, argument_types); + assertNoParameters(name, parameters); + + AggregateFunctionPtr res; + + if (isDecimal(argument_types[0]) || isDecimal(argument_types[1])) + { + throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::NOT_IMPLEMENTED); + } + else + { + res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], argument_types)); + } + + if (!res) + { + throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::NOT_IMPLEMENTED); + } + + return res; +} + +} + + +void registerAggregateFunctionStudentTTest(AggregateFunctionFactory & factory) +{ + factory.registerFunction("StudentTTest", createAggregateFunctionStudentTTest, AggregateFunctionFactory::CaseInsensitive); +} + +} diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.h b/src/AggregateFunctions/AggregateFunctionStudentTTest.h new file mode 100644 index 00000000000..b03f9178709 --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.h @@ -0,0 +1,230 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} + +namespace DB +{ + +template +struct AggregateFunctionStudentTTestData final +{ + size_t size_x = 0; + size_t size_y = 0; + X sum_x = static_cast(0); + Y sum_y = static_cast(0); + X square_sum_x = static_cast(0); + Y square_sum_y = static_cast(0); + Float64 mean_x = static_cast(0); + Float64 mean_y = static_cast(0); + + void add(X x, Y y) + { + sum_x += x; + sum_y += y; + size_x++; + size_y++; + mean_x = static_cast(sum_x) / size_x; + mean_y = static_cast(sum_y) / size_y; + square_sum_x += x * x; + square_sum_y += y * y; + } + + void merge(const AggregateFunctionStudentTTestData &other) + { + sum_x += other.sum_x; + sum_y += other.sum_y; + size_x += other.size_x; + size_y += other.size_y; + mean_x = static_cast(sum_x) / size_x; + mean_y = static_cast(sum_y) / size_y; + square_sum_x += other.square_sum_x; + square_sum_y += other.square_sum_y; + } + + void serialize(WriteBuffer &buf) const + { + writeBinary(mean_x, buf); + writeBinary(mean_y, buf); + writeBinary(sum_x, buf); + writeBinary(sum_y, buf); + writeBinary(square_sum_x, buf); + writeBinary(square_sum_y, buf); + writeBinary(size_x, buf); + writeBinary(size_y, buf); + } + + void deserialize(ReadBuffer &buf) + { + readBinary(mean_x, buf); + readBinary(mean_y, buf); + readBinary(sum_x, buf); + readBinary(sum_y, buf); + readBinary(square_sum_x, buf); + readBinary(square_sum_y, buf); + readBinary(size_x, buf); + readBinary(size_y, buf); + } + + size_t getSizeY() const + { + return size_y; + } + + size_t getSizeX() const + { + return size_x; + } + + Float64 getSSquared() const + { + /// TODO: Update comment with Tex. + /// The original formulae looks like ... + /// But we made some mathematical transformations not to store original sequences. + /// Also we dropped sqrt, because later it will be squared later. + const Float64 all_x = square_sum_x + size_x * std::pow(mean_x, 2) - 2 * mean_x * sum_x; + const Float64 all_y = square_sum_y + size_y * std::pow(mean_y, 2) - 2 * mean_y * sum_y; + return static_cast(all_x + all_y) / (size_x + size_y - 2); + } + + + Float64 getTStatisticSquared() const + { + if (size_x == 0 || size_y == 0) + { + throw Exception("Division by zero encountered in Aggregate function StudentTTest", ErrorCodes::BAD_ARGUMENTS); + } + + if (mean_x - mean_y < 1e-8) + { + return static_cast(0.0); + } + + return std::pow(mean_x - mean_y, 2) / getStandartErrorSquared(); + } + + + Float64 getStandartErrorSquared() const + { + return getSSquared() * (1 / size_x + 1 / size_y); + } + + Float64 getDegreesOfFreedom() const + { + return static_cast(size_x + size_y - 2); + } + + static Float64 integrateSimpson(Float64 a, Float64 b, std::function func, size_t iterations = 1e6) + { + double h = (b - a) / iterations; + Float64 sum_odds = 0.0; + for (size_t i = 1; i < iterations; i += 2) + sum_odds += func(a + i * h); + Float64 sum_evens = 0.0; + for (size_t i = 2; i < iterations; i += 2) + sum_evens += func(a + i * h); + return (func(a) + func(b) + 2 * sum_evens + 4 * sum_odds) * h / 3; + } + + Float64 getPValue() const + { + const Float64 v = getDegreesOfFreedom(); + const Float64 t = getTStatisticSquared(); + std::cout << "getDegreesOfFreedom " << v << " getTStatisticSquared " << t << std::endl; + auto f = [&v] (double x) { return std::pow(x, v/2 - 1) / std::sqrt(1 - x); }; + Float64 numenator = integrateSimpson(0, v / (t + v), f); + Float64 denominator = std::exp(std::lgammal(v/2) + std::lgammal(0.5) - std::lgammal(v/2 + 0.5)); + return numenator / denominator; + } + + Float64 getResult() const + { + return getPValue(); + } +}; + +/// Returns p-value +/// https://cpb-us-w2.wpmucdn.com/voices.uchicago.edu/dist/9/1193/files/2016/01/05b-TandP.pdf +template +class AggregateFunctionStudentTTest : + public IAggregateFunctionDataHelper,AggregateFunctionStudentTTest> +{ + +public: + AggregateFunctionStudentTTest(const DataTypes & arguments) + : IAggregateFunctionDataHelper, AggregateFunctionStudentTTest> ({arguments}, {}) + {} + + String getName() const override + { + return "StudentTTest"; + } + + DataTypePtr getReturnType() const override + { + return std::make_shared>(); + } + + void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override + { + auto col_x = assert_cast *>(columns[0]); + auto col_y = assert_cast *>(columns[1]); + + X x = col_x->getData()[row_num]; + Y y = col_y->getData()[row_num]; + + this->data(place).add(x, y); + } + + void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override + { + this->data(place).merge(this->data(rhs)); + } + + void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override + { + this->data(place).serialize(buf); + } + + void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override + { + this->data(place).deserialize(buf); + } + + void insertResultInto(AggregateDataPtr place, IColumn & to, Arena * /*arena*/) const override + { + size_t size_x = this->data(place).getSizeX(); + size_t size_y = this->data(place).getSizeY(); + + if (size_x < 2 || size_y < 2) + { + throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS); + } + + auto & column = static_cast &>(to); + column.getData().push_back(this->data(place).getResult()); + } + +}; + +}; diff --git a/src/AggregateFunctions/registerAggregateFunctions.cpp b/src/AggregateFunctions/registerAggregateFunctions.cpp index fd914443ba2..9fd02ba9d6c 100644 --- a/src/AggregateFunctions/registerAggregateFunctions.cpp +++ b/src/AggregateFunctions/registerAggregateFunctions.cpp @@ -46,6 +46,7 @@ void registerAggregateFunctions() registerAggregateFunctionCategoricalIV(factory); registerAggregateFunctionAggThrow(factory); registerAggregateFunctionWelchTTest(factory); + registerAggregateFunctionStudentTTest(factory); registerAggregateFunctionRankCorrelation(factory); } diff --git a/src/AggregateFunctions/registerAggregateFunctions.h b/src/AggregateFunctions/registerAggregateFunctions.h index de239258fa0..abbba56ed32 100644 --- a/src/AggregateFunctions/registerAggregateFunctions.h +++ b/src/AggregateFunctions/registerAggregateFunctions.h @@ -36,6 +36,7 @@ void registerAggregateFunctionMoving(AggregateFunctionFactory &); void registerAggregateFunctionCategoricalIV(AggregateFunctionFactory &); void registerAggregateFunctionAggThrow(AggregateFunctionFactory &); void registerAggregateFunctionWelchTTest(AggregateFunctionFactory &); +void registerAggregateFunctionStudentTTest(AggregateFunctionFactory &); void registerAggregateFunctionRankCorrelation(AggregateFunctionFactory &); class AggregateFunctionCombinatorFactory; From dbaada559782898e6a992b048e9f6fc58df22f2a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 15:15:47 +0300 Subject: [PATCH 064/325] Another test (cherry picked from commit da87861285e63369bd79e176ce375a8d6ea18b85) --- .../01502_log_tinylog_deadlock_race.reference | 6 ++ .../01502_log_tinylog_deadlock_race.sh | 85 +++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference create mode 100755 tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh diff --git a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference new file mode 100644 index 00000000000..4bf85ae79f3 --- /dev/null +++ b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference @@ -0,0 +1,6 @@ +Testing TinyLog +Done TinyLog +Testing StripeLog +Done StripeLog +Testing Log +Done Log diff --git a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh new file mode 100755 index 00000000000..a5b2ff6db8f --- /dev/null +++ b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +set -e + +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + + +function thread_create { + while true; do + $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS $1 (x UInt64, s Array(Nullable(String))) ENGINE = $2" + sleep 0.0$RANDOM + done +} + +function thread_drop { + while true; do + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $1" + sleep 0.0$RANDOM + done +} + +function thread_rename { + while true; do + $CLICKHOUSE_CLIENT --query "RENAME TABLE $1 TO $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' + sleep 0.0$RANDOM + done +} + +function thread_select { + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM $1 FORMAT Null" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done +} + +function thread_insert { + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT rand64(1), [toString(rand64(2))] FROM numbers($2)" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done +} + +function thread_insert_select { + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT * FROM $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done +} + +export -f thread_create +export -f thread_drop +export -f thread_rename +export -f thread_select +export -f thread_insert +export -f thread_insert_select + + +# Do randomized queries and expect nothing extraordinary happens. + +function test_with_engine { + echo "Testing $1" + + timeout 10 bash -c "thread_create t1 $1" & + timeout 10 bash -c "thread_create t2 $1" & + timeout 10 bash -c 'thread_drop t1' & + timeout 10 bash -c 'thread_drop t2' & + timeout 10 bash -c 'thread_rename t1 t2' & + timeout 10 bash -c 'thread_rename t2 t1' & + timeout 10 bash -c 'thread_select t1' & + timeout 10 bash -c 'thread_select t2' & + timeout 10 bash -c 'thread_insert t1 5' & + timeout 10 bash -c 'thread_insert t2 10' & + timeout 10 bash -c 'thread_insert_select t1 t2' & + timeout 10 bash -c 'thread_insert_select t2 t1' & + + wait + echo "Done $1" +} + +test_with_engine TinyLog +test_with_engine StripeLog +test_with_engine Log From b3fc6d9b3385a47cc9553c32f1181797b64486e2 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 13 Oct 2020 16:31:02 +0300 Subject: [PATCH 065/325] fix IF [NOT] EXISTS failure --- src/Interpreters/InterpreterDropQuery.cpp | 24 ++++++++----------- src/Interpreters/InterpreterDropQuery.h | 2 +- .../01502_log_tinylog_deadlock_race.reference | 8 ++----- .../01502_log_tinylog_deadlock_race.sh | 7 +++--- 4 files changed, 17 insertions(+), 24 deletions(-) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index c70431e5238..a9bc738f614 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -47,7 +47,7 @@ BlockIO InterpreterDropQuery::execute() if (!drop.table.empty()) { if (!drop.is_dictionary) - return executeToTable({drop.database, drop.table, drop.uuid}, drop); + return executeToTable(drop); else return executeToDictionary(drop.database, drop.table, drop.kind, drop.if_exists, drop.temporary, drop.no_ddl_lock); } @@ -58,29 +58,25 @@ BlockIO InterpreterDropQuery::execute() } -BlockIO InterpreterDropQuery::executeToTable( - const StorageID & table_id_, - const ASTDropQuery & query) +BlockIO InterpreterDropQuery::executeToTable(const ASTDropQuery & query) { - if (query.temporary || table_id_.database_name.empty()) + auto table_id = StorageID(query); + if (query.temporary || table_id.database_name.empty()) { - if (context.tryResolveStorageID(table_id_, Context::ResolveExternal)) - return executeToTemporaryTable(table_id_.getTableName(), query.kind); + if (context.tryResolveStorageID(table_id, Context::ResolveExternal)) + return executeToTemporaryTable(table_id.getTableName(), query.kind); + else + table_id.database_name = context.getCurrentDatabase(); } if (query.temporary) { if (query.if_exists) return {}; - throw Exception("Temporary table " + backQuoteIfNeed(table_id_.table_name) + " doesn't exist", + throw Exception("Temporary table " + backQuoteIfNeed(table_id.table_name) + " doesn't exist", ErrorCodes::UNKNOWN_TABLE); } - auto table_id = query.if_exists ? context.tryResolveStorageID(table_id_, Context::ResolveOrdinary) - : context.resolveStorageID(table_id_, Context::ResolveOrdinary); - if (!table_id) - return {}; - auto ddl_guard = (!query.no_ddl_lock ? DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name) : nullptr); /// If table was already dropped by anyone, an exception will be thrown @@ -255,7 +251,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next()) { query.table = iterator->name(); - executeToTable({query.database, query.table}, query); + executeToTable(query); } } diff --git a/src/Interpreters/InterpreterDropQuery.h b/src/Interpreters/InterpreterDropQuery.h index b54736b5c21..c5d9aacdfd5 100644 --- a/src/Interpreters/InterpreterDropQuery.h +++ b/src/Interpreters/InterpreterDropQuery.h @@ -31,7 +31,7 @@ private: BlockIO executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists); - BlockIO executeToTable(const StorageID & table_id, const ASTDropQuery & query); + BlockIO executeToTable(const ASTDropQuery & query); BlockIO executeToDictionary(const String & database_name, const String & dictionary_name, ASTDropQuery::Kind kind, bool if_exists, bool is_temporary, bool no_ddl_lock); diff --git a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference index 4bf85ae79f3..c62a2b18918 100644 --- a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference +++ b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference @@ -1,6 +1,2 @@ -Testing TinyLog -Done TinyLog -Testing StripeLog -Done StripeLog -Testing Log -Done Log +Testing Memory +Done Memory diff --git a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh index a5b2ff6db8f..f0b5f0a3568 100755 --- a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh +++ b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh @@ -80,6 +80,7 @@ function test_with_engine { echo "Done $1" } -test_with_engine TinyLog -test_with_engine StripeLog -test_with_engine Log +#test_with_engine TinyLog +#test_with_engine StripeLog +#test_with_engine Log +test_with_engine Memory From cb8d132cca22da7531f345371f75ad6d4e793d61 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 13 Oct 2020 18:00:36 +0300 Subject: [PATCH 066/325] fix deadlock with DDLGuard --- src/Interpreters/DatabaseCatalog.cpp | 27 +++++++++--- src/Interpreters/DatabaseCatalog.h | 4 +- src/Interpreters/InterpreterDropQuery.cpp | 1 + .../0_stateless/01150_ddl_guard_rwr.reference | 0 .../0_stateless/01150_ddl_guard_rwr.sh | 43 +++++++++++++++++++ 5 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/01150_ddl_guard_rwr.reference create mode 100755 tests/queries/0_stateless/01150_ddl_guard_rwr.sh diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 203e2292c08..03f0e057821 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -530,7 +530,7 @@ std::unique_ptr DatabaseCatalog::getDDLGuard(const String & database, std::unique_lock lock(ddl_guards_mutex); auto db_guard_iter = ddl_guards.try_emplace(database).first; DatabaseGuard & db_guard = db_guard_iter->second; - return std::make_unique(db_guard.first, db_guard.second, std::move(lock), table); + return std::make_unique(db_guard.first, db_guard.second, std::move(lock), table, database); } std::unique_lock DatabaseCatalog::getExclusiveDDLGuardForDatabase(const String & database) @@ -832,7 +832,7 @@ void DatabaseCatalog::waitTableFinallyDropped(const UUID & uuid) } -DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock guards_lock_, const String & elem) +DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock guards_lock_, const String & elem, const String & database_name) : map(map_), db_mutex(db_mutex_), guards_lock(std::move(guards_lock_)) { it = map.emplace(elem, Entry{std::make_unique(), 0}).first; @@ -841,14 +841,19 @@ DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_locksecond.mutex); bool is_database = elem.empty(); if (!is_database) - db_mutex.lock_shared(); + { + + bool locked_database_for_read = db_mutex.try_lock_shared(); + if (!locked_database_for_read) + { + removeTableLock(); + throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} is currently dropped or renamed", database_name); + } + } } -DDLGuard::~DDLGuard() +void DDLGuard::removeTableLock() { - bool is_database = it->first.empty(); - if (!is_database) - db_mutex.unlock_shared(); guards_lock.lock(); --it->second.counter; if (!it->second.counter) @@ -858,4 +863,12 @@ DDLGuard::~DDLGuard() } } +DDLGuard::~DDLGuard() +{ + bool is_database = it->first.empty(); + if (!is_database) + db_mutex.unlock_shared(); + removeTableLock(); +} + } diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 7bc6923bde4..c6f50117564 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -51,7 +51,7 @@ public: /// NOTE: using std::map here (and not std::unordered_map) to avoid iterator invalidation on insertion. using Map = std::map; - DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock guards_lock_, const String & elem); + DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock guards_lock_, const String & elem, const String & database_name); ~DDLGuard(); private: @@ -60,6 +60,8 @@ private: Map::iterator it; std::unique_lock guards_lock; std::unique_lock table_lock; + + void removeTableLock(); }; diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index a9bc738f614..b29f2893db9 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -247,6 +247,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS ASTDropQuery query; query.kind = kind; + query.if_exists = true; query.database = database_name; for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next()) { diff --git a/tests/queries/0_stateless/01150_ddl_guard_rwr.reference b/tests/queries/0_stateless/01150_ddl_guard_rwr.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01150_ddl_guard_rwr.sh b/tests/queries/0_stateless/01150_ddl_guard_rwr.sh new file mode 100755 index 00000000000..c14e4c38f54 --- /dev/null +++ b/tests/queries/0_stateless/01150_ddl_guard_rwr.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS test_01150" +$CLICKHOUSE_CLIENT --query "CREATE DATABASE test_01150" + +$CLICKHOUSE_CLIENT --query "CREATE TABLE test_01150.t1 (x UInt64, s Array(Nullable(String))) ENGINE = Memory" +$CLICKHOUSE_CLIENT --query "CREATE TABLE test_01150.t2 (x UInt64, s Array(Nullable(String))) ENGINE = Memory" + +function thread_detach_attach { + while true; do + $CLICKHOUSE_CLIENT --query "DETACH DATABASE test_01150" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (219)' + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "ATTACH DATABASE test_01150" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (82)' + sleep 0.0$RANDOM + done +} + +function thread_rename { + while true; do + $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t1 TO test_01150.t2_tmp, test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (81|60|57|521)' + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (81|60|57|521)' + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (81|60|57|521)' + sleep 0.0$RANDOM + done +} + +export -f thread_detach_attach +export -f thread_rename + +timeout 20 bash -c "thread_detach_attach" & +timeout 20 bash -c 'thread_rename' & +wait +sleep 1 + +$CLICKHOUSE_CLIENT --query "ATTACH DATABASE IF NOT EXISTS test_01150" +$CLICKHOUSE_CLIENT --query "DROP DATABASE test_01150"; From 8a64b65e51713d429bd744ca7288ef858041a2bd Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 13 Oct 2020 20:45:59 +0300 Subject: [PATCH 067/325] fix --- src/Interpreters/InterpreterDropQuery.cpp | 2 ++ .../0_stateless/01516_drop_table_stress.sh | 18 +++++++----------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index b29f2893db9..a250ab1afd4 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -88,6 +88,8 @@ BlockIO InterpreterDropQuery::executeToTable(const ASTDropQuery & query) if (query_ptr->as().is_view && !table->isView()) throw Exception("Table " + table_id.getNameForLogs() + " is not a View", ErrorCodes::LOGICAL_ERROR); + table_id = table->getStorageID(); + if (query.kind == ASTDropQuery::Kind::Detach) { context.checkAccess(table->isView() ? AccessType::DROP_VIEW : AccessType::DROP_TABLE, table_id); diff --git a/tests/queries/0_stateless/01516_drop_table_stress.sh b/tests/queries/0_stateless/01516_drop_table_stress.sh index 3d6218c4549..3e2fd613a36 100755 --- a/tests/queries/0_stateless/01516_drop_table_stress.sh +++ b/tests/queries/0_stateless/01516_drop_table_stress.sh @@ -12,21 +12,17 @@ function drop_database() function drop_table() { - ${CLICKHOUSE_CLIENT} -nm <&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db_01516.data1;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db_01516.data2;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" } function create() { - ${CLICKHOUSE_CLIENT} -nm <&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS db_01516.data2 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS db_01516.data3 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" } for _ in {1..100}; do From 744013d4b8e1624c528697b594982d3b696345d6 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Tue, 13 Oct 2020 21:46:15 +0300 Subject: [PATCH 068/325] test in comparison with scipy --- .../AggregateFunctionStudentTTest.h | 56 ++++++++++++++----- .../AggregateFunctionWelchTTest.h | 43 ++++++++++++-- src/AggregateFunctions/ya.make | 4 +- .../0_stateless/01322_student_ttest.reference | 4 ++ .../0_stateless/01322_student_ttest.sql | 19 +++++++ .../0_stateless/01322_welch_ttest.reference | 10 +++- .../queries/0_stateless/01322_welch_ttest.sql | 27 +++++++-- 7 files changed, 137 insertions(+), 26 deletions(-) create mode 100644 tests/queries/0_stateless/01322_student_ttest.reference create mode 100644 tests/queries/0_stateless/01322_student_ttest.sql diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.h b/src/AggregateFunctions/AggregateFunctionStudentTTest.h index b03f9178709..ac05a11d334 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.h +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.h @@ -115,18 +115,22 @@ struct AggregateFunctionStudentTTestData final throw Exception("Division by zero encountered in Aggregate function StudentTTest", ErrorCodes::BAD_ARGUMENTS); } - if (mean_x - mean_y < 1e-8) - { - return static_cast(0.0); - } - return std::pow(mean_x - mean_y, 2) / getStandartErrorSquared(); } + Float64 getTStatistic() const + { + if (size_x == 0 || size_y == 0) + { + throw Exception("Division by zero encountered in Aggregate function StudentTTest", ErrorCodes::BAD_ARGUMENTS); + } + + return (mean_x - mean_y) / std::sqrt(getStandartErrorSquared()); + } Float64 getStandartErrorSquared() const { - return getSSquared() * (1 / size_x + 1 / size_y); + return getSSquared() * (1.0 / static_cast(size_x) + 1.0 / static_cast(size_y)); } Float64 getDegreesOfFreedom() const @@ -150,20 +154,23 @@ struct AggregateFunctionStudentTTestData final { const Float64 v = getDegreesOfFreedom(); const Float64 t = getTStatisticSquared(); - std::cout << "getDegreesOfFreedom " << v << " getTStatisticSquared " << t << std::endl; + std::cout << "getDegreesOfFreedom() " << getDegreesOfFreedom() << std::endl; + std::cout << "getTStatisticSquared() " << getTStatisticSquared() << std::endl; auto f = [&v] (double x) { return std::pow(x, v/2 - 1) / std::sqrt(1 - x); }; Float64 numenator = integrateSimpson(0, v / (t + v), f); Float64 denominator = std::exp(std::lgammal(v/2) + std::lgammal(0.5) - std::lgammal(v/2 + 0.5)); + std::cout << "numenator " << numenator << std::endl; + std::cout << "denominator " << denominator << std::endl; return numenator / denominator; } - Float64 getResult() const + std::pair getResult() const { - return getPValue(); + return std::make_pair(getTStatistic(), getPValue()); } }; -/// Returns p-value +/// Returns tuple of (t-statistic, p-value) /// https://cpb-us-w2.wpmucdn.com/voices.uchicago.edu/dist/9/1193/files/2016/01/05b-TandP.pdf template class AggregateFunctionStudentTTest : @@ -182,7 +189,22 @@ public: DataTypePtr getReturnType() const override { - return std::make_shared>(); + DataTypes types + { + std::make_shared>(), + std::make_shared>(), + }; + + Strings names + { + "t-statistic", + "p-value" + }; + + return std::make_shared( + std::move(types), + std::move(names) + ); } void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override @@ -221,8 +243,16 @@ public: throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS); } - auto & column = static_cast &>(to); - column.getData().push_back(this->data(place).getResult()); + Float64 t_statistic = 0.0; + Float64 p_value = 0.0; + std::tie(t_statistic, p_value) = this->data(place).getResult(); + + auto & column_tuple = assert_cast(to); + auto & column_stat = assert_cast &>(column_tuple.getColumn(0)); + auto & column_value = assert_cast &>(column_tuple.getColumn(1)); + + column_stat.getData().push_back(t_statistic); + column_value.getData().push_back(p_value); } }; diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index e445278e9e7..36641b826b1 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -122,6 +122,16 @@ struct AggregateFunctionWelchTTestData final return std::pow(mean_x - mean_y, 2) / (getSxSquared() / size_x + getSySquared() / size_y); } + Float64 getTStatistic() const + { + if (size_x == 0 || size_y == 0) + { + throw Exception("Division by zero encountered in Aggregate function WelchTTest", ErrorCodes::BAD_ARGUMENTS); + } + + return (mean_x - mean_y) / std::sqrt(getSxSquared() / size_x + getSySquared() / size_y); + } + Float64 getDegreesOfFreedom() const { auto sx = getSxSquared(); @@ -154,9 +164,9 @@ struct AggregateFunctionWelchTTestData final return numenator / denominator; } - Float64 getResult() const + std::pair getResult() const { - return getPValue(); + return std::make_pair(getTStatistic(), getPValue()); } }; @@ -178,7 +188,22 @@ public: DataTypePtr getReturnType() const override { - return std::make_shared>(); + DataTypes types + { + std::make_shared>(), + std::make_shared>(), + }; + + Strings names + { + "t-statistic", + "p-value" + }; + + return std::make_shared( + std::move(types), + std::move(names) + ); } void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override @@ -217,8 +242,16 @@ public: throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS); } - auto & column = static_cast &>(to); - column.getData().push_back(this->data(place).getResult()); + Float64 t_statistic = 0.0; + Float64 p_value = 0.0; + std::tie(t_statistic, p_value) = this->data(place).getResult(); + + auto & column_tuple = assert_cast(to); + auto & column_stat = assert_cast &>(column_tuple.getColumn(0)); + auto & column_value = assert_cast &>(column_tuple.getColumn(1)); + + column_stat.getData().push_back(t_statistic); + column_value.getData().push_back(p_value); } }; diff --git a/src/AggregateFunctions/ya.make b/src/AggregateFunctions/ya.make index f5a869b2f78..1578e0c80ea 100644 --- a/src/AggregateFunctions/ya.make +++ b/src/AggregateFunctions/ya.make @@ -42,6 +42,7 @@ SRCS( AggregateFunctionState.cpp AggregateFunctionStatistics.cpp AggregateFunctionStatisticsSimple.cpp + AggregateFunctionStudentTTest.cpp AggregateFunctionSum.cpp AggregateFunctionSumMap.cpp AggregateFunctionTimeSeriesGroupSum.cpp @@ -49,12 +50,13 @@ SRCS( AggregateFunctionUniqCombined.cpp AggregateFunctionUniq.cpp AggregateFunctionUniqUpTo.cpp + AggregateFunctionWelchTTest.cpp AggregateFunctionWindowFunnel.cpp parseAggregateFunctionParameters.cpp registerAggregateFunctions.cpp UniqCombinedBiasData.cpp UniqVariadicHash.cpp - AggregateFunctionWelchTTest.cpp + ) END() diff --git a/tests/queries/0_stateless/01322_student_ttest.reference b/tests/queries/0_stateless/01322_student_ttest.reference new file mode 100644 index 00000000000..02e44744629 --- /dev/null +++ b/tests/queries/0_stateless/01322_student_ttest.reference @@ -0,0 +1,4 @@ +-2.610898982580138 0.00916587538237954 +-2.610898982580134 0.0091658753823792 +-28.740781574102936 7.667329672103986e-133 +-28.74078157410298 0 diff --git a/tests/queries/0_stateless/01322_student_ttest.sql b/tests/queries/0_stateless/01322_student_ttest.sql new file mode 100644 index 00000000000..3636e239fe8 --- /dev/null +++ b/tests/queries/0_stateless/01322_student_ttest.sql @@ -0,0 +1,19 @@ +DROP TABLE IF EXISTS student_ttest; + +/*Check t-stat and p-value and compare it with scipy.stat implementation + First: a=1, sigma (not sigma^2)=5, size=500 + Second: a=1, sigma = 5, size = 500 */ +CREATE TABLE student_ttest (left Float64, right Float64) ENGINE = Memory; +INSERT INTO student_ttest VALUES (0.88854,-2.90702), (-5.76966,3.61651), (6.76618,4.27458), (3.55546,4.82133), (-9.76948,9.59483), (4.92323,1.00424), (-0.36352,2.04147), (0.97018,-3.58214), (4.61656,6.59543), (-6.78292,-1.00532), (4.02008,-3.59794), (12.41838,-2.82434), (5.14417,-3.13194), (3.86836,9.90977), (-1.26199,0.523), (12.44106,4.62779), (3.28349,-2.56872), (1.77261,2.25807), (-8.94748,1.04044), (-1.01449,-2.35744), (-1.26377,10.81531), (6.79682,-9.68469), (6.32333,3.80885), (-8.21214,12.70435), (-1.68565,-6.01112), (9.7557,1.89065), (3.66694,5.08892), (1.39967,3.45254), (-5.52035,11.58151), (-10.95601,0.85035), (0.93877,8.38397), (1.45933,1.17169), (-5.40551,4.74621), (-0.83857,-1.66614), (8.50794,4.2414), (-6.68686,1.68765), (5.03099,1.85223), (1.56251,9.10111), (4.17381,-2.38085), (-2.92644,-14.79595), (5.11068,-3.8938), (2.09617,-3.41864), (11.7787,-3.15282), (6.50336,-0.56684), (0.62098,12.87997), (-7.97121,6.89115), (3.81902,12.921), (0.33151,-7.94908), (10.68584,2.45687), (0.56007,2.14957), (-7.38621,7.55081), (5.05882,-3.71534), (2.34616,-2.41064), (11.3806,-0.80734), (5.95276,-4.75651), (-3.01429,2.05241), (5.98169,-5.44523), (0.96985,-2.75054), (-1.15932,-13.00131), (2.11547,-2.74451), (2.49668,-1.39004), (-12.49569,-3.02854), (-4.94667,7.65112), (-3.64215,1.1245), (-8.35595,6.74117), (3.211,-0.75777), (2.33805,8.93451), (2.38608,-8.85559), (-3.2862,-0.36405), (-0.80454,4.02742), (-0.53483,6.88718), (10.66445,-1.05124), (-0.37619,3.04085), (0.48246,3.32368), (7.41919,1.147), (0.42414,3.41554), (-2.32335,-3.47851), (-0.70223,-0.47684), (-5.9332,-0.55605), (-1.20561,-0.17006), (3.39865,2.26218), (9.61739,12.45494), (-0.78651,-1.84097), (-4.00256,1.64934), (-7.99646,-7.07496), (8.72923,-9.99462), (0.71859,6.09954), (-1.62726,-1.05319), (5.11234,3.04757), (-0.95625,0.93899), (-3.75573,-4.63243), (1.03141,-7.43322), (-3.33588,-7.298), (1.51804,-6.59016), (-3.30935,-6.11649), (-1.97507,0.56682), (4.06456,2.00661), (3.27195,-2.79814), (-7.81761,2.84482), (-3.81785,3.65348), (-4.18311,-4.22807), (-11.33313,-4.54336), (-0.25221,-3.63343), (7.2514,2.96878), (5.30301,6.11661), (2.46762,-1.70919), (4.22716,-4.71133), (0.33916,6.09652), (9.7638,-6.83454), (-7.58684,0.18006), (-4.09888,1.51676), (4.26617,-5.31646), (-0.56744,-3.21215), (4.65125,-5.07599), (-1.30301,-2.36591), (4.53771,3.55724), (9.96929,4.8904), (3.72939,-3.22586), (-2.29818,-1.74928), (3.09417,5.73458), (0.82251,1.41188), (5.29975,2.86255), (2.8685,2.90179), (-5.73321,-2.19949), (-1.85651,1.72727), (-1.07984,1.76939), (9.78342,-0.12848), (-13.49652,-0.52), (3.68791,3.48333), (1.9998,7.8262), (1.11674,0.09099), (9.43869,7.77017), (4.07029,9.49484), (5.32715,1.42825), (7.16504,1.99624), (6.66096,4.00419), (-5.7111,1.07925), (-0.38575,-0.09987), (4.49165,-5.48733), (-3.36489,-1.83517), (7.71814,2.38059), (-1.58966,1.42075), (-1.61063,-1.11968), (-0.91602,-6.46035), (0.73459,7.66576), (-3.24463,4.6307), (6.3947,5.55989), (-2.77845,3.16684), (4.45899,5.07671), (-8.84186,-10.20566), (2.62276,-4.73386), (1.774,1.28353), (4.3692,6.75679), (0.05942,12.09895), (-1.44042,7.0049), (-2.53594,7.16156), (-2.24752,-0.64311), (4.98874,-0.66747), (4.05434,3.99996), (-2.56483,9.07298), (-6.79286,-4.60971), (-2.06165,0.70744), (-0.26056,2.56774), (1.89567,9.32424), (-3.15145,3.95087), (-7.31321,7.11372), (0.28936,-0.89284), (-0.63111,8.6155), (0.22611,-0.14141), (-9.3377,-4.86319), (-5.76638,-6.95801), (3.87306,4.44883), (6.7011,4.6156), (9.03915,-2.3579), (-1.21835,-5.1186), (0.82892,8.12819), (2.80656,2.78392), (-1.34746,-4.30221), (-1.99912,-1.47506), (0.6036,6.8598), (-3.46117,0.47636), (5.23732,0.95383), (-1.86702,7.79779), (-5.86115,-2.61767), (6.48523,-10.5087), (-7.40158,-2.74299), (-1.38913,3.87369), (4.94613,-1.07093), (-2.07818,4.98864), (2.39808,-7.50772), (4.89238,6.41316), (4.39481,1.39061), (5.20425,-3.1747), (13.62598,-2.13621), (-2.86293,-0.02203), (-3.62396,0.89025), (-4.28695,-5.87746), (4.66425,3.60026), (2.20871,-0.23178), (1.60382,-2.1897), (-9.87024,-5.85101), (-7.37302,-1.6053), (-4.17814,3.6184), (2.5148,-8.53795), (3.21708,-0.35987), (-11.48089,2.15301), (1.19821,-6.60692), (-0.07436,9.54341), (-1.10652,1.11511), (4.03395,2.94025), (-4.35883,12.05657), (2.04013,3.75156), (0.52264,7.95597), (8.14004,-0.99449), (-8.86949,0.90597), (-0.35807,-7.90627), (-10.71113,3.50863), (-2.13755,-1.47493), (0.50715,4.11671), (6.30826,10.06325), (2.37527,-1.06059), (0.20872,-1.37737), (-5.85729,-0.42542), (-4.97217,-3.90267), (-9.78434,9.35037), (-1.53277,-7.91219), (0.14827,-4.69945), (-1.053,3.63776), (1.74558,3.46492), (11.17194,2.84518), (9.35487,-3.04301), (-9.17209,8.82764), (10.41814,7.80134), (7.41206,7.87755), (3.71775,7.01035), (-2.04674,2.43271), (6.18037,11.36418), (5.6383,-6.92659), (-0.90058,5.95541), (-1.27073,3.59436), (-2.3473,5.18429), (-8.44271,4.20225), (2.75551,0.5029), (-1.15521,4.03074), (4.08722,5.23152), (-1.70399,10.65409), (7.24114,-0.69845), (-8.43976,11.70096), (-1.53052,5.80692), (-0.00526,-8.1819), (-4.04813,4.31485), (-2.84299,5.7227), (-5.201,5.67398), (7.75774,-1.75826), (-2.85791,7.54164), (-3.86071,-1.79026), (-1.80029,-1.7395), (-5.26015,5.65042), (-3.158,0.38765), (7.71014,-4.64719), (-4.84866,-10.22048), (-8.38785,-2.05447), (7.67021,-2.43441), (4.96521,-5.38551), (-0.40919,5.47764), (-3.25711,8.26637), (3.07685,-3.6421), (2.89376,-11.66269), (-10.47331,3.972), (-3.48942,5.46642), (1.13906,-3.72304), (-8.57454,5.75251), (-3.38963,5.12841), (-2.3195,0.59067), (-1.60694,5.21138), (-5.57406,-4.58702), (-0.93075,-8.737), (-11.76579,-2.12737), (10.68283,0.22888), (8.74324,-1.46448), (7.66409,2.40311), (4.76715,-5.21814), (0.44539,13.94749), (-1.35941,-2.77448), (4.18849,-3.7867), (-6.17097,3.4954), (0.27977,3.12586), (-1.45006,-7.01485), (-4.81694,-3.20727), (-3.0297,6.31415), (0.02145,2.37521), (2.46883,8.13787), (9.60317,2.15956), (-9.93898,-0.40842), (1.05549,-7.27283), (5.55366,4.27575), (-3.80722,-2.89126), (-4.18851,6.84344), (1.00351,7.0869), (3.11385,-5.18837), (-5.17623,2.67648), (-3.18396,-6.57021), (-6.65302,0.60429), (-0.50832,-1.04921), (-4.04375,7.12873), (4.52707,1.68973), (6.63124,-2.58404), (-3.72082,-3.83114), (5.79825,-7.26546), (-2.0158,-5.07153), (-2.78369,-0.80395), (-1.91821,2.09455), (6.31714,4.33374), (-1.80869,8.54335), (8.55586,0.80566), (2.40826,-8.38085), (-8.46361,7.54812), (5.04452,8.78007), (-0.84665,1.5857), (2.30903,8.43855), (-3.71837,-1.90846), (-0.69419,-1.2434), (3.6733,7.16172), (-1.96098,-3.44129), (2.36747,-6.37542), (-12.03622,-4.99486), (4.38481,4.99033), (2.93955,-1.83734), (2.16804,-2.83289), (-0.08218,-4.13997), (-3.97934,1.40163), (-7.43985,8.57867), (0.91666,-1.87639), (7.23432,3.41667), (-6.13303,6.31762), (-10.23217,1.58473), (-6.21681,1.63625), (-0.80934,-6.93618), (0.17914,3.58046), (2.13338,-6.8097), (6.97656,4.69978), (6.90455,-1.72912), (6.25943,5.29491), (-6.04019,-1.63062), (-7.30909,5.83818), (1.4589,17.0769), (12.00208,4.54301), (2.22457,-1.33801), (-2.45912,5.64339), (-6.92213,1.26913), (4.05547,-1.01553), (0.04709,4.8316), (-7.70952,3.08635), (-1.47883,-2.27738), (1.3701,-1.13761), (-4.92928,10.08698), (-2.75872,5.33827), (-0.09178,2.84345), (2.62642,-1.51132), (-1.14623,13.46078), (2.76609,8.58965), (4.94404,-2.36683), (-7.01764,-1.8217), (-10.91568,1.96981), (-2.49738,2.31718), (0.73576,3.66493), (2.25436,1.93104), (-1.72956,5.20332), (2.41054,3.20519), (5.72149,3.34631), (-6.41371,7.0087), (3.38217,-7.96126), (1.24133,-0.62182), (10.03634,-4.65227), (-2.37303,10.6572), (-1.35543,4.50891), (-1.4387,9.74298), (-4.0976,3.85707), (-0.82501,6.41144), (-1.93498,1.48649), (5.59955,2.28076), (5.46656,2.75342), (2.43568,-5.40401), (-0.23926,7.11389), (-4.9945,5.74368), (-4.96655,6.78345), (-0.59258,3.83773), (2.02497,0.70959), (0.67583,0.57434), (3.16522,1.5888), (-1.9673,3.94889), (-6.75319,5.8234), (-6.69723,7.78366), (0.81148,9.08354), (4.44531,-7.99182), (-4.43522,-2.77033), (-5.28602,-10.29342), (-3.58829,1.76251), (-7.97395,2.09266), (-2.84891,4.20614), (-3.95112,-3.63064), (3.54945,-2.17794), (12.12376,-2.66225), (-3.12347,-2.74707), (3.65209,-1.93431), (9.34031,1.38629), (-0.26348,4.12816), (-5.23968,-1.58902), (2.22336,-5.08864), (-10.70405,-2.30491), (-4.41319,2.64605), (-5.94912,1.16158), (1.8147,2.63534), (7.69287,1.4956), (9.46125,-4.60768), (4.72497,0.60771), (-0.57565,3.29549), (-1.12303,-1.42592), (2.90272,0.8883), (-4.4584,-1.10612), (4.28819,-2.57296), (11.64512,5.88085), (-1.80395,7.40745), (2.51605,13.48116), (-3.18439,5.53539), (-0.70213,-1.46014), (-7.68383,3.73304), (-8.32268,3.5435), (-8.71115,-3.89151), (9.96933,4.16265), (0.95675,2.32663), (3.35114,5.31735), (-2.66008,6.33485), (7.75456,2.1339), (0.73568,0.82708), (0.3483,-2.95155), (-1.09203,-6.76019), (-7.76963,-4.20179), (5.81902,8.78354), (-3.41424,1.41863), (-0.39209,7.65689), (4.67608,-6.52601), (0.68753,-4.4426), (5.17179,-4.49483), (4.98983,-3.91479), (-0.12659,-2.84562), (3.25267,2.58974), (1.50184,2.24424), (2.94507,-4.65846), (-0.42333,8.4062), (-3.66227,8.20262), (8.90812,-8.63752), (4.74411,4.97966), (2.22018,-0.35563), (-2.07976,-4.72116), (4.8711,-2.95997), (0.5023,2.73959), (6.31569,-0.23956), (-4.36903,10.13915), (3.82146,11.83775), (-6.99477,-2.50332), (3.61225,-0.58181), (14.69335,-7.62836), (0.58368,2.26478), (4.65341,-3.50179), (-3.14272,-2.08023), (2.67048,4.07256), (4.64963,-1.40826), (-2.70828,-2.33644), (1.42923,3.00197), (5.84498,4.23668), (-4.76568,-2.24647), (0.19907,1.0445), (1.67486,-0.31901), (5.32145,8.62657), (-8.03477,3.92817), (3.46776,0.08462), (4.66374,10.15884), (-5.37394,0.4113), (5.39045,4.45847), (-1.44756,5.82941), (-1.64419,6.59202), (3.39699,-3.73441), (-2.94659,-5.86969), (-2.38437,-4.56543), (-0.23958,-1.32636), (6.88389,-0.17884), (-2.7172,-3.56181), (-1.53419,-0.66932), (7.38841,6.87538), (-5.44178,0.73527), (-0.89287,-0.24177), (2.93546,-0.8657), (-0.26901,-0.22977), (-4.70044,1.02095), (2.25846,6.16311), (-9.28813,-5.68027), (6.04268,-3.7619), (4.41693,4.22959), (1.75714,-1.5249); +SELECT '-2.610898982580138', '0.00916587538237954'; +SELECT roundBankers(StudentTTest(left, right).1, 16) as t_stat, roundBankers(StudentTTest(left, right).2, 16) as p_value from student_ttest; +DROP TABLE IF EXISTS student_ttest; + +/*Check t-stat and p-value and compare it with scipy.stat implementation + First: a=1, sigma (not sigma^2)=5, size=500 + Second: a=1, sigma = 5, size = 500 */ +CREATE TABLE student_ttest (left Float64, right Float64) ENGINE = Memory; +INSERT INTO student_ttest VALUES (4.52546,8.69444), (3.73628,3.81414), (-0.39478,12.38442), (5.15633,8.9738), (0.50539,9.19594), (-5.34036,7.21009), (0.19336,4.97743), (8.35729,4.94756), (6.95818,19.80911), (-2.93812,13.75358), (8.30807,16.56373), (-3.3517,9.72882), (4.16279,4.64509), (-3.17231,17.76854), (1.93545,4.80693), (11.06606,8.79505), (-4.22678,10.88868), (-1.99975,6.21932), (-4.51178,15.11614), (-4.50711,13.24703), (1.89786,14.76476), (-6.19638,-0.6117), (-3.70188,17.48993), (5.01334,12.11847), (1.79036,4.87439), (2.14435,18.56479), (3.0282,1.23712), (2.35528,5.41596), (-12.18535,4.54994), (5.59709,11.37668), (-12.92336,9.5982), (-0.04281,6.59822), (-0.16923,1.16703), (0.88924,8.88418), (-4.68414,10.95047), (8.01099,5.52787), (2.61686,-1.11647), (-2.76895,14.49946), (3.32165,3.27585), (-0.85135,-0.42025), (1.21368,6.37906), (4.38673,2.5242), (6.20964,8.1405), (-1.23172,6.46732), (4.65516,9.89332), (-1.87143,10.4374), (0.86429,-1.06465), (2.51184,6.84902), (-1.88822,10.96576), (-1.61802,7.83319), (1.93653,14.39823), (-3.66631,7.02594), (-1.05294,13.46629), (-10.74718,10.39531), (16.49295,11.27348), (-7.65494,9.32187), (-3.39303,12.32667), (-4.89418,8.98905), (3.2521,9.54757), (0.05831,5.98325), (-3.00409,3.47248), (5.76702,9.26966), (2.67674,5.77816), (10.52623,6.32966), (-0.54501,9.49313), (-4.89835,6.21337), (3.52457,10.00242), (-0.0451,6.25167), (-6.61226,15.64671), (9.02391,2.78968), (5.52571,6.55442), (4.54352,3.68819), (-3.8394,9.55934), (-7.75295,4.166), (5.91167,12.32471), (1.38897,7.10969), (6.24166,16.31723), (5.58536,12.99482), (4.7591,10.11585), (-2.58336,10.29455), (-1.91263,18.27524), (3.31575,12.84435), (5.3507,13.11954), (-15.22081,12.84147), (-0.84775,15.55658), (-4.538,11.45329), (6.71177,7.50912), (0.52882,8.56226), (2.0242,8.63104), (5.69146,15.68026), (4.63328,21.6361), (0.22984,6.23925), (-2.84052,8.65714), (7.91867,9.9423), (1.11001,12.28213), (-0.11251,3.11279), (-0.20905,13.58128), (0.03287,16.51407), (-1.59397,16.60476), (-5.39405,12.02022), (-7.1233,12.11035), (4.51517,9.47832), (-0.70967,6.40742), (5.67299,8.87252), (-0.33835,15.14265), (-1.83047,2.23572), (-0.62877,11.57144), (-7.23148,18.87737), (0.1802,12.1833), (11.73325,11.17519), (2.17603,16.80422), (-0.11683,6.81423), (-1.29102,12.12546), (-0.23201,8.06153), (-6.8643,10.97228), (-6.85153,7.30596), (-4.77163,15.44026), (6.11721,8.00993), (5.96406,12.60196), (3.59135,13.96832), (-0.60095,14.03207), (3.11163,4.53758), (-0.18831,8.08297), (0.67657,4.90451), (-3.16117,8.14253), (0.26957,19.88605), (2.18653,13.85254), (-5.94611,23.01839), (-4.39352,6.02084), (-3.71525,9.60319), (5.11103,1.90511), (1.33998,10.35237), (1.01629,16.27082), (-3.36917,12.52379), (-3.99661,11.37435), (8.19336,13.61823), (2.89168,15.77622), (-11.10373,15.17254), (11.68005,6.711), (3.08282,4.74205), (-6.81506,10.09812), (-2.34587,6.61722), (-2.68725,10.34164), (0.3577,8.96602), (-3.05682,12.32157), (9.08062,11.75711), (-0.77913,13.49499), (10.35215,8.57713), (6.82565,11.50313), (-1.24674,1.13097), (5.18822,7.83205), (-3.70743,5.77957), (1.40319,15.5519), (5.89432,10.82676), (1.43152,11.51218), (6.70638,9.29779), (9.76613,9.77021), (4.27604,9.94114), (-2.63141,15.54513), (-7.8133,19.10736), (-0.06668,15.04205), (1.05391,9.03114), (4.41797,24.0104), (0.09337,9.94205), (6.16075,2.5925), (7.49413,8.82726), (-3.52872,10.0209), (-2.17126,8.1635), (-3.87605,4.24074), (3.26607,7.67291), (-3.28045,5.21642), (2.1429,11.2808), (1.53386,6.88172), (0.21169,5.98743), (-0.63674,17.97249), (5.84893,6.46323), (-0.63498,15.37416), (8.29526,2.89957), (-1.08358,17.13044), (-2.306,11.06355), (2.86991,3.09625), (-0.76074,-2.33019), (5.49191,7.42675), (1.82883,15.06792), (-3.70497,8.81116), (-0.53232,19.17446), (-11.49722,18.77181), (3.44877,14.06443), (-1.8596,12.81241), (-10.34851,2.72299), (1.13093,18.67739), (-10.93389,11.63275), (-3.39703,2.23891), (0.19749,13.01195), (-3.68389,7.43402), (-4.67863,8.14599), (10.78916,16.65328), (0.37675,1.362), (3.98094,3.87957), (-3.64775,11.16134), (-4.8443,6.25357), (1.102,4.21945), (8.72112,12.50047), (-1.47361,6.45486), (6.24183,18.99924), (6.83569,18.09508), (-3.11684,13.59528), (4.91306,3.39681), (-0.03628,13.33157), (5.1282,5.8945), (-2.38558,5.61212), (2.33351,8.41149), (-0.97191,13.78608), (-0.05588,6.08609), (-4.70019,12.76962), (-5.12371,3.26206), (0.65606,0.25528), (-0.11574,11.9083), (4.4238,4.35071), (6.93399,11.19855), (3.68712,13.87404), (-0.01187,6.87986), (1.8332,8.32566), (5.81322,22.51334), (-4.04709,2.5226), (-8.26397,16.84498), (-2.11273,6.26108), (5.28396,13.84824), (0.73054,6.03262), (6.43559,14.12668), (4.35565,16.01939), (-1.05545,8.19237), (5.00087,18.01595), (-2.72239,9.45609), (7.32313,6.90459), (2.11548,12.83115), (-3.40953,10.603), (6.97051,13.70439), (-0.45567,6.1633), (1.31699,4.1151), (-1.49871,8.20499), (7.14772,11.67903), (0.79277,7.30851), (6.9698,6.50941), (2.08733,7.3949), (-3.55962,12.80075), (0.75601,5.62043), (1.21,18.2542), (-2.17877,17.9393), (1.83206,16.4569), (5.72463,8.78811), (7.42257,4.85949), (0.97829,-3.36394), (7.54238,5.38683), (9.91081,12.26083), (-4.61743,10.27907), (-4.40799,11.5144), (9.99854,11.57335), (8.53725,1.94203), (3.2905,7.78228), (0.38634,11.79385), (-2.53374,10.18415), (4.94758,14.67613), (4.79624,4.70301), (5.57664,12.72151), (-6.44871,-3.35508), (3.34431,17.63775), (0.14209,2.53883), (10.88431,14.01483), (0.31846,12.4387), (-0.54703,11.15408), (-4.67791,7.74882), (-5.68011,13.60956), (-4.93362,7.81991), (1.2271,10.90969), (5.27512,8.19828), (-3.84611,-1.18523), (6.81706,0.5916), (10.33033,0.35805), (5.13979,12.98364), (3.66534,11.38628), (-2.07219,13.94644), (10.65442,2.03781), (-3.31751,10.74447), (-1.82011,12.35656), (-0.39886,7.08701), (1.77052,2.69871), (1.29049,19.66653), (7.92344,7.88636), (-2.92595,10.36916), (-2.67107,1.632), (5.64708,11.86081), (0.34639,13.47602), (-3.04356,6.60204), (3.98828,7.01303), (-1.36695,20.19992), (-8.48462,18.88249), (-4.04669,11.34367), (9.84561,12.97305), (-6.1537,9.5776), (0.82433,17.91364), (1.92449,18.3247), (2.51288,9.9211), (0.40965,7.14257), (2.89183,6.59133), (3.84347,12.35274), (0.66829,10.57523), (-3.45094,12.12859), (1.3544,9.47177), (-9.85456,0.60659), (5.25689,4.72996), (-5.26018,4.51121), (-6.16912,13.28893), (-1.77163,8.09014), (3.96687,8.02511), (0.70893,13.85406), (-5.45342,1.75412), (-3.89706,6.00641), (3.11868,6.35554), (4.41714,7.11293), (7.64841,8.30442), (0.00489,12.63024), (3.2263,12.38966), (-5.33042,7.6801), (2.52189,11.33744), (-7.40308,4.67713), (0.67891,7.62276), (2.49343,2.14478), (5.43133,15.32988), (-0.67541,1.52299), (-0.60299,17.00017), (-6.32903,8.29701), (-3.44336,10.92961), (-0.23963,6.78449), (6.94686,7.02698), (6.59442,11.51719), (-4.18532,9.97926), (-1.8228,7.44251), (-0.29443,7.58541), (2.99821,4.76058), (2.51942,12.88959), (-3.49176,9.974), (-0.57979,17.03689), (8.69471,11.14554), (-1.19427,11.7392), (-3.17119,11.50029), (-2.99566,19.41759), (-3.34493,9.65127), (-2.33826,9.87673), (-5.04164,14.13485), (-0.48214,9.78034), (7.45097,1.57826), (3.04787,3.72091), (2.92632,9.4054), (1.39694,23.22816), (4.38686,-0.12571), (3.25753,6.97343), (7.14218,10.09049), (-4.04341,11.78393), (-9.19352,3.01909), (2.78473,16.09448), (0.33331,6.25485), (9.89238,7.13164), (6.00566,7.75879), (-1.7511,9.56834), (4.77815,6.14824), (5.07457,13.53454), (2.56132,8.26364), (2.38317,8.7095), (-1.63486,10.61607), (-1.46871,10.64418), (-5.8681,23.9106), (-2.96227,11.38978), (-1.90638,11.4383), (-13.3052,18.41498), (-2.14705,3.70959), (-9.62069,19.95918), (2.29313,9.53847), (0.22162,14.04957), (-1.83956,13.70151), (4.1853,5.45046), (6.05965,10.95061), (-0.23737,9.55156), (6.07452,17.92345), (4.34629,6.23976), (4.02922,8.71029), (3.62622,13.58736), (-3.95825,8.78527), (-1.63412,11.14213), (-1.25727,12.23717), (5.06323,16.44557), (-0.66176,0.47144), (2.36606,9.7198), (-5.77792,13.50981), (4.535,14.27806), (1.02031,13.50793), (4.49345,7.47381), (-4.99791,11.07844), (2.46716,9.89844), (3.65471,21.48548), (11.2283,6.92085), (6.69743,4.44074), (-5.60375,19.98074), (0.28683,7.92826), (-0.85737,16.6313), (4.26726,17.17618), (-3.4322,13.80807), (-2.07039,5.37083), (-2.26798,9.73962), (-0.99818,10.66273), (0.41335,8.90639), (5.18124,12.24596), (-5.01858,16.89203), (2.05561,12.69184), (-0.12117,15.59077), (0.99471,6.94287), (6.89979,-0.1801), (-4.18527,3.25318), (-6.35104,8.08804), (3.89734,13.78384), (-1.979,0.46434), (3.15404,7.78224), (3.52672,9.10987), (2.48372,-0.89391), (-6.13089,14.3696), (2.2968,3.01763), (-2.74324,8.03559), (-0.12876,7.24609), (-1.51135,11.86271), (-3.92434,6.28196), (-1.71254,8.9725), (-1.25878,14.46114), (2.03021,9.50216), (4.31726,16.30413), (-3.02908,1.02795), (9.7093,1.88717), (-3.36284,9.80106), (6.70938,4.53487), (0.42762,16.34543), (5.04726,7.71098), (2.78386,2.74639), (6.83022,6.51875), (-3.02109,10.42308), (-0.65382,13.57901), (-15.58675,0.52784), (5.89746,4.4708), (-4.11598,6.39619), (-1.37208,14.57666), (10.08082,2.71602), (5.35686,12.53905), (1.93331,11.4292), (10.47444,12.44641), (-2.36872,14.50894), (6.50752,17.64374), (2.54603,11.03218), (-0.4332,9.82789), (5.26572,10.11104), (2.09016,2.16137), (1.15513,10.24054), (14.95941,12.86909), (-3.85505,15.22845), (-2.36239,5.05411), (1.64338,10.84836), (-4.25074,11.15717), (7.29744,0.91782), (-1.18964,13.29961), (5.60612,15.11314), (-3.77011,11.54004), (6.67642,-0.94238), (-0.06862,19.32581), (5.60514,10.20744), (3.7341,6.54857), (9.59001,8.69108), (3.30093,8.2296), (-2.75658,8.4474), (4.71994,6.81178), (0.74699,5.99415), (2.91095,13.99336), (-7.36829,8.7469), (-5.29487,8.62349), (3.31079,1.84212), (1.06974,4.4762), (-1.18424,9.25421), (-7.415,10.44229), (3.40595,12.21649), (-7.63085,10.45968), (1.13336,15.34722), (-0.0096,5.50868), (0.8928,10.93609), (-0.5943,2.78631), (7.48306,11.86145), (10.11943,18.67385), (5.60459,10.64051), (4.00189,12.75565), (2.35823,6.63666), (0.33475,12.19343), (3.47072,9.08636), (-6.68867,11.67256), (3.31031,20.31392), (2.17159,11.66443); +SELECT -28.740781574102936, 7.667329672103986e-133; +SELECT roundBankers(StudentTTest(left, right).1, 16) as t_stat, roundBankers(StudentTTest(left, right).2, 16) as p_value from student_ttest; +DROP TABLE IF EXISTS student_ttest; diff --git a/tests/queries/0_stateless/01322_welch_ttest.reference b/tests/queries/0_stateless/01322_welch_ttest.reference index 015dd503b7e..d06853a0a5e 100644 --- a/tests/queries/0_stateless/01322_welch_ttest.reference +++ b/tests/queries/0_stateless/01322_welch_ttest.reference @@ -1,6 +1,10 @@ 0.021378001462867 -0.021378 +0.0213780014628671 0.090773324285671 -0.09077332 +0.0907733242891952 0.00339907162713746 -0.00339907 +0.0033990715715539 +-0.5028215369186904 0.6152361677168877 +-0.5028215369187079 0.6152361677170834 +14.971190998235835 5.898143508382202e-44 +14.971190998235837 0 diff --git a/tests/queries/0_stateless/01322_welch_ttest.sql b/tests/queries/0_stateless/01322_welch_ttest.sql index 073e71f69fe..2a045e70b32 100644 --- a/tests/queries/0_stateless/01322_welch_ttest.sql +++ b/tests/queries/0_stateless/01322_welch_ttest.sql @@ -1,18 +1,37 @@ +/*Check only p-value first*/ DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (27.5,27.1), (21.0,22.0), (19.0,20.8), (23.6,23.4), (17.0,23.4), (17.9,23.5), (16.9,25.8), (20.1,22.0), (21.9,24.8), (22.6,20.2), (23.1,21.9), (19.6,22.1), (19.0,22.9), (21.7,20.5), (21.4,24.4); SELECT '0.021378001462867'; -SELECT roundBankers(WelchTTest(left, right), 8) from welch_ttest; +SELECT roundBankers(WelchTTest(left, right).2, 16) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (30.02,29.89), (29.99,29.93), (30.11,29.72), (29.97,29.98), (30.01,30.02), (29.99,29.98); SELECT '0.090773324285671'; -SELECT roundBankers(WelchTTest(left, right), 8) from welch_ttest; +SELECT roundBankers(WelchTTest(left, right).2, 16) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (0.010268,0.159258), (0.000167,0.136278), (0.000167,0.122389); SELECT '0.00339907162713746'; -SELECT roundBankers(WelchTTest(left, right), 8) from welch_ttest; -DROP TABLE IF EXISTS welch_ttest; \ No newline at end of file +SELECT roundBankers(WelchTTest(left, right).2, 16) from welch_ttest; +DROP TABLE IF EXISTS welch_ttest; + +/*Check t-stat and p-value and compare it with scipy.stat implementation + First: a=10, sigma (not sigma^2)=5, size=500 + Second: a=10, sigma = 10, size = 500 */ +CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; +INSERT INTO welch_ttest VALUES (14.72789,-8.65656), (9.61661,22.98234), (13.57615,23.80821), (3.98392,13.33939), (11.98889,-4.05537), (10.99422,23.5155), (5.44792,-6.45272), (20.29346,17.7903), (7.05926,11.463), (9.22732,5.28021), (12.06847,8.39157), (13.52612,6.02464), (8.24597,14.43732), (9.35245,15.76584), (10.12297,1.54391), (15.80624,1.24897), (13.68613,27.1507), (10.72729,7.71091), (5.62078,15.71846), (6.12229,32.97808), (6.03801,-1.79334), (8.95585,-9.23439), (24.04613,11.27838), (9.04757,0.72703), (2.68263,18.51557), (15.43935,9.16619), (2.89423,17.29624), (4.01423,-1.30208), (4.30568,-3.48018), (11.99948,10.12082), (8.40574,-8.01318), (10.86642,-14.22264), (9.4266,16.58174), (-8.12752,-0.55975), (7.91634,5.61449), (7.3967,1.44626), (2.26431,7.89158), (14.20118,1.13369), (6.68233,-0.82609), (15.46221,12.23365), (7.88467,12.45443), (11.20011,14.46915), (8.92027,13.72627), (10.27926,18.41459), (5.14395,29.66702), (5.62178,1.51619), (12.84383,10.40078), (9.98009,3.33266), (-0.69789,6.12036), (11.41386,11.86553), (7.76863,6.59422), (7.21743,22.0948), (1.81176,1.79623), (9.43762,14.29513), (19.22117,19.69162), (2.97128,-7.98033), (14.32851,5.48433), (7.54959,-2.28474), (3.81545,9.91876), (10.1281,10.64097), (2.48596,0.22523), (10.0461,17.01773), (3.59714,22.37388), (9.73522,14.04215), (18.8077,23.1244), (3.15148,18.96958), (12.26062,8.42663), (5.66707,3.7165), (6.58623,14.29366), (17.30902,23.50886), (9.91391,26.33722), (5.36946,26.72396), (15.73637,13.26287), (16.96281,12.97607), (11.54063,17.41838), (18.37358,8.63875), (11.38255,17.08943), (10.53256,23.15356), (8.08833,-4.4965), (16.27556,7.58895), (2.42969,26.04074), (9.56127,6.84245), (7.32998,20.56287), (9.19511,3.84735), (9.66903,-2.76304), (4.15029,13.1615), (8.83511,8.21954), (14.60617,-3.49943), (14.06143,22.12419), (5.39556,7.08323), (10.11871,16.12937), (10.56619,-0.32672), (14.4462,16.5942), (10.42106,7.68977), (7.75551,11.39484), (11.00418,-5.11987), (4.47226,20.87404), (16.35461,8.01007), (18.55174,3.26497), (11.82044,5.61253), (7.39454,20.69182), (11.27767,0.0296), (6.83827,21.904), (7.76858,22.46572), (15.97614,3.63685), (14.53781,-5.10846), (12.99546,14.86389), (16.91151,5.47188), (9.65012,18.44095), (14.25487,16.71368), (14.03618,6.36704), (2.57382,8.82663), (2.50779,14.6727), (14.24787,7.98383), (13.34666,2.65568), (7.31102,21.45827), (10.22981,11.77948), (17.4435,4.71979), (21.2074,3.17951), (6.64191,13.90226), (18.7086,15.50578), (14.78686,10.8026), (9.85287,16.91369), (4.48263,9.90552), (14.17469,13.87322), (14.4342,4.12366), (19.2481,-3.78985), (3.47165,1.7599), (8.28712,3.43715), (8.81657,-3.45246), (0.92319,23.64571), (20.41106,-4.96877), (6.76127,3.93514), (22.00242,1.49914), (8.66129,12.71519), (10.9929,5.11521), (17.95494,4.79872), (17.20996,20.89391), (12.18888,5.363), (12.14257,8.02765), (15.81243,14.30804), (4.43362,11.49002), (1.17567,14.25281), (15.60881,7.6573), (9.34833,15.49686), (6.33513,3.29327), (-0.83095,2.27236), (12.43268,12.58104), (6.63207,19.19128), (11.96877,15.25901), (14.81029,6.5221), (21.84876,10.10965), (3.75896,12.75249), (6.91307,16.50977), (13.73015,-8.6697), (8.63753,8.28553), (15.71679,1.44315), (1.74565,4.65869), (9.16895,0.98149), (5.70685,0.16623), (5.00117,17.66332), (13.06888,4.35346), (7.51204,6.52742), (15.34885,-1.06631), (5.20264,-5.28454), (8.59043,14.25583), (6.45619,8.74058), (14.61979,1.89553), (11.7075,-0.92959), (14.04901,10.30289), (4.20525,-6.3744), (15.1733,-8.1706), (3.12934,10.95369), (8.08049,4.94384), (15.41273,28.40568), (16.90751,3.7004), (5.86893,2.52363), (7.1086,4.07997), (4.418,7.8849), (12.0614,17.95409), (7.07887,16.67021), (3.61585,11.34377), (11.73001,-0.07446), (10.80449,22.00223), (8.40311,3.31778), (9.91276,18.50719), (16.4164,-3.58655), (5.25034,6.5394), (15.20283,12.40459), (10.42909,16.59866), (9.53888,7.54176), (14.68939,-1.51044), (6.60007,12.69758), (18.31058,2.9842), (7.01885,2.49187), (18.71631,2.04113), (10.50002,-2.46544), (10.7517,15.18368), (4.23224,-0.04058), (2.28924,-0.4127), (8.56059,10.5526), (8.25095,12.03982), (9.15673,12.10923), (13.28409,11.54954), (8.4513,-1.18613), (2.83911,11.30984), (2.79676,23.54105), (9.11055,10.67321), (7.18529,24.09196), (-4.1258,7.5008), (5.28306,12.52233), (6.82757,4.30673), (10.89035,9.35793), (5.24822,4.44472), (11.935,-7.00679), (6.45675,8.56241), (10.18088,23.73891), (4.9932,15.62708), (18.09939,16.09205), (8.11738,12.52074), (5.37883,14.58927), (10.50339,-4.80187), (16.64093,8.47964), (14.77263,7.75477), (13.71385,12.6893), (6.98746,7.14147), (10.74635,12.12654), (5.49432,12.32334), (13.46078,7.98909), (10.67565,3.26652), (9.0291,20.53684), (11.51417,32.3369), (13.07118,19.74911), (9.5049,-4.62897), (8.50611,8.26483), (6.47606,20.88451), (13.06526,-2.12982), (19.08658,25.61459), (9.49741,5.32091), (10.60865,-4.1196), (2.28996,7.57937), (8.12846,21.15847), (5.62241,6.46355), (4.07712,7.74846), (17.98526,19.62636), (9.466,28.34629), (11.38904,26.73919), (5.91826,20.40427), (1.52059,3.03378), (18.79161,10.2537), (18.20669,7.47745), (-1.67829,10.79184), (18.01586,3.91962), (16.31577,19.97973), (7.88281,18.87711), (8.46179,12.56157), (10.31113,11.46033), (14.88377,3.78661), (1.31835,-9.45748), (2.53176,12.06033), (9.48625,-0.74615), (3.97936,13.2815), (11.52319,24.78052), (13.24178,5.83337), (7.58739,17.4111), (10.00959,19.70331), (9.73361,11.78446), (8.35716,-1.366), (1.65491,1.37458), (11.11521,16.31483), (6.08355,32.63464), (10.04582,-3.79736), (11.58237,19.17984), (16.40249,-0.27705), (1.9691,-3.69456), (13.22776,28.38058), (2.67059,-1.36876), (9.83651,-25.63301), (2.12539,3.58644), (9.27114,-6.85667), (9.0699,13.42225), (2.78179,12.04671), (12.49311,28.99468), (12.97662,7.87662), (15.06359,2.61119), (16.91565,-3.56022), (5.92011,1.50022), (5.81304,14.55836), (8.46425,9.35831), (9.48705,16.9366), (4.68191,29.23126), (5.70028,15.31386), (-0.78798,13.46112), (10.03442,7.39667), (15.45433,11.15599), (9.43845,9.80499), (3.05825,22.64923), (6.92126,8.67693), (14.05905,18.67335), (19.71579,-3.19127), (15.0131,22.94716), (4.50386,17.86834), (1.31061,16.98267), (10.81197,15.91653), (14.32942,11.79718), (9.26469,18.50208), (7.27679,8.90755), (22.69295,10.44843), (12.03763,4.67433), (7.34876,6.82287), (16.60689,10.82228), (7.48786,-4.18631), (15.78602,20.3872), (17.21048,11.84735), (13.93482,21.25376), (9.69911,10.55032), (12.24315,12.19023), (10.58131,0.63369), (19.57006,7.92381), (9.8856,17.90933), (11.70302,15.30781), (7.89864,10.01877), (12.24831,0.88744), (16.93707,22.20967), (9.65467,-4.23117), (4.221,21.50819), (15.45229,11.27421), (12.83088,-16.23179), (7.58313,33.43085), (12.895,5.15093), (10.02471,1.34505), (13.36059,6.027), (5.07864,-10.43035), (9.72017,27.45998), (11.05809,19.24886), (15.28528,-4.44761), (13.99834,5.453), (19.26989,12.73758), (9.41846,11.2897), (11.65425,31.032), (8.49638,7.39168), (6.38592,11.95245), (-4.69837,26.279), (12.22061,-1.0255), (9.41331,10.36675), (13.2075,11.58439), (12.97005,27.8405), (11.44352,13.1707), (9.79805,31.39133), (6.93116,27.08301), (10.07691,-2.14368), (22.05892,4.08476), (7.80353,21.5573), (-2.17276,16.69822), (0.61509,7.69955), (8.35842,8.32793), (17.77108,6.49235), (14.70841,-7.3284), (1.27992,10.58264), (15.62699,-6.17006), (9.32914,34.55782), (15.41866,10.93221), (10.82009,44.24299), (3.29902,14.6224), (9.21998,-7.42798), (7.93845,15.52351), (10.33344,11.33982), (12.06399,10.46716), (5.5308,13.0986), (8.38727,-4.25988), (18.11104,9.55316), (8.86565,0.75489), (19.41825,25.99212), (9.52376,-0.81401), (3.94552,3.49551), (9.37587,22.99402), (15.44954,10.99628), (15.90527,23.70223), (13.18927,2.71482), (7.01646,22.82309), (9.06005,31.25686), (9.06431,4.86318), (5.76006,-1.06476), (9.18705,15.10298), (-3.48446,-0.61015), (15.89817,17.81246), (12.94719,-1.55788), (23.69426,18.09709), (17.47755,9.11271), (15.61528,9.94682), (0.54832,-7.33194), (14.32916,-4.67293), (9.55305,21.81717), (13.79891,7.16318), (0.82544,13.25649), (13.34875,13.88776), (9.07614,4.95793), (5.19621,17.65303), (2.1451,14.47382), (9.87726,13.19373), (8.45439,31.86093), (-1.41842,5.73161), (7.93598,10.96492), (11.23151,6.97951), (17.84458,1.75136), (7.02237,10.96144), (10.7842,15.08137), (4.42832,9.95311), (4.45044,7.07729), (1.50938,3.08148), (21.21651,22.37954), (6.2097,8.51951), (6.84354,2.88746), (18.53804,26.73509), (12.01072,-2.88939), (4.8345,-2.82367), (20.41587,-0.35783), (14.48353,14.22076), (8.71116,11.50295), (12.42818,7.10171), (14.89244,8.28488), (8.03033,0.54178), (5.25917,13.8022), (2.30092,15.62157), (10.22504,10.79173), (15.37573,28.18946), (7.13666,30.43524), (4.45018,2.54914), (10.18405,9.89421), (3.91025,13.08631), (14.52304,4.68761), (13.14771,5.61516), (11.99219,22.88072), (9.21345,7.4735), (8.85106,11.27382), (12.91887,2.39559), (15.62308,-3.31889), (11.88034,9.61957), (15.12097,23.01381), (11.58168,-1.23467), (16.83051,9.07691), (5.25405,15.78056), (2.19976,12.28421), (4.56716,9.44888), (16.46053,13.16928), (5.61995,4.33357), (8.67704,2.21737), (5.62789,33.17833), (9.84815,13.25407), (13.05834,-2.47961), (11.74205,6.41401), (3.88393,18.8439), (16.15321,-4.63375), (4.83925,-8.2909), (13.00334,12.18221), (4.4028,-2.95356), (4.35794,19.61659), (4.47478,12.45056), (2.38713,-4.17198), (4.25235,21.9641), (10.87509,11.96416), (9.82411,12.74573), (13.61518,10.47873), (10.25507,12.73295), (4.0335,11.31373), (10.69881,9.9827), (5.70321,5.87138), (6.96244,4.24372), (9.35874,-23.72256), (6.28076,28.41337), (8.29015,4.88103), (6.88653,3.61902), (7.70687,8.93586), (8.2001,16.40759), (6.73415,27.84494), (3.82052,5.6001), (3.94469,14.51379), (15.82384,13.5576), (2.54004,12.92213), (10.74876,3.90686), (12.60517,17.07104), (17.7024,15.84268), (4.6722,17.38777), (13.67341,16.54766), (6.4565,5.94487), (12.95699,17.02804), (4.56912,7.66386), (5.58464,10.43088), (4.0638,6.16059), (13.05559,20.46178), (5.38269,20.02888), (0.16354,20.95949), (7.23962,6.50808), (7.38577,7.22366), (8.50951,8.06659), (13.72574,16.08241), (17.80421,13.83514), (3.01135,-0.33454), (8.02608,12.98848), (14.23847,12.99024); +SELECT -0.5028215369186904, 0.6152361677168877; +SELECT roundBankers(WelchTTest(left, right).1, 16) as t_stat, roundBankers(WelchTTest(left, right).2, 16) as p_value from welch_ttest; +DROP TABLE IF EXISTS welch_ttest; + +/*Check t-stat and p-value and compare it with scipy.stat implementation + First: a=10, sigma (not sigma^2)=5, size=500 + Second: a=1, sigma = 12, size = 500 */ +CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; +INSERT INTO welch_ttest VALUES (4.82025,-2.69857), (6.13896,15.80943), (15.20277,7.31555), (14.15351,3.96517), (7.21338,4.77809), (8.55506,9.6472), (13.80816,-26.41717), (11.28411,-10.85635), (7.4612,-1.4376), (7.43759,-0.96308), (12.9832,2.84315), (-5.74783,5.79467), (12.47114,-3.06091), (15.14223,-14.62902), (3.40603,22.08022), (9.27323,-2.11982), (7.88547,-4.84824), (8.56456,-10.50447), (4.59731,2.4891), (7.91213,9.90324), (7.33894,-22.66866), (21.74811,-0.97103), (11.92111,-16.57608), (0.18828,-3.78749), (10.47314,25.84511), (20.37396,5.30797), (11.04991,-18.19466), (13.30083,11.72708), (14.28065,0.2891), (2.86942,-9.83474), (24.96072,6.69942), (14.20164,18.09604), (18.28769,18.52651), (10.50949,1.38201), (9.22273,7.64615), (11.77608,17.66598), (8.56872,-2.44141), (13.74535,-9.01598), (11.65209,27.69142), (12.51894,4.06946), (17.76256,-15.0077), (13.52122,-10.49648), (8.70796,-4.88322), (6.04749,-25.09805), (16.33064,-4.64024), (8.35636,20.94434), (14.03496,24.12126), (11.05834,-14.10962), (14.49261,10.6512), (2.59383,14.50687), (8.01022,-19.88081), (4.05458,-11.55271), (13.26384,13.16921), (14.62058,16.63864), (10.52489,-24.08114), (8.46357,-9.09949), (6.4147,-10.54702), (9.70071,0.20813), (12.47581,8.19066), (4.38333,-2.70523), (17.54172,-0.23954), (10.12109,7.19398), (7.73186,-7.1618), (14.0279,-7.44322), (11.6621,-17.92031), (17.47045,-1.58146), (15.50223,9.18338), (15.46034,3.25838), (13.39964,-14.30234), (14.98025,1.84695), (15.87912,31.13794), (17.67374,-0.85067), (9.64073,19.02787), (12.84904,-3.09594), (7.70278,13.45584), (13.03156,-5.48104), (9.04512,-22.74928), (15.97014,-8.03697), (8.96389,17.31143), (11.48009,-16.65231), (9.71153,-18.58713), (13.00084,-16.52641), (12.39803,14.95261), (13.08188,12.56762), (5.82244,15.00188), (10.81871,1.85858), (8.2539,2.1926), (7.52114,-2.4095), (9.11488,21.56873), (8.37482,3.35509), (14.48652,-4.98672), (11.42152,35.08603), (16.03111,-10.01602), (13.14057,-3.85153), (-2.26351,-6.81974), (15.50394,19.56525), (14.88603,-9.35488), (13.37257,0.24268), (11.84026,-3.51488), (7.66558,-0.37066), (6.24584,24.20888), (3.6312,-11.73537), (2.7018,0.01282), (5.63656,0.03963), (5.82643,-9.65589), (10.06745,-0.37429), (-0.5831,5.61255), (14.84202,0.49984), (9.5524,-10.15066), (19.71713,-14.54314), (14.23109,16.56889), (8.69105,-7.73873), (5.33742,-3.76422), (7.30372,1.40722), (7.93342,2.28818), (15.20884,-13.12643), (7.53839,5.17082), (13.45311,4.79089), (11.04473,-17.42643), (10.76673,8.72548), (15.44145,-3.70285), (14.06596,16.77893), (9.14873,13.382), (12.88372,19.98418), (8.74994,0.00483), (10.53263,-4.75951), (16.16694,2.35391), (8.37197,21.65809), (3.43739,-9.2714), (4.72799,-18.38253), (9.08802,7.23097), (11.2531,14.97927), (5.16115,-4.02197), (10.20895,-29.8189), (18.70884,-12.8554), (15.88924,-7.60124), (3.38758,-14.90158), (6.46449,-3.31486), (10.21088,31.38144), (14.08458,-8.61288), (15.74508,15.31895), (19.31896,-10.19488), (13.19641,13.796), (11.95409,-0.32912), (10.70718,-0.0684), (1.05245,-30.06834), (10.04772,24.93912), (17.01369,-3.26506), (10.2286,-8.29751), (19.58323,-5.39189), (7.02892,-25.08603), (4.16866,-1.45318), (8.94326,16.72724), (4.99854,-3.38467), (8.88352,-26.00478), (18.65422,7.28369), (17.32328,16.96226), (9.33492,16.5858), (14.94788,10.46583), (8.05863,3.84345), (14.6737,-2.99382), (10.93801,1.42078), (0.54036,-11.0123), (-0.34242,2.09909), (5.89076,1.21064), (3.15189,15.36079), (1.94421,-21.61349), (6.38698,22.7726), (10.50654,10.50512), (8.95362,-6.95825), (6.23711,9.20036), (11.75359,15.66902), (12.42155,3.28098), (-1.55472,-9.05692), (4.6688,0.32882), (10.48087,-1.64934), (11.74615,-4.81406), (9.26822,-5.06006), (7.55517,19.97493), (12.76005,2.88646), (16.47102,-0.34552), (11.31297,7.55186), (14.37437,-22.96115), (2.38799,31.29166), (6.44577,6.18798), (5.07471,-2.52715), (11.55123,-11.58799), (7.76795,14.13596), (10.60116,13.45069), (14.40885,12.15179), (11.58158,3.44491), (8.81648,-8.78006), (12.92299,18.32087), (11.26939,11.91757), (17.95014,-2.00179), (2.95002,10.88411), (17.41959,9.09327), (11.12455,6.62484), (8.78541,8.87178), (14.36413,11.52254), (12.98554,-14.15988), (12.58505,-17.19515), (15.49789,14.03089), (11.70999,-2.4095), (0.65596,-16.83575), (11.08202,2.71469), (14.75752,4.84351), (6.84385,-1.17651), (9.27245,-3.37529), (13.78243,-19.92137), (17.4863,4.48952), (4.01777,-12.4906), (11.82861,-5.65277), (13.86551,8.50819), (6.16591,-19.61261), (8.71589,12.54156), (16.77195,11.06784), (17.23243,-12.59285), (-2.12941,3.43683), (5.66629,-3.00325), (12.45153,12.49082), (1.63971,7.20955), (13.84031,17.6547), (4.6144,15.8619), (5.26169,24.3048), (9.27769,-8.05434), (9.14288,-6.06901), (9.71953,-15.69515), (9.38446,-11.13917), (1.64788,-3.90757), (11.72922,-2.57038), (13.68926,5.14065), (9.42952,17.8497), (12.05574,-8.64665), (9.09148,-18.68331), (5.32273,5.8567), (20.25258,-20.93884), (10.14599,4.40583), (10.82156,14.35985), (5.75736,4.18134), (7.13567,4.3635), (9.29746,9.35428), (5.1618,2.8908), (10.076,16.01017), (21.65669,-1.48499), (13.35486,-9.97949), (6.79957,1.03055), (8.76243,-2.79697), (14.59294,6.85977), (16.90609,4.73213), (10.50337,2.7815), (-0.07923,-2.46866), (13.51648,18.39425), (12.0676,-0.80378), (0.86482,-0.22982), (9.03563,-16.11608), (5.38751,3.0862), (17.16866,3.20779), (2.78702,10.50146), (11.15548,-0.21305), (12.30843,11.21012), (8.04897,-0.99825), (9.95814,18.39633), (11.29308,-3.39003), (14.13032,-0.64411), (21.05877,-1.39932), (3.57386,15.45319), (7.96631,-0.66044), (3.30484,-15.2223), (18.61856,-34.39907), (16.35184,-3.57836), (7.65236,16.82828), (18.02895,1.66624), (9.79458,15.43475), (16.7274,8.17776), (8.84453,5.50486), (13.05709,10.43082), (10.91447,-6.63332), (8.40171,2.28008), (16.95211,16.37203), (11.82194,5.16313), (19.87978,-8.85281), (12.88455,13.26692), (-0.00947,-7.46842), (12.28109,8.43091), (6.96462,-13.18172), (13.75282,-0.72401), (14.39141,22.3881), (11.07193,10.65448), (12.88039,2.81289), (11.38253,10.92405), (21.02707,-8.95358), (7.51955,19.80653), (6.31984,-12.86527), (15.6543,5.38826), (14.80315,-6.83501), (8.38024,-15.7647), (21.7516,-27.67412), (14.31336,8.6499), (15.04703,-4.89542), (5.73787,16.76167), (13.16911,12.84284), (12.40695,-17.27324), (9.88968,-4.18726), (8.46703,-14.62366), (8.70637,-5.49863), (8.03551,-16.22846), (5.9757,10.60329), (12.22951,6.46781), (3.14736,1.70458), (10.51266,10.77448), (18.593,0.8463), (10.82213,13.0482), (7.14216,-4.36264), (6.81154,3.22647), (-0.6486,2.38828), (20.56136,6.7946), (11.35367,-0.25254), (11.38205,1.2497), (17.14,1.6544), (14.91215,4.1019), (15.50207,11.27839), (5.93162,-5.04127), (3.74869,18.11674), (14.11532,0.51231), (7.38954,-0.51029), (5.45764,13.52556), (18.33733,16.10171), (9.91923,5.68197), (2.38991,-2.85904), (14.16756,-8.89167), (2.39791,6.24489), (6.92586,10.85319), (5.32474,-0.39816), (2.28812,3.87079), (5.71718,-3.1867), (5.84197,1.55322), (2.76206,16.86779), (19.05928,-14.60321), (11.51788,-1.81952), (6.56648,-3.11624), (3.35735,1.24193), (7.55948,10.18179), (19.99908,4.69796), (13.00634,0.69032), (18.36886,11.7723), (11.14675,7.62896), (16.72931,9.89741), (12.50106,9.11484), (6.00605,-3.84676), (23.06653,-0.4777), (5.39694,0.95958), (9.53167,-7.95056), (12.76944,-10.97474), (7.20604,-6.54861), (13.25391,34.74933), (13.7341,27.39463), (10.85292,4.18299), (-7.75835,6.02476), (10.29728,-1.99397), (13.70099,1.26478), (10.17959,23.37106), (9.98399,10.49682), (12.69389,-11.04354), (-0.28848,-12.22284), (-2.18319,-9.87635), (13.36378,28.90511), (10.09232,6.77613), (5.49489,0.55352), (5.46156,0.37031), (0.94225,7.1418), (12.79205,3.24897), (10.09593,-1.60918), (6.06218,3.1675), (0.89463,-17.97072), (11.88986,-5.61743), (10.79733,14.1422), (1.51371,14.87695), (2.20967,-4.65961), (15.45732,-0.99174), (16.5262,-2.96623), (5.99724,-9.02263), (8.3613,-17.2088), (15.68183,2.78608), (15.32117,6.74239), (14.15674,4.8524), (6.64553,7.46731), (4.20777,1.04894), (-0.10521,-12.8023), (-0.88169,-17.18188), (1.85913,-5.08801), (9.73673,22.13942), (0.30926,-0.36384), (6.17559,17.80564), (11.76602,7.67504), (5.68385,1.59779), (14.57088,4.10942), (12.81509,0.61074), (9.85682,-14.40767), (12.06376,10.59906), (6.08874,16.57017), (11.63921,-15.17526), (14.86722,-6.98549), (10.41035,-0.64548), (2.93794,3.23756), (12.21841,14.65504), (0.23804,4.583), (3.14845,12.72378), (7.29748,5.26547), (3.06134,0.81781), (13.77684,9.38273), (16.21992,10.37636), (5.33511,10.70325), (9.68959,-0.83043), (9.44169,-7.53149), (18.08012,-9.09147), (4.04224,-19.51381), (8.77918,-28.44508), (10.18324,6.44392), (9.38914,11.10201), (11.76995,-2.86184), (14.19963,8.30673), (6.88817,8.8797), (16.56123,10.68053), (15.39885,15.62919), (5.21241,8.00579), (4.44408,6.4651), (17.87587,-4.50029), (12.53337,18.04514), (13.60916,11.12996), (6.60104,-5.14007), (7.35453,9.43857), (18.61572,3.13476), (6.10437,4.9772), (13.08682,-17.45782), (12.15404,0.05552), (4.90789,-1.90283), (2.13353,2.67908), (12.49593,-2.62243), (11.93056,-3.22767), (13.29408,-8.70222), (5.70038,-23.11605), (8.40271,21.6757), (5.19456,12.70076), (-5.51028,4.4322), (14.0329,11.69344), (10.38365,9.18052), (6.56812,-2.2549), (4.21129,-2.15615), (9.7157,20.29765), (9.88553,-0.29536), (13.45346,15.50109), (4.97752,8.79187), (12.77595,5.11533), (8.56465,-20.44436), (4.27703,-3.00909), (18.12502,-4.48291), (12.45735,21.84462), (12.42912,1.94225), (12.08125,-2.81908), (10.85779,17.19418), (4.36013,-9.33528), (11.85062,-0.17346), (8.47776,0.03958), (9.60822,-35.17786), (11.3069,8.36887), (14.25525,-9.02292), (1.55168,-10.98804), (14.57782,0.29335), (7.84786,4.29634), (9.87774,3.87718), (14.75575,-9.08532), (3.68774,7.13922), (9.37667,-7.62463), (20.28676,-10.5666), (12.10027,4.68165), (8.01819,-3.30172), (18.78158,13.04852), (20.85402,13.45616), (18.98069,2.41043), (16.1429,-0.36501), (9.24047,-15.67383), (14.12487,17.92217), (10.18841,8.42106), (-3.04478,3.22063), (5.7552,-7.31753), (9.30376,21.99596), (11.42837,-36.8273), (6.02364,-20.46391), (8.86984,5.74179), (10.91177,-15.83178), (10.04418,14.90454), (18.10774,-8.84645), (7.49384,3.72036), (9.11556,4.6877), (9.7051,16.35418), (5.23268,3.15441), (9.04647,2.39907), (8.81547,-17.58664), (2.65098,-13.18269); +SELECT 14.971190998235835, 5.898143508382202e-44; +SELECT roundBankers(WelchTTest(left, right).1, 16) as t_stat, roundBankers(WelchTTest(left, right).2, 16) as p_value from welch_ttest; +DROP TABLE IF EXISTS welch_ttest; From a750b76818b6fc6273a6d7fc8e5785d9ff0068c7 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Tue, 13 Oct 2020 21:50:04 +0300 Subject: [PATCH 069/325] better --- tests/queries/0_stateless/01322_student_ttest.sql | 2 +- tests/queries/0_stateless/01322_welch_ttest.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01322_student_ttest.sql b/tests/queries/0_stateless/01322_student_ttest.sql index 3636e239fe8..babc8457bc0 100644 --- a/tests/queries/0_stateless/01322_student_ttest.sql +++ b/tests/queries/0_stateless/01322_student_ttest.sql @@ -14,6 +14,6 @@ DROP TABLE IF EXISTS student_ttest; Second: a=1, sigma = 5, size = 500 */ CREATE TABLE student_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO student_ttest VALUES (4.52546,8.69444), (3.73628,3.81414), (-0.39478,12.38442), (5.15633,8.9738), (0.50539,9.19594), (-5.34036,7.21009), (0.19336,4.97743), (8.35729,4.94756), (6.95818,19.80911), (-2.93812,13.75358), (8.30807,16.56373), (-3.3517,9.72882), (4.16279,4.64509), (-3.17231,17.76854), (1.93545,4.80693), (11.06606,8.79505), (-4.22678,10.88868), (-1.99975,6.21932), (-4.51178,15.11614), (-4.50711,13.24703), (1.89786,14.76476), (-6.19638,-0.6117), (-3.70188,17.48993), (5.01334,12.11847), (1.79036,4.87439), (2.14435,18.56479), (3.0282,1.23712), (2.35528,5.41596), (-12.18535,4.54994), (5.59709,11.37668), (-12.92336,9.5982), (-0.04281,6.59822), (-0.16923,1.16703), (0.88924,8.88418), (-4.68414,10.95047), (8.01099,5.52787), (2.61686,-1.11647), (-2.76895,14.49946), (3.32165,3.27585), (-0.85135,-0.42025), (1.21368,6.37906), (4.38673,2.5242), (6.20964,8.1405), (-1.23172,6.46732), (4.65516,9.89332), (-1.87143,10.4374), (0.86429,-1.06465), (2.51184,6.84902), (-1.88822,10.96576), (-1.61802,7.83319), (1.93653,14.39823), (-3.66631,7.02594), (-1.05294,13.46629), (-10.74718,10.39531), (16.49295,11.27348), (-7.65494,9.32187), (-3.39303,12.32667), (-4.89418,8.98905), (3.2521,9.54757), (0.05831,5.98325), (-3.00409,3.47248), (5.76702,9.26966), (2.67674,5.77816), (10.52623,6.32966), (-0.54501,9.49313), (-4.89835,6.21337), (3.52457,10.00242), (-0.0451,6.25167), (-6.61226,15.64671), (9.02391,2.78968), (5.52571,6.55442), (4.54352,3.68819), (-3.8394,9.55934), (-7.75295,4.166), (5.91167,12.32471), (1.38897,7.10969), (6.24166,16.31723), (5.58536,12.99482), (4.7591,10.11585), (-2.58336,10.29455), (-1.91263,18.27524), (3.31575,12.84435), (5.3507,13.11954), (-15.22081,12.84147), (-0.84775,15.55658), (-4.538,11.45329), (6.71177,7.50912), (0.52882,8.56226), (2.0242,8.63104), (5.69146,15.68026), (4.63328,21.6361), (0.22984,6.23925), (-2.84052,8.65714), (7.91867,9.9423), (1.11001,12.28213), (-0.11251,3.11279), (-0.20905,13.58128), (0.03287,16.51407), (-1.59397,16.60476), (-5.39405,12.02022), (-7.1233,12.11035), (4.51517,9.47832), (-0.70967,6.40742), (5.67299,8.87252), (-0.33835,15.14265), (-1.83047,2.23572), (-0.62877,11.57144), (-7.23148,18.87737), (0.1802,12.1833), (11.73325,11.17519), (2.17603,16.80422), (-0.11683,6.81423), (-1.29102,12.12546), (-0.23201,8.06153), (-6.8643,10.97228), (-6.85153,7.30596), (-4.77163,15.44026), (6.11721,8.00993), (5.96406,12.60196), (3.59135,13.96832), (-0.60095,14.03207), (3.11163,4.53758), (-0.18831,8.08297), (0.67657,4.90451), (-3.16117,8.14253), (0.26957,19.88605), (2.18653,13.85254), (-5.94611,23.01839), (-4.39352,6.02084), (-3.71525,9.60319), (5.11103,1.90511), (1.33998,10.35237), (1.01629,16.27082), (-3.36917,12.52379), (-3.99661,11.37435), (8.19336,13.61823), (2.89168,15.77622), (-11.10373,15.17254), (11.68005,6.711), (3.08282,4.74205), (-6.81506,10.09812), (-2.34587,6.61722), (-2.68725,10.34164), (0.3577,8.96602), (-3.05682,12.32157), (9.08062,11.75711), (-0.77913,13.49499), (10.35215,8.57713), (6.82565,11.50313), (-1.24674,1.13097), (5.18822,7.83205), (-3.70743,5.77957), (1.40319,15.5519), (5.89432,10.82676), (1.43152,11.51218), (6.70638,9.29779), (9.76613,9.77021), (4.27604,9.94114), (-2.63141,15.54513), (-7.8133,19.10736), (-0.06668,15.04205), (1.05391,9.03114), (4.41797,24.0104), (0.09337,9.94205), (6.16075,2.5925), (7.49413,8.82726), (-3.52872,10.0209), (-2.17126,8.1635), (-3.87605,4.24074), (3.26607,7.67291), (-3.28045,5.21642), (2.1429,11.2808), (1.53386,6.88172), (0.21169,5.98743), (-0.63674,17.97249), (5.84893,6.46323), (-0.63498,15.37416), (8.29526,2.89957), (-1.08358,17.13044), (-2.306,11.06355), (2.86991,3.09625), (-0.76074,-2.33019), (5.49191,7.42675), (1.82883,15.06792), (-3.70497,8.81116), (-0.53232,19.17446), (-11.49722,18.77181), (3.44877,14.06443), (-1.8596,12.81241), (-10.34851,2.72299), (1.13093,18.67739), (-10.93389,11.63275), (-3.39703,2.23891), (0.19749,13.01195), (-3.68389,7.43402), (-4.67863,8.14599), (10.78916,16.65328), (0.37675,1.362), (3.98094,3.87957), (-3.64775,11.16134), (-4.8443,6.25357), (1.102,4.21945), (8.72112,12.50047), (-1.47361,6.45486), (6.24183,18.99924), (6.83569,18.09508), (-3.11684,13.59528), (4.91306,3.39681), (-0.03628,13.33157), (5.1282,5.8945), (-2.38558,5.61212), (2.33351,8.41149), (-0.97191,13.78608), (-0.05588,6.08609), (-4.70019,12.76962), (-5.12371,3.26206), (0.65606,0.25528), (-0.11574,11.9083), (4.4238,4.35071), (6.93399,11.19855), (3.68712,13.87404), (-0.01187,6.87986), (1.8332,8.32566), (5.81322,22.51334), (-4.04709,2.5226), (-8.26397,16.84498), (-2.11273,6.26108), (5.28396,13.84824), (0.73054,6.03262), (6.43559,14.12668), (4.35565,16.01939), (-1.05545,8.19237), (5.00087,18.01595), (-2.72239,9.45609), (7.32313,6.90459), (2.11548,12.83115), (-3.40953,10.603), (6.97051,13.70439), (-0.45567,6.1633), (1.31699,4.1151), (-1.49871,8.20499), (7.14772,11.67903), (0.79277,7.30851), (6.9698,6.50941), (2.08733,7.3949), (-3.55962,12.80075), (0.75601,5.62043), (1.21,18.2542), (-2.17877,17.9393), (1.83206,16.4569), (5.72463,8.78811), (7.42257,4.85949), (0.97829,-3.36394), (7.54238,5.38683), (9.91081,12.26083), (-4.61743,10.27907), (-4.40799,11.5144), (9.99854,11.57335), (8.53725,1.94203), (3.2905,7.78228), (0.38634,11.79385), (-2.53374,10.18415), (4.94758,14.67613), (4.79624,4.70301), (5.57664,12.72151), (-6.44871,-3.35508), (3.34431,17.63775), (0.14209,2.53883), (10.88431,14.01483), (0.31846,12.4387), (-0.54703,11.15408), (-4.67791,7.74882), (-5.68011,13.60956), (-4.93362,7.81991), (1.2271,10.90969), (5.27512,8.19828), (-3.84611,-1.18523), (6.81706,0.5916), (10.33033,0.35805), (5.13979,12.98364), (3.66534,11.38628), (-2.07219,13.94644), (10.65442,2.03781), (-3.31751,10.74447), (-1.82011,12.35656), (-0.39886,7.08701), (1.77052,2.69871), (1.29049,19.66653), (7.92344,7.88636), (-2.92595,10.36916), (-2.67107,1.632), (5.64708,11.86081), (0.34639,13.47602), (-3.04356,6.60204), (3.98828,7.01303), (-1.36695,20.19992), (-8.48462,18.88249), (-4.04669,11.34367), (9.84561,12.97305), (-6.1537,9.5776), (0.82433,17.91364), (1.92449,18.3247), (2.51288,9.9211), (0.40965,7.14257), (2.89183,6.59133), (3.84347,12.35274), (0.66829,10.57523), (-3.45094,12.12859), (1.3544,9.47177), (-9.85456,0.60659), (5.25689,4.72996), (-5.26018,4.51121), (-6.16912,13.28893), (-1.77163,8.09014), (3.96687,8.02511), (0.70893,13.85406), (-5.45342,1.75412), (-3.89706,6.00641), (3.11868,6.35554), (4.41714,7.11293), (7.64841,8.30442), (0.00489,12.63024), (3.2263,12.38966), (-5.33042,7.6801), (2.52189,11.33744), (-7.40308,4.67713), (0.67891,7.62276), (2.49343,2.14478), (5.43133,15.32988), (-0.67541,1.52299), (-0.60299,17.00017), (-6.32903,8.29701), (-3.44336,10.92961), (-0.23963,6.78449), (6.94686,7.02698), (6.59442,11.51719), (-4.18532,9.97926), (-1.8228,7.44251), (-0.29443,7.58541), (2.99821,4.76058), (2.51942,12.88959), (-3.49176,9.974), (-0.57979,17.03689), (8.69471,11.14554), (-1.19427,11.7392), (-3.17119,11.50029), (-2.99566,19.41759), (-3.34493,9.65127), (-2.33826,9.87673), (-5.04164,14.13485), (-0.48214,9.78034), (7.45097,1.57826), (3.04787,3.72091), (2.92632,9.4054), (1.39694,23.22816), (4.38686,-0.12571), (3.25753,6.97343), (7.14218,10.09049), (-4.04341,11.78393), (-9.19352,3.01909), (2.78473,16.09448), (0.33331,6.25485), (9.89238,7.13164), (6.00566,7.75879), (-1.7511,9.56834), (4.77815,6.14824), (5.07457,13.53454), (2.56132,8.26364), (2.38317,8.7095), (-1.63486,10.61607), (-1.46871,10.64418), (-5.8681,23.9106), (-2.96227,11.38978), (-1.90638,11.4383), (-13.3052,18.41498), (-2.14705,3.70959), (-9.62069,19.95918), (2.29313,9.53847), (0.22162,14.04957), (-1.83956,13.70151), (4.1853,5.45046), (6.05965,10.95061), (-0.23737,9.55156), (6.07452,17.92345), (4.34629,6.23976), (4.02922,8.71029), (3.62622,13.58736), (-3.95825,8.78527), (-1.63412,11.14213), (-1.25727,12.23717), (5.06323,16.44557), (-0.66176,0.47144), (2.36606,9.7198), (-5.77792,13.50981), (4.535,14.27806), (1.02031,13.50793), (4.49345,7.47381), (-4.99791,11.07844), (2.46716,9.89844), (3.65471,21.48548), (11.2283,6.92085), (6.69743,4.44074), (-5.60375,19.98074), (0.28683,7.92826), (-0.85737,16.6313), (4.26726,17.17618), (-3.4322,13.80807), (-2.07039,5.37083), (-2.26798,9.73962), (-0.99818,10.66273), (0.41335,8.90639), (5.18124,12.24596), (-5.01858,16.89203), (2.05561,12.69184), (-0.12117,15.59077), (0.99471,6.94287), (6.89979,-0.1801), (-4.18527,3.25318), (-6.35104,8.08804), (3.89734,13.78384), (-1.979,0.46434), (3.15404,7.78224), (3.52672,9.10987), (2.48372,-0.89391), (-6.13089,14.3696), (2.2968,3.01763), (-2.74324,8.03559), (-0.12876,7.24609), (-1.51135,11.86271), (-3.92434,6.28196), (-1.71254,8.9725), (-1.25878,14.46114), (2.03021,9.50216), (4.31726,16.30413), (-3.02908,1.02795), (9.7093,1.88717), (-3.36284,9.80106), (6.70938,4.53487), (0.42762,16.34543), (5.04726,7.71098), (2.78386,2.74639), (6.83022,6.51875), (-3.02109,10.42308), (-0.65382,13.57901), (-15.58675,0.52784), (5.89746,4.4708), (-4.11598,6.39619), (-1.37208,14.57666), (10.08082,2.71602), (5.35686,12.53905), (1.93331,11.4292), (10.47444,12.44641), (-2.36872,14.50894), (6.50752,17.64374), (2.54603,11.03218), (-0.4332,9.82789), (5.26572,10.11104), (2.09016,2.16137), (1.15513,10.24054), (14.95941,12.86909), (-3.85505,15.22845), (-2.36239,5.05411), (1.64338,10.84836), (-4.25074,11.15717), (7.29744,0.91782), (-1.18964,13.29961), (5.60612,15.11314), (-3.77011,11.54004), (6.67642,-0.94238), (-0.06862,19.32581), (5.60514,10.20744), (3.7341,6.54857), (9.59001,8.69108), (3.30093,8.2296), (-2.75658,8.4474), (4.71994,6.81178), (0.74699,5.99415), (2.91095,13.99336), (-7.36829,8.7469), (-5.29487,8.62349), (3.31079,1.84212), (1.06974,4.4762), (-1.18424,9.25421), (-7.415,10.44229), (3.40595,12.21649), (-7.63085,10.45968), (1.13336,15.34722), (-0.0096,5.50868), (0.8928,10.93609), (-0.5943,2.78631), (7.48306,11.86145), (10.11943,18.67385), (5.60459,10.64051), (4.00189,12.75565), (2.35823,6.63666), (0.33475,12.19343), (3.47072,9.08636), (-6.68867,11.67256), (3.31031,20.31392), (2.17159,11.66443); -SELECT -28.740781574102936, 7.667329672103986e-133; +SELECT '-28.740781574102936', '7.667329672103986e-133'; SELECT roundBankers(StudentTTest(left, right).1, 16) as t_stat, roundBankers(StudentTTest(left, right).2, 16) as p_value from student_ttest; DROP TABLE IF EXISTS student_ttest; diff --git a/tests/queries/0_stateless/01322_welch_ttest.sql b/tests/queries/0_stateless/01322_welch_ttest.sql index 2a045e70b32..5a5b52ab612 100644 --- a/tests/queries/0_stateless/01322_welch_ttest.sql +++ b/tests/queries/0_stateless/01322_welch_ttest.sql @@ -23,7 +23,7 @@ DROP TABLE IF EXISTS welch_ttest; Second: a=10, sigma = 10, size = 500 */ CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (14.72789,-8.65656), (9.61661,22.98234), (13.57615,23.80821), (3.98392,13.33939), (11.98889,-4.05537), (10.99422,23.5155), (5.44792,-6.45272), (20.29346,17.7903), (7.05926,11.463), (9.22732,5.28021), (12.06847,8.39157), (13.52612,6.02464), (8.24597,14.43732), (9.35245,15.76584), (10.12297,1.54391), (15.80624,1.24897), (13.68613,27.1507), (10.72729,7.71091), (5.62078,15.71846), (6.12229,32.97808), (6.03801,-1.79334), (8.95585,-9.23439), (24.04613,11.27838), (9.04757,0.72703), (2.68263,18.51557), (15.43935,9.16619), (2.89423,17.29624), (4.01423,-1.30208), (4.30568,-3.48018), (11.99948,10.12082), (8.40574,-8.01318), (10.86642,-14.22264), (9.4266,16.58174), (-8.12752,-0.55975), (7.91634,5.61449), (7.3967,1.44626), (2.26431,7.89158), (14.20118,1.13369), (6.68233,-0.82609), (15.46221,12.23365), (7.88467,12.45443), (11.20011,14.46915), (8.92027,13.72627), (10.27926,18.41459), (5.14395,29.66702), (5.62178,1.51619), (12.84383,10.40078), (9.98009,3.33266), (-0.69789,6.12036), (11.41386,11.86553), (7.76863,6.59422), (7.21743,22.0948), (1.81176,1.79623), (9.43762,14.29513), (19.22117,19.69162), (2.97128,-7.98033), (14.32851,5.48433), (7.54959,-2.28474), (3.81545,9.91876), (10.1281,10.64097), (2.48596,0.22523), (10.0461,17.01773), (3.59714,22.37388), (9.73522,14.04215), (18.8077,23.1244), (3.15148,18.96958), (12.26062,8.42663), (5.66707,3.7165), (6.58623,14.29366), (17.30902,23.50886), (9.91391,26.33722), (5.36946,26.72396), (15.73637,13.26287), (16.96281,12.97607), (11.54063,17.41838), (18.37358,8.63875), (11.38255,17.08943), (10.53256,23.15356), (8.08833,-4.4965), (16.27556,7.58895), (2.42969,26.04074), (9.56127,6.84245), (7.32998,20.56287), (9.19511,3.84735), (9.66903,-2.76304), (4.15029,13.1615), (8.83511,8.21954), (14.60617,-3.49943), (14.06143,22.12419), (5.39556,7.08323), (10.11871,16.12937), (10.56619,-0.32672), (14.4462,16.5942), (10.42106,7.68977), (7.75551,11.39484), (11.00418,-5.11987), (4.47226,20.87404), (16.35461,8.01007), (18.55174,3.26497), (11.82044,5.61253), (7.39454,20.69182), (11.27767,0.0296), (6.83827,21.904), (7.76858,22.46572), (15.97614,3.63685), (14.53781,-5.10846), (12.99546,14.86389), (16.91151,5.47188), (9.65012,18.44095), (14.25487,16.71368), (14.03618,6.36704), (2.57382,8.82663), (2.50779,14.6727), (14.24787,7.98383), (13.34666,2.65568), (7.31102,21.45827), (10.22981,11.77948), (17.4435,4.71979), (21.2074,3.17951), (6.64191,13.90226), (18.7086,15.50578), (14.78686,10.8026), (9.85287,16.91369), (4.48263,9.90552), (14.17469,13.87322), (14.4342,4.12366), (19.2481,-3.78985), (3.47165,1.7599), (8.28712,3.43715), (8.81657,-3.45246), (0.92319,23.64571), (20.41106,-4.96877), (6.76127,3.93514), (22.00242,1.49914), (8.66129,12.71519), (10.9929,5.11521), (17.95494,4.79872), (17.20996,20.89391), (12.18888,5.363), (12.14257,8.02765), (15.81243,14.30804), (4.43362,11.49002), (1.17567,14.25281), (15.60881,7.6573), (9.34833,15.49686), (6.33513,3.29327), (-0.83095,2.27236), (12.43268,12.58104), (6.63207,19.19128), (11.96877,15.25901), (14.81029,6.5221), (21.84876,10.10965), (3.75896,12.75249), (6.91307,16.50977), (13.73015,-8.6697), (8.63753,8.28553), (15.71679,1.44315), (1.74565,4.65869), (9.16895,0.98149), (5.70685,0.16623), (5.00117,17.66332), (13.06888,4.35346), (7.51204,6.52742), (15.34885,-1.06631), (5.20264,-5.28454), (8.59043,14.25583), (6.45619,8.74058), (14.61979,1.89553), (11.7075,-0.92959), (14.04901,10.30289), (4.20525,-6.3744), (15.1733,-8.1706), (3.12934,10.95369), (8.08049,4.94384), (15.41273,28.40568), (16.90751,3.7004), (5.86893,2.52363), (7.1086,4.07997), (4.418,7.8849), (12.0614,17.95409), (7.07887,16.67021), (3.61585,11.34377), (11.73001,-0.07446), (10.80449,22.00223), (8.40311,3.31778), (9.91276,18.50719), (16.4164,-3.58655), (5.25034,6.5394), (15.20283,12.40459), (10.42909,16.59866), (9.53888,7.54176), (14.68939,-1.51044), (6.60007,12.69758), (18.31058,2.9842), (7.01885,2.49187), (18.71631,2.04113), (10.50002,-2.46544), (10.7517,15.18368), (4.23224,-0.04058), (2.28924,-0.4127), (8.56059,10.5526), (8.25095,12.03982), (9.15673,12.10923), (13.28409,11.54954), (8.4513,-1.18613), (2.83911,11.30984), (2.79676,23.54105), (9.11055,10.67321), (7.18529,24.09196), (-4.1258,7.5008), (5.28306,12.52233), (6.82757,4.30673), (10.89035,9.35793), (5.24822,4.44472), (11.935,-7.00679), (6.45675,8.56241), (10.18088,23.73891), (4.9932,15.62708), (18.09939,16.09205), (8.11738,12.52074), (5.37883,14.58927), (10.50339,-4.80187), (16.64093,8.47964), (14.77263,7.75477), (13.71385,12.6893), (6.98746,7.14147), (10.74635,12.12654), (5.49432,12.32334), (13.46078,7.98909), (10.67565,3.26652), (9.0291,20.53684), (11.51417,32.3369), (13.07118,19.74911), (9.5049,-4.62897), (8.50611,8.26483), (6.47606,20.88451), (13.06526,-2.12982), (19.08658,25.61459), (9.49741,5.32091), (10.60865,-4.1196), (2.28996,7.57937), (8.12846,21.15847), (5.62241,6.46355), (4.07712,7.74846), (17.98526,19.62636), (9.466,28.34629), (11.38904,26.73919), (5.91826,20.40427), (1.52059,3.03378), (18.79161,10.2537), (18.20669,7.47745), (-1.67829,10.79184), (18.01586,3.91962), (16.31577,19.97973), (7.88281,18.87711), (8.46179,12.56157), (10.31113,11.46033), (14.88377,3.78661), (1.31835,-9.45748), (2.53176,12.06033), (9.48625,-0.74615), (3.97936,13.2815), (11.52319,24.78052), (13.24178,5.83337), (7.58739,17.4111), (10.00959,19.70331), (9.73361,11.78446), (8.35716,-1.366), (1.65491,1.37458), (11.11521,16.31483), (6.08355,32.63464), (10.04582,-3.79736), (11.58237,19.17984), (16.40249,-0.27705), (1.9691,-3.69456), (13.22776,28.38058), (2.67059,-1.36876), (9.83651,-25.63301), (2.12539,3.58644), (9.27114,-6.85667), (9.0699,13.42225), (2.78179,12.04671), (12.49311,28.99468), (12.97662,7.87662), (15.06359,2.61119), (16.91565,-3.56022), (5.92011,1.50022), (5.81304,14.55836), (8.46425,9.35831), (9.48705,16.9366), (4.68191,29.23126), (5.70028,15.31386), (-0.78798,13.46112), (10.03442,7.39667), (15.45433,11.15599), (9.43845,9.80499), (3.05825,22.64923), (6.92126,8.67693), (14.05905,18.67335), (19.71579,-3.19127), (15.0131,22.94716), (4.50386,17.86834), (1.31061,16.98267), (10.81197,15.91653), (14.32942,11.79718), (9.26469,18.50208), (7.27679,8.90755), (22.69295,10.44843), (12.03763,4.67433), (7.34876,6.82287), (16.60689,10.82228), (7.48786,-4.18631), (15.78602,20.3872), (17.21048,11.84735), (13.93482,21.25376), (9.69911,10.55032), (12.24315,12.19023), (10.58131,0.63369), (19.57006,7.92381), (9.8856,17.90933), (11.70302,15.30781), (7.89864,10.01877), (12.24831,0.88744), (16.93707,22.20967), (9.65467,-4.23117), (4.221,21.50819), (15.45229,11.27421), (12.83088,-16.23179), (7.58313,33.43085), (12.895,5.15093), (10.02471,1.34505), (13.36059,6.027), (5.07864,-10.43035), (9.72017,27.45998), (11.05809,19.24886), (15.28528,-4.44761), (13.99834,5.453), (19.26989,12.73758), (9.41846,11.2897), (11.65425,31.032), (8.49638,7.39168), (6.38592,11.95245), (-4.69837,26.279), (12.22061,-1.0255), (9.41331,10.36675), (13.2075,11.58439), (12.97005,27.8405), (11.44352,13.1707), (9.79805,31.39133), (6.93116,27.08301), (10.07691,-2.14368), (22.05892,4.08476), (7.80353,21.5573), (-2.17276,16.69822), (0.61509,7.69955), (8.35842,8.32793), (17.77108,6.49235), (14.70841,-7.3284), (1.27992,10.58264), (15.62699,-6.17006), (9.32914,34.55782), (15.41866,10.93221), (10.82009,44.24299), (3.29902,14.6224), (9.21998,-7.42798), (7.93845,15.52351), (10.33344,11.33982), (12.06399,10.46716), (5.5308,13.0986), (8.38727,-4.25988), (18.11104,9.55316), (8.86565,0.75489), (19.41825,25.99212), (9.52376,-0.81401), (3.94552,3.49551), (9.37587,22.99402), (15.44954,10.99628), (15.90527,23.70223), (13.18927,2.71482), (7.01646,22.82309), (9.06005,31.25686), (9.06431,4.86318), (5.76006,-1.06476), (9.18705,15.10298), (-3.48446,-0.61015), (15.89817,17.81246), (12.94719,-1.55788), (23.69426,18.09709), (17.47755,9.11271), (15.61528,9.94682), (0.54832,-7.33194), (14.32916,-4.67293), (9.55305,21.81717), (13.79891,7.16318), (0.82544,13.25649), (13.34875,13.88776), (9.07614,4.95793), (5.19621,17.65303), (2.1451,14.47382), (9.87726,13.19373), (8.45439,31.86093), (-1.41842,5.73161), (7.93598,10.96492), (11.23151,6.97951), (17.84458,1.75136), (7.02237,10.96144), (10.7842,15.08137), (4.42832,9.95311), (4.45044,7.07729), (1.50938,3.08148), (21.21651,22.37954), (6.2097,8.51951), (6.84354,2.88746), (18.53804,26.73509), (12.01072,-2.88939), (4.8345,-2.82367), (20.41587,-0.35783), (14.48353,14.22076), (8.71116,11.50295), (12.42818,7.10171), (14.89244,8.28488), (8.03033,0.54178), (5.25917,13.8022), (2.30092,15.62157), (10.22504,10.79173), (15.37573,28.18946), (7.13666,30.43524), (4.45018,2.54914), (10.18405,9.89421), (3.91025,13.08631), (14.52304,4.68761), (13.14771,5.61516), (11.99219,22.88072), (9.21345,7.4735), (8.85106,11.27382), (12.91887,2.39559), (15.62308,-3.31889), (11.88034,9.61957), (15.12097,23.01381), (11.58168,-1.23467), (16.83051,9.07691), (5.25405,15.78056), (2.19976,12.28421), (4.56716,9.44888), (16.46053,13.16928), (5.61995,4.33357), (8.67704,2.21737), (5.62789,33.17833), (9.84815,13.25407), (13.05834,-2.47961), (11.74205,6.41401), (3.88393,18.8439), (16.15321,-4.63375), (4.83925,-8.2909), (13.00334,12.18221), (4.4028,-2.95356), (4.35794,19.61659), (4.47478,12.45056), (2.38713,-4.17198), (4.25235,21.9641), (10.87509,11.96416), (9.82411,12.74573), (13.61518,10.47873), (10.25507,12.73295), (4.0335,11.31373), (10.69881,9.9827), (5.70321,5.87138), (6.96244,4.24372), (9.35874,-23.72256), (6.28076,28.41337), (8.29015,4.88103), (6.88653,3.61902), (7.70687,8.93586), (8.2001,16.40759), (6.73415,27.84494), (3.82052,5.6001), (3.94469,14.51379), (15.82384,13.5576), (2.54004,12.92213), (10.74876,3.90686), (12.60517,17.07104), (17.7024,15.84268), (4.6722,17.38777), (13.67341,16.54766), (6.4565,5.94487), (12.95699,17.02804), (4.56912,7.66386), (5.58464,10.43088), (4.0638,6.16059), (13.05559,20.46178), (5.38269,20.02888), (0.16354,20.95949), (7.23962,6.50808), (7.38577,7.22366), (8.50951,8.06659), (13.72574,16.08241), (17.80421,13.83514), (3.01135,-0.33454), (8.02608,12.98848), (14.23847,12.99024); -SELECT -0.5028215369186904, 0.6152361677168877; +SELECT '-0.5028215369186904', '0.6152361677168877'; SELECT roundBankers(WelchTTest(left, right).1, 16) as t_stat, roundBankers(WelchTTest(left, right).2, 16) as p_value from welch_ttest; DROP TABLE IF EXISTS welch_ttest; @@ -32,6 +32,6 @@ DROP TABLE IF EXISTS welch_ttest; Second: a=1, sigma = 12, size = 500 */ CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (4.82025,-2.69857), (6.13896,15.80943), (15.20277,7.31555), (14.15351,3.96517), (7.21338,4.77809), (8.55506,9.6472), (13.80816,-26.41717), (11.28411,-10.85635), (7.4612,-1.4376), (7.43759,-0.96308), (12.9832,2.84315), (-5.74783,5.79467), (12.47114,-3.06091), (15.14223,-14.62902), (3.40603,22.08022), (9.27323,-2.11982), (7.88547,-4.84824), (8.56456,-10.50447), (4.59731,2.4891), (7.91213,9.90324), (7.33894,-22.66866), (21.74811,-0.97103), (11.92111,-16.57608), (0.18828,-3.78749), (10.47314,25.84511), (20.37396,5.30797), (11.04991,-18.19466), (13.30083,11.72708), (14.28065,0.2891), (2.86942,-9.83474), (24.96072,6.69942), (14.20164,18.09604), (18.28769,18.52651), (10.50949,1.38201), (9.22273,7.64615), (11.77608,17.66598), (8.56872,-2.44141), (13.74535,-9.01598), (11.65209,27.69142), (12.51894,4.06946), (17.76256,-15.0077), (13.52122,-10.49648), (8.70796,-4.88322), (6.04749,-25.09805), (16.33064,-4.64024), (8.35636,20.94434), (14.03496,24.12126), (11.05834,-14.10962), (14.49261,10.6512), (2.59383,14.50687), (8.01022,-19.88081), (4.05458,-11.55271), (13.26384,13.16921), (14.62058,16.63864), (10.52489,-24.08114), (8.46357,-9.09949), (6.4147,-10.54702), (9.70071,0.20813), (12.47581,8.19066), (4.38333,-2.70523), (17.54172,-0.23954), (10.12109,7.19398), (7.73186,-7.1618), (14.0279,-7.44322), (11.6621,-17.92031), (17.47045,-1.58146), (15.50223,9.18338), (15.46034,3.25838), (13.39964,-14.30234), (14.98025,1.84695), (15.87912,31.13794), (17.67374,-0.85067), (9.64073,19.02787), (12.84904,-3.09594), (7.70278,13.45584), (13.03156,-5.48104), (9.04512,-22.74928), (15.97014,-8.03697), (8.96389,17.31143), (11.48009,-16.65231), (9.71153,-18.58713), (13.00084,-16.52641), (12.39803,14.95261), (13.08188,12.56762), (5.82244,15.00188), (10.81871,1.85858), (8.2539,2.1926), (7.52114,-2.4095), (9.11488,21.56873), (8.37482,3.35509), (14.48652,-4.98672), (11.42152,35.08603), (16.03111,-10.01602), (13.14057,-3.85153), (-2.26351,-6.81974), (15.50394,19.56525), (14.88603,-9.35488), (13.37257,0.24268), (11.84026,-3.51488), (7.66558,-0.37066), (6.24584,24.20888), (3.6312,-11.73537), (2.7018,0.01282), (5.63656,0.03963), (5.82643,-9.65589), (10.06745,-0.37429), (-0.5831,5.61255), (14.84202,0.49984), (9.5524,-10.15066), (19.71713,-14.54314), (14.23109,16.56889), (8.69105,-7.73873), (5.33742,-3.76422), (7.30372,1.40722), (7.93342,2.28818), (15.20884,-13.12643), (7.53839,5.17082), (13.45311,4.79089), (11.04473,-17.42643), (10.76673,8.72548), (15.44145,-3.70285), (14.06596,16.77893), (9.14873,13.382), (12.88372,19.98418), (8.74994,0.00483), (10.53263,-4.75951), (16.16694,2.35391), (8.37197,21.65809), (3.43739,-9.2714), (4.72799,-18.38253), (9.08802,7.23097), (11.2531,14.97927), (5.16115,-4.02197), (10.20895,-29.8189), (18.70884,-12.8554), (15.88924,-7.60124), (3.38758,-14.90158), (6.46449,-3.31486), (10.21088,31.38144), (14.08458,-8.61288), (15.74508,15.31895), (19.31896,-10.19488), (13.19641,13.796), (11.95409,-0.32912), (10.70718,-0.0684), (1.05245,-30.06834), (10.04772,24.93912), (17.01369,-3.26506), (10.2286,-8.29751), (19.58323,-5.39189), (7.02892,-25.08603), (4.16866,-1.45318), (8.94326,16.72724), (4.99854,-3.38467), (8.88352,-26.00478), (18.65422,7.28369), (17.32328,16.96226), (9.33492,16.5858), (14.94788,10.46583), (8.05863,3.84345), (14.6737,-2.99382), (10.93801,1.42078), (0.54036,-11.0123), (-0.34242,2.09909), (5.89076,1.21064), (3.15189,15.36079), (1.94421,-21.61349), (6.38698,22.7726), (10.50654,10.50512), (8.95362,-6.95825), (6.23711,9.20036), (11.75359,15.66902), (12.42155,3.28098), (-1.55472,-9.05692), (4.6688,0.32882), (10.48087,-1.64934), (11.74615,-4.81406), (9.26822,-5.06006), (7.55517,19.97493), (12.76005,2.88646), (16.47102,-0.34552), (11.31297,7.55186), (14.37437,-22.96115), (2.38799,31.29166), (6.44577,6.18798), (5.07471,-2.52715), (11.55123,-11.58799), (7.76795,14.13596), (10.60116,13.45069), (14.40885,12.15179), (11.58158,3.44491), (8.81648,-8.78006), (12.92299,18.32087), (11.26939,11.91757), (17.95014,-2.00179), (2.95002,10.88411), (17.41959,9.09327), (11.12455,6.62484), (8.78541,8.87178), (14.36413,11.52254), (12.98554,-14.15988), (12.58505,-17.19515), (15.49789,14.03089), (11.70999,-2.4095), (0.65596,-16.83575), (11.08202,2.71469), (14.75752,4.84351), (6.84385,-1.17651), (9.27245,-3.37529), (13.78243,-19.92137), (17.4863,4.48952), (4.01777,-12.4906), (11.82861,-5.65277), (13.86551,8.50819), (6.16591,-19.61261), (8.71589,12.54156), (16.77195,11.06784), (17.23243,-12.59285), (-2.12941,3.43683), (5.66629,-3.00325), (12.45153,12.49082), (1.63971,7.20955), (13.84031,17.6547), (4.6144,15.8619), (5.26169,24.3048), (9.27769,-8.05434), (9.14288,-6.06901), (9.71953,-15.69515), (9.38446,-11.13917), (1.64788,-3.90757), (11.72922,-2.57038), (13.68926,5.14065), (9.42952,17.8497), (12.05574,-8.64665), (9.09148,-18.68331), (5.32273,5.8567), (20.25258,-20.93884), (10.14599,4.40583), (10.82156,14.35985), (5.75736,4.18134), (7.13567,4.3635), (9.29746,9.35428), (5.1618,2.8908), (10.076,16.01017), (21.65669,-1.48499), (13.35486,-9.97949), (6.79957,1.03055), (8.76243,-2.79697), (14.59294,6.85977), (16.90609,4.73213), (10.50337,2.7815), (-0.07923,-2.46866), (13.51648,18.39425), (12.0676,-0.80378), (0.86482,-0.22982), (9.03563,-16.11608), (5.38751,3.0862), (17.16866,3.20779), (2.78702,10.50146), (11.15548,-0.21305), (12.30843,11.21012), (8.04897,-0.99825), (9.95814,18.39633), (11.29308,-3.39003), (14.13032,-0.64411), (21.05877,-1.39932), (3.57386,15.45319), (7.96631,-0.66044), (3.30484,-15.2223), (18.61856,-34.39907), (16.35184,-3.57836), (7.65236,16.82828), (18.02895,1.66624), (9.79458,15.43475), (16.7274,8.17776), (8.84453,5.50486), (13.05709,10.43082), (10.91447,-6.63332), (8.40171,2.28008), (16.95211,16.37203), (11.82194,5.16313), (19.87978,-8.85281), (12.88455,13.26692), (-0.00947,-7.46842), (12.28109,8.43091), (6.96462,-13.18172), (13.75282,-0.72401), (14.39141,22.3881), (11.07193,10.65448), (12.88039,2.81289), (11.38253,10.92405), (21.02707,-8.95358), (7.51955,19.80653), (6.31984,-12.86527), (15.6543,5.38826), (14.80315,-6.83501), (8.38024,-15.7647), (21.7516,-27.67412), (14.31336,8.6499), (15.04703,-4.89542), (5.73787,16.76167), (13.16911,12.84284), (12.40695,-17.27324), (9.88968,-4.18726), (8.46703,-14.62366), (8.70637,-5.49863), (8.03551,-16.22846), (5.9757,10.60329), (12.22951,6.46781), (3.14736,1.70458), (10.51266,10.77448), (18.593,0.8463), (10.82213,13.0482), (7.14216,-4.36264), (6.81154,3.22647), (-0.6486,2.38828), (20.56136,6.7946), (11.35367,-0.25254), (11.38205,1.2497), (17.14,1.6544), (14.91215,4.1019), (15.50207,11.27839), (5.93162,-5.04127), (3.74869,18.11674), (14.11532,0.51231), (7.38954,-0.51029), (5.45764,13.52556), (18.33733,16.10171), (9.91923,5.68197), (2.38991,-2.85904), (14.16756,-8.89167), (2.39791,6.24489), (6.92586,10.85319), (5.32474,-0.39816), (2.28812,3.87079), (5.71718,-3.1867), (5.84197,1.55322), (2.76206,16.86779), (19.05928,-14.60321), (11.51788,-1.81952), (6.56648,-3.11624), (3.35735,1.24193), (7.55948,10.18179), (19.99908,4.69796), (13.00634,0.69032), (18.36886,11.7723), (11.14675,7.62896), (16.72931,9.89741), (12.50106,9.11484), (6.00605,-3.84676), (23.06653,-0.4777), (5.39694,0.95958), (9.53167,-7.95056), (12.76944,-10.97474), (7.20604,-6.54861), (13.25391,34.74933), (13.7341,27.39463), (10.85292,4.18299), (-7.75835,6.02476), (10.29728,-1.99397), (13.70099,1.26478), (10.17959,23.37106), (9.98399,10.49682), (12.69389,-11.04354), (-0.28848,-12.22284), (-2.18319,-9.87635), (13.36378,28.90511), (10.09232,6.77613), (5.49489,0.55352), (5.46156,0.37031), (0.94225,7.1418), (12.79205,3.24897), (10.09593,-1.60918), (6.06218,3.1675), (0.89463,-17.97072), (11.88986,-5.61743), (10.79733,14.1422), (1.51371,14.87695), (2.20967,-4.65961), (15.45732,-0.99174), (16.5262,-2.96623), (5.99724,-9.02263), (8.3613,-17.2088), (15.68183,2.78608), (15.32117,6.74239), (14.15674,4.8524), (6.64553,7.46731), (4.20777,1.04894), (-0.10521,-12.8023), (-0.88169,-17.18188), (1.85913,-5.08801), (9.73673,22.13942), (0.30926,-0.36384), (6.17559,17.80564), (11.76602,7.67504), (5.68385,1.59779), (14.57088,4.10942), (12.81509,0.61074), (9.85682,-14.40767), (12.06376,10.59906), (6.08874,16.57017), (11.63921,-15.17526), (14.86722,-6.98549), (10.41035,-0.64548), (2.93794,3.23756), (12.21841,14.65504), (0.23804,4.583), (3.14845,12.72378), (7.29748,5.26547), (3.06134,0.81781), (13.77684,9.38273), (16.21992,10.37636), (5.33511,10.70325), (9.68959,-0.83043), (9.44169,-7.53149), (18.08012,-9.09147), (4.04224,-19.51381), (8.77918,-28.44508), (10.18324,6.44392), (9.38914,11.10201), (11.76995,-2.86184), (14.19963,8.30673), (6.88817,8.8797), (16.56123,10.68053), (15.39885,15.62919), (5.21241,8.00579), (4.44408,6.4651), (17.87587,-4.50029), (12.53337,18.04514), (13.60916,11.12996), (6.60104,-5.14007), (7.35453,9.43857), (18.61572,3.13476), (6.10437,4.9772), (13.08682,-17.45782), (12.15404,0.05552), (4.90789,-1.90283), (2.13353,2.67908), (12.49593,-2.62243), (11.93056,-3.22767), (13.29408,-8.70222), (5.70038,-23.11605), (8.40271,21.6757), (5.19456,12.70076), (-5.51028,4.4322), (14.0329,11.69344), (10.38365,9.18052), (6.56812,-2.2549), (4.21129,-2.15615), (9.7157,20.29765), (9.88553,-0.29536), (13.45346,15.50109), (4.97752,8.79187), (12.77595,5.11533), (8.56465,-20.44436), (4.27703,-3.00909), (18.12502,-4.48291), (12.45735,21.84462), (12.42912,1.94225), (12.08125,-2.81908), (10.85779,17.19418), (4.36013,-9.33528), (11.85062,-0.17346), (8.47776,0.03958), (9.60822,-35.17786), (11.3069,8.36887), (14.25525,-9.02292), (1.55168,-10.98804), (14.57782,0.29335), (7.84786,4.29634), (9.87774,3.87718), (14.75575,-9.08532), (3.68774,7.13922), (9.37667,-7.62463), (20.28676,-10.5666), (12.10027,4.68165), (8.01819,-3.30172), (18.78158,13.04852), (20.85402,13.45616), (18.98069,2.41043), (16.1429,-0.36501), (9.24047,-15.67383), (14.12487,17.92217), (10.18841,8.42106), (-3.04478,3.22063), (5.7552,-7.31753), (9.30376,21.99596), (11.42837,-36.8273), (6.02364,-20.46391), (8.86984,5.74179), (10.91177,-15.83178), (10.04418,14.90454), (18.10774,-8.84645), (7.49384,3.72036), (9.11556,4.6877), (9.7051,16.35418), (5.23268,3.15441), (9.04647,2.39907), (8.81547,-17.58664), (2.65098,-13.18269); -SELECT 14.971190998235835, 5.898143508382202e-44; +SELECT '14.971190998235835', '5.898143508382202e-44'; SELECT roundBankers(WelchTTest(left, right).1, 16) as t_stat, roundBankers(WelchTTest(left, right).2, 16) as p_value from welch_ttest; DROP TABLE IF EXISTS welch_ttest; From f9204135d8212640803fc7fdd5ce69e8902fd176 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Wed, 14 Oct 2020 16:54:02 +0300 Subject: [PATCH 070/325] add lgammal.c --- base/glibc-compatibility/musl/lgammal.c | 340 ++++++++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 base/glibc-compatibility/musl/lgammal.c diff --git a/base/glibc-compatibility/musl/lgammal.c b/base/glibc-compatibility/musl/lgammal.c new file mode 100644 index 00000000000..cc4a5e3b54d --- /dev/null +++ b/base/glibc-compatibility/musl/lgammal.c @@ -0,0 +1,340 @@ +/* origin: OpenBSD /usr/src/lib/libm/src/ld80/e_lgammal.c */ +/* + * ==================================================== + * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + * + * Developed at SunPro, a Sun Microsystems, Inc. business. + * Permission to use, copy, modify, and distribute this + * software is freely granted, provided that this notice + * is preserved. + * ==================================================== + */ +/* + * Copyright (c) 2008 Stephen L. Moshier + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +/* lgammal(x) + * Reentrant version of the logarithm of the Gamma function + * with user provide pointer for the sign of Gamma(x). + * + * Method: + * 1. Argument Reduction for 0 < x <= 8 + * Since gamma(1+s)=s*gamma(s), for x in [0,8], we may + * reduce x to a number in [1.5,2.5] by + * lgamma(1+s) = log(s) + lgamma(s) + * for example, + * lgamma(7.3) = log(6.3) + lgamma(6.3) + * = log(6.3*5.3) + lgamma(5.3) + * = log(6.3*5.3*4.3*3.3*2.3) + lgamma(2.3) + * 2. Polynomial approximation of lgamma around its + * minimun ymin=1.461632144968362245 to maintain monotonicity. + * On [ymin-0.23, ymin+0.27] (i.e., [1.23164,1.73163]), use + * Let z = x-ymin; + * lgamma(x) = -1.214862905358496078218 + z^2*poly(z) + * 2. Rational approximation in the primary interval [2,3] + * We use the following approximation: + * s = x-2.0; + * lgamma(x) = 0.5*s + s*P(s)/Q(s) + * Our algorithms are based on the following observation + * + * zeta(2)-1 2 zeta(3)-1 3 + * lgamma(2+s) = s*(1-Euler) + --------- * s - --------- * s + ... + * 2 3 + * + * where Euler = 0.5771... is the Euler constant, which is very + * close to 0.5. + * + * 3. For x>=8, we have + * lgamma(x)~(x-0.5)log(x)-x+0.5*log(2pi)+1/(12x)-1/(360x**3)+.... + * (better formula: + * lgamma(x)~(x-0.5)*(log(x)-1)-.5*(log(2pi)-1) + ...) + * Let z = 1/x, then we approximation + * f(z) = lgamma(x) - (x-0.5)(log(x)-1) + * by + * 3 5 11 + * w = w0 + w1*z + w2*z + w3*z + ... + w6*z + * + * 4. For negative x, since (G is gamma function) + * -x*G(-x)*G(x) = pi/sin(pi*x), + * we have + * G(x) = pi/(sin(pi*x)*(-x)*G(-x)) + * since G(-x) is positive, sign(G(x)) = sign(sin(pi*x)) for x<0 + * Hence, for x<0, signgam = sign(sin(pi*x)) and + * lgamma(x) = log(|Gamma(x)|) + * = log(pi/(|x*sin(pi*x)|)) - lgamma(-x); + * Note: one should avoid compute pi*(-x) directly in the + * computation of sin(pi*(-x)). + * + * 5. Special Cases + * lgamma(2+s) ~ s*(1-Euler) for tiny s + * lgamma(1)=lgamma(2)=0 + * lgamma(x) ~ -log(x) for tiny x + * lgamma(0) = lgamma(inf) = inf + * lgamma(-integer) = +-inf + * + */ + +static const long double pi = 3.14159265358979323846264L, + +/* lgam(1+x) = 0.5 x + x a(x)/b(x) + -0.268402099609375 <= x <= 0 + peak relative error 6.6e-22 */ +a0 = -6.343246574721079391729402781192128239938E2L, +a1 = 1.856560238672465796768677717168371401378E3L, +a2 = 2.404733102163746263689288466865843408429E3L, +a3 = 8.804188795790383497379532868917517596322E2L, +a4 = 1.135361354097447729740103745999661157426E2L, +a5 = 3.766956539107615557608581581190400021285E0L, + +b0 = 8.214973713960928795704317259806842490498E3L, +b1 = 1.026343508841367384879065363925870888012E4L, +b2 = 4.553337477045763320522762343132210919277E3L, +b3 = 8.506975785032585797446253359230031874803E2L, +b4 = 6.042447899703295436820744186992189445813E1L, +/* b5 = 1.000000000000000000000000000000000000000E0 */ + + +tc = 1.4616321449683623412626595423257213284682E0L, +tf = -1.2148629053584961146050602565082954242826E-1, /* double precision */ +/* tt = (tail of tf), i.e. tf + tt has extended precision. */ +tt = 3.3649914684731379602768989080467587736363E-18L, +/* lgam ( 1.4616321449683623412626595423257213284682E0 ) = +-1.2148629053584960809551455717769158215135617312999903886372437313313530E-1 */ + +/* lgam (x + tc) = tf + tt + x g(x)/h(x) + -0.230003726999612341262659542325721328468 <= x + <= 0.2699962730003876587373404576742786715318 + peak relative error 2.1e-21 */ +g0 = 3.645529916721223331888305293534095553827E-18L, +g1 = 5.126654642791082497002594216163574795690E3L, +g2 = 8.828603575854624811911631336122070070327E3L, +g3 = 5.464186426932117031234820886525701595203E3L, +g4 = 1.455427403530884193180776558102868592293E3L, +g5 = 1.541735456969245924860307497029155838446E2L, +g6 = 4.335498275274822298341872707453445815118E0L, + +h0 = 1.059584930106085509696730443974495979641E4L, +h1 = 2.147921653490043010629481226937850618860E4L, +h2 = 1.643014770044524804175197151958100656728E4L, +h3 = 5.869021995186925517228323497501767586078E3L, +h4 = 9.764244777714344488787381271643502742293E2L, +h5 = 6.442485441570592541741092969581997002349E1L, +/* h6 = 1.000000000000000000000000000000000000000E0 */ + + +/* lgam (x+1) = -0.5 x + x u(x)/v(x) + -0.100006103515625 <= x <= 0.231639862060546875 + peak relative error 1.3e-21 */ +u0 = -8.886217500092090678492242071879342025627E1L, +u1 = 6.840109978129177639438792958320783599310E2L, +u2 = 2.042626104514127267855588786511809932433E3L, +u3 = 1.911723903442667422201651063009856064275E3L, +u4 = 7.447065275665887457628865263491667767695E2L, +u5 = 1.132256494121790736268471016493103952637E2L, +u6 = 4.484398885516614191003094714505960972894E0L, + +v0 = 1.150830924194461522996462401210374632929E3L, +v1 = 3.399692260848747447377972081399737098610E3L, +v2 = 3.786631705644460255229513563657226008015E3L, +v3 = 1.966450123004478374557778781564114347876E3L, +v4 = 4.741359068914069299837355438370682773122E2L, +v5 = 4.508989649747184050907206782117647852364E1L, +/* v6 = 1.000000000000000000000000000000000000000E0 */ + + +/* lgam (x+2) = .5 x + x s(x)/r(x) + 0 <= x <= 1 + peak relative error 7.2e-22 */ +s0 = 1.454726263410661942989109455292824853344E6L, +s1 = -3.901428390086348447890408306153378922752E6L, +s2 = -6.573568698209374121847873064292963089438E6L, +s3 = -3.319055881485044417245964508099095984643E6L, +s4 = -7.094891568758439227560184618114707107977E5L, +s5 = -6.263426646464505837422314539808112478303E4L, +s6 = -1.684926520999477529949915657519454051529E3L, + +r0 = -1.883978160734303518163008696712983134698E7L, +r1 = -2.815206082812062064902202753264922306830E7L, +r2 = -1.600245495251915899081846093343626358398E7L, +r3 = -4.310526301881305003489257052083370058799E6L, +r4 = -5.563807682263923279438235987186184968542E5L, +r5 = -3.027734654434169996032905158145259713083E4L, +r6 = -4.501995652861105629217250715790764371267E2L, +/* r6 = 1.000000000000000000000000000000000000000E0 */ + + +/* lgam(x) = ( x - 0.5 ) * log(x) - x + LS2PI + 1/x w(1/x^2) + x >= 8 + Peak relative error 1.51e-21 +w0 = LS2PI - 0.5 */ +w0 = 4.189385332046727417803e-1L, +w1 = 8.333333333333331447505E-2L, +w2 = -2.777777777750349603440E-3L, +w3 = 7.936507795855070755671E-4L, +w4 = -5.952345851765688514613E-4L, +w5 = 8.412723297322498080632E-4L, +w6 = -1.880801938119376907179E-3L, +w7 = 4.885026142432270781165E-3L; + +/* sin(pi*x) assuming x > 2^-1000, if sin(pi*x)==0 the sign is arbitrary */ +static long double sin_pi(long double x) +{ + int n; + + /* spurious inexact if odd int */ + x *= 0.5; + x = 2.0*(x - floorl(x)); /* x mod 2.0 */ + + n = (int)(x*4.0); + n = (n+1)/2; + x -= n*0.5f; + x *= pi; + + switch (n) { + default: /* case 4: */ + case 0: return __sinl(x, 0.0, 0); + case 1: return __cosl(x, 0.0); + case 2: return __sinl(-x, 0.0, 0); + case 3: return -__cosl(x, 0.0); + } +} + +#include +#include + +long double __lgammal_r(long double x, int *sg) { + long double t, y, z, nadj, p, p1, p2, q, r, w; + union ldshape u = {x}; + uint32_t ix = (u.i.se & 0x7fffU)<<16 | u.i.m>>48; + int sign = u.i.se >> 15; + int i; + + *sg = 1; + + /* purge off +-inf, NaN, +-0, tiny and negative arguments */ + if (ix >= 0x7fff0000) + return x * x; + if (ix < 0x3fc08000) { /* |x|<2**-63, return -log(|x|) */ + if (sign) { + *sg = -1; + x = -x; + } + return -logl(x); + } + if (sign) { + x = -x; + t = sin_pi(x); + if (t == 0.0) + return 1.0 / (x-x); /* -integer */ + if (t > 0.0) + *sg = -1; + else + t = -t; + nadj = logl(pi / (t * x)); + } + + /* purge off 1 and 2 (so the sign is ok with downward rounding) */ + if ((ix == 0x3fff8000 || ix == 0x40008000) && u.i.m == 0) { + r = 0; + } else if (ix < 0x40008000) { /* x < 2.0 */ + if (ix <= 0x3ffee666) { /* 8.99993896484375e-1 */ + /* lgamma(x) = lgamma(x+1) - log(x) */ + r = -logl(x); + if (ix >= 0x3ffebb4a) { /* 7.31597900390625e-1 */ + y = x - 1.0; + i = 0; + } else if (ix >= 0x3ffced33) { /* 2.31639862060546875e-1 */ + y = x - (tc - 1.0); + i = 1; + } else { /* x < 0.23 */ + y = x; + i = 2; + } + } else { + r = 0.0; + if (ix >= 0x3fffdda6) { /* 1.73162841796875 */ + /* [1.7316,2] */ + y = x - 2.0; + i = 0; + } else if (ix >= 0x3fff9da6) { /* 1.23162841796875 */ + /* [1.23,1.73] */ + y = x - tc; + i = 1; + } else { + /* [0.9, 1.23] */ + y = x - 1.0; + i = 2; + } + } + switch (i) { + case 0: + p1 = a0 + y * (a1 + y * (a2 + y * (a3 + y * (a4 + y * a5)))); + p2 = b0 + y * (b1 + y * (b2 + y * (b3 + y * (b4 + y)))); + r += 0.5 * y + y * p1/p2; + break; + case 1: + p1 = g0 + y * (g1 + y * (g2 + y * (g3 + y * (g4 + y * (g5 + y * g6))))); + p2 = h0 + y * (h1 + y * (h2 + y * (h3 + y * (h4 + y * (h5 + y))))); + p = tt + y * p1/p2; + r += (tf + p); + break; + case 2: + p1 = y * (u0 + y * (u1 + y * (u2 + y * (u3 + y * (u4 + y * (u5 + y * u6)))))); + p2 = v0 + y * (v1 + y * (v2 + y * (v3 + y * (v4 + y * (v5 + y))))); + r += (-0.5 * y + p1 / p2); + } + } else if (ix < 0x40028000) { /* 8.0 */ + /* x < 8.0 */ + i = (int)x; + y = x - (double)i; + p = y * (s0 + y * (s1 + y * (s2 + y * (s3 + y * (s4 + y * (s5 + y * s6)))))); + q = r0 + y * (r1 + y * (r2 + y * (r3 + y * (r4 + y * (r5 + y * (r6 + y)))))); + r = 0.5 * y + p / q; + z = 1.0; + /* lgamma(1+s) = log(s) + lgamma(s) */ + switch (i) { + case 7: + z *= (y + 6.0); /* FALLTHRU */ + case 6: + z *= (y + 5.0); /* FALLTHRU */ + case 5: + z *= (y + 4.0); /* FALLTHRU */ + case 4: + z *= (y + 3.0); /* FALLTHRU */ + case 3: + z *= (y + 2.0); /* FALLTHRU */ + r += logl(z); + break; + } + } else if (ix < 0x40418000) { /* 2^66 */ + /* 8.0 <= x < 2**66 */ + t = logl(x); + z = 1.0 / x; + y = z * z; + w = w0 + z * (w1 + y * (w2 + y * (w3 + y * (w4 + y * (w5 + y * (w6 + y * w7)))))); + r = (x - 0.5) * (t - 1.0) + w; + } else /* 2**66 <= x <= inf */ + r = x * (logl(x) - 1.0); + if (sign) + r = nadj - r; + return r; +} + +int signgam; + +long double lgammal(long double x) +{ + return lgammal_r(x, &signgam); +} + From 252b9b2ec1212ca4569f56b2012d69cc83c8606b Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Wed, 14 Oct 2020 16:54:55 +0300 Subject: [PATCH 071/325] better --- base/glibc-compatibility/musl/lgammal.c | 26 ++----------------------- 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/base/glibc-compatibility/musl/lgammal.c b/base/glibc-compatibility/musl/lgammal.c index cc4a5e3b54d..1e720163a5d 100644 --- a/base/glibc-compatibility/musl/lgammal.c +++ b/base/glibc-compatibility/musl/lgammal.c @@ -187,31 +187,9 @@ w5 = 8.412723297322498080632E-4L, w6 = -1.880801938119376907179E-3L, w7 = 4.885026142432270781165E-3L; -/* sin(pi*x) assuming x > 2^-1000, if sin(pi*x)==0 the sign is arbitrary */ -static long double sin_pi(long double x) -{ - int n; - - /* spurious inexact if odd int */ - x *= 0.5; - x = 2.0*(x - floorl(x)); /* x mod 2.0 */ - - n = (int)(x*4.0); - n = (n+1)/2; - x -= n*0.5f; - x *= pi; - - switch (n) { - default: /* case 4: */ - case 0: return __sinl(x, 0.0, 0); - case 1: return __cosl(x, 0.0); - case 2: return __sinl(-x, 0.0, 0); - case 3: return -__cosl(x, 0.0); - } -} - #include #include +#include "libm.h" long double __lgammal_r(long double x, int *sg) { long double t, y, z, nadj, p, p1, p2, q, r, w; @@ -234,7 +212,7 @@ long double __lgammal_r(long double x, int *sg) { } if (sign) { x = -x; - t = sin_pi(x); + t = sin(pi * x); if (t == 0.0) return 1.0 / (x-x); /* -integer */ if (t > 0.0) From 72fddba2b1153598d9cc61c82b1ce49fbafde688 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 14 Oct 2020 17:08:51 +0300 Subject: [PATCH 072/325] more fasttest fixes --- docker/test/fasttest/run.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index a0830ba5f12..d77d7233893 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -33,6 +33,12 @@ server_pid=none function stop_server { + if kill -0 -- "$server_pid" + then + echo "ClickHouse server pid '$server_pid' is not running" + return 0 + fi + for _ in {1..60} do if ! pkill -f "clickhouse-server" && ! kill -- "$server_pid" ; then break ; fi @@ -167,8 +173,7 @@ clickhouse-client --version clickhouse-test --help mkdir -p "$FASTTEST_DATA"{,/client-config} -cp -a "$FASTTEST_SOURCE/programs/server/"{config,users}.xml "$FASTTEST_DATA" -cp -a "$FASTTEST_SOURCE/programs/server/"{config,users}.xml "$FASTTEST_DATA" +cp -a "$FASTTEST_SOURCE/programs/server/"{config,users}.{xml,d} "$FASTTEST_DATA" "$FASTTEST_SOURCE/tests/config/install.sh" "$FASTTEST_DATA" "$FASTTEST_DATA/client-config" # doesn't support SSL rm -f "$FASTTEST_DATA/config.d/secure_ports.xml" From ead6d5992c32921da59947ff2a7689ddcc9ca7d1 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 14 Oct 2020 18:53:14 +0300 Subject: [PATCH 073/325] fixup --- docker/test/fasttest/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index d77d7233893..299b748e4eb 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -33,7 +33,7 @@ server_pid=none function stop_server { - if kill -0 -- "$server_pid" + if ! kill -0 -- "$server_pid" then echo "ClickHouse server pid '$server_pid' is not running" return 0 @@ -274,7 +274,7 @@ then stop_server ||: # Clean the data so that there is no interference from the previous test run. - rm -rf "$FASTTEST_DATA"/{meta,}data ||: + rm -rf "$FASTTEST_DATA"/{{meta,}data,user_files} ||: start_server From f8852fcced01b368dcc6984e610d976494a61fe7 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Wed, 14 Oct 2020 20:01:03 +0300 Subject: [PATCH 074/325] Update 01150_ddl_guard_rwr.sh --- tests/queries/0_stateless/01150_ddl_guard_rwr.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01150_ddl_guard_rwr.sh b/tests/queries/0_stateless/01150_ddl_guard_rwr.sh index c14e4c38f54..43804075938 100755 --- a/tests/queries/0_stateless/01150_ddl_guard_rwr.sh +++ b/tests/queries/0_stateless/01150_ddl_guard_rwr.sh @@ -39,5 +39,6 @@ timeout 20 bash -c 'thread_rename' & wait sleep 1 +$CLICKHOUSE_CLIENT --query "DETACH DATABASE IF EXISTS test_01150" $CLICKHOUSE_CLIENT --query "ATTACH DATABASE IF NOT EXISTS test_01150" $CLICKHOUSE_CLIENT --query "DROP DATABASE test_01150"; From ff7601a52cea90d5594a35969417747219dd9585 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Wed, 14 Oct 2020 21:41:35 +0300 Subject: [PATCH 075/325] add python test and fix build --- base/glibc-compatibility/musl/lgammal.c | 4 +- .../AggregateFunctionStudentTTest.cpp | 2 +- .../AggregateFunctionStudentTTest.h | 28 ++--- .../AggregateFunctionWelchTTest.cpp | 2 +- .../AggregateFunctionWelchTTest.h | 8 +- .../queries/0_stateless/01322_ttest_scipy.py | 108 ++++++++++++++++++ .../0_stateless/01322_ttest_scipy.reference | 0 7 files changed, 126 insertions(+), 26 deletions(-) create mode 100644 tests/queries/0_stateless/01322_ttest_scipy.py create mode 100644 tests/queries/0_stateless/01322_ttest_scipy.reference diff --git a/base/glibc-compatibility/musl/lgammal.c b/base/glibc-compatibility/musl/lgammal.c index 1e720163a5d..534abf41894 100644 --- a/base/glibc-compatibility/musl/lgammal.c +++ b/base/glibc-compatibility/musl/lgammal.c @@ -309,10 +309,10 @@ long double __lgammal_r(long double x, int *sg) { return r; } -int signgam; +int signgam_lgammal; long double lgammal(long double x) { - return lgammal_r(x, &signgam); + return lgammal_r(x, &signgam_lgammal); } diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp index b6f32409946..b03e961a24a 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp @@ -50,7 +50,7 @@ AggregateFunctionPtr createAggregateFunctionStudentTTest(const std::string & nam void registerAggregateFunctionStudentTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("StudentTTest", createAggregateFunctionStudentTTest, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("studentTTest", createAggregateFunctionStudentTTest, AggregateFunctionFactory::CaseInsensitive); } } diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.h b/src/AggregateFunctions/AggregateFunctionStudentTTest.h index ac05a11d334..2f3d35dbeab 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.h +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.h @@ -98,8 +98,8 @@ struct AggregateFunctionStudentTTestData final Float64 getSSquared() const { - /// TODO: Update comment with Tex. - /// The original formulae looks like ... + /// The original formulae looks like + /// \frac{\sum_{i = 1}^{n_x}{(x_i - \bar{x}) ^ 2} + \sum_{i = 1}^{n_y}{(y_i - \bar{y}) ^ 2}}{n_x + n_y - 2} /// But we made some mathematical transformations not to store original sequences. /// Also we dropped sqrt, because later it will be squared later. const Float64 all_x = square_sum_x + size_x * std::pow(mean_x, 2) - 2 * mean_x * sum_x; @@ -110,26 +110,19 @@ struct AggregateFunctionStudentTTestData final Float64 getTStatisticSquared() const { - if (size_x == 0 || size_y == 0) - { - throw Exception("Division by zero encountered in Aggregate function StudentTTest", ErrorCodes::BAD_ARGUMENTS); - } - return std::pow(mean_x - mean_y, 2) / getStandartErrorSquared(); } Float64 getTStatistic() const { - if (size_x == 0 || size_y == 0) - { - throw Exception("Division by zero encountered in Aggregate function StudentTTest", ErrorCodes::BAD_ARGUMENTS); - } - return (mean_x - mean_y) / std::sqrt(getStandartErrorSquared()); } Float64 getStandartErrorSquared() const { + if (size_x == 0 || size_y == 0) + throw Exception("Division by zero encountered in Aggregate function StudentTTest", ErrorCodes::BAD_ARGUMENTS); + return getSSquared() * (1.0 / static_cast(size_x) + 1.0 / static_cast(size_y)); } @@ -138,9 +131,10 @@ struct AggregateFunctionStudentTTestData final return static_cast(size_x + size_y - 2); } - static Float64 integrateSimpson(Float64 a, Float64 b, std::function func, size_t iterations = 1e6) + static Float64 integrateSimpson(Float64 a, Float64 b, std::function func) { - double h = (b - a) / iterations; + const size_t iterations = std::max(1e6, 1e4 * std::abs(std::round(b))); + const long double h = (b - a) / iterations; Float64 sum_odds = 0.0; for (size_t i = 1; i < iterations; i += 2) sum_odds += func(a + i * h); @@ -154,13 +148,9 @@ struct AggregateFunctionStudentTTestData final { const Float64 v = getDegreesOfFreedom(); const Float64 t = getTStatisticSquared(); - std::cout << "getDegreesOfFreedom() " << getDegreesOfFreedom() << std::endl; - std::cout << "getTStatisticSquared() " << getTStatisticSquared() << std::endl; auto f = [&v] (double x) { return std::pow(x, v/2 - 1) / std::sqrt(1 - x); }; Float64 numenator = integrateSimpson(0, v / (t + v), f); Float64 denominator = std::exp(std::lgammal(v/2) + std::lgammal(0.5) - std::lgammal(v/2 + 0.5)); - std::cout << "numenator " << numenator << std::endl; - std::cout << "denominator " << denominator << std::endl; return numenator / denominator; } @@ -184,7 +174,7 @@ public: String getName() const override { - return "StudentTTest"; + return "studentTTest"; } DataTypePtr getReturnType() const override diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index d9fce97680c..00607171c41 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -50,7 +50,7 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("welchTTest", createAggregateFunctionWelchTTest, AggregateFunctionFactory::CaseInsensitive); } } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 36641b826b1..3332a6c363e 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -142,8 +142,9 @@ struct AggregateFunctionWelchTTestData final return numerator / (denominator_first + denominator_second); } - static Float64 integrateSimpson(Float64 a, Float64 b, std::function func, size_t iterations = 1e6) + static Float64 integrateSimpson(Float64 a, Float64 b, std::function func) { + size_t iterations = std::max(1e6, 1e4 * std::abs(std::round(b))); double h = (b - a) / iterations; Float64 sum_odds = 0.0; for (size_t i = 1; i < iterations; i += 2) @@ -170,7 +171,8 @@ struct AggregateFunctionWelchTTestData final } }; -/// Returns p-value +/// Returns tuple of (t-statistic, p-value) +/// https://cpb-us-w2.wpmucdn.com/voices.uchicago.edu/dist/9/1193/files/2016/01/05b-TandP.pdf template class AggregateFunctionWelchTTest : public IAggregateFunctionDataHelper,AggregateFunctionWelchTTest> @@ -183,7 +185,7 @@ public: String getName() const override { - return "WelchTTest"; + return "welchTTest"; } DataTypePtr getReturnType() const override diff --git a/tests/queries/0_stateless/01322_ttest_scipy.py b/tests/queries/0_stateless/01322_ttest_scipy.py new file mode 100644 index 00000000000..d8255cd8062 --- /dev/null +++ b/tests/queries/0_stateless/01322_ttest_scipy.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +import os +import io +import sys +import requests +import time +import pandas as pd +import numpy as np +from scipy import stats + +CLICKHOUSE_HOST = os.environ.get('CLICKHOUSE_HOST', '127.0.0.1') +CLICKHOUSE_PORT_HTTP = os.environ.get('CLICKHOUSE_PORT_HTTP', '8123') +CLICKHOUSE_SERVER_URL_STR = 'http://' + ':'.join(str(s) for s in [CLICKHOUSE_HOST, CLICKHOUSE_PORT_HTTP]) + "/" + +class ClickHouseClient: + def __init__(self, host = CLICKHOUSE_SERVER_URL_STR): + self.host = host + + def query(self, query, connection_timeout = 1500): + NUMBER_OF_TRIES = 30 + DELAY = 10 + + for i in range(NUMBER_OF_TRIES): + r = requests.post( + self.host, + params = {'timeout_before_checking_execution_speed': 120, 'max_execution_time': 6000}, + timeout = connection_timeout, + data = query) + if r.status_code == 200: + return r.text + else: + print('ATTENTION: try #%d failed' % i) + if i != (NUMBER_OF_TRIES-1): + print(query) + print(r.text) + time.sleep(DELAY*(i+1)) + else: + raise ValueError(r.text) + + def query_return_df(self, query, connection_timeout = 1500): + data = self.query(query, connection_timeout) + df = pd.read_csv(io.StringIO(data), sep = '\t') + return df + + def query_with_data(self, query, content): + content = content.encode('utf-8') + r = requests.post(self.host, data=content) + result = r.text + if r.status_code == 200: + return result + else: + raise ValueError(r.text) + +def test_and_check(name, a, b, t_stat, p_value): + client = ClickHouseClient() + client.query("DROP TABLE IF EXISTS ttest;") + client.query("CREATE TABLE ttest (left Float64, right Float64) ENGINE = Memory;"); + client.query("INSERT INTO ttest VALUES {};".format(", ".join(['({},{})'.format(i, j) for i,j in zip(a, b)]))) + + real = client.query_return_df( + "SELECT roundBankers({}(left, right).1, 16) as t_stat, ".format(name) + + "roundBankers({}(left, right).2, 16) as p_value ".format(name) + + "FROM ttest FORMAT TabSeparatedWithNames;") + real_t_stat = real['t_stat'][0] + real_p_value = real['p_value'][0] + assert(abs(real_t_stat - np.float64(t_stat) < 1e-4)), "clickhouse_t_stat {}, scipy_t_stat {}".format(real_t_stat, t_stat) + assert(abs(real_p_value - np.float64(p_value)) < 1e-4), "clickhouse_p_value {}, scipy_p_value {}".format(real_p_value, p_value) + client.query("DROP TABLE IF EXISTS ttest;") + + +def test_student(): + rvs1 = np.round(stats.norm.rvs(loc=1, scale=5,size=500), 5) + rvs2 = np.round(stats.norm.rvs(loc=10, scale=5,size=500), 5) + s, p = stats.ttest_ind(rvs1, rvs2, equal_var = True) + test_and_check("studentTTest", rvs1, rvs2, s, p) + + rvs1 = np.round(stats.norm.rvs(loc=0, scale=5,size=500), 5) + rvs2 = np.round(stats.norm.rvs(loc=0, scale=5,size=500), 5) + s, p = stats.ttest_ind(rvs1, rvs2, equal_var = True) + test_and_check("studentTTest", rvs1, rvs2, s, p) + + + rvs1 = np.round(stats.norm.rvs(loc=0, scale=10,size=65536), 5) + rvs2 = np.round(stats.norm.rvs(loc=5, scale=1,size=65536), 5) + s, p = stats.ttest_ind(rvs1, rvs2, equal_var = True) + test_and_check("studentTTest", rvs1, rvs2, s, p) + +def test_welch(): + rvs1 = np.round(stats.norm.rvs(loc=1, scale=15,size=500), 5) + rvs2 = np.round(stats.norm.rvs(loc=10, scale=5,size=500), 5) + s, p = stats.ttest_ind(rvs1, rvs2, equal_var = True) + test_and_check("studentTTest", rvs1, rvs2, s, p) + + rvs1 = np.round(stats.norm.rvs(loc=0, scale=7,size=500), 5) + rvs2 = np.round(stats.norm.rvs(loc=0, scale=3,size=500), 5) + s, p = stats.ttest_ind(rvs1, rvs2, equal_var = True) + test_and_check("studentTTest", rvs1, rvs2, s, p) + + + rvs1 = np.round(stats.norm.rvs(loc=0, scale=10,size=65536), 5) + rvs2 = np.round(stats.norm.rvs(loc=5, scale=1,size=65536), 5) + s, p = stats.ttest_ind(rvs1, rvs2, equal_var = True) + test_and_check("studentTTest", rvs1, rvs2, s, p) + +if __name__ == "__main__": + test_student() + test_welch() + print("Ok.") \ No newline at end of file diff --git a/tests/queries/0_stateless/01322_ttest_scipy.reference b/tests/queries/0_stateless/01322_ttest_scipy.reference new file mode 100644 index 00000000000..e69de29bb2d From 1f200f8bd20a08f7180a00bfc77aaf56a5169aba Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Wed, 14 Oct 2020 21:43:38 +0300 Subject: [PATCH 076/325] add reference --- tests/queries/0_stateless/01322_ttest_scipy.reference | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01322_ttest_scipy.reference b/tests/queries/0_stateless/01322_ttest_scipy.reference index e69de29bb2d..587579af915 100644 --- a/tests/queries/0_stateless/01322_ttest_scipy.reference +++ b/tests/queries/0_stateless/01322_ttest_scipy.reference @@ -0,0 +1 @@ +Ok. From 575354a6d39c91d37fd8ed3795c78c97985a946c Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Wed, 14 Oct 2020 21:52:23 +0300 Subject: [PATCH 077/325] add scipy to dockerfile --- docker/test/stateless/Dockerfile | 3 +++ docker/test/stateless_unbundled/Dockerfile | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 33eb1c29103..ca7ecf9cb9c 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -16,6 +16,7 @@ RUN apt-get update -y \ python3-lxml \ python3-requests \ python3-termcolor \ + python3-pip \ qemu-user-static \ sudo \ telnet \ @@ -23,6 +24,8 @@ RUN apt-get update -y \ unixodbc \ wget +RUN pip3 install numpy, scipy, pandas + RUN mkdir -p /tmp/clickhouse-odbc-tmp \ && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \ && cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib/ \ diff --git a/docker/test/stateless_unbundled/Dockerfile b/docker/test/stateless_unbundled/Dockerfile index f2fd28e4078..2f4663fa061 100644 --- a/docker/test/stateless_unbundled/Dockerfile +++ b/docker/test/stateless_unbundled/Dockerfile @@ -58,6 +58,7 @@ RUN apt-get --allow-unauthenticated update -y \ python3-lxml \ python3-requests \ python3-termcolor \ + python3-pip \ qemu-user-static \ sudo \ telnet \ @@ -66,7 +67,9 @@ RUN apt-get --allow-unauthenticated update -y \ unixodbc \ unixodbc-dev \ wget \ - zlib1g-dev + zlib1g-dev\ + +RUN pip3 install numpy, scipy, pandas RUN mkdir -p /tmp/clickhouse-odbc-tmp \ && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \ From 2d29eab934a6543653ebd1381215a9bf762452af Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Wed, 14 Oct 2020 22:00:20 +0300 Subject: [PATCH 078/325] better --- docker/test/stateless_unbundled/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/stateless_unbundled/Dockerfile b/docker/test/stateless_unbundled/Dockerfile index 2f4663fa061..528f1e0520c 100644 --- a/docker/test/stateless_unbundled/Dockerfile +++ b/docker/test/stateless_unbundled/Dockerfile @@ -67,7 +67,7 @@ RUN apt-get --allow-unauthenticated update -y \ unixodbc \ unixodbc-dev \ wget \ - zlib1g-dev\ + zlib1g-dev RUN pip3 install numpy, scipy, pandas From b1e15530614f29320cd0b21541d00e3fc3523515 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Wed, 14 Oct 2020 22:43:08 +0300 Subject: [PATCH 079/325] style + docker + tests --- docker/test/stateless/Dockerfile | 2 +- docker/test/stateless_unbundled/Dockerfile | 2 +- src/AggregateFunctions/AggregateFunctionStudentTTest.cpp | 4 ---- src/AggregateFunctions/AggregateFunctionStudentTTest.h | 2 +- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 7 ------- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 2 +- tests/queries/0_stateless/01322_ttest_scipy.sh | 8 ++++++++ 7 files changed, 12 insertions(+), 15 deletions(-) create mode 100755 tests/queries/0_stateless/01322_ttest_scipy.sh diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index ca7ecf9cb9c..8f2c7358bb8 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -24,7 +24,7 @@ RUN apt-get update -y \ unixodbc \ wget -RUN pip3 install numpy, scipy, pandas +RUN pip3 install numpy scipy pandas RUN mkdir -p /tmp/clickhouse-odbc-tmp \ && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \ diff --git a/docker/test/stateless_unbundled/Dockerfile b/docker/test/stateless_unbundled/Dockerfile index 528f1e0520c..345ba905412 100644 --- a/docker/test/stateless_unbundled/Dockerfile +++ b/docker/test/stateless_unbundled/Dockerfile @@ -69,7 +69,7 @@ RUN apt-get --allow-unauthenticated update -y \ wget \ zlib1g-dev -RUN pip3 install numpy, scipy, pandas +RUN pip3 install numpy scipy pandas RUN mkdir -p /tmp/clickhouse-odbc-tmp \ && wget -nv -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \ diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp index b03e961a24a..a2c36e43488 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp @@ -11,7 +11,6 @@ namespace ErrorCodes { -extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int NOT_IMPLEMENTED; } @@ -44,13 +43,10 @@ AggregateFunctionPtr createAggregateFunctionStudentTTest(const std::string & nam return res; } - } - void registerAggregateFunctionStudentTTest(AggregateFunctionFactory & factory) { factory.registerFunction("studentTTest", createAggregateFunctionStudentTTest, AggregateFunctionFactory::CaseInsensitive); } - } diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.h b/src/AggregateFunctions/AggregateFunctionStudentTTest.h index 2f3d35dbeab..2a4ec40e3c1 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.h +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.h @@ -163,7 +163,7 @@ struct AggregateFunctionStudentTTestData final /// Returns tuple of (t-statistic, p-value) /// https://cpb-us-w2.wpmucdn.com/voices.uchicago.edu/dist/9/1193/files/2016/01/05b-TandP.pdf template -class AggregateFunctionStudentTTest : +class AggregateFunctionStudentTTest : public IAggregateFunctionDataHelper,AggregateFunctionStudentTTest> { diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 00607171c41..483c99dde9b 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -6,12 +6,8 @@ #include #include - -// the return type is boolean (we use UInt8 as we do not have boolean in clickhouse) - namespace ErrorCodes { -extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int NOT_IMPLEMENTED; } @@ -44,13 +40,10 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, return res; } - } - void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { factory.registerFunction("welchTTest", createAggregateFunctionWelchTTest, AggregateFunctionFactory::CaseInsensitive); } - } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 3332a6c363e..b262ebb70af 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -174,7 +174,7 @@ struct AggregateFunctionWelchTTestData final /// Returns tuple of (t-statistic, p-value) /// https://cpb-us-w2.wpmucdn.com/voices.uchicago.edu/dist/9/1193/files/2016/01/05b-TandP.pdf template -class AggregateFunctionWelchTTest : +class AggregateFunctionWelchTTest : public IAggregateFunctionDataHelper,AggregateFunctionWelchTTest> { diff --git a/tests/queries/0_stateless/01322_ttest_scipy.sh b/tests/queries/0_stateless/01322_ttest_scipy.sh new file mode 100755 index 00000000000..10dc79614d4 --- /dev/null +++ b/tests/queries/0_stateless/01322_ttest_scipy.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +# We should have correct env vars from shell_config.sh to run this test + +python3 "$CURDIR"/01322_ttest_scipy.py \ No newline at end of file From 80f3de1359e119851f1de45e4250f4cf5f87c63d Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Thu, 15 Oct 2020 20:39:04 +0800 Subject: [PATCH 080/325] ISSUES-15883 try fix collate name --- src/Parsers/MySQL/ASTAlterCommand.cpp | 6 +-- src/Parsers/MySQL/ASTDeclareColumn.cpp | 4 +- src/Parsers/MySQL/ASTDeclareOption.cpp | 39 +++++--------------- src/Parsers/MySQL/ASTDeclareOption.h | 4 +- src/Parsers/MySQL/ASTDeclareTableOptions.cpp | 8 ++-- 5 files changed, 21 insertions(+), 40 deletions(-) diff --git a/src/Parsers/MySQL/ASTAlterCommand.cpp b/src/Parsers/MySQL/ASTAlterCommand.cpp index b6f2b925de0..92461635265 100644 --- a/src/Parsers/MySQL/ASTAlterCommand.cpp +++ b/src/Parsers/MySQL/ASTAlterCommand.cpp @@ -303,9 +303,9 @@ static inline bool parseOtherCommand(IParser::Pos & pos, ASTPtr & node, Expected OptionDescribe("ENABLE KEYS", "enable_keys", std::make_shared()), OptionDescribe("DISABLE KEYS", "enable_keys", std::make_shared()), /// TODO: with collate - OptionDescribe("CONVERT TO CHARACTER SET", "charset", std::make_shared()), - OptionDescribe("CHARACTER SET", "charset", std::make_shared()), - OptionDescribe("DEFAULT CHARACTER SET", "charset", std::make_shared()), + OptionDescribe("CONVERT TO CHARACTER SET", "charset", std::make_shared()), + OptionDescribe("CHARACTER SET", "charset", std::make_shared()), + OptionDescribe("DEFAULT CHARACTER SET", "charset", std::make_shared()), OptionDescribe("LOCK", "lock", std::make_shared()) } }; diff --git a/src/Parsers/MySQL/ASTDeclareColumn.cpp b/src/Parsers/MySQL/ASTDeclareColumn.cpp index 6d21f934858..3913c828ec3 100644 --- a/src/Parsers/MySQL/ASTDeclareColumn.cpp +++ b/src/Parsers/MySQL/ASTDeclareColumn.cpp @@ -51,8 +51,8 @@ static inline bool parseColumnDeclareOptions(IParser::Pos & pos, ASTPtr & node, OptionDescribe("UNIQUE", "unique_key", std::make_unique()), OptionDescribe("KEY", "primary_key", std::make_unique()), OptionDescribe("COMMENT", "comment", std::make_unique()), - OptionDescribe("CHARACTER SET", "charset_name", std::make_unique()), - OptionDescribe("COLLATE", "collate", std::make_unique()), + OptionDescribe("CHARACTER SET", "charset_name", std::make_unique()), + OptionDescribe("COLLATE", "collate", std::make_unique()), OptionDescribe("COLUMN_FORMAT", "column_format", std::make_unique()), OptionDescribe("STORAGE", "storage", std::make_unique()), OptionDescribe("AS", "generated", std::make_unique()), diff --git a/src/Parsers/MySQL/ASTDeclareOption.cpp b/src/Parsers/MySQL/ASTDeclareOption.cpp index 92ac5f0343e..17be639b630 100644 --- a/src/Parsers/MySQL/ASTDeclareOption.cpp +++ b/src/Parsers/MySQL/ASTDeclareOption.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { @@ -94,41 +95,21 @@ bool ParserAlwaysFalse::parseImpl(IParser::Pos & /*pos*/, ASTPtr & node, Expecte return true; } -bool ParserCharsetName::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected &) +bool ParserCharsetOrCollateName::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & expected) { - /// Identifier in backquotes or in double quotes - if (pos->type == TokenType::QuotedIdentifier) - { - ReadBufferFromMemory buf(pos->begin, pos->size()); - String s; + ParserIdentifier p_identifier; + ParserStringLiteral p_string_literal; - if (*pos->begin == '`') - readBackQuotedStringWithSQLStyle(s, buf); - else - readDoubleQuotedStringWithSQLStyle(s, buf); - - if (s.empty()) /// Identifiers "empty string" are not allowed. - return false; - - node = std::make_shared(s); - ++pos; + if (p_identifier.parse(pos, node, expected)) return true; - } - else if (pos->type == TokenType::BareWord) + else { - const char * begin = pos->begin; - - while (true) + if (p_string_literal.parse(pos, node, expected)) { - if (!isWhitespaceASCII(*pos->end) && pos->type != TokenType::EndOfStream) - ++pos; - else - break; + const auto & string_value = node->as()->value.safeGet(); + node = std::make_shared(string_value); + return true; } - - node = std::make_shared(String(begin, pos->end)); - ++pos; - return true; } return false; diff --git a/src/Parsers/MySQL/ASTDeclareOption.h b/src/Parsers/MySQL/ASTDeclareOption.h index 24800371061..2502618b209 100644 --- a/src/Parsers/MySQL/ASTDeclareOption.h +++ b/src/Parsers/MySQL/ASTDeclareOption.h @@ -61,10 +61,10 @@ public: /// Copy and paste from ParserIdentifier, /// the difference is that multiple tokens are glued if there is no whitespace ASCII between them -struct ParserCharsetName : public IParserBase +struct ParserCharsetOrCollateName : public IParserBase { protected: - const char * getName() const override { return "charset name"; } + const char * getName() const override { return "charset or collate name"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected &) override; }; diff --git a/src/Parsers/MySQL/ASTDeclareTableOptions.cpp b/src/Parsers/MySQL/ASTDeclareTableOptions.cpp index 87b99cdf1ac..c903c7d2fa7 100644 --- a/src/Parsers/MySQL/ASTDeclareTableOptions.cpp +++ b/src/Parsers/MySQL/ASTDeclareTableOptions.cpp @@ -68,12 +68,12 @@ bool ParserDeclareTableOptions::parseImpl(IParser::Pos & pos, ASTPtr & node, Exp { OptionDescribe("AUTO_INCREMENT", "auto_increment", std::make_shared()), OptionDescribe("AVG_ROW_LENGTH", "avg_row_length", std::make_shared()), - OptionDescribe("CHARSET", "character_set", std::make_shared()), - OptionDescribe("DEFAULT CHARSET", "character_set", std::make_shared()), - OptionDescribe("CHARACTER SET", "character_set", std::make_shared()), + OptionDescribe("CHARSET", "character_set", std::make_shared()), + OptionDescribe("DEFAULT CHARSET", "character_set", std::make_shared()), + OptionDescribe("CHARACTER SET", "character_set", std::make_shared()), OptionDescribe("DEFAULT CHARACTER SET", "character_set", std::make_shared()), OptionDescribe("CHECKSUM", "checksum", std::make_shared>()), - OptionDescribe("COLLATE", "collate", std::make_shared()), + OptionDescribe("COLLATE", "collate", std::make_shared()), OptionDescribe("DEFAULT COLLATE", "collate", std::make_shared()), OptionDescribe("COMMENT", "comment", std::make_shared()), OptionDescribe("COMPRESSION", "compression", std::make_shared()), From c8aa007a455372ab0b71a60f3405aea9beb5c5d6 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Thu, 15 Oct 2020 20:42:10 +0800 Subject: [PATCH 081/325] ISSUES-15883 modify comment --- src/Parsers/MySQL/ASTDeclareOption.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Parsers/MySQL/ASTDeclareOption.h b/src/Parsers/MySQL/ASTDeclareOption.h index 2502618b209..a9529924567 100644 --- a/src/Parsers/MySQL/ASTDeclareOption.h +++ b/src/Parsers/MySQL/ASTDeclareOption.h @@ -59,8 +59,7 @@ public: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; -/// Copy and paste from ParserIdentifier, -/// the difference is that multiple tokens are glued if there is no whitespace ASCII between them +/// identifier, string literal, binary keyword struct ParserCharsetOrCollateName : public IParserBase { protected: From ee8a9d2cf12220fb1420e2186cb8735256ed4fa5 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Thu, 15 Oct 2020 18:57:17 +0300 Subject: [PATCH 082/325] Don't touch MySQL database if it's unnecessary --- src/Databases/DatabaseLazy.h | 4 ++++ src/Databases/IDatabase.h | 4 ++++ src/Databases/MySQL/DatabaseConnectionMySQL.h | 6 ++++++ src/Interpreters/AsynchronousMetrics.cpp | 4 ++-- src/Server/ReplicasStatusHandler.cpp | 4 ++-- src/Storages/System/StorageSystemDistributionQueue.cpp | 4 ++-- src/Storages/System/StorageSystemGraphite.cpp | 4 ++-- src/Storages/System/StorageSystemMutations.cpp | 4 ++-- src/Storages/System/StorageSystemPartsBase.cpp | 6 +++--- src/Storages/System/StorageSystemReplicas.cpp | 4 ++-- src/Storages/System/StorageSystemReplicationQueue.cpp | 4 ++-- 11 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/Databases/DatabaseLazy.h b/src/Databases/DatabaseLazy.h index 13c14863efb..2d091297c91 100644 --- a/src/Databases/DatabaseLazy.h +++ b/src/Databases/DatabaseLazy.h @@ -22,6 +22,10 @@ public: String getEngineName() const override { return "Lazy"; } + bool canContainMergeTreeTables() const override { return false; } + + bool canContainDistributedTables() const override { return false; } + void loadStoredObjects( Context & context, bool has_force_restore_data_flag, bool force_attach) override; diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index b28bd5fd599..fadec5fe7a9 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -147,6 +147,10 @@ public: /// Get name of database engine. virtual String getEngineName() const = 0; + virtual bool canContainMergeTreeTables() const { return true; } + + virtual bool canContainDistributedTables() const { return true; } + /// Load a set of existing tables. /// You can call only once, right after the object is created. virtual void loadStoredObjects(Context & /*context*/, bool /*has_force_restore_data_flag*/, bool /*force_attach*/ = false) {} diff --git a/src/Databases/MySQL/DatabaseConnectionMySQL.h b/src/Databases/MySQL/DatabaseConnectionMySQL.h index 7bf5e8c1d88..d8694e71db2 100644 --- a/src/Databases/MySQL/DatabaseConnectionMySQL.h +++ b/src/Databases/MySQL/DatabaseConnectionMySQL.h @@ -42,6 +42,12 @@ public: String getEngineName() const override { return "MySQL"; } + bool canContainMergeTreeTables() const override { return false; } + + bool canContainDistributedTables() const override { return false; } + + bool shouldBeEmptyOnDetach() const override { return false; } + bool empty() const override; DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name) override; diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index feb2036a0d6..e1a9a820ebb 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -233,8 +233,8 @@ void AsynchronousMetrics::update() for (const auto & db : databases) { - /// Lazy database can not contain MergeTree tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain MergeTree tables + if (!db.second->canContainMergeTreeTables()) continue; for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index bc5436f00ee..1aa5c10afd7 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -43,8 +43,8 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request /// Iterate through all the replicated tables. for (const auto & db : databases) { - /// Lazy database can not contain replicated tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain replicated tables + if (!db.second->canContainMergeTreeTables()) continue; for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) diff --git a/src/Storages/System/StorageSystemDistributionQueue.cpp b/src/Storages/System/StorageSystemDistributionQueue.cpp index 2459be0ba71..39ccea64e26 100644 --- a/src/Storages/System/StorageSystemDistributionQueue.cpp +++ b/src/Storages/System/StorageSystemDistributionQueue.cpp @@ -38,8 +38,8 @@ void StorageSystemDistributionQueue::fillData(MutableColumns & res_columns, cons std::map> tables; for (const auto & db : DatabaseCatalog::instance().getDatabases()) { - /// Lazy database can not contain distributed tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain distributed tables + if (!db.second->canContainDistributedTables()) continue; const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); diff --git a/src/Storages/System/StorageSystemGraphite.cpp b/src/Storages/System/StorageSystemGraphite.cpp index ffa789a4751..93bc16785b2 100644 --- a/src/Storages/System/StorageSystemGraphite.cpp +++ b/src/Storages/System/StorageSystemGraphite.cpp @@ -32,8 +32,8 @@ static StorageSystemGraphite::Configs getConfigs(const Context & context) for (const auto & db : databases) { - /// Lazy database can not contain MergeTree tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain MergeTree tables + if (!db.second->canContainMergeTreeTables()) continue; for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp index 32f672b8401..f66f57ef5d1 100644 --- a/src/Storages/System/StorageSystemMutations.cpp +++ b/src/Storages/System/StorageSystemMutations.cpp @@ -44,8 +44,8 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, const Contex std::map> merge_tree_tables; for (const auto & db : DatabaseCatalog::instance().getDatabases()) { - /// Lazy database can not contain MergeTree tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain MergeTree tables + if (!db.second->canContainMergeTreeTables()) continue; const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index faa2ec0e1c3..d10346af89f 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -83,9 +83,9 @@ StoragesInfoStream::StoragesInfoStream(const SelectQueryInfo & query_info, const MutableColumnPtr database_column_mut = ColumnString::create(); for (const auto & database : databases) { - /// Lazy database can not contain MergeTree tables - /// and it's unnecessary to load all tables of Lazy database just to filter all of them. - if (database.second->getEngineName() != "Lazy") + /// Checck if database can contain MergeTree tables, + /// if not it's unnecessary to load all tables of database just to filter all of them. + if (database.second->canContainMergeTreeTables()) database_column_mut->insert(database.first); } block_to_filter.insert(ColumnWithTypeAndName( diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index 7ab6e939815..973ccfbf464 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -74,8 +74,8 @@ Pipe StorageSystemReplicas::read( std::map> replicated_tables; for (const auto & db : DatabaseCatalog::instance().getDatabases()) { - /// Lazy database can not contain replicated tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain replicated tables + if (db.second->canContainMergeTreeTables()) continue; const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) diff --git a/src/Storages/System/StorageSystemReplicationQueue.cpp b/src/Storages/System/StorageSystemReplicationQueue.cpp index f04d8759507..9cd5e8b8ff3 100644 --- a/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -55,8 +55,8 @@ void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, const std::map> replicated_tables; for (const auto & db : DatabaseCatalog::instance().getDatabases()) { - /// Lazy database can not contain replicated tables - if (db.second->getEngineName() == "Lazy") + /// Check if database can contain replicated tables + if (!db.second->canContainMergeTreeTables()) continue; const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); From f02840449e6b3c5bfcdb4d7cb03e6489e8e22b1f Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 15 Oct 2020 20:27:18 +0300 Subject: [PATCH 083/325] fix the test --- .../01455_opentelemetry_distributed.sh | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh index d3114ab66ff..446f713c11e 100755 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh @@ -63,14 +63,24 @@ check_log echo "===sampled===" query_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") -for _ in {1..200} +for i in {1..200} do ${CLICKHOUSE_CLIENT} \ --opentelemetry_start_trace_probability=0.1 \ - --query_id "$query_id" \ - --query "select 1 from remote('127.0.0.2', system, one) format Null" -done + --query_id "$query_id-$i" \ + --query "select 1 from remote('127.0.0.2', system, one) format Null" \ + & + # clickhouse-client is slow to start, so run them in parallel, but not too + # much. + if [[ $((i % 10)) -eq 0 ]] + then + wait + fi +done +wait + +${CLICKHOUSE_CLIENT} -q "system flush logs" ${CLICKHOUSE_CLIENT} -q " with count(*) as c -- expect 200 * 0.1 = 20 sampled events on average @@ -78,6 +88,8 @@ ${CLICKHOUSE_CLIENT} -q " from system.opentelemetry_log array join attribute.names as name, attribute.values as value where name = 'clickhouse.query_id' - and value = '$query_id' + and operation_name = 'query' + and parent_span_id = 0 -- only account for the initial queries + and value like '$query_id-%' ; " From dc8e165ce4fa98d715f2d3817057a4eb2d5b47cf Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 15 Oct 2020 20:27:26 +0300 Subject: [PATCH 084/325] improve fasttest usability --- docker/test/fasttest/Dockerfile | 1 - docker/test/fasttest/run.sh | 100 +++++++++--------- tests/clickhouse-test | 8 +- .../0_stateless/01293_show_clusters.reference | 6 -- .../0_stateless/01293_show_clusters.sql | 4 +- 5 files changed, 58 insertions(+), 61 deletions(-) diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 3cfa57bd747..6547a98c58b 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -56,7 +56,6 @@ RUN apt-get update \ python3-lxml \ python3-requests \ python3-termcolor \ - qemu-user-static \ rename \ software-properties-common \ tzdata \ diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index ad44cfc3cf9..f12ecbb2c9c 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -191,63 +191,65 @@ stop_server ||: start_server TESTS_TO_SKIP=( - parquet - avro - h3 - odbc - mysql - sha256 - _orc_ - arrow - 01098_temporary_and_external_tables - 01083_expressions_in_engine_arguments - hdfs - 00911_tautological_compare - protobuf - capnproto - java_hash - hashing - secure - 00490_special_line_separators_and_characters_outside_of_bmp - 00436_convert_charset 00105_shard_collations - 01354_order_by_tuple_collate_const - 01292_create_user - 01098_msgpack_format - 00929_multi_match_edit_distance - 00926_multimatch - 00834_cancel_http_readonly_queries_on_client_close - brotli - parallel_alter + 00109_shard_totals_after_having + 00110_external_sort 00302_http_compression 00417_kill_query - 01294_lazy_database_concurrent - 01193_metadata_loading - base64 - 01031_mutations_interpreter_and_context - json - client - 01305_replica_create_drop_zookeeper - 01092_memory_profiler - 01355_ilike - 01281_unsucceeded_insert_select_queries_counter - live_view - limit_memory - memory_limit - memory_leak - 00110_external_sort + 00436_convert_charset + 00490_special_line_separators_and_characters_outside_of_bmp + 00652_replicated_mutations_zookeeper 00682_empty_parts_merge 00701_rollup - 00109_shard_totals_after_having - ddl_dictionaries + 00834_cancel_http_readonly_queries_on_client_close + 00911_tautological_compare + 00926_multimatch + 00929_multi_match_edit_distance + 01031_mutations_interpreter_and_context + 01053_ssd_dictionary # this test mistakenly requires acces to /var/lib/clickhouse -- can't run this locally, disabled + 01083_expressions_in_engine_arguments + 01092_memory_profiler + 01098_msgpack_format + 01098_temporary_and_external_tables + 01103_check_cpu_instructions_at_startup # avoid dependency on qemu -- invonvenient when running locally + 01193_metadata_loading + 01238_http_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently 01251_dict_is_in_infinite_loop 01259_dictionary_custom_settings_ddl 01268_dictionary_direct_layout 01280_ssd_complex_key_dictionary - 00652_replicated_mutations_zookeeper - 01411_bayesian_ab_testing - 01238_http_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently 01281_group_by_limit_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently + 01281_unsucceeded_insert_select_queries_counter + 01292_create_user + 01294_lazy_database_concurrent + 01305_replica_create_drop_zookeeper + 01354_order_by_tuple_collate_const + 01355_ilike + 01411_bayesian_ab_testing + _orc_ + arrow + avro + base64 + brotli + capnproto + client + ddl_dictionaries + h3 + hashing + hdfs + java_hash + json + limit_memory + live_view + memory_leak + memory_limit + mysql + odbc + parallel_alter + parquet + protobuf + secure + sha256 # Not sure why these two fail even in sequential mode. Disabled for now # to make some progress. @@ -258,7 +260,7 @@ TESTS_TO_SKIP=( 01460_DistributedFilesToInsert ) -time clickhouse-test -j 8 --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt" +time clickhouse-test -j 8 --order=random --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt" # substr is to remove semicolon after test name readarray -t FAILED_TESTS < <(awk '/FAIL|TIMEOUT|ERROR/ { print substr($3, 1, length($3)-1) }' "$FASTTEST_OUTPUT/test_log.txt" | tee "$FASTTEST_OUTPUT/failed-parallel-tests.txt") @@ -281,7 +283,7 @@ then echo "Going to run again: ${FAILED_TESTS[*]}" - clickhouse-test --no-long --testname --shard --zookeeper "${FAILED_TESTS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a "$FASTTEST_OUTPUT/test_log.txt" + clickhouse-test --order=random --no-long --testname --shard --zookeeper "${FAILED_TESTS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a "$FASTTEST_OUTPUT/test_log.txt" else echo "No failed tests" fi diff --git a/tests/clickhouse-test b/tests/clickhouse-test index be8f9551db2..d5736001d60 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -353,20 +353,22 @@ def run_tests_array(all_tests_with_params): if os.path.isfile(stdout_file): print(", result:\n") - print(open(stdout_file).read()) + print('\n'.join(open(stdout_file).read().split('\n')[:100])) elif stderr: failures += 1 failures_chain += 1 print(MSG_FAIL, end='') print_test_time(total_time) - print(" - having stderror:\n{}".format(stderr)) + print(" - having stderror:\n{}".format( + '\n'.join(stderr.split('\n')[:100]))) elif 'Exception' in stdout: failures += 1 failures_chain += 1 print(MSG_FAIL, end='') print_test_time(total_time) - print(" - having exception:\n{}".format(stdout)) + print(" - having exception:\n{}".format( + '\n'.join(stdout.split('\n')[:100]))) elif not os.path.isfile(reference_file): print(MSG_UNKNOWN, end='') print_test_time(total_time) diff --git a/tests/queries/0_stateless/01293_show_clusters.reference b/tests/queries/0_stateless/01293_show_clusters.reference index 39e25143131..590ca348458 100644 --- a/tests/queries/0_stateless/01293_show_clusters.reference +++ b/tests/queries/0_stateless/01293_show_clusters.reference @@ -1,8 +1,2 @@ -test_cluster_two_shards -test_cluster_two_shards_different_databases -test_cluster_two_shards_localhost test_shard_localhost -test_shard_localhost_secure -test_unavailable_shard -test_cluster_two_shards test_shard_localhost 1 1 1 localhost ::1 9000 1 default 0 0 diff --git a/tests/queries/0_stateless/01293_show_clusters.sql b/tests/queries/0_stateless/01293_show_clusters.sql index af450680dac..ad5e51531e3 100644 --- a/tests/queries/0_stateless/01293_show_clusters.sql +++ b/tests/queries/0_stateless/01293_show_clusters.sql @@ -1,3 +1,3 @@ -show clusters; -show clusters like 'test%' limit 1; +-- don't show all clusters to reduce dependency on the configuration of server +show clusters like 'test_shard%' limit 1; show cluster 'test_shard_localhost'; From a80bbf6e0eff4db2222776b18a5f1a578784ba82 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Thu, 15 Oct 2020 22:54:17 +0300 Subject: [PATCH 085/325] fasttest + rm .py --- docker/test/fasttest/Dockerfile | 3 +++ ...test_scipy.py => 01322_ttest_scipy.python} | 0 .../queries/0_stateless/01322_ttest_scipy.sh | 2 +- .../01521_skip_unused_shards_bugfix.sql | 25 +++++++++++++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) rename tests/queries/0_stateless/{01322_ttest_scipy.py => 01322_ttest_scipy.python} (100%) create mode 100644 tests/queries/0_stateless/01521_skip_unused_shards_bugfix.sql diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 3cfa57bd747..23d7504230a 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -53,6 +53,7 @@ RUN apt-get update \ ninja-build \ psmisc \ python3 \ + python3-pip \ python3-lxml \ python3-requests \ python3-termcolor \ @@ -63,6 +64,8 @@ RUN apt-get update \ unixodbc \ --yes --no-install-recommends +RUN pip3 install numpy scipy pandas + # This symlink required by gcc to find lld compiler RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld diff --git a/tests/queries/0_stateless/01322_ttest_scipy.py b/tests/queries/0_stateless/01322_ttest_scipy.python similarity index 100% rename from tests/queries/0_stateless/01322_ttest_scipy.py rename to tests/queries/0_stateless/01322_ttest_scipy.python diff --git a/tests/queries/0_stateless/01322_ttest_scipy.sh b/tests/queries/0_stateless/01322_ttest_scipy.sh index 10dc79614d4..31c1acf3e60 100755 --- a/tests/queries/0_stateless/01322_ttest_scipy.sh +++ b/tests/queries/0_stateless/01322_ttest_scipy.sh @@ -5,4 +5,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # We should have correct env vars from shell_config.sh to run this test -python3 "$CURDIR"/01322_ttest_scipy.py \ No newline at end of file +python3 "$CURDIR"/01322_ttest_scipy.python diff --git a/tests/queries/0_stateless/01521_skip_unused_shards_bugfix.sql b/tests/queries/0_stateless/01521_skip_unused_shards_bugfix.sql new file mode 100644 index 00000000000..a6dd118bad3 --- /dev/null +++ b/tests/queries/0_stateless/01521_skip_unused_shards_bugfix.sql @@ -0,0 +1,25 @@ +DROP TABLE IF EXISTS mv; +DROP DATABASE IF EXISTS dict_01521; +CREATE DATABASE dict_01521; + +CREATE TABLE dict_01521.sharding_table (key UInt64, val UInt64) Engine=Memory(); + +CREATE DICTIONARY dict_01521.sharding_dict +( + key UInt64 DEFAULT 0, + val UInt8 DEFAULT 1 +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'sharding_table' PASSWORD '' DB 'dict_01521')) +LIFETIME(MIN 0 MAX 0) +LAYOUT(HASHED()); + +INSERT INTO dict_01521.sharding_table VALUES (150, 1), (151, 2); + +CREATE TABLE table_first (a UInt64, b UInt64) ENGINE = Memory; +CREATE TABLE table_second (a UInt64, b UInt64) ENGINE = Memory; + +CREATE TABLE table_distr (a Int) ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), 't_local'); + + + From 708fedbcf8bc700d21bdab7ecf66b590abb6fd75 Mon Sep 17 00:00:00 2001 From: MyroTk Date: Thu, 15 Oct 2020 22:23:49 +0200 Subject: [PATCH 086/325] testflows RBAC tests for views, distributed tables, public tables, and privileges: alter, updated select, updated insert, and show tables --- tests/testflows/example/regression.py | 6 +- tests/testflows/helpers/argparser.py | 15 +- tests/testflows/helpers/cluster.py | 89 +- tests/testflows/ldap/regression.py | 6 +- .../configs/clickhouse/config.d/remote.xml | 73 +- .../rbac/configs/clickhouse/config.d/ssl.xml | 1 + .../rbac/configs/clickhouse/config.xml | 20 +- .../docker-compose/clickhouse-service.yml | 0 .../rbac/docker-compose/docker-compose.yml | 4 +- .../rbac/docker-compose/zookeeper-service.yml | 8 +- tests/testflows/rbac/helper/__init__.py | 0 tests/testflows/rbac/helper/common.py | 124 + tests/testflows/rbac/helper/errors.py | 123 + tests/testflows/rbac/helper/tables.py | 41 + tests/testflows/rbac/regression.py | 118 +- tests/testflows/rbac/requirements/__init__.py | 2 +- .../rbac/requirements/requirements.md | 959 ++++++- .../rbac/requirements/requirements.py | 1332 +++++++++- .../rbac/tests/privileges/__init__.py | 7 + .../rbac/tests/privileges/alter/__init__.py | 0 .../tests/privileges/alter/alter_column.py | 993 ++++++++ .../privileges/alter/alter_constraint.py | 629 +++++ .../tests/privileges/alter/alter_index.py | 815 ++++++ .../tests/privileges/alter/alter_settings.py | 476 ++++ .../rbac/tests/privileges/alter/alter_ttl.py | 605 +++++ .../tests/privileges/distributed_table.py | 1000 ++++++++ .../rbac/tests/privileges/feature.py | 24 +- .../testflows/rbac/tests/privileges/insert.py | 570 +++-- .../rbac/tests/privileges/public_tables.py | 38 + .../testflows/rbac/tests/privileges/select.py | 419 ++- .../rbac/tests/privileges/show_tables.py | 62 + .../rbac/tests/syntax/alter_quota.py | 8 +- .../testflows/rbac/tests/syntax/alter_role.py | 2 +- .../rbac/tests/syntax/alter_row_policy.py | 2 +- .../tests/syntax/alter_settings_profile.py | 10 +- .../testflows/rbac/tests/syntax/alter_user.py | 2 +- .../rbac/tests/syntax/create_quota.py | 6 +- .../rbac/tests/syntax/create_role.py | 6 +- .../rbac/tests/syntax/create_row_policy.py | 8 +- .../tests/syntax/create_settings_profile.py | 4 +- .../rbac/tests/syntax/create_user.py | 2 +- .../testflows/rbac/tests/syntax/drop_quota.py | 2 +- .../testflows/rbac/tests/syntax/drop_role.py | 6 +- .../rbac/tests/syntax/drop_row_policy.py | 2 +- .../tests/syntax/drop_settings_profile.py | 2 +- .../testflows/rbac/tests/syntax/drop_user.py | 2 +- .../rbac/tests/syntax/grant_privilege.py | 6 +- .../testflows/rbac/tests/syntax/grant_role.py | 10 +- .../rbac/tests/syntax/revoke_privilege.py | 52 +- .../rbac/tests/syntax/revoke_role.py | 6 +- .../rbac/tests/syntax/set_default_role.py | 4 +- tests/testflows/rbac/tests/syntax/set_role.py | 2 +- .../rbac/tests/syntax/show_create_role.py | 2 +- tests/testflows/rbac/tests/views/__init__.py | 0 tests/testflows/rbac/tests/views/feature.py | 20 + tests/testflows/rbac/tests/views/live_view.py | 1141 +++++++++ .../rbac/tests/views/materialized_view.py | 2268 +++++++++++++++++ tests/testflows/rbac/tests/views/view.py | 1162 +++++++++ tests/testflows/regression.py | 4 +- 59 files changed, 12433 insertions(+), 867 deletions(-) mode change 100644 => 100755 tests/testflows/helpers/cluster.py mode change 100644 => 100755 tests/testflows/rbac/docker-compose/clickhouse-service.yml mode change 100644 => 100755 tests/testflows/rbac/docker-compose/docker-compose.yml mode change 100644 => 100755 tests/testflows/rbac/docker-compose/zookeeper-service.yml create mode 100644 tests/testflows/rbac/helper/__init__.py create mode 100755 tests/testflows/rbac/helper/common.py create mode 100755 tests/testflows/rbac/helper/errors.py create mode 100755 tests/testflows/rbac/helper/tables.py mode change 100644 => 100755 tests/testflows/rbac/requirements/requirements.py create mode 100755 tests/testflows/rbac/tests/privileges/alter/__init__.py create mode 100755 tests/testflows/rbac/tests/privileges/alter/alter_column.py create mode 100755 tests/testflows/rbac/tests/privileges/alter/alter_constraint.py create mode 100755 tests/testflows/rbac/tests/privileges/alter/alter_index.py create mode 100755 tests/testflows/rbac/tests/privileges/alter/alter_settings.py create mode 100755 tests/testflows/rbac/tests/privileges/alter/alter_ttl.py create mode 100755 tests/testflows/rbac/tests/privileges/distributed_table.py create mode 100755 tests/testflows/rbac/tests/privileges/public_tables.py mode change 100644 => 100755 tests/testflows/rbac/tests/privileges/select.py create mode 100755 tests/testflows/rbac/tests/privileges/show_tables.py create mode 100755 tests/testflows/rbac/tests/views/__init__.py create mode 100755 tests/testflows/rbac/tests/views/feature.py create mode 100755 tests/testflows/rbac/tests/views/live_view.py create mode 100755 tests/testflows/rbac/tests/views/materialized_view.py create mode 100755 tests/testflows/rbac/tests/views/view.py diff --git a/tests/testflows/example/regression.py b/tests/testflows/example/regression.py index 2c0a778d39b..cb58b42ba4c 100755 --- a/tests/testflows/example/regression.py +++ b/tests/testflows/example/regression.py @@ -2,7 +2,7 @@ import sys from testflows.core import * -append_path(sys.path, "..") +append_path(sys.path, "..") from helpers.cluster import Cluster from helpers.argparser import argparser @@ -10,13 +10,13 @@ from helpers.argparser import argparser @TestFeature @Name("example") @ArgumentParser(argparser) -def regression(self, local, clickhouse_binary_path): +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """Simple example of how you can use TestFlows to test ClickHouse. """ nodes = { "clickhouse": ("clickhouse1",), } - + with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: self.context.cluster = cluster diff --git a/tests/testflows/helpers/argparser.py b/tests/testflows/helpers/argparser.py index 033c15a3bfe..03014becb76 100644 --- a/tests/testflows/helpers/argparser.py +++ b/tests/testflows/helpers/argparser.py @@ -1,5 +1,12 @@ import os +def onoff(v): + if v in ["yes", "1", "on"]: + return True + elif v in ["no", "0", "off"]: + return False + raise ValueError(f"invalid {v}") + def argparser(parser): """Default argument parser for regressions. """ @@ -10,4 +17,10 @@ def argparser(parser): parser.add_argument("--clickhouse-binary-path", type=str, dest="clickhouse_binary_path", help="path to ClickHouse binary, default: /usr/bin/clickhouse", metavar="path", - default=os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH", "/usr/bin/clickhouse")) \ No newline at end of file + default=os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH", "/usr/bin/clickhouse")) + + parser.add_argument("--stress", action="store_true", default=False, + help="enable stress testing (might take a long time)") + + parser.add_argument("--parallel", type=onoff, default=True, choices=["yes", "no", "on", "off", 0, 1], + help="enable parallelism for tests that support it") \ No newline at end of file diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py old mode 100644 new mode 100755 index 8fda8ac43d8..490a9f4e17e --- a/tests/testflows/helpers/cluster.py +++ b/tests/testflows/helpers/cluster.py @@ -7,6 +7,7 @@ import tempfile from testflows.core import * from testflows.asserts import error from testflows.connect import Shell +from testflows.uexpect import ExpectTimeoutError class QueryRuntimeException(Exception): """Exception during query execution on the server. @@ -78,32 +79,43 @@ class ClickHouseNode(Node): def query(self, sql, message=None, exitcode=None, steps=True, no_checks=False, raise_on_exception=False, step=By, settings=None, *args, **kwargs): """Execute and check query. - :param sql: sql query :param message: expected message that should be in the output, default: None :param exitcode: expected exitcode, default: None """ + settings = list(settings or []) + + if hasattr(current().context, "default_query_settings"): + settings += current().context.default_query_settings + if len(sql) > 1024: with tempfile.NamedTemporaryFile("w", encoding="utf-8") as query: query.write(sql) query.flush() command = f"cat \"{query.name}\" | {self.cluster.docker_compose} exec -T {self.name} clickhouse client -n" - for setting in settings or []: + for setting in settings: name, value = setting command += f" --{name} \"{value}\"" description = f""" echo -e \"{sql[:100]}...\" > {query.name} {command} """ - with step("executing command", description=description) if steps else NullStep(): - r = self.cluster.bash(None)(command, *args, **kwargs) + with step("executing command", description=description, format_description=False) if steps else NullStep(): + try: + r = self.cluster.bash(None)(command, *args, **kwargs) + except ExpectTimeoutError: + self.cluster.close_bash(None) else: command = f"echo -e \"{sql}\" | clickhouse client -n" - for setting in settings or []: + for setting in settings: name, value = setting command += f" --{name} \"{value}\"" - with step("executing command", description=command) if steps else NullStep(): - r = self.cluster.bash(self.name)(command, *args, **kwargs) + with step("executing command", description=command, format_description=False) if steps else NullStep(): + try: + r = self.cluster.bash(self.name)(command, *args, **kwargs) + except ExpectTimeoutError: + self.cluster.close_bash(self.name) + raise if no_checks: return r @@ -134,6 +146,7 @@ class Cluster(object): docker_compose="docker-compose", docker_compose_project_dir=None, docker_compose_file="docker-compose.yml"): + self.terminating = False self._bash = {} self.clickhouse_binary_path = clickhouse_binary_path self.configs_dir = configs_dir @@ -183,11 +196,19 @@ class Cluster(object): def bash(self, node, timeout=120): """Returns thread-local bash terminal to a specific node. - :param node: name of the service """ + test = current() + + if self.terminating: + if test and (test.cflags & MANDATORY): + pass + else: + raise InterruptedError("terminating") + current_thread = threading.current_thread() - id = f"{current_thread.ident}-{node}" + id = f"{current_thread.name}-{node}" + with self.lock: if self._bash.get(id) is None: if node is None: @@ -196,9 +217,30 @@ class Cluster(object): self._bash[id] = Shell(command=[ "/bin/bash", "--noediting", "-c", f"{self.docker_compose} exec {node} bash --noediting" ], name=node).__enter__() + self._bash[id].timeout = timeout + + # clean up any stale open shells for threads that have exited + active_thread_names = {thread.name for thread in threading.enumerate()} + + for bash_id in list(self._bash.keys()): + thread_name, node_name = bash_id.rsplit("-", 1) + if thread_name not in active_thread_names: + self._bash[bash_id].__exit__(None, None, None) + del self._bash[bash_id] + return self._bash[id] + def close_bash(self, node): + current_thread = threading.current_thread() + id = f"{current_thread.name}-{node}" + + with self.lock: + if self._bash.get(id) is None: + return + self._bash[id].__exit__(None, None, None) + del self._bash[id] + def __enter__(self): with Given("docker-compose cluster"): self.up() @@ -210,20 +252,21 @@ class Cluster(object): self.down() finally: with self.lock: - for shell in list(self._bash.values()): + for shell in self._bash.values(): shell.__exit__(type, value, traceback) def node(self, name): """Get object with node bound methods. - :param name: name of service name """ if name.startswith("clickhouse"): return ClickHouseNode(self, name) return Node(self, name) - def down(self, timeout=120): + def down(self, timeout=300): """Bring cluster down by executing docker-compose down.""" + self.terminating = True + try: bash = self.bash(None) with self.lock: @@ -235,7 +278,7 @@ class Cluster(object): else: self._bash[id] = shell finally: - return self.command(None, f"{self.docker_compose} down", timeout=timeout) + return self.command(None, f"{self.docker_compose} down", bash=bash, timeout=timeout) def up(self, timeout=30*60): if self.local: @@ -264,7 +307,7 @@ class Cluster(object): if cmd.exitcode != 0: continue with And("executing docker-compose down just in case it is up"): - cmd = self.command(None, f"{self.docker_compose} down 2>&1 | tee", exitcode=None, timeout=timeout) + cmd = self.command(None, f"{self.docker_compose} down --remove-orphans 2>&1 | tee", exitcode=None, timeout=timeout) if cmd.exitcode != 0: continue with And("executing docker-compose up"): @@ -285,22 +328,26 @@ class Cluster(object): for name in self.nodes["clickhouse"]: self.node(name).wait_healthy() - def command(self, node, command, message=None, exitcode=None, steps=True, *args, **kwargs): + def command(self, node, command, message=None, exitcode=None, steps=True, bash=None, *args, **kwargs): """Execute and check command. - :param node: name of the service :param command: command :param message: expected message that should be in the output, default: None :param exitcode: expected exitcode, default: None :param steps: don't break command into steps, default: True """ - debug(f"command() {node}, {command}") - with By("executing command", description=command) if steps else NullStep(): - r = self.bash(node)(command, *args, **kwargs) + with By("executing command", description=command, format_description=False) if steps else NullStep(): + if bash is None: + bash = self.bash(node) + try: + r = bash(command, *args, **kwargs) + except ExpectTimeoutError: + self.close_bash(node) + raise if exitcode is not None: - with Then(f"exitcode should be {exitcode}") if steps else NullStep(): + with Then(f"exitcode should be {exitcode}", format_name=False) if steps else NullStep(): assert r.exitcode == exitcode, error(r.output) if message is not None: - with Then(f"output should contain message", description=message) if steps else NullStep(): + with Then(f"output should contain message", description=message, format_description=False) if steps else NullStep(): assert message in r.output, error(r.output) return r diff --git a/tests/testflows/ldap/regression.py b/tests/testflows/ldap/regression.py index 567807fc0a8..1c0edcb57d9 100755 --- a/tests/testflows/ldap/regression.py +++ b/tests/testflows/ldap/regression.py @@ -2,7 +2,7 @@ import sys from testflows.core import * -append_path(sys.path, "..") +append_path(sys.path, "..") from helpers.cluster import Cluster from helpers.argparser import argparser @@ -33,13 +33,13 @@ xfails = { RQ_SRS_007_LDAP_Authentication("1.0") ) @XFails(xfails) -def regression(self, local, clickhouse_binary_path): +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """ClickHouse integration with LDAP regression module. """ nodes = { "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), } - + with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: self.context.cluster = cluster diff --git a/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml b/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml index ada8eec5fc9..a7ed0d6e2b4 100644 --- a/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml +++ b/tests/testflows/rbac/configs/clickhouse/config.d/remote.xml @@ -58,9 +58,44 @@ 9440 1 - - - + + + + + + clickhouse1 + 9440 + 1 + + + + + + + clickhouse1 + 9000 + + + + + clickhouse2 + 9000 + + + + + + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + + clickhouse2 @@ -73,8 +108,20 @@ 9000 - - + + + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + clickhouse1 @@ -94,6 +141,22 @@ + + + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + diff --git a/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml b/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml index ca65ffd5e04..768d2250b79 100644 --- a/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml +++ b/tests/testflows/rbac/configs/clickhouse/config.d/ssl.xml @@ -3,6 +3,7 @@ /etc/clickhouse-server/ssl/server.crt /etc/clickhouse-server/ssl/server.key + /etc/clickhouse-server/ssl/dhparam.pem none true diff --git a/tests/testflows/rbac/configs/clickhouse/config.xml b/tests/testflows/rbac/configs/clickhouse/config.xml index 65187edf806..4ec12232539 100644 --- a/tests/testflows/rbac/configs/clickhouse/config.xml +++ b/tests/testflows/rbac/configs/clickhouse/config.xml @@ -69,7 +69,7 @@ - + 0.0.0.0 /var/lib/clickhouse/access/ + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + users.xml @@ -160,7 +172,7 @@ - + @@ -220,7 +232,7 @@ See https://clickhouse.yandex/docs/en/table_engines/replication/ --> - + - + - + + 86400 + diff --git a/tests/integration/test_MemoryTracking/test.py b/tests/integration/test_MemoryTracking/test.py index 6009e8cc953..a0ad8dc519d 100644 --- a/tests/integration/test_MemoryTracking/test.py +++ b/tests/integration/test_MemoryTracking/test.py @@ -8,7 +8,10 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/no_system_log.xml']) +node = cluster.add_instance('node', main_configs=[ + 'configs/no_system_log.xml', + 'configs/asynchronous_metrics_update_period_s.xml', +]) logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(logging.StreamHandler()) From 21315127a0adfade3050c1d4f1662077beed9068 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Sat, 24 Oct 2020 10:05:54 +0800 Subject: [PATCH 162/325] remove unused codes in AggregateFunctionGroupBitmapData --- .../AggregateFunctionGroupBitmapData.h | 124 ------------------ 1 file changed, 124 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h b/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h index 20788318b3d..d80e5e81f19 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h +++ b/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h @@ -12,7 +12,6 @@ // TODO: find out what it is. On github, they have proper interface headers like // this one: https://github.com/RoaringBitmap/CRoaring/blob/master/include/roaring/roaring.h #include -void * containerptr_roaring_bitmap_add(roaring_bitmap_t * r, uint32_t val, uint8_t * typecode, int * index); namespace DB { @@ -599,129 +598,6 @@ public: } } -private: - - /// To read and write the DB Buffer directly, migrate code from CRoaring - void db_roaring_bitmap_add_many(DB::ReadBuffer & db_buf, roaring_bitmap_t * r, size_t n_args) - { - void * container = nullptr; // hold value of last container touched - uint8_t typecode = 0; // typecode of last container touched - uint32_t prev = 0; // previous valued inserted - size_t i = 0; // index of value - int containerindex = 0; - if (n_args == 0) - return; - uint32_t val; - readBinary(val, db_buf); - container = containerptr_roaring_bitmap_add(r, val, &typecode, &containerindex); - prev = val; - ++i; - for (; i < n_args; ++i) - { - readBinary(val, db_buf); - if (((prev ^ val) >> 16) == 0) - { // no need to seek the container, it is at hand - // because we already have the container at hand, we can do the - // insertion - // automatically, bypassing the roaring_bitmap_add call - uint8_t newtypecode = typecode; - void * container2 = container_add(container, val & 0xFFFF, typecode, &newtypecode); - // rare instance when we need to - if (container2 != container) - { - // change the container type - container_free(container, typecode); - ra_set_container_at_index(&r->high_low_container, containerindex, container2, newtypecode); - typecode = newtypecode; - container = container2; - } - } - else - { - container = containerptr_roaring_bitmap_add(r, val, &typecode, &containerindex); - } - prev = val; - } - } - - void db_ra_to_uint32_array(DB::WriteBuffer & db_buf, roaring_array_t * ra) const - { - size_t ctr = 0; - for (Int32 i = 0; i < ra->size; ++i) - { - Int32 num_added = db_container_to_uint32_array(db_buf, ra->containers[i], ra->typecodes[i], (static_cast(ra->keys[i])) << 16); - ctr += num_added; - } - } - - UInt32 db_container_to_uint32_array(DB::WriteBuffer & db_buf, const void * container, uint8_t typecode, UInt32 base) const - { - container = container_unwrap_shared(container, &typecode); - switch (typecode) - { - case BITSET_CONTAINER_TYPE_CODE: - return db_bitset_container_to_uint32_array(db_buf, static_cast(container), base); - case ARRAY_CONTAINER_TYPE_CODE: - return db_array_container_to_uint32_array(db_buf, static_cast(container), base); - case RUN_CONTAINER_TYPE_CODE: - return db_run_container_to_uint32_array(db_buf, static_cast(container), base); - } - return 0; - } - - UInt32 db_bitset_container_to_uint32_array(DB::WriteBuffer & db_buf, const bitset_container_t * cont, UInt32 base) const - { - return static_cast(db_bitset_extract_setbits(db_buf, cont->array, BITSET_CONTAINER_SIZE_IN_WORDS, base)); - } - - size_t db_bitset_extract_setbits(DB::WriteBuffer & db_buf, UInt64 * bitset, size_t length, UInt32 base) const - { - UInt32 outpos = 0; - for (size_t i = 0; i < length; ++i) - { - UInt64 w = bitset[i]; - while (w != 0) - { - UInt64 t = w & (~w + 1); // on x64, should compile to BLSI (careful: the Intel compiler seems to fail) - UInt32 r = __builtin_ctzll(w); // on x64, should compile to TZCNT - UInt32 val = r + base; - writePODBinary(val, db_buf); - outpos++; - w ^= t; - } - base += 64; - } - return outpos; - } - - int db_array_container_to_uint32_array(DB::WriteBuffer & db_buf, const array_container_t * cont, UInt32 base) const - { - UInt32 outpos = 0; - for (Int32 i = 0; i < cont->cardinality; ++i) - { - const UInt32 val = base + cont->array[i]; - writePODBinary(val, db_buf); - outpos++; - } - return outpos; - } - - int db_run_container_to_uint32_array(DB::WriteBuffer & db_buf, const run_container_t * cont, UInt32 base) const - { - UInt32 outpos = 0; - for (Int32 i = 0; i < cont->n_runs; ++i) - { - UInt32 run_start = base + cont->runs[i].value; - UInt16 le = cont->runs[i].length; - for (Int32 j = 0; j <= le; ++j) - { - UInt32 val = run_start + j; - writePODBinary(val, db_buf); - outpos++; - } - } - return outpos; - } }; template From e354108e532b59c98f61ca4e72415543178a9451 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Sat, 24 Oct 2020 10:54:02 +0800 Subject: [PATCH 163/325] ISSUES-15883 trigger CI From 74558a4e27d14ff7dc6ae21b5e7dc10cf3f48d06 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Oct 2020 06:41:47 +0300 Subject: [PATCH 164/325] Better diagnostics when client has dropped connection --- src/Server/TCPHandler.cpp | 26 +++++++++++++++++++++++--- src/Server/TCPHandler.h | 1 + 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 406e29ba4ab..a37f88f9306 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -277,6 +277,9 @@ void TCPHandler::runImpl() /// Do it before sending end of stream, to have a chance to show log message in client. query_scope->logPeakMemoryUsage(); + if (state.is_connection_closed) + break; + sendLogs(); sendEndOfStream(); @@ -444,7 +447,11 @@ bool TCPHandler::readDataNext(const size_t & poll_interval, const int & receive_ /// If client disconnected. if (in->eof()) + { + LOG_INFO(log, "Client has dropped the connection, cancel the query."); + state.is_connection_closed = true; return false; + } /// We accept and process data. And if they are over, then we leave. if (!receivePacket()) @@ -477,9 +484,8 @@ void TCPHandler::readData(const Settings & connection_settings) std::tie(poll_interval, receive_timeout) = getReadTimeouts(connection_settings); sendLogs(); - while (true) - if (!readDataNext(poll_interval, receive_timeout)) - return; + while (readDataNext(poll_interval, receive_timeout)) + ; } @@ -567,6 +573,9 @@ void TCPHandler::processOrdinaryQuery() sendProgress(); } + if (state.is_connection_closed) + return; + sendData({}); } @@ -632,6 +641,9 @@ void TCPHandler::processOrdinaryQueryWithProcessors() sendLogs(); } + if (state.is_connection_closed) + return; + sendData({}); } @@ -1179,6 +1191,14 @@ bool TCPHandler::isQueryCancelled() /// During request execution the only packet that can come from the client is stopping the query. if (static_cast(*in).poll(0)) { + if (in->eof()) + { + LOG_INFO(log, "Client has dropped the connection, cancel the query."); + state.is_cancelled = true; + state.is_connection_closed = true; + return true; + } + UInt64 packet_type = 0; readVarUInt(packet_type, *in); diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 12149d9a66f..2f2bf35e59e 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -57,6 +57,7 @@ struct QueryState /// Is request cancelled bool is_cancelled = false; + bool is_connection_closed = false; /// empty or not bool is_empty = true; /// Data was sent. From 2bbb663fdd2cadc1fae7b3a78767132a1b9363e8 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 24 Oct 2020 07:30:39 +0300 Subject: [PATCH 165/325] Update programs/local/LocalServer.cpp Co-authored-by: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> --- programs/local/LocalServer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 3a1aecbc1cf..db1838c38f7 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -48,7 +48,6 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; - extern const int LOGICAL_ERROR; extern const int SYNTAX_ERROR; extern const int CANNOT_LOAD_CONFIG; extern const int FILE_ALREADY_EXISTS; From 0f3caaffe400ef44d82630a65551cd42763872a2 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 24 Oct 2020 07:48:15 +0300 Subject: [PATCH 166/325] Trigger CI --- tests/integration/test_disabled_mysql_server/test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_disabled_mysql_server/test.py b/tests/integration/test_disabled_mysql_server/test.py index 34dd094af3d..df5f123c6bd 100644 --- a/tests/integration/test_disabled_mysql_server/test.py +++ b/tests/integration/test_disabled_mysql_server/test.py @@ -51,7 +51,6 @@ def test_disabled_mysql_server(started_cluster): with PartitionManager() as pm: clickhouse_node.query("CREATE DATABASE test_db ENGINE = MySQL('mysql1:3306', 'test_db', 'root', 'clickhouse')") - pm._add_rule({'source': clickhouse_node.ip_address, 'destination_port': 3306, 'action': 'DROP'}) clickhouse_node.query("SELECT * FROM system.parts") From 0748377ab17e719caab311a8480bc23c52768df2 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 24 Oct 2020 07:59:08 +0300 Subject: [PATCH 167/325] Revert "Optionally upload clickhouse binary in fast test" --- docker/test/fasttest/Dockerfile | 1 - docker/test/fasttest/run.sh | 3 --- 2 files changed, 4 deletions(-) diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 0fff738e718..6547a98c58b 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -79,7 +79,6 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone ENV COMMIT_SHA='' ENV PULL_REQUEST_NUMBER='' -ENV COPY_CLICKHOUSE_BINARY_TO_OUTPUT=0 COPY run.sh / CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 74fdbeedc08..91fe84a04cd 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -172,9 +172,6 @@ function build ( cd "$FASTTEST_BUILD" time ninja clickhouse-bundle | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/build_log.txt" -if [ "$COPY_CLICKHOUSE_BINARY_TO_OUTPUT" -eq "1" ]; then - cp programs/clickhouse "$FASTTEST_OUTPUT/clickhouse" -fi ccache --show-stats ||: ) } From c3c6ac39e012fd3eba4bc33925fdfbb39be70984 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 24 Oct 2020 09:06:49 +0300 Subject: [PATCH 168/325] Tune TTL of the background query in 01541_max_memory_usage_for_user --- tests/queries/0_stateless/01541_max_memory_usage_for_user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01541_max_memory_usage_for_user.sh b/tests/queries/0_stateless/01541_max_memory_usage_for_user.sh index 7544ecfb026..945f202a803 100755 --- a/tests/queries/0_stateless/01541_max_memory_usage_for_user.sh +++ b/tests/queries/0_stateless/01541_max_memory_usage_for_user.sh @@ -35,7 +35,7 @@ function execute_tcp_one_session() # one users query in background (to avoid reseting max_memory_usage_for_user) # --max_block_size=1 to make it killable (check the state each 1 second, 1 row) # (the test takes ~40 seconds in debug build, so 60 seconds is ok) -${CLICKHOUSE_CLIENT} --max_block_size=1 --format Null -q 'SELECT sleepEachRow(1) FROM numbers(60)' & +${CLICKHOUSE_CLIENT} --max_block_size=1 --format Null -q 'SELECT sleepEachRow(1) FROM numbers(600)' & # trap sleep_query_pid=$! function cleanup() From 85c69aad7f3c4fbda047ca2edbead8bae8b0f39a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Oct 2020 09:23:54 +0300 Subject: [PATCH 169/325] Add check for BOM --- utils/check-style/check-style | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index a65ffeb8c67..4983782c00d 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -94,3 +94,8 @@ find $ROOT_PATH/{src,programs,utils} -name '*.h' | while read file; do [[ $(head # Check for executable bit on non-executable files find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} '(' -name '*.cpp' -or -name '*.h' -or -name '*.sql' -or -name '*.xml' -or -name '*.reference' -or -name '*.txt' -or -name '*.md' ')' -and -executable | grep -q '.' && echo "These files should not be executable." + +# Check for BOM +find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xEF\xBB\xBF' && echo "Files should not have UTF-8 BOM" +find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xFF\xFE' && echo "Files should not have UTF-16LE BOM" +find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xFE\xFF' && echo "Files should not have UTF-16BE BOM" From 97d84dd85549dc7c6f635d86f70bf2bfa1b1fc89 Mon Sep 17 00:00:00 2001 From: Xianda Ke Date: Sat, 24 Oct 2020 17:37:51 +0800 Subject: [PATCH 170/325] minor fix. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit redundant variable, duplicated defination of the variable aad_data compiler error: variable ‘aad_value’ set but not used [-Werror=unused-but-set-variable] --- src/Functions/FunctionsAES.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index 6600931118e..10c4a27e509 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -307,11 +307,6 @@ private: } const auto input_value = input_column->getDataAt(r); - auto aad_value = StringRef{}; - if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM && !std::is_same_v>) - { - aad_value = aad_column->getDataAt(r); - } if constexpr (mode != CipherMode::MySQLCompatibility) { From 34b9d15b66c1b468fb2d84f7097076bef8d17a63 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 24 Oct 2020 21:34:54 +0300 Subject: [PATCH 171/325] Update ThreadStatusExt.cpp --- src/Interpreters/ThreadStatusExt.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index d324307b487..2166c78ef7c 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -300,7 +300,7 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits) performance_counters.setParent(&ProfileEvents::global_counters); memory_tracker.reset(); - /// Must reset pointer to thread_group's memory_tracker, because it will be destroyed two lines below. + /// Must reset pointer to thread_group's memory_tracker, because it will be destroyed two lines below (will reset to its parent). memory_tracker.setParent(thread_group->memory_tracker.getParent()); query_id.clear(); From 1d170f57457fd575714797615b65f2304746e06c Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Sat, 24 Oct 2020 21:46:10 +0300 Subject: [PATCH 172/325] ASTTableIdentifier Part #1: improve internal representation of ASTIdentifier name (#16149) * Use only |name_parts| as primary name source * Restore legacy logic for table restoration * Fix build * Fix tests * Add pytest server config * Fix tests * Fixes due to review --- programs/client/Client.cpp | 2 +- programs/copier/Internals.cpp | 6 +- src/Compression/CompressionFactory.cpp | 4 +- src/DataTypes/DataTypeFactory.cpp | 2 +- .../getDictionaryConfigurationFromAST.cpp | 4 +- src/Interpreters/ActionsVisitor.cpp | 2 +- src/Interpreters/AddDefaultDatabaseVisitor.h | 15 +-- src/Interpreters/ArrayJoinedColumnsVisitor.h | 14 +-- src/Interpreters/CollectJoinOnKeysVisitor.cpp | 12 +-- .../ExtractExpressionInfoVisitor.cpp | 2 +- src/Interpreters/IdentifierSemantic.cpp | 31 +++--- src/Interpreters/IdentifierSemantic.h | 10 +- .../JoinToSubqueryTransformVisitor.cpp | 28 +++--- src/Interpreters/JoinedTables.cpp | 8 +- src/Interpreters/QueryNormalizer.cpp | 8 +- src/Interpreters/RenameColumnVisitor.cpp | 5 +- .../RequiredSourceColumnsData.cpp | 4 +- .../RequiredSourceColumnsVisitor.cpp | 7 +- .../TranslateQualifiedNamesVisitor.cpp | 11 ++- .../evaluateConstantExpression.cpp | 4 +- src/Interpreters/getClusterName.cpp | 2 +- src/Parsers/ASTColumnsTransformers.cpp | 2 +- src/Parsers/ASTIdentifier.cpp | 99 +++++++++++-------- src/Parsers/ASTIdentifier.h | 43 ++++---- src/Parsers/ExpressionElementParsers.cpp | 12 +-- src/Parsers/ExpressionListParsers.cpp | 14 ++- src/Parsers/MySQL/ASTDeclareConstraint.cpp | 2 +- src/Parsers/MySQL/ASTDeclareIndex.cpp | 18 ++-- src/Parsers/MySQL/ASTDeclarePartition.cpp | 2 +- src/Parsers/MySQL/ASTDeclareReference.cpp | 2 +- src/Parsers/MySQL/ASTDeclareSubPartition.cpp | 2 +- .../MySQL/tests/gtest_column_parser.cpp | 8 +- .../MySQL/tests/gtest_constraint_parser.cpp | 10 +- .../MySQL/tests/gtest_create_parser.cpp | 2 +- .../MySQL/tests/gtest_index_parser.cpp | 12 +-- .../tests/gtest_partition_options_parser.cpp | 38 +++---- .../MySQL/tests/gtest_partition_parser.cpp | 34 +++---- .../MySQL/tests/gtest_reference_parser.cpp | 22 ++--- .../MySQL/tests/gtest_subpartition_parser.cpp | 8 +- .../tests/gtest_table_options_parser.cpp | 24 ++--- src/Parsers/ParserCreateQuery.cpp | 4 +- src/Parsers/ParserDictionary.cpp | 4 +- src/Parsers/ParserSystemQuery.cpp | 4 +- src/Parsers/tests/gtest_dictionary_parser.cpp | 2 +- src/Storages/AlterCommands.cpp | 10 +- src/Storages/MutationCommands.cpp | 8 +- src/Storages/StorageDistributed.cpp | 2 +- .../System/StorageSystemZooKeeper.cpp | 2 +- tests/queries/server.py | 41 +++++--- utils/db-generator/query_db_generator.cpp | 14 +-- 50 files changed, 322 insertions(+), 304 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 07f2a231afe..3cd584c0e55 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1896,7 +1896,7 @@ private: if (has_vertical_output_suffix) throw Exception("Output format already specified", ErrorCodes::CLIENT_OUTPUT_FORMAT_SPECIFIED); const auto & id = query_with_output->format->as(); - current_format = id.name; + current_format = id.name(); } } diff --git a/programs/copier/Internals.cpp b/programs/copier/Internals.cpp index 24b5d616af4..0f607ea5faf 100644 --- a/programs/copier/Internals.cpp +++ b/programs/copier/Internals.cpp @@ -168,11 +168,11 @@ ASTPtr extractOrderBy(const ASTPtr & storage_ast) throw Exception("ORDER BY cannot be empty", ErrorCodes::BAD_ARGUMENTS); } -/// Wraps only identifiers with backticks. +/// Wraps only identifiers with backticks. std::string wrapIdentifiersWithBackticks(const ASTPtr & root) { if (auto identifier = std::dynamic_pointer_cast(root)) - return backQuote(identifier->name); + return backQuote(identifier->name()); if (auto function = std::dynamic_pointer_cast(root)) return function->name + '(' + wrapIdentifiersWithBackticks(function->arguments) + ')'; @@ -214,7 +214,7 @@ Names extractPrimaryKeyColumnNames(const ASTPtr & storage_ast) for (size_t i = 0; i < sorting_key_size; ++i) { /// Column name could be represented as a f_1(f_2(...f_n(column_name))). - /// Each f_i could take one or more parameters. + /// Each f_i could take one or more parameters. /// We will wrap identifiers with backticks to allow non-standart identifier names. String sorting_key_column = sorting_key_expr_list->children[i]->getColumnName(); diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index 91b4aa4b8de..fe6a5b2dacd 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -76,7 +76,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr ASTPtr codec_arguments; if (const auto * family_name = inner_codec_ast->as()) { - codec_family_name = family_name->name; + codec_family_name = family_name->name(); codec_arguments = {}; } else if (const auto * ast_func = inner_codec_ast->as()) @@ -207,7 +207,7 @@ CompressionCodecPtr CompressionCodecFactory::get(const ASTPtr & ast, const IData ASTPtr codec_arguments; if (const auto * family_name = inner_codec_ast->as()) { - codec_family_name = family_name->name; + codec_family_name = family_name->name(); codec_arguments = {}; } else if (const auto * ast_func = inner_codec_ast->as()) diff --git a/src/DataTypes/DataTypeFactory.cpp b/src/DataTypes/DataTypeFactory.cpp index 9386f4b39f1..5052a065163 100644 --- a/src/DataTypes/DataTypeFactory.cpp +++ b/src/DataTypes/DataTypeFactory.cpp @@ -43,7 +43,7 @@ DataTypePtr DataTypeFactory::get(const ASTPtr & ast) const if (const auto * ident = ast->as()) { - return get(ident->name, {}); + return get(ident->name(), {}); } if (const auto * lit = ast->as()) diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp index b1962e48eea..430c1d591dd 100644 --- a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp +++ b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp @@ -172,7 +172,7 @@ Names getPrimaryKeyColumns(const ASTExpressionList * primary_key) for (size_t index = 0; index != children.size(); ++index) { const ASTIdentifier * key_part = children[index]->as(); - result.push_back(key_part->name); + result.push_back(key_part->name()); } return result; } @@ -367,7 +367,7 @@ void buildConfigurationFromFunctionWithKeyValueArguments( if (const auto * identifier = pair->second->as(); identifier) { - AutoPtr value(doc->createTextNode(identifier->name)); + AutoPtr value(doc->createTextNode(identifier->name())); current_xml_element->appendChild(value); } else if (const auto * literal = pair->second->as(); literal) diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index e0e921b003b..96da40e8f6c 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -582,7 +582,7 @@ void ActionsMatcher::visit(const ASTIdentifier & identifier, const ASTPtr & ast, /// Special check for WITH statement alias. Add alias action to be able to use this alias. if (identifier.prefer_alias_to_column_name && !identifier.alias.empty()) - data.addAlias(identifier.name, identifier.alias); + data.addAlias(identifier.name(), identifier.alias); } } diff --git a/src/Interpreters/AddDefaultDatabaseVisitor.h b/src/Interpreters/AddDefaultDatabaseVisitor.h index 9322232c154..bb684c5547a 100644 --- a/src/Interpreters/AddDefaultDatabaseVisitor.h +++ b/src/Interpreters/AddDefaultDatabaseVisitor.h @@ -24,11 +24,12 @@ namespace DB class AddDefaultDatabaseVisitor { public: - AddDefaultDatabaseVisitor(const String & database_name_, bool only_replace_current_database_function_ = false, std::ostream * ostr_ = nullptr) - : database_name(database_name_), - only_replace_current_database_function(only_replace_current_database_function_), - visit_depth(0), - ostr(ostr_) + explicit AddDefaultDatabaseVisitor( + const String & database_name_, bool only_replace_current_database_function_ = false, std::ostream * ostr_ = nullptr) + : database_name(database_name_) + , only_replace_current_database_function(only_replace_current_database_function_) + , visit_depth(0) + , ostr(ostr_) {} void visitDDL(ASTPtr & ast) const @@ -105,7 +106,7 @@ private: void visit(const ASTIdentifier & identifier, ASTPtr & ast) const { if (!identifier.compound()) - ast = createTableIdentifier(database_name, identifier.name); + ast = createTableIdentifier(database_name, identifier.name()); } void visit(ASTSubquery & subquery, ASTPtr &) const @@ -116,7 +117,7 @@ private: void visit(ASTFunction & function, ASTPtr &) const { bool is_operator_in = false; - for (auto name : {"in", "notIn", "globalIn", "globalNotIn"}) + for (const auto * name : {"in", "notIn", "globalIn", "globalNotIn"}) { if (function.name == name) { diff --git a/src/Interpreters/ArrayJoinedColumnsVisitor.h b/src/Interpreters/ArrayJoinedColumnsVisitor.h index 56832914b80..94f6bdaf138 100644 --- a/src/Interpreters/ArrayJoinedColumnsVisitor.h +++ b/src/Interpreters/ArrayJoinedColumnsVisitor.h @@ -98,33 +98,33 @@ private: if (!IdentifierSemantic::getColumnName(node)) return; - auto split = Nested::splitName(node.name); /// ParsedParams, Key1 + auto split = Nested::splitName(node.name()); /// ParsedParams, Key1 - if (array_join_alias_to_name.count(node.name)) + if (array_join_alias_to_name.count(node.name())) { /// ARRAY JOIN was written with an array column. Example: SELECT K1 FROM ... ARRAY JOIN ParsedParams.Key1 AS K1 - array_join_result_to_source[node.name] = array_join_alias_to_name[node.name]; /// K1 -> ParsedParams.Key1 + array_join_result_to_source[node.name()] = array_join_alias_to_name[node.name()]; /// K1 -> ParsedParams.Key1 } else if (array_join_alias_to_name.count(split.first) && !split.second.empty()) { /// ARRAY JOIN was written with a nested table. Example: SELECT PP.KEY1 FROM ... ARRAY JOIN ParsedParams AS PP - array_join_result_to_source[node.name] /// PP.Key1 -> ParsedParams.Key1 + array_join_result_to_source[node.name()] /// PP.Key1 -> ParsedParams.Key1 = Nested::concatenateName(array_join_alias_to_name[split.first], split.second); } - else if (array_join_name_to_alias.count(node.name)) + else if (array_join_name_to_alias.count(node.name())) { /** Example: SELECT ParsedParams.Key1 FROM ... ARRAY JOIN ParsedParams.Key1 AS PP.Key1. * That is, the query uses the original array, replicated by itself. */ array_join_result_to_source[ /// PP.Key1 -> ParsedParams.Key1 - array_join_name_to_alias[node.name]] = node.name; + array_join_name_to_alias[node.name()]] = node.name(); } else if (array_join_name_to_alias.count(split.first) && !split.second.empty()) { /** Example: SELECT ParsedParams.Key1 FROM ... ARRAY JOIN ParsedParams AS PP. */ array_join_result_to_source[ /// PP.Key1 -> ParsedParams.Key1 - Nested::concatenateName(array_join_name_to_alias[split.first], split.second)] = node.name; + Nested::concatenateName(array_join_name_to_alias[split.first], split.second)] = node.name(); } } }; diff --git a/src/Interpreters/CollectJoinOnKeysVisitor.cpp b/src/Interpreters/CollectJoinOnKeysVisitor.cpp index e0fce4854d2..48273e32209 100644 --- a/src/Interpreters/CollectJoinOnKeysVisitor.cpp +++ b/src/Interpreters/CollectJoinOnKeysVisitor.cpp @@ -144,11 +144,11 @@ std::pair CollectJoinOnKeysMatcher::getTableNumbers(const ASTPtr const ASTIdentifier * CollectJoinOnKeysMatcher::unrollAliases(const ASTIdentifier * identifier, const Aliases & aliases) { - if (identifier->compound()) + if (identifier->supposedToBeCompound()) return identifier; UInt32 max_attempts = 100; - for (auto it = aliases.find(identifier->name); it != aliases.end();) + for (auto it = aliases.find(identifier->name()); it != aliases.end();) { const ASTIdentifier * parent = identifier; identifier = it->second->as(); @@ -156,12 +156,12 @@ const ASTIdentifier * CollectJoinOnKeysMatcher::unrollAliases(const ASTIdentifie break; /// not a column alias if (identifier == parent) break; /// alias to itself with the same name: 'a as a' - if (identifier->compound()) + if (identifier->supposedToBeCompound()) break; /// not an alias. Break to prevent cycle through short names: 'a as b, t1.b as a' - it = aliases.find(identifier->name); + it = aliases.find(identifier->name()); if (!max_attempts--) - throw Exception("Cannot unroll aliases for '" + identifier->name + "'", ErrorCodes::LOGICAL_ERROR); + throw Exception("Cannot unroll aliases for '" + identifier->name() + "'", ErrorCodes::LOGICAL_ERROR); } return identifier; @@ -186,7 +186,7 @@ size_t CollectJoinOnKeysMatcher::getTableForIdentifiers(std::vectorname; + const String & name = identifier->name(); bool in_left_table = data.left_table.hasColumn(name); bool in_right_table = data.right_table.hasColumn(name); diff --git a/src/Interpreters/ExtractExpressionInfoVisitor.cpp b/src/Interpreters/ExtractExpressionInfoVisitor.cpp index 5f7754d315a..c730f49fe90 100644 --- a/src/Interpreters/ExtractExpressionInfoVisitor.cpp +++ b/src/Interpreters/ExtractExpressionInfoVisitor.cpp @@ -41,7 +41,7 @@ void ExpressionInfoMatcher::visit(const ASTIdentifier & identifier, const ASTPtr const auto & table = data.tables[index]; // TODO: make sure no collision ever happens - if (table.hasColumn(identifier.name)) + if (table.hasColumn(identifier.name())) { data.unique_reference_tables_pos.emplace(index); break; diff --git a/src/Interpreters/IdentifierSemantic.cpp b/src/Interpreters/IdentifierSemantic.cpp index 256a3784c77..a1fc533eb7f 100644 --- a/src/Interpreters/IdentifierSemantic.cpp +++ b/src/Interpreters/IdentifierSemantic.cpp @@ -51,7 +51,7 @@ std::optional tryChooseTable(const ASTIdentifier & identifier, const std if ((best_match != ColumnMatch::NoMatch) && same_match) { if (!allow_ambiguous) - throw Exception("Ambiguous column '" + identifier.name + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME); + throw Exception("Ambiguous column '" + identifier.name() + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME); best_match = ColumnMatch::Ambiguous; return {}; } @@ -66,7 +66,7 @@ std::optional tryChooseTable(const ASTIdentifier & identifier, const std std::optional IdentifierSemantic::getColumnName(const ASTIdentifier & node) { if (!node.semantic->special) - return node.name; + return node.name(); return {}; } @@ -75,14 +75,14 @@ std::optional IdentifierSemantic::getColumnName(const ASTPtr & ast) if (ast) if (const auto * id = ast->as()) if (!id->semantic->special) - return id->name; + return id->name(); return {}; } std::optional IdentifierSemantic::getTableName(const ASTIdentifier & node) { if (node.semantic->special) - return node.name; + return node.name(); return {}; } @@ -91,7 +91,7 @@ std::optional IdentifierSemantic::getTableName(const ASTPtr & ast) if (ast) if (const auto * id = ast->as()) if (id->semantic->special) - return id->name; + return id->name(); return {}; } @@ -151,7 +151,7 @@ StorageID IdentifierSemantic::extractDatabaseAndTable(const ASTIdentifier & iden if (identifier.name_parts.size() == 2) return { identifier.name_parts[0], identifier.name_parts[1], identifier.uuid }; - return { "", identifier.name, identifier.uuid }; + return { "", identifier.name_parts[0], identifier.uuid }; } std::optional IdentifierSemantic::extractNestedName(const ASTIdentifier & identifier, const String & table_name) @@ -232,16 +232,8 @@ void IdentifierSemantic::setColumnShortName(ASTIdentifier & identifier, const Da if (!to_strip) return; - std::vector stripped(identifier.name_parts.begin() + to_strip, identifier.name_parts.end()); - - DB::String new_name; - for (const auto & part : stripped) - { - if (!new_name.empty()) - new_name += '.'; - new_name += part; - } - identifier.name.swap(new_name); + identifier.name_parts = std::vector(identifier.name_parts.begin() + to_strip, identifier.name_parts.end()); + identifier.resetFullName(); } void IdentifierSemantic::setColumnLongName(ASTIdentifier & identifier, const DatabaseAndTableWithAlias & db_and_table) @@ -249,10 +241,11 @@ void IdentifierSemantic::setColumnLongName(ASTIdentifier & identifier, const Dat String prefix = db_and_table.getQualifiedNamePrefix(); if (!prefix.empty()) { - String short_name = identifier.shortName(); - identifier.name = prefix + short_name; prefix.resize(prefix.size() - 1); /// crop dot - identifier.name_parts = {prefix, short_name}; + identifier.name_parts = {prefix, identifier.shortName()}; + identifier.resetFullName(); + identifier.semantic->table = prefix; + identifier.semantic->legacy_compound = true; } } diff --git a/src/Interpreters/IdentifierSemantic.h b/src/Interpreters/IdentifierSemantic.h index ca5a923c2ea..80b55ba0537 100644 --- a/src/Interpreters/IdentifierSemantic.h +++ b/src/Interpreters/IdentifierSemantic.h @@ -10,10 +10,12 @@ namespace DB struct IdentifierSemanticImpl { - bool special = false; /// for now it's 'not a column': tables, subselects and some special stuff like FORMAT - bool can_be_alias = true; /// if it's a cropped name it could not be an alias - bool covered = false; /// real (compound) name is hidden by an alias (short name) - std::optional membership; /// table position in join + bool special = false; /// for now it's 'not a column': tables, subselects and some special stuff like FORMAT + bool can_be_alias = true; /// if it's a cropped name it could not be an alias + bool covered = false; /// real (compound) name is hidden by an alias (short name) + std::optional membership; /// table position in join + String table = {}; /// store table name for columns just to support legacy logic. + bool legacy_compound = false; /// true if identifier supposed to be comply for legacy |compound()| behavior }; /// Static class to manipulate IdentifierSemanticImpl via ASTIdentifier diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index cdd7ec3ebf9..372bbfbe648 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -117,7 +117,7 @@ private: throw Exception("Logical error: qualified asterisk must have exactly one child", ErrorCodes::LOGICAL_ERROR); ASTIdentifier & identifier = child->children[0]->as(); - data.addTableColumns(identifier.name); + data.addTableColumns(identifier.name()); } else data.new_select_expression_list->children.push_back(child); @@ -228,7 +228,7 @@ struct CollectColumnIdentifiersMatcher void addIdentirier(const ASTIdentifier & ident) { for (const auto & aliases : ignored) - if (aliases.count(ident.name)) + if (aliases.count(ident.name())) return; identifiers.push_back(const_cast(&ident)); } @@ -293,7 +293,7 @@ struct CheckAliasDependencyVisitorData void visit(ASTIdentifier & ident, ASTPtr &) { - if (!dependency && aliases.count(ident.name)) + if (!dependency && aliases.count(ident.name())) dependency = &ident; } }; @@ -467,7 +467,7 @@ std::vector normalizeColumnNamesExtractNeeded( for (ASTIdentifier * ident : identifiers) { - bool got_alias = aliases.count(ident->name); + bool got_alias = aliases.count(ident->name()); bool allow_ambiguous = got_alias; /// allow ambiguous column overridden by an alias if (auto table_pos = IdentifierSemantic::chooseTableColumnMatch(*ident, tables, allow_ambiguous)) @@ -475,12 +475,12 @@ std::vector normalizeColumnNamesExtractNeeded( if (!ident->isShort()) { if (got_alias) - throw Exception("Alias clashes with qualified column '" + ident->name + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME); + throw Exception("Alias clashes with qualified column '" + ident->name() + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME); String short_name = ident->shortName(); String original_long_name; if (public_identifiers.count(ident)) - original_long_name = ident->name; + original_long_name = ident->name(); size_t count = countTablesWithColumn(tables, short_name); @@ -488,7 +488,7 @@ std::vector normalizeColumnNamesExtractNeeded( { const auto & table = tables[*table_pos]; IdentifierSemantic::setColumnLongName(*ident, table.table); /// table.column -> table_alias.column - auto & unique_long_name = ident->name; + const auto & unique_long_name = ident->name(); /// For tables moved into subselects we need unique short names for clashed names if (*table_pos != last_table_pos) @@ -512,7 +512,7 @@ std::vector normalizeColumnNamesExtractNeeded( needed_columns[*table_pos].no_clashes.emplace(ident->shortName()); } else if (!got_alias) - throw Exception("Unknown column name '" + ident->name + "'", ErrorCodes::UNKNOWN_IDENTIFIER); + throw Exception("Unknown column name '" + ident->name() + "'", ErrorCodes::UNKNOWN_IDENTIFIER); } return needed_columns; @@ -613,12 +613,12 @@ void JoinToSubqueryTransformMatcher::visit(ASTSelectQuery & select, ASTPtr & ast { for (auto * ident : on_identifiers) { - auto it = data.aliases.find(ident->name); - if (!on_aliases.count(ident->name) && it != data.aliases.end()) + auto it = data.aliases.find(ident->name()); + if (!on_aliases.count(ident->name()) && it != data.aliases.end()) { auto alias_expression = it->second; alias_pushdown[table_pos].push_back(alias_expression); - on_aliases[ident->name] = alias_expression; + on_aliases[ident->name()] = alias_expression; } } } @@ -638,14 +638,14 @@ void JoinToSubqueryTransformMatcher::visit(ASTSelectQuery & select, ASTPtr & ast CheckAliasDependencyVisitor(check).visit(expr.second); if (check.dependency) throw Exception("Cannot rewrite JOINs. Alias '" + expr.first + - "' used in ON section depends on another alias '" + check.dependency->name + "'", + "' used in ON section depends on another alias '" + check.dependency->name() + "'", ErrorCodes::NOT_IMPLEMENTED); } /// Check same name in aliases, USING and ON sections. Cannot push down alias to ON through USING cause of name masquerading. for (auto * ident : using_identifiers) - if (on_aliases.count(ident->name)) - throw Exception("Cannot rewrite JOINs. Alias '" + ident->name + "' appears both in ON and USING", ErrorCodes::NOT_IMPLEMENTED); + if (on_aliases.count(ident->name())) + throw Exception("Cannot rewrite JOINs. Alias '" + ident->name() + "' appears both in ON and USING", ErrorCodes::NOT_IMPLEMENTED); using_identifiers.clear(); /// Replace pushdowned expressions with aliases names in original expression lists. diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index d38a3fa68dc..c0511122c1e 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -49,7 +49,7 @@ void replaceJoinedTable(const ASTSelectQuery & select_query) if (table_expr.database_and_table_name) { const auto & table_id = table_expr.database_and_table_name->as(); - String expr = "(select * from " + table_id.name + ") as " + table_id.shortName(); + String expr = "(select * from " + table_id.name() + ") as " + table_id.shortName(); // FIXME: since the expression "a as b" exposes both "a" and "b" names, which is not equivalent to "(select * from a) as b", // we can't replace aliased tables. @@ -99,7 +99,7 @@ private: match == IdentifierSemantic::ColumnMatch::DbAndTable) { if (rewritten) - throw Exception("Failed to rewrite distributed table names. Ambiguous column '" + identifier.name + "'", + throw Exception("Failed to rewrite distributed table names. Ambiguous column '" + identifier.name() + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME); /// Table has an alias. So we set a new name qualified by table alias. IdentifierSemantic::setColumnLongName(identifier, table); @@ -114,10 +114,10 @@ private: bool rewritten = false; for (const auto & table : data) { - if (identifier.name == table.table) + if (identifier.name() == table.table) { if (rewritten) - throw Exception("Failed to rewrite distributed table. Ambiguous column '" + identifier.name + "'", + throw Exception("Failed to rewrite distributed table. Ambiguous column '" + identifier.name() + "'", ErrorCodes::AMBIGUOUS_COLUMN_NAME); identifier.setShortName(table.alias); rewritten = true; diff --git a/src/Interpreters/QueryNormalizer.cpp b/src/Interpreters/QueryNormalizer.cpp index 07d4888b555..3252626959d 100644 --- a/src/Interpreters/QueryNormalizer.cpp +++ b/src/Interpreters/QueryNormalizer.cpp @@ -73,8 +73,8 @@ void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data) return; /// If it is an alias, but not a parent alias (for constructs like "SELECT column + 1 AS column"). - auto it_alias = data.aliases.find(node.name); - if (it_alias != data.aliases.end() && current_alias != node.name) + auto it_alias = data.aliases.find(node.name()); + if (it_alias != data.aliases.end() && current_alias != node.name()) { if (!IdentifierSemantic::canBeAlias(node)) return; @@ -89,7 +89,7 @@ void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data) String node_alias = ast->tryGetAlias(); if (current_asts.count(alias_node.get()) /// We have loop of multiple aliases - || (node.name == our_alias_or_name && our_name && node_alias == *our_name)) /// Our alias points to node.name, direct loop + || (node.name() == our_alias_or_name && our_name && node_alias == *our_name)) /// Our alias points to node.name, direct loop throw Exception("Cyclic aliases", ErrorCodes::CYCLIC_ALIASES); /// Let's replace it with the corresponding tree node. @@ -97,7 +97,7 @@ void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data) { /// Avoid infinite recursion here auto opt_name = IdentifierSemantic::getColumnName(alias_node); - bool is_cycle = opt_name && *opt_name == node.name; + bool is_cycle = opt_name && *opt_name == node.name(); if (!is_cycle) { diff --git a/src/Interpreters/RenameColumnVisitor.cpp b/src/Interpreters/RenameColumnVisitor.cpp index f94353cf646..22bbfc7a83f 100644 --- a/src/Interpreters/RenameColumnVisitor.cpp +++ b/src/Interpreters/RenameColumnVisitor.cpp @@ -3,10 +3,13 @@ namespace DB { + void RenameColumnData::visit(ASTIdentifier & identifier, ASTPtr &) const { + // TODO(ilezhankin): make proper rename std::optional identifier_column_name = IdentifierSemantic::getColumnName(identifier); if (identifier_column_name && identifier_column_name == column_name) - identifier.name = rename_to; + identifier.setShortName(rename_to); } + } diff --git a/src/Interpreters/RequiredSourceColumnsData.cpp b/src/Interpreters/RequiredSourceColumnsData.cpp index ec9f2ca4817..9118acc38c9 100644 --- a/src/Interpreters/RequiredSourceColumnsData.cpp +++ b/src/Interpreters/RequiredSourceColumnsData.cpp @@ -27,7 +27,7 @@ void RequiredSourceColumnsData::addColumnIdentifier(const ASTIdentifier & node) /// There should be no complex cases after query normalization. Names to aliases: one-to-many. String alias = node.tryGetAlias(); - required_names[node.name].addInclusion(alias); + required_names[node.name()].addInclusion(alias); } bool RequiredSourceColumnsData::addArrayJoinAliasIfAny(const IAST & ast) @@ -42,7 +42,7 @@ bool RequiredSourceColumnsData::addArrayJoinAliasIfAny(const IAST & ast) void RequiredSourceColumnsData::addArrayJoinIdentifier(const ASTIdentifier & node) { - array_join_columns.insert(node.name); + array_join_columns.insert(node.name()); } size_t RequiredSourceColumnsData::nameInclusion(const String & name) const diff --git a/src/Interpreters/RequiredSourceColumnsVisitor.cpp b/src/Interpreters/RequiredSourceColumnsVisitor.cpp index e546a40f28d..5a265b59414 100644 --- a/src/Interpreters/RequiredSourceColumnsVisitor.cpp +++ b/src/Interpreters/RequiredSourceColumnsVisitor.cpp @@ -34,7 +34,7 @@ std::vector RequiredSourceColumnsMatcher::extractNamesFromLambda(const A if (!identifier) throw Exception("lambda argument declarations must be identifiers", ErrorCodes::TYPE_MISMATCH); - names.push_back(identifier->name); + names.push_back(identifier->name()); } return names; @@ -132,10 +132,11 @@ void RequiredSourceColumnsMatcher::visit(const ASTSelectQuery & select, const AS void RequiredSourceColumnsMatcher::visit(const ASTIdentifier & node, const ASTPtr &, Data & data) { - if (node.name.empty()) + // FIXME(ilezhankin): shouldn't ever encounter + if (node.name().empty()) throw Exception("Expected not empty name", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - if (!data.private_aliases.count(node.name)) + if (!data.private_aliases.count(node.name())) data.addColumnIdentifier(node); } diff --git a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp index 32d8841d7b4..98ed2166c40 100644 --- a/src/Interpreters/TranslateQualifiedNamesVisitor.cpp +++ b/src/Interpreters/TranslateQualifiedNamesVisitor.cpp @@ -104,7 +104,7 @@ void TranslateQualifiedNamesMatcher::visit(ASTIdentifier & identifier, ASTPtr &, if (data.unknownColumn(table_pos, identifier)) { String table_name = data.tables[table_pos].table.getQualifiedNamePrefix(false); - throw Exception("There's no column '" + identifier.name + "' in table '" + table_name + "'", + throw Exception("There's no column '" + identifier.name() + "' in table '" + table_name + "'", ErrorCodes::UNKNOWN_IDENTIFIER); } @@ -175,9 +175,12 @@ void TranslateQualifiedNamesMatcher::visit(ASTSelectQuery & select, const ASTPtr static void addIdentifier(ASTs & nodes, const DatabaseAndTableWithAlias & table, const String & column_name) { + std::vector parts = {column_name}; + String table_name = table.getQualifiedNamePrefix(false); - auto identifier = std::make_shared(std::vector{table_name, column_name}); - nodes.emplace_back(identifier); + if (!table_name.empty()) parts.insert(parts.begin(), table_name); + + nodes.emplace_back(std::make_shared(std::move(parts))); } /// Replace *, alias.*, database.table.* with a list of columns. @@ -354,7 +357,7 @@ void RestoreQualifiedNamesMatcher::visit(ASTIdentifier & identifier, ASTPtr &, D { if (IdentifierSemantic::getMembership(identifier)) { - identifier.restoreCompoundName(); + identifier.restoreTable(); // TODO(ilezhankin): should restore qualified name here - why exactly here? if (data.rename) data.changeTable(identifier); } diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index e1f53c72801..02ef3426483 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -72,7 +72,7 @@ ASTPtr evaluateConstantExpressionAsLiteral(const ASTPtr & node, const Context & ASTPtr evaluateConstantExpressionOrIdentifierAsLiteral(const ASTPtr & node, const Context & context) { if (const auto * id = node->as()) - return std::make_shared(id->name); + return std::make_shared(id->name()); return evaluateConstantExpressionAsLiteral(node, context); } @@ -113,7 +113,7 @@ namespace const auto & name = name_and_type.name; const auto & type = name_and_type.type; - if (name == identifier->name) + if (name == identifier->name()) { ColumnWithTypeAndName column; Field converted = convertFieldToType(value, *type); diff --git a/src/Interpreters/getClusterName.cpp b/src/Interpreters/getClusterName.cpp index 60040ce4cb0..01e45e1d650 100644 --- a/src/Interpreters/getClusterName.cpp +++ b/src/Interpreters/getClusterName.cpp @@ -19,7 +19,7 @@ namespace ErrorCodes std::string getClusterName(const IAST & node) { if (const auto * ast_id = node.as()) - return ast_id->name; + return ast_id->name(); if (const auto * ast_lit = node.as()) return ast_lit->value.safeGet(); diff --git a/src/Parsers/ASTColumnsTransformers.cpp b/src/Parsers/ASTColumnsTransformers.cpp index 43d54f07ab8..fee606aec26 100644 --- a/src/Parsers/ASTColumnsTransformers.cpp +++ b/src/Parsers/ASTColumnsTransformers.cpp @@ -71,7 +71,7 @@ void ASTColumnsExceptTransformer::transform(ASTs & nodes) const { for (const auto & except_child : children) { - if (except_child->as().name == id->shortName()) + if (except_child->as().name() == id->shortName()) return true; } } diff --git a/src/Parsers/ASTIdentifier.cpp b/src/Parsers/ASTIdentifier.cpp index 9117be46e51..d980300a22a 100644 --- a/src/Parsers/ASTIdentifier.cpp +++ b/src/Parsers/ASTIdentifier.cpp @@ -1,10 +1,10 @@ -#include #include -#include + #include #include #include #include +#include namespace DB @@ -16,6 +16,27 @@ namespace ErrorCodes extern const int SYNTAX_ERROR; } +ASTIdentifier::ASTIdentifier(const String & short_name) + : full_name(short_name), name_parts{short_name}, semantic(std::make_shared()) +{ + assert(!full_name.empty()); +} + +ASTIdentifier::ASTIdentifier(std::vector && name_parts_, bool special) + : name_parts(name_parts_), semantic(std::make_shared()) +{ + assert(!name_parts.empty()); + for (const auto & part [[maybe_unused]] : name_parts) + assert(!part.empty()); + + semantic->special = special; + semantic->legacy_compound = true; + + if (!special && name_parts.size() >= 2) + semantic->table = name_parts.end()[-2]; + + resetFullName(); +} ASTPtr ASTIdentifier::clone() const { @@ -24,51 +45,29 @@ ASTPtr ASTIdentifier::clone() const return ret; } -std::shared_ptr ASTIdentifier::createSpecial(const String & name, std::vector && name_parts) +bool ASTIdentifier::supposedToBeCompound() const { - auto ret = std::make_shared(name, std::move(name_parts)); - ret->semantic->special = true; - return ret; + return semantic->legacy_compound; } -ASTIdentifier::ASTIdentifier(const String & name_, std::vector && name_parts_) - : name(name_) - , name_parts(name_parts_) - , semantic(std::make_shared()) -{ - if (!name_parts.empty() && name_parts[0].empty()) - name_parts.erase(name_parts.begin()); - - if (name.empty()) - { - if (name_parts.size() == 2) - name = name_parts[0] + '.' + name_parts[1]; - else if (name_parts.size() == 1) - name = name_parts[0]; - } -} - -ASTIdentifier::ASTIdentifier(std::vector && name_parts_) - : ASTIdentifier("", std::move(name_parts_)) -{} - void ASTIdentifier::setShortName(const String & new_name) { - name = new_name; - name_parts.clear(); + assert(!new_name.empty()); + + full_name = new_name; + name_parts = {new_name}; bool special = semantic->special; *semantic = IdentifierSemanticImpl(); semantic->special = special; } -void ASTIdentifier::restoreCompoundName() +const String & ASTIdentifier::name() const { - if (name_parts.empty()) - return; - name = name_parts[0]; - for (size_t i = 1; i < name_parts.size(); ++i) - name += '.' + name_parts[i]; + assert(!name_parts.empty()); + assert(!full_name.empty()); + + return full_name; } void ASTIdentifier::formatImplWithoutAlias(const FormatSettings & settings, FormatState &, FormatStateStacked) const @@ -93,20 +92,29 @@ void ASTIdentifier::formatImplWithoutAlias(const FormatSettings & settings, Form } else { - format_element(name); + format_element(shortName()); } } void ASTIdentifier::appendColumnNameImpl(WriteBuffer & ostr) const { - writeString(name, ostr); + writeString(name(), ostr); +} + +void ASTIdentifier::restoreTable() +{ + if (!compound()) + { + name_parts.insert(name_parts.begin(), semantic->table); + resetFullName(); + } } void ASTIdentifier::resetTable(const String & database_name, const String & table_name) { auto ast = createTableIdentifier(database_name, table_name); auto & ident = ast->as(); - name.swap(ident.name); + full_name.swap(ident.full_name); name_parts.swap(ident.name_parts); uuid = ident.uuid; } @@ -117,6 +125,13 @@ void ASTIdentifier::updateTreeHashImpl(SipHash & hash_state) const IAST::updateTreeHashImpl(hash_state); } +void ASTIdentifier::resetFullName() +{ + full_name = name_parts[0]; + for (size_t i = 1; i < name_parts.size(); ++i) + full_name += '.' + name_parts[i]; +} + ASTPtr createTableIdentifier(const String & database_name, const String & table_name) { assert(database_name != "_temporary_and_external_tables"); @@ -127,9 +142,9 @@ ASTPtr createTableIdentifier(const StorageID & table_id) { std::shared_ptr res; if (table_id.database_name.empty()) - res = ASTIdentifier::createSpecial(table_id.table_name); + res = std::make_shared(std::vector{table_id.table_name}, true); else - res = ASTIdentifier::createSpecial(table_id.database_name + "." + table_id.table_name, {table_id.database_name, table_id.table_name}); + res = std::make_shared(std::vector{table_id.database_name, table_id.table_name}, true); res->uuid = table_id.uuid; return res; } @@ -156,7 +171,7 @@ bool tryGetIdentifierNameInto(const IAST * ast, String & name) { if (const auto * node = ast->as()) { - name = node->name; + name = node->name(); return true; } } @@ -180,7 +195,7 @@ StorageID getTableIdentifier(const ASTPtr & ast) if (identifier.name_parts.size() == 2) return { identifier.name_parts[0], identifier.name_parts[1], identifier.uuid }; - return { "", identifier.name, identifier.uuid }; + return { "", identifier.name_parts[0], identifier.uuid }; } } diff --git a/src/Parsers/ASTIdentifier.h b/src/Parsers/ASTIdentifier.h index 5c06fa7fa38..59f698eab1c 100644 --- a/src/Parsers/ASTIdentifier.h +++ b/src/Parsers/ASTIdentifier.h @@ -18,59 +18,54 @@ struct StorageID; class ASTIdentifier : public ASTWithAlias { public: - /// The composite identifier will have a concatenated name (of the form a.b.c), - /// and individual components will be available inside the name_parts. - String name; UUID uuid = UUIDHelpers::Nil; - ASTIdentifier(const String & name_, std::vector && name_parts_ = {}); - ASTIdentifier(std::vector && name_parts_); + explicit ASTIdentifier(const String & short_name); + explicit ASTIdentifier(std::vector && name_parts, bool special = false); /** Get the text that identifies this element. */ - String getID(char delim) const override { return "Identifier" + (delim + name); } + String getID(char delim) const override { return "Identifier" + (delim + name()); } ASTPtr clone() const override; - void collectIdentifierNames(IdentifierNameSet & set) const override - { - set.insert(name); - } + void collectIdentifierNames(IdentifierNameSet & set) const override { set.insert(name()); } - bool compound() const { return !name_parts.empty(); } - bool isShort() const { return name_parts.empty() || name == name_parts.back(); } + bool compound() const { return name_parts.size() > 1; } + bool isShort() const { return name_parts.size() == 1; } + bool supposedToBeCompound() const; // TODO(ilezhankin): get rid of this void setShortName(const String & new_name); - /// Restore name field from name_parts in case it was cropped by analyzer but we need a full form for future (re)analyze. - void restoreCompoundName(); + /// The composite identifier will have a concatenated name (of the form a.b.c), + /// and individual components will be available inside the name_parts. + const String & shortName() const { return name_parts.back(); } + const String & name() const; - const String & shortName() const - { - if (!name_parts.empty()) - return name_parts.back(); - return name; - } + void restoreTable(); // TODO(ilezhankin): get rid of this - void resetTable(const String & database_name, const String & table_name); + // FIXME: used only when it's needed to rewrite distributed table name to real remote table name. + void resetTable(const String & database_name, const String & table_name); // TODO(ilezhankin): get rid of this void updateTreeHashImpl(SipHash & hash_state) const override; protected: + String full_name; + std::vector name_parts; + void formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; void appendColumnNameImpl(WriteBuffer & ostr) const override; private: using ASTWithAlias::children; /// ASTIdentifier is child free - std::vector name_parts; std::shared_ptr semantic; /// pimpl - static std::shared_ptr createSpecial(const String & name, std::vector && name_parts = {}); - friend struct IdentifierSemantic; friend ASTPtr createTableIdentifier(const StorageID & table_id); friend void setIdentifierSpecial(ASTPtr & ast); friend StorageID getTableIdentifier(const ASTPtr & ast); + + void resetFullName(); }; diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index b26e73287d0..3c45bd005a9 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -184,16 +184,10 @@ bool ParserCompoundIdentifier::parseImpl(Pos & pos, ASTPtr & node, Expected & ex .parse(pos, id_list, expected)) return false; - String name; std::vector parts; const auto & list = id_list->as(); for (const auto & child : list.children) - { - if (!name.empty()) - name += '.'; parts.emplace_back(getIdentifierName(child)); - name += parts.back(); - } ParserKeyword s_uuid("UUID"); UUID uuid = UUIDHelpers::Nil; @@ -207,9 +201,7 @@ bool ParserCompoundIdentifier::parseImpl(Pos & pos, ASTPtr & node, Expected & ex uuid = parseFromString(ast_uuid->as()->value.get()); } - if (parts.size() == 1) - parts.clear(); - node = std::make_shared(name, std::move(parts)); + node = std::make_shared(std::move(parts)); node->as()->uuid = uuid; return true; @@ -1651,7 +1643,7 @@ bool ParserFunctionWithKeyValueArguments::parseImpl(Pos & pos, ASTPtr & node, Ex } auto function = std::make_shared(left_bracket_found); - function->name = Poco::toLower(typeid_cast(*identifier.get()).name); + function->name = Poco::toLower(identifier->as()->name()); function->elements = expr_list_args; function->children.push_back(function->elements); node = function; diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index d6678bb9a78..ad03d949174 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -1,13 +1,11 @@ -#include +#include + #include #include -#include -#include -#include -#include #include - -#include +#include +#include +#include #include @@ -750,7 +748,7 @@ bool ParserKeyValuePair::parseImpl(Pos & pos, ASTPtr & node, Expected & expected } auto pair = std::make_shared(with_brackets); - pair->first = Poco::toLower(typeid_cast(*identifier.get()).name); + pair->first = Poco::toLower(identifier->as()->name()); pair->set(pair->second, value); node = pair; return true; diff --git a/src/Parsers/MySQL/ASTDeclareConstraint.cpp b/src/Parsers/MySQL/ASTDeclareConstraint.cpp index 0f447fb3b40..96184dfc89d 100644 --- a/src/Parsers/MySQL/ASTDeclareConstraint.cpp +++ b/src/Parsers/MySQL/ASTDeclareConstraint.cpp @@ -63,7 +63,7 @@ bool ParserDeclareConstraint::parseImpl(IParser::Pos & pos, ASTPtr & node, Expec declare_constraint->check_expression = index_check_expression; if (constraint_symbol) - declare_constraint->constraint_name = constraint_symbol->as()->name; + declare_constraint->constraint_name = constraint_symbol->as()->name(); node = declare_constraint; return true; diff --git a/src/Parsers/MySQL/ASTDeclareIndex.cpp b/src/Parsers/MySQL/ASTDeclareIndex.cpp index 8e6e9d43793..c5b4686e976 100644 --- a/src/Parsers/MySQL/ASTDeclareIndex.cpp +++ b/src/Parsers/MySQL/ASTDeclareIndex.cpp @@ -73,7 +73,7 @@ static inline bool parseDeclareOrdinaryIndex(IParser::Pos & pos, String & index_ index_type = "SPATIAL"; if (p_identifier.parse(pos, temp_node, expected)) - index_name = temp_node->as()->name; + index_name = temp_node->as()->name(); } else if (ParserKeyword("FULLTEXT").ignore(pos, expected)) { @@ -82,7 +82,7 @@ static inline bool parseDeclareOrdinaryIndex(IParser::Pos & pos, String & index_ index_type = "FULLTEXT"; if (p_identifier.parse(pos, temp_node, expected)) - index_name = temp_node->as()->name; + index_name = temp_node->as()->name(); } else { @@ -94,14 +94,14 @@ static inline bool parseDeclareOrdinaryIndex(IParser::Pos & pos, String & index_ index_type = "KEY_BTREE"; /// default index type if (p_identifier.parse(pos, temp_node, expected)) - index_name = temp_node->as()->name; + index_name = temp_node->as()->name(); if (ParserKeyword("USING").ignore(pos, expected)) { if (!p_identifier.parse(pos, temp_node, expected)) return false; - index_type = "KEY_" + temp_node->as()->name; + index_type = "KEY_" + temp_node->as()->name(); } } @@ -122,7 +122,7 @@ static inline bool parseDeclareConstraintIndex(IParser::Pos & pos, String & inde if (!p_identifier.parse(pos, temp_node, expected)) return false; - index_name = temp_node->as()->name; + index_name = temp_node->as()->name(); } } @@ -132,7 +132,7 @@ static inline bool parseDeclareConstraintIndex(IParser::Pos & pos, String & inde ParserKeyword("INDEX").ignore(pos, expected); if (p_identifier.parse(pos, temp_node, expected)) - index_name = temp_node->as()->name; /// reset index_name + index_name = temp_node->as()->name(); /// reset index_name index_type = "UNIQUE_BTREE"; /// default btree index_type if (ParserKeyword("USING").ignore(pos, expected)) @@ -140,7 +140,7 @@ static inline bool parseDeclareConstraintIndex(IParser::Pos & pos, String & inde if (!p_identifier.parse(pos, temp_node, expected)) return false; - index_type = "UNIQUE_" + temp_node->as()->name; + index_type = "UNIQUE_" + temp_node->as()->name(); } } else if (ParserKeyword("PRIMARY KEY").ignore(pos, expected)) @@ -151,14 +151,14 @@ static inline bool parseDeclareConstraintIndex(IParser::Pos & pos, String & inde if (!p_identifier.parse(pos, temp_node, expected)) return false; - index_type = "PRIMARY_KEY_" + temp_node->as()->name; + index_type = "PRIMARY_KEY_" + temp_node->as()->name(); } } else if (ParserKeyword("FOREIGN KEY").ignore(pos, expected)) { index_type = "FOREIGN"; if (p_identifier.parse(pos, temp_node, expected)) - index_name = temp_node->as()->name; /// reset index_name + index_name = temp_node->as()->name(); /// reset index_name } return true; diff --git a/src/Parsers/MySQL/ASTDeclarePartition.cpp b/src/Parsers/MySQL/ASTDeclarePartition.cpp index 8e1d27778b5..76f864fcc44 100644 --- a/src/Parsers/MySQL/ASTDeclarePartition.cpp +++ b/src/Parsers/MySQL/ASTDeclarePartition.cpp @@ -107,7 +107,7 @@ bool ParserDeclarePartition::parseImpl(IParser::Pos & pos, ASTPtr & node, Expect partition_declare->less_than = less_than; partition_declare->in_expression = in_expression; partition_declare->subpartitions = subpartitions; - partition_declare->partition_name = partition_name->as()->name; + partition_declare->partition_name = partition_name->as()->name(); if (options) { diff --git a/src/Parsers/MySQL/ASTDeclareReference.cpp b/src/Parsers/MySQL/ASTDeclareReference.cpp index 434b9561eda..862d35e2b76 100644 --- a/src/Parsers/MySQL/ASTDeclareReference.cpp +++ b/src/Parsers/MySQL/ASTDeclareReference.cpp @@ -95,7 +95,7 @@ bool ParserDeclareReference::parseImpl(IParser::Pos & pos, ASTPtr & node, Expect declare_reference->on_delete_option = delete_option; declare_reference->on_update_option = update_option; declare_reference->reference_expression = expression; - declare_reference->reference_table_name = table_name->as()->name; + declare_reference->reference_table_name = table_name->as()->name(); node = declare_reference; return true; diff --git a/src/Parsers/MySQL/ASTDeclareSubPartition.cpp b/src/Parsers/MySQL/ASTDeclareSubPartition.cpp index 1b2d9c081e6..d77fba271c4 100644 --- a/src/Parsers/MySQL/ASTDeclareSubPartition.cpp +++ b/src/Parsers/MySQL/ASTDeclareSubPartition.cpp @@ -41,7 +41,7 @@ bool ParserDeclareSubPartition::parseImpl(Pos & pos, ASTPtr & node, Expected & e auto subpartition_declare = std::make_shared(); subpartition_declare->options = options; - subpartition_declare->logical_name = logical_name->as()->name; + subpartition_declare->logical_name = logical_name->as()->name(); if (options) { diff --git a/src/Parsers/MySQL/tests/gtest_column_parser.cpp b/src/Parsers/MySQL/tests/gtest_column_parser.cpp index ef6371f71d9..de4c64be817 100644 --- a/src/Parsers/MySQL/tests/gtest_column_parser.cpp +++ b/src/Parsers/MySQL/tests/gtest_column_parser.cpp @@ -29,9 +29,9 @@ TEST(ParserColumn, AllNonGeneratedColumnOption) EXPECT_EQ(declare_options->changes["unique_key"]->as()->value.safeGet(), 1); EXPECT_EQ(declare_options->changes["primary_key"]->as()->value.safeGet(), 1); EXPECT_EQ(declare_options->changes["comment"]->as()->value.safeGet(), "column comment"); - EXPECT_EQ(declare_options->changes["collate"]->as()->name, "utf-8"); - EXPECT_EQ(declare_options->changes["column_format"]->as()->name, "FIXED"); - EXPECT_EQ(declare_options->changes["storage"]->as()->name, "MEMORY"); + EXPECT_EQ(declare_options->changes["collate"]->as()->name(), "utf-8"); + EXPECT_EQ(declare_options->changes["column_format"]->as()->name(), "FIXED"); + EXPECT_EQ(declare_options->changes["storage"]->as()->name(), "MEMORY"); EXPECT_TRUE(declare_options->changes["reference"]->as()); EXPECT_TRUE(declare_options->changes["constraint"]->as()); } @@ -52,7 +52,7 @@ TEST(ParserColumn, AllGeneratedColumnOption) EXPECT_EQ(declare_options->changes["unique_key"]->as()->value.safeGet(), 1); EXPECT_EQ(declare_options->changes["primary_key"]->as()->value.safeGet(), 1); EXPECT_EQ(declare_options->changes["comment"]->as()->value.safeGet(), "column comment"); - EXPECT_EQ(declare_options->changes["collate"]->as()->name, "utf-8"); + EXPECT_EQ(declare_options->changes["collate"]->as()->name(), "utf-8"); EXPECT_EQ(declare_options->changes["generated"]->as()->value.safeGet(), 1); EXPECT_EQ(declare_options->changes["is_stored"]->as()->value.safeGet(), 1); EXPECT_TRUE(declare_options->changes["reference"]->as()); diff --git a/src/Parsers/MySQL/tests/gtest_constraint_parser.cpp b/src/Parsers/MySQL/tests/gtest_constraint_parser.cpp index de885bf36c8..9c9124c9f58 100644 --- a/src/Parsers/MySQL/tests/gtest_constraint_parser.cpp +++ b/src/Parsers/MySQL/tests/gtest_constraint_parser.cpp @@ -18,7 +18,7 @@ TEST(ParserConstraint, CheckConstraint) EXPECT_EQ(ast_constraint_01->as()->constraint_name, "symbol_name"); auto * check_expression_01 = ast_constraint_01->as()->check_expression->as(); EXPECT_EQ(check_expression_01->name, "equals"); - EXPECT_EQ(check_expression_01->arguments->children[0]->as()->name, "col_01"); + EXPECT_EQ(check_expression_01->arguments->children[0]->as()->name(), "col_01"); EXPECT_EQ(check_expression_01->arguments->children[1]->as()->value.safeGet(), 1); String constraint_02 = "CONSTRAINT CHECK col_01 = 1"; @@ -26,7 +26,7 @@ TEST(ParserConstraint, CheckConstraint) EXPECT_EQ(ast_constraint_02->as()->constraint_name, ""); auto * check_expression_02 = ast_constraint_02->as()->check_expression->as(); EXPECT_EQ(check_expression_02->name, "equals"); - EXPECT_EQ(check_expression_02->arguments->children[0]->as()->name, "col_01"); + EXPECT_EQ(check_expression_02->arguments->children[0]->as()->name(), "col_01"); EXPECT_EQ(check_expression_02->arguments->children[1]->as()->value.safeGet(), 1); String constraint_03 = "CHECK col_01 = 1"; @@ -34,7 +34,7 @@ TEST(ParserConstraint, CheckConstraint) EXPECT_EQ(ast_constraint_03->as()->constraint_name, ""); auto * check_expression_03 = ast_constraint_03->as()->check_expression->as(); EXPECT_EQ(check_expression_03->name, "equals"); - EXPECT_EQ(check_expression_03->arguments->children[0]->as()->name, "col_01"); + EXPECT_EQ(check_expression_03->arguments->children[0]->as()->name(), "col_01"); EXPECT_EQ(check_expression_03->arguments->children[1]->as()->value.safeGet(), 1); String constraint_04 = "CONSTRAINT CHECK col_01 = 1 ENFORCED"; @@ -43,7 +43,7 @@ TEST(ParserConstraint, CheckConstraint) EXPECT_EQ(ast_constraint_04->as()->constraint_name, ""); auto * check_expression_04 = ast_constraint_04->as()->check_expression->as(); EXPECT_EQ(check_expression_04->name, "equals"); - EXPECT_EQ(check_expression_04->arguments->children[0]->as()->name, "col_01"); + EXPECT_EQ(check_expression_04->arguments->children[0]->as()->name(), "col_01"); EXPECT_EQ(check_expression_04->arguments->children[1]->as()->value.safeGet(), 1); String constraint_05 = "CONSTRAINT CHECK col_01 = 1 NOT ENFORCED"; @@ -52,6 +52,6 @@ TEST(ParserConstraint, CheckConstraint) EXPECT_EQ(ast_constraint_05->as()->constraint_name, ""); auto * check_expression_05 = ast_constraint_05->as()->check_expression->as(); EXPECT_EQ(check_expression_05->name, "equals"); - EXPECT_EQ(check_expression_05->arguments->children[0]->as()->name, "col_01"); + EXPECT_EQ(check_expression_05->arguments->children[0]->as()->name(), "col_01"); EXPECT_EQ(check_expression_05->arguments->children[1]->as()->value.safeGet(), 1); } diff --git a/src/Parsers/MySQL/tests/gtest_create_parser.cpp b/src/Parsers/MySQL/tests/gtest_create_parser.cpp index 92c0070aa88..1aaba8d67e4 100644 --- a/src/Parsers/MySQL/tests/gtest_create_parser.cpp +++ b/src/Parsers/MySQL/tests/gtest_create_parser.cpp @@ -28,7 +28,7 @@ TEST(CreateTableParser, SimpleCreate) EXPECT_EQ(ast->as()->columns_list->as()->columns->children.size(), 1); EXPECT_EQ(ast->as()->columns_list->as()->indices->children.size(), 1); EXPECT_EQ(ast->as()->columns_list->as()->constraints->children.size(), 1); - EXPECT_EQ(ast->as()->table_options->as()->changes["engine"]->as()->name, "INNODB"); + EXPECT_EQ(ast->as()->table_options->as()->changes["engine"]->as()->name(), "INNODB"); EXPECT_TRUE(ast->as()->partition_options->as()); } diff --git a/src/Parsers/MySQL/tests/gtest_index_parser.cpp b/src/Parsers/MySQL/tests/gtest_index_parser.cpp index 02b3b10acff..a8be6787b2c 100644 --- a/src/Parsers/MySQL/tests/gtest_index_parser.cpp +++ b/src/Parsers/MySQL/tests/gtest_index_parser.cpp @@ -17,13 +17,13 @@ TEST(ParserIndex, AllIndexOptions) ASTPtr ast = parseQuery(p_index, input.data(), input.data() + input.size(), "", 0, 0); ASTDeclareIndex * declare_index = ast->as(); - EXPECT_EQ(declare_index->index_columns->children[0]->as()->name, "col_01"); + EXPECT_EQ(declare_index->index_columns->children[0]->as()->name(), "col_01"); EXPECT_EQ(declare_index->index_columns->children[1]->as()->name, "col_02"); EXPECT_EQ(declare_index->index_columns->children[1]->as()->arguments->children[0]->as()->value.safeGet(), 100); - EXPECT_EQ(declare_index->index_columns->children[2]->as()->name, "col_03"); + EXPECT_EQ(declare_index->index_columns->children[2]->as()->name(), "col_03"); ASTDeclareOptions * declare_options = declare_index->index_options->as(); EXPECT_EQ(declare_options->changes["key_block_size"]->as()->value.safeGet(), 3); - EXPECT_EQ(declare_options->changes["index_type"]->as()->name, "HASH"); + EXPECT_EQ(declare_options->changes["index_type"]->as()->name(), "HASH"); EXPECT_EQ(declare_options->changes["comment"]->as()->value.safeGet(), "index comment"); EXPECT_EQ(declare_options->changes["visible"]->as()->value.safeGet(), 1); } @@ -36,12 +36,12 @@ TEST(ParserIndex, OptionalIndexOptions) ASTPtr ast = parseQuery(p_index, input.data(), input.data() + input.size(), "", 0, 0); ASTDeclareIndex * declare_index = ast->as(); - EXPECT_EQ(declare_index->index_columns->children[0]->as()->name, "col_01"); + EXPECT_EQ(declare_index->index_columns->children[0]->as()->name(), "col_01"); EXPECT_EQ(declare_index->index_columns->children[1]->as()->name, "col_02"); EXPECT_EQ(declare_index->index_columns->children[1]->as()->arguments->children[0]->as()->value.safeGet(), 100); - EXPECT_EQ(declare_index->index_columns->children[2]->as()->name, "col_03"); + EXPECT_EQ(declare_index->index_columns->children[2]->as()->name(), "col_03"); ASTDeclareOptions * declare_options = declare_index->index_options->as(); - EXPECT_EQ(declare_options->changes["index_type"]->as()->name, "HASH"); + EXPECT_EQ(declare_options->changes["index_type"]->as()->name(), "HASH"); EXPECT_EQ(declare_options->changes["visible"]->as()->value.safeGet(), 0); EXPECT_EQ(declare_options->changes["key_block_size"]->as()->value.safeGet(), 3); } diff --git a/src/Parsers/MySQL/tests/gtest_partition_options_parser.cpp b/src/Parsers/MySQL/tests/gtest_partition_options_parser.cpp index 1651efcb966..01b757e5891 100644 --- a/src/Parsers/MySQL/tests/gtest_partition_options_parser.cpp +++ b/src/Parsers/MySQL/tests/gtest_partition_options_parser.cpp @@ -18,14 +18,14 @@ TEST(ParserPartitionOptions, HashPatitionOptions) ASTDeclarePartitionOptions * declare_partition_options_01 = ast_01->as(); EXPECT_EQ(declare_partition_options_01->partition_type, "hash"); - EXPECT_EQ(declare_partition_options_01->partition_expression->as()->name, "col_01"); + EXPECT_EQ(declare_partition_options_01->partition_expression->as()->name(), "col_01"); String linear_hash_partition = "PARTITION BY LINEAR HASH(col_01)"; ASTPtr ast_02 = parseQuery(p_partition_options, linear_hash_partition.data(), linear_hash_partition.data() + linear_hash_partition.size(), "", 0, 0); ASTDeclarePartitionOptions * declare_partition_options_02 = ast_02->as(); EXPECT_EQ(declare_partition_options_02->partition_type, "linear_hash"); - EXPECT_EQ(declare_partition_options_02->partition_expression->as()->name, "col_01"); + EXPECT_EQ(declare_partition_options_02->partition_expression->as()->name(), "col_01"); } TEST(ParserPartitionOptions, KeyPatitionOptions) @@ -37,7 +37,7 @@ TEST(ParserPartitionOptions, KeyPatitionOptions) ASTDeclarePartitionOptions * declare_partition_options_01 = ast_01->as(); EXPECT_EQ(declare_partition_options_01->partition_type, "key"); - EXPECT_EQ(declare_partition_options_01->partition_expression->as()->name, "col_01"); + EXPECT_EQ(declare_partition_options_01->partition_expression->as()->name(), "col_01"); String linear_key_partition = "PARTITION BY LINEAR KEY(col_01, col_02)"; ASTPtr ast_02 = parseQuery(p_partition_options, linear_key_partition.data(), linear_key_partition.data() + linear_key_partition.size(), "", 0, 0); @@ -45,15 +45,15 @@ TEST(ParserPartitionOptions, KeyPatitionOptions) ASTDeclarePartitionOptions * declare_partition_options_02 = ast_02->as(); EXPECT_EQ(declare_partition_options_02->partition_type, "linear_key"); ASTPtr columns_list = declare_partition_options_02->partition_expression->as()->arguments; - EXPECT_EQ(columns_list->children[0]->as()->name, "col_01"); - EXPECT_EQ(columns_list->children[1]->as()->name, "col_02"); + EXPECT_EQ(columns_list->children[0]->as()->name(), "col_01"); + EXPECT_EQ(columns_list->children[1]->as()->name(), "col_02"); String key_partition_with_algorithm = "PARTITION BY KEY ALGORITHM=1 (col_01)"; ASTPtr ast_03 = parseQuery(p_partition_options, key_partition_with_algorithm.data(), key_partition_with_algorithm.data() + key_partition_with_algorithm.size(), "", 0, 0); ASTDeclarePartitionOptions * declare_partition_options_03 = ast_03->as(); EXPECT_EQ(declare_partition_options_03->partition_type, "key_1"); - EXPECT_EQ(declare_partition_options_03->partition_expression->as()->name, "col_01"); + EXPECT_EQ(declare_partition_options_03->partition_expression->as()->name(), "col_01"); } TEST(ParserPartitionOptions, RangePatitionOptions) @@ -65,7 +65,7 @@ TEST(ParserPartitionOptions, RangePatitionOptions) ASTDeclarePartitionOptions * declare_partition_options_01 = ast_01->as(); EXPECT_EQ(declare_partition_options_01->partition_type, "range"); - EXPECT_EQ(declare_partition_options_01->partition_expression->as()->name, "col_01"); + EXPECT_EQ(declare_partition_options_01->partition_expression->as()->name(), "col_01"); String range_columns_partition = "PARTITION BY RANGE COLUMNS(col_01, col_02)"; ASTPtr ast_02 = parseQuery(p_partition_options, range_columns_partition.data(), range_columns_partition.data() + range_columns_partition.size(), "", 0, 0); @@ -73,8 +73,8 @@ TEST(ParserPartitionOptions, RangePatitionOptions) ASTDeclarePartitionOptions * declare_partition_options_02 = ast_02->as(); EXPECT_EQ(declare_partition_options_02->partition_type, "range"); ASTPtr columns_list = declare_partition_options_02->partition_expression->as()->arguments; - EXPECT_EQ(columns_list->children[0]->as()->name, "col_01"); - EXPECT_EQ(columns_list->children[1]->as()->name, "col_02"); + EXPECT_EQ(columns_list->children[0]->as()->name(), "col_01"); + EXPECT_EQ(columns_list->children[1]->as()->name(), "col_02"); } TEST(ParserPartitionOptions, ListPatitionOptions) @@ -86,7 +86,7 @@ TEST(ParserPartitionOptions, ListPatitionOptions) ASTDeclarePartitionOptions * declare_partition_options_01 = ast_01->as(); EXPECT_EQ(declare_partition_options_01->partition_type, "list"); - EXPECT_EQ(declare_partition_options_01->partition_expression->as()->name, "col_01"); + EXPECT_EQ(declare_partition_options_01->partition_expression->as()->name(), "col_01"); String range_columns_partition = "PARTITION BY LIST COLUMNS(col_01, col_02)"; ASTPtr ast_02 = parseQuery(p_partition_options, range_columns_partition.data(), range_columns_partition.data() + range_columns_partition.size(), "", 0, 0); @@ -94,8 +94,8 @@ TEST(ParserPartitionOptions, ListPatitionOptions) ASTDeclarePartitionOptions * declare_partition_options_02 = ast_02->as(); EXPECT_EQ(declare_partition_options_02->partition_type, "list"); ASTPtr columns_list = declare_partition_options_02->partition_expression->as()->arguments; - EXPECT_EQ(columns_list->children[0]->as()->name, "col_01"); - EXPECT_EQ(columns_list->children[1]->as()->name, "col_02"); + EXPECT_EQ(columns_list->children[0]->as()->name(), "col_01"); + EXPECT_EQ(columns_list->children[1]->as()->name(), "col_02"); } TEST(ParserPartitionOptions, PatitionNumberOptions) @@ -107,7 +107,7 @@ TEST(ParserPartitionOptions, PatitionNumberOptions) ASTDeclarePartitionOptions * declare_partition_options = ast->as(); EXPECT_EQ(declare_partition_options->partition_type, "key"); - EXPECT_EQ(declare_partition_options->partition_expression->as()->name, "col_01"); + EXPECT_EQ(declare_partition_options->partition_expression->as()->name(), "col_01"); EXPECT_EQ(declare_partition_options->partition_numbers->as()->value.safeGet(), 2); } @@ -120,10 +120,10 @@ TEST(ParserPartitionOptions, PatitionWithSubpartitionOptions) ASTDeclarePartitionOptions * declare_partition_options = ast->as(); EXPECT_EQ(declare_partition_options->partition_type, "key"); - EXPECT_EQ(declare_partition_options->partition_expression->as()->name, "col_01"); + EXPECT_EQ(declare_partition_options->partition_expression->as()->name(), "col_01"); EXPECT_EQ(declare_partition_options->partition_numbers->as()->value.safeGet(), 3); EXPECT_EQ(declare_partition_options->subpartition_type, "hash"); - EXPECT_EQ(declare_partition_options->subpartition_expression->as()->name, "col_02"); + EXPECT_EQ(declare_partition_options->subpartition_expression->as()->name(), "col_02"); EXPECT_EQ(declare_partition_options->subpartition_numbers->as()->value.safeGet(), 4); } @@ -138,10 +138,10 @@ TEST(ParserPartitionOptions, PatitionOptionsWithDeclarePartition) ASTDeclarePartitionOptions * declare_partition_options = ast->as(); EXPECT_EQ(declare_partition_options->partition_type, "key"); - EXPECT_EQ(declare_partition_options->partition_expression->as()->name, "col_01"); + EXPECT_EQ(declare_partition_options->partition_expression->as()->name(), "col_01"); EXPECT_EQ(declare_partition_options->partition_numbers->as()->value.safeGet(), 3); EXPECT_EQ(declare_partition_options->subpartition_type, "hash"); - EXPECT_EQ(declare_partition_options->subpartition_expression->as()->name, "col_02"); + EXPECT_EQ(declare_partition_options->subpartition_expression->as()->name(), "col_02"); EXPECT_EQ(declare_partition_options->subpartition_numbers->as()->value.safeGet(), 4); EXPECT_TRUE(declare_partition_options->declare_partitions->as()->children[0]->as()); } @@ -157,10 +157,10 @@ TEST(ParserPartitionOptions, PatitionOptionsWithDeclarePartitions) ASTDeclarePartitionOptions * declare_partition_options = ast->as(); EXPECT_EQ(declare_partition_options->partition_type, "key"); - EXPECT_EQ(declare_partition_options->partition_expression->as()->name, "col_01"); + EXPECT_EQ(declare_partition_options->partition_expression->as()->name(), "col_01"); EXPECT_EQ(declare_partition_options->partition_numbers->as()->value.safeGet(), 3); EXPECT_EQ(declare_partition_options->subpartition_type, "hash"); - EXPECT_EQ(declare_partition_options->subpartition_expression->as()->name, "col_02"); + EXPECT_EQ(declare_partition_options->subpartition_expression->as()->name(), "col_02"); EXPECT_EQ(declare_partition_options->subpartition_numbers->as()->value.safeGet(), 4); EXPECT_TRUE(declare_partition_options->declare_partitions->as()->children[0]->as()); EXPECT_TRUE(declare_partition_options->declare_partitions->as()->children[1]->as()); diff --git a/src/Parsers/MySQL/tests/gtest_partition_parser.cpp b/src/Parsers/MySQL/tests/gtest_partition_parser.cpp index 48e8a9f53c6..458c7acd553 100644 --- a/src/Parsers/MySQL/tests/gtest_partition_parser.cpp +++ b/src/Parsers/MySQL/tests/gtest_partition_parser.cpp @@ -22,13 +22,13 @@ TEST(ParserPartition, AllPatitionOptions) ASTDeclarePartition * declare_partition = ast->as(); EXPECT_EQ(declare_partition->partition_name, "partition_name"); ASTDeclareOptions * declare_options = declare_partition->options->as(); - EXPECT_EQ(declare_options->changes["engine"]->as()->name, "engine_name"); + EXPECT_EQ(declare_options->changes["engine"]->as()->name(), "engine_name"); EXPECT_EQ(declare_options->changes["comment"]->as()->value.safeGet(), "partition comment"); EXPECT_EQ(declare_options->changes["data_directory"]->as()->value.safeGet(), "data_directory"); EXPECT_EQ(declare_options->changes["index_directory"]->as()->value.safeGet(), "index_directory"); EXPECT_EQ(declare_options->changes["min_rows"]->as()->value.safeGet(), 0); EXPECT_EQ(declare_options->changes["max_rows"]->as()->value.safeGet(), 1000); - EXPECT_EQ(declare_options->changes["tablespace"]->as()->name, "table_space_name"); + EXPECT_EQ(declare_options->changes["tablespace"]->as()->name(), "table_space_name"); } TEST(ParserPartition, OptionalPatitionOptions) @@ -40,10 +40,10 @@ TEST(ParserPartition, OptionalPatitionOptions) ASTDeclarePartition * declare_partition = ast->as(); EXPECT_EQ(declare_partition->partition_name, "partition_name"); ASTDeclareOptions * declare_options = declare_partition->options->as(); - EXPECT_EQ(declare_options->changes["engine"]->as()->name, "engine_name"); + EXPECT_EQ(declare_options->changes["engine"]->as()->name(), "engine_name"); EXPECT_EQ(declare_options->changes["min_rows"]->as()->value.safeGet(), 0); EXPECT_EQ(declare_options->changes["max_rows"]->as()->value.safeGet(), 1000); - EXPECT_EQ(declare_options->changes["tablespace"]->as()->name, "table_space_name"); + EXPECT_EQ(declare_options->changes["tablespace"]->as()->name(), "table_space_name"); } TEST(ParserPartition, PatitionOptionsWithLessThan) @@ -56,16 +56,16 @@ TEST(ParserPartition, PatitionOptionsWithLessThan) EXPECT_EQ(declare_partition_01->partition_name, "partition_01"); EXPECT_EQ(declare_partition_01->less_than->as()->value.safeGet(), 1991); ASTDeclareOptions * declare_options_01 = declare_partition_01->options->as(); - EXPECT_EQ(declare_options_01->changes["engine"]->as()->name, "engine_name"); + EXPECT_EQ(declare_options_01->changes["engine"]->as()->name(), "engine_name"); String partition_02 = "PARTITION partition_02 VALUES LESS THAN MAXVALUE STORAGE engine = engine_name"; ASTPtr ast_partition_02 = parseQuery(p_partition, partition_02.data(), partition_02.data() + partition_02.size(), "", 0, 0); ASTDeclarePartition * declare_partition_02 = ast_partition_02->as(); EXPECT_EQ(declare_partition_02->partition_name, "partition_02"); - EXPECT_EQ(declare_partition_02->less_than->as()->name, "MAXVALUE"); + EXPECT_EQ(declare_partition_02->less_than->as()->name(), "MAXVALUE"); ASTDeclareOptions * declare_options_02 = declare_partition_02->options->as(); - EXPECT_EQ(declare_options_02->changes["engine"]->as()->name, "engine_name"); + EXPECT_EQ(declare_options_02->changes["engine"]->as()->name(), "engine_name"); String partition_03 = "PARTITION partition_03 VALUES LESS THAN (50, MAXVALUE) STORAGE engine = engine_name"; ASTPtr ast_partition_03 = parseQuery(p_partition, partition_03.data(), partition_03.data() + partition_03.size(), "", 0, 0); @@ -74,9 +74,9 @@ TEST(ParserPartition, PatitionOptionsWithLessThan) EXPECT_EQ(declare_partition_03->partition_name, "partition_03"); ASTPtr declare_partition_03_argument = declare_partition_03->less_than->as()->arguments; EXPECT_EQ(declare_partition_03_argument->children[0]->as()->value.safeGet(), 50); - EXPECT_EQ(declare_partition_03_argument->children[1]->as()->name, "MAXVALUE"); + EXPECT_EQ(declare_partition_03_argument->children[1]->as()->name(), "MAXVALUE"); ASTDeclareOptions * declare_options_03 = declare_partition_03->options->as(); - EXPECT_EQ(declare_options_03->changes["engine"]->as()->name, "engine_name"); + EXPECT_EQ(declare_options_03->changes["engine"]->as()->name(), "engine_name"); String partition_04 = "PARTITION partition_04 VALUES LESS THAN (MAXVALUE, MAXVALUE) STORAGE engine = engine_name"; ASTPtr ast_partition_04 = parseQuery(p_partition, partition_04.data(), partition_04.data() + partition_04.size(), "", 0, 0); @@ -84,10 +84,10 @@ TEST(ParserPartition, PatitionOptionsWithLessThan) ASTDeclarePartition * declare_partition_04 = ast_partition_04->as(); EXPECT_EQ(declare_partition_04->partition_name, "partition_04"); ASTPtr declare_partition_04_argument = declare_partition_04->less_than->as()->arguments; - EXPECT_EQ(declare_partition_04_argument->children[0]->as()->name, "MAXVALUE"); - EXPECT_EQ(declare_partition_04_argument->children[1]->as()->name, "MAXVALUE"); + EXPECT_EQ(declare_partition_04_argument->children[0]->as()->name(), "MAXVALUE"); + EXPECT_EQ(declare_partition_04_argument->children[1]->as()->name(), "MAXVALUE"); ASTDeclareOptions * declare_options_04 = declare_partition_04->options->as(); - EXPECT_EQ(declare_options_04->changes["engine"]->as()->name, "engine_name"); + EXPECT_EQ(declare_options_04->changes["engine"]->as()->name(), "engine_name"); } TEST(ParserPartition, PatitionOptionsWithInExpression) @@ -101,9 +101,9 @@ TEST(ParserPartition, PatitionOptionsWithInExpression) ASTPtr declare_partition_01_argument = declare_partition_01->in_expression->as()->arguments; EXPECT_TRUE(declare_partition_01_argument->children[0]->as()->value.isNull()); EXPECT_EQ(declare_partition_01_argument->children[1]->as()->value.safeGet(), 1991); - EXPECT_EQ(declare_partition_01_argument->children[2]->as()->name, "MAXVALUE"); + EXPECT_EQ(declare_partition_01_argument->children[2]->as()->name(), "MAXVALUE"); ASTDeclareOptions * declare_options_01 = declare_partition_01->options->as(); - EXPECT_EQ(declare_options_01->changes["engine"]->as()->name, "engine_name"); + EXPECT_EQ(declare_options_01->changes["engine"]->as()->name(), "engine_name"); String partition_02 = "PARTITION partition_02 VALUES IN ((NULL, 1991), (1991, NULL), (MAXVALUE, MAXVALUE)) STORAGE engine = engine_name"; ASTPtr ast_partition_02 = parseQuery(p_partition, partition_02.data(), partition_02.data() + partition_02.size(), "", 0, 0); @@ -121,11 +121,11 @@ TEST(ParserPartition, PatitionOptionsWithInExpression) EXPECT_TRUE(argument_02->as()->value.safeGet()[1].isNull()); ASTPtr argument_03 = declare_partition_02_argument->children[2]->as()->arguments; - EXPECT_EQ(argument_03->as()->children[0]->as()->name, "MAXVALUE"); - EXPECT_EQ(argument_03->as()->children[1]->as()->name, "MAXVALUE"); + EXPECT_EQ(argument_03->as()->children[0]->as()->name(), "MAXVALUE"); + EXPECT_EQ(argument_03->as()->children[1]->as()->name(), "MAXVALUE"); ASTDeclareOptions * declare_options_02 = declare_partition_02->options->as(); - EXPECT_EQ(declare_options_02->changes["engine"]->as()->name, "engine_name"); + EXPECT_EQ(declare_options_02->changes["engine"]->as()->name(), "engine_name"); } TEST(ParserPartition, PatitionOptionsWithSubpartitions) diff --git a/src/Parsers/MySQL/tests/gtest_reference_parser.cpp b/src/Parsers/MySQL/tests/gtest_reference_parser.cpp index 694558b9cc3..7447f16fc7c 100644 --- a/src/Parsers/MySQL/tests/gtest_reference_parser.cpp +++ b/src/Parsers/MySQL/tests/gtest_reference_parser.cpp @@ -14,14 +14,14 @@ TEST(ParserReference, SimpleReference) String reference_01 = "REFERENCES table_name (ref_col_01)"; ASTPtr ast_reference_01 = parseQuery(p_reference, reference_01.data(), reference_01.data() + reference_01.size(), "", 0, 0); EXPECT_EQ(ast_reference_01->as()->reference_table_name, "table_name"); - EXPECT_EQ(ast_reference_01->as()->reference_expression->as()->name, "ref_col_01"); + EXPECT_EQ(ast_reference_01->as()->reference_expression->as()->name(), "ref_col_01"); String reference_02 = "REFERENCES table_name (ref_col_01, ref_col_02)"; ASTPtr ast_reference_02 = parseQuery(p_reference, reference_02.data(), reference_02.data() + reference_02.size(), "", 0, 0); EXPECT_EQ(ast_reference_02->as()->reference_table_name, "table_name"); ASTPtr arguments = ast_reference_02->as()->reference_expression->as()->arguments; - EXPECT_EQ(arguments->children[0]->as()->name, "ref_col_01"); - EXPECT_EQ(arguments->children[1]->as()->name, "ref_col_02"); + EXPECT_EQ(arguments->children[0]->as()->name(), "ref_col_01"); + EXPECT_EQ(arguments->children[1]->as()->name(), "ref_col_02"); } TEST(ParserReference, ReferenceDifferenceKind) @@ -30,19 +30,19 @@ TEST(ParserReference, ReferenceDifferenceKind) String reference_01 = "REFERENCES table_name (ref_col_01) MATCH FULL"; ASTPtr ast_reference_01 = parseQuery(p_reference, reference_01.data(), reference_01.data() + reference_01.size(), "", 0, 0); EXPECT_EQ(ast_reference_01->as()->reference_table_name, "table_name"); - EXPECT_EQ(ast_reference_01->as()->reference_expression->as()->name, "ref_col_01"); + EXPECT_EQ(ast_reference_01->as()->reference_expression->as()->name(), "ref_col_01"); EXPECT_EQ(ast_reference_01->as()->kind, ASTDeclareReference::MATCH_FULL); String reference_02 = "REFERENCES table_name (ref_col_01) MATCH PARTIAL"; ASTPtr ast_reference_02 = parseQuery(p_reference, reference_02.data(), reference_02.data() + reference_02.size(), "", 0, 0); EXPECT_EQ(ast_reference_02->as()->reference_table_name, "table_name"); - EXPECT_EQ(ast_reference_02->as()->reference_expression->as()->name, "ref_col_01"); + EXPECT_EQ(ast_reference_02->as()->reference_expression->as()->name(), "ref_col_01"); EXPECT_EQ(ast_reference_02->as()->kind, ASTDeclareReference::MATCH_PARTIAL); String reference_03 = "REFERENCES table_name (ref_col_01) MATCH SIMPLE"; ASTPtr ast_reference_03 = parseQuery(p_reference, reference_03.data(), reference_03.data() + reference_03.size(), "", 0, 0); EXPECT_EQ(ast_reference_03->as()->reference_table_name, "table_name"); - EXPECT_EQ(ast_reference_03->as()->reference_expression->as()->name, "ref_col_01"); + EXPECT_EQ(ast_reference_03->as()->reference_expression->as()->name(), "ref_col_01"); EXPECT_EQ(ast_reference_03->as()->kind, ASTDeclareReference::MATCH_SIMPLE); } @@ -52,7 +52,7 @@ TEST(ParserReference, ReferenceDifferenceOption) String reference_01 = "REFERENCES table_name (ref_col_01) MATCH FULL ON DELETE RESTRICT ON UPDATE RESTRICT"; ASTPtr ast_reference_01 = parseQuery(p_reference, reference_01.data(), reference_01.data() + reference_01.size(), "", 0, 0); EXPECT_EQ(ast_reference_01->as()->reference_table_name, "table_name"); - EXPECT_EQ(ast_reference_01->as()->reference_expression->as()->name, "ref_col_01"); + EXPECT_EQ(ast_reference_01->as()->reference_expression->as()->name(), "ref_col_01"); EXPECT_EQ(ast_reference_01->as()->kind, ASTDeclareReference::MATCH_FULL); EXPECT_EQ(ast_reference_01->as()->on_delete_option, ASTDeclareReference::RESTRICT); EXPECT_EQ(ast_reference_01->as()->on_update_option, ASTDeclareReference::RESTRICT); @@ -60,7 +60,7 @@ TEST(ParserReference, ReferenceDifferenceOption) String reference_02 = "REFERENCES table_name (ref_col_01) MATCH FULL ON DELETE CASCADE ON UPDATE CASCADE"; ASTPtr ast_reference_02 = parseQuery(p_reference, reference_02.data(), reference_02.data() + reference_02.size(), "", 0, 0); EXPECT_EQ(ast_reference_02->as()->reference_table_name, "table_name"); - EXPECT_EQ(ast_reference_02->as()->reference_expression->as()->name, "ref_col_01"); + EXPECT_EQ(ast_reference_02->as()->reference_expression->as()->name(), "ref_col_01"); EXPECT_EQ(ast_reference_02->as()->kind, ASTDeclareReference::MATCH_FULL); EXPECT_EQ(ast_reference_02->as()->on_delete_option, ASTDeclareReference::CASCADE); EXPECT_EQ(ast_reference_02->as()->on_update_option, ASTDeclareReference::CASCADE); @@ -68,7 +68,7 @@ TEST(ParserReference, ReferenceDifferenceOption) String reference_03 = "REFERENCES table_name (ref_col_01) MATCH FULL ON DELETE SET NULL ON UPDATE SET NULL"; ASTPtr ast_reference_03 = parseQuery(p_reference, reference_03.data(), reference_03.data() + reference_03.size(), "", 0, 0); EXPECT_EQ(ast_reference_03->as()->reference_table_name, "table_name"); - EXPECT_EQ(ast_reference_03->as()->reference_expression->as()->name, "ref_col_01"); + EXPECT_EQ(ast_reference_03->as()->reference_expression->as()->name(), "ref_col_01"); EXPECT_EQ(ast_reference_03->as()->kind, ASTDeclareReference::MATCH_FULL); EXPECT_EQ(ast_reference_03->as()->on_delete_option, ASTDeclareReference::SET_NULL); EXPECT_EQ(ast_reference_03->as()->on_update_option, ASTDeclareReference::SET_NULL); @@ -76,7 +76,7 @@ TEST(ParserReference, ReferenceDifferenceOption) String reference_04 = "REFERENCES table_name (ref_col_01) MATCH FULL ON UPDATE NO ACTION ON DELETE NO ACTION"; ASTPtr ast_reference_04 = parseQuery(p_reference, reference_04.data(), reference_04.data() + reference_04.size(), "", 0, 0); EXPECT_EQ(ast_reference_04->as()->reference_table_name, "table_name"); - EXPECT_EQ(ast_reference_04->as()->reference_expression->as()->name, "ref_col_01"); + EXPECT_EQ(ast_reference_04->as()->reference_expression->as()->name(), "ref_col_01"); EXPECT_EQ(ast_reference_04->as()->kind, ASTDeclareReference::MATCH_FULL); EXPECT_EQ(ast_reference_04->as()->on_delete_option, ASTDeclareReference::NO_ACTION); EXPECT_EQ(ast_reference_04->as()->on_update_option, ASTDeclareReference::NO_ACTION); @@ -84,7 +84,7 @@ TEST(ParserReference, ReferenceDifferenceOption) String reference_05 = "REFERENCES table_name (ref_col_01) MATCH FULL ON UPDATE SET DEFAULT ON DELETE SET DEFAULT"; ASTPtr ast_reference_05 = parseQuery(p_reference, reference_05.data(), reference_05.data() + reference_05.size(), "", 0, 0); EXPECT_EQ(ast_reference_05->as()->reference_table_name, "table_name"); - EXPECT_EQ(ast_reference_05->as()->reference_expression->as()->name, "ref_col_01"); + EXPECT_EQ(ast_reference_05->as()->reference_expression->as()->name(), "ref_col_01"); EXPECT_EQ(ast_reference_05->as()->kind, ASTDeclareReference::MATCH_FULL); EXPECT_EQ(ast_reference_05->as()->on_delete_option, ASTDeclareReference::SET_DEFAULT); EXPECT_EQ(ast_reference_05->as()->on_update_option, ASTDeclareReference::SET_DEFAULT); diff --git a/src/Parsers/MySQL/tests/gtest_subpartition_parser.cpp b/src/Parsers/MySQL/tests/gtest_subpartition_parser.cpp index 5c1cf3710ab..b375f73c55c 100644 --- a/src/Parsers/MySQL/tests/gtest_subpartition_parser.cpp +++ b/src/Parsers/MySQL/tests/gtest_subpartition_parser.cpp @@ -19,13 +19,13 @@ TEST(ParserSubpartition, AllSubpatitionOptions) ASTDeclareSubPartition * declare_subpartition = ast->as(); EXPECT_EQ(declare_subpartition->logical_name, "subpartition_name"); ASTDeclareOptions * declare_options = declare_subpartition->options->as(); - EXPECT_EQ(declare_options->changes["engine"]->as()->name, "engine_name"); + EXPECT_EQ(declare_options->changes["engine"]->as()->name(), "engine_name"); EXPECT_EQ(declare_options->changes["comment"]->as()->value.safeGet(), "subpartition comment"); EXPECT_EQ(declare_options->changes["data_directory"]->as()->value.safeGet(), "data_directory"); EXPECT_EQ(declare_options->changes["index_directory"]->as()->value.safeGet(), "index_directory"); EXPECT_EQ(declare_options->changes["min_rows"]->as()->value.safeGet(), 0); EXPECT_EQ(declare_options->changes["max_rows"]->as()->value.safeGet(), 1000); - EXPECT_EQ(declare_options->changes["tablespace"]->as()->name, "table_space_name"); + EXPECT_EQ(declare_options->changes["tablespace"]->as()->name(), "table_space_name"); } TEST(ParserSubpartition, OptionalSubpatitionOptions) @@ -37,9 +37,9 @@ TEST(ParserSubpartition, OptionalSubpatitionOptions) ASTDeclareSubPartition * declare_subpartition = ast->as(); EXPECT_EQ(declare_subpartition->logical_name, "subpartition_name"); ASTDeclareOptions * declare_options = declare_subpartition->options->as(); - EXPECT_EQ(declare_options->changes["engine"]->as()->name, "engine_name"); + EXPECT_EQ(declare_options->changes["engine"]->as()->name(), "engine_name"); EXPECT_EQ(declare_options->changes["min_rows"]->as()->value.safeGet(), 0); EXPECT_EQ(declare_options->changes["max_rows"]->as()->value.safeGet(), 1000); - EXPECT_EQ(declare_options->changes["tablespace"]->as()->name, "table_space_name"); + EXPECT_EQ(declare_options->changes["tablespace"]->as()->name(), "table_space_name"); } diff --git a/src/Parsers/MySQL/tests/gtest_table_options_parser.cpp b/src/Parsers/MySQL/tests/gtest_table_options_parser.cpp index b051f6149bb..b252ee51ace 100644 --- a/src/Parsers/MySQL/tests/gtest_table_options_parser.cpp +++ b/src/Parsers/MySQL/tests/gtest_table_options_parser.cpp @@ -23,9 +23,9 @@ TEST(ParserTableOptions, AllSubpatitionOptions) ASTDeclareOptions * declare_options = ast->as(); EXPECT_EQ(declare_options->changes["auto_increment"]->as()->value.safeGet(), 1); EXPECT_EQ(declare_options->changes["avg_row_length"]->as()->value.safeGet(), 3); - EXPECT_EQ(declare_options->changes["character_set"]->as()->name, "utf-8"); + EXPECT_EQ(declare_options->changes["character_set"]->as()->name(), "utf-8"); EXPECT_EQ(declare_options->changes["checksum"]->as()->value.safeGet(), 1); - EXPECT_EQ(declare_options->changes["collate"]->as()->name, "utf8_bin"); + EXPECT_EQ(declare_options->changes["collate"]->as()->name(), "utf8_bin"); EXPECT_EQ(declare_options->changes["comment"]->as()->value.safeGet(), "table option comment"); EXPECT_EQ(declare_options->changes["compression"]->as()->value.safeGet(), "LZ4"); EXPECT_EQ(declare_options->changes["connection"]->as()->value.safeGet(), "connect_string"); @@ -33,23 +33,23 @@ TEST(ParserTableOptions, AllSubpatitionOptions) EXPECT_EQ(declare_options->changes["index_directory"]->as()->value.safeGet(), "index_directory"); EXPECT_EQ(declare_options->changes["delay_key_write"]->as()->value.safeGet(), 0); EXPECT_EQ(declare_options->changes["encryption"]->as()->value.safeGet(), "Y"); - EXPECT_EQ(declare_options->changes["engine"]->as()->name, "INNODB"); - EXPECT_EQ(declare_options->changes["insert_method"]->as()->name, "NO"); + EXPECT_EQ(declare_options->changes["engine"]->as()->name(), "INNODB"); + EXPECT_EQ(declare_options->changes["insert_method"]->as()->name(), "NO"); EXPECT_EQ(declare_options->changes["key_block_size"]->as()->value.safeGet(), 3); EXPECT_EQ(declare_options->changes["max_rows"]->as()->value.safeGet(), 1000); EXPECT_EQ(declare_options->changes["min_rows"]->as()->value.safeGet(), 0); - EXPECT_EQ(declare_options->changes["pack_keys"]->as()->name, "DEFAULT"); + EXPECT_EQ(declare_options->changes["pack_keys"]->as()->name(), "DEFAULT"); EXPECT_EQ(declare_options->changes["password"]->as()->value.safeGet(), "password"); - EXPECT_EQ(declare_options->changes["row_format"]->as()->name, "DYNAMIC"); - EXPECT_EQ(declare_options->changes["stats_auto_recalc"]->as()->name, "DEFAULT"); - EXPECT_EQ(declare_options->changes["stats_persistent"]->as()->name, "DEFAULT"); + EXPECT_EQ(declare_options->changes["row_format"]->as()->name(), "DYNAMIC"); + EXPECT_EQ(declare_options->changes["stats_auto_recalc"]->as()->name(), "DEFAULT"); + EXPECT_EQ(declare_options->changes["stats_persistent"]->as()->name(), "DEFAULT"); EXPECT_EQ(declare_options->changes["stats_sample_pages"]->as()->value.safeGet(), 3); - EXPECT_EQ(declare_options->changes["tablespace"]->as()->name, "tablespace_name"); + EXPECT_EQ(declare_options->changes["tablespace"]->as()->name(), "tablespace_name"); ASTPtr arguments = declare_options->changes["union"]->as()->arguments; - EXPECT_EQ(arguments->children[0]->as()->name, "table_01"); - EXPECT_EQ(arguments->children[1]->as()->name, "table_02"); + EXPECT_EQ(arguments->children[0]->as()->name(), "table_01"); + EXPECT_EQ(arguments->children[1]->as()->name(), "table_02"); } TEST(ParserTableOptions, OptionalTableOptions) @@ -60,5 +60,5 @@ TEST(ParserTableOptions, OptionalTableOptions) ASTDeclareOptions * declare_options = ast->as(); EXPECT_EQ(declare_options->changes["auto_increment"]->as()->value.safeGet(), 1); - EXPECT_EQ(declare_options->changes["stats_auto_recalc"]->as()->name, "DEFAULT"); + EXPECT_EQ(declare_options->changes["stats_auto_recalc"]->as()->name(), "DEFAULT"); } diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 1afdfac0461..6416e08d93b 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -114,7 +114,7 @@ bool ParserIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expe return false; auto index = std::make_shared(); - index->name = name->as().name; + index->name = name->as().name(); index->granularity = granularity->as().value.safeGet(); index->set(index->expr, expr); index->set(index->type, type); @@ -143,7 +143,7 @@ bool ParserConstraintDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & return false; auto constraint = std::make_shared(); - constraint->name = name->as().name; + constraint->name = name->as().name(); constraint->set(constraint->expr, expr); node = constraint; diff --git a/src/Parsers/ParserDictionary.cpp b/src/Parsers/ParserDictionary.cpp index d69e4b02aed..77cd480d595 100644 --- a/src/Parsers/ParserDictionary.cpp +++ b/src/Parsers/ParserDictionary.cpp @@ -95,9 +95,9 @@ bool ParserDictionaryRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expec return false; if (pair.first == "min") - res->min_attr_name = identifier->name; + res->min_attr_name = identifier->name(); else if (pair.first == "max") - res->max_attr_name = identifier->name; + res->max_attr_name = identifier->name(); else return false; } diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index 296f4187e3a..020b7993c2d 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -137,7 +137,7 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & { ASTPtr ast; if (ParserIdentifier{}.parse(pos, ast, expected)) - storage_policy_str = ast->as().name; + storage_policy_str = ast->as().name(); else return false; @@ -145,7 +145,7 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & return false; if (ParserIdentifier{}.parse(pos, ast, expected)) - volume_str = ast->as().name; + volume_str = ast->as().name(); else return false; } diff --git a/src/Parsers/tests/gtest_dictionary_parser.cpp b/src/Parsers/tests/gtest_dictionary_parser.cpp index 6b777af77a2..c2bde5fa8f1 100644 --- a/src/Parsers/tests/gtest_dictionary_parser.cpp +++ b/src/Parsers/tests/gtest_dictionary_parser.cpp @@ -86,7 +86,7 @@ TEST(ParserDictionaryDDL, SimpleDictionary) auto * primary_key = create->dictionary->primary_key; EXPECT_EQ(primary_key->children.size(), 1); - EXPECT_EQ(primary_key->children[0]->as()->name, "key_column"); + EXPECT_EQ(primary_key->children[0]->as()->name(), "key_column"); /// range test auto * range = create->dictionary->range; diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 7beb0a4d706..8cae7866748 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -208,7 +208,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ command.index_name = ast_index_decl.name; if (command_ast->index) - command.after_index_name = command_ast->index->as().name; + command.after_index_name = command_ast->index->as().name(); command.if_not_exists = command_ast->if_not_exists; @@ -235,7 +235,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ command.ast = command_ast->clone(); command.if_exists = command_ast->if_exists; command.type = AlterCommand::DROP_CONSTRAINT; - command.constraint_name = command_ast->constraint->as().name; + command.constraint_name = command_ast->constraint->as().name(); return command; } @@ -244,7 +244,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ AlterCommand command; command.ast = command_ast->clone(); command.type = AlterCommand::DROP_INDEX; - command.index_name = command_ast->index->as().name; + command.index_name = command_ast->index->as().name(); command.if_exists = command_ast->if_exists; if (command_ast->clear_index) command.clear = true; @@ -290,8 +290,8 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ AlterCommand command; command.ast = command_ast->clone(); command.type = AlterCommand::RENAME_COLUMN; - command.column_name = command_ast->column->as().name; - command.rename_to = command_ast->rename_to->as().name; + command.column_name = command_ast->column->as().name(); + command.rename_to = command_ast->rename_to->as().name(); command.if_exists = command_ast->if_exists; return command; } diff --git a/src/Storages/MutationCommands.cpp b/src/Storages/MutationCommands.cpp index f3569c344d9..ba998dd5951 100644 --- a/src/Storages/MutationCommands.cpp +++ b/src/Storages/MutationCommands.cpp @@ -57,7 +57,7 @@ std::optional MutationCommand::parse(ASTAlterCommand * command, res.type = MATERIALIZE_INDEX; res.partition = command->partition; res.predicate = nullptr; - res.index_name = command->index->as().name; + res.index_name = command->index->as().name(); return res; } else if (parse_alter_commands && command->type == ASTAlterCommand::MODIFY_COLUMN) @@ -88,7 +88,7 @@ std::optional MutationCommand::parse(ASTAlterCommand * command, MutationCommand res; res.ast = command->ptr(); res.type = MutationCommand::Type::DROP_INDEX; - res.column_name = command->index->as().name; + res.column_name = command->index->as().name(); if (command->partition) res.partition = command->partition; if (command->clear_index) @@ -100,8 +100,8 @@ std::optional MutationCommand::parse(ASTAlterCommand * command, MutationCommand res; res.ast = command->ptr(); res.type = MutationCommand::Type::RENAME_COLUMN; - res.column_name = command->column->as().name; - res.rename_to = command->rename_to->as().name; + res.column_name = command->column->as().name(); + res.rename_to = command->rename_to->as().name(); return res; } else if (command->type == ASTAlterCommand::MATERIALIZE_TTL) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index b858239d637..9046940b3f7 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -271,7 +271,7 @@ std::optional getOptimizedQueryProcessingStage(const if (!id) return false; /// TODO: if GROUP BY contains multiIf()/if() it should contain only columns from sharding_key - if (!sharding_key_block.has(id->name)) + if (!sharding_key_block.has(id->name())) return false; } return true; diff --git a/src/Storages/System/StorageSystemZooKeeper.cpp b/src/Storages/System/StorageSystemZooKeeper.cpp index 81a42f1fe63..a3660cf2dec 100644 --- a/src/Storages/System/StorageSystemZooKeeper.cpp +++ b/src/Storages/System/StorageSystemZooKeeper.cpp @@ -74,7 +74,7 @@ static bool extractPathImpl(const IAST & elem, String & res, const Context & con else return false; - if (ident->name != "path") + if (ident->name() != "path") return false; auto evaluated = evaluateConstantExpressionAsLiteral(value, context); diff --git a/tests/queries/server.py b/tests/queries/server.py index e9f7361a6fe..c4f8968e08a 100644 --- a/tests/queries/server.py +++ b/tests/queries/server.py @@ -137,19 +137,34 @@ ServerThread.DEFAULT_SERVER_CONFIG = \ - - - localhost - {tcp_port} - - - - - localhost - {tcp_port} - - - + + + localhost + {tcp_port} + + + + + localhost + {tcp_port} + + + + + + + + 127.0.0.1 + {tcp_port} + + + + + 127.0.0.2 + {tcp_port} + + + diff --git a/utils/db-generator/query_db_generator.cpp b/utils/db-generator/query_db_generator.cpp index 84ad07056b9..c8aae4a56f3 100644 --- a/utils/db-generator/query_db_generator.cpp +++ b/utils/db-generator/query_db_generator.cpp @@ -622,7 +622,7 @@ FuncRet arrayJoinFunc(DB::ASTPtr ch, std::map & columns) { auto ident = std::dynamic_pointer_cast(arg); if (ident) - indents.insert(ident->name); + indents.insert(ident->name()); } for (const auto & indent : indents) { @@ -654,7 +654,7 @@ FuncRet inFunc(DB::ASTPtr ch, std::map & columns) auto ident = std::dynamic_pointer_cast(arg); if (ident) { - indents.insert(ident->name); + indents.insert(ident->name()); } auto literal = std::dynamic_pointer_cast(arg); if (literal) @@ -734,7 +734,7 @@ FuncRet arrayFunc(DB::ASTPtr ch, std::map & columns) if (ident) { no_indent = false; - indents.insert(ident->name); + indents.insert(ident->name()); } auto literal = std::dynamic_pointer_cast(arg); if (literal) @@ -784,7 +784,7 @@ FuncRet arithmeticFunc(DB::ASTPtr ch, std::map & columns) if (ident) { no_indent = false; - indents.insert(ident->name); + indents.insert(ident->name()); } auto literal = std::dynamic_pointer_cast(arg); if (literal) @@ -848,7 +848,7 @@ FuncRet likeFunc(DB::ASTPtr ch, std::map & columns) { auto ident = std::dynamic_pointer_cast(arg); if (ident) - indents.insert(ident->name); + indents.insert(ident->name()); auto literal = std::dynamic_pointer_cast(arg); if (literal) { @@ -905,7 +905,7 @@ FuncRet simpleFunc(DB::ASTPtr ch, std::map & columns) if (ident) { no_indent = false; - indents.insert(ident->name); + indents.insert(ident->name()); } auto literal = std::dynamic_pointer_cast(arg); if (literal) @@ -1046,7 +1046,7 @@ std::set getIndent(DB::ASTPtr ch) std::set ret = {}; auto x = std::dynamic_pointer_cast(ch); if (x) - ret.insert(x->name); + ret.insert(x->name()); for (const auto & child : (*ch).children) { auto child_ind = getIndent(child); From 2e23fc242fe3cc295571c126f487a874b8e8fb8f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Oct 2020 22:48:16 +0300 Subject: [PATCH 173/325] Remove flaky LIVE VIEW test --- .../__init__.py | 0 .../configs/remote_servers.xml | 18 -- .../configs/set_distributed_defaults.xml | 35 --- .../test_distributed_over_live_view/test.py | 276 ------------------ 4 files changed, 329 deletions(-) delete mode 100644 tests/integration/test_distributed_over_live_view/__init__.py delete mode 100644 tests/integration/test_distributed_over_live_view/configs/remote_servers.xml delete mode 100644 tests/integration/test_distributed_over_live_view/configs/set_distributed_defaults.xml delete mode 100644 tests/integration/test_distributed_over_live_view/test.py diff --git a/tests/integration/test_distributed_over_live_view/__init__.py b/tests/integration/test_distributed_over_live_view/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_distributed_over_live_view/configs/remote_servers.xml b/tests/integration/test_distributed_over_live_view/configs/remote_servers.xml deleted file mode 100644 index ebce4697529..00000000000 --- a/tests/integration/test_distributed_over_live_view/configs/remote_servers.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - node1 - 9000 - - - - - node2 - 9000 - - - - - diff --git a/tests/integration/test_distributed_over_live_view/configs/set_distributed_defaults.xml b/tests/integration/test_distributed_over_live_view/configs/set_distributed_defaults.xml deleted file mode 100644 index 194eb1ebb87..00000000000 --- a/tests/integration/test_distributed_over_live_view/configs/set_distributed_defaults.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - - 3 - 1000 - 1 - - - 5 - 3000 - 1 - - - - - - - - ::/0 - - default - default - - - - - ::/0 - - delays - default - - - - - diff --git a/tests/integration/test_distributed_over_live_view/test.py b/tests/integration/test_distributed_over_live_view/test.py deleted file mode 100644 index 78b90024ebf..00000000000 --- a/tests/integration/test_distributed_over_live_view/test.py +++ /dev/null @@ -1,276 +0,0 @@ - - -import sys -import time - -import pytest -from helpers.cluster import ClickHouseCluster -from helpers.uclient import client, prompt, end_of_block - -cluster = ClickHouseCluster(__file__) - -# log = sys.stdout -log = None - -NODES = {'node' + str(i): cluster.add_instance( - 'node' + str(i), - main_configs=['configs/remote_servers.xml'], - user_configs=['configs/set_distributed_defaults.xml'], -) for i in (1, 2)} - -CREATE_TABLES_SQL = ''' -DROP TABLE IF EXISTS lv_over_base_table; -DROP TABLE IF EXISTS distributed_table; -DROP TABLE IF EXISTS base_table; - -SET allow_experimental_live_view = 1; - -CREATE TABLE - base_table( - node String, - key Int32, - value Int32 - ) -ENGINE = Memory; - -CREATE LIVE VIEW lv_over_base_table AS SELECT * FROM base_table; - -CREATE TABLE - distributed_table -AS base_table -ENGINE = Distributed(test_cluster, default, base_table, rand()); -''' - -INSERT_SQL_TEMPLATE = "INSERT INTO base_table VALUES ('{node_id}', {key}, {value})" - - -@pytest.fixture(scope="function") -def started_cluster(): - try: - cluster.start() - for node_index, (node_name, node) in enumerate(NODES.items()): - node.query(CREATE_TABLES_SQL) - for i in range(0, 2): - sql = INSERT_SQL_TEMPLATE.format(node_id=node_name, key=i, value=i + (node_index * 10)) - node.query(sql) - yield cluster - - finally: - cluster.shutdown() - -def poll_query(node, query, expected, timeout): - """Repeatedly execute query until either expected result is returned or timeout occurs. - """ - start_time = time.time() - while node.query(query) != expected and time.time() - start_time < timeout: - pass - assert node.query(query) == expected - -@pytest.mark.parametrize("node", list(NODES.values())[:1]) -@pytest.mark.parametrize("source", ["lv_over_distributed_table"]) -class TestLiveViewOverDistributedSuite: - def test_distributed_over_live_view_order_by_node(self, started_cluster, node, source): - node0, node1 = list(NODES.values()) - - select_query = "SELECT * FROM distributed_over_lv ORDER BY node, key FORMAT CSV" - select_query_dist_table = "SELECT * FROM distributed_table ORDER BY node, key FORMAT CSV" - select_count_query = "SELECT count() FROM distributed_over_lv" - - with client(name="client1> ", log=log, command=" ".join(node0.client.command)) as client1, \ - client(name="client2> ", log=log, command=" ".join(node1.client.command)) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send("DROP TABLE IF EXISTS distributed_over_lv") - client1.expect(prompt) - client1.send( - "CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") - client1.expect(prompt) - - client1.send(select_query) - client1.expect('"node1",0,0') - client1.expect('"node1",1,1') - client1.expect('"node2",0,10') - client1.expect('"node2",1,11') - client1.expect(prompt) - - client1.send("INSERT INTO distributed_table VALUES ('node1', 1, 3), ('node1', 2, 3)") - client1.expect(prompt) - client2.send("INSERT INTO distributed_table VALUES ('node1', 3, 3)") - client2.expect(prompt) - - poll_query(node0, select_count_query, "7\n", timeout=60) - print("\n--DEBUG1--") - print(select_query) - print(node0.query(select_query)) - print("---------") - print("\n--DEBUG2--") - print(select_query_dist_table) - print(node0.query(select_query_dist_table)) - print("---------") - - client1.send(select_query) - client1.expect('"node1",0,0') - client1.expect('"node1",1,1') - client1.expect('"node1",1,3') - client1.expect('"node1",2,3') - client1.expect('"node1",3,3') - client1.expect('"node2",0,10') - client1.expect('"node2",1,11') - client1.expect(prompt) - - def test_distributed_over_live_view_order_by_key(self, started_cluster, node, source): - node0, node1 = list(NODES.values()) - - select_query = "SELECT * FROM distributed_over_lv ORDER BY key, node FORMAT CSV" - select_count_query = "SELECT count() FROM distributed_over_lv" - - with client(name="client1> ", log=log, command=" ".join(node0.client.command)) as client1, \ - client(name="client2> ", log=log, command=" ".join(node1.client.command)) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send("DROP TABLE IF EXISTS distributed_over_lv") - client1.expect(prompt) - client1.send( - "CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") - client1.expect(prompt) - - client1.send(select_query) - client1.expect('"node1",0,0') - client1.expect('"node2",0,10') - client1.expect('"node1",1,1') - client1.expect('"node2",1,11') - client1.expect(prompt) - - client1.send("INSERT INTO distributed_table VALUES ('node1', 1, 3), ('node1', 2, 3)") - client1.expect(prompt) - client2.send("INSERT INTO distributed_table VALUES ('node1', 3, 3)") - client2.expect(prompt) - - poll_query(node0, select_count_query, "7\n", timeout=60) - - client1.send(select_query) - client1.expect('"node1",0,0') - client1.expect('"node2",0,10') - client1.expect('"node1",1,1') - client1.expect('"node1",1,3') - client1.expect('"node2",1,11') - client1.expect('"node1",2,3') - client1.expect('"node1",3,3') - client1.expect(prompt) - - def test_distributed_over_live_view_group_by_node(self, started_cluster, node, source): - node0, node1 = list(NODES.values()) - - select_query = "SELECT node, SUM(value) FROM distributed_over_lv GROUP BY node ORDER BY node FORMAT CSV" - - with client(name="client1> ", log=log, command=" ".join(node0.client.command)) as client1, \ - client(name="client2> ", log=log, command=" ".join(node1.client.command)) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send("DROP TABLE IF EXISTS distributed_over_lv") - client1.expect(prompt) - client1.send( - "CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") - client1.expect(prompt) - - client1.send(select_query) - client1.expect('"node1",1') - client1.expect('"node2",21') - client1.expect(prompt) - - client2.send("INSERT INTO distributed_table VALUES ('node1', 2, 2)") - client2.expect(prompt) - - poll_query(node0, select_query, '"node1",3\n"node2",21\n', timeout=60) - - client1.send(select_query) - client1.expect('"node1",3') - client1.expect('"node2",21') - client1.expect(prompt) - - client1.send("INSERT INTO distributed_table VALUES ('node1', 1, 3), ('node1', 3, 3)") - client1.expect(prompt) - client2.send("INSERT INTO distributed_table VALUES ('node1', 3, 3)") - client2.expect(prompt) - - poll_query(node0, select_query, '"node1",12\n"node2",21\n', timeout=60) - - client1.send(select_query) - client1.expect('"node1",12') - client1.expect('"node2",21') - client1.expect(prompt) - - def test_distributed_over_live_view_group_by_key(self, started_cluster, node, source): - node0, node1 = list(NODES.values()) - - select_query = "SELECT key, SUM(value) FROM distributed_over_lv GROUP BY key ORDER BY key FORMAT CSV" - - with client(name="client1> ", log=log, command=" ".join(node0.client.command)) as client1, \ - client(name="client2> ", log=log, command=" ".join(node1.client.command)) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send("DROP TABLE IF EXISTS distributed_over_lv") - client1.expect(prompt) - client1.send( - "CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") - client1.expect(prompt) - - client1.send(select_query) - client1.expect('0,10') - client1.expect('1,12') - client1.expect(prompt) - - client2.send("INSERT INTO distributed_table VALUES ('node1', 2, 2)") - client2.expect(prompt) - - poll_query(node0, "SELECT count() FROM (%s)" % select_query.rsplit("FORMAT")[0], "3\n", timeout=60) - - client1.send(select_query) - client1.expect('0,10') - client1.expect('1,12') - client1.expect('2,2') - client1.expect(prompt) - - client2.send("INSERT INTO distributed_table VALUES ('node1', 1, 3), ('node1', 3, 3)") - client2.expect(prompt) - - poll_query(node0, "SELECT count() FROM (%s)" % select_query.rsplit("FORMAT")[0], "4\n", timeout=60) - - client1.send(select_query) - client1.expect('0,10') - client1.expect('1,15') - client1.expect('2,2') - client1.expect('3,3') - client1.expect(prompt) - - def test_distributed_over_live_view_sum(self, started_cluster, node, source): - node0, node1 = list(NODES.values()) - - with client(name="client1> ", log=log, command=" ".join(node0.client.command)) as client1, \ - client(name="client2> ", log=log, command=" ".join(node1.client.command)) as client2: - client1.expect(prompt) - client2.expect(prompt) - - client1.send("DROP TABLE IF EXISTS distributed_over_lv") - client1.expect(prompt) - client1.send( - "CREATE TABLE distributed_over_lv AS lv_over_base_table ENGINE = Distributed(test_cluster, default, lv_over_base_table)") - client1.expect(prompt) - - client1.send("SELECT sum(value) FROM distributed_over_lv") - client1.expect(r"22" + end_of_block) - client1.expect(prompt) - - client2.send("INSERT INTO distributed_table VALUES ('node1', 2, 2)") - client2.expect(prompt) - - poll_query(node0, "SELECT sum(value) FROM distributed_over_lv", "24\n", timeout=60) - - client2.send("INSERT INTO distributed_table VALUES ('node1', 3, 3), ('node1', 4, 4)") - client2.expect(prompt) - - poll_query(node0, "SELECT sum(value) FROM distributed_over_lv", "31\n", timeout=60) From 9d50921e529402ec7dbd76b61506fa40df84e2f2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Oct 2020 23:52:32 +0300 Subject: [PATCH 174/325] Fix performance test "functions logical" after move to clang --- src/Functions/FunctionsLogical.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index 58e1c52a60c..defa4f4493b 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -290,10 +290,9 @@ private: /// Apply target function by feeding it "batches" of N columns -/// Combining 10 columns per pass is the fastest for large columns sizes. -/// For small columns sizes - more columns is faster. +/// Combining 8 columns per pass is the fastest method, because it's the maximum when clang vectorizes a loop. template < - typename Op, template typename OperationApplierImpl, size_t N = 10> + typename Op, template typename OperationApplierImpl, size_t N = 8> struct OperationApplier { template From 8b21ef5d4fd6c9d2e3a49135d819676ed09fffd0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 25 Oct 2020 00:50:52 +0300 Subject: [PATCH 175/325] Remove excessive exclamation marks --- .../AggregateFunctionTimeSeriesGroupSum.h | 2 +- src/Core/tests/CMakeLists.txt | 3 --- src/Core/tests/gtest_move_field.cpp | 22 ++++++++++++++++ src/Core/tests/move_field.cpp | 25 ------------------- src/Interpreters/tests/two_level_hash_map.cpp | 2 +- 5 files changed, 24 insertions(+), 30 deletions(-) create mode 100644 src/Core/tests/gtest_move_field.cpp delete mode 100644 src/Core/tests/move_field.cpp diff --git a/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h b/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h index be0a3eb4af5..b755fbf081b 100644 --- a/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h +++ b/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h @@ -92,7 +92,7 @@ struct AggregateFunctionTimeSeriesGroupSumData it_ss->second.add(t, v); } if (result.size() > 0 && t < result.back().first) - throw Exception{"timeSeriesGroupSum or timeSeriesGroupRateSum must order by timestamp asc!!!", ErrorCodes::LOGICAL_ERROR}; + throw Exception{"timeSeriesGroupSum or timeSeriesGroupRateSum must order by timestamp asc.", ErrorCodes::LOGICAL_ERROR}; if (result.size() > 0 && t == result.back().first) { //do not add new point diff --git a/src/Core/tests/CMakeLists.txt b/src/Core/tests/CMakeLists.txt index d609e49f247..cd6450633ff 100644 --- a/src/Core/tests/CMakeLists.txt +++ b/src/Core/tests/CMakeLists.txt @@ -5,9 +5,6 @@ target_include_directories (string_pool SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLU add_executable (field field.cpp) target_link_libraries (field PRIVATE dbms) -add_executable (move_field move_field.cpp) -target_link_libraries (move_field PRIVATE clickhouse_common_io) - add_executable (string_ref_hash string_ref_hash.cpp) target_link_libraries (string_ref_hash PRIVATE clickhouse_common_io) diff --git a/src/Core/tests/gtest_move_field.cpp b/src/Core/tests/gtest_move_field.cpp new file mode 100644 index 00000000000..9c807039c6a --- /dev/null +++ b/src/Core/tests/gtest_move_field.cpp @@ -0,0 +1,22 @@ +#include +#include + +using namespace DB; + +GTEST_TEST(Field, Move) +{ + Field f; + + f = Field{String{"Hello, world (1)"}}; + ASSERT_EQ(f.get(), "Hello, world (1)"); + f = Field{String{"Hello, world (2)"}}; + ASSERT_EQ(f.get(), "Hello, world (2)"); + f = Field{Array{Field{String{"Hello, world (3)"}}}}; + ASSERT_EQ(f.get()[0].get(), "Hello, world (3)"); + f = String{"Hello, world (4)"}; + ASSERT_EQ(f.get(), "Hello, world (4)"); + f = Array{Field{String{"Hello, world (5)"}}}; + ASSERT_EQ(f.get()[0].get(), "Hello, world (5)"); + f = Array{String{"Hello, world (6)"}}; + ASSERT_EQ(f.get()[0].get(), "Hello, world (6)"); +} diff --git a/src/Core/tests/move_field.cpp b/src/Core/tests/move_field.cpp deleted file mode 100644 index 2780abffc40..00000000000 --- a/src/Core/tests/move_field.cpp +++ /dev/null @@ -1,25 +0,0 @@ -#include -#include - - -int main(int, char **) -{ - using namespace DB; - - Field f; - - f = Field{String{"Hello, world"}}; - std::cerr << f.get() << "\n"; - f = Field{String{"Hello, world!"}}; - std::cerr << f.get() << "\n"; - f = Field{Array{Field{String{"Hello, world!!"}}}}; - std::cerr << f.get()[0].get() << "\n"; - f = String{"Hello, world!!!"}; - std::cerr << f.get() << "\n"; - f = Array{Field{String{"Hello, world!!!!"}}}; - std::cerr << f.get()[0].get() << "\n"; - f = Array{String{"Hello, world!!!!!"}}; - std::cerr << f.get()[0].get() << "\n"; - - return 0; -} diff --git a/src/Interpreters/tests/two_level_hash_map.cpp b/src/Interpreters/tests/two_level_hash_map.cpp index f79be16e095..33fd5ee8305 100644 --- a/src/Interpreters/tests/two_level_hash_map.cpp +++ b/src/Interpreters/tests/two_level_hash_map.cpp @@ -128,7 +128,7 @@ int main(int argc, char ** argv) std::cerr << "sum_counts: " << sum_counts << ", elems: " << elems << std::endl; if (sum_counts != n) - std::cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" << std::endl; + std::cerr << "Error!" << std::endl; } return 0; From 942828f4d5c810235cdcd281de90ffb4350ef07b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 25 Oct 2020 00:54:17 +0300 Subject: [PATCH 176/325] Check style --- utils/check-style/check-style | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 4983782c00d..ef569c9f73e 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -99,3 +99,6 @@ find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} '(' -name '*. find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xEF\xBB\xBF' && echo "Files should not have UTF-8 BOM" find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xFF\xFE' && echo "Files should not have UTF-16LE BOM" find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xFE\xFF' && echo "Files should not have UTF-16BE BOM" + +# Too many exclamation marks +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -F '!!!' && echo "Too many exclamation marks (looks dirty, unconfident)." From 2613012fd1f888f5ec72f1ee0b4296e7a672596a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 25 Oct 2020 04:43:06 +0300 Subject: [PATCH 177/325] Improve performance of FunctionsLogical a little by adding "restrict" --- src/Functions/FunctionsLogical.cpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index defa4f4493b..3e19516daaa 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -299,28 +299,29 @@ struct OperationApplier static void apply(Columns & in, ResultData & result_data, bool use_result_data_as_input = false) { if (!use_result_data_as_input) - doBatchedApply(in, result_data); + doBatchedApply(in, result_data.data(), result_data.size()); while (!in.empty()) - doBatchedApply(in, result_data); + doBatchedApply(in, result_data.data(), result_data.size()); } - template - static void NO_INLINE doBatchedApply(Columns & in, ResultData & result_data) + template + static void NO_INLINE doBatchedApply(Columns & in, Result * __restrict result_data, size_t size) { if (N > in.size()) { OperationApplier - ::template doBatchedApply(in, result_data); + ::template doBatchedApply(in, result_data, size); return; } const OperationApplierImpl operation_applier_impl(in); - size_t i = 0; - for (auto & res : result_data) + for (size_t i = 0; i < size; ++i) + { if constexpr (CarryResult) - res = Op::apply(res, operation_applier_impl.apply(i++)); + result_data[i] = Op::apply(result_data[i], operation_applier_impl.apply(i)); else - res = operation_applier_impl.apply(i++); + result_data[i] = operation_applier_impl.apply(i); + } in.erase(in.end() - N, in.end()); } @@ -331,7 +332,7 @@ template < struct OperationApplier { template - static void NO_INLINE doBatchedApply(Columns &, Result &) + static void NO_INLINE doBatchedApply(Columns &, Result &, size_t) { throw Exception( "OperationApplier<...>::apply(...): not enough arguments to run this method", From 4bf32cb15bf78827bd86c1310741ebe005acc8b9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 16 Oct 2020 02:46:36 +0300 Subject: [PATCH 178/325] Fix destruction order of Suggest #16035 --- programs/client/Client.cpp | 9 ++++++--- programs/client/Suggest.h | 13 ++++--------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 3cd584c0e55..8ccdfd8af65 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -218,6 +218,8 @@ private: QueryFuzzer fuzzer; int query_fuzzer_runs = 0; + std::optional suggest; + /// We will format query_id in interactive mode in various ways, the default is just to print Query id: ... std::vector> query_id_formats; @@ -577,10 +579,11 @@ private: if (print_time_to_stderr) throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS); + suggest.emplace(); if (server_revision >= Suggest::MIN_SERVER_REVISION && !config().getBool("disable_suggestion", false)) { /// Load suggestion data from the server. - Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit")); + suggest->load(connection_parameters, config().getInt("suggestion_limit")); } /// Load command history if present. @@ -607,7 +610,7 @@ private: highlight_callback = highlight; ReplxxLineReader lr( - Suggest::instance(), + *suggest, history_file, config().has("multiline"), query_extenders, @@ -615,7 +618,7 @@ private: highlight_callback); #elif defined(USE_READLINE) && USE_READLINE - ReadlineLineReader lr(Suggest::instance(), history_file, config().has("multiline"), query_extenders, query_delimiters); + ReadlineLineReader lr(*suggest, history_file, config().has("multiline"), query_extenders, query_delimiters); #else LineReader lr(history_file, config().has("multiline"), query_extenders, query_delimiters); #endif diff --git a/programs/client/Suggest.h b/programs/client/Suggest.h index b13289ac322..03332088cbe 100644 --- a/programs/client/Suggest.h +++ b/programs/client/Suggest.h @@ -18,10 +18,11 @@ namespace ErrorCodes class Suggest : public LineReader::Suggest, boost::noncopyable { public: - static Suggest & instance() + Suggest(); + ~Suggest() { - static Suggest instance; - return instance; + if (loading_thread.joinable()) + loading_thread.join(); } void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit); @@ -30,12 +31,6 @@ public: static constexpr int MIN_SERVER_REVISION = 54406; private: - Suggest(); - ~Suggest() - { - if (loading_thread.joinable()) - loading_thread.join(); - } void loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit); void fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query); From 2efd740dc7cbd3159e269eab5e03e1a48fbfb8c6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 16 Oct 2020 03:07:56 +0300 Subject: [PATCH 179/325] Added a test --- .../01526_client_start_and_exit.reference | 1 + .../01526_client_start_and_exit.sh | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 tests/queries/0_stateless/01526_client_start_and_exit.reference create mode 100755 tests/queries/0_stateless/01526_client_start_and_exit.sh diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.reference b/tests/queries/0_stateless/01526_client_start_and_exit.reference new file mode 100644 index 00000000000..7326d960397 --- /dev/null +++ b/tests/queries/0_stateless/01526_client_start_and_exit.reference @@ -0,0 +1 @@ +Ok diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.sh b/tests/queries/0_stateless/01526_client_start_and_exit.sh new file mode 100755 index 00000000000..d3fd2cb73b2 --- /dev/null +++ b/tests/queries/0_stateless/01526_client_start_and_exit.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +# Create a huge amount of tables, so Suggest will take a time to load +seq 1 1000 | sed -r -e 's/(.+)/CREATE TABLE IF NOT EXISTS test\1 (x UInt8) ENGINE = Memory;/' | ${CLICKHOUSE_CLIENT} -n + +function stress() +{ + while true; do + ./"$CURDIR"/01526_client_start_and_exit.expect + done +} + +export -f stress + +for _ in {1..10}; do + timeout 3 bash -c stress & +done + +wait +echo 'Ok' + +seq 1 1000 | sed -r -e 's/(.+)/DROP TABLE test\1;/' | ${CLICKHOUSE_CLIENT} -n From 90a3e95aabac19ab4dae30819dc579a88428c297 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 16 Oct 2020 03:38:32 +0300 Subject: [PATCH 180/325] Add missing file --- .../0_stateless/01526_client_start_and_exit.expect | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 tests/queries/0_stateless/01526_client_start_and_exit.expect diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.expect b/tests/queries/0_stateless/01526_client_start_and_exit.expect new file mode 100755 index 00000000000..4ad0eea5d97 --- /dev/null +++ b/tests/queries/0_stateless/01526_client_start_and_exit.expect @@ -0,0 +1,12 @@ +#!/usr/bin/expect -f + +log_user 0 +set timeout 5 +match_max 100000 + +if ![info exists env(CLICKHOUSE_PORT_TCP)] {set env(CLICKHOUSE_PORT_TCP) 9000} + +spawn clickhouse-client --port "$env(CLICKHOUSE_PORT_TCP)" +expect ":) " +send -- "\4" +expect eof From 8cc463f549d29e5c11ea8664e5c17eb05ab50033 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 25 Oct 2020 05:31:58 +0300 Subject: [PATCH 181/325] Better test --- .../queries/0_stateless/01526_client_start_and_exit.expect | 4 ++-- .../0_stateless/01526_client_start_and_exit.reference | 2 +- tests/queries/0_stateless/01526_client_start_and_exit.sh | 7 ++----- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.expect b/tests/queries/0_stateless/01526_client_start_and_exit.expect index 4ad0eea5d97..003439ffa54 100755 --- a/tests/queries/0_stateless/01526_client_start_and_exit.expect +++ b/tests/queries/0_stateless/01526_client_start_and_exit.expect @@ -1,12 +1,12 @@ #!/usr/bin/expect -f -log_user 0 +log_user 1 set timeout 5 match_max 100000 if ![info exists env(CLICKHOUSE_PORT_TCP)] {set env(CLICKHOUSE_PORT_TCP) 9000} -spawn clickhouse-client --port "$env(CLICKHOUSE_PORT_TCP)" +spawn bash -c "clickhouse-client --port $env(CLICKHOUSE_PORT_TCP) && echo $?" expect ":) " send -- "\4" expect eof diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.reference b/tests/queries/0_stateless/01526_client_start_and_exit.reference index 7326d960397..e3e2e7b22af 100644 --- a/tests/queries/0_stateless/01526_client_start_and_exit.reference +++ b/tests/queries/0_stateless/01526_client_start_and_exit.reference @@ -1 +1 @@ -Ok +Loaded 10000 queries. diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.sh b/tests/queries/0_stateless/01526_client_start_and_exit.sh index d3fd2cb73b2..abcf51d8c50 100755 --- a/tests/queries/0_stateless/01526_client_start_and_exit.sh +++ b/tests/queries/0_stateless/01526_client_start_and_exit.sh @@ -4,12 +4,12 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh # Create a huge amount of tables, so Suggest will take a time to load -seq 1 1000 | sed -r -e 's/(.+)/CREATE TABLE IF NOT EXISTS test\1 (x UInt8) ENGINE = Memory;/' | ${CLICKHOUSE_CLIENT} -n +${CLICKHOUSE_CLIENT} -q "SELECT 'CREATE TABLE test_' || hex(randomPrintableASCII(40)) || '(x UInt8) Engine=Memory;' FROM numbers(10000)" --format=TSVRaw | ${CLICKHOUSE_BENCHMARK} -c32 -i 10000 -d 0 2>&1 | grep -F 'Loaded 10000 queries' function stress() { while true; do - ./"$CURDIR"/01526_client_start_and_exit.expect + ./"$CURDIR"/01526_client_start_and_exit.expect | grep -v -P 'ClickHouse client|Connecting|Connected|:\) Bye\.|^\s*$|spawn bash|^0\s*$' done } @@ -20,6 +20,3 @@ for _ in {1..10}; do done wait -echo 'Ok' - -seq 1 1000 | sed -r -e 's/(.+)/DROP TABLE test\1;/' | ${CLICKHOUSE_CLIENT} -n From 98f073a3a541db7e9bed9ed4d78057bde3eef4ac Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sun, 25 Oct 2020 05:38:54 +0300 Subject: [PATCH 182/325] Text updated ant translated to Russian --- .../{crash_log.md => crash-log.md} | 16 +++++-- docs/ru/operations/system-tables/crash-log.md | 48 +++++++++++++++++++ 2 files changed, 59 insertions(+), 5 deletions(-) rename docs/en/operations/system-tables/{crash_log.md => crash-log.md} (77%) create mode 100644 docs/ru/operations/system-tables/crash-log.md diff --git a/docs/en/operations/system-tables/crash_log.md b/docs/en/operations/system-tables/crash-log.md similarity index 77% rename from docs/en/operations/system-tables/crash_log.md rename to docs/en/operations/system-tables/crash-log.md index d38ce31584f..5bdf402513a 100644 --- a/docs/en/operations/system-tables/crash_log.md +++ b/docs/en/operations/system-tables/crash-log.md @@ -1,4 +1,4 @@ -## system.crash_log {#system-tables_crash_log} +# system.crash_log {#system-tables_crash_log} Contains information about stack traces for fatal errors. The table does not exist in the database by default, it is created only when fatal errors occur. @@ -10,11 +10,11 @@ Columns: - `signal` ([Int32](../../sql-reference/data-types/int-uint.md)) — Signal number. - `thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Thread ID. - `query_id` ([String](../../sql-reference/data-types/string.md)) — Query ID. -- `trace` ([Array](../../sql-reference/data-types/array.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Array of traces. -- `trace_full` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — Array of full traces. +- `trace` ([Array](../../sql-reference/data-types/array.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Stack trace at the moment of crash. Each element is a virtual memory address inside ClickHouse server process. +- `trace_full` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — Stack trace at the moment of crash. Each element contains a called method inside ClickHouse server process. - `version` ([String](../../sql-reference/data-types/string.md)) — ClickHouse server version. - `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ClickHouse server revision. -- `build_id` ([String](../../sql-reference/data-types/string.md)) — ClickHouse server build. +- `build_id` ([String](../../sql-reference/data-types/string.md)) — BuildID that is generated by compiler. **Example** @@ -23,9 +23,12 @@ Query: ``` sql SELECT * FROM system.crash_log ORDER BY event_time DESC LIMIT 1; ``` + Result (not full): ``` text +Row 1: +────── event_date: 2020-10-14 event_time: 2020-10-14 15:47:40 timestamp_ns: 1602679660271312710 @@ -39,4 +42,7 @@ revision: 54442 build_id: ``` -[Original article](https://clickhouse.tech/docs/en/operations/system_tables/crash_log) +**See also** +- [trace_log](../../operations/system_tables/trace_log.md) system table + +[Original article](https://clickhouse.tech/docs/en/operations/system_tables/crash-log) diff --git a/docs/ru/operations/system-tables/crash-log.md b/docs/ru/operations/system-tables/crash-log.md new file mode 100644 index 00000000000..19e9d19b198 --- /dev/null +++ b/docs/ru/operations/system-tables/crash-log.md @@ -0,0 +1,48 @@ +# system.crash_log {#system-tables_crash_log} + +Содержит информацию о трассировках стека для фатальных ошибок. Таблица не содержится в базе данных по умолчанию, а создается только при возникновении фатальных ошибок. + +Колонки: + +- `event_date` ([Datetime](../../sql-reference/data-types/datetime.md)) — Дата события. +- `event_time` ([Datetime](../../sql-reference/data-types/datetime.md)) — Время события. +- `timestamp_ns` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Время события с наносекундами. +- `signal` ([Int32](../../sql-reference/data-types/int-uint.md)) — Номер сигнала, пришедшего в поток. +- `thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Идентификатор треда. +- `query_id` ([String](../../sql-reference/data-types/string.md)) — Идентификатор запроса. +- `trace` ([Array](../../sql-reference/data-types/array.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Трассировка стека в момент ошибки. Представляет собой список физических адресов, по которым расположены вызываемые методы. +- `trace_full` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — Трассировка стека в момент ошибки. Содержит вызываемые методы. +- `version` ([String](../../sql-reference/data-types/string.md)) — Версия сервера ClickHouse. +- `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Ревизия сборки сервера ClickHouse. +- `build_id` ([String](../../sql-reference/data-types/string.md)) — BuildID, сгенерированный компилятором. + +**Пример** + +Запрос: + +``` sql +SELECT * FROM system.crash_log ORDER BY event_time DESC LIMIT 1; +``` + +Результат (приведён не полностью): + +``` text +Row 1: +────── +event_date: 2020-10-14 +event_time: 2020-10-14 15:47:40 +timestamp_ns: 1602679660271312710 +signal: 11 +thread_id: 23624 +query_id: 428aab7c-8f5c-44e9-9607-d16b44467e69 +trace: [188531193,...] +trace_full: ['3. DB::(anonymous namespace)::FunctionFormatReadableTimeDelta::executeImpl(std::__1::vector >&, std::__1::vector > const&, unsigned long, unsigned long) const @ 0xb3cc1f9 in /home/username/work/ClickHouse/build/programs/clickhouse',...] +version: ClickHouse 20.11.1.1 +revision: 54442 +build_id: +``` + +**См. также** +- Системная таблица [trace_log](../../operations/system_tables/trace_log.md) + +[Original article](https://clickhouse.tech/docs/en/operations/system_tables/crash-log) From a2d888f19fb29abb7556ce799b55b11a31fbc6d8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Oct 2020 07:13:17 +0300 Subject: [PATCH 183/325] Fix processing very large entries in queue --- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 16 +++++++--------- .../MergeTree/ReplicatedMergeTreeQueue.h | 13 +++++++++++++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 45e16e81208..1d4f6b38be8 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -486,20 +486,18 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper { std::sort(log_entries.begin(), log_entries.end()); - /// ZK contains a limit on the number or total size of operations in a multi-request. - /// If the limit is exceeded, the connection is simply closed. - /// The constant is selected with a margin. The default limit in ZK is 1 MB of data in total. - /// The average size of the node value in this case is less than 10 kilobytes. - static constexpr auto MAX_MULTI_OPS = 100; - - for (size_t entry_idx = 0, num_entries = log_entries.size(); entry_idx < num_entries; entry_idx += MAX_MULTI_OPS) + for (size_t entry_idx = 0, num_entries = log_entries.size(); entry_idx < num_entries; entry_idx += current_multi_batch_size) { auto begin = log_entries.begin() + entry_idx; - auto end = entry_idx + MAX_MULTI_OPS >= log_entries.size() + auto end = entry_idx + current_multi_batch_size >= log_entries.size() ? log_entries.end() - : (begin + MAX_MULTI_OPS); + : (begin + current_multi_batch_size); auto last = end - 1; + /// Increase the batch size exponentially, so it will saturate to MAX_MULTI_OPS. + if (current_multi_batch_size < MAX_MULTI_OPS) + current_multi_batch_size = std::min(MAX_MULTI_OPS, current_multi_batch_size * 2); + String last_entry = *last; if (!startsWith(last_entry, "log-")) throw Exception("Error in zookeeper data: unexpected node " + last_entry + " in " + zookeeper_path + "/log", diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 88a61f50225..06dcddedd3e 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -259,6 +259,19 @@ private: ~CurrentlyExecuting(); }; + /// ZK contains a limit on the number or total size of operations in a multi-request. + /// If the limit is exceeded, the connection is simply closed. + /// The constant is selected with a margin. The default limit in ZK is 1 MB of data in total. + /// The average size of the node value in this case is less than 10 kilobytes. + static constexpr auto MAX_MULTI_OPS = 100; + + /// Very large queue entries may appear occasionally. + /// We cannot process MAX_MULTI_OPS at once because it will fail. + /// But we have to process more than one entry at once because otherwise lagged replicas keep up slowly. + /// Let's start with one entry per transaction and icrease it exponentially towards MAX_MULTI_OPS. + /// It will allow to make some progress before failing and remain operational even in extreme cases. + size_t current_multi_batch_size = 1; + public: ReplicatedMergeTreeQueue(StorageReplicatedMergeTree & storage_); ~ReplicatedMergeTreeQueue(); From 22a89fca010d879310dd6cbbf7b172f0a918d4e0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Oct 2020 07:21:46 +0300 Subject: [PATCH 184/325] Fix processing very large entries in queue --- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 6 +++++- src/Storages/MergeTree/ReplicatedMergeTreeQueue.h | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 1d4f6b38be8..a1f20016cb3 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -57,6 +57,7 @@ bool ReplicatedMergeTreeQueue::isVirtualPart(const MergeTreeData::DataPartPtr & return virtual_parts.getContainingPart(data_part->info) != data_part->name; } + bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) { auto queue_path = replica_path + "/queue"; @@ -68,6 +69,9 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) { std::lock_guard pull_logs_lock(pull_logs_to_queue_mutex); + /// Reset batch size on initialization to recover from possible errors of too large batch size. + current_multi_batch_size = 1; + String log_pointer_str = zookeeper->get(replica_path + "/log_pointer"); log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); @@ -496,7 +500,7 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper /// Increase the batch size exponentially, so it will saturate to MAX_MULTI_OPS. if (current_multi_batch_size < MAX_MULTI_OPS) - current_multi_batch_size = std::min(MAX_MULTI_OPS, current_multi_batch_size * 2); + current_multi_batch_size = std::min(MAX_MULTI_OPS, current_multi_batch_size * 2); String last_entry = *last; if (!startsWith(last_entry, "log-")) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 06dcddedd3e..93b79c8336c 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -263,7 +263,7 @@ private: /// If the limit is exceeded, the connection is simply closed. /// The constant is selected with a margin. The default limit in ZK is 1 MB of data in total. /// The average size of the node value in this case is less than 10 kilobytes. - static constexpr auto MAX_MULTI_OPS = 100; + static constexpr size_t MAX_MULTI_OPS = 100; /// Very large queue entries may appear occasionally. /// We cannot process MAX_MULTI_OPS at once because it will fail. From 300f07bdba532ccdd789ce6300cf4388683a36ed Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sun, 25 Oct 2020 05:58:39 +0300 Subject: [PATCH 185/325] Links fixed. --- docs/en/operations/system-tables/crash-log.md | 4 ++-- docs/ru/operations/system-tables/crash-log.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/operations/system-tables/crash-log.md b/docs/en/operations/system-tables/crash-log.md index 5bdf402513a..5b338237b83 100644 --- a/docs/en/operations/system-tables/crash-log.md +++ b/docs/en/operations/system-tables/crash-log.md @@ -43,6 +43,6 @@ build_id: ``` **See also** -- [trace_log](../../operations/system_tables/trace_log.md) system table +- [trace_log](../../operations/system-tables/trace_log.md) system table -[Original article](https://clickhouse.tech/docs/en/operations/system_tables/crash-log) +[Original article](https://clickhouse.tech/docs/en/operations/system-tables/crash-log) diff --git a/docs/ru/operations/system-tables/crash-log.md b/docs/ru/operations/system-tables/crash-log.md index 19e9d19b198..7b645a06b2d 100644 --- a/docs/ru/operations/system-tables/crash-log.md +++ b/docs/ru/operations/system-tables/crash-log.md @@ -43,6 +43,6 @@ build_id: ``` **См. также** -- Системная таблица [trace_log](../../operations/system_tables/trace_log.md) +- Системная таблица [trace_log](../../operations/system-tables/trace_log.md) -[Original article](https://clickhouse.tech/docs/en/operations/system_tables/crash-log) +[Original article](https://clickhouse.tech/docs/en/operations/system-tables/crash-log) From 6762f967687d8f0e504f0c8f42d4fe39beefad9f Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 25 Oct 2020 08:25:21 +0300 Subject: [PATCH 186/325] Update 01526_client_start_and_exit.sh --- tests/queries/0_stateless/01526_client_start_and_exit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.sh b/tests/queries/0_stateless/01526_client_start_and_exit.sh index abcf51d8c50..0b3a2ee6b4f 100755 --- a/tests/queries/0_stateless/01526_client_start_and_exit.sh +++ b/tests/queries/0_stateless/01526_client_start_and_exit.sh @@ -9,7 +9,7 @@ ${CLICKHOUSE_CLIENT} -q "SELECT 'CREATE TABLE test_' || hex(randomPrintableASCII function stress() { while true; do - ./"$CURDIR"/01526_client_start_and_exit.expect | grep -v -P 'ClickHouse client|Connecting|Connected|:\) Bye\.|^\s*$|spawn bash|^0\s*$' + "$CURDIR"/01526_client_start_and_exit.expect | grep -v -P 'ClickHouse client|Connecting|Connected|:\) Bye\.|^\s*$|spawn bash|^0\s*$' done } From 440ae2bc57f29d9a5af3218dacdbc93dc6d8ffc0 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 25 Oct 2020 11:45:29 +0300 Subject: [PATCH 187/325] UUID safe cast functions added 1. Added readUUIDTextImpl, readUUIDText, tryReadUUIDText functions in ReadHelpers. 2. Added toUUIDOrNull, toUUIDOrZero functions based on ReadHelpers read implementations. 3. Updated documentation. --- .../sql-reference/functions/uuid-functions.md | 48 +++++++++++++++++++ src/Functions/FunctionsConversion.cpp | 4 ++ src/Functions/FunctionsConversion.h | 17 ++++++- src/IO/ReadHelpers.h | 38 +++++++++++++-- .../01528_to_uuid_or_null_or_zero.reference | 8 ++++ .../01528_to_uuid_or_null_or_zero.sql | 19 ++++++++ 6 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/01528_to_uuid_or_null_or_zero.reference create mode 100644 tests/queries/0_stateless/01528_to_uuid_or_null_or_zero.sql diff --git a/docs/en/sql-reference/functions/uuid-functions.md b/docs/en/sql-reference/functions/uuid-functions.md index f608c643ee8..b747ac07bb8 100644 --- a/docs/en/sql-reference/functions/uuid-functions.md +++ b/docs/en/sql-reference/functions/uuid-functions.md @@ -61,6 +61,54 @@ SELECT toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0') AS uuid └──────────────────────────────────────┘ ``` +## toUUIDOrNull (x) {#touuidornull-x} + +It takes an argument of type String and tries to parse it into UUID. If failed, returns NULL. + +``` sql +toUUIDOrNull(String) +``` + +**Returned value** + +The Nullable UUID type value. + +**Usage example** + +``` sql +SELECT toUUIDOrNull('61f0c404-5cb3-11e7-907b-a6006ad3dba0T') AS uuid +``` + +``` text +┌─uuid─┐ +│ ᴺᵁᴸᴸ │ +└──────┘ +``` + +## toUUIDOrZero (x) {#touuidorzero-x} + +It takes an argument of type String and tries to parse it into UUID. If failed, returns zero UUID. + +``` sql +toUUIDOrZero(String) +``` + +**Returned value** + +The UUID type value. + +**Usage example** + +``` sql +SELECT toUUIDOrZero('61f0c404-5cb3-11e7-907b-a6006ad3dba0T') AS uuid +``` + +``` text +┌─────────────────────────────────uuid─┐ +│ 00000000-0000-0000-0000-000000000000 │ +└──────────────────────────────────────┘ +``` + ## UUIDStringToNum {#uuidstringtonum} Accepts a string containing 36 characters in the format `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`, and returns it as a set of bytes in a [FixedString(16)](../../sql-reference/data-types/fixedstring.md). diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 3f38614f584..df962800385 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -68,6 +68,8 @@ void registerFunctionsConversion(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); @@ -90,6 +92,8 @@ void registerFunctionsConversion(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 1cbf8cc3925..6e8847b7f09 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -568,7 +568,7 @@ template <> inline void parseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *) { UUID tmp; - readText(tmp, rb); + readUUIDText(tmp, rb); x = tmp; } @@ -602,6 +602,17 @@ inline bool tryParseImpl(DataTypeDateTime::FieldType & x, Read return true; } +template <> +inline bool tryParseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *) +{ + UUID tmp; + if (!tryReadUUIDText(tmp, rb)) + return false; + + x = tmp; + return true; +} + /** Throw exception with verbose message when string value is not parsed completely. */ @@ -1754,6 +1765,7 @@ struct NameToDecimal32OrZero { static constexpr auto name = "toDecimal32OrZero"; struct NameToDecimal64OrZero { static constexpr auto name = "toDecimal64OrZero"; }; struct NameToDecimal128OrZero { static constexpr auto name = "toDecimal128OrZero"; }; struct NameToDecimal256OrZero { static constexpr auto name = "toDecimal256OrZero"; }; +struct NameToUUIDOrZero { static constexpr auto name = "toUUIDOrZero"; }; using FunctionToUInt8OrZero = FunctionConvertFromString; using FunctionToUInt16OrZero = FunctionConvertFromString; @@ -1775,6 +1787,7 @@ using FunctionToDecimal32OrZero = FunctionConvertFromString, NameToDecimal64OrZero, ConvertFromStringExceptionMode::Zero>; using FunctionToDecimal128OrZero = FunctionConvertFromString, NameToDecimal128OrZero, ConvertFromStringExceptionMode::Zero>; using FunctionToDecimal256OrZero = FunctionConvertFromString, NameToDecimal256OrZero, ConvertFromStringExceptionMode::Zero>; +using FunctionToUUIDOrZero = FunctionConvertFromString; struct NameToUInt8OrNull { static constexpr auto name = "toUInt8OrNull"; }; struct NameToUInt16OrNull { static constexpr auto name = "toUInt16OrNull"; }; @@ -1796,6 +1809,7 @@ struct NameToDecimal32OrNull { static constexpr auto name = "toDecimal32OrNull"; struct NameToDecimal64OrNull { static constexpr auto name = "toDecimal64OrNull"; }; struct NameToDecimal128OrNull { static constexpr auto name = "toDecimal128OrNull"; }; struct NameToDecimal256OrNull { static constexpr auto name = "toDecimal256OrNull"; }; +struct NameToUUIDOrNull { static constexpr auto name = "toUUIDOrNull"; }; using FunctionToUInt8OrNull = FunctionConvertFromString; using FunctionToUInt16OrNull = FunctionConvertFromString; @@ -1817,6 +1831,7 @@ using FunctionToDecimal32OrNull = FunctionConvertFromString, NameToDecimal64OrNull, ConvertFromStringExceptionMode::Null>; using FunctionToDecimal128OrNull = FunctionConvertFromString, NameToDecimal128OrNull, ConvertFromStringExceptionMode::Null>; using FunctionToDecimal256OrNull = FunctionConvertFromString, NameToDecimal256OrNull, ConvertFromStringExceptionMode::Null>; +using FunctionToUUIDOrNull = FunctionConvertFromString; struct NameParseDateTimeBestEffort { static constexpr auto name = "parseDateTimeBestEffort"; }; struct NameParseDateTimeBestEffortUS { static constexpr auto name = "parseDateTimeBestEffortUS"; }; diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 90a56af3c34..d79328889f1 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -619,9 +619,11 @@ inline bool tryReadDateText(DayNum & date, ReadBuffer & buf) return readDateTextImpl(date, buf); } - -inline void readUUIDText(UUID & uuid, ReadBuffer & buf) +template +inline ReturnType readUUIDTextImpl(UUID & uuid, ReadBuffer & buf) { + static constexpr bool throw_exception = std::is_same_v; + char s[36]; size_t size = buf.read(s, 32); @@ -634,21 +636,49 @@ inline void readUUIDText(UUID & uuid, ReadBuffer & buf) if (size != 36) { s[size] = 0; - throw Exception(std::string("Cannot parse uuid ") + s, ErrorCodes::CANNOT_PARSE_UUID); + + if constexpr (throw_exception) + { + throw Exception(std::string("Cannot parse uuid ") + s, ErrorCodes::CANNOT_PARSE_UUID); + } + else + { + return ReturnType(false); + } } parseUUID(reinterpret_cast(s), std::reverse_iterator(reinterpret_cast(&uuid) + 16)); } else parseUUIDWithoutSeparator(reinterpret_cast(s), std::reverse_iterator(reinterpret_cast(&uuid) + 16)); + + return ReturnType(true); } else { s[size] = 0; - throw Exception(std::string("Cannot parse uuid ") + s, ErrorCodes::CANNOT_PARSE_UUID); + + if constexpr (throw_exception) + { + throw Exception(std::string("Cannot parse uuid ") + s, ErrorCodes::CANNOT_PARSE_UUID); + } + else + { + return ReturnType(false); + } } } +inline void readUUIDText(UUID & uuid, ReadBuffer & buf) +{ + return readUUIDTextImpl(uuid, buf); +} + +inline bool tryReadUUIDText(UUID & uuid, ReadBuffer & buf) +{ + return readUUIDTextImpl(uuid, buf); +} + template inline T parse(const char * data, size_t size); diff --git a/tests/queries/0_stateless/01528_to_uuid_or_null_or_zero.reference b/tests/queries/0_stateless/01528_to_uuid_or_null_or_zero.reference new file mode 100644 index 00000000000..041e329748e --- /dev/null +++ b/tests/queries/0_stateless/01528_to_uuid_or_null_or_zero.reference @@ -0,0 +1,8 @@ +61f0c404-5cb3-11e7-907b-a6006ad3dba0 +\N +00000000-0000-0000-0000-000000000000 +61f0c404-5cb3-11e7-907b-a6006ad3dba0 +61f0c404-5cb3-11e7-907b-a6006ad3dba0 +\N +61f0c404-5cb3-11e7-907b-a6006ad3dba0 +00000000-0000-0000-0000-000000000000 diff --git a/tests/queries/0_stateless/01528_to_uuid_or_null_or_zero.sql b/tests/queries/0_stateless/01528_to_uuid_or_null_or_zero.sql new file mode 100644 index 00000000000..ae6a1b2db04 --- /dev/null +++ b/tests/queries/0_stateless/01528_to_uuid_or_null_or_zero.sql @@ -0,0 +1,19 @@ +DROP TABLE IF EXISTS to_uuid_test; + +SELECT toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0'); +SELECT toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0T'); --{serverError 6} +SELECT toUUIDOrNull('61f0c404-5cb3-11e7-907b-a6006ad3dba0T'); +SELECT toUUIDOrZero('59f0c404-5cb3-11e7-907b-a6006ad3dba0T'); + +CREATE TABLE to_uuid_test (value String) ENGINE = TinyLog(); + +INSERT INTO to_uuid_test VALUES ('61f0c404-5cb3-11e7-907b-a6006ad3dba0'); +SELECT toUUID(value) FROM to_uuid_test; + +INSERT INTO to_uuid_test VALUES ('61f0c404-5cb3-11e7-907b-a6006ad3dba0T'); +SELECT toUUID(value) FROM to_uuid_test; -- {serverError 6} +SELECT toUUIDOrNull(value) FROM to_uuid_test; +SELECT toUUIDOrZero(value) FROM to_uuid_test; + +DROP TABLE to_uuid_test; + From edc8d6e5e76560eca7b59feb62eb1c06c4167d9d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 25 Oct 2020 14:14:52 +0300 Subject: [PATCH 188/325] Fix async Distributed INSERT w/ prefer_localhost_replica=0 and internal_replication --- programs/server/config.xml | 16 +++++++++ src/Interpreters/Cluster.cpp | 11 ++++-- ..._directory_monitor_batch_inserts.reference | 16 +++++++++ ...ibuted_directory_monitor_batch_inserts.sql | 36 ++++++++++++++++++- 4 files changed, 75 insertions(+), 4 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index 9850d77abb7..5bdec5377fd 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -392,6 +392,22 @@ + + + true + + 127.0.0.1 + 9000 + + + + true + + 127.0.0.2 + 9000 + + + diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index 8a98e8282a6..9c2766ae7d6 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -614,13 +614,18 @@ const std::string & Cluster::ShardInfo::pathForInsert(bool prefer_localhost_repl if (!has_internal_replication) throw Exception("internal_replication is not set", ErrorCodes::LOGICAL_ERROR); - if (dir_name_for_internal_replication.empty() || dir_name_for_internal_replication_with_local.empty()) - throw Exception("Directory name for async inserts is empty", ErrorCodes::LOGICAL_ERROR); - if (prefer_localhost_replica) + { + if (dir_name_for_internal_replication.empty()) + throw Exception("Directory name for async inserts is empty", ErrorCodes::LOGICAL_ERROR); return dir_name_for_internal_replication; + } else + { + if (dir_name_for_internal_replication_with_local.empty()) + throw Exception("Directory name for async inserts is empty", ErrorCodes::LOGICAL_ERROR); return dir_name_for_internal_replication_with_local; + } } bool Cluster::maybeCrossReplication() const diff --git a/tests/queries/0_stateless/01040_distributed_directory_monitor_batch_inserts.reference b/tests/queries/0_stateless/01040_distributed_directory_monitor_batch_inserts.reference index 5565ed6787f..03e58c13ff2 100644 --- a/tests/queries/0_stateless/01040_distributed_directory_monitor_batch_inserts.reference +++ b/tests/queries/0_stateless/01040_distributed_directory_monitor_batch_inserts.reference @@ -1,4 +1,20 @@ +test_cluster_two_shards prefer_localhost_replica=0 +0 0 1 +1 +test_cluster_two_shards prefer_localhost_replica=1 +0 0 1 +1 +test_cluster_two_shards_internal_replication prefer_localhost_replica=0 +0 +0 +1 +1 +test_cluster_two_shards_internal_replication prefer_localhost_replica=1 +0 +0 +1 +1 diff --git a/tests/queries/0_stateless/01040_distributed_directory_monitor_batch_inserts.sql b/tests/queries/0_stateless/01040_distributed_directory_monitor_batch_inserts.sql index dbec319ab76..dec748789c8 100644 --- a/tests/queries/0_stateless/01040_distributed_directory_monitor_batch_inserts.sql +++ b/tests/queries/0_stateless/01040_distributed_directory_monitor_batch_inserts.sql @@ -7,6 +7,40 @@ DROP TABLE IF EXISTS dist_test_01040; CREATE TABLE test_01040 (key UInt64) ENGINE=TinyLog(); CREATE TABLE dist_test_01040 AS test_01040 Engine=Distributed(test_cluster_two_shards, currentDatabase(), test_01040, key); + +-- internal_replication=false +SELECT 'test_cluster_two_shards prefer_localhost_replica=0'; +SET prefer_localhost_replica=0; INSERT INTO dist_test_01040 SELECT toUInt64(number) FROM numbers(2); SYSTEM FLUSH DISTRIBUTED dist_test_01040; -SELECT * FROM dist_test_01040; +SELECT * FROM dist_test_01040 ORDER BY key; +TRUNCATE TABLE test_01040; + +SELECT 'test_cluster_two_shards prefer_localhost_replica=1'; +SET prefer_localhost_replica=1; +INSERT INTO dist_test_01040 SELECT toUInt64(number) FROM numbers(2); +SYSTEM FLUSH DISTRIBUTED dist_test_01040; +SELECT * FROM dist_test_01040 ORDER BY key; +TRUNCATE TABLE test_01040; + +DROP TABLE dist_test_01040; + +-- internal_replication=true +CREATE TABLE dist_test_01040 AS test_01040 Engine=Distributed(test_cluster_two_shards_internal_replication, currentDatabase(), test_01040, key); +SELECT 'test_cluster_two_shards_internal_replication prefer_localhost_replica=0'; +SET prefer_localhost_replica=0; +INSERT INTO dist_test_01040 SELECT toUInt64(number) FROM numbers(2); +SYSTEM FLUSH DISTRIBUTED dist_test_01040; +SELECT * FROM dist_test_01040 ORDER BY key; +TRUNCATE TABLE test_01040; + +SELECT 'test_cluster_two_shards_internal_replication prefer_localhost_replica=1'; +SET prefer_localhost_replica=1; +INSERT INTO dist_test_01040 SELECT toUInt64(number) FROM numbers(2); +SYSTEM FLUSH DISTRIBUTED dist_test_01040; +SELECT * FROM dist_test_01040 ORDER BY key; +TRUNCATE TABLE test_01040; + + +DROP TABLE dist_test_01040; +DROP TABLE test_01040; From f97e6beb70984d65aa705dfe3b4bba467d824d2f Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 25 Oct 2020 15:34:18 +0300 Subject: [PATCH 189/325] Remove message broker pool from context --- src/Interpreters/Context.cpp | 13 ------------- src/Interpreters/Context.h | 1 - 2 files changed, 14 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 7f2ada8a426..9d2ea6ded86 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -335,7 +335,6 @@ struct ContextShared std::optional background_move_pool; /// The thread pool for the background moves performed by the tables. std::optional schedule_pool; /// A thread pool that can run different jobs in background (used in replicated tables) std::optional distributed_schedule_pool; /// A thread pool that can run different jobs in background (used for distributed sends) - std::optional message_broker_schedule_pool; /// A thread pool that can run different jobs in background (used in kafka streaming) MultiVersion macros; /// Substitutions extracted from config. std::unique_ptr ddl_worker; /// Process ddl commands from zk. /// Rules for selecting the compression settings, depending on the size of the part. @@ -438,7 +437,6 @@ struct ContextShared schedule_pool.reset(); distributed_schedule_pool.reset(); ddl_worker.reset(); - message_broker_schedule_pool.reset(); /// Stop trace collector if any trace_collector.reset(); @@ -1441,17 +1439,6 @@ BackgroundSchedulePool & Context::getDistributedSchedulePool() return *shared->distributed_schedule_pool; } -BackgroundSchedulePool & Context::getMessageBrokerSchedulePool() -{ - auto lock = getLock(); - if (!shared->message_broker_schedule_pool) - shared->message_broker_schedule_pool.emplace( - settings.background_message_broker_schedule_pool_size, - CurrentMetrics::BackgroundMessageBrokerSchedulePoolTask, - "BgMBSchPool"); - return *shared->message_broker_schedule_pool; -} - void Context::setDDLWorker(std::unique_ptr ddl_worker) { auto lock = getLock(); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index bd5e17fe2e4..075fc3837ef 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -511,7 +511,6 @@ public: BackgroundProcessingPool & getBackgroundPool(); BackgroundProcessingPool & getBackgroundMovePool(); BackgroundSchedulePool & getSchedulePool(); - BackgroundSchedulePool & getMessageBrokerSchedulePool(); BackgroundSchedulePool & getDistributedSchedulePool(); void setDDLWorker(std::unique_ptr ddl_worker); From 4bcbcfed1f243a1759b10b3df58409e23909f82b Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Sun, 25 Oct 2020 09:40:19 -0400 Subject: [PATCH 190/325] Fixing procedure of getting log file size in LDAP tests. Increasing default timeouts. --- .../ldap/authentication/tests/common.py | 16 ++++++++-------- .../ldap/external_user_directory/tests/common.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/testflows/ldap/authentication/tests/common.py b/tests/testflows/ldap/authentication/tests/common.py index 4e3d1e16647..1c662b3898b 100644 --- a/tests/testflows/ldap/authentication/tests/common.py +++ b/tests/testflows/ldap/authentication/tests/common.py @@ -47,7 +47,7 @@ ASCII_CHARS = string.ascii_lowercase + string.ascii_uppercase + string.digits def randomword(length, chars=ASCII_CHARS): return ''.join(random.choice(chars) for i in range(length)) -def restart(node=None, safe=False, timeout=20): +def restart(node=None, safe=False, timeout=60): """Restart ClickHouse server and wait for config to be reloaded. """ with When("I restart ClickHouse server node"): @@ -62,7 +62,7 @@ def restart(node=None, safe=False, timeout=20): with And("getting current log size"): logsize = \ - node.command("ls -s --block-size=1 /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[ + node.command("stat --format=%s /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[ 0].strip() with And("restarting ClickHouse server"): @@ -78,7 +78,7 @@ def restart(node=None, safe=False, timeout=20): f"ConfigReloader: Loaded config '/etc/clickhouse-server/config.xml', performed update on configuration", timeout=timeout) -def add_config(config, timeout=20, restart=False): +def add_config(config, timeout=60, restart=False): """Add dynamic configuration file to ClickHouse. :param node: node @@ -108,7 +108,7 @@ def add_config(config, timeout=20, restart=False): with And("I get the current log size"): logsize = \ - node.command("ls -s --block-size=1 /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[ + node.command("stat --format=%s /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[ 0].strip() with And("I restart ClickHouse to apply the config changes"): @@ -189,7 +189,7 @@ def create_ldap_servers_config_content(servers, config_d_dir="/etc/clickhouse-se @contextmanager def ldap_servers(servers, config_d_dir="/etc/clickhouse-server/config.d", config_file="ldap_servers.xml", - timeout=20, restart=False): + timeout=60, restart=False): """Add LDAP servers configuration. """ config = create_ldap_servers_config_content(servers, config_d_dir, config_file) @@ -236,7 +236,7 @@ def add_users_identified_with_ldap(*users): @contextmanager def ldap_authenticated_users(*users, config_d_dir="/etc/clickhouse-server/users.d", - config_file=None, timeout=20, restart=True, config=None, rbac=False): + config_file=None, timeout=60, restart=True, config=None, rbac=False): """Add LDAP authenticated users. """ if rbac: @@ -248,7 +248,7 @@ def ldap_authenticated_users(*users, config_d_dir="/etc/clickhouse-server/users. config = create_ldap_users_config_content(*users, config_d_dir=config_d_dir, config_file=config_file) return add_config(config, restart=restart) -def invalid_server_config(servers, message=None, tail=13, timeout=20): +def invalid_server_config(servers, message=None, tail=13, timeout=60): """Check that ClickHouse errors when trying to load invalid LDAP servers configuration file. """ node = current().context.node @@ -277,7 +277,7 @@ def invalid_server_config(servers, message=None, tail=13, timeout=20): with By("removing the config file", description=config.path): node.command(f"rm -rf {config.path}", exitcode=0) -def invalid_user_config(servers, config, message=None, tail=13, timeout=20): +def invalid_user_config(servers, config, message=None, tail=13, timeout=60): """Check that ClickHouse errors when trying to load invalid LDAP users configuration file. """ node = current().context.node diff --git a/tests/testflows/ldap/external_user_directory/tests/common.py b/tests/testflows/ldap/external_user_directory/tests/common.py index d6f414e617a..38b53ca6e9f 100644 --- a/tests/testflows/ldap/external_user_directory/tests/common.py +++ b/tests/testflows/ldap/external_user_directory/tests/common.py @@ -129,7 +129,7 @@ def create_entries_ldap_external_user_directory_config_content(entries, config_d return Config(content, path, name, uid, "config.xml") -def invalid_ldap_external_user_directory_config(server, roles, message, tail=20, timeout=20, config=None): +def invalid_ldap_external_user_directory_config(server, roles, message, tail=20, timeout=60, config=None): """Check that ClickHouse errors when trying to load invalid LDAP external user directory configuration file. """ @@ -181,7 +181,7 @@ def invalid_ldap_external_user_directory_config(server, roles, message, tail=20, @contextmanager def ldap_external_user_directory(server, roles, config_d_dir="/etc/clickhouse-server/config.d", - config_file=None, timeout=20, restart=True, config=None): + config_file=None, timeout=60, restart=True, config=None): """Add LDAP external user directory. """ if config_file is None: From f8f2d62d704c92ef59bb6428b07aa1f2e257902d Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sun, 25 Oct 2020 20:27:21 +0300 Subject: [PATCH 191/325] minor improvements in DatabaseCatalog --- src/Common/CurrentMetrics.cpp | 1 + src/Databases/DatabaseAtomic.cpp | 1 + src/Databases/DatabaseMemory.cpp | 3 + src/Databases/DatabaseWithDictionaries.cpp | 4 ++ src/Interpreters/DatabaseCatalog.cpp | 72 +++++++++++++++++++--- src/Interpreters/DatabaseCatalog.h | 9 ++- 6 files changed, 79 insertions(+), 11 deletions(-) diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index ef0b82666dd..3d6a2d6f99c 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -54,6 +54,7 @@ M(LocalThread, "Number of threads in local thread pools. Should be similar to GlobalThreadActive.") \ M(LocalThreadActive, "Number of threads in local thread pools running a task.") \ M(DistributedFilesToInsert, "Number of pending files to process for asynchronous insertion into Distributed tables. Number of files for every shard is summed.") \ + M(TablesToDropQueueSize, "Number of dropped tables, that are waiting for background data removal.") \ namespace CurrentMetrics { diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index a9dbae8ec92..6c1ca1e8fce 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -269,6 +269,7 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora database_name, query.database); not_in_use = cleanupDetachedTables(); assertDetachedTableNotInUse(query.uuid); + DatabaseCatalog::instance().addUUIDMapping(query.uuid, {}, {}); renameNoReplace(table_metadata_tmp_path, table_metadata_path); attachTableUnlocked(query.table, table, lock); /// Should never throw table_name_to_path.emplace(query.table, table_data_path); diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 5eacb846d52..357acb32371 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -53,6 +53,9 @@ void DatabaseMemory::dropTable( } table->is_dropped = true; create_queries.erase(table_name); + UUID table_uuid = table->getStorageID().uuid; + if (table_uuid != UUIDHelpers::Nil) + DatabaseCatalog::instance().removeUUIDMappingFinally(table_uuid); } ASTPtr DatabaseMemory::getCreateDatabaseQuery() const diff --git a/src/Databases/DatabaseWithDictionaries.cpp b/src/Databases/DatabaseWithDictionaries.cpp index ed85028d04d..6c5173c986f 100644 --- a/src/Databases/DatabaseWithDictionaries.cpp +++ b/src/Databases/DatabaseWithDictionaries.cpp @@ -223,6 +223,10 @@ void DatabaseWithDictionaries::removeDictionary(const Context &, const String & attachDictionary(dictionary_name, attach_info); throw; } + + UUID dict_uuid = attach_info.create_query->as()->uuid; + if (dict_uuid != UUIDHelpers::Nil) + DatabaseCatalog::instance().removeUUIDMappingFinally(dict_uuid); } DatabaseDictionariesIteratorPtr DatabaseWithDictionaries::getDictionariesIterator(const FilterByNameFunction & filter_by_dictionary_name) diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index cfe1046aceb..e130646be11 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -13,9 +13,16 @@ #include #include #include +#include #include + +namespace CurrentMetrics +{ + extern const Metric TablesToDropQueueSize; +} + namespace DB { @@ -411,36 +418,71 @@ DatabasePtr DatabaseCatalog::getSystemDatabase() const return getDatabase(SYSTEM_DATABASE); } -void DatabaseCatalog::addUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table) +void DatabaseCatalog::addUUIDMapping(const UUID & uuid, const DatabasePtr & database, const StoragePtr & table) { assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); + assert((database && table) || (!database && !table)); UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; std::lock_guard lock{map_part.mutex}; - auto [_, inserted] = map_part.map.try_emplace(uuid, std::move(database), std::move(table)); + auto [it, inserted] = map_part.map.try_emplace(uuid, database, table); + if (inserted) + return; + + auto & prev_database = it->second.first; + auto & prev_table = it->second.second; + assert((prev_database && prev_table) || (!prev_database && !prev_table)); + + if (!prev_table && table) + { + /// It's empty mapping, it was created to "lock" UUID and prevent collision. Just update it. + prev_database = database; + prev_table = table; + return; + } + + /// We are trying to replace existing mapping (prev_table != nullptr), it's logical error + if (table) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} already exists", toString(uuid)); /// Normally this should never happen, but it's possible when the same UUIDs are explicitly specified in different CREATE queries, /// so it's not LOGICAL_ERROR - if (!inserted) - throw Exception("Mapping for table with UUID=" + toString(uuid) + " already exists", ErrorCodes::TABLE_ALREADY_EXISTS); + throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Mapping for table with UUID={} already exists. It happened due to UUID collision, " + "most likely because some not random UUIDs were manually specified in CREATE queries.", toString(uuid)); } void DatabaseCatalog::removeUUIDMapping(const UUID & uuid) +{ + assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); + UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; + std::lock_guard lock{map_part.mutex}; + auto it = map_part.map.find(uuid); + if (it == map_part.map.end()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} doesn't exist", toString(uuid)); + it->second = {}; +} + +void DatabaseCatalog::removeUUIDMappingFinally(const UUID & uuid) { assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; std::lock_guard lock{map_part.mutex}; if (!map_part.map.erase(uuid)) - throw Exception("Mapping for table with UUID=" + toString(uuid) + " doesn't exist", ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} doesn't exist", toString(uuid)); } void DatabaseCatalog::updateUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table) { assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); + assert(database && table); UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; std::lock_guard lock{map_part.mutex}; auto it = map_part.map.find(uuid); if (it == map_part.map.end()) - throw Exception("Mapping for table with UUID=" + toString(uuid) + " doesn't exist", ErrorCodes::LOGICAL_ERROR); - it->second = std::make_pair(std::move(database), std::move(table)); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} doesn't exist", toString(uuid)); + auto & prev_database = it->second.first; + auto & prev_table = it->second.second; + assert(prev_database && prev_table); + prev_database = std::move(database); + prev_table = std::move(table); } std::unique_ptr DatabaseCatalog::database_catalog; @@ -631,6 +673,8 @@ void DatabaseCatalog::loadMarkedAsDroppedTables() dropped_metadata.emplace(std::move(full_path), std::move(dropped_id)); } + LOG_INFO(log, "Found {} partially dropped tables. Will load them and retry removal.", dropped_metadata.size()); + ThreadPool pool; for (const auto & elem : dropped_metadata) { @@ -695,6 +739,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr LOG_WARNING(log, "Cannot parse metadata of partially dropped table {} from {}. Will remove metadata file and data directory. Garbage may be left in /store directory and ZooKeeper.", table_id.getNameForLogs(), dropped_metadata_path); } + addUUIDMapping(table_id.uuid, {}, {}); drop_time = Poco::File(dropped_metadata_path).getLastModified().epochTime(); } @@ -704,6 +749,8 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr else tables_marked_dropped.push_back({table_id, table, dropped_metadata_path, drop_time}); tables_marked_dropped_ids.insert(table_id.uuid); + CurrentMetrics::add(CurrentMetrics::TablesToDropQueueSize, 1); + /// If list of dropped tables was empty, start a drop task if (drop_task && tables_marked_dropped.size() == 1) (*drop_task)->schedule(); @@ -732,6 +779,10 @@ void DatabaseCatalog::dropTableDataTask() LOG_INFO(log, "Will try drop {}", table.table_id.getNameForLogs()); tables_marked_dropped.erase(it); } + else + { + LOG_TRACE(log, "No tables to drop. Queue size: {}", tables_marked_dropped.size()); + } need_reschedule = !tables_marked_dropped.empty(); } catch (...) @@ -770,7 +821,7 @@ void DatabaseCatalog::dropTableDataTask() (*drop_task)->scheduleAfter(reschedule_time_ms); } -void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) const +void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) { if (table.table) { @@ -789,6 +840,9 @@ void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) const LOG_INFO(log, "Removing metadata {} of dropped table {}", table.metadata_path, table.table_id.getNameForLogs()); Poco::File(table.metadata_path).remove(); + + removeUUIDMappingFinally(table.table_id.uuid); + CurrentMetrics::sub(CurrentMetrics::TablesToDropQueueSize, 1); } String DatabaseCatalog::getPathForUUID(const UUID & uuid) @@ -826,6 +880,8 @@ void DatabaseCatalog::waitTableFinallyDropped(const UUID & uuid) { if (uuid == UUIDHelpers::Nil) return; + + LOG_DEBUG(log, "Waiting for table {} to be finally dropped", toString(uuid)); std::unique_lock lock{tables_marked_dropped_mutex}; wait_table_finally_dropped.wait(lock, [&]() { diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index c6f50117564..46646f2ff05 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -165,10 +165,13 @@ public: void updateDependency(const StorageID & old_from, const StorageID & old_where,const StorageID & new_from, const StorageID & new_where); /// If table has UUID, addUUIDMapping(...) must be called when table attached to some database - /// and removeUUIDMapping(...) must be called when it detached. + /// removeUUIDMapping(...) must be called when it detached, + /// and removeUUIDMappingFinally(...) must be called when table is dropped and its data removed from disk. + /// To "lock" some UUID and prevent collision, addUUIDMapping(...) may be called with nullptr arguments. /// Such tables can be accessed by persistent UUID instead of database and table name. - void addUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table); + void addUUIDMapping(const UUID & uuid, const DatabasePtr & database, const StoragePtr & table); void removeUUIDMapping(const UUID & uuid); + void removeUUIDMappingFinally(const UUID & uuid); /// For moving table between databases void updateUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table); @@ -222,7 +225,7 @@ private: void loadMarkedAsDroppedTables(); void dropTableDataTask(); - void dropTableFinally(const TableMarkedAsDropped & table) const; + void dropTableFinally(const TableMarkedAsDropped & table); static constexpr size_t reschedule_time_ms = 100; From 934f64a2fdd8578097eb175f3cebd2ba771764aa Mon Sep 17 00:00:00 2001 From: tavplubix Date: Mon, 26 Oct 2020 00:51:30 +0300 Subject: [PATCH 192/325] Update DatabaseCatalog.cpp --- src/Interpreters/DatabaseCatalog.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index e130646be11..90e1a32827f 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -162,7 +162,12 @@ void DatabaseCatalog::shutdownImpl() tables_marked_dropped.clear(); std::lock_guard lock(databases_mutex); - assert(std::find_if_not(uuid_map.begin(), uuid_map.end(), [](const auto & elem) { return elem.map.empty(); }) == uuid_map.end()); + assert(std::find_if(uuid_map.begin(), uuid_map.end(), [](const auto & elem) + { + const auto & not_empty_mapping = [] (const auto & mapping) { return mapping.second.second; }; + auto it = std::find_if(elem.map.begin(), elem.map.end(), not_empty_mapping); + return it != elem.map.end(); + }) == uuid_map.end()); databases.clear(); db_uuid_map.clear(); view_dependencies.clear(); From be95d3d854dc9bc96995944882482b090d34b44c Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Sun, 25 Oct 2020 21:09:14 -0400 Subject: [PATCH 193/325] Fixing another issue in LDAP tests. --- .../ldap/authentication/tests/common.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/testflows/ldap/authentication/tests/common.py b/tests/testflows/ldap/authentication/tests/common.py index 1c662b3898b..0fd9670fae0 100644 --- a/tests/testflows/ldap/authentication/tests/common.py +++ b/tests/testflows/ldap/authentication/tests/common.py @@ -85,19 +85,29 @@ def add_config(config, timeout=60, restart=False): :param config: configuration file description :param timeout: timeout, default: 20 sec """ - def check_preprocessed_config_is_updated(): + def check_preprocessed_config_is_updated(after_removal=False): """Check that preprocessed config is updated. """ started = time.time() command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" + while time.time() - started < timeout: exitcode = node.command(command, steps=False).exitcode - if exitcode == 0: - break + if after_removal: + if exitcode == 1: + break + else: + if exitcode == 0: + break time.sleep(1) + if settings.debug: node.command(f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name}") - assert exitcode == 0, error() + + if after_removal: + assert exitcode == 1, error() + else: + assert exitcode == 0, error() def wait_for_config_to_be_loaded(): """Wait for config to be loaded. @@ -160,7 +170,7 @@ def add_config(config, timeout=60, restart=False): node.command(f"rm -rf {config.path}", exitcode=0) with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): - check_preprocessed_config_is_updated() + check_preprocessed_config_is_updated(after_removal=True) with And("I wait for config to be reloaded"): wait_for_config_to_be_loaded() From dd84fb572fa5267472749b2c2f507c907ca6c3d0 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Mon, 26 Oct 2020 04:33:28 +0300 Subject: [PATCH 194/325] More diagnostics. --- tests/integration/test_multiple_disks/test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index b0159d16501..07478d99657 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -449,8 +449,11 @@ def test_jbod_overflow(start_cluster, name, engine): data.append(get_random_string(1024 * 1024)) # 1MB row node1.query("INSERT INTO {} VALUES {}".format(name, ','.join(["('" + x + "')" for x in data]))) + for p in ("/jbod1", "/jbod2", "/external"): + print(node1.exec_in_container([f"bash", "-c", f"find {p} | xargs -n1 du -sh"])) + used_disks = get_used_disks_for_table(node1, name) - assert all(disk == 'jbod1' for disk in used_disks) + assert set(used_disks) == {'jbod1'} # should go to the external disk (jbod is overflown) data = [] # 10MB in total From d5e277f08026d88c2292faa3d3ce4d2c25373c02 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 26 Oct 2020 06:35:57 +0300 Subject: [PATCH 195/325] Better exceptions rethrow --- src/Databases/MySQL/MaterializeMySQLSettings.cpp | 6 ++---- src/Interpreters/ActionsVisitor.cpp | 4 ++-- src/Storages/JoinSettings.cpp | 6 ++---- src/Storages/Kafka/KafkaSettings.cpp | 6 ++---- src/Storages/MergeTree/MergeTreeSettings.cpp | 12 ++++-------- src/Storages/RabbitMQ/RabbitMQSettings.cpp | 6 ++---- src/Storages/SetSettings.cpp | 6 ++---- 7 files changed, 16 insertions(+), 30 deletions(-) diff --git a/src/Databases/MySQL/MaterializeMySQLSettings.cpp b/src/Databases/MySQL/MaterializeMySQLSettings.cpp index 609ce011f91..a8672bf488e 100644 --- a/src/Databases/MySQL/MaterializeMySQLSettings.cpp +++ b/src/Databases/MySQL/MaterializeMySQLSettings.cpp @@ -8,7 +8,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -25,9 +24,8 @@ void MaterializeMySQLSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for database " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for database " + storage_def.engine->name); + throw; } } else diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 96da40e8f6c..aeb2db0523c 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -648,12 +648,12 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & { function_builder = FunctionFactory::instance().get(node.name, data.context); } - catch (DB::Exception & e) + catch (Exception & e) { auto hints = AggregateFunctionFactory::instance().getHints(node.name); if (!hints.empty()) e.addMessage("Or unknown aggregate function " + node.name + ". Maybe you meant: " + toString(hints)); - e.rethrow(); + throw; } Names argument_names; diff --git a/src/Storages/JoinSettings.cpp b/src/Storages/JoinSettings.cpp index 15637d67dea..8a2699746da 100644 --- a/src/Storages/JoinSettings.cpp +++ b/src/Storages/JoinSettings.cpp @@ -10,7 +10,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -27,9 +26,8 @@ void JoinSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for storage " + storage_def.engine->name); + throw; } } else diff --git a/src/Storages/Kafka/KafkaSettings.cpp b/src/Storages/Kafka/KafkaSettings.cpp index 4d80419af2d..6ef74511d83 100644 --- a/src/Storages/Kafka/KafkaSettings.cpp +++ b/src/Storages/Kafka/KafkaSettings.cpp @@ -10,7 +10,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -27,9 +26,8 @@ void KafkaSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for storage " + storage_def.engine->name); + throw; } } else diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index 00580c8d8bb..f7a9a37e6dd 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -11,8 +11,6 @@ namespace DB namespace ErrorCodes { - extern const int INVALID_CONFIG_PARAMETER; - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -34,9 +32,8 @@ void MergeTreeSettings::loadFromConfig(const String & config_elem, const Poco::U catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " in MergeTree config", ErrorCodes::INVALID_CONFIG_PARAMETER); - else - e.rethrow(); + e.addMessage("in MergeTree config"); + throw; } } @@ -51,9 +48,8 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for storage " + storage_def.engine->name); + throw; } } else diff --git a/src/Storages/RabbitMQ/RabbitMQSettings.cpp b/src/Storages/RabbitMQ/RabbitMQSettings.cpp index f956c520749..93495cdd8ae 100644 --- a/src/Storages/RabbitMQ/RabbitMQSettings.cpp +++ b/src/Storages/RabbitMQ/RabbitMQSettings.cpp @@ -9,7 +9,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -26,9 +25,8 @@ void RabbitMQSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for storage " + storage_def.engine->name); + throw; } } else diff --git a/src/Storages/SetSettings.cpp b/src/Storages/SetSettings.cpp index f7ff1c446f2..baa3d231067 100644 --- a/src/Storages/SetSettings.cpp +++ b/src/Storages/SetSettings.cpp @@ -10,7 +10,6 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int UNKNOWN_SETTING; } @@ -27,9 +26,8 @@ void SetSettings::loadFromQuery(ASTStorage & storage_def) catch (Exception & e) { if (e.code() == ErrorCodes::UNKNOWN_SETTING) - throw Exception(e.message() + " for storage " + storage_def.engine->name, ErrorCodes::BAD_ARGUMENTS); - else - e.rethrow(); + e.addMessage("for storage " + storage_def.engine->name); + throw; } } else From 3aa49204d628dcd74090f06449e2cec26f8856a3 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 26 Oct 2020 07:39:20 +0300 Subject: [PATCH 196/325] Update MergeTreeSettings.cpp --- src/Storages/MergeTree/MergeTreeSettings.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index f7a9a37e6dd..15ff62e0aa6 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -12,6 +12,7 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_SETTING; + extern const int BAD_ARGUMENTS; } IMPLEMENT_SETTINGS_TRAITS(MergeTreeSettingsTraits, LIST_OF_MERGE_TREE_SETTINGS) From 18458f36e773ae2a04b35bc244d2751308a553f1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 26 Oct 2020 08:20:37 +0300 Subject: [PATCH 197/325] Fix trash --- docs/en/operations/system-tables/crash-log.md | 6 +++--- docs/ru/operations/system-tables/crash-log.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/operations/system-tables/crash-log.md b/docs/en/operations/system-tables/crash-log.md index 5b338237b83..5e9fec53429 100644 --- a/docs/en/operations/system-tables/crash-log.md +++ b/docs/en/operations/system-tables/crash-log.md @@ -1,6 +1,6 @@ -# system.crash_log {#system-tables_crash_log} +# system.crash_log {#system-tables_crash_log} -Contains information about stack traces for fatal errors. The table does not exist in the database by default, it is created only when fatal errors occur. +Contains information about stack traces for fatal errors. The table does not exist in the database by default, it is created only when fatal errors occur. Columns: @@ -39,7 +39,7 @@ trace: [188531193,...] trace_full: ['3. DB::(anonymous namespace)::FunctionFormatReadableTimeDelta::executeImpl(std::__1::vector >&, std::__1::vector > const&, unsigned long, unsigned long) const @ 0xb3cc1f9 in /home/username/work/ClickHouse/build/programs/clickhouse',...] version: ClickHouse 20.11.1.1 revision: 54442 -build_id: +build_id: ``` **See also** diff --git a/docs/ru/operations/system-tables/crash-log.md b/docs/ru/operations/system-tables/crash-log.md index 7b645a06b2d..d2b3ae5c6f5 100644 --- a/docs/ru/operations/system-tables/crash-log.md +++ b/docs/ru/operations/system-tables/crash-log.md @@ -1,6 +1,6 @@ -# system.crash_log {#system-tables_crash_log} +# system.crash_log {#system-tables_crash_log} -Содержит информацию о трассировках стека для фатальных ошибок. Таблица не содержится в базе данных по умолчанию, а создается только при возникновении фатальных ошибок. +Содержит информацию о трассировках стека для фатальных ошибок. Таблица не содержится в базе данных по умолчанию, а создается только при возникновении фатальных ошибок. Колонки: @@ -39,7 +39,7 @@ trace: [188531193,...] trace_full: ['3. DB::(anonymous namespace)::FunctionFormatReadableTimeDelta::executeImpl(std::__1::vector >&, std::__1::vector > const&, unsigned long, unsigned long) const @ 0xb3cc1f9 in /home/username/work/ClickHouse/build/programs/clickhouse',...] version: ClickHouse 20.11.1.1 revision: 54442 -build_id: +build_id: ``` **См. также** From f491903d416fd7d683af36896d9c1105b004048c Mon Sep 17 00:00:00 2001 From: Robert Hodges Date: Sun, 25 Oct 2020 22:23:54 -0700 Subject: [PATCH 198/325] Update cloud.md Adding description of Altinity.Cloud to cloud platforms. --- docs/en/commercial/cloud.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/en/commercial/cloud.md b/docs/en/commercial/cloud.md index 07d843f724a..06216517db8 100644 --- a/docs/en/commercial/cloud.md +++ b/docs/en/commercial/cloud.md @@ -18,4 +18,14 @@ toc_title: Cloud - Encryption and isolation - Automated maintenance +## Altinity.Cloud {#altinity.cloud} + +[Altinity.Cloud](https://altinity.com/cloud-database/) is a fully managed ClickHouse-as-a-Service for the Amazon public cloud. +- Fast deployment of ClickHouse clusters on Amazon resources +- Easy scale-out/scale-in as well as vertical scaling of nodes +- Isolated per-tenant VPCs with public endpoint or VPC peering +- Configurable storage types and volume configurations +- Cross-AZ scaling for performance and high availability +- Built-in monitoring and SQL query editor + {## [Original article](https://clickhouse.tech/docs/en/commercial/cloud/) ##} From a5b56c8b84473cd73caf1ddce0927f2f1a6685d8 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Mon, 26 Oct 2020 10:23:27 +0300 Subject: [PATCH 199/325] Empty commit to re-run checks. From 79895130b75f872346be67603f3f48f09a7cada3 Mon Sep 17 00:00:00 2001 From: filimonov <1549571+filimonov@users.noreply.github.com> Date: Mon, 26 Oct 2020 08:59:15 +0100 Subject: [PATCH 200/325] Better grammar in comments, CI rerun --- programs/local/LocalServer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index db1838c38f7..f85b5bde965 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -122,8 +122,8 @@ void LocalServer::tryInitPath() } else { - // Path is not provided explicitly - use unique path in the system temporary directory - // (or in current dir if temporary don't exists) + // The path is not provided explicitly - use a unique path in the system temporary directory + // (or in the current dir if temporary don't exist) Poco::Logger * log = &logger(); std::filesystem::path parent_folder; std::filesystem::path default_path; @@ -144,7 +144,7 @@ void LocalServer::tryInitPath() LOG_DEBUG(log, "Will create working directory inside current directory: {}", parent_folder.string()); } - /// we can have other clickhouse-local running simultaneously, even with same pid (for ex. - several docker mounting same folder) + /// we can have another clickhouse-local running simultaneously, even with the same PID (for ex. - several dockers mounting the same folder) /// or it can be some leftovers from other clickhouse-local runs /// as we can't accurately distinguish those situations we don't touch any existent folders /// we just try to pick some free name for our working folder From 54f823aafa3f00c5454a6538200ca5b30d63d70a Mon Sep 17 00:00:00 2001 From: tavplubix Date: Mon, 26 Oct 2020 12:42:54 +0300 Subject: [PATCH 201/325] Update 01114_database_atomic.sh --- tests/queries/0_stateless/01114_database_atomic.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01114_database_atomic.sh b/tests/queries/0_stateless/01114_database_atomic.sh index f00a1c4a4d8..d220491d152 100755 --- a/tests/queries/0_stateless/01114_database_atomic.sh +++ b/tests/queries/0_stateless/01114_database_atomic.sh @@ -65,9 +65,9 @@ $CLICKHOUSE_CLIENT -q "SELECT count() FROM test_01114_1.mt" # result: 5 $CLICKHOUSE_CLIENT -q "SELECT tuple(s, sleepEachRow(3)) FROM test_01114_1.mt" > /dev/null & # 15s sleep 1 -$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_1" && echo "dropped" +$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_1" --database_atomic_wait_for_drop_and_detach_synchronously=0 && echo "dropped" wait # for INSERT $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01114_2.mt" # result: 30, 435 -$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_2" +$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01114_2" --database_atomic_wait_for_drop_and_detach_synchronously=0 From cb943599036c6366fb15ba2ef06ec2a576460af1 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Mon, 26 Oct 2020 12:45:30 +0300 Subject: [PATCH 202/325] Update 01107_atomic_db_detach_attach.sh --- tests/queries/0_stateless/01107_atomic_db_detach_attach.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh index 7dd3e4e37e6..173bf44e1f1 100755 --- a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh +++ b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh @@ -12,16 +12,16 @@ sleep 1 $CLICKHOUSE_CLIENT -q "DETACH TABLE test_01107.mt" --database_atomic_wait_for_drop_and_detach_synchronously=0 $CLICKHOUSE_CLIENT -q "ATTACH TABLE test_01107.mt" 2>&1 | grep -F "Code: 57" > /dev/null && echo "OK" -$CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" 2>&1 | grep -F "Code: 219" > /dev/null && echo "OK" +$CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" --database_atomic_wait_for_drop_and_detach_synchronously=0 2>&1 | grep -F "Code: 219" > /dev/null && echo "OK" wait $CLICKHOUSE_CLIENT -q "ATTACH TABLE test_01107.mt" $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01107.mt" -$CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" +$CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" --database_atomic_wait_for_drop_and_detach_synchronously=0 $CLICKHOUSE_CLIENT -q "ATTACH DATABASE test_01107" $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01107.mt" $CLICKHOUSE_CLIENT -q "INSERT INTO test_01107.mt SELECT number + sleepEachRow(1) FROM numbers(5)" && echo "end" & sleep 1 -$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01107" && sleep 1 && echo "dropped" +$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01107" --database_atomic_wait_for_drop_and_detach_synchronously=0 && sleep 1 && echo "dropped" wait From fe9db9866c3345a512c6993e62ab669257c6d1cf Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 26 Oct 2020 12:56:50 +0300 Subject: [PATCH 203/325] Check exception message is poisoned in ExpressionActions. --- src/Interpreters/ExpressionActions.cpp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 8513384d31a..6ce9fe2c793 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -22,6 +22,15 @@ # include "config_core.h" #endif +#include + +#if defined(MEMORY_SANITIZER) + #include +#endif + +#if defined(ADDRESS_SANITIZER) + #include +#endif namespace ProfileEvents { @@ -623,6 +632,22 @@ void ExpressionActions::execute(Block & block, bool dry_run) const } catch (Exception & e) { +#if defined(MEMORY_SANITIZER) + const auto & msg = e.message(); + if (__msan_test_shadow(msg.data(), msg.size()) != -1) + { + LOG_FATAL(&Poco::Logger::get("ExpressionActions"), "Poisoned exception message (msan): {}", e.getStackTraceString()); + } +#endif + +#if defined(ADDRESS_SANITIZER) + const auto & msg = e.message(); + if (__asan_region_is_poisoned(msg.data(), msg.size())) + { + LOG_FATAL(&Poco::Logger::get("ExpressionActions"), "Poisoned exception message (asan): {}", e.getStackTraceString()); + } +#endif + e.addMessage(fmt::format("while executing '{}'", action.toString())); throw; } From 8f03e526a4b50f61428e0bf7020cb628d99d2668 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Mon, 26 Oct 2020 13:09:41 +0300 Subject: [PATCH 204/325] DOCSUP-3200: Edit and translate to Russian (#16204) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Edit and translate Отредактировал английскую версию текстов документов и перевел изменения на русский язык. * Update trace_log.md Исправляю битые ссылки. * Fixing broken links Исправил битые ссылки. * Fixing broken links Поправил битую ссылку, добавив раздел в русскую версию. Co-authored-by: Dmitriy --- docs/en/faq/integration/json-import.md | 2 +- docs/en/interfaces/formats.md | 6 +- .../third-party/client-libraries.md | 2 +- docs/en/operations/system-tables/query_log.md | 8 +- .../system-tables/query_thread_log.md | 2 +- docs/en/operations/system-tables/text_log.md | 2 +- docs/en/operations/system-tables/trace_log.md | 2 +- .../statements/alter/sample-by.md | 10 +- docs/en/sql-reference/statements/drop.md | 64 ++-- docs/ru/interfaces/formats.md | 297 ++++++++++++------ docs/ru/operations/system-tables/query_log.md | 100 +++--- .../system-tables/query_thread_log.md | 84 ++--- docs/ru/operations/system-tables/text_log.md | 49 ++- docs/ru/operations/system-tables/trace_log.md | 33 +- .../statements/alter/sample-by.md | 20 ++ docs/ru/sql-reference/statements/drop.md | 64 ++-- docs/ru/sql-reference/statements/grant.md | 2 + 17 files changed, 452 insertions(+), 295 deletions(-) create mode 100644 docs/ru/sql-reference/statements/alter/sample-by.md diff --git a/docs/en/faq/integration/json-import.md b/docs/en/faq/integration/json-import.md index 067b407a079..fb94f226f2b 100644 --- a/docs/en/faq/integration/json-import.md +++ b/docs/en/faq/integration/json-import.md @@ -30,4 +30,4 @@ Instead of inserting data manually, you might consider to use one of [client lib - `input_format_import_nested_json` allows to insert nested JSON objects into columns of [Nested](../../sql-reference/data-types/nested-data-structures/nested.md) type. !!! note "Note" - Settings are specified as `GET` parameters for the HTTP interface or as additional command-line arguments prefixed with `--` for the CLI interface. + Settings are specified as `GET` parameters for the HTTP interface or as additional command-line arguments prefixed with `--` for the `CLI` interface. \ No newline at end of file diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index d96d48bdca3..d310705d1c1 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -460,7 +460,7 @@ See also the [JSONEachRow](#jsoneachrow) format. ## JSONString {#jsonstring} -Differs from JSON only in that data fields are output in strings, not in typed json values. +Differs from JSON only in that data fields are output in strings, not in typed JSON values. Example: @@ -596,7 +596,7 @@ When inserting the data, you should provide a separate JSON value for each row. ## JSONEachRowWithProgress {#jsoneachrowwithprogress} ## JSONStringEachRowWithProgress {#jsonstringeachrowwithprogress} -Differs from JSONEachRow/JSONStringEachRow in that ClickHouse will also yield progress information as JSON objects. +Differs from `JSONEachRow`/`JSONStringEachRow` in that ClickHouse will also yield progress information as JSON values. ```json {"row":{"'hello'":"hello","multiply(42, number)":"0","range(5)":[0,1,2,3,4]}} @@ -608,7 +608,7 @@ Differs from JSONEachRow/JSONStringEachRow in that ClickHouse will also yield pr ## JSONCompactEachRowWithNamesAndTypes {#jsoncompacteachrowwithnamesandtypes} ## JSONCompactStringEachRowWithNamesAndTypes {#jsoncompactstringeachrowwithnamesandtypes} -Differs from JSONCompactEachRow/JSONCompactStringEachRow in that the column names and types are written as the first two rows. +Differs from `JSONCompactEachRow`/`JSONCompactStringEachRow` in that the column names and types are written as the first two rows. ```json ["'hello'", "multiply(42, number)", "range(5)"] diff --git a/docs/en/interfaces/third-party/client-libraries.md b/docs/en/interfaces/third-party/client-libraries.md index bb98c5f936b..4e8347c9a6e 100644 --- a/docs/en/interfaces/third-party/client-libraries.md +++ b/docs/en/interfaces/third-party/client-libraries.md @@ -6,7 +6,7 @@ toc_title: Client Libraries # Client Libraries from Third-party Developers {#client-libraries-from-third-party-developers} !!! warning "Disclaimer" - Yandex does **not** maintain the libraries listed below and haven’t done any extensive testing to ensure their quality. + Yandex does **not** maintain the libraries listed below and hasn’t done any extensive testing to ensure their quality. - Python - [infi.clickhouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) diff --git a/docs/en/operations/system-tables/query_log.md b/docs/en/operations/system-tables/query_log.md index ddd5d327d59..32b2bdf2133 100644 --- a/docs/en/operations/system-tables/query_log.md +++ b/docs/en/operations/system-tables/query_log.md @@ -20,8 +20,8 @@ The `system.query_log` table registers two kinds of queries: Each query creates one or two rows in the `query_log` table, depending on the status (see the `type` column) of the query: -1. If the query execution was successful, two rows with the `QueryStart` and `QueryFinish` types are created . -2. If an error occurred during query processing, two events with the `QueryStart` and `ExceptionWhileProcessing` types are created . +1. If the query execution was successful, two rows with the `QueryStart` and `QueryFinish` types are created. +2. If an error occurred during query processing, two events with the `QueryStart` and `ExceptionWhileProcessing` types are created. 3. If an error occurred before launching the query, a single event with the `ExceptionBeforeStart` type is created. Columns: @@ -37,8 +37,8 @@ Columns: - `query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Start time of query execution. - `query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Start time of query execution with microsecond precision. - `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Duration of query execution in milliseconds. -- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number or rows read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_rows` includes the total number of rows read at all replicas. Each replica sends it’s `read_rows` value, and the server-initiator of the query summarize all received and local values. The cache volumes doesn’t affect this value. -- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number or bytes read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_bytes` includes the total number of rows read at all replicas. Each replica sends it’s `read_bytes` value, and the server-initiator of the query summarize all received and local values. The cache volumes doesn’t affect this value. +- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number of rows read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_rows` includes the total number of rows read at all replicas. Each replica sends it’s `read_rows` value, and the server-initiator of the query summarizes all received and local values. The cache volumes don’t affect this value. +- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number of bytes read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_bytes` includes the total number of rows read at all replicas. Each replica sends it’s `read_bytes` value, and the server-initiator of the query summarizes all received and local values. The cache volumes don’t affect this value. - `written_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0. - `written_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0. - `result_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of rows in a result of the `SELECT` query, or a number of rows in the `INSERT` query. diff --git a/docs/en/operations/system-tables/query_thread_log.md b/docs/en/operations/system-tables/query_thread_log.md index a6f090e99f3..0ae2e7d5d3b 100644 --- a/docs/en/operations/system-tables/query_thread_log.md +++ b/docs/en/operations/system-tables/query_thread_log.md @@ -1,6 +1,6 @@ # system.query_thread_log {#system_tables-query_thread_log} -Contains information about threads which execute queries, for example, thread name, thread start time, duration of query processing. +Contains information about threads that execute queries, for example, thread name, thread start time, duration of query processing. To start logging: diff --git a/docs/en/operations/system-tables/text_log.md b/docs/en/operations/system-tables/text_log.md index 34f37c2cae7..f5f53c95653 100644 --- a/docs/en/operations/system-tables/text_log.md +++ b/docs/en/operations/system-tables/text_log.md @@ -1,6 +1,6 @@ # system.text_log {#system_tables-text_log} -Contains logging entries. Logging level which goes to this table can be limited with `text_log.level` server setting. +Contains logging entries. The logging level which goes to this table can be limited to the `text_log.level` server setting. Columns: diff --git a/docs/en/operations/system-tables/trace_log.md b/docs/en/operations/system-tables/trace_log.md index bc955ebe4de..8107f60b808 100644 --- a/docs/en/operations/system-tables/trace_log.md +++ b/docs/en/operations/system-tables/trace_log.md @@ -18,7 +18,7 @@ Columns: - `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ClickHouse server build revision. - When connecting to server by `clickhouse-client`, you see the string similar to `Connected to ClickHouse server version 19.18.1 revision 54429.`. This field contains the `revision`, but not the `version` of a server. + When connecting to the server by `clickhouse-client`, you see the string similar to `Connected to ClickHouse server version 19.18.1 revision 54429.`. This field contains the `revision`, but not the `version` of a server. - `timer_type` ([Enum8](../../sql-reference/data-types/enum.md)) — Timer type: diff --git a/docs/en/sql-reference/statements/alter/sample-by.md b/docs/en/sql-reference/statements/alter/sample-by.md index bea6364e4ea..df8ff90f196 100644 --- a/docs/en/sql-reference/statements/alter/sample-by.md +++ b/docs/en/sql-reference/statements/alter/sample-by.md @@ -5,16 +5,16 @@ toc_title: SAMPLE BY # Manipulating Sampling-Key Expressions {#manipulations-with-sampling-key-expressions} +Syntax: + ``` sql ALTER TABLE [db].name [ON CLUSTER cluster] MODIFY SAMPLE BY new_expression ``` The command changes the [sampling key](../../../engines/table-engines/mergetree-family/mergetree.md) of the table to `new_expression` (an expression or a tuple of expressions). -The command is lightweight in a sense that it only changes metadata. The primary key must contain the new sample key. +The command is lightweight in the sense that it only changes metadata. The primary key must contain the new sample key. !!! note "Note" - It only works for tables in the [`MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md) family (including -[replicated](../../../engines/table-engines/mergetree-family/replication.md) tables). - - + It only works for tables in the [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) family (including +[replicated](../../../engines/table-engines/mergetree-family/replication.md) tables). \ No newline at end of file diff --git a/docs/en/sql-reference/statements/drop.md b/docs/en/sql-reference/statements/drop.md index de4eda64ef6..07761b6b821 100644 --- a/docs/en/sql-reference/statements/drop.md +++ b/docs/en/sql-reference/statements/drop.md @@ -5,84 +5,96 @@ toc_title: DROP # DROP Statements {#drop} -Deletes existing entity. If `IF EXISTS` clause is specified, these queries doesn’t return an error if the entity doesn’t exist. +Deletes existing entity. If the `IF EXISTS` clause is specified, these queries don’t return an error if the entity doesn’t exist. ## DROP DATABASE {#drop-database} +Deletes all tables inside the `db` database, then deletes the `db` database itself. + +Syntax: + ``` sql DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] ``` -Deletes all tables inside the `db` database, then deletes the ‘db’ database itself. - ## DROP TABLE {#drop-table} +Deletes the table. + +Syntax: + ``` sql DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` -Deletes the table. - ## DROP DICTIONARY {#drop-dictionary} +Deletes the dictionary. + +Syntax: + ``` sql DROP DICTIONARY [IF EXISTS] [db.]name ``` -Deletes the dictionary. - ## DROP USER {#drop-user-statement} +Deletes a user. + +Syntax: + ``` sql DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` -Deletes a user. - ## DROP ROLE {#drop-role-statement} +Deletes a role. The deleted role is revoked from all the entities where it was assigned. + +Syntax: + ``` sql DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` -Deletes a role. - -Deleted role is revoked from all the entities where it was assigned. - ## DROP ROW POLICY {#drop-row-policy-statement} +Deletes a row policy. Deleted row policy is revoked from all the entities where it was assigned. + +Syntax: + ``` sql DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] ``` -Deletes a row policy. - -Deleted row policy is revoked from all the entities where it was assigned. - ## DROP QUOTA {#drop-quota-statement} +Deletes a quota. The deleted quota is revoked from all the entities where it was assigned. + +Syntax: + ``` sql DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` -Deletes a quota. - -Deleted quota is revoked from all the entities where it was assigned. - ## DROP SETTINGS PROFILE {#drop-settings-profile-statement} +Deletes a settings profile. The deleted settings profile is revoked from all the entities where it was assigned. + +Syntax: + ``` sql DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` -Deletes a settings profile. - -Deleted settings profile is revoked from all the entities where it was assigned. - ## DROP VIEW {#drop-view} +Deletes a view. Views can be deleted by a `DROP TABLE` command as well but `DROP VIEW` checks that `[db.]name` is a view. + +Syntax: + ``` sql DROP VIEW [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` -Deletes a view. Views can be deleted by a `DROP TABLE` command as well but `DROP VIEW` checks that `[db.]name` is a view. +[Оriginal article](https://clickhouse.tech/docs/en/sql-reference/statements/drop/) \ No newline at end of file diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 2745139998f..e94810be8d3 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -4,41 +4,53 @@ ClickHouse может принимать (`INSERT`) и отдавать (`SELECT Поддерживаемые форматы и возможность использовать их в запросах `INSERT` и `SELECT` перечислены в таблице ниже. -| Формат | INSERT | SELECT | -|-----------------------------------------------------------------|--------|--------| -| [TabSeparated](#tabseparated) | ✔ | ✔ | -| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ | -| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ | -| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ | -| [Template](#format-template) | ✔ | ✔ | -| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ | -| [CSV](#csv) | ✔ | ✔ | -| [CSVWithNames](#csvwithnames) | ✔ | ✔ | -| [CustomSeparated](#format-customseparated) | ✔ | ✔ | -| [Values](#data-format-values) | ✔ | ✔ | -| [Vertical](#vertical) | ✗ | ✔ | -| [JSON](#json) | ✗ | ✔ | -| [JSONCompact](#jsoncompact) | ✗ | ✔ | -| [JSONEachRow](#jsoneachrow) | ✔ | ✔ | -| [TSKV](#tskv) | ✔ | ✔ | -| [Pretty](#pretty) | ✗ | ✔ | -| [PrettyCompact](#prettycompact) | ✗ | ✔ | -| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ | -| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ | -| [PrettySpace](#prettyspace) | ✗ | ✔ | -| [Protobuf](#protobuf) | ✔ | ✔ | -| [ProtobufSingle](#protobufsingle) | ✔ | ✔ | -| [Parquet](#data-format-parquet) | ✔ | ✔ | -| [Arrow](#data-format-arrow) | ✔ | ✔ | -| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ | -| [ORC](#data-format-orc) | ✔ | ✗ | -| [RowBinary](#rowbinary) | ✔ | ✔ | -| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | -| [Native](#native) | ✔ | ✔ | -| [Null](#null) | ✗ | ✔ | -| [XML](#xml) | ✗ | ✔ | -| [CapnProto](#capnproto) | ✔ | ✗ | -| [LineAsString](#lineasstring) | ✔ | ✗ | +| Формат | INSERT | SELECT | +|-----------------------------------------------------------------------------------------|--------|--------| +| [TabSeparated](#tabseparated) | ✔ | ✔ | +| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ | +| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ | +| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ | +| [Template](#format-template) | ✔ | ✔ | +| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ | +| [CSV](#csv) | ✔ | ✔ | +| [CSVWithNames](#csvwithnames) | ✔ | ✔ | +| [CustomSeparated](#format-customseparated) | ✔ | ✔ | +| [Values](#data-format-values) | ✔ | ✔ | +| [Vertical](#vertical) | ✗ | ✔ | +| [VerticalRaw](#verticalraw) | ✗ | ✔ | +| [JSON](#json) | ✗ | ✔ | +| [JSONString](#jsonstring) | ✗ | ✔ | +| [JSONCompact](#jsoncompact) | ✗ | ✔ | +| [JSONCompactString](#jsoncompactstring) | ✗ | ✔ | +| [JSONEachRow](#jsoneachrow) | ✔ | ✔ | +| [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ | +| [JSONStringEachRow](#jsonstringeachrow) | ✔ | ✔ | +| [JSONStringEachRowWithProgress](#jsonstringeachrowwithprogress) | ✗ | ✔ | +| [JSONCompactEachRow](#jsoncompacteachrow) | ✔ | ✔ | +| [JSONCompactEachRowWithNamesAndTypes](#jsoncompacteachrowwithnamesandtypes) | ✔ | ✔ | +| [JSONCompactStringEachRow](#jsoncompactstringeachrow) | ✔ | ✔ | +| [JSONCompactStringEachRowWithNamesAndTypes](#jsoncompactstringeachrowwithnamesandtypes) | ✔ | ✔ | +| [TSKV](#tskv) | ✔ | ✔ | +| [Pretty](#pretty) | ✗ | ✔ | +| [PrettyCompact](#prettycompact) | ✗ | ✔ | +| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ | +| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ | +| [PrettySpace](#prettyspace) | ✗ | ✔ | +| [Protobuf](#protobuf) | ✔ | ✔ | +| [ProtobufSingle](#protobufsingle) | ✔ | ✔ | +| [Avro](#data-format-avro) | ✔ | ✔ | +| [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ | +| [Parquet](#data-format-parquet) | ✔ | ✔ | +| [Arrow](#data-format-arrow) | ✔ | ✔ | +| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ | +| [ORC](#data-format-orc) | ✔ | ✗ | +| [RowBinary](#rowbinary) | ✔ | ✔ | +| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | +| [Native](#native) | ✔ | ✔ | +| [Null](#null) | ✗ | ✔ | +| [XML](#xml) | ✗ | ✔ | +| [CapnProto](#capnproto) | ✔ | ✗ | +| [LineAsString](#lineasstring) | ✔ | ✗ | Вы можете регулировать некоторые параметры работы с форматами с помощью настроек ClickHouse. За дополнительной информацией обращайтесь к разделу [Настройки](../operations/settings/settings.md). @@ -364,62 +376,41 @@ SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTA "meta": [ { - "name": "SearchPhrase", + "name": "'hello'", "type": "String" }, { - "name": "c", + "name": "multiply(42, number)", "type": "UInt64" + }, + { + "name": "range(5)", + "type": "Array(UInt8)" } ], "data": [ { - "SearchPhrase": "", - "c": "8267016" + "'hello'": "hello", + "multiply(42, number)": "0", + "range(5)": [0,1,2,3,4] }, { - "SearchPhrase": "bathroom interior design", - "c": "2166" + "'hello'": "hello", + "multiply(42, number)": "42", + "range(5)": [0,1,2,3,4] }, { - "SearchPhrase": "yandex", - "c": "1655" - }, - { - "SearchPhrase": "spring 2014 fashion", - "c": "1549" - }, - { - "SearchPhrase": "freeform photos", - "c": "1480" + "'hello'": "hello", + "multiply(42, number)": "84", + "range(5)": [0,1,2,3,4] } ], - "totals": - { - "SearchPhrase": "", - "c": "8873898" - }, + "rows": 3, - "extremes": - { - "min": - { - "SearchPhrase": "", - "c": "1480" - }, - "max": - { - "SearchPhrase": "", - "c": "8267016" - } - }, - - "rows": 5, - - "rows_before_limit_at_least": 141137 + "rows_before_limit_at_least": 3 } ``` @@ -438,65 +429,167 @@ JSON совместим с JavaScript. Для этого, дополнитель ClickHouse поддерживает [NULL](../sql-reference/syntax.md), который при выводе JSON будет отображен как `null`. Чтобы включить отображение в результате значений `+nan`, `-nan`, `+inf`, `-inf`, установите параметр [output_format_json_quote_denormals](../operations/settings/settings.md#settings-output_format_json_quote_denormals) равным 1. -Смотрите также формат [JSONEachRow](#jsoneachrow) . +Смотрите также формат [JSONEachRow](#jsoneachrow). + +## JSONString {#jsonstring} + +Отличается от JSON только тем, что поля данных выводятся в строках, а не в типизированных значениях JSON. + +Пример: + +```json +{ + "meta": + [ + { + "name": "'hello'", + "type": "String" + }, + { + "name": "multiply(42, number)", + "type": "UInt64" + }, + { + "name": "range(5)", + "type": "Array(UInt8)" + } + ], + + "data": + [ + { + "'hello'": "hello", + "multiply(42, number)": "0", + "range(5)": "[0,1,2,3,4]" + }, + { + "'hello'": "hello", + "multiply(42, number)": "42", + "range(5)": "[0,1,2,3,4]" + }, + { + "'hello'": "hello", + "multiply(42, number)": "84", + "range(5)": "[0,1,2,3,4]" + } + ], + + "rows": 3, + + "rows_before_limit_at_least": 3 +} +``` ## JSONCompact {#jsoncompact} +## JSONCompactString {#jsoncompactstring} Отличается от JSON только тем, что строчки данных выводятся в массивах, а не в object-ах. Пример: ``` json +// JSONCompact { "meta": [ { - "name": "SearchPhrase", + "name": "'hello'", "type": "String" }, { - "name": "c", + "name": "multiply(42, number)", "type": "UInt64" + }, + { + "name": "range(5)", + "type": "Array(UInt8)" } ], "data": [ - ["", "8267016"], - ["интерьер ванной комнаты", "2166"], - ["яндекс", "1655"], - ["весна 2014 мода", "1549"], - ["фриформ фото", "1480"] + ["hello", "0", [0,1,2,3,4]], + ["hello", "42", [0,1,2,3,4]], + ["hello", "84", [0,1,2,3,4]] ], - "totals": ["","8873898"], + "rows": 3, - "extremes": - { - "min": ["","1480"], - "max": ["","8267016"] - }, - - "rows": 5, - - "rows_before_limit_at_least": 141137 + "rows_before_limit_at_least": 3 } ``` -Этот формат подходит только для вывода результата выполнения запроса, но не для парсинга (приёма данных для вставки в таблицу). -Смотрите также формат `JSONEachRow`. +```json +// JSONCompactString +{ + "meta": + [ + { + "name": "'hello'", + "type": "String" + }, + { + "name": "multiply(42, number)", + "type": "UInt64" + }, + { + "name": "range(5)", + "type": "Array(UInt8)" + } + ], -## JSONEachRow {#jsoneachrow} + "data": + [ + ["hello", "0", "[0,1,2,3,4]"], + ["hello", "42", "[0,1,2,3,4]"], + ["hello", "84", "[0,1,2,3,4]"] + ], -При использовании этого формата, ClickHouse выводит каждую запись как объект JSON (каждый объект отдельной строкой), при этом данные в целом — невалидный JSON. + "rows": 3, -``` json -{"SearchPhrase":"дизайн штор","count()":"1064"} -{"SearchPhrase":"баку","count()":"1000"} -{"SearchPhrase":"","count":"8267016"} + "rows_before_limit_at_least": 3 +} ``` -При вставке данных необходимо каждую запись передавать как отдельный объект JSON. +## JSONEachRow {#jsoneachrow} +## JSONStringEachRow {#jsonstringeachrow} +## JSONCompactEachRow {#jsoncompacteachrow} +## JSONCompactStringEachRow {#jsoncompactstringeachrow} + +При использовании этих форматов ClickHouse выводит каждую запись как значения JSON (каждое значение отдельной строкой), при этом данные в целом — невалидный JSON. + +``` json +{"some_int":42,"some_str":"hello","some_tuple":[1,"a"]} // JSONEachRow +[42,"hello",[1,"a"]] // JSONCompactEachRow +["42","hello","(2,'a')"] // JSONCompactStringsEachRow +``` + +При вставке данных вы должны предоставить отдельное значение JSON для каждой строки. + +## JSONEachRowWithProgress {#jsoneachrowwithprogress} +## JSONStringEachRowWithProgress {#jsonstringeachrowwithprogress} + +Отличается от `JSONEachRow`/`JSONStringEachRow` тем, что ClickHouse будет выдавать информацию о ходе выполнения в виде значений JSON. + +```json +{"row":{"'hello'":"hello","multiply(42, number)":"0","range(5)":[0,1,2,3,4]}} +{"row":{"'hello'":"hello","multiply(42, number)":"42","range(5)":[0,1,2,3,4]}} +{"row":{"'hello'":"hello","multiply(42, number)":"84","range(5)":[0,1,2,3,4]}} +{"progress":{"read_rows":"3","read_bytes":"24","written_rows":"0","written_bytes":"0","total_rows_to_read":"3"}} +``` + +## JSONCompactEachRowWithNamesAndTypes {#jsoncompacteachrowwithnamesandtypes} +## JSONCompactStringEachRowWithNamesAndTypes {#jsoncompactstringeachrowwithnamesandtypes} + +Отличается от `JSONCompactEachRow`/`JSONCompactStringEachRow` тем, что имена и типы столбцов записываются как первые две строки. + +```json +["'hello'", "multiply(42, number)", "range(5)"] +["String", "UInt64", "Array(UInt8)"] +["hello", "0", [0,1,2,3,4]] +["hello", "42", [0,1,2,3,4]] +["hello", "84", [0,1,2,3,4]] +``` ### Вставка данных {#vstavka-dannykh} @@ -784,6 +877,10 @@ test: string with 'quotes' and with some special Этот формат подходит только для вывода результата выполнения запроса, но не для парсинга (приёма данных для вставки в таблицу). +## VerticalRaw {#verticalraw} + +Аналогичен [Vertical](#vertical), но с отключенным выходом. Этот формат подходит только для вывода результата выполнения запроса, но не для парсинга (приёма данных для вставки в таблицу). + ## XML {#xml} Формат XML подходит только для вывода данных, не для парсинга. Пример: diff --git a/docs/ru/operations/system-tables/query_log.md b/docs/ru/operations/system-tables/query_log.md index 27ed93e874e..39f685288d8 100644 --- a/docs/ru/operations/system-tables/query_log.md +++ b/docs/ru/operations/system-tables/query_log.md @@ -33,11 +33,12 @@ ClickHouse не удаляет данные из таблица автомати - `'ExceptionWhileProcessing' = 4` — исключение во время обработки запроса. - `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата начала запроса. - `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала запроса. +- `event_time_microseconds` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала запроса с точностью до микросекунд. - `query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала обработки запроса. - `query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время начала обработки запроса с точностью до микросекунд. - `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — длительность выполнения запроса в миллисекундах. -- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Общее количество строк, считанных из всех таблиц и табличных функций, участвующих в запросе. Включает в себя обычные подзапросы, подзапросы для `IN` и `JOIN`. Для распределенных запросов `read_rows` включает в себя общее количество строк, прочитанных на всех репликах. Каждая реплика передает собственное значение `read_rows`, а сервер-инициатор запроса суммирует все полученные и локальные значения. Объемы кэша не учитываюся. -- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Общее количество байтов, считанных из всех таблиц и табличных функций, участвующих в запросе. Включает в себя обычные подзапросы, подзапросы для `IN` и `JOIN`. Для распределенных запросов `read_bytes` включает в себя общее количество байтов, прочитанных на всех репликах. Каждая реплика передает собственное значение `read_bytes`, а сервер-инициатор запроса суммирует все полученные и локальные значения. Объемы кэша не учитываюся. +- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — общее количество строк, считанных из всех таблиц и табличных функций, участвующих в запросе. Включает в себя обычные подзапросы, подзапросы для `IN` и `JOIN`. Для распределенных запросов `read_rows` включает в себя общее количество строк, прочитанных на всех репликах. Каждая реплика передает собственное значение `read_rows`, а сервер-инициатор запроса суммирует все полученные и локальные значения. Объемы кэша не учитываюся. +- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — общее количество байтов, считанных из всех таблиц и табличных функций, участвующих в запросе. Включает в себя обычные подзапросы, подзапросы для `IN` и `JOIN`. Для распределенных запросов `read_bytes` включает в себя общее количество байтов, прочитанных на всех репликах. Каждая реплика передает собственное значение `read_bytes`, а сервер-инициатор запроса суммирует все полученные и локальные значения. Объемы кэша не учитываюся. - `written_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — количество записанных строк для запросов `INSERT`. Для других запросов, значение столбца 0. - `written_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — объём записанных данных в байтах для запросов `INSERT`. Для других запросов, значение столбца 0. - `result_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — количество строк в результате запроса `SELECT` или количество строк в запросе `INSERT`. @@ -76,64 +77,67 @@ ClickHouse не удаляет данные из таблица автомати - `quota_key` ([String](../../sql-reference/data-types/string.md)) — «ключ квоты» из настроек [квот](quotas.md) (см. `keyed`). - `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ревизия ClickHouse. - `thread_numbers` ([Array(UInt32)](../../sql-reference/data-types/array.md)) — количество потоков, участвующих в обработке запросов. -- `ProfileEvents.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — Счетчики для изменения различных метрик. Описание метрик можно получить из таблицы [system.events](#system_tables-events)(#system_tables-events +- `ProfileEvents.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — счетчики для изменения различных метрик. Описание метрик можно получить из таблицы [system.events](#system_tables-events)(#system_tables-events - `ProfileEvents.Values` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — метрики, перечисленные в столбце `ProfileEvents.Names`. - `Settings.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — имена настроек, которые меняются, когда клиент выполняет запрос. Чтобы разрешить логирование изменений настроек, установите параметр `log_query_settings` равным 1. -- `Settings.Values` ([Array(String)](../../sql-reference/data-types/array.md)) — Значения настроек, которые перечислены в столбце `Settings.Names`. +- `Settings.Values` ([Array(String)](../../sql-reference/data-types/array.md)) — значения настроек, которые перечислены в столбце `Settings.Names`. **Пример** ``` sql -SELECT * FROM system.query_log LIMIT 1 FORMAT Vertical; +SELECT * FROM system.query_log LIMIT 1 \G ``` ``` text Row 1: ────── -type: QueryStart -event_date: 2020-05-13 -event_time: 2020-05-13 14:02:28 -query_start_time: 2020-05-13 14:02:28 -query_duration_ms: 0 -read_rows: 0 -read_bytes: 0 -written_rows: 0 -written_bytes: 0 -result_rows: 0 -result_bytes: 0 -memory_usage: 0 -query: SELECT 1 -exception_code: 0 -exception: -stack_trace: -is_initial_query: 1 -user: default -query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a -address: ::ffff:127.0.0.1 -port: 57720 -initial_user: default -initial_query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a -initial_address: ::ffff:127.0.0.1 -initial_port: 57720 -interface: 1 -os_user: bayonet -client_hostname: clickhouse.ru-central1.internal -client_name: ClickHouse client -client_revision: 54434 -client_version_major: 20 -client_version_minor: 4 -client_version_patch: 1 -http_method: 0 -http_user_agent: -quota_key: -revision: 54434 -thread_ids: [] -ProfileEvents.Names: [] -ProfileEvents.Values: [] -Settings.Names: ['use_uncompressed_cache','load_balancing','log_queries','max_memory_usage'] -Settings.Values: ['0','random','1','10000000000'] - +type: QueryStart +event_date: 2020-09-11 +event_time: 2020-09-11 10:08:17 +event_time_microseconds: 2020-09-11 10:08:17.063321 +query_start_time: 2020-09-11 10:08:17 +query_start_time_microseconds: 2020-09-11 10:08:17.063321 +query_duration_ms: 0 +read_rows: 0 +read_bytes: 0 +written_rows: 0 +written_bytes: 0 +result_rows: 0 +result_bytes: 0 +memory_usage: 0 +current_database: default +query: INSERT INTO test1 VALUES +exception_code: 0 +exception: +stack_trace: +is_initial_query: 1 +user: default +query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef +address: ::ffff:127.0.0.1 +port: 33452 +initial_user: default +initial_query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef +initial_address: ::ffff:127.0.0.1 +initial_port: 33452 +interface: 1 +os_user: bharatnc +client_hostname: tower +client_name: ClickHouse +client_revision: 54437 +client_version_major: 20 +client_version_minor: 7 +client_version_patch: 2 +http_method: 0 +http_user_agent: +quota_key: +revision: 54440 +thread_ids: [] +ProfileEvents.Names: [] +ProfileEvents.Values: [] +Settings.Names: ['use_uncompressed_cache','load_balancing','log_queries','max_memory_usage','allow_introspection_functions'] +Settings.Values: ['0','random','1','10000000000','1'] ``` + **Смотрите также** - [system.query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log) — в этой таблице содержится информация о цепочке каждого выполненного запроса. diff --git a/docs/ru/operations/system-tables/query_thread_log.md b/docs/ru/operations/system-tables/query_thread_log.md index 11571f594d5..052baf98035 100644 --- a/docs/ru/operations/system-tables/query_thread_log.md +++ b/docs/ru/operations/system-tables/query_thread_log.md @@ -15,6 +15,7 @@ ClickHouse не удаляет данные из таблицы автомати - `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата завершения выполнения запроса потоком. - `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — дата и время завершения выполнения запроса потоком. +- `event_time_microseconds` ([DateTime](../../sql-reference/data-types/datetime.md)) — дата и время завершения выполнения запроса потоком с точностью до микросекунд. - `query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время начала обработки запроса. - `query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — время начала обработки запроса с точностью до микросекунд. - `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — длительность обработки запроса в миллисекундах. @@ -24,7 +25,7 @@ ClickHouse не удаляет данные из таблицы автомати - `written_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — объём записанных данных в байтах для запросов `INSERT`. Для других запросов, значение столбца 0. - `memory_usage` ([Int64](../../sql-reference/data-types/int-uint.md)) — разница между выделенной и освобождённой памятью в контексте потока. - `peak_memory_usage` ([Int64](../../sql-reference/data-types/int-uint.md)) — максимальная разница между выделенной и освобождённой памятью в контексте потока. -- `thread_name` ([String](../../sql-reference/data-types/string.md)) — Имя потока. +- `thread_name` ([String](../../sql-reference/data-types/string.md)) — имя потока. - `thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — tid (ID потока операционной системы). - `master_thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — tid (ID потока операционной системы) главного потока. - `query` ([String](../../sql-reference/data-types/string.md)) — текст запроса. @@ -56,56 +57,57 @@ ClickHouse не удаляет данные из таблицы автомати - `http_user_agent` ([String](../../sql-reference/data-types/string.md)) — HTTP заголовок `UserAgent`. - `quota_key` ([String](../../sql-reference/data-types/string.md)) — «ключ квоты» из настроек [квот](quotas.md) (см. `keyed`). - `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ревизия ClickHouse. -- `ProfileEvents.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — Счетчики для изменения различных метрик для данного потока. Описание метрик можно получить из таблицы [system.events](#system_tables-events). +- `ProfileEvents.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — счетчики для изменения различных метрик для данного потока. Описание метрик можно получить из таблицы [system.events](#system_tables-events). - `ProfileEvents.Values` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — метрики для данного потока, перечисленные в столбце `ProfileEvents.Names`. **Пример** ``` sql - SELECT * FROM system.query_thread_log LIMIT 1 FORMAT Vertical + SELECT * FROM system.query_thread_log LIMIT 1 \G ``` ``` text Row 1: ────── -event_date: 2020-05-13 -event_time: 2020-05-13 14:02:28 -query_start_time: 2020-05-13 14:02:28 -query_duration_ms: 0 -read_rows: 1 -read_bytes: 1 -written_rows: 0 -written_bytes: 0 -memory_usage: 0 -peak_memory_usage: 0 -thread_name: QueryPipelineEx -thread_id: 28952 -master_thread_id: 28924 -query: SELECT 1 -is_initial_query: 1 -user: default -query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a -address: ::ffff:127.0.0.1 -port: 57720 -initial_user: default -initial_query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a -initial_address: ::ffff:127.0.0.1 -initial_port: 57720 -interface: 1 -os_user: bayonet -client_hostname: clickhouse.ru-central1.internal -client_name: ClickHouse client -client_revision: 54434 -client_version_major: 20 -client_version_minor: 4 -client_version_patch: 1 -http_method: 0 -http_user_agent: -quota_key: -revision: 54434 -ProfileEvents.Names: ['ContextLock','RealTimeMicroseconds','UserTimeMicroseconds','OSCPUWaitMicroseconds','OSCPUVirtualTimeMicroseconds'] -ProfileEvents.Values: [1,97,81,5,81] -... +event_date: 2020-09-11 +event_time: 2020-09-11 10:08:17 +event_time_microseconds: 2020-09-11 10:08:17.134042 +query_start_time: 2020-09-11 10:08:17 +query_start_time_microseconds: 2020-09-11 10:08:17.063150 +query_duration_ms: 70 +read_rows: 0 +read_bytes: 0 +written_rows: 1 +written_bytes: 12 +memory_usage: 4300844 +peak_memory_usage: 4300844 +thread_name: TCPHandler +thread_id: 638133 +master_thread_id: 638133 +query: INSERT INTO test1 VALUES +is_initial_query: 1 +user: default +query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef +address: ::ffff:127.0.0.1 +port: 33452 +initial_user: default +initial_query_id: 50a320fd-85a8-49b8-8761-98a86bcbacef +initial_address: ::ffff:127.0.0.1 +initial_port: 33452 +interface: 1 +os_user: bharatnc +client_hostname: tower +client_name: ClickHouse +client_revision: 54437 +client_version_major: 20 +client_version_minor: 7 +client_version_patch: 2 +http_method: 0 +http_user_agent: +quota_key: +revision: 54440 +ProfileEvents.Names: ['Query','InsertQuery','FileOpen','WriteBufferFromFileDescriptorWrite','WriteBufferFromFileDescriptorWriteBytes','ReadCompressedBytes','CompressedReadBufferBlocks','CompressedReadBufferBytes','IOBufferAllocs','IOBufferAllocBytes','FunctionExecute','CreatedWriteBufferOrdinary','DiskWriteElapsedMicroseconds','NetworkReceiveElapsedMicroseconds','NetworkSendElapsedMicroseconds','InsertedRows','InsertedBytes','SelectedRows','SelectedBytes','MergeTreeDataWriterRows','MergeTreeDataWriterUncompressedBytes','MergeTreeDataWriterCompressedBytes','MergeTreeDataWriterBlocks','MergeTreeDataWriterBlocksAlreadySorted','ContextLock','RWLockAcquiredReadLocks','RealTimeMicroseconds','UserTimeMicroseconds','SoftPageFaults','OSCPUVirtualTimeMicroseconds','OSWriteBytes','OSReadChars','OSWriteChars'] +ProfileEvents.Values: [1,1,11,11,591,148,3,71,29,6533808,1,11,72,18,47,1,12,1,12,1,12,189,1,1,10,2,70853,2748,49,2747,45056,422,1520] ``` **Смотрите также** diff --git a/docs/ru/operations/system-tables/text_log.md b/docs/ru/operations/system-tables/text_log.md index 01e34c914e6..141c3680c07 100644 --- a/docs/ru/operations/system-tables/text_log.md +++ b/docs/ru/operations/system-tables/text_log.md @@ -4,12 +4,13 @@ Столбцы: -- `event_date` (Date) — Дата создания записи. -- `event_time` (DateTime) — Время создания записи. -- `microseconds` (UInt32) — Время создания записи в микросекундах. -- `thread_name` (String) — Название потока, из которого была сделана запись. -- `thread_id` (UInt64) — Идентификатор потока ОС. -- `level` (Enum8) — Уровень логирования записи. Возможные значения: +- `event_date` (Date) — дата создания записи. +- `event_time` (DateTime) — время создания записи. +- `event_time_microseconds` (DateTime) — время создания записи с точностью до микросекунд. +- `microseconds` (UInt32) — время создания записи в микросекундах. +- `thread_name` (String) — название потока, из которого была сделана запись. +- `thread_id` (UInt64) — идентификатор потока ОС. +- `level` (Enum8) — уровень логирования записи. Возможные значения: - `1` или `'Fatal'`. - `2` или `'Critical'`. - `3` или `'Error'`. @@ -18,11 +19,35 @@ - `6` или `'Information'`. - `7` или `'Debug'`. - `8` или `'Trace'`. -- `query_id` (String) — Идентификатор запроса. -- `logger_name` (LowCardinality(String)) — Название логгера (`DDLWorker`). -- `message` (String) — Само тело записи. -- `revision` (UInt32) — Ревизия ClickHouse. -- `source_file` (LowCardinality(String)) — Исходный файл, из которого была сделана запись. -- `source_line` (UInt64) — Исходная строка, из которой была сделана запись. +- `query_id` (String) — идентификатор запроса. +- `logger_name` (LowCardinality(String)) — название логгера (`DDLWorker`). +- `message` (String) — само тело записи. +- `revision` (UInt32) — ревизия ClickHouse. +- `source_file` (LowCardinality(String)) — исходный файл, из которого была сделана запись. +- `source_line` (UInt64) — исходная строка, из которой была сделана запись. + +**Пример** + +``` sql +SELECT * FROM system.text_log LIMIT 1 \G +``` + +``` text +Row 1: +────── +event_date: 2020-09-10 +event_time: 2020-09-10 11:23:07 +event_time_microseconds: 2020-09-10 11:23:07.871397 +microseconds: 871397 +thread_name: clickhouse-serv +thread_id: 564917 +level: Information +query_id: +logger_name: DNSCacheUpdater +message: Update period 15 seconds +revision: 54440 +source_file: /ClickHouse/src/Interpreters/DNSCacheUpdater.cpp; void DB::DNSCacheUpdater::start() +source_line: 45 +``` [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/text_log) diff --git a/docs/ru/operations/system-tables/trace_log.md b/docs/ru/operations/system-tables/trace_log.md index 8bab8ff646c..3f0a16199d5 100644 --- a/docs/ru/operations/system-tables/trace_log.md +++ b/docs/ru/operations/system-tables/trace_log.md @@ -6,26 +6,28 @@ ClickHouse создает эту таблицу когда утсановлен Для анализа stack traces, используйте функции интроспекции `addressToLine`, `addressToSymbol` и `demangle`. -Колонки: +Столбцы: -- `event_date`([Date](../../sql-reference/data-types/date.md)) — Дата в момент снятия экземпляра стэка адресов вызова. +- `event_date`([Date](../../sql-reference/data-types/date.md)) — дата в момент снятия экземпляра стэка адресов вызова. -- `event_time`([DateTime](../../sql-reference/data-types/datetime.md)) — Дата и время в момент снятия экземпляра стэка адресов вызова. +- `event_time`([DateTime](../../sql-reference/data-types/datetime.md)) — дата и время в момент снятия экземпляра стэка адресов вызова. + +- `event_time_microseconds` ([DateTime](../../sql-reference/data-types/datetime.md)) — дата и время в момент снятия экземпляра стэка адресов вызова с точностью до микросекунд. - `revision`([UInt32](../../sql-reference/data-types/int-uint.md)) — ревизия сборки сервера ClickHouse. Во время соединения с сервером через `clickhouse-client`, вы видите строку похожую на `Connected to ClickHouse server version 19.18.1 revision 54429.`. Это поле содержит номер после `revision`, но не содержит строку после `version`. -- `timer_type`([Enum8](../../sql-reference/data-types/enum.md)) — Тип таймера: +- `timer_type`([Enum8](../../sql-reference/data-types/enum.md)) — тип таймера: - `Real` означает wall-clock время. - `CPU` означает относительное CPU время. -- `thread_number`([UInt32](../../sql-reference/data-types/int-uint.md)) — Идентификатор треда. +- `thread_number`([UInt32](../../sql-reference/data-types/int-uint.md)) — идентификатор треда. -- `query_id`([String](../../sql-reference/data-types/string.md)) — Идентификатор запроса который может быть использован для получения деталей о запросе из таблицы [query_log](query_log.md#system_tables-query_log) system table. +- `query_id`([String](../../sql-reference/data-types/string.md)) — идентификатор запроса который может быть использован для получения деталей о запросе из таблицы [query_log](query_log.md#system_tables-query_log) system table. -- `trace`([Array(UInt64)](../../sql-reference/data-types/array.md)) — Трассировка стека адресов вызова в момент семплирования. Каждый элемент массива это адрес виртуальной памяти внутри процесса сервера ClickHouse. +- `trace`([Array(UInt64)](../../sql-reference/data-types/array.md)) — трассировка стека адресов вызова в момент семплирования. Каждый элемент массива — это адрес виртуальной памяти внутри процесса сервера ClickHouse. **Пример** @@ -36,13 +38,16 @@ SELECT * FROM system.trace_log LIMIT 1 \G ``` text Row 1: ────── -event_date: 2019-11-15 -event_time: 2019-11-15 15:09:38 -revision: 54428 -timer_type: Real -thread_number: 48 -query_id: acc4d61f-5bd1-4a3e-bc91-2180be37c915 -trace: [94222141367858,94222152240175,94222152325351,94222152329944,94222152330796,94222151449980,94222144088167,94222151682763,94222144088167,94222151682763,94222144088167,94222144058283,94222144059248,94222091840750,94222091842302,94222091831228,94222189631488,140509950166747,140509942945935] +event_date: 2020-09-10 +event_time: 2020-09-10 11:23:09 +event_time_microseconds: 2020-09-10 11:23:09.872924 +timestamp_ns: 1599762189872924510 +revision: 54440 +trace_type: Memory +thread_id: 564963 +query_id: +trace: [371912858,371912789,371798468,371799717,371801313,371790250,624462773,566365041,566440261,566445834,566460071,566459914,566459842,566459580,566459469,566459389,566459341,566455774,371993941,371988245,372158848,372187428,372187309,372187093,372185478,140222123165193,140222122205443] +size: 5244400 ``` [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/trace_log) diff --git a/docs/ru/sql-reference/statements/alter/sample-by.md b/docs/ru/sql-reference/statements/alter/sample-by.md new file mode 100644 index 00000000000..2235e345a45 --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/sample-by.md @@ -0,0 +1,20 @@ +--- +toc_priority: 41 +toc_title: SAMPLE BY +--- + +# Manipulating Sampling-Key Expressions {#manipulations-with-sampling-key-expressions} + +Синтаксис: + +``` sql +ALTER TABLE [db].name [ON CLUSTER cluster] MODIFY SAMPLE BY new_expression +``` + +Команда меняет [ключ сэмплирования](../../../engines/table-engines/mergetree-family/mergetree.md) таблицы на `new_expression` (выражение или ряд выражений). + +Эта команда является упрощенной в том смысле, что она изменяет только метаданные. Первичный ключ должен содержать новый ключ сэмплирования. + +!!! note "Note" + Это работает только для таблиц в семействе [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) (включая +[реплицируемые](../../../engines/table-engines/mergetree-family/replication.md) таблицы). \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/drop.md b/docs/ru/sql-reference/statements/drop.md index 22e553cfdac..3a6ac22b071 100644 --- a/docs/ru/sql-reference/statements/drop.md +++ b/docs/ru/sql-reference/statements/drop.md @@ -5,106 +5,96 @@ toc_title: DROP # DROP {#drop} -Удаляет существующий объект. -Если указано `IF EXISTS` - не выдавать ошибку, если объекта не существует. +Удаляет существующий объект. Если указано `IF EXISTS` - не выдавать ошибку, если объекта не существует. ## DROP DATABASE {#drop-database} +Удаляет все таблицы в базе данных `db`, затем удаляет саму базу данных `db`. + +Синтаксис: + ``` sql DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] ``` -Удаляет все таблицы в базе данных db, затем удаляет саму базу данных db. - - ## DROP TABLE {#drop-table} +Удаляет таблицу. + +Синтаксис: + ``` sql DROP [TEMPORARY] TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` -Удаляет таблицу. - - ## DROP DICTIONARY {#drop-dictionary} +Удаляет словарь. + +Синтаксис: + ``` sql DROP DICTIONARY [IF EXISTS] [db.]name ``` -Удаляет словарь. - - ## DROP USER {#drop-user-statement} Удаляет пользователя. -### Синтаксис {#drop-user-syntax} +Синтаксис: ```sql DROP USER [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` - ## DROP ROLE {#drop-role-statement} -Удаляет роль. +Удаляет роль. При удалении роль отзывается у всех объектов системы доступа, которым она присвоена. -При удалении роль отзывается у всех объектов системы доступа, которым она присвоена. - -### Синтаксис {#drop-role-syntax} +Синтаксис: ```sql DROP ROLE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` - ## DROP ROW POLICY {#drop-row-policy-statement} -Удаляет политику доступа к строкам. +Удаляет политику доступа к строкам. При удалении политика отзывается у всех объектов системы доступа, которым она присвоена. -При удалении политика отзывается у всех объектов системы доступа, которым она присвоена. - -### Синтаксис {#drop-row-policy-syntax} +Синтаксис: ``` sql DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER cluster_name] ``` - ## DROP QUOTA {#drop-quota-statement} -Удаляет квоту. +Удаляет квоту. При удалении квота отзывается у всех объектов системы доступа, которым она присвоена. -При удалении квота отзывается у всех объектов системы доступа, которым она присвоена. - -### Синтаксис {#drop-quota-syntax} +Синтаксис: ``` sql DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` - ## DROP SETTINGS PROFILE {#drop-settings-profile-statement} -Удаляет профиль настроек. +Удаляет профиль настроек. При удалении профиль отзывается у всех объектов системы доступа, которым он присвоен. -При удалении профиль отзывается у всех объектов системы доступа, которым он присвоен. - -### Синтаксис {#drop-settings-profile-syntax} +Синтаксис: ``` sql DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] ``` - ## DROP VIEW {#drop-view} +Удаляет представление. Представления могут быть удалены и командой `DROP TABLE`, но команда `DROP VIEW` проверяет, что `[db.]name` является представлением. + +Синтаксис: + ``` sql DROP VIEW [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` -Удаляет представление. Представления могут быть удалены и командой `DROP TABLE`, но команда `DROP VIEW` проверяет, что `[db.]name` является представлением. - - -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/drop/) +[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/drop/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md index 8eea84ac594..69137095c4d 100644 --- a/docs/ru/sql-reference/statements/grant.md +++ b/docs/ru/sql-reference/statements/grant.md @@ -79,6 +79,7 @@ GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION - `ALTER RENAME COLUMN` - `ALTER INDEX` - `ALTER ORDER BY` + - `ALTER SAMPLE BY` - `ALTER ADD INDEX` - `ALTER DROP INDEX` - `ALTER MATERIALIZE INDEX` @@ -264,6 +265,7 @@ GRANT INSERT(x,y) ON db.table TO john - `ALTER RENAME COLUMN`. Уровень: `COLUMN`. Алиасы: `RENAME COLUMN` - `ALTER INDEX`. Уровень: `GROUP`. Алиасы: `INDEX` - `ALTER ORDER BY`. Уровень: `TABLE`. Алиасы: `ALTER MODIFY ORDER BY`, `MODIFY ORDER BY` + - `ALTER SAMPLE BY`. Уровень: `TABLE`. Алиасы: `ALTER MODIFY SAMPLE BY`, `MODIFY SAMPLE BY` - `ALTER ADD INDEX`. Уровень: `TABLE`. Алиасы: `ADD INDEX` - `ALTER DROP INDEX`. Уровень: `TABLE`. Алиасы: `DROP INDEX` - `ALTER MATERIALIZE INDEX`. Уровень: `TABLE`. Алиасы: `MATERIALIZE INDEX` From 3151de1b4e7f2864362c5fe397b6fb17010ffce9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 26 Oct 2020 13:26:52 +0300 Subject: [PATCH 205/325] Update ColumnConst include. --- src/Columns/ColumnConst.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Columns/ColumnConst.cpp b/src/Columns/ColumnConst.cpp index b7fb22aeb0e..550a44a23a2 100644 --- a/src/Columns/ColumnConst.cpp +++ b/src/Columns/ColumnConst.cpp @@ -6,6 +6,8 @@ #include #include +#include + #if defined(MEMORY_SANITIZER) #include #endif From 9ec78855cded9eb5a84e7fed300ff7f8a4b497c2 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Mon, 26 Oct 2020 13:29:30 +0300 Subject: [PATCH 206/325] DOCSUP-2806: Add meta header in RU (#15801) * DOCSUP-2806: Add meta intro. * DOCSUP-2806: Update meta intro. * DOCSUP-2806: Fix meta. * DOCSUP-2806: Add quotes for meta headers. * DOCSUP-2806: Remove quotes from meta headers. * DOCSUP-2806: Add meta headers. * DOCSUP-2806: Fix quotes in meta headers. * DOCSUP-2806: Update meta headers. * DOCSUP-2806: Fix link to nowhere in EN. * DOCSUP-2806: Fix link (settings to tune) * DOCSUP-2806: Fix links. * DOCSUP-2806:Fix links EN * DOCSUP-2806: Fix build errors. * DOCSUP-2806: Fix meta intro. * DOCSUP-2806: Fix toc_priority in examples datasets TOC. * DOCSUP-2806: Fix items order in toc. * DOCSUP-2806: Fix order in toc. * DOCSUP-2806: Fix toc order. * DOCSUP-2806: Fix order in toc. * DOCSUP-2806: Fix toc index in create * DOCSUP-2806: Fix toc order in create. Co-authored-by: romanzhukov Co-authored-by: alexey-milovidov --- .../example-datasets/amplab-benchmark.md | 2 +- docs/en/getting-started/example-datasets/criteo.md | 2 +- docs/en/getting-started/example-datasets/index.md | 4 ++-- docs/en/getting-started/example-datasets/metrica.md | 2 +- docs/en/getting-started/example-datasets/nyc-taxi.md | 2 +- docs/en/getting-started/example-datasets/ontime.md | 2 +- .../getting-started/example-datasets/star-schema.md | 2 +- docs/en/getting-started/example-datasets/wikistat.md | 2 +- docs/en/sql-reference/functions/encoding-functions.md | 2 +- docs/en/sql-reference/operators/index.md | 2 +- docs/en/sql-reference/statements/alter/index.md | 2 +- docs/en/sql-reference/statements/attach.md | 2 +- docs/en/sql-reference/statements/check-table.md | 2 +- docs/en/sql-reference/statements/create/database.md | 2 +- docs/en/sql-reference/statements/create/dictionary.md | 2 +- docs/en/sql-reference/statements/create/index.md | 2 +- docs/en/sql-reference/statements/create/quota.md | 2 +- docs/en/sql-reference/statements/create/role.md | 2 +- docs/en/sql-reference/statements/create/row-policy.md | 2 +- .../statements/create/settings-profile.md | 2 +- docs/en/sql-reference/statements/create/table.md | 2 +- docs/en/sql-reference/statements/create/user.md | 2 +- docs/en/sql-reference/statements/create/view.md | 2 +- docs/en/sql-reference/statements/describe-table.md | 2 +- docs/en/sql-reference/statements/detach.md | 2 +- docs/en/sql-reference/statements/drop.md | 2 +- docs/en/sql-reference/statements/exists.md | 2 +- docs/en/sql-reference/statements/grant.md | 2 +- docs/en/sql-reference/statements/insert-into.md | 2 +- docs/en/sql-reference/statements/kill.md | 2 +- docs/en/sql-reference/statements/optimize.md | 2 +- docs/en/sql-reference/statements/rename.md | 2 +- docs/en/sql-reference/statements/revoke.md | 2 +- docs/en/sql-reference/statements/select/index.md | 2 +- docs/en/sql-reference/statements/set-role.md | 2 +- docs/en/sql-reference/statements/set.md | 2 +- docs/en/sql-reference/statements/show.md | 2 +- docs/en/sql-reference/statements/system.md | 2 +- docs/en/sql-reference/statements/truncate.md | 2 +- docs/en/sql-reference/statements/use.md | 2 +- docs/ru/commercial/cloud.md | 5 +++++ docs/ru/development/architecture.md | 5 +++++ docs/ru/development/browse-code.md | 6 ++++++ docs/ru/development/contrib.md | 6 ++++++ docs/ru/development/developer-instruction.md | 5 +++++ docs/ru/development/style.md | 6 ++++++ docs/ru/engines/database-engines/index.md | 7 +++++++ docs/ru/engines/database-engines/lazy.md | 5 +++++ docs/ru/engines/database-engines/mysql.md | 5 +++++ docs/ru/engines/index.md | 6 +++--- docs/ru/engines/table-engines/index.md | 7 +++++++ docs/ru/engines/table-engines/integrations/hdfs.md | 5 +++++ docs/ru/engines/table-engines/integrations/index.md | 2 +- docs/ru/engines/table-engines/integrations/jdbc.md | 5 +++++ docs/ru/engines/table-engines/integrations/kafka.md | 5 +++++ docs/ru/engines/table-engines/integrations/mysql.md | 5 +++++ docs/ru/engines/table-engines/integrations/odbc.md | 5 +++++ docs/ru/engines/table-engines/log-family/index.md | 4 ++-- docs/ru/engines/table-engines/log-family/log.md | 5 +++++ docs/ru/engines/table-engines/log-family/stripelog.md | 5 +++++ docs/ru/engines/table-engines/log-family/tinylog.md | 5 +++++ .../mergetree-family/aggregatingmergetree.md | 5 +++++ .../mergetree-family/collapsingmergetree.md | 5 +++++ .../mergetree-family/custom-partitioning-key.md | 6 ++++++ .../mergetree-family/graphitemergetree.md | 5 +++++ .../engines/table-engines/mergetree-family/index.md | 3 +-- .../mergetree-family/replacingmergetree.md | 5 +++++ .../table-engines/mergetree-family/replication.md | 5 +++++ .../mergetree-family/summingmergetree.md | 5 +++++ .../mergetree-family/versionedcollapsingmergetree.md | 5 +++++ docs/ru/engines/table-engines/special/buffer.md | 5 +++++ docs/ru/engines/table-engines/special/dictionary.md | 5 +++++ docs/ru/engines/table-engines/special/distributed.md | 5 +++++ .../ru/engines/table-engines/special/external-data.md | 5 +++++ docs/ru/engines/table-engines/special/file.md | 5 +++++ docs/ru/engines/table-engines/special/index.md | 2 +- docs/ru/engines/table-engines/special/join.md | 5 +++++ .../engines/table-engines/special/materializedview.md | 5 +++++ docs/ru/engines/table-engines/special/memory.md | 5 +++++ docs/ru/engines/table-engines/special/merge.md | 5 +++++ docs/ru/engines/table-engines/special/null.md | 5 ++++- docs/ru/engines/table-engines/special/set.md | 5 +++++ docs/ru/engines/table-engines/special/url.md | 5 +++++ docs/ru/engines/table-engines/special/view.md | 5 +++++ docs/ru/faq/index.md | 2 +- .../example-datasets/amplab-benchmark.md | 5 +++++ docs/ru/getting-started/example-datasets/criteo.md | 5 +++++ docs/ru/getting-started/example-datasets/index.md | 7 +++---- docs/ru/getting-started/example-datasets/metrica.md | 5 +++++ docs/ru/getting-started/example-datasets/nyc-taxi.md | 5 +++++ docs/ru/getting-started/example-datasets/ontime.md | 5 +++++ .../getting-started/example-datasets/star-schema.md | 5 +++++ docs/ru/getting-started/example-datasets/wikistat.md | 5 +++++ docs/ru/getting-started/index.md | 3 +-- docs/ru/getting-started/install.md | 5 +++++ docs/ru/getting-started/playground.md | 5 +++++ docs/ru/guides/apply-catboost-model.md | 5 +++++ docs/ru/interfaces/cli.md | 5 +++++ docs/ru/interfaces/cpp.md | 5 +++++ docs/ru/interfaces/formats.md | 5 +++++ docs/ru/interfaces/http.md | 5 +++++ docs/ru/interfaces/jdbc.md | 5 +++++ docs/ru/interfaces/mysql.md | 5 +++++ docs/ru/interfaces/odbc.md | 6 ++++++ docs/ru/interfaces/tcp.md | 5 +++++ docs/ru/interfaces/third-party/client-libraries.md | 5 +++++ docs/ru/interfaces/third-party/gui.md | 6 ++++++ docs/ru/interfaces/third-party/index.md | 3 +-- docs/ru/interfaces/third-party/integrations.md | 5 +++++ docs/ru/interfaces/third-party/proxy.md | 5 +++++ docs/ru/introduction/distinctive-features.md | 5 +++++ docs/ru/introduction/history.md | 6 ++++++ docs/ru/introduction/performance.md | 5 +++++ docs/ru/operations/access-rights.md | 5 +++++ docs/ru/operations/backup.md | 5 +++++ docs/ru/operations/configuration-files.md | 6 ++++++ docs/ru/operations/index.md | 2 +- docs/ru/operations/monitoring.md | 5 +++++ docs/ru/operations/quotas.md | 5 +++++ docs/ru/operations/requirements.md | 5 +++++ .../server-configuration-parameters/settings.md | 5 +++++ .../ru/operations/settings/constraints-on-settings.md | 5 +++++ docs/ru/operations/settings/index.md | 4 ++-- .../ru/operations/settings/permissions-for-queries.md | 5 +++++ docs/ru/operations/settings/query-complexity.md | 5 +++++ docs/ru/operations/settings/settings-profiles.md | 5 +++++ docs/ru/operations/settings/settings-users.md | 5 +++++ docs/ru/operations/settings/settings.md | 2 +- docs/ru/operations/system-tables/index.md | 2 +- docs/ru/operations/tips.md | 5 +++++ docs/ru/operations/troubleshooting.md | 5 +++++ docs/ru/operations/update.md | 5 +++++ docs/ru/operations/utilities/clickhouse-copier.md | 5 +++++ docs/ru/operations/utilities/clickhouse-local.md | 5 +++++ docs/ru/operations/utilities/index.md | 6 ++++++ .../sql-reference/aggregate-functions/combinators.md | 6 ++++++ .../aggregate-functions/parametric-functions.md | 5 +++++ docs/ru/sql-reference/data-types/boolean.md | 5 +++++ docs/ru/sql-reference/data-types/date.md | 5 +++++ docs/ru/sql-reference/data-types/datetime.md | 5 +++++ docs/ru/sql-reference/data-types/datetime64.md | 5 +++++ docs/ru/sql-reference/data-types/decimal.md | 5 +++++ docs/ru/sql-reference/data-types/domains/index.md | 4 ++-- docs/ru/sql-reference/data-types/domains/ipv4.md | 5 +++++ docs/ru/sql-reference/data-types/domains/ipv6.md | 5 +++++ docs/ru/sql-reference/data-types/enum.md | 5 +++++ docs/ru/sql-reference/data-types/fixedstring.md | 5 +++++ docs/ru/sql-reference/data-types/float.md | 5 +++++ docs/ru/sql-reference/data-types/int-uint.md | 5 +++++ .../data-types/nested-data-structures/index.md | 7 +++++++ .../data-types/special-data-types/expression.md | 5 +++++ .../data-types/special-data-types/index.md | 7 +++++++ .../data-types/special-data-types/interval.md | 5 +++++ .../data-types/special-data-types/nothing.md | 5 +++++ .../data-types/special-data-types/set.md | 5 +++++ docs/ru/sql-reference/data-types/string.md | 5 +++++ docs/ru/sql-reference/data-types/uuid.md | 6 ++++++ .../external-dicts-dict-hierarchical.md | 5 +++++ .../external-dicts-dict-layout.md | 5 +++++ .../external-dicts-dict-lifetime.md | 5 +++++ .../external-dicts-dict-sources.md | 5 +++++ .../external-dicts-dict-structure.md | 5 +++++ .../external-dictionaries/external-dicts-dict.md | 5 +++++ .../external-dictionaries/external-dicts.md | 6 ++++++ .../dictionaries/external-dictionaries/index.md | 2 +- docs/ru/sql-reference/dictionaries/index.md | 6 ++++++ docs/ru/sql-reference/dictionaries/internal-dicts.md | 5 +++++ docs/ru/sql-reference/distributed-ddl.md | 2 +- .../sql-reference/functions/arithmetic-functions.md | 5 +++++ docs/ru/sql-reference/functions/array-functions.md | 5 +++++ docs/ru/sql-reference/functions/array-join.md | 5 +++++ docs/ru/sql-reference/functions/bit-functions.md | 5 +++++ docs/ru/sql-reference/functions/bitmap-functions.md | 7 ++++++- .../sql-reference/functions/comparison-functions.md | 5 +++++ .../sql-reference/functions/conditional-functions.md | 5 +++++ .../ru/sql-reference/functions/date-time-functions.md | 5 +++++ docs/ru/sql-reference/functions/encoding-functions.md | 5 +++++ docs/ru/sql-reference/functions/ext-dict-functions.md | 5 +++++ .../ru/sql-reference/functions/functions-for-nulls.md | 5 +++++ docs/ru/sql-reference/functions/geo/coordinates.md | 3 ++- docs/ru/sql-reference/functions/geo/geohash.md | 2 +- docs/ru/sql-reference/functions/geo/h3.md | 2 +- docs/ru/sql-reference/functions/geo/index.md | 2 +- docs/ru/sql-reference/functions/hash-functions.md | 5 +++++ docs/ru/sql-reference/functions/in-functions.md | 5 +++++ docs/ru/sql-reference/functions/index.md | 6 ++++++ docs/ru/sql-reference/functions/introspection.md | 5 +++++ .../sql-reference/functions/ip-address-functions.md | 5 +++++ docs/ru/sql-reference/functions/json-functions.md | 5 +++++ docs/ru/sql-reference/functions/logical-functions.md | 5 +++++ .../functions/machine-learning-functions.md | 5 +++++ docs/ru/sql-reference/functions/math-functions.md | 5 +++++ docs/ru/sql-reference/functions/other-functions.md | 11 ++++++++--- docs/ru/sql-reference/functions/random-functions.md | 5 +++++ docs/ru/sql-reference/functions/rounding-functions.md | 5 +++++ .../functions/splitting-merging-functions.md | 5 +++++ docs/ru/sql-reference/functions/string-functions.md | 5 +++++ .../functions/string-replace-functions.md | 5 +++++ .../functions/string-search-functions.md | 5 +++++ .../functions/type-conversion-functions.md | 5 +++++ docs/ru/sql-reference/functions/url-functions.md | 5 +++++ docs/ru/sql-reference/functions/uuid-functions.md | 5 +++++ docs/ru/sql-reference/functions/ym-dict-functions.md | 5 +++++ docs/ru/sql-reference/operators/index.md | 5 +++++ docs/ru/sql-reference/statements/alter/column.md | 2 +- docs/ru/sql-reference/statements/alter/constraint.md | 2 +- docs/ru/sql-reference/statements/alter/index.md | 2 +- docs/ru/sql-reference/statements/alter/index/index.md | 2 +- docs/ru/sql-reference/statements/attach.md | 2 +- docs/ru/sql-reference/statements/check-table.md | 2 +- docs/ru/sql-reference/statements/create/database.md | 4 ++-- docs/ru/sql-reference/statements/create/dictionary.md | 4 ++-- docs/ru/sql-reference/statements/create/index.md | 4 ++-- docs/ru/sql-reference/statements/create/quota.md | 4 ++-- docs/ru/sql-reference/statements/create/role.md | 4 ++-- docs/ru/sql-reference/statements/create/row-policy.md | 4 ++-- .../statements/create/settings-profile.md | 4 ++-- docs/ru/sql-reference/statements/create/table.md | 4 ++-- docs/ru/sql-reference/statements/create/user.md | 4 ++-- docs/ru/sql-reference/statements/create/view.md | 4 ++-- docs/ru/sql-reference/statements/describe-table.md | 2 +- docs/ru/sql-reference/statements/detach.md | 2 +- docs/ru/sql-reference/statements/drop.md | 2 +- docs/ru/sql-reference/statements/exists.md | 2 +- docs/ru/sql-reference/statements/grant.md | 5 +++++ docs/ru/sql-reference/statements/insert-into.md | 2 +- docs/ru/sql-reference/statements/kill.md | 2 +- docs/ru/sql-reference/statements/optimize.md | 2 +- docs/ru/sql-reference/statements/rename.md | 2 +- docs/ru/sql-reference/statements/revoke.md | 5 +++++ docs/ru/sql-reference/statements/select/array-join.md | 4 ++++ docs/ru/sql-reference/statements/select/distinct.md | 4 ++++ docs/ru/sql-reference/statements/select/format.md | 4 ++++ docs/ru/sql-reference/statements/select/from.md | 4 ++++ docs/ru/sql-reference/statements/select/group-by.md | 4 ++++ docs/ru/sql-reference/statements/select/having.md | 4 ++++ docs/ru/sql-reference/statements/select/index.md | 6 ++++-- .../sql-reference/statements/select/into-outfile.md | 4 ++++ docs/ru/sql-reference/statements/select/join.md | 4 ++++ docs/ru/sql-reference/statements/select/limit-by.md | 4 ++++ docs/ru/sql-reference/statements/select/prewhere.md | 4 ++++ docs/ru/sql-reference/statements/select/sample.md | 4 ++++ docs/ru/sql-reference/statements/select/union-all.md | 4 ++++ docs/ru/sql-reference/statements/select/where.md | 4 ++++ docs/ru/sql-reference/statements/select/with.md | 4 ++++ docs/ru/sql-reference/statements/set-role.md | 2 +- docs/ru/sql-reference/statements/set.md | 2 +- docs/ru/sql-reference/statements/show.md | 5 +++++ docs/ru/sql-reference/statements/system.md | 5 +++++ docs/ru/sql-reference/statements/truncate.md | 2 +- docs/ru/sql-reference/statements/use.md | 2 +- docs/ru/sql-reference/syntax.md | 5 +++++ docs/ru/sql-reference/table-functions/file.md | 5 +++++ docs/ru/sql-reference/table-functions/generate.md | 5 +++++ docs/ru/sql-reference/table-functions/hdfs.md | 5 +++++ docs/ru/sql-reference/table-functions/input.md | 5 +++++ docs/ru/sql-reference/table-functions/jdbc.md | 5 +++++ docs/ru/sql-reference/table-functions/merge.md | 5 +++++ docs/ru/sql-reference/table-functions/mysql.md | 5 +++++ docs/ru/sql-reference/table-functions/numbers.md | 5 +++++ docs/ru/sql-reference/table-functions/odbc.md | 5 +++++ docs/ru/sql-reference/table-functions/remote.md | 5 +++++ docs/ru/sql-reference/table-functions/url.md | 5 +++++ docs/ru/whats-new/index.md | 2 +- docs/ru/whats-new/security-changelog.md | 5 +++++ 265 files changed, 998 insertions(+), 118 deletions(-) diff --git a/docs/en/getting-started/example-datasets/amplab-benchmark.md b/docs/en/getting-started/example-datasets/amplab-benchmark.md index 1b740f6194c..27ddfd27f78 100644 --- a/docs/en/getting-started/example-datasets/amplab-benchmark.md +++ b/docs/en/getting-started/example-datasets/amplab-benchmark.md @@ -1,5 +1,5 @@ --- -toc_priority: 17 +toc_priority: 19 toc_title: AMPLab Big Data Benchmark --- diff --git a/docs/en/getting-started/example-datasets/criteo.md b/docs/en/getting-started/example-datasets/criteo.md index a38a1b14210..261d1606fa4 100644 --- a/docs/en/getting-started/example-datasets/criteo.md +++ b/docs/en/getting-started/example-datasets/criteo.md @@ -1,5 +1,5 @@ --- -toc_priority: 19 +toc_priority: 18 toc_title: Terabyte Click Logs from Criteo --- diff --git a/docs/en/getting-started/example-datasets/index.md b/docs/en/getting-started/example-datasets/index.md index eedf4196b23..35ac90f9beb 100644 --- a/docs/en/getting-started/example-datasets/index.md +++ b/docs/en/getting-started/example-datasets/index.md @@ -1,6 +1,6 @@ --- toc_folder_title: Example Datasets -toc_priority: 15 +toc_priority: 14 toc_title: Introduction --- @@ -18,4 +18,4 @@ The list of documented datasets: - [New York Taxi Data](../../getting-started/example-datasets/nyc-taxi.md) - [OnTime](../../getting-started/example-datasets/ontime.md) -[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets) +[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets) \ No newline at end of file diff --git a/docs/en/getting-started/example-datasets/metrica.md b/docs/en/getting-started/example-datasets/metrica.md index 4131dca78fe..b036973b255 100644 --- a/docs/en/getting-started/example-datasets/metrica.md +++ b/docs/en/getting-started/example-datasets/metrica.md @@ -1,5 +1,5 @@ --- -toc_priority: 14 +toc_priority: 15 toc_title: Yandex.Metrica Data --- diff --git a/docs/en/getting-started/example-datasets/nyc-taxi.md b/docs/en/getting-started/example-datasets/nyc-taxi.md index a180f19be8a..9b9a12ba724 100644 --- a/docs/en/getting-started/example-datasets/nyc-taxi.md +++ b/docs/en/getting-started/example-datasets/nyc-taxi.md @@ -1,5 +1,5 @@ --- -toc_priority: 16 +toc_priority: 20 toc_title: New York Taxi Data --- diff --git a/docs/en/getting-started/example-datasets/ontime.md b/docs/en/getting-started/example-datasets/ontime.md index 91b6913e223..c2c8d5d930e 100644 --- a/docs/en/getting-started/example-datasets/ontime.md +++ b/docs/en/getting-started/example-datasets/ontime.md @@ -1,5 +1,5 @@ --- -toc_priority: 15 +toc_priority: 21 toc_title: OnTime --- diff --git a/docs/en/getting-started/example-datasets/star-schema.md b/docs/en/getting-started/example-datasets/star-schema.md index 7940807ee49..8a019e7ded3 100644 --- a/docs/en/getting-started/example-datasets/star-schema.md +++ b/docs/en/getting-started/example-datasets/star-schema.md @@ -1,5 +1,5 @@ --- -toc_priority: 20 +toc_priority: 16 toc_title: Star Schema Benchmark --- diff --git a/docs/en/getting-started/example-datasets/wikistat.md b/docs/en/getting-started/example-datasets/wikistat.md index 874af8c9423..619711582f4 100644 --- a/docs/en/getting-started/example-datasets/wikistat.md +++ b/docs/en/getting-started/example-datasets/wikistat.md @@ -1,5 +1,5 @@ --- -toc_priority: 18 +toc_priority: 17 toc_title: WikiStat --- diff --git a/docs/en/sql-reference/functions/encoding-functions.md b/docs/en/sql-reference/functions/encoding-functions.md index 44008fbbcbe..bc3f5ca4345 100644 --- a/docs/en/sql-reference/functions/encoding-functions.md +++ b/docs/en/sql-reference/functions/encoding-functions.md @@ -6,7 +6,7 @@ toc_title: Encoding # Encoding Functions {#encoding-functions} ## char {#char} - + Returns the string with the length as the number of passed arguments and each byte has the value of corresponding argument. Accepts multiple arguments of numeric types. If the value of argument is out of range of UInt8 data type, it is converted to UInt8 with possible rounding and overflow. **Syntax** diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 3fe3384fffc..e5554b58e4a 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -1,5 +1,5 @@ --- -toc_priority: 37 +toc_priority: 38 toc_title: Operators --- diff --git a/docs/en/sql-reference/statements/alter/index.md b/docs/en/sql-reference/statements/alter/index.md index 60b7375f76d..30603122096 100644 --- a/docs/en/sql-reference/statements/alter/index.md +++ b/docs/en/sql-reference/statements/alter/index.md @@ -1,5 +1,5 @@ --- -toc_priority: 36 +toc_priority: 35 toc_title: ALTER --- diff --git a/docs/en/sql-reference/statements/attach.md b/docs/en/sql-reference/statements/attach.md index 6260e15cfa9..035441ef5f1 100644 --- a/docs/en/sql-reference/statements/attach.md +++ b/docs/en/sql-reference/statements/attach.md @@ -1,5 +1,5 @@ --- -toc_priority: 42 +toc_priority: 40 toc_title: ATTACH --- diff --git a/docs/en/sql-reference/statements/check-table.md b/docs/en/sql-reference/statements/check-table.md index b55955eeb40..450447acaf8 100644 --- a/docs/en/sql-reference/statements/check-table.md +++ b/docs/en/sql-reference/statements/check-table.md @@ -1,5 +1,5 @@ --- -toc_priority: 43 +toc_priority: 41 toc_title: CHECK --- diff --git a/docs/en/sql-reference/statements/create/database.md b/docs/en/sql-reference/statements/create/database.md index e874672471d..bdb31d44b0b 100644 --- a/docs/en/sql-reference/statements/create/database.md +++ b/docs/en/sql-reference/statements/create/database.md @@ -1,5 +1,5 @@ --- -toc_priority: 1 +toc_priority: 35 toc_title: DATABASE --- diff --git a/docs/en/sql-reference/statements/create/dictionary.md b/docs/en/sql-reference/statements/create/dictionary.md index 5313c59396e..b1098c54703 100644 --- a/docs/en/sql-reference/statements/create/dictionary.md +++ b/docs/en/sql-reference/statements/create/dictionary.md @@ -1,5 +1,5 @@ --- -toc_priority: 4 +toc_priority: 38 toc_title: DICTIONARY --- diff --git a/docs/en/sql-reference/statements/create/index.md b/docs/en/sql-reference/statements/create/index.md index 71ace2b664c..902a4348bac 100644 --- a/docs/en/sql-reference/statements/create/index.md +++ b/docs/en/sql-reference/statements/create/index.md @@ -1,6 +1,6 @@ --- toc_folder_title: CREATE -toc_priority: 35 +toc_priority: 34 toc_title: Overview --- diff --git a/docs/en/sql-reference/statements/create/quota.md b/docs/en/sql-reference/statements/create/quota.md index 7919dc0f6db..29752050b69 100644 --- a/docs/en/sql-reference/statements/create/quota.md +++ b/docs/en/sql-reference/statements/create/quota.md @@ -1,5 +1,5 @@ --- -toc_priority: 8 +toc_priority: 42 toc_title: QUOTA --- diff --git a/docs/en/sql-reference/statements/create/role.md b/docs/en/sql-reference/statements/create/role.md index 8a682875d63..19db2b79b61 100644 --- a/docs/en/sql-reference/statements/create/role.md +++ b/docs/en/sql-reference/statements/create/role.md @@ -1,5 +1,5 @@ --- -toc_priority: 6 +toc_priority: 40 toc_title: ROLE --- diff --git a/docs/en/sql-reference/statements/create/row-policy.md b/docs/en/sql-reference/statements/create/row-policy.md index b1e7c8e9006..9f8b6b87d3b 100644 --- a/docs/en/sql-reference/statements/create/row-policy.md +++ b/docs/en/sql-reference/statements/create/row-policy.md @@ -1,5 +1,5 @@ --- -toc_priority: 7 +toc_priority: 41 toc_title: ROW POLICY --- diff --git a/docs/en/sql-reference/statements/create/settings-profile.md b/docs/en/sql-reference/statements/create/settings-profile.md index 6fcd1d4e840..196c336bb4d 100644 --- a/docs/en/sql-reference/statements/create/settings-profile.md +++ b/docs/en/sql-reference/statements/create/settings-profile.md @@ -1,5 +1,5 @@ --- -toc_priority: 9 +toc_priority: 43 toc_title: SETTINGS PROFILE --- diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 9aecc6c07f7..35780856ef2 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -1,5 +1,5 @@ --- -toc_priority: 2 +toc_priority: 36 toc_title: TABLE --- diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index 98c29363102..273882fd639 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -1,5 +1,5 @@ --- -toc_priority: 5 +toc_priority: 39 toc_title: USER --- diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index dc1200e5979..17a6c26c084 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -1,5 +1,5 @@ --- -toc_priority: 3 +toc_priority: 37 toc_title: VIEW --- diff --git a/docs/en/sql-reference/statements/describe-table.md b/docs/en/sql-reference/statements/describe-table.md index b7ffec1db07..bc197bf0f72 100644 --- a/docs/en/sql-reference/statements/describe-table.md +++ b/docs/en/sql-reference/statements/describe-table.md @@ -1,5 +1,5 @@ --- -toc_priority: 44 +toc_priority: 42 toc_title: DESCRIBE --- diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index 0d236c132df..62a7c0cc1e0 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -1,5 +1,5 @@ --- -toc_priority: 45 +toc_priority: 43 toc_title: DETACH --- diff --git a/docs/en/sql-reference/statements/drop.md b/docs/en/sql-reference/statements/drop.md index 07761b6b821..4317a20419e 100644 --- a/docs/en/sql-reference/statements/drop.md +++ b/docs/en/sql-reference/statements/drop.md @@ -1,5 +1,5 @@ --- -toc_priority: 46 +toc_priority: 44 toc_title: DROP --- diff --git a/docs/en/sql-reference/statements/exists.md b/docs/en/sql-reference/statements/exists.md index 2e8072125c7..3b0f4b66343 100644 --- a/docs/en/sql-reference/statements/exists.md +++ b/docs/en/sql-reference/statements/exists.md @@ -1,5 +1,5 @@ --- -toc_priority: 47 +toc_priority: 45 toc_title: EXISTS --- diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index c3e18ca3c75..f3829de2fbb 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -1,5 +1,5 @@ --- -toc_priority: 39 +toc_priority: 38 toc_title: GRANT --- diff --git a/docs/en/sql-reference/statements/insert-into.md b/docs/en/sql-reference/statements/insert-into.md index a2afbb2e10a..b49314a1785 100644 --- a/docs/en/sql-reference/statements/insert-into.md +++ b/docs/en/sql-reference/statements/insert-into.md @@ -1,5 +1,5 @@ --- -toc_priority: 34 +toc_priority: 33 toc_title: INSERT INTO --- diff --git a/docs/en/sql-reference/statements/kill.md b/docs/en/sql-reference/statements/kill.md index 1e2d2e925f7..d3f2d9bb5c6 100644 --- a/docs/en/sql-reference/statements/kill.md +++ b/docs/en/sql-reference/statements/kill.md @@ -1,5 +1,5 @@ --- -toc_priority: 48 +toc_priority: 46 toc_title: KILL --- diff --git a/docs/en/sql-reference/statements/optimize.md b/docs/en/sql-reference/statements/optimize.md index 374ef926d6c..a67f282e793 100644 --- a/docs/en/sql-reference/statements/optimize.md +++ b/docs/en/sql-reference/statements/optimize.md @@ -1,5 +1,5 @@ --- -toc_priority: 49 +toc_priority: 47 toc_title: OPTIMIZE --- diff --git a/docs/en/sql-reference/statements/rename.md b/docs/en/sql-reference/statements/rename.md index 315aaac40b5..4f14ad016a3 100644 --- a/docs/en/sql-reference/statements/rename.md +++ b/docs/en/sql-reference/statements/rename.md @@ -1,5 +1,5 @@ --- -toc_priority: 50 +toc_priority: 48 toc_title: RENAME --- diff --git a/docs/en/sql-reference/statements/revoke.md b/docs/en/sql-reference/statements/revoke.md index 8269c1f34a9..71a76546096 100644 --- a/docs/en/sql-reference/statements/revoke.md +++ b/docs/en/sql-reference/statements/revoke.md @@ -1,5 +1,5 @@ --- -toc_priority: 40 +toc_priority: 39 toc_title: REVOKE --- diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index 07be8c2bf45..3107f791eb9 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -1,7 +1,7 @@ --- title: SELECT Query toc_folder_title: SELECT -toc_priority: 33 +toc_priority: 32 toc_title: Overview --- diff --git a/docs/en/sql-reference/statements/set-role.md b/docs/en/sql-reference/statements/set-role.md index 564c8ec859f..cf14a9c6d75 100644 --- a/docs/en/sql-reference/statements/set-role.md +++ b/docs/en/sql-reference/statements/set-role.md @@ -1,5 +1,5 @@ --- -toc_priority: 52 +toc_priority: 51 toc_title: SET ROLE --- diff --git a/docs/en/sql-reference/statements/set.md b/docs/en/sql-reference/statements/set.md index 4a5bbf0baf6..c6c8d28257d 100644 --- a/docs/en/sql-reference/statements/set.md +++ b/docs/en/sql-reference/statements/set.md @@ -1,5 +1,5 @@ --- -toc_priority: 51 +toc_priority: 49 toc_title: SET --- diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index 81aca1261de..e1f2ef3488a 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -1,5 +1,5 @@ --- -toc_priority: 38 +toc_priority: 37 toc_title: SHOW --- diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 8230177f249..75303fde19e 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -1,5 +1,5 @@ --- -toc_priority: 37 +toc_priority: 36 toc_title: SYSTEM --- diff --git a/docs/en/sql-reference/statements/truncate.md b/docs/en/sql-reference/statements/truncate.md index a8cf802616f..a13936c88ab 100644 --- a/docs/en/sql-reference/statements/truncate.md +++ b/docs/en/sql-reference/statements/truncate.md @@ -1,5 +1,5 @@ --- -toc_priority: 53 +toc_priority: 52 toc_title: TRUNCATE --- diff --git a/docs/en/sql-reference/statements/use.md b/docs/en/sql-reference/statements/use.md index 2932542f052..841c23d333d 100644 --- a/docs/en/sql-reference/statements/use.md +++ b/docs/en/sql-reference/statements/use.md @@ -1,5 +1,5 @@ --- -toc_priority: 54 +toc_priority: 53 toc_title: USE --- diff --git a/docs/ru/commercial/cloud.md b/docs/ru/commercial/cloud.md index 6cbc0e6743b..e6b0309c456 100644 --- a/docs/ru/commercial/cloud.md +++ b/docs/ru/commercial/cloud.md @@ -1,3 +1,8 @@ +--- +toc_priority: 1 +toc_title: "\u041f\u043e\u0441\u0442\u0430\u0432\u0449\u0438\u043a\u0438\u0020\u043e\u0431\u043b\u0430\u0447\u043d\u044b\u0445\u0020\u0443\u0441\u043b\u0443\u0433\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +--- + # Поставщики облачных услуг ClickHouse {#clickhouse-cloud-service-providers} !!! info "Инфо" diff --git a/docs/ru/development/architecture.md b/docs/ru/development/architecture.md index e66b20f764a..0d8ddb6f795 100644 --- a/docs/ru/development/architecture.md +++ b/docs/ru/development/architecture.md @@ -1,3 +1,8 @@ +--- +toc_priority: 62 +toc_title: "\u041e\u0431\u0437\u043e\u0440\u0020\u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u044b\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +--- + # Обзор архитектуры ClickHouse {#overview-of-clickhouse-architecture} ClickHouse - полноценная колоночная СУБД. Данные хранятся в колонках, а в процессе обработки - в массивах (векторах или фрагментах (chunk’ах) колонок). По возможности операции выполняются на массивах, а не на индивидуальных значениях. Это называется “векторизованное выполнения запросов” (vectorized query execution), и помогает снизить стоимость фактической обработки данных. diff --git a/docs/ru/development/browse-code.md b/docs/ru/development/browse-code.md index 814b213a6a7..ac17cf0e6f5 100644 --- a/docs/ru/development/browse-code.md +++ b/docs/ru/development/browse-code.md @@ -1,3 +1,9 @@ +--- +toc_priority: 71 +toc_title: "\u041d\u0430\u0432\u0438\u0433\u0430\u0446\u0438\u044f\u0020\u043f\u043e\u0020\u043a\u043e\u0434\u0443\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +--- + + # Навигация по коду ClickHouse {#navigatsiia-po-kodu-clickhouse} Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse.tech/codebrowser/html_report///ClickHouse/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексация и поиск. Слепок кода обновляется ежедневно. diff --git a/docs/ru/development/contrib.md b/docs/ru/development/contrib.md index c640ecee79d..e65ab4819e8 100644 --- a/docs/ru/development/contrib.md +++ b/docs/ru/development/contrib.md @@ -1,3 +1,9 @@ +--- +toc_priority: 70 +toc_title: "\u0418\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u043c\u044b\u0435\u0020\u0441\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0435\u0020\u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0438" +--- + + # Используемые сторонние библиотеки {#ispolzuemye-storonnie-biblioteki} | Библиотека | Лицензия | diff --git a/docs/ru/development/developer-instruction.md b/docs/ru/development/developer-instruction.md index 9edcd4cf9f9..b0c84d9de7c 100644 --- a/docs/ru/development/developer-instruction.md +++ b/docs/ru/development/developer-instruction.md @@ -1,3 +1,8 @@ +--- +toc_priority: 61 +toc_title: "\u0418\u043d\u0441\u0442\u0440\u0443\u043a\u0446\u0438\u044f\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a\u043e\u0432" +--- + # Инструкция для разработчиков Сборка ClickHouse поддерживается на Linux, FreeBSD, Mac OS X. diff --git a/docs/ru/development/style.md b/docs/ru/development/style.md index 951537e8113..671293a7bbd 100644 --- a/docs/ru/development/style.md +++ b/docs/ru/development/style.md @@ -1,3 +1,9 @@ +--- +toc_priority: 68 +toc_title: "\u041a\u0430\u043a\u0020\u043f\u0438\u0441\u0430\u0442\u044c\u0020\u043a\u043e\u0434\u0020\u043d\u0430\u0020\u0043\u002b\u002b" +--- + + # Как писать код на C++ {#kak-pisat-kod-na-c} ## Общее {#obshchee} diff --git a/docs/ru/engines/database-engines/index.md b/docs/ru/engines/database-engines/index.md index 79be85518e2..d3dd729e302 100644 --- a/docs/ru/engines/database-engines/index.md +++ b/docs/ru/engines/database-engines/index.md @@ -1,3 +1,10 @@ +--- +toc_folder_title: "\u0414\u0432\u0438\u0436\u043a\u0438\u0020\u0431\u0430\u0437\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_priority: 27 +toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +--- + + # Движки баз данных {#dvizhki-baz-dannykh} Движки баз данных обеспечивают работу с таблицами. diff --git a/docs/ru/engines/database-engines/lazy.md b/docs/ru/engines/database-engines/lazy.md index c1538ec91bc..c01aae0284e 100644 --- a/docs/ru/engines/database-engines/lazy.md +++ b/docs/ru/engines/database-engines/lazy.md @@ -1,3 +1,8 @@ +--- +toc_priority: 31 +toc_title: Lazy +--- + # Lazy {#lazy} Сохраняет таблицы только в оперативной памяти `expiration_time_in_seconds` через несколько секунд после последнего доступа. Может использоваться только с таблицами \*Log. diff --git a/docs/ru/engines/database-engines/mysql.md b/docs/ru/engines/database-engines/mysql.md index e293cb6ec77..2f8335de73a 100644 --- a/docs/ru/engines/database-engines/mysql.md +++ b/docs/ru/engines/database-engines/mysql.md @@ -1,3 +1,8 @@ +--- +toc_priority: 30 +toc_title: MySQL +--- + # MySQL {#mysql} Позволяет подключаться к базам данных на удалённом MySQL сервере и выполнять запросы `INSERT` и `SELECT` для обмена данными между ClickHouse и MySQL. diff --git a/docs/ru/engines/index.md b/docs/ru/engines/index.md index 333e65dcb7b..28ccc8bcfe6 100644 --- a/docs/ru/engines/index.md +++ b/docs/ru/engines/index.md @@ -1,6 +1,6 @@ --- -toc_folder_title: "\u0414\u0432\u0438\u0436\u043A\u0438" +toc_folder_title: "\u0045\u006e\u0067\u0069\u006e\u0065\u0073" +toc_hidden: true toc_priority: 25 +toc_title: hidden --- - - diff --git a/docs/ru/engines/table-engines/index.md b/docs/ru/engines/table-engines/index.md index 6a954313c60..740588c50a4 100644 --- a/docs/ru/engines/table-engines/index.md +++ b/docs/ru/engines/table-engines/index.md @@ -1,3 +1,10 @@ +--- +toc_folder_title: "\u0414\u0432\u0438\u0436\u043a\u0438\u0020\u0442\u0430\u0431\u043b\u0438\u0446" +toc_priority: 26 +toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +--- + + # Движки таблиц {#table_engines} Движок таблицы (тип таблицы) определяет: diff --git a/docs/ru/engines/table-engines/integrations/hdfs.md b/docs/ru/engines/table-engines/integrations/hdfs.md index b5234b38d7d..bd8e760fce4 100644 --- a/docs/ru/engines/table-engines/integrations/hdfs.md +++ b/docs/ru/engines/table-engines/integrations/hdfs.md @@ -1,3 +1,8 @@ +--- +toc_priority: 4 +toc_title: HDFS +--- + # HDFS {#table_engines-hdfs} Управляет данными в HDFS. Данный движок похож на движки [File](../special/file.md#table_engines-file) и [URL](../special/url.md#table_engines-url). diff --git a/docs/ru/engines/table-engines/integrations/index.md b/docs/ru/engines/table-engines/integrations/index.md index 0186f6cf19e..02189cf9e55 100644 --- a/docs/ru/engines/table-engines/integrations/index.md +++ b/docs/ru/engines/table-engines/integrations/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: Integrations +toc_folder_title: "\u0414\u0432\u0438\u0436\u043a\u0438\u0020\u0442\u0430\u0431\u043b\u0438\u0446\u0020\u0434\u043b\u044f\u0020\u0438\u043d\u0442\u0435\u0433\u0440\u0430\u0446\u0438\u0438" toc_priority: 30 --- diff --git a/docs/ru/engines/table-engines/integrations/jdbc.md b/docs/ru/engines/table-engines/integrations/jdbc.md index cded0c51fa0..d7d438e0633 100644 --- a/docs/ru/engines/table-engines/integrations/jdbc.md +++ b/docs/ru/engines/table-engines/integrations/jdbc.md @@ -1,3 +1,8 @@ +--- +toc_priority: 2 +toc_title: JDBC +--- + # JDBC {#table-engine-jdbc} Позволяет ClickHouse подключаться к внешним базам данных с помощью [JDBC](https://en.wikipedia.org/wiki/Java_Database_Connectivity). diff --git a/docs/ru/engines/table-engines/integrations/kafka.md b/docs/ru/engines/table-engines/integrations/kafka.md index 8c47e97f497..bcca349f743 100644 --- a/docs/ru/engines/table-engines/integrations/kafka.md +++ b/docs/ru/engines/table-engines/integrations/kafka.md @@ -1,3 +1,8 @@ +--- +toc_priority: 5 +toc_title: Kafka +--- + # Kafka {#kafka} Движок работает с [Apache Kafka](http://kafka.apache.org/). diff --git a/docs/ru/engines/table-engines/integrations/mysql.md b/docs/ru/engines/table-engines/integrations/mysql.md index 9891fe79959..3370e9b06d0 100644 --- a/docs/ru/engines/table-engines/integrations/mysql.md +++ b/docs/ru/engines/table-engines/integrations/mysql.md @@ -1,3 +1,8 @@ +--- +toc_priority: 3 +toc_title: MySQL +--- + # MySQL {#mysql} Движок MySQL позволяет выполнять запросы `SELECT` над данными, хранящимися на удалённом MySQL сервере. diff --git a/docs/ru/engines/table-engines/integrations/odbc.md b/docs/ru/engines/table-engines/integrations/odbc.md index aecdbbf038d..97317d647c8 100644 --- a/docs/ru/engines/table-engines/integrations/odbc.md +++ b/docs/ru/engines/table-engines/integrations/odbc.md @@ -1,3 +1,8 @@ +--- +toc_priority: 1 +toc_title: ODBC +--- + # ODBC {#table-engine-odbc} Позволяет ClickHouse подключаться к внешним базам данных с помощью [ODBC](https://en.wikipedia.org/wiki/Open_Database_Connectivity). diff --git a/docs/ru/engines/table-engines/log-family/index.md b/docs/ru/engines/table-engines/log-family/index.md index d2cb0df5a8a..7c6d2f81d7c 100644 --- a/docs/ru/engines/table-engines/log-family/index.md +++ b/docs/ru/engines/table-engines/log-family/index.md @@ -1,6 +1,6 @@ --- -toc_folder_title: Семейство Log -toc_title: Введение +toc_folder_title: "\u0421\u0435\u043c\u0435\u0439\u0441\u0442\u0432\u043e\u0020\u004c\u006f\u0067" +toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" toc_priority: 29 --- diff --git a/docs/ru/engines/table-engines/log-family/log.md b/docs/ru/engines/table-engines/log-family/log.md index 826738d6a3d..fad331454c7 100644 --- a/docs/ru/engines/table-engines/log-family/log.md +++ b/docs/ru/engines/table-engines/log-family/log.md @@ -1,3 +1,8 @@ +--- +toc_priority: 33 +toc_title: Log +--- + # Log {#log} Движок относится к семейству движков Log. Смотрите общие свойства и различия движков в статье [Семейство Log](index.md). diff --git a/docs/ru/engines/table-engines/log-family/stripelog.md b/docs/ru/engines/table-engines/log-family/stripelog.md index 9523a354ee7..e505aae4c52 100644 --- a/docs/ru/engines/table-engines/log-family/stripelog.md +++ b/docs/ru/engines/table-engines/log-family/stripelog.md @@ -1,3 +1,8 @@ +--- +toc_priority: 32 +toc_title: StripeLog +--- + # StripeLog {#stripelog} Движок относится к семейству движков Log. Смотрите общие свойства и различия движков в статье [Семейство Log](index.md). diff --git a/docs/ru/engines/table-engines/log-family/tinylog.md b/docs/ru/engines/table-engines/log-family/tinylog.md index 69bc88b133e..d5c24d41ca4 100644 --- a/docs/ru/engines/table-engines/log-family/tinylog.md +++ b/docs/ru/engines/table-engines/log-family/tinylog.md @@ -1,3 +1,8 @@ +--- +toc_priority: 34 +toc_title: TinyLog +--- + # TinyLog {#tinylog} Движок относится к семейству движков Log. Смотрите общие свойства и различия движков в статье [Семейство Log](index.md). diff --git a/docs/ru/engines/table-engines/mergetree-family/aggregatingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/aggregatingmergetree.md index 3e18f890af7..99b4ec06765 100644 --- a/docs/ru/engines/table-engines/mergetree-family/aggregatingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/aggregatingmergetree.md @@ -1,3 +1,8 @@ +--- +toc_priority: 35 +toc_title: AggregatingMergeTree +--- + # AggregatingMergeTree {#aggregatingmergetree} Движок наследует функциональность [MergeTree](mergetree.md#table_engines-mergetree), изменяя логику слияния кусков данных. Все строки с одинаковым первичным ключом (точнее, с одинаковым [ключом сортировки](mergetree.md)) ClickHouse заменяет на одну (в пределах одного куска данных), которая хранит объединение состояний агрегатных функций. diff --git a/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md index f7bc14cc647..8ea3a5a7c92 100644 --- a/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md @@ -1,3 +1,8 @@ +--- +toc_priority: 36 +toc_title: CollapsingMergeTree +--- + # CollapsingMergeTree {#table_engine-collapsingmergetree} Движок наследует функциональность от [MergeTree](mergetree.md) и добавляет в алгоритм слияния кусков данных логику сворачивания (удаления) строк. diff --git a/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md index 32b5eee7fab..2d26528d964 100644 --- a/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -1,3 +1,9 @@ +--- +toc_priority: 32 +toc_title: "\u041f\u0440\u043e\u0438\u0437\u0432\u043e\u043b\u044c\u043d\u044b\u0439\u0020\u043a\u043b\u044e\u0447\u0020\u043f\u0430\u0440\u0442\u0438\u0446\u0438\u043e\u043d\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f" +--- + + # Произвольный ключ партиционирования {#proizvolnyi-kliuch-partitsionirovaniia} Партиционирование данных доступно для таблиц семейства [MergeTree](mergetree.md) (включая [реплицированные таблицы](replication.md)). Таблицы [MaterializedView](../special/materializedview.md#materializedview), созданные на основе таблиц MergeTree, также поддерживают партиционирование. diff --git a/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md index 861a8b6e633..e47c9127711 100644 --- a/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md @@ -1,3 +1,8 @@ +--- +toc_priority: 38 +toc_title: GraphiteMergeTree +--- + # GraphiteMergeTree {#graphitemergetree} Движок предназначен для прореживания и агрегирования/усреднения (rollup) данных [Graphite](http://graphite.readthedocs.io/en/latest/index.html). Он может быть интересен разработчикам, которые хотят использовать ClickHouse как хранилище данных для Graphite. diff --git a/docs/ru/engines/table-engines/mergetree-family/index.md b/docs/ru/engines/table-engines/mergetree-family/index.md index 9e989d807da..abdfdd77d7f 100644 --- a/docs/ru/engines/table-engines/mergetree-family/index.md +++ b/docs/ru/engines/table-engines/mergetree-family/index.md @@ -1,6 +1,5 @@ --- toc_folder_title: MergeTree Family toc_priority: 28 +toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" --- - - diff --git a/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md index 4aa1eb556f3..1228371e8ea 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md @@ -1,3 +1,8 @@ +--- +toc_priority: 33 +toc_title: ReplacingMergeTree +--- + # ReplacingMergeTree {#replacingmergetree} Движок отличается от [MergeTree](mergetree.md#table_engines-mergetree) тем, что выполняет удаление дублирующихся записей с одинаковым значением [ключа сортировки](mergetree.md)). diff --git a/docs/ru/engines/table-engines/mergetree-family/replication.md b/docs/ru/engines/table-engines/mergetree-family/replication.md index 410cf06c0c5..426c456ba3a 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replication.md +++ b/docs/ru/engines/table-engines/mergetree-family/replication.md @@ -1,3 +1,8 @@ +--- +toc_priority: 31 +toc_title: "\u0420\u0435\u043f\u043b\u0438\u043a\u0430\u0446\u0438\u044f\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +--- + # Репликация данных {#table_engines-replication} Репликация поддерживается только для таблиц семейства MergeTree: diff --git a/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md index 070cd109c30..7b9c11adc2e 100644 --- a/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md @@ -1,3 +1,8 @@ +--- +toc_priority: 34 +toc_title: SummingMergeTree +--- + # SummingMergeTree {#summingmergetree} Движок наследует функциональность [MergeTree](mergetree.md#table_engines-mergetree). Отличие заключается в том, что для таблиц `SummingMergeTree` при слиянии кусков данных ClickHouse все строки с одинаковым первичным ключом (точнее, с одинаковым [ключом сортировки](mergetree.md)) заменяет на одну, которая хранит только суммы значений из столбцов с цифровым типом данных. Если ключ сортировки подобран таким образом, что одному значению ключа соответствует много строк, это значительно уменьшает объём хранения и ускоряет последующую выборку данных. diff --git a/docs/ru/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md index bf280eb52bc..2adb8cc0d77 100644 --- a/docs/ru/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md @@ -1,3 +1,8 @@ +--- +toc_priority: 37 +toc_title: VersionedCollapsingMergeTree +--- + # VersionedCollapsingMergeTree {#versionedcollapsingmergetree} Движок: diff --git a/docs/ru/engines/table-engines/special/buffer.md b/docs/ru/engines/table-engines/special/buffer.md index 72d2822af98..986fe9adbb9 100644 --- a/docs/ru/engines/table-engines/special/buffer.md +++ b/docs/ru/engines/table-engines/special/buffer.md @@ -1,3 +1,8 @@ +--- +toc_priority: 45 +toc_title: Buffer +--- + # Buffer {#buffer} Буферизует записываемые данные в оперативке, периодически сбрасывая их в другую таблицу. При чтении, производится чтение данных одновременно из буфера и из другой таблицы. diff --git a/docs/ru/engines/table-engines/special/dictionary.md b/docs/ru/engines/table-engines/special/dictionary.md index b35c8cecc8f..048da157b2d 100644 --- a/docs/ru/engines/table-engines/special/dictionary.md +++ b/docs/ru/engines/table-engines/special/dictionary.md @@ -1,3 +1,8 @@ +--- +toc_priority: 35 +toc_title: Dictionary +--- + # Dictionary {#dictionary} Движок `Dictionary` отображает данные [словаря](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) как таблицу ClickHouse. diff --git a/docs/ru/engines/table-engines/special/distributed.md b/docs/ru/engines/table-engines/special/distributed.md index c8f7fe6eba7..7ab0b916337 100644 --- a/docs/ru/engines/table-engines/special/distributed.md +++ b/docs/ru/engines/table-engines/special/distributed.md @@ -1,3 +1,8 @@ +--- +toc_priority: 33 +toc_title: Distributed +--- + # Distributed {#distributed} **Движок Distributed не хранит данные самостоятельно**, а позволяет обрабатывать запросы распределённо, на нескольких серверах. Чтение автоматически распараллеливается. При чтении будут использованы индексы таблиц на удалённых серверах, если есть. diff --git a/docs/ru/engines/table-engines/special/external-data.md b/docs/ru/engines/table-engines/special/external-data.md index 41708978367..7e383c0c12d 100644 --- a/docs/ru/engines/table-engines/special/external-data.md +++ b/docs/ru/engines/table-engines/special/external-data.md @@ -1,3 +1,8 @@ +--- +toc_priority: 45 +toc_title: "\u0412\u043d\u0435\u0448\u043d\u0438\u0435\u0020\u0434\u0430\u043d\u043d\u044b\u0435\u0020\u0434\u043b\u044f\u0020\u043e\u0431\u0440\u0430\u0431\u043e\u0442\u043a\u0438\u0020\u0437\u0430\u043f\u0440\u043e\u0441\u0430" +--- + # Внешние данные для обработки запроса {#vneshnie-dannye-dlia-obrabotki-zaprosa} ClickHouse позволяет отправить на сервер данные, необходимые для обработки одного запроса, вместе с запросом SELECT. Такие данные будут положены во временную таблицу (см. раздел «Временные таблицы») и смогут использоваться в запросе (например, в операторах IN). diff --git a/docs/ru/engines/table-engines/special/file.md b/docs/ru/engines/table-engines/special/file.md index 907988a47aa..6a55ef31732 100644 --- a/docs/ru/engines/table-engines/special/file.md +++ b/docs/ru/engines/table-engines/special/file.md @@ -1,3 +1,8 @@ +--- +toc_priority: 37 +toc_title: File +--- + # File(Format) {#table_engines-file} Управляет данными в одном файле на диске в указанном формате. diff --git a/docs/ru/engines/table-engines/special/index.md b/docs/ru/engines/table-engines/special/index.md index dcf46c503b2..0d86461dd2d 100644 --- a/docs/ru/engines/table-engines/special/index.md +++ b/docs/ru/engines/table-engines/special/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: Special +toc_folder_title: "\u0421\u043f\u0435\u0446\u0438\u0430\u043b\u044c\u043d\u044b\u0435\u0020\u0434\u0432\u0438\u0436\u043a\u0438\u0020\u0442\u0430\u0431\u043b\u0438\u0446" toc_priority: 31 --- diff --git a/docs/ru/engines/table-engines/special/join.md b/docs/ru/engines/table-engines/special/join.md index aa734f5ca55..65bece4724c 100644 --- a/docs/ru/engines/table-engines/special/join.md +++ b/docs/ru/engines/table-engines/special/join.md @@ -1,3 +1,8 @@ +--- +toc_priority: 40 +toc_title: Join +--- + # Join {#join} Подготовленная структура данных для использования в операциях [JOIN](../../../engines/table-engines/special/join.md#select-join). diff --git a/docs/ru/engines/table-engines/special/materializedview.md b/docs/ru/engines/table-engines/special/materializedview.md index ae7a9965a24..1281d1db9ab 100644 --- a/docs/ru/engines/table-engines/special/materializedview.md +++ b/docs/ru/engines/table-engines/special/materializedview.md @@ -1,3 +1,8 @@ +--- +toc_priority: 43 +toc_title: MaterializedView +--- + # MaterializedView {#materializedview} Используется для реализации материализованных представлений (подробнее см. запрос [CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query)). Для хранения данных, использует другой движок, который был указан при создании представления. При чтении из таблицы, просто использует этот движок. diff --git a/docs/ru/engines/table-engines/special/memory.md b/docs/ru/engines/table-engines/special/memory.md index 6de7978d134..9ca189ef3b2 100644 --- a/docs/ru/engines/table-engines/special/memory.md +++ b/docs/ru/engines/table-engines/special/memory.md @@ -1,3 +1,8 @@ +--- +toc_priority: 44 +toc_title: Memory +--- + # Memory {#memory} Хранит данные в оперативке, в несжатом виде. Данные хранятся именно в таком виде, в каком они получаются при чтении. То есть, само чтение из этой таблицы полностью бесплатно. diff --git a/docs/ru/engines/table-engines/special/merge.md b/docs/ru/engines/table-engines/special/merge.md index 4f6e1751591..656aa7cfd6b 100644 --- a/docs/ru/engines/table-engines/special/merge.md +++ b/docs/ru/engines/table-engines/special/merge.md @@ -1,3 +1,8 @@ +--- +toc_priority: 36 +toc_title: Merge +--- + # Merge {#merge} Движок `Merge` (не путайте с движком `MergeTree`) не хранит данные самостоятельно, а позволяет читать одновременно из произвольного количества других таблиц. diff --git a/docs/ru/engines/table-engines/special/null.md b/docs/ru/engines/table-engines/special/null.md index ac8c7942f79..2c3af1ce11e 100644 --- a/docs/ru/engines/table-engines/special/null.md +++ b/docs/ru/engines/table-engines/special/null.md @@ -1,4 +1,7 @@ -# Null {#null} +--- +toc_priority: 38 +toc_title: 'Null' +--- При записи в таблицу типа Null, данные игнорируются. При чтении из таблицы типа Null, возвращается пустота. diff --git a/docs/ru/engines/table-engines/special/set.md b/docs/ru/engines/table-engines/special/set.md index 497d52821d1..73fe3c3cf50 100644 --- a/docs/ru/engines/table-engines/special/set.md +++ b/docs/ru/engines/table-engines/special/set.md @@ -1,3 +1,8 @@ +--- +toc_priority: 39 +toc_title: Set +--- + # Set {#set} Представляет собой множество, постоянно находящееся в оперативке. Предназначено для использования в правой части оператора IN (смотрите раздел «Операторы IN»). diff --git a/docs/ru/engines/table-engines/special/url.md b/docs/ru/engines/table-engines/special/url.md index 69c1567a1c2..cdb5afddf75 100644 --- a/docs/ru/engines/table-engines/special/url.md +++ b/docs/ru/engines/table-engines/special/url.md @@ -1,3 +1,8 @@ +--- +toc_priority: 41 +toc_title: URL +--- + # URL(URL, Format) {#table_engines-url} Управляет данными на удаленном HTTP/HTTPS сервере. Данный движок похож diff --git a/docs/ru/engines/table-engines/special/view.md b/docs/ru/engines/table-engines/special/view.md index a732a675d4c..18813a55da2 100644 --- a/docs/ru/engines/table-engines/special/view.md +++ b/docs/ru/engines/table-engines/special/view.md @@ -1,3 +1,8 @@ +--- +toc_priority: 42 +toc_title: View +--- + # View {#table_engines-view} Используется для реализации представлений (подробнее см. запрос `CREATE VIEW`). Не хранит данные, а хранит только указанный запрос `SELECT`. При чтении из таблицы, выполняет его (с удалением из запроса всех ненужных столбцов). diff --git a/docs/ru/faq/index.md b/docs/ru/faq/index.md index 2ee9d51e83b..d9eccf59c5c 100644 --- a/docs/ru/faq/index.md +++ b/docs/ru/faq/index.md @@ -1,6 +1,6 @@ --- toc_folder_title: F.A.Q. +toc_hidden: true toc_priority: 76 --- - diff --git a/docs/ru/getting-started/example-datasets/amplab-benchmark.md b/docs/ru/getting-started/example-datasets/amplab-benchmark.md index b503fc909f3..bc59672ab26 100644 --- a/docs/ru/getting-started/example-datasets/amplab-benchmark.md +++ b/docs/ru/getting-started/example-datasets/amplab-benchmark.md @@ -1,3 +1,8 @@ +--- +toc_priority: 19 +toc_title: AMPLab Big Data Benchmark +--- + # AMPLab Big Data Benchmark {#amplab-big-data-benchmark} См. https://amplab.cs.berkeley.edu/benchmark/ diff --git a/docs/ru/getting-started/example-datasets/criteo.md b/docs/ru/getting-started/example-datasets/criteo.md index 4abb8c43f35..7a58da5b695 100644 --- a/docs/ru/getting-started/example-datasets/criteo.md +++ b/docs/ru/getting-started/example-datasets/criteo.md @@ -1,3 +1,8 @@ +--- +toc_priority: 18 +toc_title: "\u0422\u0435\u0440\u0430\u0431\u0430\u0439\u0442\u0020\u043b\u043e\u0433\u043e\u0432\u0020\u043a\u043b\u0438\u043a\u043e\u0432\u0020\u043e\u0442\u0020\u0043\u0072\u0069\u0074\u0065\u006f" +--- + # Терабайт логов кликов от Criteo {#terabait-logov-klikov-ot-criteo} Скачайте данные с http://labs.criteo.com/downloads/download-terabyte-click-logs/ diff --git a/docs/ru/getting-started/example-datasets/index.md b/docs/ru/getting-started/example-datasets/index.md index c349e369b1d..eff944a7980 100644 --- a/docs/ru/getting-started/example-datasets/index.md +++ b/docs/ru/getting-started/example-datasets/index.md @@ -1,8 +1,7 @@ --- -toc_folder_title: "\u0422\u0435\u0441\u0442\u043E\u0432\u044B\u0435 \u043C\u0430\u0441\ - \u0441\u0438\u0432\u044B \u0434\u0430\u043D\u043D\u044B\u0445" -toc_priority: 12 -toc_title: "\u041E\u0431\u0437\u043E\u0440" +toc_folder_title: "\u0422\u0435\u0441\u0442\u043e\u0432\u044b\u0435\u0020\u043c\u0430\u0441\u0441\u0438\u0432\u044b\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_priority: 14 +toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" --- # Тестовые массивы данных {#testovye-massivy-dannykh} diff --git a/docs/ru/getting-started/example-datasets/metrica.md b/docs/ru/getting-started/example-datasets/metrica.md index ab2fe3d6c64..e8a3163376c 100644 --- a/docs/ru/getting-started/example-datasets/metrica.md +++ b/docs/ru/getting-started/example-datasets/metrica.md @@ -1,3 +1,8 @@ +--- +toc_priority: 15 +toc_title: "\u0410\u043d\u043e\u043d\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u044b\u0435\u0020\u0434\u0430\u043d\u043d\u044b\u0435\u0020\u042f\u043d\u0434\u0435\u043a\u0441\u002e\u041c\u0435\u0442\u0440\u0438\u043a\u0438" +--- + # Анонимизированные данные Яндекс.Метрики {#anonimizirovannye-dannye-iandeks-metriki} Датасет состоит из двух таблиц, содержащих анонимизированные данные о хитах (`hits_v1`) и визитах (`visits_v1`) Яндекс.Метрики. Каждую из таблиц можно скачать в виде сжатого `.tsv.xz`-файла или в виде уже готовых партиций. Также можно скачать расширенную версию таблицы `hits`, содержащую 100 миллионов строк в виде [архива c файлами TSV](https://clickhouse-datasets.s3.yandex.net/hits/tsv/hits_100m_obfuscated_v1.tsv.xz) и в виде [готовых партиций](https://clickhouse-datasets.s3.yandex.net/hits/partitions/hits_100m_obfuscated_v1.tar.xz). diff --git a/docs/ru/getting-started/example-datasets/nyc-taxi.md b/docs/ru/getting-started/example-datasets/nyc-taxi.md index 64202b2e765..1f981324261 100644 --- a/docs/ru/getting-started/example-datasets/nyc-taxi.md +++ b/docs/ru/getting-started/example-datasets/nyc-taxi.md @@ -1,3 +1,8 @@ +--- +toc_priority: 20 +toc_title: "\u0414\u0430\u043d\u043d\u044b\u0435\u0020\u043e\u0020\u0442\u0430\u043a\u0441\u0438\u0020\u0432\u0020\u041d\u044c\u044e\u002d\u0419\u043e\u0440\u043a\u0435" +--- + # Данные о такси в Нью-Йорке {#dannye-o-taksi-v-niu-iorke} Этот датасет может быть получен двумя способами: diff --git a/docs/ru/getting-started/example-datasets/ontime.md b/docs/ru/getting-started/example-datasets/ontime.md index 7e7a6be2196..4d3eea14da6 100644 --- a/docs/ru/getting-started/example-datasets/ontime.md +++ b/docs/ru/getting-started/example-datasets/ontime.md @@ -1,3 +1,8 @@ +--- +toc_priority: 21 +toc_title: OnTime +--- + # OnTime {#ontime} Этот датасет может быть получен двумя способами: diff --git a/docs/ru/getting-started/example-datasets/star-schema.md b/docs/ru/getting-started/example-datasets/star-schema.md index d938b1e8620..f7502e8409d 100644 --- a/docs/ru/getting-started/example-datasets/star-schema.md +++ b/docs/ru/getting-started/example-datasets/star-schema.md @@ -1,3 +1,8 @@ +--- +toc_priority: 16 +toc_title: Star Schema Benchmark +--- + # Star Schema Benchmark {#star-schema-benchmark} Компиляция dbgen: diff --git a/docs/ru/getting-started/example-datasets/wikistat.md b/docs/ru/getting-started/example-datasets/wikistat.md index 0e3e269fe9f..c5a877ff8fd 100644 --- a/docs/ru/getting-started/example-datasets/wikistat.md +++ b/docs/ru/getting-started/example-datasets/wikistat.md @@ -1,3 +1,8 @@ +--- +toc_priority: 17 +toc_title: WikiStat +--- + # WikiStat {#wikistat} См: http://dumps.wikimedia.org/other/pagecounts-raw/ diff --git a/docs/ru/getting-started/index.md b/docs/ru/getting-started/index.md index b2b292c4725..ab72ce4a1d2 100644 --- a/docs/ru/getting-started/index.md +++ b/docs/ru/getting-started/index.md @@ -1,6 +1,5 @@ --- -toc_folder_title: "\u041D\u0430\u0447\u0430\u043B\u043E \u0440\u0430\u0431\u043E\u0442\ - \u044B" +toc_folder_title: "\u041d\u0430\u0447\u0430\u043b\u043e\u0020\u0440\u0430\u0431\u043e\u0442\u044b" toc_hidden: true toc_priority: 8 toc_title: hidden diff --git a/docs/ru/getting-started/install.md b/docs/ru/getting-started/install.md index 3e9270ef3bc..fb14ecfe599 100644 --- a/docs/ru/getting-started/install.md +++ b/docs/ru/getting-started/install.md @@ -1,3 +1,8 @@ +--- +toc_priority: 11 +toc_title: "\u0423\u0441\u0442\u0430\u043d\u043e\u0432\u043a\u0430" +--- + # Установка {#ustanovka} ## Системные требования {#sistemnye-trebovaniia} diff --git a/docs/ru/getting-started/playground.md b/docs/ru/getting-started/playground.md index e3bae5c9756..86a5cd5272c 100644 --- a/docs/ru/getting-started/playground.md +++ b/docs/ru/getting-started/playground.md @@ -1,3 +1,8 @@ +--- +toc_priority: 14 +toc_title: Playground +--- + # ClickHouse Playground {#clickhouse-playground} [ClickHouse Playground](https://play.clickhouse.tech) позволяет пользователям экспериментировать с ClickHouse, мгновенно выполняя запросы без настройки своего сервера или кластера. diff --git a/docs/ru/guides/apply-catboost-model.md b/docs/ru/guides/apply-catboost-model.md index 3515d2731c2..026b4d9d75e 100644 --- a/docs/ru/guides/apply-catboost-model.md +++ b/docs/ru/guides/apply-catboost-model.md @@ -1,3 +1,8 @@ +--- +toc_priority: 41 +toc_title: "\u041f\u0440\u0438\u043c\u0435\u043d\u0435\u043d\u0438\u0435\u0020\u043c\u043e\u0434\u0435\u043b\u0438\u0020\u0043\u0061\u0074\u0042\u006f\u006f\u0073\u0074\u0020\u0432\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +--- + # Применение модели CatBoost в ClickHouse {#applying-catboost-model-in-clickhouse} [CatBoost](https://catboost.ai) — открытая программная библиотека разработанная компанией [Яндекс](https://yandex.ru/company/) для машинного обучения, которая использует схему градиентного бустинга. diff --git a/docs/ru/interfaces/cli.md b/docs/ru/interfaces/cli.md index 816b5962280..222af33f952 100644 --- a/docs/ru/interfaces/cli.md +++ b/docs/ru/interfaces/cli.md @@ -1,3 +1,8 @@ +--- +toc_priority: 17 +toc_title: "\u041a\u043b\u0438\u0435\u043d\u0442\u0020\u043a\u043e\u043c\u0430\u043d\u0434\u043d\u043e\u0439\u0020\u0441\u0442\u0440\u043e\u043a\u0438" +--- + # Клиент командной строки {#klient-komandnoi-stroki} ClickHouse предоставляет собственный клиент командной строки: `clickhouse-client`. Клиент поддерживает запуск с аргументами командной строки и с конфигурационными файлами. Подробнее читайте в разделе [Конфигурирование](#interfaces_cli_configuration). diff --git a/docs/ru/interfaces/cpp.md b/docs/ru/interfaces/cpp.md index 7136af3197b..264b4f82500 100644 --- a/docs/ru/interfaces/cpp.md +++ b/docs/ru/interfaces/cpp.md @@ -1,3 +1,8 @@ +--- +toc_priority: 24 +toc_title: "\u0043\u002b\u002b\u0020\u043a\u043b\u0438\u0435\u043d\u0442\u0441\u043a\u0430\u044f\u0020\u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0430" +--- + # C++ клиентская библиотека {#c-klientskaia-biblioteka} См. README в репозитории [clickhouse-cpp](https://github.com/ClickHouse/clickhouse-cpp). diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index e94810be8d3..042c62e310c 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1,3 +1,8 @@ +--- +toc_priority: 21 +toc_title: "\u0424\u043e\u0440\u043c\u0430\u0442\u044b\u0020\u0432\u0445\u043e\u0434\u043d\u044b\u0445\u0020\u0438\u0020\u0432\u044b\u0445\u043e\u0434\u043d\u044b\u0445\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +--- + # Форматы входных и выходных данных {#formats} ClickHouse может принимать (`INSERT`) и отдавать (`SELECT`) данные в различных форматах. diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index ffe4b2e5276..279f2916c78 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -1,3 +1,8 @@ +--- +toc_priority: 19 +toc_title: "\u0048\u0054\u0054\u0050\u002d\u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441" +--- + # HTTP-интерфейс {#http-interface} HTTP интерфейс позволяет использовать ClickHouse на любой платформе, из любого языка программирования. У нас он используется для работы из Java и Perl, а также из shell-скриптов. В других отделах, HTTP интерфейс используется из Perl, Python и Go. HTTP интерфейс более ограничен по сравнению с родным интерфейсом, но является более совместимым. diff --git a/docs/ru/interfaces/jdbc.md b/docs/ru/interfaces/jdbc.md index e83cd2cfacf..196dba64933 100644 --- a/docs/ru/interfaces/jdbc.md +++ b/docs/ru/interfaces/jdbc.md @@ -1,3 +1,8 @@ +--- +toc_priority: 22 +toc_title: "\u004a\u0044\u0042\u0043\u002d\u0434\u0440\u0430\u0439\u0432\u0435\u0440" +--- + # JDBC-драйвер {#jdbc-draiver} - **[Официальный драйвер](https://github.com/ClickHouse/clickhouse-jdbc)** diff --git a/docs/ru/interfaces/mysql.md b/docs/ru/interfaces/mysql.md index 79368aa1ba2..fa0003e0bea 100644 --- a/docs/ru/interfaces/mysql.md +++ b/docs/ru/interfaces/mysql.md @@ -1,3 +1,8 @@ +--- +toc_priority: 20 +toc_title: "\u004d\u0079\u0053\u0051\u004c\u002d\u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441" +--- + # MySQL-интерфейс {#mysql-interface} ClickHouse поддерживает взаимодействие по протоколу MySQL. Данная функция включается настройкой [mysql_port](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-mysql_port) в конфигурационном файле: diff --git a/docs/ru/interfaces/odbc.md b/docs/ru/interfaces/odbc.md index 0f4a6279539..728c4bd6979 100644 --- a/docs/ru/interfaces/odbc.md +++ b/docs/ru/interfaces/odbc.md @@ -1,3 +1,9 @@ +--- +toc_priority: 23 +toc_title: "\u004f\u0044\u0042\u0043\u002d\u0434\u0440\u0430\u0439\u0432\u0435\u0440" +--- + + # ODBC-драйвер {#odbc-draiver} - [Официальный драйвер](https://github.com/ClickHouse/clickhouse-odbc). diff --git a/docs/ru/interfaces/tcp.md b/docs/ru/interfaces/tcp.md index 6bdfb286ac2..d89646f15b7 100644 --- a/docs/ru/interfaces/tcp.md +++ b/docs/ru/interfaces/tcp.md @@ -1,3 +1,8 @@ +--- +toc_priority: 18 +toc_title: "\u0420\u043e\u0434\u043d\u043e\u0439\u0020\u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441\u0020\u0028\u0054\u0043\u0050\u0029" +--- + # Родной интерфейс (TCP) {#rodnoi-interfeis-tcp} Нативный протокол используется в [клиенте командной строки](cli.md), для взаимодействия между серверами во время обработки распределенных запросов, а также в других программах на C++. К сожалению, у родного протокола ClickHouse пока нет формальной спецификации, но в нем можно разобраться с использованием исходного кода ClickHouse (начиная с [примерно этого места](https://github.com/ClickHouse/ClickHouse/tree/master/src/Client)) и/или путем перехвата и анализа TCP трафика. diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md index 66a2252b23a..f35acb9e968 100644 --- a/docs/ru/interfaces/third-party/client-libraries.md +++ b/docs/ru/interfaces/third-party/client-libraries.md @@ -1,3 +1,8 @@ +--- +toc_priority: 26 +toc_title: "\u041a\u043b\u0438\u0435\u043d\u0442\u0441\u043a\u0438\u0435\u0020\u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0438\u0020\u043e\u0442\u0020\u0441\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0445\u0020\u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a\u043e\u0432" +--- + # Клиентские библиотеки от сторонних разработчиков {#klientskie-biblioteki-ot-storonnikh-razrabotchikov} !!! warning "Disclaimer" diff --git a/docs/ru/interfaces/third-party/gui.md b/docs/ru/interfaces/third-party/gui.md index f7eaa5cc77f..1fabdb8a31c 100644 --- a/docs/ru/interfaces/third-party/gui.md +++ b/docs/ru/interfaces/third-party/gui.md @@ -1,3 +1,9 @@ +--- +toc_priority: 28 +toc_title: "\u0412\u0438\u0437\u0443\u0430\u043b\u044c\u043d\u044b\u0435\u0020\u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441\u044b\u0020\u043e\u0442\u0020\u0441\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0445\u0020\u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a\u043e\u0432" +--- + + # Визуальные интерфейсы от сторонних разработчиков {#vizualnye-interfeisy-ot-storonnikh-razrabotchikov} ## С открытым исходным кодом {#s-otkrytym-iskhodnym-kodom} diff --git a/docs/ru/interfaces/third-party/index.md b/docs/ru/interfaces/third-party/index.md index bfdd554aebd..a57169df73b 100644 --- a/docs/ru/interfaces/third-party/index.md +++ b/docs/ru/interfaces/third-party/index.md @@ -1,6 +1,5 @@ --- -toc_folder_title: "\u041E\u0442 \u0441\u0442\u043E\u0440\u043E\u043D\u043D\u0438\u0445\ - \ \u0440\u0430\u0437\u0440\u0430\u0431\u043E\u0442\u0447\u0438\u043A\u043E\u0432" +toc_folder_title: "\u0421\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0435\u0020\u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441\u044b" toc_priority: 24 --- diff --git a/docs/ru/interfaces/third-party/integrations.md b/docs/ru/interfaces/third-party/integrations.md index 527f65343db..60d6181ab3f 100644 --- a/docs/ru/interfaces/third-party/integrations.md +++ b/docs/ru/interfaces/third-party/integrations.md @@ -1,3 +1,8 @@ +--- +toc_priority: 27 +toc_title: "\u0411\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0438\u0020\u0434\u043b\u044f\u0020\u0438\u043d\u0442\u0435\u0433\u0440\u0430\u0446\u0438\u0438\u0020\u043e\u0442\u0020\u0441\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0445\u0020\u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a\u043e\u0432" +--- + # Библиотеки для интеграции от сторонних разработчиков {#biblioteki-dlia-integratsii-ot-storonnikh-razrabotchikov} !!! warning "Disclaimer" diff --git a/docs/ru/interfaces/third-party/proxy.md b/docs/ru/interfaces/third-party/proxy.md index 225c3fee150..fc66ecde293 100644 --- a/docs/ru/interfaces/third-party/proxy.md +++ b/docs/ru/interfaces/third-party/proxy.md @@ -1,3 +1,8 @@ +--- +toc_priority: 29 +toc_title: "\u041f\u0440\u043e\u043a\u0441\u0438\u002d\u0441\u0435\u0440\u0432\u0435\u0440\u044b\u0020\u043e\u0442\u0020\u0441\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0445\u0020\u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a\u043e\u0432" +--- + # Прокси-серверы от сторонних разработчиков {#proksi-servery-ot-storonnikh-razrabotchikov} ## chproxy {#chproxy} diff --git a/docs/ru/introduction/distinctive-features.md b/docs/ru/introduction/distinctive-features.md index 98dbe6df134..4eeeef4a443 100644 --- a/docs/ru/introduction/distinctive-features.md +++ b/docs/ru/introduction/distinctive-features.md @@ -1,3 +1,8 @@ +--- +toc_priority: 4 +toc_title: "\u041e\u0442\u043b\u0438\u0447\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u0435\u0020\u0432\u043e\u0437\u043c\u043e\u0436\u043d\u043e\u0441\u0442\u0438\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +--- + # Отличительные возможности ClickHouse {#otlichitelnye-vozmozhnosti-clickhouse} ## По-настоящему столбцовая СУБД {#po-nastoiashchemu-stolbtsovaia-subd} diff --git a/docs/ru/introduction/history.md b/docs/ru/introduction/history.md index 65254b0f4f0..ab740954bbe 100644 --- a/docs/ru/introduction/history.md +++ b/docs/ru/introduction/history.md @@ -1,3 +1,9 @@ +--- +toc_priority: 7 +toc_title: "\u0418\u0441\u0442\u043e\u0440\u0438\u044f\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +--- + + # История ClickHouse {#istoriia-clickhouse} ClickHouse изначально разрабатывался для обеспечения работы [Яндекс.Метрики](https://metrika.yandex.ru/), [второй крупнейшей в мире](http://w3techs.com/technologies/overview/traffic_analysis/all) платформы для веб аналитики, и продолжает быть её ключевым компонентом. При более 13 триллионах записей в базе данных и более 20 миллиардах событий в сутки, ClickHouse позволяет генерировать индивидуально настроенные отчёты на лету напрямую из неагрегированных данных. Данная статья вкратце демонстрирует какие цели исторически стояли перед ClickHouse на ранних этапах его развития. diff --git a/docs/ru/introduction/performance.md b/docs/ru/introduction/performance.md index cbefa10cf7c..c449e76a6ea 100644 --- a/docs/ru/introduction/performance.md +++ b/docs/ru/introduction/performance.md @@ -1,3 +1,8 @@ +--- +toc_priority: 6 +toc_title: "\u041f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c" +--- + # Производительность {#proizvoditelnost} По результатам внутреннего тестирования в Яндексе, ClickHouse обладает наиболее высокой производительностью (как наиболее высокой пропускной способностью на длинных запросах, так и наиболее низкой задержкой на коротких запросах), при соответствующем сценарии работы, среди доступных для тестирования систем подобного класса. Результаты тестирования можно посмотреть на [отдельной странице](https://clickhouse.tech/benchmark/dbms/). diff --git a/docs/ru/operations/access-rights.md b/docs/ru/operations/access-rights.md index 27dbc2fbf62..9973de91161 100644 --- a/docs/ru/operations/access-rights.md +++ b/docs/ru/operations/access-rights.md @@ -1,3 +1,8 @@ +--- +toc_priority: 48 +toc_title: "\u0423\u043f\u0440\u0430\u0432\u043b\u0435\u043d\u0438\u0435\u0020\u0434\u043e\u0441\u0442\u0443\u043f\u043e\u043c" +--- + # Управление доступом {#access-control} ClickHouse поддерживает управление доступом на основе подхода [RBAC](https://ru.wikipedia.org/wiki/Управление_доступом_на_основе_ролей). diff --git a/docs/ru/operations/backup.md b/docs/ru/operations/backup.md index 89fb0403543..0dcb6fd307d 100644 --- a/docs/ru/operations/backup.md +++ b/docs/ru/operations/backup.md @@ -1,3 +1,8 @@ +--- +toc_priority: 49 +toc_title: "\u0420\u0435\u0437\u0435\u0440\u0432\u043d\u043e\u0435\u0020\u043a\u043e\u043f\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u0435\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +--- + # Резервное копирование данных {#rezervnoe-kopirovanie-dannykh} [Репликация](../engines/table-engines/mergetree-family/replication.md) обеспечивает защиту от аппаратных сбоев, но не защищает от человеческих ошибок: случайного удаления данных, удаления не той таблицы, которую надо было, или таблицы на не том кластере, а также программных ошибок, которые приводят к неправильной обработке данных или их повреждению. Во многих случаях подобные ошибки влияют на все реплики. ClickHouse имеет встроенные средства защиты для предотвращения некоторых типов ошибок — например, по умолчанию [не получится удалить таблицы \*MergeTree, содержащие более 50 Гб данных, одной командой](https://github.com/ClickHouse/ClickHouse/blob/v18.14.18-stable/programs/server/config.xml#L322-L330). Однако эти средства защиты не охватывают все возможные случаи и могут быть обойдены. diff --git a/docs/ru/operations/configuration-files.md b/docs/ru/operations/configuration-files.md index df56ab9ac7d..a4cc9182427 100644 --- a/docs/ru/operations/configuration-files.md +++ b/docs/ru/operations/configuration-files.md @@ -1,3 +1,9 @@ +--- +toc_priority: 50 +toc_title: "\u041a\u043e\u043d\u0444\u0438\u0433\u0443\u0440\u0430\u0446\u0438\u043e\u043d\u043d\u044b\u0435\u0020\u0444\u0430\u0439\u043b\u044b" +--- + + # Конфигурационные файлы {#configuration_files} Основной конфигурационный файл сервера - `config.xml`. Он расположен в директории `/etc/clickhouse-server/`. diff --git a/docs/ru/operations/index.md b/docs/ru/operations/index.md index 0a15959d652..74a1d135967 100644 --- a/docs/ru/operations/index.md +++ b/docs/ru/operations/index.md @@ -1,7 +1,7 @@ --- toc_folder_title: "\u042d\u043a\u0441\u043f\u043b\u0443\u0430\u0442\u0430\u0446\u0438\u044f" toc_priority: 41 -toc_title: intro +toc_title: "\u042d\u043a\u0441\u043f\u043b\u0443\u0430\u0442\u0430\u0446\u0438\u044f" --- # Эксплуатация {#operations} diff --git a/docs/ru/operations/monitoring.md b/docs/ru/operations/monitoring.md index a1013f5eae5..52d0b5ecc8a 100644 --- a/docs/ru/operations/monitoring.md +++ b/docs/ru/operations/monitoring.md @@ -1,3 +1,8 @@ +--- +toc_priority: 45 +toc_title: "\u041c\u043e\u043d\u0438\u0442\u043e\u0440\u0438\u043d\u0433" +--- + # Мониторинг {#monitoring} Вы можете отслеживать: diff --git a/docs/ru/operations/quotas.md b/docs/ru/operations/quotas.md index 399e80d2011..92533eef0c1 100644 --- a/docs/ru/operations/quotas.md +++ b/docs/ru/operations/quotas.md @@ -1,3 +1,8 @@ +--- +toc_priority: 51 +toc_title: "\u041a\u0432\u043e\u0442\u044b" +--- + # Квоты {#quotas} Квоты позволяют ограничить использование ресурсов за некоторый интервал времени, или просто подсчитывать использование ресурсов. diff --git a/docs/ru/operations/requirements.md b/docs/ru/operations/requirements.md index e1ecafd9f3b..36a7dd30b34 100644 --- a/docs/ru/operations/requirements.md +++ b/docs/ru/operations/requirements.md @@ -1,3 +1,8 @@ +--- +toc_priority: 44 +toc_title: "\u0422\u0440\u0435\u0431\u043e\u0432\u0430\u043d\u0438\u044f" +--- + # Требования {#trebovaniia} ## Процессор {#protsessor} diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 2ca881d19f0..17966ef0547 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -1,3 +1,8 @@ +--- +toc_priority: 57 +toc_title: "\u041a\u043e\u043d\u0444\u0438\u0433\u0443\u0440\u0430\u0446\u0438\u043e\u043d\u043d\u044b\u0435\u0020\u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b\u0020\u0441\u0435\u0440\u0432\u0435\u0440\u0430" +--- + # Конфигурационные параметры сервера {#server-configuration-parameters-reference} ## builtin_dictionaries_reload_interval {#builtin-dictionaries-reload-interval} diff --git a/docs/ru/operations/settings/constraints-on-settings.md b/docs/ru/operations/settings/constraints-on-settings.md index dd324caecc6..b23be22958c 100644 --- a/docs/ru/operations/settings/constraints-on-settings.md +++ b/docs/ru/operations/settings/constraints-on-settings.md @@ -1,3 +1,8 @@ +--- +toc_priority: 62 +toc_title: "\u041e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u044f\u0020\u043d\u0430\u0020\u0438\u0437\u043c\u0435\u043d\u0435\u043d\u0438\u0435\u0020\u043d\u0430\u0441\u0442\u0440\u043e\u0435\u043a" +--- + # Ограничения на изменение настроек {#constraints-on-settings} Ограничения на изменение настроек могут находиться внутри секции `profiles` файла `user.xml` и запрещают пользователю менять некоторые настройки с помощью запроса `SET`. diff --git a/docs/ru/operations/settings/index.md b/docs/ru/operations/settings/index.md index edc8d2d3014..c24b7053c46 100644 --- a/docs/ru/operations/settings/index.md +++ b/docs/ru/operations/settings/index.md @@ -1,7 +1,7 @@ --- -toc_folder_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +toc_folder_title: Settings toc_priority: 55 -toc_title: "\u041d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0438" +toc_title: Introduction --- # Настройки {#session-settings-intro} diff --git a/docs/ru/operations/settings/permissions-for-queries.md b/docs/ru/operations/settings/permissions-for-queries.md index cb65e652331..ae896dac77c 100644 --- a/docs/ru/operations/settings/permissions-for-queries.md +++ b/docs/ru/operations/settings/permissions-for-queries.md @@ -1,3 +1,8 @@ +--- +toc_priority: 58 +toc_title: "\u0420\u0430\u0437\u0440\u0435\u0448\u0435\u043d\u0438\u044f\u0020\u0434\u043b\u044f\u0020\u0437\u0430\u043f\u0440\u043e\u0441\u043e\u0432" +--- + # Разрешения для запросов {#permissions_for_queries} Запросы в ClickHouse можно разделить на несколько типов: diff --git a/docs/ru/operations/settings/query-complexity.md b/docs/ru/operations/settings/query-complexity.md index aeb5445a054..a62e7523207 100644 --- a/docs/ru/operations/settings/query-complexity.md +++ b/docs/ru/operations/settings/query-complexity.md @@ -1,3 +1,8 @@ +--- +toc_priority: 59 +toc_title: "\u041e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u044f\u0020\u043d\u0430\u0020\u0441\u043b\u043e\u0436\u043d\u043e\u0441\u0442\u044c\u0020\u0437\u0430\u043f\u0440\u043e\u0441\u0430" +--- + # Ограничения на сложность запроса {#restrictions-on-query-complexity} Ограничения на сложность запроса - часть настроек. diff --git a/docs/ru/operations/settings/settings-profiles.md b/docs/ru/operations/settings/settings-profiles.md index d1e24490120..10feda01850 100644 --- a/docs/ru/operations/settings/settings-profiles.md +++ b/docs/ru/operations/settings/settings-profiles.md @@ -1,3 +1,8 @@ +--- +toc_priority: 61 +toc_title: "\u041f\u0440\u043e\u0444\u0438\u043b\u0438\u0020\u043d\u0430\u0441\u0442\u0440\u043e\u0435\u043a" +--- + # Профили настроек {#settings-profiles} Профиль настроек — это набор настроек, сгруппированных под одним именем. diff --git a/docs/ru/operations/settings/settings-users.md b/docs/ru/operations/settings/settings-users.md index 7c12780823a..2069922d0ea 100644 --- a/docs/ru/operations/settings/settings-users.md +++ b/docs/ru/operations/settings/settings-users.md @@ -1,3 +1,8 @@ +--- +toc_priority: 63 +toc_title: "\u041d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0438\u0020\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u0435\u0439" +--- + # Настройки пользователей {#nastroiki-polzovatelei} Раздел `users` конфигурационного файла `user.xml` содержит настройки для пользователей. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 8a65f3781c1..3192b523d48 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1,6 +1,6 @@ --- toc_priority: 60 -toc_title: Settings +toc_title: "\u041d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0438" --- # Настройки {#settings} diff --git a/docs/ru/operations/system-tables/index.md b/docs/ru/operations/system-tables/index.md index 6fa989d3d0d..93ea1c92068 100644 --- a/docs/ru/operations/system-tables/index.md +++ b/docs/ru/operations/system-tables/index.md @@ -1,6 +1,6 @@ --- toc_priority: 52 -toc_title: Системные таблицы +toc_title: "\u0421\u0438\u0441\u0442\u0435\u043c\u043d\u044b\u0435\u0020\u0442\u0430\u0431\u043b\u0438\u0446\u044b" --- # Системные таблицы {#system-tables} diff --git a/docs/ru/operations/tips.md b/docs/ru/operations/tips.md index f076fab1311..40035309c03 100644 --- a/docs/ru/operations/tips.md +++ b/docs/ru/operations/tips.md @@ -1,3 +1,8 @@ +--- +toc_priority: 58 +toc_title: "\u0421\u043e\u0432\u0435\u0442\u044b\u0020\u043f\u043e\u0020\u044d\u043a\u0441\u043f\u043b\u0443\u0430\u0442\u0430\u0446\u0438\u0438" +--- + # Советы по эксплуатации {#sovety-po-ekspluatatsii} ## CPU Scaling Governor {#cpu-scaling-governor} diff --git a/docs/ru/operations/troubleshooting.md b/docs/ru/operations/troubleshooting.md index cb86dfef5e8..3df2a1dd46c 100644 --- a/docs/ru/operations/troubleshooting.md +++ b/docs/ru/operations/troubleshooting.md @@ -1,3 +1,8 @@ +--- +toc_priority: 46 +toc_title: "\u0423\u0441\u0442\u0440\u0430\u043d\u0435\u043d\u0438\u0435\u0020\u043d\u0435\u0438\u0441\u043f\u0440\u0430\u0432\u043d\u043e\u0441\u0442\u0435\u0439" +--- + # Устранение неисправностей {#ustranenie-neispravnostei} - [Установка дистрибутива](#troubleshooting-installation-errors) diff --git a/docs/ru/operations/update.md b/docs/ru/operations/update.md index f1998864f40..c74b28b3fd7 100644 --- a/docs/ru/operations/update.md +++ b/docs/ru/operations/update.md @@ -1,3 +1,8 @@ +--- +toc_priority: 47 +toc_title: "\u041e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0438\u0435\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +--- + # Обновление ClickHouse {#obnovlenie-clickhouse} Если ClickHouse установлен с помощью deb-пакетов, выполните следующие команды на сервере: diff --git a/docs/ru/operations/utilities/clickhouse-copier.md b/docs/ru/operations/utilities/clickhouse-copier.md index b43f5ccaf7a..64e3c1eee12 100644 --- a/docs/ru/operations/utilities/clickhouse-copier.md +++ b/docs/ru/operations/utilities/clickhouse-copier.md @@ -1,3 +1,8 @@ +--- +toc_priority: 59 +toc_title: clickhouse-copier +--- + # clickhouse-copier {#clickhouse-copier} Копирует данные из таблиц одного кластера в таблицы другого (или этого же) кластера. diff --git a/docs/ru/operations/utilities/clickhouse-local.md b/docs/ru/operations/utilities/clickhouse-local.md index 7dfa9587686..962b63e2b70 100644 --- a/docs/ru/operations/utilities/clickhouse-local.md +++ b/docs/ru/operations/utilities/clickhouse-local.md @@ -1,3 +1,8 @@ +--- +toc_priority: 60 +toc_title: clickhouse-local +--- + # clickhouse-local {#clickhouse-local} Принимает на вход данные, которые можно представить в табличном виде и выполняет над ними операции, заданные на [языке запросов](../../operations/utilities/clickhouse-local.md) ClickHouse. diff --git a/docs/ru/operations/utilities/index.md b/docs/ru/operations/utilities/index.md index 91ee649ee3a..5b55ebd798d 100644 --- a/docs/ru/operations/utilities/index.md +++ b/docs/ru/operations/utilities/index.md @@ -1,3 +1,9 @@ +--- +toc_folder_title: "\u0423\u0442\u0438\u043b\u0438\u0442\u044b" +toc_priority: 56 +toc_title: "\u041e\u0431\u0437\u043e\u0440" +--- + # Утилиты ClickHouse {#utility-clickhouse} - [clickhouse-local](clickhouse-local.md) diff --git a/docs/ru/sql-reference/aggregate-functions/combinators.md b/docs/ru/sql-reference/aggregate-functions/combinators.md index ca65f6ac615..ef014906423 100644 --- a/docs/ru/sql-reference/aggregate-functions/combinators.md +++ b/docs/ru/sql-reference/aggregate-functions/combinators.md @@ -1,3 +1,9 @@ +--- +toc_priority: 37 +toc_title: "\u041a\u043e\u043c\u0431\u0438\u043d\u0430\u0442\u043e\u0440\u044b\u0020\u0430\u0433\u0440\u0435\u0433\u0430\u0442\u043d\u044b\u0445\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0439" +--- + + # Комбинаторы агрегатных функций {#aggregate_functions_combinators} К имени агрегатной функции может быть приписан некоторый суффикс. При этом, работа агрегатной функции некоторым образом модифицируется. diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index 7cc0bdef581..f20acaa45c3 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 38 +toc_title: "\u041f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u0438\u0447\u0435\u0441\u043a\u0438\u0435\u0020\u0430\u0433\u0440\u0435\u0433\u0430\u0442\u043d\u044b\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +--- + # Параметрические агрегатные функции {#aggregate_functions_parametric} Некоторые агрегатные функции могут принимать не только столбцы-аргументы (по которым производится свёртка), но и набор параметров - констант для инициализации. Синтаксис - две пары круглых скобок вместо одной. Первая - для параметров, вторая - для аргументов. diff --git a/docs/ru/sql-reference/data-types/boolean.md b/docs/ru/sql-reference/data-types/boolean.md index f868ebf6d14..bb0cd50c739 100644 --- a/docs/ru/sql-reference/data-types/boolean.md +++ b/docs/ru/sql-reference/data-types/boolean.md @@ -1,3 +1,8 @@ +--- +toc_priority: 43 +toc_title: "\u0411\u0443\u043b\u0435\u0432\u044b\u0020\u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f" +--- + # Булевы значения {#bulevy-znacheniia} Отдельного типа для булевых значений нет. Для них используется тип UInt8, в котором используются только значения 0 и 1. diff --git a/docs/ru/sql-reference/data-types/date.md b/docs/ru/sql-reference/data-types/date.md index 97f58ec30f6..9bcae2c1d72 100644 --- a/docs/ru/sql-reference/data-types/date.md +++ b/docs/ru/sql-reference/data-types/date.md @@ -1,3 +1,8 @@ +--- +toc_priority: 47 +toc_title: Date +--- + # Date {#data-type-date} Дата. Хранится в двух байтах в виде (беззнакового) числа дней, прошедших от 1970-01-01. Позволяет хранить значения от чуть больше, чем начала unix-эпохи до верхнего порога, определяющегося константой на этапе компиляции (сейчас - до 2106 года, последний полностью поддерживаемый год - 2105). diff --git a/docs/ru/sql-reference/data-types/datetime.md b/docs/ru/sql-reference/data-types/datetime.md index d60e81c74d5..87c5da68f35 100644 --- a/docs/ru/sql-reference/data-types/datetime.md +++ b/docs/ru/sql-reference/data-types/datetime.md @@ -1,3 +1,8 @@ +--- +toc_priority: 48 +toc_title: DateTime +--- + # DateTime {#data_type-datetime} Позволяет хранить момент времени, который может быть представлен как календарная дата и время. diff --git a/docs/ru/sql-reference/data-types/datetime64.md b/docs/ru/sql-reference/data-types/datetime64.md index bc9394276ee..0a602e44636 100644 --- a/docs/ru/sql-reference/data-types/datetime64.md +++ b/docs/ru/sql-reference/data-types/datetime64.md @@ -1,3 +1,8 @@ +--- +toc_priority: 49 +toc_title: DateTime64 +--- + # DateTime64 {#data_type-datetime64} Позволяет хранить момент времени, который может быть представлен как календарная дата и время, с заданной суб-секундной точностью. diff --git a/docs/ru/sql-reference/data-types/decimal.md b/docs/ru/sql-reference/data-types/decimal.md index 29192cb8819..bdcd3c767b9 100644 --- a/docs/ru/sql-reference/data-types/decimal.md +++ b/docs/ru/sql-reference/data-types/decimal.md @@ -1,3 +1,8 @@ +--- +toc_priority: 42 +toc_title: Decimal +--- + # Decimal(P, S), Decimal32(S), Decimal64(S), Decimal128(S), Decimal256(S) {#decimalp-s-decimal32s-decimal64s-decimal128s} Знаковые дробные числа с сохранением точности операций сложения, умножения и вычитания. Для деления осуществляется отбрасывание (не округление) знаков, не попадающих в младший десятичный разряд. diff --git a/docs/ru/sql-reference/data-types/domains/index.md b/docs/ru/sql-reference/data-types/domains/index.md index fe5c7ab7349..4449469b1bc 100644 --- a/docs/ru/sql-reference/data-types/domains/index.md +++ b/docs/ru/sql-reference/data-types/domains/index.md @@ -1,6 +1,6 @@ --- -toc_folder_title: Домены -toc_title_title: Обзор +toc_folder_title: "\u0414\u043e\u043c\u0435\u043d\u044b" +toc_title_title: "\u041e\u0431\u0437\u043e\u0440" toc_priority: 56 --- diff --git a/docs/ru/sql-reference/data-types/domains/ipv4.md b/docs/ru/sql-reference/data-types/domains/ipv4.md index 68b67bcca60..57d6f12ab17 100644 --- a/docs/ru/sql-reference/data-types/domains/ipv4.md +++ b/docs/ru/sql-reference/data-types/domains/ipv4.md @@ -1,3 +1,8 @@ +--- +toc_priority: 59 +toc_title: IPv4 +--- + ## IPv4 {#ipv4} `IPv4` — это домен, базирующийся на типе данных `UInt32` предназначенный для хранения адресов IPv4. Он обеспечивает компактное хранение данных с удобным для человека форматом ввода-вывода, и явно отображаемым типом данных в структуре таблицы. diff --git a/docs/ru/sql-reference/data-types/domains/ipv6.md b/docs/ru/sql-reference/data-types/domains/ipv6.md index c88ee74adea..04c5fd0d491 100644 --- a/docs/ru/sql-reference/data-types/domains/ipv6.md +++ b/docs/ru/sql-reference/data-types/domains/ipv6.md @@ -1,3 +1,8 @@ +--- +toc_priority: 60 +toc_title: IPv6 +--- + ## IPv6 {#ipv6} `IPv6` — это домен, базирующийся на типе данных `FixedString(16)`, предназначенный для хранения адресов IPv6. Он обеспечивает компактное хранение данных с удобным для человека форматом ввода-вывода, и явно отображаемым типом данных в структуре таблицы. diff --git a/docs/ru/sql-reference/data-types/enum.md b/docs/ru/sql-reference/data-types/enum.md index 792c82f7410..b86d15c19a8 100644 --- a/docs/ru/sql-reference/data-types/enum.md +++ b/docs/ru/sql-reference/data-types/enum.md @@ -1,3 +1,8 @@ +--- +toc_priority: 50 +toc_title: Enum +--- + # Enum {#enum} Перечисляемый тип данных, содержащий именованные значения. diff --git a/docs/ru/sql-reference/data-types/fixedstring.md b/docs/ru/sql-reference/data-types/fixedstring.md index 4a26b2f76af..21115418e30 100644 --- a/docs/ru/sql-reference/data-types/fixedstring.md +++ b/docs/ru/sql-reference/data-types/fixedstring.md @@ -1,3 +1,8 @@ +--- +toc_priority: 45 +toc_title: FixedString(N) +--- + # FixedString {#fixedstring} Строка фиксированной длины `N` байт (не символов, не кодовых точек). diff --git a/docs/ru/sql-reference/data-types/float.md b/docs/ru/sql-reference/data-types/float.md index 91d4b655e2a..f2e85f35041 100644 --- a/docs/ru/sql-reference/data-types/float.md +++ b/docs/ru/sql-reference/data-types/float.md @@ -1,3 +1,8 @@ +--- +toc_priority: 41 +toc_title: Float32, Float64 +--- + # Float32, Float64 {#float32-float64} [Числа с плавающей запятой](https://en.wikipedia.org/wiki/IEEE_754). diff --git a/docs/ru/sql-reference/data-types/int-uint.md b/docs/ru/sql-reference/data-types/int-uint.md index 3a33c95e4c3..c45c639aace 100644 --- a/docs/ru/sql-reference/data-types/int-uint.md +++ b/docs/ru/sql-reference/data-types/int-uint.md @@ -1,3 +1,8 @@ +--- +toc_priority: 40 +toc_title: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64 +--- + # UInt8, UInt16, UInt32, UInt64, UInt256, Int8, Int16, Int32, Int64, Int128, Int256 {#uint8-uint16-uint32-uint64-int8-int16-int32-int64} Целые числа фиксированной длины, без знака или со знаком. diff --git a/docs/ru/sql-reference/data-types/nested-data-structures/index.md b/docs/ru/sql-reference/data-types/nested-data-structures/index.md index 73d9019c96a..d53cabc6652 100644 --- a/docs/ru/sql-reference/data-types/nested-data-structures/index.md +++ b/docs/ru/sql-reference/data-types/nested-data-structures/index.md @@ -1,3 +1,10 @@ +--- +toc_folder_title: "\u0412\u043b\u043e\u0436\u0435\u043d\u043d\u044b\u0435\u0020\u0441\u0442\u0440\u0443\u043a\u0442\u0443\u0440\u044b\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_hidden: true +toc_priority: 54 +toc_title: hidden +--- + # Вложенные структуры данных {#vlozhennye-struktury-dannykh} [Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/nested_data_structures/) diff --git a/docs/ru/sql-reference/data-types/special-data-types/expression.md b/docs/ru/sql-reference/data-types/special-data-types/expression.md index 1f4b960fb10..718fcc886a6 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/expression.md +++ b/docs/ru/sql-reference/data-types/special-data-types/expression.md @@ -1,3 +1,8 @@ +--- +toc_priority: 58 +toc_title: Expression +--- + # Expression {#expression} Используется для представления лямбда-выражений в функциях высшего порядка. diff --git a/docs/ru/sql-reference/data-types/special-data-types/index.md b/docs/ru/sql-reference/data-types/special-data-types/index.md index d5aff1501db..29c057472ea 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/index.md +++ b/docs/ru/sql-reference/data-types/special-data-types/index.md @@ -1,3 +1,10 @@ +--- +toc_folder_title: "\u0421\u043b\u0443\u0436\u0435\u0431\u043d\u044b\u0435\u0020\u0442\u0438\u043f\u044b\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_hidden: true +toc_priority: 55 +toc_title: hidden +--- + # Служебные типы данных {#sluzhebnye-tipy-dannykh} Значения служебных типов данных не могут сохраняться в таблицу и выводиться в качестве результата, а возникают как промежуточный результат выполнения запроса. diff --git a/docs/ru/sql-reference/data-types/special-data-types/interval.md b/docs/ru/sql-reference/data-types/special-data-types/interval.md index a77d05ab8be..31240b49c97 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/interval.md +++ b/docs/ru/sql-reference/data-types/special-data-types/interval.md @@ -1,3 +1,8 @@ +--- +toc_priority: 61 +toc_title: Interval +--- + # Interval {#data-type-interval} Семейство типов данных, представляющих интервалы дат и времени. Оператор [INTERVAL](../../../sql-reference/data-types/special-data-types/interval.md#operator-interval) возвращает значения этих типов. diff --git a/docs/ru/sql-reference/data-types/special-data-types/nothing.md b/docs/ru/sql-reference/data-types/special-data-types/nothing.md index 9644f102522..c6a9cb868d8 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/nothing.md +++ b/docs/ru/sql-reference/data-types/special-data-types/nothing.md @@ -1,3 +1,8 @@ +--- +toc_priority: 60 +toc_title: Nothing +--- + # Nothing {#nothing} Этот тип данных предназначен только для того, чтобы представлять [NULL](../../../sql-reference/data-types/special-data-types/nothing.md), т.е. отсутствие значения. diff --git a/docs/ru/sql-reference/data-types/special-data-types/set.md b/docs/ru/sql-reference/data-types/special-data-types/set.md index 9a99ed0ca9b..4c2f4ed2c66 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/set.md +++ b/docs/ru/sql-reference/data-types/special-data-types/set.md @@ -1,3 +1,8 @@ +--- +toc_priority: 59 +toc_title: Set +--- + # Set {#set} Используется для представления правой части выражения IN. diff --git a/docs/ru/sql-reference/data-types/string.md b/docs/ru/sql-reference/data-types/string.md index 4669d154df2..798caec4d62 100644 --- a/docs/ru/sql-reference/data-types/string.md +++ b/docs/ru/sql-reference/data-types/string.md @@ -1,3 +1,8 @@ +--- +toc_priority: 44 +toc_title: String +--- + # String {#string} Строки произвольной длины. Длина не ограничена. Значение может содержать произвольный набор байт, включая нулевые байты. diff --git a/docs/ru/sql-reference/data-types/uuid.md b/docs/ru/sql-reference/data-types/uuid.md index 9d667a50526..b780190f6f4 100644 --- a/docs/ru/sql-reference/data-types/uuid.md +++ b/docs/ru/sql-reference/data-types/uuid.md @@ -1,3 +1,9 @@ +--- +toc_priority: 46 +toc_title: UUID +--- + + # UUID {#uuid-data-type} Универсальный уникальный идентификатор (UUID) - это 16-байтовое число, используемое для идентификации записей. Подробнее про UUID читайте на [Википедии](https://en.wikipedia.org/wiki/Universally_unique_identifier). diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md index 0869b409b0b..350e391dbed 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md @@ -1,3 +1,8 @@ +--- +toc_priority: 45 +toc_title: "\u0418\u0435\u0440\u0430\u0440\u0445\u0438\u0447\u0435\u0441\u043a\u0438\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0438" +--- + # Иерархические словари {#ierarkhicheskie-slovari} ClickHouse поддерживает иерархические словари с [числовыми ключом](external-dicts-dict-structure.md#ext_dict-numeric-key). diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index 61275cc6652..979f1f2e5b9 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -1,3 +1,8 @@ +--- +toc_priority: 41 +toc_title: "\u0425\u0440\u0430\u043d\u0435\u043d\u0438\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0435\u0439\u0020\u0432\u0020\u043f\u0430\u043c\u044f\u0442\u0438" +--- + # Хранение словарей в памяти {#dicts-external-dicts-dict-layout} Словари можно размещать в памяти множеством способов. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md index 2f287795296..4dbf4be9f96 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md @@ -1,3 +1,8 @@ +--- +toc_priority: 42 +toc_title: "\u041e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0438\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0435\u0439" +--- + # Обновление словарей {#obnovlenie-slovarei} ClickHouse периодически обновляет словари. Интервал обновления для полностью загружаемых словарей и интервал инвалидации для кэшируемых словарей определяется в теге `` в секундах. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 61d98f37cfd..0015edfdf72 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -1,3 +1,8 @@ +--- +toc_priority: 43 +toc_title: "\u0418\u0441\u0442\u043e\u0447\u043d\u0438\u043a\u0438\u0020\u0432\u043d\u0435\u0448\u043d\u0438\u0445\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0435\u0439" +--- + # Источники внешних словарей {#dicts-external-dicts-dict-sources} Внешний словарь можно подключить из множества источников. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md index 4c3b4eb22e4..bf87ce61b9e 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md @@ -1,3 +1,8 @@ +--- +toc_priority: 44 +toc_title: "\u041a\u043b\u044e\u0447\u0020\u0438\u0020\u043f\u043e\u043b\u044f\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u044f" +--- + # Ключ и поля словаря {#kliuch-i-polia-slovaria} Секция `` описывает ключ словаря и поля, доступные для запросов. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md index 4ecc9b6c093..ff18f906926 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md @@ -1,3 +1,8 @@ +--- +toc_priority: 40 +toc_title: "\u041d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0430\u0020\u0432\u043d\u0435\u0448\u043d\u0435\u0433\u043e\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u044f" +--- + # Настройка внешнего словаря {#dicts-external-dicts-dict} XML-конфигурация словаря имеет следующую структуру: diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md index 734de8cffdf..c18af68c15e 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md @@ -1,3 +1,9 @@ +--- +toc_priority: 39 +toc_title: "\u0412\u043d\u0435\u0448\u043d\u0438\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0438" +--- + + # Внешние словари {#dicts-external-dicts} Существует возможность подключать собственные словари из различных источников данных. Источником данных для словаря может быть локальный текстовый/исполняемый файл, HTTP(s) ресурс или другая СУБД. Подробнее смотрите в разделе «[Источники внешних словарей](external-dicts-dict-sources.md)». diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/index.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/index.md index 4098ac38060..b448858b1fa 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/index.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: External Dictionaries +toc_folder_title: "\u0412\u043d\u0435\u0448\u043d\u0438\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0438" toc_priority: 37 --- diff --git a/docs/ru/sql-reference/dictionaries/index.md b/docs/ru/sql-reference/dictionaries/index.md index 8625a29b1e1..5a4119b4dd5 100644 --- a/docs/ru/sql-reference/dictionaries/index.md +++ b/docs/ru/sql-reference/dictionaries/index.md @@ -1,3 +1,9 @@ +--- +toc_folder_title: "\u0421\u043b\u043e\u0432\u0430\u0440\u0438" +toc_priority: 35 +toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +--- + # Словари {#slovari} Словарь — это отображение (`ключ -> атрибуты`), которое удобно использовать для различного вида справочников. diff --git a/docs/ru/sql-reference/dictionaries/internal-dicts.md b/docs/ru/sql-reference/dictionaries/internal-dicts.md index 0e8e7c82a6e..d8103efa6ae 100644 --- a/docs/ru/sql-reference/dictionaries/internal-dicts.md +++ b/docs/ru/sql-reference/dictionaries/internal-dicts.md @@ -1,3 +1,8 @@ +--- +toc_priority: 39 +toc_title: "\u0412\u0441\u0442\u0440\u043e\u0435\u043d\u043d\u044b\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0438" +--- + # Встроенные словари {#internal_dicts} ClickHouse содержит встроенную возможность работы с геобазой. diff --git a/docs/ru/sql-reference/distributed-ddl.md b/docs/ru/sql-reference/distributed-ddl.md index a06a931680a..275709320f6 100644 --- a/docs/ru/sql-reference/distributed-ddl.md +++ b/docs/ru/sql-reference/distributed-ddl.md @@ -1,6 +1,6 @@ --- toc_priority: 32 -toc_title: Распределенные DDL запросы +toc_title: "\u0420\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u043d\u044b\u0435\u0020\u0044\u0044\u004c\u0020\u0437\u0430\u043f\u0440\u043e\u0441\u044b\u000a" --- # Распределенные DDL запросы (секция ON CLUSTER) {#raspredelennye-ddl-zaprosy-sektsiia-on-cluster} diff --git a/docs/ru/sql-reference/functions/arithmetic-functions.md b/docs/ru/sql-reference/functions/arithmetic-functions.md index 8513737f025..16c3e8fd8f0 100644 --- a/docs/ru/sql-reference/functions/arithmetic-functions.md +++ b/docs/ru/sql-reference/functions/arithmetic-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 34 +toc_title: "\u0410\u0440\u0438\u0444\u043c\u0435\u0442\u0438\u0447\u0435\u0441\u043a\u0438\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +--- + # Арифметические функции {#arifmeticheskie-funktsii} Для всех арифметических функций, тип результата вычисляется, как минимальный числовой тип, который может вместить результат, если такой тип есть. Минимум берётся одновременно по числу бит, знаковости и «плавучести». Если бит не хватает, то берётся тип максимальной битности. diff --git a/docs/ru/sql-reference/functions/array-functions.md b/docs/ru/sql-reference/functions/array-functions.md index 91c0443c85d..62181eebb4b 100644 --- a/docs/ru/sql-reference/functions/array-functions.md +++ b/docs/ru/sql-reference/functions/array-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 35 +toc_title: "\u041c\u0430\u0441\u0441\u0438\u0432\u044b" +--- + # Массивы {#functions-for-working-with-arrays} ## empty {#function-empty} diff --git a/docs/ru/sql-reference/functions/array-join.md b/docs/ru/sql-reference/functions/array-join.md index afbc7bfe3e8..2ed3d25fa92 100644 --- a/docs/ru/sql-reference/functions/array-join.md +++ b/docs/ru/sql-reference/functions/array-join.md @@ -1,3 +1,8 @@ +--- +toc_priority: 61 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u044f\u0020\u0041\u0072\u0072\u0061\u0079\u004a\u006f\u0069\u006e" +--- + # Функция ArrayJoin {#functions_arrayjoin} Это совсем необычная функция. diff --git a/docs/ru/sql-reference/functions/bit-functions.md b/docs/ru/sql-reference/functions/bit-functions.md index 9d78dd861e9..8c7808437a5 100644 --- a/docs/ru/sql-reference/functions/bit-functions.md +++ b/docs/ru/sql-reference/functions/bit-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 48 +toc_title: "\u0411\u0438\u0442\u043e\u0432\u044b\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +--- + # Битовые функции {#bitovye-funktsii} Битовые функции работают для любой пары типов из UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64. diff --git a/docs/ru/sql-reference/functions/bitmap-functions.md b/docs/ru/sql-reference/functions/bitmap-functions.md index aa154831b48..b21ddea94e4 100644 --- a/docs/ru/sql-reference/functions/bitmap-functions.md +++ b/docs/ru/sql-reference/functions/bitmap-functions.md @@ -1,4 +1,9 @@ -# Функции для битмапов {#bitmap-functions} +--- +toc_priority: 49 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0431\u0438\u0442\u043c\u0430\u043f\u043e\u0432" +--- + +# Функции для битовых масок {#bitmap-functions} ## bitmapBuild {#bitmap_functions-bitmapbuild} diff --git a/docs/ru/sql-reference/functions/comparison-functions.md b/docs/ru/sql-reference/functions/comparison-functions.md index 9b921d6fe55..a98c97ec96c 100644 --- a/docs/ru/sql-reference/functions/comparison-functions.md +++ b/docs/ru/sql-reference/functions/comparison-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 36 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0441\u0440\u0430\u0432\u043d\u0435\u043d\u0438\u044f" +--- + # Функции сравнения {#funktsii-sravneniia} Функции сравнения возвращают всегда 0 или 1 (UInt8). diff --git a/docs/ru/sql-reference/functions/conditional-functions.md b/docs/ru/sql-reference/functions/conditional-functions.md index 7efb6f7bfc5..83268b68959 100644 --- a/docs/ru/sql-reference/functions/conditional-functions.md +++ b/docs/ru/sql-reference/functions/conditional-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 43 +toc_title: "\u0423\u0441\u043b\u043e\u0432\u043d\u044b\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +--- + # Условные функции {#uslovnye-funktsii} ## if {#if} diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index aa9fdee478d..d24de2faae1 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 39 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0434\u0430\u0442\u0430\u043c\u0438\u0020\u0438\u0020\u0432\u0440\u0435\u043c\u0435\u043d\u0435\u043c" +--- + # Функции для работы с датами и временем {#funktsii-dlia-raboty-s-datami-i-vremenem} Поддержка часовых поясов diff --git a/docs/ru/sql-reference/functions/encoding-functions.md b/docs/ru/sql-reference/functions/encoding-functions.md index b8ea481364d..6f1c2aad6cb 100644 --- a/docs/ru/sql-reference/functions/encoding-functions.md +++ b/docs/ru/sql-reference/functions/encoding-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 52 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043a\u043e\u0434\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f" +--- + # Функции кодирования {#funktsii-kodirovaniia} ## char {#char} diff --git a/docs/ru/sql-reference/functions/ext-dict-functions.md b/docs/ru/sql-reference/functions/ext-dict-functions.md index 792afd1775d..6054ed141d4 100644 --- a/docs/ru/sql-reference/functions/ext-dict-functions.md +++ b/docs/ru/sql-reference/functions/ext-dict-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 58 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0432\u043d\u0435\u0448\u043d\u0438\u043c\u0438\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u044f\u043c\u0438" +--- + # Функции для работы с внешними словарями {#ext_dict_functions} Информацию о подключении и настройке внешних словарей смотрите в разделе [Внешние словари](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md). diff --git a/docs/ru/sql-reference/functions/functions-for-nulls.md b/docs/ru/sql-reference/functions/functions-for-nulls.md index 5fbfb4bd27b..17da1ea9194 100644 --- a/docs/ru/sql-reference/functions/functions-for-nulls.md +++ b/docs/ru/sql-reference/functions/functions-for-nulls.md @@ -1,3 +1,8 @@ +--- +toc_priority: 63 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u004e\u0075\u006c\u006c\u0061\u0062\u006c\u0065\u002d\u0430\u0440\u0433\u0443\u043c\u0435\u043d\u0442\u0430\u043c\u0438" +--- + # Функции для работы с Nullable-аргументами {#funktsii-dlia-raboty-s-nullable-argumentami} ## isNull {#isnull} diff --git a/docs/ru/sql-reference/functions/geo/coordinates.md b/docs/ru/sql-reference/functions/geo/coordinates.md index 9d65139e69e..1931a9b932f 100644 --- a/docs/ru/sql-reference/functions/geo/coordinates.md +++ b/docs/ru/sql-reference/functions/geo/coordinates.md @@ -1,5 +1,6 @@ --- -toc_title: Функции для работы с географическими координатами +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0433\u0435\u043e\u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u0438\u043c\u0438\u0020\u043a\u043e\u043e\u0440\u0434\u0438\u043d\u0430\u0442\u0430\u043c\u0438" +toc_priority: 62 --- # Функции для работы с географическими координатами {#geographical-coordinates} diff --git a/docs/ru/sql-reference/functions/geo/geohash.md b/docs/ru/sql-reference/functions/geo/geohash.md index 7273d58b1d9..38c64f11b10 100644 --- a/docs/ru/sql-reference/functions/geo/geohash.md +++ b/docs/ru/sql-reference/functions/geo/geohash.md @@ -1,5 +1,5 @@ --- -toc_title: Geohash +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0441\u0438\u0441\u0442\u0435\u043c\u043e\u0439\u0020\u0047\u0065\u006f\u0068\u0061\u0073\u0068" --- # Функции для работы с системой Geohash {#geohash} diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index d9a5e72721b..69d06b5dfa6 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -1,5 +1,5 @@ --- -toc_title: Индексы H3 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0438\u043d\u0434\u0435\u043a\u0441\u0430\u043c\u0438\u0020\u0048\u0033" --- # Функции для работы с индексами H3 {#h3index} diff --git a/docs/ru/sql-reference/functions/geo/index.md b/docs/ru/sql-reference/functions/geo/index.md index bfea32a245b..cedaafaa31d 100644 --- a/docs/ru/sql-reference/functions/geo/index.md +++ b/docs/ru/sql-reference/functions/geo/index.md @@ -1,6 +1,6 @@ --- toc_priority: 62 -toc_folder_title: Гео-данные +toc_folder_title: "\u0413\u0435\u043e\u002d\u0434\u0430\u043d\u043d\u044b\u0435" toc_title: hidden --- diff --git a/docs/ru/sql-reference/functions/hash-functions.md b/docs/ru/sql-reference/functions/hash-functions.md index 201fc934cea..92fc69227f4 100644 --- a/docs/ru/sql-reference/functions/hash-functions.md +++ b/docs/ru/sql-reference/functions/hash-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 50 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0445\u044d\u0448\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f" +--- + # Функции хэширования {#funktsii-kheshirovaniia} Функции хэширования могут использоваться для детерминированного псевдослучайного разбрасывания элементов. diff --git a/docs/ru/sql-reference/functions/in-functions.md b/docs/ru/sql-reference/functions/in-functions.md index 679fcbccc21..e137187a36b 100644 --- a/docs/ru/sql-reference/functions/in-functions.md +++ b/docs/ru/sql-reference/functions/in-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 60 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438\u0020\u043e\u043f\u0435\u0440\u0430\u0442\u043e\u0440\u0430\u0020\u0049\u004e" +--- + # Функции для реализации оператора IN {#funktsii-dlia-realizatsii-operatora-in} ## in, notIn, globalIn, globalNotIn {#in-functions} diff --git a/docs/ru/sql-reference/functions/index.md b/docs/ru/sql-reference/functions/index.md index 9c1c0c5ca9d..25d3b6de067 100644 --- a/docs/ru/sql-reference/functions/index.md +++ b/docs/ru/sql-reference/functions/index.md @@ -1,3 +1,9 @@ +--- +toc_folder_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438" +toc_priority: 32 +toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +--- + # Функции {#funktsii} Функции бывают как минимум\* двух видов - обычные функции (называются просто, функциями) и агрегатные функции. Это совершенно разные вещи. Обычные функции работают так, как будто применяются к каждой строке по отдельности (для каждой строки, результат вычисления функции не зависит от других строк). Агрегатные функции аккумулируют множество значений из разных строк (то есть, зависят от целого множества строк). diff --git a/docs/ru/sql-reference/functions/introspection.md b/docs/ru/sql-reference/functions/introspection.md index c09efd35093..9f4f2ebd1e9 100644 --- a/docs/ru/sql-reference/functions/introspection.md +++ b/docs/ru/sql-reference/functions/introspection.md @@ -1,3 +1,8 @@ +--- +toc_priority: 65 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0438\u043d\u0442\u0440\u043e\u0441\u043f\u0435\u043a\u0446\u0438\u0438" +--- + # Функции интроспекции {#introspection-functions} Функции из этого раздела могут использоваться для интроспекции [ELF](https://en.wikipedia.org/wiki/Executable_and_Linkable_Format) и [DWARF](https://en.wikipedia.org/wiki/DWARF) в целях профилирования запросов. diff --git a/docs/ru/sql-reference/functions/ip-address-functions.md b/docs/ru/sql-reference/functions/ip-address-functions.md index a9a0a7f919a..724fb97c0d5 100644 --- a/docs/ru/sql-reference/functions/ip-address-functions.md +++ b/docs/ru/sql-reference/functions/ip-address-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 55 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0049\u0050\u002d\u0430\u0434\u0440\u0435\u0441\u0430\u043c\u0438" +--- + # Функции для работы с IP-адресами {#funktsii-dlia-raboty-s-ip-adresami} ## IPv4NumToString(num) {#ipv4numtostringnum} diff --git a/docs/ru/sql-reference/functions/json-functions.md b/docs/ru/sql-reference/functions/json-functions.md index 752b70b7c5f..69b8f8f98f5 100644 --- a/docs/ru/sql-reference/functions/json-functions.md +++ b/docs/ru/sql-reference/functions/json-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 56 +toc_title: JSON +--- + # Функции для работы с JSON {#funktsii-dlia-raboty-s-json} В Яндекс.Метрике пользователями передаётся JSON в качестве параметров визитов. Для работы с таким JSON-ом, реализованы некоторые функции. (Хотя в большинстве случаев, JSON-ы дополнительно обрабатываются заранее, и полученные значения кладутся в отдельные столбцы в уже обработанном виде.) Все эти функции исходят из сильных допущений о том, каким может быть JSON, и при этом стараются почти ничего не делать. diff --git a/docs/ru/sql-reference/functions/logical-functions.md b/docs/ru/sql-reference/functions/logical-functions.md index de2a3d51729..9b1ee6a66a7 100644 --- a/docs/ru/sql-reference/functions/logical-functions.md +++ b/docs/ru/sql-reference/functions/logical-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 37 +toc_title: "\u041b\u043e\u0433\u0438\u0447\u0435\u0441\u043a\u0438\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +--- + # Логические функции {#logicheskie-funktsii} Логические функции принимают любые числовые типы, а возвращают число типа UInt8, равное 0 или 1. diff --git a/docs/ru/sql-reference/functions/machine-learning-functions.md b/docs/ru/sql-reference/functions/machine-learning-functions.md index e9b0e8c9bc9..2ffdfd05613 100644 --- a/docs/ru/sql-reference/functions/machine-learning-functions.md +++ b/docs/ru/sql-reference/functions/machine-learning-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 64 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043c\u0430\u0448\u0438\u043d\u043d\u043e\u0433\u043e\u0020\u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f" +--- + # Функции машинного обучения {#funktsii-mashinnogo-obucheniia} ## evalMLMethod (prediction) {#machine_learning_methods-evalmlmethod} diff --git a/docs/ru/sql-reference/functions/math-functions.md b/docs/ru/sql-reference/functions/math-functions.md index e52b14d26c6..6df366d129f 100644 --- a/docs/ru/sql-reference/functions/math-functions.md +++ b/docs/ru/sql-reference/functions/math-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 44 +toc_title: "\u041c\u0430\u0442\u0435\u043c\u0430\u0442\u0438\u0447\u0435\u0441\u043a\u0438\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +--- + # Математические функции {#matematicheskie-funktsii} Все функции возвращают число типа Float64. Точность результата близка к максимально возможной, но результат может не совпадать с наиболее близким к соответствующему вещественному числу машинно представимым числом. diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index bb5371ff449..9367f3be00c 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 66 +toc_title: "\u041f\u0440\u043e\u0447\u0438\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +--- + # Прочие функции {#other-functions} ## hostName() {#hostname} @@ -1468,7 +1473,7 @@ SELECT getSetting('custom_a'); ## isDecimalOverflow {#is-decimal-overflow} -Проверяет, находится ли число [Decimal](../../sql-reference/data-types/decimal.md#decimalp-s-decimal32s-decimal64s-decimal128s) вне собственной (или заданной) области значений. +Проверяет, находится ли число [Decimal](../../sql-reference/data-types/decimal.md) вне собственной (или заданной) области значений. **Синтаксис** @@ -1478,7 +1483,7 @@ isDecimalOverflow(d, [p]) **Параметры** -- `d` — число. [Decimal](../../sql-reference/data-types/decimal.md#decimalp-s-decimal32s-decimal64s-decimal128s). +- `d` — число. [Decimal](../../sql-reference/data-types/decimal.md). - `p` — точность. Необязательный параметр. Если опущен, используется исходная точность первого аргумента. Использование этого параметра может быть полезно для извлечения данных в другую СУБД или файл. [UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges). **Возвращаемое значение** @@ -1515,7 +1520,7 @@ countDigits(x) **Параметры** -- `x` — [целое](../../sql-reference/data-types/int-uint.md#uint8-uint16-uint32-uint64-int8-int16-int32-int64) или [дробное](../../sql-reference/data-types/decimal.md#decimalp-s-decimal32s-decimal64s-decimal128s) число. +- `x` — [целое](../../sql-reference/data-types/int-uint.md#uint8-uint16-uint32-uint64-int8-int16-int32-int64) или [дробное](../../sql-reference/data-types/decimal.md) число. **Возвращаемое значение** diff --git a/docs/ru/sql-reference/functions/random-functions.md b/docs/ru/sql-reference/functions/random-functions.md index 21dcfeeb3c0..f3889504fa6 100644 --- a/docs/ru/sql-reference/functions/random-functions.md +++ b/docs/ru/sql-reference/functions/random-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 51 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0433\u0435\u043d\u0435\u0440\u0430\u0446\u0438\u0438\u0020\u043f\u0441\u0435\u0432\u0434\u043e\u0441\u043b\u0443\u0447\u0430\u0439\u043d\u044b\u0445\u0020\u0447\u0438\u0441\u0435\u043b" +--- + # Функции генерации псевдослучайных чисел {#functions-for-generating-pseudo-random-numbers} Используются не криптографические генераторы псевдослучайных чисел. diff --git a/docs/ru/sql-reference/functions/rounding-functions.md b/docs/ru/sql-reference/functions/rounding-functions.md index c643bccf3e2..78033160396 100644 --- a/docs/ru/sql-reference/functions/rounding-functions.md +++ b/docs/ru/sql-reference/functions/rounding-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 45 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043e\u043a\u0440\u0443\u0433\u043b\u0435\u043d\u0438\u044f" +--- + # Функции округления {#funktsii-okrugleniia} ## floor(x\[, N\]) {#floorx-n} diff --git a/docs/ru/sql-reference/functions/splitting-merging-functions.md b/docs/ru/sql-reference/functions/splitting-merging-functions.md index bf4e76c3bb1..d451eabc407 100644 --- a/docs/ru/sql-reference/functions/splitting-merging-functions.md +++ b/docs/ru/sql-reference/functions/splitting-merging-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 47 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0440\u0430\u0437\u0431\u0438\u0435\u043d\u0438\u044f\u0020\u0438\u0020\u0441\u043b\u0438\u044f\u043d\u0438\u044f\u0020\u0441\u0442\u0440\u043e\u043a\u0020\u0438\u0020\u043c\u0430\u0441\u0441\u0438\u0432\u043e\u0432" +--- + # Функции разбиения и слияния строк и массивов {#funktsii-razbieniia-i-sliianiia-strok-i-massivov} ## splitByChar(separator, s) {#splitbycharseparator-s} diff --git a/docs/ru/sql-reference/functions/string-functions.md b/docs/ru/sql-reference/functions/string-functions.md index 58ed582b399..cc488fb2d9c 100644 --- a/docs/ru/sql-reference/functions/string-functions.md +++ b/docs/ru/sql-reference/functions/string-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 40 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u043e\u0020\u0441\u0442\u0440\u043e\u043a\u0430\u043c\u0438" +--- + # Функции для работы со строками {#funktsii-dlia-raboty-so-strokami} ## empty {#empty} diff --git a/docs/ru/sql-reference/functions/string-replace-functions.md b/docs/ru/sql-reference/functions/string-replace-functions.md index 8a2db9bf472..f334d6804f9 100644 --- a/docs/ru/sql-reference/functions/string-replace-functions.md +++ b/docs/ru/sql-reference/functions/string-replace-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 42 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043f\u043e\u0438\u0441\u043a\u0430\u0020\u0438\u0020\u0437\u0430\u043c\u0435\u043d\u044b\u0020\u0432\u0020\u0441\u0442\u0440\u043e\u043a\u0430\u0445" +--- + # Функции поиска и замены в строках {#funktsii-poiska-i-zameny-v-strokakh} ## replaceOne(haystack, pattern, replacement) {#replaceonehaystack-pattern-replacement} diff --git a/docs/ru/sql-reference/functions/string-search-functions.md b/docs/ru/sql-reference/functions/string-search-functions.md index 078a09a8aa4..d2f1119783b 100644 --- a/docs/ru/sql-reference/functions/string-search-functions.md +++ b/docs/ru/sql-reference/functions/string-search-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 41 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043f\u043e\u0438\u0441\u043a\u0430\u0020\u0432\u0020\u0441\u0442\u0440\u043e\u043a\u0430\u0445" +--- + # Функции поиска в строках {#funktsii-poiska-v-strokakh} Во всех функциях, поиск регистрозависимый по умолчанию. Существуют варианты функций для регистронезависимого поиска. diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index 3bee1a3656f..773850b65ce 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 38 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043f\u0440\u0435\u043e\u0431\u0440\u0430\u0437\u043e\u0432\u0430\u043d\u0438\u044f\u0020\u0442\u0438\u043f\u043e\u0432" +--- + # Функции преобразования типов {#funktsii-preobrazovaniia-tipov} ## Общие проблемы преобразования чисел {#numeric-conversion-issues} diff --git a/docs/ru/sql-reference/functions/url-functions.md b/docs/ru/sql-reference/functions/url-functions.md index 54b752d7c68..8f10a1ebd2b 100644 --- a/docs/ru/sql-reference/functions/url-functions.md +++ b/docs/ru/sql-reference/functions/url-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 54 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0055\u0052\u004c" +--- + # Функции для работы с URL {#funktsii-dlia-raboty-s-url} Все функции работают не по RFC - то есть, максимально упрощены ради производительности. diff --git a/docs/ru/sql-reference/functions/uuid-functions.md b/docs/ru/sql-reference/functions/uuid-functions.md index b2b567c6a06..389ce751ce0 100644 --- a/docs/ru/sql-reference/functions/uuid-functions.md +++ b/docs/ru/sql-reference/functions/uuid-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 53 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0055\u0055\u0049\u0044" +--- + # Функции для работы с UUID {#funktsii-dlia-raboty-s-uuid} ## generateUUIDv4 {#uuid-function-generate} diff --git a/docs/ru/sql-reference/functions/ym-dict-functions.md b/docs/ru/sql-reference/functions/ym-dict-functions.md index 7ac27c0d285..c3b04e4ab66 100644 --- a/docs/ru/sql-reference/functions/ym-dict-functions.md +++ b/docs/ru/sql-reference/functions/ym-dict-functions.md @@ -1,3 +1,8 @@ +--- +toc_priority: 59 +toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u043e\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u044f\u043c\u0438\u0020\u042f\u043d\u0434\u0435\u043a\u0441\u002e\u041c\u0435\u0442\u0440\u0438\u043a\u0438" +--- + # Функции для работы со словарями Яндекс.Метрики {#ym-dict-functions} Чтобы указанные ниже функции работали, в конфиге сервера должны быть указаны пути и адреса для получения всех словарей Яндекс.Метрики. Словари загружаются при первом вызове любой из этих функций. Если справочники не удаётся загрузить - будет выкинуто исключение. diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index 08594193d4c..3befb18687d 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -1,3 +1,8 @@ +--- +toc_priority: 38 +toc_title: "\u041e\u043f\u0435\u0440\u0430\u0442\u043e\u0440\u044b" +--- + # Операторы {#operatory} Все операторы преобразуются в соответствующие функции на этапе парсинга запроса, с учётом их приоритетов и ассоциативности. diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md index 811539d60d3..a8fe0ccf642 100644 --- a/docs/ru/sql-reference/statements/alter/column.md +++ b/docs/ru/sql-reference/statements/alter/column.md @@ -1,6 +1,6 @@ --- toc_priority: 37 -toc_title: COLUMN +toc_title: "\u041c\u0430\u043d\u0438\u043f\u0443\u043b\u044f\u0446\u0438\u0438\u0020\u0441\u043e\u0020\u0441\u0442\u043e\u043b\u0431\u0446\u0430\u043c\u0438" --- # Манипуляции со столбцами {#manipuliatsii-so-stolbtsami} diff --git a/docs/ru/sql-reference/statements/alter/constraint.md b/docs/ru/sql-reference/statements/alter/constraint.md index bacdff9ff57..e26db208493 100644 --- a/docs/ru/sql-reference/statements/alter/constraint.md +++ b/docs/ru/sql-reference/statements/alter/constraint.md @@ -1,6 +1,6 @@ --- toc_priority: 43 -toc_title: CONSTRAINT +toc_title: "\u041c\u0430\u043d\u0438\u043f\u0443\u043b\u044f\u0446\u0438\u0438\u0020\u0441\u0020\u043e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u044f\u043c\u0438" --- # Манипуляции с ограничениями (constraints) {#manipuliatsii-s-ogranicheniiami-constraints} diff --git a/docs/ru/sql-reference/statements/alter/index.md b/docs/ru/sql-reference/statements/alter/index.md index 035be934eb4..830c4a5745b 100644 --- a/docs/ru/sql-reference/statements/alter/index.md +++ b/docs/ru/sql-reference/statements/alter/index.md @@ -1,5 +1,5 @@ --- -toc_priority: 36 +toc_priority: 35 toc_title: ALTER --- diff --git a/docs/ru/sql-reference/statements/alter/index/index.md b/docs/ru/sql-reference/statements/alter/index/index.md index 863f8b875dd..2cadbbe065e 100644 --- a/docs/ru/sql-reference/statements/alter/index/index.md +++ b/docs/ru/sql-reference/statements/alter/index/index.md @@ -1,7 +1,7 @@ --- toc_hidden_folder: true toc_priority: 42 -toc_title: INDEX +toc_title: "\u041c\u0430\u043d\u0438\u043f\u0443\u043b\u044f\u0446\u0438\u0438\u0020\u0441\u0020\u0438\u043d\u0434\u0435\u043a\u0441\u0430\u043c\u0438" --- # Манипуляции с индексами {#manipuliatsii-s-indeksami} diff --git a/docs/ru/sql-reference/statements/attach.md b/docs/ru/sql-reference/statements/attach.md index 2aaf133130b..259ab893e63 100644 --- a/docs/ru/sql-reference/statements/attach.md +++ b/docs/ru/sql-reference/statements/attach.md @@ -1,5 +1,5 @@ --- -toc_priority: 42 +toc_priority: 40 toc_title: ATTACH --- diff --git a/docs/ru/sql-reference/statements/check-table.md b/docs/ru/sql-reference/statements/check-table.md index fa37b24851f..3dc135d87c6 100644 --- a/docs/ru/sql-reference/statements/check-table.md +++ b/docs/ru/sql-reference/statements/check-table.md @@ -1,5 +1,5 @@ --- -toc_priority: 43 +toc_priority: 41 toc_title: CHECK --- diff --git a/docs/ru/sql-reference/statements/create/database.md b/docs/ru/sql-reference/statements/create/database.md index a6c8b81c8b8..e6c561f8e0b 100644 --- a/docs/ru/sql-reference/statements/create/database.md +++ b/docs/ru/sql-reference/statements/create/database.md @@ -1,6 +1,6 @@ --- -toc_priority: 1 -toc_title: База данных +toc_priority: 35 +toc_title: "\u0411\u0430\u0437\u0430\u0020\u0434\u0430\u043d\u043d\u044b\u0445" --- # CREATE DATABASE {#query-language-create-database} diff --git a/docs/ru/sql-reference/statements/create/dictionary.md b/docs/ru/sql-reference/statements/create/dictionary.md index da9443e1b3a..a20dc812e02 100644 --- a/docs/ru/sql-reference/statements/create/dictionary.md +++ b/docs/ru/sql-reference/statements/create/dictionary.md @@ -1,6 +1,6 @@ --- -toc_priority: 4 -toc_title: Словарь +toc_priority: 38 +toc_title: "\u0421\u043b\u043e\u0432\u0430\u0440\u044c" --- # CREATE DICTIONARY {#create-dictionary-query} diff --git a/docs/ru/sql-reference/statements/create/index.md b/docs/ru/sql-reference/statements/create/index.md index f6399cfba11..28ddce2afe3 100644 --- a/docs/ru/sql-reference/statements/create/index.md +++ b/docs/ru/sql-reference/statements/create/index.md @@ -1,7 +1,7 @@ --- toc_folder_title: CREATE -toc_priority: 35 -toc_title: Обзор +toc_priority: 34 +toc_title: "\u041e\u0431\u0437\u043e\u0440" --- # Запросы CREATE {#create-queries} diff --git a/docs/ru/sql-reference/statements/create/quota.md b/docs/ru/sql-reference/statements/create/quota.md index 6351de2d38a..fe18869bf2e 100644 --- a/docs/ru/sql-reference/statements/create/quota.md +++ b/docs/ru/sql-reference/statements/create/quota.md @@ -1,6 +1,6 @@ --- -toc_priority: 8 -toc_title: Квота +toc_priority: 42 +toc_title: "\u041a\u0432\u043e\u0442\u0430" --- # CREATE QUOTA {#create-quota-statement} diff --git a/docs/ru/sql-reference/statements/create/role.md b/docs/ru/sql-reference/statements/create/role.md index b8c0fc2b453..b9e529fb213 100644 --- a/docs/ru/sql-reference/statements/create/role.md +++ b/docs/ru/sql-reference/statements/create/role.md @@ -1,6 +1,6 @@ --- -toc_priority: 6 -toc_title: Роль +toc_priority: 40 +toc_title: "\u0420\u043e\u043b\u044c" --- # CREATE ROLE {#create-role-statement} diff --git a/docs/ru/sql-reference/statements/create/row-policy.md b/docs/ru/sql-reference/statements/create/row-policy.md index 7ec28761452..a62e275a046 100644 --- a/docs/ru/sql-reference/statements/create/row-policy.md +++ b/docs/ru/sql-reference/statements/create/row-policy.md @@ -1,6 +1,6 @@ --- -toc_priority: 7 -toc_title: Политика доступа +toc_priority: 41 +toc_title: "\u041f\u043e\u043b\u0438\u0442\u0438\u043a\u0430\u0020\u0434\u043e\u0441\u0442\u0443\u043f\u0430" --- # CREATE ROW POLICY {#create-row-policy-statement} diff --git a/docs/ru/sql-reference/statements/create/settings-profile.md b/docs/ru/sql-reference/statements/create/settings-profile.md index 70c90b53565..9d525023af2 100644 --- a/docs/ru/sql-reference/statements/create/settings-profile.md +++ b/docs/ru/sql-reference/statements/create/settings-profile.md @@ -1,6 +1,6 @@ --- -toc_priority: 9 -toc_title: Профиль настроек +toc_priority: 43 +toc_title: "\u041f\u0440\u043e\u0444\u0438\u043b\u044c\u0020\u043d\u0430\u0441\u0442\u0440\u043e\u0435\u043a" --- # CREATE SETTINGS PROFILE {#create-settings-profile-statement} diff --git a/docs/ru/sql-reference/statements/create/table.md b/docs/ru/sql-reference/statements/create/table.md index c6093bbc6de..d54ec189a1a 100644 --- a/docs/ru/sql-reference/statements/create/table.md +++ b/docs/ru/sql-reference/statements/create/table.md @@ -1,6 +1,6 @@ --- -toc_priority: 2 -toc_title: Таблица +toc_priority: 36 +toc_title: "\u0422\u0430\u0431\u043b\u0438\u0446\u0430" --- # CREATE TABLE {#create-table-query} diff --git a/docs/ru/sql-reference/statements/create/user.md b/docs/ru/sql-reference/statements/create/user.md index 3c04b4df86c..e7af1659a1b 100644 --- a/docs/ru/sql-reference/statements/create/user.md +++ b/docs/ru/sql-reference/statements/create/user.md @@ -1,6 +1,6 @@ --- -toc_priority: 5 -toc_title: Пользователь +toc_priority: 39 +toc_title: "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c" --- # CREATE USER {#create-user-statement} diff --git a/docs/ru/sql-reference/statements/create/view.md b/docs/ru/sql-reference/statements/create/view.md index caa3d04659e..891e33bc9b3 100644 --- a/docs/ru/sql-reference/statements/create/view.md +++ b/docs/ru/sql-reference/statements/create/view.md @@ -1,6 +1,6 @@ --- -toc_priority: 3 -toc_title: Представление +toc_priority: 37 +toc_title: "\u041f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u0435" --- # CREATE VIEW {#create-view} diff --git a/docs/ru/sql-reference/statements/describe-table.md b/docs/ru/sql-reference/statements/describe-table.md index 5b38dca5833..64ed61de232 100644 --- a/docs/ru/sql-reference/statements/describe-table.md +++ b/docs/ru/sql-reference/statements/describe-table.md @@ -1,5 +1,5 @@ --- -toc_priority: 44 +toc_priority: 42 toc_title: DESCRIBE --- diff --git a/docs/ru/sql-reference/statements/detach.md b/docs/ru/sql-reference/statements/detach.md index 172e86179c8..00d0a4b20c6 100644 --- a/docs/ru/sql-reference/statements/detach.md +++ b/docs/ru/sql-reference/statements/detach.md @@ -1,5 +1,5 @@ --- -toc_priority: 45 +toc_priority: 43 toc_title: DETACH --- diff --git a/docs/ru/sql-reference/statements/drop.md b/docs/ru/sql-reference/statements/drop.md index 3a6ac22b071..514a92db91f 100644 --- a/docs/ru/sql-reference/statements/drop.md +++ b/docs/ru/sql-reference/statements/drop.md @@ -1,5 +1,5 @@ --- -toc_priority: 46 +toc_priority: 44 toc_title: DROP --- diff --git a/docs/ru/sql-reference/statements/exists.md b/docs/ru/sql-reference/statements/exists.md index 865d23ad622..0b2fd69273c 100644 --- a/docs/ru/sql-reference/statements/exists.md +++ b/docs/ru/sql-reference/statements/exists.md @@ -1,5 +1,5 @@ --- -toc_priority: 47 +toc_priority: 45 toc_title: EXISTS --- diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md index 69137095c4d..d38e2ea38a0 100644 --- a/docs/ru/sql-reference/statements/grant.md +++ b/docs/ru/sql-reference/statements/grant.md @@ -1,3 +1,8 @@ +--- +toc_priority: 38 +toc_title: GRANT +--- + # GRANT - Присваивает [привилегии](#grant-privileges) пользователям или ролям ClickHouse. diff --git a/docs/ru/sql-reference/statements/insert-into.md b/docs/ru/sql-reference/statements/insert-into.md index 281b708f3f9..8ea7c83bec8 100644 --- a/docs/ru/sql-reference/statements/insert-into.md +++ b/docs/ru/sql-reference/statements/insert-into.md @@ -1,5 +1,5 @@ --- -toc_priority: 34 +toc_priority: 33 toc_title: INSERT INTO --- diff --git a/docs/ru/sql-reference/statements/kill.md b/docs/ru/sql-reference/statements/kill.md index dd2f24ffa27..e2556a7f782 100644 --- a/docs/ru/sql-reference/statements/kill.md +++ b/docs/ru/sql-reference/statements/kill.md @@ -1,5 +1,5 @@ --- -toc_priority: 48 +toc_priority: 46 toc_title: KILL --- diff --git a/docs/ru/sql-reference/statements/optimize.md b/docs/ru/sql-reference/statements/optimize.md index dc70d86a1a0..9b94c31a8f7 100644 --- a/docs/ru/sql-reference/statements/optimize.md +++ b/docs/ru/sql-reference/statements/optimize.md @@ -1,5 +1,5 @@ --- -toc_priority: 49 +toc_priority: 47 toc_title: OPTIMIZE --- diff --git a/docs/ru/sql-reference/statements/rename.md b/docs/ru/sql-reference/statements/rename.md index 9f6069d8a94..94bf3c682a1 100644 --- a/docs/ru/sql-reference/statements/rename.md +++ b/docs/ru/sql-reference/statements/rename.md @@ -1,5 +1,5 @@ --- -toc_priority: 50 +toc_priority: 48 toc_title: RENAME --- diff --git a/docs/ru/sql-reference/statements/revoke.md b/docs/ru/sql-reference/statements/revoke.md index 1d2928bb76e..339746b8591 100644 --- a/docs/ru/sql-reference/statements/revoke.md +++ b/docs/ru/sql-reference/statements/revoke.md @@ -1,3 +1,8 @@ +--- +toc_priority: 39 +toc_title: REVOKE +--- + # REVOKE Отзывает привилегии у пользователей или ролей. diff --git a/docs/ru/sql-reference/statements/select/array-join.md b/docs/ru/sql-reference/statements/select/array-join.md index f8f11ba1b17..a3abf9e5e2e 100644 --- a/docs/ru/sql-reference/statements/select/array-join.md +++ b/docs/ru/sql-reference/statements/select/array-join.md @@ -1,3 +1,7 @@ +--- +toc_title: ARRAY JOIN +--- + # Секция ARRAY JOIN {#select-array-join-clause} Типовая операция для таблиц, содержащих столбец-массив — произвести новую таблицу, которая будет иметь столбец с каждым отдельным элементом массивов из изначального столбца, в то время как значения других столбцов дублируются. Это основной сценарий использования секции `ARRAY JOIN`. diff --git a/docs/ru/sql-reference/statements/select/distinct.md b/docs/ru/sql-reference/statements/select/distinct.md index 62e2e25b7e5..9d620079f6b 100644 --- a/docs/ru/sql-reference/statements/select/distinct.md +++ b/docs/ru/sql-reference/statements/select/distinct.md @@ -1,3 +1,7 @@ +--- +toc_title: DISTINCT +--- + # Секция DISTINCT {#select-distinct} Если указан `SELECT DISTINCT`, то в результате запроса останутся только уникальные строки. Таким образом, из всех наборов полностью совпадающих строк в результате останется только одна строка. diff --git a/docs/ru/sql-reference/statements/select/format.md b/docs/ru/sql-reference/statements/select/format.md index dad0ef0d62c..18972ddd82b 100644 --- a/docs/ru/sql-reference/statements/select/format.md +++ b/docs/ru/sql-reference/statements/select/format.md @@ -1,3 +1,7 @@ +--- +toc_title: FORMAT +--- + # Секция FORMAT {#format-clause} ClickHouse поддерживает широкий спектр [форматов сериализации](../../../interfaces/formats.md) это может быть использовано, в частности, для результатов запросов. Существует несколько способов выбора формата для `SELECT`, один из них заключается в том, чтобы указать `FORMAT format` в конце запроса, чтобы получить результирующие данные в любом конкретном формате. diff --git a/docs/ru/sql-reference/statements/select/from.md b/docs/ru/sql-reference/statements/select/from.md index ac0ab1dcd3f..491bbfe892b 100644 --- a/docs/ru/sql-reference/statements/select/from.md +++ b/docs/ru/sql-reference/statements/select/from.md @@ -1,3 +1,7 @@ +--- +toc_title: FROM +--- + # Секция FROM {#select-from} В секции `FROM` указывается источник, из которого будут читаться данные: diff --git a/docs/ru/sql-reference/statements/select/group-by.md b/docs/ru/sql-reference/statements/select/group-by.md index 9581f477af5..a0454ef1d91 100644 --- a/docs/ru/sql-reference/statements/select/group-by.md +++ b/docs/ru/sql-reference/statements/select/group-by.md @@ -1,3 +1,7 @@ +--- +toc_title: GROUP BY +--- + # Секция GROUP BY {#select-group-by-clause} Секция `GROUP BY` переключает `SELECT` запрос в режим агрегации, который работает следующим образом: diff --git a/docs/ru/sql-reference/statements/select/having.md b/docs/ru/sql-reference/statements/select/having.md index 83f58c5566f..dc701df906f 100644 --- a/docs/ru/sql-reference/statements/select/having.md +++ b/docs/ru/sql-reference/statements/select/having.md @@ -1,3 +1,7 @@ +--- +toc_title: HAVING +--- + # Секция HAVING {#having-clause} Позволяет фильтровать результаты агрегации, полученные с помощью [GROUP BY](group-by.md). Разница с [WHERE](where.md) в том, что `WHERE` выполняется перед агрегацией, в то время как `HAVING` выполняется после него. diff --git a/docs/ru/sql-reference/statements/select/index.md b/docs/ru/sql-reference/statements/select/index.md index b735d37118c..f5fe2788370 100644 --- a/docs/ru/sql-reference/statements/select/index.md +++ b/docs/ru/sql-reference/statements/select/index.md @@ -1,6 +1,8 @@ --- -toc_priority: 33 -toc_title: SELECT +title: "\u0421\u0438\u043d\u0442\u0430\u043a\u0441\u0438\u0441\u0020\u0437\u0430\u043f\u0440\u043e\u0441\u043e\u0432\u0020\u0053\u0045\u004c\u0045\u0043\u0054" +toc_folder_title: SELECT +toc_priority: 32 +toc_title: "\u041e\u0431\u0437\u043e\u0440" --- # Синтаксис запросов SELECT {#select-queries-syntax} diff --git a/docs/ru/sql-reference/statements/select/into-outfile.md b/docs/ru/sql-reference/statements/select/into-outfile.md index 0f5cf01e9d1..f956903f8b4 100644 --- a/docs/ru/sql-reference/statements/select/into-outfile.md +++ b/docs/ru/sql-reference/statements/select/into-outfile.md @@ -1,3 +1,7 @@ +--- +toc_title: INTO OUTFILE +--- + # Секция INTO OUTFILE {#into-outfile-clause} Чтобы перенаправить вывод `SELECT` запроса в указанный файл на стороне клиента, добавьте к нему секцию `INTO OUTFILE filename` (где filename — строковый литерал). diff --git a/docs/ru/sql-reference/statements/select/join.md b/docs/ru/sql-reference/statements/select/join.md index 6f1b5e2cde6..c5548d74156 100644 --- a/docs/ru/sql-reference/statements/select/join.md +++ b/docs/ru/sql-reference/statements/select/join.md @@ -1,3 +1,7 @@ +--- +toc_title: JOIN +--- + # Секция JOIN {#select-join} Join создаёт новую таблицу путем объединения столбцов из одной или нескольких таблиц с использованием общих для каждой из них значений. Это обычная операция в базах данных с поддержкой SQL, которая соответствует join из [реляционной алгебры](https://en.wikipedia.org/wiki/Relational_algebra#Joins_and_join-like_operators). Частный случай соединения одной таблицы часто называют «self-join». diff --git a/docs/ru/sql-reference/statements/select/limit-by.md b/docs/ru/sql-reference/statements/select/limit-by.md index ea5d467ae4f..fba81c023b5 100644 --- a/docs/ru/sql-reference/statements/select/limit-by.md +++ b/docs/ru/sql-reference/statements/select/limit-by.md @@ -1,3 +1,7 @@ +--- +toc_title: LIMIT BY +--- + # Секция LIMIT BY {#limit-by-clause} Запрос с секцией `LIMIT n BY expressions` выбирает первые `n` строк для каждого отличного значения `expressions`. Ключ `LIMIT BY` может содержать любое количество [выражений](../../syntax.md#syntax-expressions). diff --git a/docs/ru/sql-reference/statements/select/prewhere.md b/docs/ru/sql-reference/statements/select/prewhere.md index 1c8595d8e0c..c2a02b1a436 100644 --- a/docs/ru/sql-reference/statements/select/prewhere.md +++ b/docs/ru/sql-reference/statements/select/prewhere.md @@ -1,3 +1,7 @@ +--- +toc_title: PREWHERE +--- + # Секция PREWHERE {#prewhere-clause} Prewhere — это оптимизация для более эффективного применения фильтрации. Она включена по умолчанию, даже если секция `PREWHERE` явно не указана. В этом случае работает автоматическое перемещение части выражения из [WHERE](where.md) до стадии prewhere. Роль секции `PREWHERE` только для управления этой оптимизацией, если вы думаете, что знаете, как сделать перемещение условия лучше, чем это происходит по умолчанию. diff --git a/docs/ru/sql-reference/statements/select/sample.md b/docs/ru/sql-reference/statements/select/sample.md index ca6b49c9ad6..a71e8256f0f 100644 --- a/docs/ru/sql-reference/statements/select/sample.md +++ b/docs/ru/sql-reference/statements/select/sample.md @@ -1,3 +1,7 @@ +--- +toc_title: SAMPLE +--- + # Секция SAMPLE {#select-sample-clause} Секция `SAMPLE` позволяет выполнять запросы приближённо. Например, чтобы посчитать статистику по всем визитам, можно обработать 1/10 всех визитов и результат домножить на 10. diff --git a/docs/ru/sql-reference/statements/select/union-all.md b/docs/ru/sql-reference/statements/select/union-all.md index bffd667fa1d..b9d1f485a7b 100644 --- a/docs/ru/sql-reference/statements/select/union-all.md +++ b/docs/ru/sql-reference/statements/select/union-all.md @@ -1,3 +1,7 @@ +--- +toc_title: UNION ALL +--- + # Секция UNION ALL {#union-all-clause} Вы можете использовать `UNION ALL` чтобы объединить любое количество `SELECT` запросы путем расширения их результатов. Пример: diff --git a/docs/ru/sql-reference/statements/select/where.md b/docs/ru/sql-reference/statements/select/where.md index 63d081db43d..8cb8e634303 100644 --- a/docs/ru/sql-reference/statements/select/where.md +++ b/docs/ru/sql-reference/statements/select/where.md @@ -1,3 +1,7 @@ +--- +toc_title: WHERE +--- + # Секция WHERE {#select-where} Позволяет задать выражение, которое ClickHouse использует для фильтрации данных перед всеми другими действиями в запросе кроме выражений, содержащихся в секции [PREWHERE](prewhere.md#prewhere-clause). Обычно, это выражение с логическими операторами. diff --git a/docs/ru/sql-reference/statements/select/with.md b/docs/ru/sql-reference/statements/select/with.md index a5be733866f..4feae232bd7 100644 --- a/docs/ru/sql-reference/statements/select/with.md +++ b/docs/ru/sql-reference/statements/select/with.md @@ -1,3 +1,7 @@ +--- +toc_title: WITH +--- + # Секция WITH {#sektsiia-with} Данная секция представляет собой [Common Table Expressions](https://ru.wikipedia.org/wiki/Иерархические_и_рекурсивные_запросы_в_SQL), то есть позволяет использовать результаты выражений из секции `WITH` в остальной части `SELECT` запроса. diff --git a/docs/ru/sql-reference/statements/set-role.md b/docs/ru/sql-reference/statements/set-role.md index 5d840fcddb7..ccbef41aa9b 100644 --- a/docs/ru/sql-reference/statements/set-role.md +++ b/docs/ru/sql-reference/statements/set-role.md @@ -1,5 +1,5 @@ --- -toc_priority: 52 +toc_priority: 50 toc_title: SET ROLE --- diff --git a/docs/ru/sql-reference/statements/set.md b/docs/ru/sql-reference/statements/set.md index 0e12e2ee7bc..b60dfcf8324 100644 --- a/docs/ru/sql-reference/statements/set.md +++ b/docs/ru/sql-reference/statements/set.md @@ -1,5 +1,5 @@ --- -toc_priority: 51 +toc_priority: 49 toc_title: SET --- diff --git a/docs/ru/sql-reference/statements/show.md b/docs/ru/sql-reference/statements/show.md index 4b226765632..4516a401de9 100644 --- a/docs/ru/sql-reference/statements/show.md +++ b/docs/ru/sql-reference/statements/show.md @@ -1,3 +1,8 @@ +--- +toc_priority: 37 +toc_title: SHOW +--- + # SHOW Queries {#show-queries} ## SHOW CREATE TABLE {#show-create-table} diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index 1ba3a7555b9..4780e9b613f 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -1,3 +1,8 @@ +--- +toc_priority: 36 +toc_title: SYSTEM +--- + # Запросы SYSTEM {#query-language-system} - [RELOAD EMBEDDED DICTIONARIES](#query_language-system-reload-emdedded-dictionaries) diff --git a/docs/ru/sql-reference/statements/truncate.md b/docs/ru/sql-reference/statements/truncate.md index f8806496e48..4909d349658 100644 --- a/docs/ru/sql-reference/statements/truncate.md +++ b/docs/ru/sql-reference/statements/truncate.md @@ -1,5 +1,5 @@ --- -toc_priority: 53 +toc_priority: 51 toc_title: TRUNCATE --- diff --git a/docs/ru/sql-reference/statements/use.md b/docs/ru/sql-reference/statements/use.md index 84b0f5ed13f..c84329ea5ff 100644 --- a/docs/ru/sql-reference/statements/use.md +++ b/docs/ru/sql-reference/statements/use.md @@ -1,5 +1,5 @@ --- -toc_priority: 54 +toc_priority: 52 toc_title: USE --- diff --git a/docs/ru/sql-reference/syntax.md b/docs/ru/sql-reference/syntax.md index 5c819002cbe..ca73d3a137e 100644 --- a/docs/ru/sql-reference/syntax.md +++ b/docs/ru/sql-reference/syntax.md @@ -1,3 +1,8 @@ +--- +toc_priority: 31 +toc_title: "\u0421\u0438\u043d\u0442\u0430\u043a\u0441\u0438\u0441" +--- + # Синтаксис {#sintaksis} В системе есть два вида парсеров: полноценный парсер SQL (recursive descent parser) и парсер форматов данных (быстрый потоковый парсер). diff --git a/docs/ru/sql-reference/table-functions/file.md b/docs/ru/sql-reference/table-functions/file.md index 4581fa081d7..d3e6e106125 100644 --- a/docs/ru/sql-reference/table-functions/file.md +++ b/docs/ru/sql-reference/table-functions/file.md @@ -1,3 +1,8 @@ +--- +toc_priority: 37 +toc_title: file +--- + # file {#file} Создаёт таблицу из файла. Данная табличная функция похожа на табличные функции [file](file.md) и [hdfs](hdfs.md). diff --git a/docs/ru/sql-reference/table-functions/generate.md b/docs/ru/sql-reference/table-functions/generate.md index 9e6d36b2a4b..47b7e43bc86 100644 --- a/docs/ru/sql-reference/table-functions/generate.md +++ b/docs/ru/sql-reference/table-functions/generate.md @@ -1,3 +1,8 @@ +--- +toc_priority: 47 +toc_title: generateRandom +--- + # generateRandom {#generaterandom} Генерирует случайные данные с заданной схемой. diff --git a/docs/ru/sql-reference/table-functions/hdfs.md b/docs/ru/sql-reference/table-functions/hdfs.md index e90f27a2eca..6edd70b7b1b 100644 --- a/docs/ru/sql-reference/table-functions/hdfs.md +++ b/docs/ru/sql-reference/table-functions/hdfs.md @@ -1,3 +1,8 @@ +--- +toc_priority: 45 +toc_title: hdfs +--- + # hdfs {#hdfs} Создаёт таблицу из файла в HDFS. Данная табличная функция похожа на табличные функции [url](url.md) и [file](file.md). diff --git a/docs/ru/sql-reference/table-functions/input.md b/docs/ru/sql-reference/table-functions/input.md index 399268f9af6..96cf7515d52 100644 --- a/docs/ru/sql-reference/table-functions/input.md +++ b/docs/ru/sql-reference/table-functions/input.md @@ -1,3 +1,8 @@ +--- +toc_priority: 46 +toc_title: input +--- + # input {#input} `input(structure)` - табличная функция, позволяющая эффективно преобразовывать и вставлять отправленные на сервер данные, diff --git a/docs/ru/sql-reference/table-functions/jdbc.md b/docs/ru/sql-reference/table-functions/jdbc.md index 20ef0d1f107..d388262606f 100644 --- a/docs/ru/sql-reference/table-functions/jdbc.md +++ b/docs/ru/sql-reference/table-functions/jdbc.md @@ -1,3 +1,8 @@ +--- +toc_priority: 43 +toc_title: jdbc +--- + # jdbc {#jdbc} `jdbc(jdbc_connection_uri, schema, table)` - возвращает таблицу, соединение с которой происходит через JDBC-драйвер. diff --git a/docs/ru/sql-reference/table-functions/merge.md b/docs/ru/sql-reference/table-functions/merge.md index d4e66391382..0822fdfe535 100644 --- a/docs/ru/sql-reference/table-functions/merge.md +++ b/docs/ru/sql-reference/table-functions/merge.md @@ -1,3 +1,8 @@ +--- +toc_priority: 38 +toc_title: merge +--- + # merge {#merge} `merge(db_name, 'tables_regexp')` - создаёт временную таблицу типа Merge. Подробнее смотрите раздел «Движки таблиц, Merge». diff --git a/docs/ru/sql-reference/table-functions/mysql.md b/docs/ru/sql-reference/table-functions/mysql.md index 99d82022df4..21841eee67a 100644 --- a/docs/ru/sql-reference/table-functions/mysql.md +++ b/docs/ru/sql-reference/table-functions/mysql.md @@ -1,3 +1,8 @@ +--- +toc_priority: 42 +toc_title: mysql +--- + # mysql {#mysql} Позволяет выполнять запросы `SELECT` над данными, хранящимися на удалённом MySQL сервере. diff --git a/docs/ru/sql-reference/table-functions/numbers.md b/docs/ru/sql-reference/table-functions/numbers.md index 79d01dd0b92..005f400e082 100644 --- a/docs/ru/sql-reference/table-functions/numbers.md +++ b/docs/ru/sql-reference/table-functions/numbers.md @@ -1,3 +1,8 @@ +--- +toc_priority: 39 +toc_title: numbers +--- + # numbers {#numbers} `numbers(N)` - возвращает таблицу с единственным столбцом `number` (UInt64), содержащим натуральные числа от `0` до `N-1`. diff --git a/docs/ru/sql-reference/table-functions/odbc.md b/docs/ru/sql-reference/table-functions/odbc.md index 38da5066cbd..19203123840 100644 --- a/docs/ru/sql-reference/table-functions/odbc.md +++ b/docs/ru/sql-reference/table-functions/odbc.md @@ -1,3 +1,8 @@ +--- +toc_priority: 44 +toc_title: odbc +--- + # odbc {#table-functions-odbc} Возвращает таблицу, подключенную через [ODBC](https://en.wikipedia.org/wiki/Open_Database_Connectivity). diff --git a/docs/ru/sql-reference/table-functions/remote.md b/docs/ru/sql-reference/table-functions/remote.md index 944500b57a0..901317a805d 100644 --- a/docs/ru/sql-reference/table-functions/remote.md +++ b/docs/ru/sql-reference/table-functions/remote.md @@ -1,3 +1,8 @@ +--- +toc_priority: 40 +toc_title: remote +--- + # remote, remoteSecure {#remote-remotesecure} Позволяет обратиться к удалённым серверам без создания таблицы типа `Distributed`. diff --git a/docs/ru/sql-reference/table-functions/url.md b/docs/ru/sql-reference/table-functions/url.md index 4c49bc76751..0cd7c24c663 100644 --- a/docs/ru/sql-reference/table-functions/url.md +++ b/docs/ru/sql-reference/table-functions/url.md @@ -1,3 +1,8 @@ +--- +toc_priority: 41 +toc_title: url +--- + # url {#url} `url(URL, format, structure)` - возвращает таблицу со столбцами, указанными в diff --git a/docs/ru/whats-new/index.md b/docs/ru/whats-new/index.md index a8c908088d0..b8ba6133454 100644 --- a/docs/ru/whats-new/index.md +++ b/docs/ru/whats-new/index.md @@ -1,6 +1,6 @@ --- toc_folder_title: "\u0427\u0442\u043E \u043D\u043E\u0432\u043E\u0433\u043E?" -toc_priority: 72 +toc_priority: 82 --- diff --git a/docs/ru/whats-new/security-changelog.md b/docs/ru/whats-new/security-changelog.md index ad55e1eeb51..1f46535833d 100644 --- a/docs/ru/whats-new/security-changelog.md +++ b/docs/ru/whats-new/security-changelog.md @@ -1,3 +1,8 @@ +--- +toc_priority: 76 +toc_title: Security Changelog +--- + # Security Changelog {#security-changelog} ## Исправлено в релизе 19.14.3.3, 2019-09-10 {#ispravleno-v-relize-19-14-3-3-2019-09-10} From 1be7a182de47b40617d20a697cc432e6da40bf05 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 26 Oct 2020 14:08:37 +0300 Subject: [PATCH 207/325] Revert "Revert "Optionally upload clickhouse binary in fast test"" --- docker/test/fasttest/Dockerfile | 1 + docker/test/fasttest/run.sh | 3 +++ 2 files changed, 4 insertions(+) diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index bed438a6579..f6c665ff3fd 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -82,6 +82,7 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone ENV COMMIT_SHA='' ENV PULL_REQUEST_NUMBER='' +ENV COPY_CLICKHOUSE_BINARY_TO_OUTPUT=0 COPY run.sh / CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 06eb0668fe6..85a5dd866f8 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -172,6 +172,9 @@ function build ( cd "$FASTTEST_BUILD" time ninja clickhouse-bundle | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/build_log.txt" +if [ "$COPY_CLICKHOUSE_BINARY_TO_OUTPUT" -eq "1" ]; then + cp programs/clickhouse "$FASTTEST_OUTPUT/clickhouse" +fi ccache --show-stats ||: ) } From 4e8bea397b8b4ef5cfd6d6757781a0d59d5ab6a5 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 26 Oct 2020 15:10:16 +0300 Subject: [PATCH 208/325] Update ExpressionActions.cpp --- src/Interpreters/ExpressionActions.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 6ce9fe2c793..36d128b4234 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -642,7 +642,7 @@ void ExpressionActions::execute(Block & block, bool dry_run) const #if defined(ADDRESS_SANITIZER) const auto & msg = e.message(); - if (__asan_region_is_poisoned(msg.data(), msg.size())) + if (__asan_region_is_poisoned(const_cast(msg.data()), msg.size())) { LOG_FATAL(&Poco::Logger::get("ExpressionActions"), "Poisoned exception message (asan): {}", e.getStackTraceString()); } From 9ed4668dbbd8b2687586e1eac181170d8c8ba5f8 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 26 Oct 2020 15:40:55 +0300 Subject: [PATCH 209/325] Refactor common part of background list --- src/Interpreters/Context.cpp | 1 + src/Storages/MergeTree/MergeList.h | 85 +++++-------------- src/Storages/MergeTree/MergeTreeData.h | 1 - .../MergeTree/MergeTreeDataMergerMutator.h | 1 - 4 files changed, 21 insertions(+), 67 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 9d2ea6ded86..197c183a06b 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeList.h b/src/Storages/MergeTree/MergeList.h index c1166c55703..cf25721f2df 100644 --- a/src/Storages/MergeTree/MergeList.h +++ b/src/Storages/MergeTree/MergeList.h @@ -1,21 +1,20 @@ #pragma once +#include +#include #include #include #include -#include -#include #include +#include +#include +#include #include #include #include #include -/** Maintains a list of currently running merges. - * For implementation of system.merges table. - */ - namespace CurrentMetrics { extern const Metric Merge; @@ -101,68 +100,36 @@ struct MergeListElement : boost::noncopyable ~MergeListElement(); }; +using MergeListEntry = BackgroundProcessListEntry; -class MergeList; - -class MergeListEntry +/** Maintains a list of currently running merges. + * For implementation of system.merges table. + */ +class MergeList final : public BackgroundProcessList { - MergeList & list; - - using container_t = std::list; - container_t::iterator it; - - CurrentMetrics::Increment num_merges {CurrentMetrics::Merge}; - -public: - MergeListEntry(const MergeListEntry &) = delete; - MergeListEntry & operator=(const MergeListEntry &) = delete; - - MergeListEntry(MergeList & list_, const container_t::iterator it_) : list(list_), it{it_} {} - ~MergeListEntry(); - - MergeListElement * operator->() { return &*it; } - const MergeListElement * operator->() const { return &*it; } -}; - - -class MergeList -{ - friend class MergeListEntry; - - using container_t = std::list; - using info_container_t = std::list; - - mutable std::mutex mutex; - container_t merges; - + using Parent = BackgroundProcessList; std::atomic merges_with_ttl_counter = 0; public: - using Entry = MergeListEntry; - using EntryPtr = std::unique_ptr; + MergeList() + : Parent(CurrentMetrics::Merge) + {} - template - EntryPtr insert(Args &&... args) + void onEntryCreate(const Parent::Entry & entry) override { - std::lock_guard lock{mutex}; - auto entry = std::make_unique(*this, merges.emplace(merges.end(), std::forward(args)...)); - if (isTTLMergeType((*entry)->merge_type)) + if (isTTLMergeType(entry->merge_type)) ++merges_with_ttl_counter; - return entry; } - info_container_t get() const + void onEntryDestroy(const Parent::Entry & entry) override { - std::lock_guard lock{mutex}; - info_container_t res; - for (const auto & merge_element : merges) - res.emplace_back(merge_element.getInfo()); - return res; + if (isTTLMergeType(entry->merge_type)) + --merges_with_ttl_counter; } void cancelPartMutations(const String & partition_id, Int64 mutation_version) { std::lock_guard lock{mutex}; - for (auto & merge_element : merges) + for (auto & merge_element : entries) { if ((partition_id.empty() || merge_element.partition_id == partition_id) && merge_element.source_data_version < mutation_version @@ -177,16 +144,4 @@ public: } }; - -inline MergeListEntry::~MergeListEntry() -{ - std::lock_guard lock{list.mutex}; - - if (isTTLMergeType(it->merge_type)) - --list.merges_with_ttl_counter; - - list.merges.erase(it); -} - - } diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 5c18661dad1..889e4fb16b4 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -35,7 +35,6 @@ namespace DB { -class MergeListEntry; class AlterCommands; class MergeTreePartsMover; class MutationCommands; diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index 0ad525d1901..570bbecb165 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -13,7 +13,6 @@ namespace DB { -class MergeListEntry; class MergeProgressCallback; /// Auxiliary struct holding metainformation for the future merged or mutated part. From bd17843bdb2a1ee2a209d11b5f1d6dd8e197c99a Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 26 Oct 2020 16:00:49 +0300 Subject: [PATCH 210/325] Update uuid-functions.md --- docs/en/sql-reference/functions/uuid-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/uuid-functions.md b/docs/en/sql-reference/functions/uuid-functions.md index b747ac07bb8..01a61c65b67 100644 --- a/docs/en/sql-reference/functions/uuid-functions.md +++ b/docs/en/sql-reference/functions/uuid-functions.md @@ -71,7 +71,7 @@ toUUIDOrNull(String) **Returned value** -The Nullable UUID type value. +The Nullable(UUID) type value. **Usage example** From ab4c43cb812a3f06628db5b04813bc62882926b3 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 26 Oct 2020 16:44:46 +0300 Subject: [PATCH 211/325] fix group by with totals/rollup/cube modifers and min/max functions over group by keys --- src/Interpreters/TreeOptimizer.cpp | 7 ++++++- .../01532_min_max_with_modifiers.reference | 20 +++++++++++++++++++ .../01532_min_max_with_modifiers.sql | 18 +++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01532_min_max_with_modifiers.reference create mode 100644 tests/queries/0_stateless/01532_min_max_with_modifiers.sql diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 9d52e30cea4..61ca933dd53 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -644,8 +644,13 @@ void TreeOptimizer::apply(ASTPtr & query, Aliases & aliases, const NameSet & sou optimizeInjectiveFunctionsInsideUniq(query, context); /// Eliminate min/max/any aggregators of functions of GROUP BY keys - if (settings.optimize_aggregators_of_group_by_keys) + if (settings.optimize_aggregators_of_group_by_keys + && !select_query->group_by_with_totals + && !select_query->group_by_with_rollup + && !select_query->group_by_with_cube) + { optimizeAggregateFunctionsOfGroupByKeys(select_query, query); + } /// Remove duplicate items from ORDER BY. optimizeDuplicatesInOrderBy(select_query); diff --git a/tests/queries/0_stateless/01532_min_max_with_modifiers.reference b/tests/queries/0_stateless/01532_min_max_with_modifiers.reference new file mode 100644 index 00000000000..9e1683ca6f9 --- /dev/null +++ b/tests/queries/0_stateless/01532_min_max_with_modifiers.reference @@ -0,0 +1,20 @@ +totals +1 1 1 +2 2 2 +3 3 3 + +0 1 3 +rollup +1 1 1 +2 2 2 +3 3 3 +0 1 3 +cube +1 1 1 +2 2 2 +3 3 3 +0 1 3 +======= +1 1 2 1 +2 2 3 1 +0 1 3 2 diff --git a/tests/queries/0_stateless/01532_min_max_with_modifiers.sql b/tests/queries/0_stateless/01532_min_max_with_modifiers.sql new file mode 100644 index 00000000000..0c8651c0f01 --- /dev/null +++ b/tests/queries/0_stateless/01532_min_max_with_modifiers.sql @@ -0,0 +1,18 @@ +SELECT 'totals'; +SELECT number % 3 + 1 AS n, min(n), max(n) FROM numbers(100) GROUP BY n WITH TOTALS; +SELECT 'rollup'; +SELECT number % 3 + 1 AS n, min(n), max(n) FROM numbers(100) GROUP BY n WITH ROLLUP; +SELECT 'cube'; +SELECT number % 3 + 1 AS n, min(n), max(n) FROM numbers(100) GROUP BY n WITH CUBE; +SELECT '======='; + +SELECT + x, + min(x) AS lower, + max(x) + 1 AS upper, + upper - lower AS range +FROM +( + SELECT arrayJoin([1, 2]) AS x +) +GROUP BY x WITH ROLLUP; From 44162c28c00654b75c98a990d065c04c270807aa Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 26 Oct 2020 19:09:43 +0300 Subject: [PATCH 212/325] Update LocalServer.cpp --- programs/local/LocalServer.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index f85b5bde965..63667307876 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -451,9 +451,7 @@ void LocalServer::setupUsers() void LocalServer::cleanup() { - // Delete the temporary directory if needed. Just in case, check that it is - // in the system temporary directory, not to delete user data if there is a - // bug. + // Delete the temporary directory if needed. if (temporary_directory_to_delete) { const auto dir = *temporary_directory_to_delete; From 79a6be08d132e6fe2a55c638e5abeb3677aafd2a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 26 Oct 2020 19:09:53 +0300 Subject: [PATCH 213/325] Update src/Interpreters/Context.cpp --- src/Interpreters/Context.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 6e5274cd5b3..328e122fdb5 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -2029,6 +2029,8 @@ void Context::reloadConfig() const void Context::shutdown() { + // Disk selector might not be initialized if there was some error during + // its initialization. Don't try to initialize it again on shutdown. if (shared->merge_tree_disk_selector) { for (auto & [disk_name, disk] : getDisksMap()) From 60db7c2aa5882b74b9b30798f750cfd68b34fbc3 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 26 Oct 2020 19:17:54 +0300 Subject: [PATCH 214/325] fix a typo in test name --- docker/test/fasttest/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 85a5dd866f8..8300c31681e 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -271,7 +271,7 @@ TESTS_TO_SKIP=( 00974_query_profiler # Look at DistributedFilesToInsert, so cannot run in parallel. - 01457_DistributedFilesToInsert + 01460_DistributedFilesToInsert 01541_max_memory_usage_for_user From e27dd538fd4e51dc7823c7530509241e97b28e51 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 26 Oct 2020 19:35:37 +0300 Subject: [PATCH 215/325] performance comparison --- docker/test/performance-comparison/compare.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 258bc0a95f7..30443f21fba 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -77,12 +77,9 @@ function restart while killall clickhouse-server; do echo . ; sleep 1 ; done echo all killed - # Disable percpu arenas because they segfault when the process is bound to - # a particular NUMA node: https://github.com/jemalloc/jemalloc/pull/1939 - # - # About the jemalloc settings: + # Change the jemalloc settings here. # https://github.com/jemalloc/jemalloc/wiki/Getting-Started - export MALLOC_CONF="percpu_arena:disabled,confirm_conf:true" + export MALLOC_CONF="confirm_conf:true" set -m # Spawn servers in their own process groups From 880f4bbd05d0a9ec3dc6b3d0aec357b9890e9ec0 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 26 Oct 2020 19:38:35 +0300 Subject: [PATCH 216/325] System fetches --- src/IO/ReadWriteBufferFromHTTP.h | 8 ++ src/Interpreters/Context.cpp | 4 + src/Interpreters/Context.h | 4 + .../MergeTree/BackgroundProcessList.h | 89 +++++++++++++++++ src/Storages/MergeTree/DataPartsExchange.cpp | 29 +++++- src/Storages/MergeTree/MergeList.h | 1 + .../MergeTree/ReplicatedFetchesList.cpp | 62 ++++++++++++ .../MergeTree/ReplicatedFetchesList.h | 95 +++++++++++++++++++ src/Storages/System/StorageSystemFetches.cpp | 59 ++++++++++++ src/Storages/System/StorageSystemFetches.h | 28 ++++++ 10 files changed, 378 insertions(+), 1 deletion(-) create mode 100644 src/Storages/MergeTree/BackgroundProcessList.h create mode 100644 src/Storages/MergeTree/ReplicatedFetchesList.cpp create mode 100644 src/Storages/MergeTree/ReplicatedFetchesList.h create mode 100644 src/Storages/System/StorageSystemFetches.cpp create mode 100644 src/Storages/System/StorageSystemFetches.h diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 48407f76938..c79a3bd953d 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -106,6 +106,7 @@ namespace detail std::vector cookies; HTTPHeaderEntries http_header_entries; RemoteHostFilter remote_host_filter; + std::function read_callback; std::istream * call(const Poco::URI uri_, Poco::Net::HTTPResponse & response) { @@ -208,6 +209,8 @@ namespace detail return false; internal_buffer = impl->buffer(); working_buffer = internal_buffer; + if (read_callback) + read_callback(count()); return true; } @@ -218,6 +221,11 @@ namespace detail return cookie.getValue(); return def; } + + void setNextReadCallback(std::function read_callback_) + { + read_callback = read_callback_; + } }; } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 197c183a06b..38e8ffdf76a 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -329,6 +330,7 @@ struct ContextShared mutable MarkCachePtr mark_cache; /// Cache of marks in compressed files. ProcessList process_list; /// Executing queries at the moment. MergeList merge_list; /// The list of executable merge (for (Replicated)?MergeTree) + ReplicatedFetchList replicated_fetch_list; ConfigurationPtr users_config; /// Config with the users, profiles and quotas sections. InterserverIOHandler interserver_io_handler; /// Handler for interserver communication. std::optional buffer_flush_schedule_pool; /// A thread pool that can do background flush for Buffer tables. @@ -506,6 +508,8 @@ ProcessList & Context::getProcessList() { return shared->process_list; } const ProcessList & Context::getProcessList() const { return shared->process_list; } MergeList & Context::getMergeList() { return shared->merge_list; } const MergeList & Context::getMergeList() const { return shared->merge_list; } +ReplicatedFetchList & Context::getReplicatedFetchList() { return shared->replicated_fetch_list; } +const ReplicatedFetchList & Context::getReplicatedFetchList() const { return shared->replicated_fetch_list; } void Context::enableNamedSessions() diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 075fc3837ef..a072906a0ba 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -65,6 +65,7 @@ class InterserverIOHandler; class BackgroundProcessingPool; class BackgroundSchedulePool; class MergeList; +class ReplicatedFetchList; class Cluster; class Compiler; class MarkCache; @@ -477,6 +478,9 @@ public: MergeList & getMergeList(); const MergeList & getMergeList() const; + ReplicatedFetchList & getReplicatedFetchList(); + const ReplicatedFetchList & getReplicatedFetchList() const; + /// If the current session is expired at the time of the call, synchronously creates and returns a new session with the startNewSession() call. /// If no ZooKeeper configured, throws an exception. std::shared_ptr getZooKeeper() const; diff --git a/src/Storages/MergeTree/BackgroundProcessList.h b/src/Storages/MergeTree/BackgroundProcessList.h new file mode 100644 index 00000000000..f90aa1c88a8 --- /dev/null +++ b/src/Storages/MergeTree/BackgroundProcessList.h @@ -0,0 +1,89 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace DB +{ + + +template +class BackgroundProcessList; + +template +class BackgroundProcessListEntry +{ + BackgroundProcessList & list; + using container_t = std::list; + typename container_t::iterator it; + CurrentMetrics::Increment metric_increment; +public: + BackgroundProcessListEntry(const BackgroundProcessListEntry &) = delete; + BackgroundProcessListEntry & operator=(const BackgroundProcessListEntry &) = delete; + + BackgroundProcessListEntry(BackgroundProcessList & list_, const typename container_t::iterator it_, const CurrentMetrics::Metric & metric) + : list(list_), it{it_}, metric_increment{metric} + { + list.onEntryCreate(*this); + } + + ~BackgroundProcessListEntry() + { + std::lock_guard lock{list.mutex}; + list.onEntryDestroy(*this); + list.entries.erase(it); + } + + ListElement * operator->() { return &*it; } + const ListElement * operator->() const { return &*it; } +}; + + +template +class BackgroundProcessList +{ +protected: + friend class BackgroundProcessListEntry; + + using container_t = std::list; + using info_container_t = std::list; + + mutable std::mutex mutex; + container_t entries; + + CurrentMetrics::Metric metric; + + BackgroundProcessList(const CurrentMetrics::Metric & metric_) + : metric(metric_) + {} +public: + + using Entry = BackgroundProcessListEntry; + using EntryPtr = std::unique_ptr; + + template + EntryPtr insert(Args &&... args) + { + std::lock_guard lock{mutex}; + auto entry = std::make_unique(*this, entries.emplace(entries.end(), std::forward(args)...), metric); + return entry; + } + + info_container_t get() const + { + std::lock_guard lock{mutex}; + info_container_t res; + for (const auto & list_element : entries) + res.emplace_back(list_element.getInfo()); + return res; + } + + virtual void onEntryCreate(const Entry & /* entry */) {} + virtual void onEntryDestroy(const Entry & /* entry */) {} + virtual inline ~BackgroundProcessList() {} +}; + +} diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 0e79404e59d..b1f1dc3f5f5 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace CurrentMetrics @@ -52,6 +53,25 @@ std::string getEndpointId(const std::string & node_id) return "DataPartsExchange:" + node_id; } + +struct ReplicatedFetchReadCallback +{ + ReplicatedFetchList::Entry & replicated_fetch_entry; + + ReplicatedFetchReadCallback(ReplicatedFetchList::Entry & replicated_fetch_entry_) + : replicated_fetch_entry(replicated_fetch_entry_) + {} + + + void operator() (size_t bytes_count) + { + replicated_fetch_entry->bytes_read_compressed = bytes_count; + replicated_fetch_entry->progress.store( + replicated_fetch_entry->bytes_read_compressed.load(std::memory_order_relaxed) / replicated_fetch_entry->total_size_bytes_compressed, + std::memory_order_relaxed); + } +}; + } std::string Service::getId(const std::string & node_id) const @@ -228,7 +248,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( throw Exception("Fetching of part was cancelled", ErrorCodes::ABORTED); /// Validation of the input that may come from malicious replica. - MergeTreePartInfo::fromPartName(part_name, data.format_version); + auto part_info = MergeTreePartInfo::fromPartName(part_name, data.format_version); const auto data_settings = data.getSettings(); Poco::URI uri; @@ -286,6 +306,13 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( /// We don't know real size of part because sender server version is too old reservation = data.makeEmptyReservationOnLargestDisk(); } + auto storage_id = data.getStorageID(); + auto entry = data.global_context.getReplicatedFetchList().insert( + storage_id.getDatabaseName(), storage_id.getTableName(), + part_info.partition_id, part_name, part_name, + replica_path, uri.toString(), interserver_scheme, to_detached, sum_files_size); + + in.setNextReadCallback(ReplicatedFetchReadCallback(*entry)); bool sync = (data_settings->min_compressed_bytes_to_fsync_after_fetch && sum_files_size >= data_settings->min_compressed_bytes_to_fsync_after_fetch); diff --git a/src/Storages/MergeTree/MergeList.h b/src/Storages/MergeTree/MergeList.h index cf25721f2df..8d97b899aac 100644 --- a/src/Storages/MergeTree/MergeList.h +++ b/src/Storages/MergeTree/MergeList.h @@ -107,6 +107,7 @@ using MergeListEntry = BackgroundProcessListEntry; */ class MergeList final : public BackgroundProcessList { +private: using Parent = BackgroundProcessList; std::atomic merges_with_ttl_counter = 0; public: diff --git a/src/Storages/MergeTree/ReplicatedFetchesList.cpp b/src/Storages/MergeTree/ReplicatedFetchesList.cpp new file mode 100644 index 00000000000..d66d0965351 --- /dev/null +++ b/src/Storages/MergeTree/ReplicatedFetchesList.cpp @@ -0,0 +1,62 @@ +#include +#include +#include +#include + +namespace DB +{ +ReplicatedFetchListElement::ReplicatedFetchListElement( + const std::string & database_, const std::string & table_, + const std::string & partition_id_, const std::string & result_part_name_, + const std::string & result_part_path_, const std::string & source_replica_path_, + const std::string & source_replica_address_, const std::string & interserver_scheme_, + UInt8 to_detached_, UInt64 total_size_bytes_compressed_) + : database(database_) + , table(table_) + , partition_id(partition_id_) + , result_part_name(result_part_name_) + , result_part_path(result_part_path_) + , source_replica_path(source_replica_path_) + , source_replica_address(source_replica_address_) + , interserver_scheme(interserver_scheme_) + , to_detached(to_detached_) + , total_size_bytes_compressed(total_size_bytes_compressed_) +{ + background_thread_memory_tracker = CurrentThread::getMemoryTracker(); + if (background_thread_memory_tracker) + { + background_thread_memory_tracker_prev_parent = background_thread_memory_tracker->getParent(); + background_thread_memory_tracker->setParent(&memory_tracker); + } +} + + +ReplicatedFetchInfo ReplicatedFetchListElement::getInfo() const +{ + ReplicatedFetchInfo res; + res.database = database; + res.table = table; + res.partition_id = partition_id; + res.result_part_name = result_part_name; + res.result_part_path = result_part_path; + res.source_replica_path = source_replica_path; + res.source_replica_address = source_replica_address; + res.interserver_scheme = interserver_scheme; + res.to_detached = to_detached; + res.elapsed = watch.elapsedSeconds(); + res.progress = progress.load(std::memory_order_relaxed); + res.bytes_read_compressed = bytes_read_compressed.load(std::memory_order_relaxed); + res.total_size_bytes_compressed = total_size_bytes_compressed; + res.memory_usage = memory_tracker.get(); + res.thread_id = thread_id; + return res; +} + +ReplicatedFetchListElement::~ReplicatedFetchListElement() +{ + /// Unplug memory_tracker from current background processing pool thread + if (background_thread_memory_tracker) + background_thread_memory_tracker->setParent(background_thread_memory_tracker_prev_parent); +} + +} diff --git a/src/Storages/MergeTree/ReplicatedFetchesList.h b/src/Storages/MergeTree/ReplicatedFetchesList.h new file mode 100644 index 00000000000..ad8edb6ad59 --- /dev/null +++ b/src/Storages/MergeTree/ReplicatedFetchesList.h @@ -0,0 +1,95 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace CurrentMetrics +{ + extern const Metric ReplicatedFetch; +} + +namespace DB +{ + +struct ReplicatedFetchInfo +{ + std::string database; + std::string table; + std::string partition_id; + + std::string result_part_name; + std::string result_part_path; + + std::string source_replica_path; + std::string source_replica_address; + std::string interserver_scheme; + + UInt8 to_detached; + + Float64 elapsed; + Float64 progress; + + UInt64 total_size_bytes_compressed; + UInt64 bytes_read_compressed; + + UInt64 memory_usage; + UInt64 thread_id; +}; + + +struct ReplicatedFetchListElement : private boost::noncopyable +{ + const std::string database; + const std::string table; + const std::string partition_id; + + const std::string result_part_name; + const std::string result_part_path; + + const std::string source_replica_path; + const std::string source_replica_address; + const std::string interserver_scheme; + + const UInt8 to_detached; + + Stopwatch watch; + std::atomic progress{}; + std::atomic is_cancelled{}; + std::atomic bytes_read_compressed{}; + UInt64 total_size_bytes_compressed{}; + + MemoryTracker memory_tracker{VariableContext::Process}; + MemoryTracker * background_thread_memory_tracker; + MemoryTracker * background_thread_memory_tracker_prev_parent = nullptr; + + UInt64 thread_id; + + ReplicatedFetchListElement( + const std::string & database_, const std::string & table_, + const std::string & partition_id_, const std::string & result_part_name_, + const std::string & result_part_path_, const std::string & source_replica_path_, + const std::string & source_replica_address_, const std::string & interserver_scheme_, + UInt8 to_detached_, UInt64 total_size_bytes_compressed_); + + ReplicatedFetchInfo getInfo() const; + + ~ReplicatedFetchListElement(); +}; + + +using ReplicatedFetchListEntry = BackgroundProcessListEntry; + +class ReplicatedFetchList final : public BackgroundProcessList +{ +private: + using Parent = BackgroundProcessList; + +public: + ReplicatedFetchList () + : Parent(CurrentMetrics::ReplicatedFetch) + {} +}; + +} diff --git a/src/Storages/System/StorageSystemFetches.cpp b/src/Storages/System/StorageSystemFetches.cpp new file mode 100644 index 00000000000..27d4eeddbfc --- /dev/null +++ b/src/Storages/System/StorageSystemFetches.cpp @@ -0,0 +1,59 @@ +#include +#include +#include +#include + +namespace DB +{ + +NamesAndTypesList StorageSystemFetches::getNamesAndTypes() +{ + return { + {"database", std::make_shared()}, + {"table", std::make_shared()}, + {"elapsed", std::make_shared()}, + {"progress", std::make_shared()}, + {"result_part_name", std::make_shared()}, + {"result_part_path", std::make_shared()}, + {"partition_id", std::make_shared()}, + {"total_size_bytes_compressed", std::make_shared()}, + {"bytes_read_compressed", std::make_shared()}, + {"source_replica_path", std::make_shared()}, + {"source_replica_address", std::make_shared()}, + {"interserver_scheme", std::make_shared()}, + {"to_detached", std::make_shared()}, + {"memory_usage", std::make_shared()}, + {"thread_id", std::make_shared()}, + }; +} + +void StorageSystemFetches::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +{ + const auto access = context.getAccess(); + const bool check_access_for_tables = !access->isGranted(AccessType::SHOW_TABLES); + + for (const auto & fetch : context.getReplicatedFetchList().get()) + { + if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, fetch.database, fetch.table)) + continue; + + size_t i = 0; + res_columns[i++]->insert(fetch.database); + res_columns[i++]->insert(fetch.table); + res_columns[i++]->insert(fetch.elapsed); + res_columns[i++]->insert(fetch.progress); + res_columns[i++]->insert(fetch.result_part_name); + res_columns[i++]->insert(fetch.result_part_path); + res_columns[i++]->insert(fetch.partition_id); + res_columns[i++]->insert(fetch.total_size_bytes_compressed); + res_columns[i++]->insert(fetch.bytes_read_compressed); + res_columns[i++]->insert(fetch.source_replica_path); + res_columns[i++]->insert(fetch.source_replica_address); + res_columns[i++]->insert(fetch.interserver_scheme); + res_columns[i++]->insert(fetch.to_detached); + res_columns[i++]->insert(fetch.memory_usage); + res_columns[i++]->insert(fetch.thread_id); + } +} + +} diff --git a/src/Storages/System/StorageSystemFetches.h b/src/Storages/System/StorageSystemFetches.h new file mode 100644 index 00000000000..be1b66193bf --- /dev/null +++ b/src/Storages/System/StorageSystemFetches.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ + +class Context; + +class StorageSystemFetches final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +{ + friend struct ext::shared_ptr_helper; +public: + std::string getName() const override { return "SystemFetches"; } + + static NamesAndTypesList getNamesAndTypes(); + +protected: + using IStorageSystemOneBlock::IStorageSystemOneBlock; + + void fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo & query_info) const override; +}; + +} From d412e0ca2c9b5a856a9476ea15e98d92c58afa8f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 26 Oct 2020 19:41:38 +0300 Subject: [PATCH 217/325] Move trash to "Unbundled" Dockerfile --- docker/packager/deb/Dockerfile | 4 ---- docker/packager/unbundled/Dockerfile | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index 4e9259b214a..8fd89d60f85 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -31,10 +31,6 @@ RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb \ && chmod +x dpkg-deb \ && cp dpkg-deb /usr/bin -RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ - && wget -nv -O /tmp/arrow-keyring.deb "https://apache.bintray.com/arrow/ubuntu/apache-arrow-archive-keyring-latest-${CODENAME}.deb" \ - && dpkg -i /tmp/arrow-keyring.deb - # Libraries from OS are only needed to test the "unbundled" build (this is not used in production). RUN apt-get update \ && apt-get install \ diff --git a/docker/packager/unbundled/Dockerfile b/docker/packager/unbundled/Dockerfile index 604f187cc58..50671011a23 100644 --- a/docker/packager/unbundled/Dockerfile +++ b/docker/packager/unbundled/Dockerfile @@ -1,6 +1,10 @@ # docker build -t yandex/clickhouse-unbundled-builder . FROM yandex/clickhouse-deb-builder +RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ + && wget -nv -O /tmp/arrow-keyring.deb "https://apache.bintray.com/arrow/ubuntu/apache-arrow-archive-keyring-latest-${CODENAME}.deb" \ + && dpkg -i /tmp/arrow-keyring.deb + # Libraries from OS are only needed to test the "unbundled" build (that is not used in production). RUN apt-get update \ && apt-get install \ From e449266a16344d60071c141fa64556b03fdefc33 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 26 Oct 2020 22:06:30 +0300 Subject: [PATCH 218/325] Fix TSan report in lgamma --- src/AggregateFunctions/AggregateFunctionStudentTTest.h | 5 +++-- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.h b/src/AggregateFunctions/AggregateFunctionStudentTTest.h index d260a6be980..a88f8151b31 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.h +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.h @@ -98,7 +98,7 @@ struct AggregateFunctionStudentTTestData final Float64 getSSquared() const { - /// The original formulae looks like + /// The original formulae looks like /// \frac{\sum_{i = 1}^{n_x}{(x_i - \bar{x}) ^ 2} + \sum_{i = 1}^{n_y}{(y_i - \bar{y}) ^ 2}}{n_x + n_y - 2} /// But we made some mathematical transformations not to store original sequences. /// Also we dropped sqrt, because later it will be squared later. @@ -150,7 +150,8 @@ struct AggregateFunctionStudentTTestData final const Float64 t = getTStatisticSquared(); auto f = [&v] (double x) { return std::pow(x, v/2 - 1) / std::sqrt(1 - x); }; Float64 numenator = integrateSimpson(0, v / (t + v), f); - Float64 denominator = std::exp(std::lgammal(v/2) + std::lgammal(0.5) - std::lgammal(v/2 + 0.5)); + int unused; + Float64 denominator = std::exp(lgammal_r(v / 2, &unused) + lgammal_r(0.5, &unused) - lgammal_r(v / 2 + 0.5, &unused)); return numenator / denominator; } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 175e0171606..5f0cc409ba9 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -18,6 +18,7 @@ #include + namespace ErrorCodes { extern const int BAD_ARGUMENTS; @@ -159,9 +160,10 @@ struct AggregateFunctionWelchTTestData final { const Float64 v = getDegreesOfFreedom(); const Float64 t = getTStatisticSquared(); - auto f = [&v] (double x) { return std::pow(x, v/2 - 1) / std::sqrt(1 - x); }; + auto f = [&v] (double x) { return std::pow(x, v / 2 - 1) / std::sqrt(1 - x); }; Float64 numenator = integrateSimpson(0, v / (t + v), f); - Float64 denominator = std::exp(std::lgammal(v/2) + std::lgammal(0.5) - std::lgammal(v/2 + 0.5)); + int unused; + Float64 denominator = std::exp(lgammal_r(v / 2, &unused) + lgammal_r(0.5, &unused) - lgammal_r(v / 2 + 0.5, &unused)); return numenator / denominator; } From 32ed8c968158780eb0e3fba4966b73019e06ace4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 26 Oct 2020 22:12:40 +0300 Subject: [PATCH 219/325] Fix trailing whitespace --- src/Access/ContextAccess.cpp | 2 +- src/Columns/ColumnNullable.cpp | 2 +- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 2 +- src/DataStreams/SquashingTransform.cpp | 2 +- src/Interpreters/InterpreterShowTablesQuery.cpp | 2 +- src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp | 2 +- src/Interpreters/PreparedSets.h | 3 ++- src/Processors/Transforms/ConvertingTransform.h | 4 +++- utils/check-style/check-style | 3 +++ 9 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 61ab4c8002d..0459022cb1a 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -100,7 +100,7 @@ namespace if (res & alter_table) res |= alter_view; - /// CREATE TABLE (on any database/table) => CREATE_TEMPORARY_TABLE (global) + /// CREATE TABLE (on any database/table) => CREATE_TEMPORARY_TABLE (global) static const AccessFlags create_temporary_table = AccessType::CREATE_TEMPORARY_TABLE; if ((level == 0) && (max_flags_with_children & create_table)) res |= create_temporary_table; diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp index bdbc941c1e7..51248a598af 100644 --- a/src/Columns/ColumnNullable.cpp +++ b/src/Columns/ColumnNullable.cpp @@ -344,7 +344,7 @@ void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_dire /// Shift all NULL values to the end. for (const auto & [first, last] : equal_ranges) { - /// Current interval is righter than limit. + /// Current interval is righter than limit. if (limit && first > limit) break; diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index abb8158781b..2db5fabbf74 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1294,7 +1294,7 @@ void ZooKeeper::receiveEvent() if (request_info.watch) { bool add_watch = false; - /// 3 indicates the ZooKeeperExistsRequest. + /// 3 indicates the ZooKeeperExistsRequest. // For exists, we set the watch on both node exist and nonexist case. // For other case like getData, we only set the watch when node exists. if (request_info.request->getOpNum() == 3) diff --git a/src/DataStreams/SquashingTransform.cpp b/src/DataStreams/SquashingTransform.cpp index c57e2351230..1f6ca8a7306 100644 --- a/src/DataStreams/SquashingTransform.cpp +++ b/src/DataStreams/SquashingTransform.cpp @@ -27,7 +27,7 @@ Block SquashingTransform::add(const Block & input_block) /* * To minimize copying, accept two types of argument: const reference for output - * stream, and rvalue reference for input stream, and decide whether to copy + * stream, and rvalue reference for input stream, and decide whether to copy * inside this function. This allows us not to copy Block unless we absolutely * have to. */ diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index ef7fd840ac5..cb5db386f5a 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -50,7 +50,7 @@ String InterpreterShowTablesQuery::getRewrittenQuery() return rewritten_query.str(); } - /// SHOW CLUSTER/CLUSTERS + /// SHOW CLUSTER/CLUSTERS if (query.clusters) { std::stringstream rewritten_query; diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp index 70916fe386d..245feae166d 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp @@ -56,7 +56,7 @@ static inline String resolveDatabase( } } - /// When USE other_database_name; CREATE TABLE table_name; + /// When USE other_database_name; CREATE TABLE table_name; /// context.getCurrentDatabase() is always return `default database` /// When USE replica_mysql_database; CREATE TABLE table_name; /// context.getCurrentDatabase() is always return replica_clickhouse_database diff --git a/src/Interpreters/PreparedSets.h b/src/Interpreters/PreparedSets.h index 0faa748303d..f486752e192 100644 --- a/src/Interpreters/PreparedSets.h +++ b/src/Interpreters/PreparedSets.h @@ -6,6 +6,7 @@ #include #include + namespace DB { @@ -16,7 +17,7 @@ struct PreparedSetKey /// if left hand sides of the IN operators have different types). static PreparedSetKey forLiteral(const IAST & ast, DataTypes types_) { - /// Remove LowCardinality types from type list because Set doesn't support LowCardinality keys now, + /// Remove LowCardinality types from type list because Set doesn't support LowCardinality keys now, /// just converts LowCardinality to ordinary types. for (auto & type : types_) type = recursiveRemoveLowCardinality(type); diff --git a/src/Processors/Transforms/ConvertingTransform.h b/src/Processors/Transforms/ConvertingTransform.h index b426a2ab525..4ae74457998 100644 --- a/src/Processors/Transforms/ConvertingTransform.h +++ b/src/Processors/Transforms/ConvertingTransform.h @@ -1,7 +1,9 @@ #pragma once + #include #include + namespace DB { @@ -46,7 +48,7 @@ private: /// How to construct result block. Position in source block, where to get each column. ColumnNumbers conversion; /// Do not check that constants are same. Use value from result_header. - /// This is needed in case run functions which are constant in query scope, + /// This is needed in case run functions which are constant in query scope, /// but may return different result being executed remotely, like `now64()` or `randConstant()`. /// In this case we replace constants from remote source to constatns from initiator. bool ignore_constant_values; diff --git a/utils/check-style/check-style b/utils/check-style/check-style index ef569c9f73e..120d41aad34 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -102,3 +102,6 @@ find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} -name '*.md' # Too many exclamation marks find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -F '!!!' && echo "Too many exclamation marks (looks dirty, unconfident)." + +# Trailing whitespaces +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -P ' $' && echo "^ Trailing whitespaces." From baf55437fa744d4bfefe6d7cf5437e3d9a4c9406 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 26 Oct 2020 22:17:01 +0300 Subject: [PATCH 220/325] Better diagnostics in check-style script --- utils/check-style/check-style | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 120d41aad34..f9818a1e2bb 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -93,15 +93,15 @@ git status -uno | grep ya.make && echo "ya.make files should be generated with u find $ROOT_PATH/{src,programs,utils} -name '*.h' | while read file; do [[ $(head -n1 $file) != '#pragma once' ]] && echo "File $file must have '#pragma once' in first line"; done # Check for executable bit on non-executable files -find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} '(' -name '*.cpp' -or -name '*.h' -or -name '*.sql' -or -name '*.xml' -or -name '*.reference' -or -name '*.txt' -or -name '*.md' ')' -and -executable | grep -q '.' && echo "These files should not be executable." +find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} '(' -name '*.cpp' -or -name '*.h' -or -name '*.sql' -or -name '*.xml' -or -name '*.reference' -or -name '*.txt' -or -name '*.md' ')' -and -executable | grep -P '.' && echo "These files should not be executable." # Check for BOM -find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xEF\xBB\xBF' && echo "Files should not have UTF-8 BOM" -find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xFF\xFE' && echo "Files should not have UTF-16LE BOM" -find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xFE\xFF' && echo "Files should not have UTF-16BE BOM" +find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xEF\xBB\xBF' | grep -P '.' && echo "Files should not have UTF-8 BOM" +find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xFF\xFE' | grep -P '.' && echo "Files should not have UTF-16LE BOM" +find $ROOT_PATH/{src,base,programs,utils,tests,docs,website,cmake} -name '*.md' -or -name '*.cpp' -or -name '*.h' | xargs grep -l -F $'\xFE\xFF' | grep -P '.' && echo "Files should not have UTF-16BE BOM" # Too many exclamation marks -find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -F '!!!' && echo "Too many exclamation marks (looks dirty, unconfident)." +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -F '!!!' | grep -P '.' && echo "Too many exclamation marks (looks dirty, unconfident)." # Trailing whitespaces -find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -P ' $' && echo "^ Trailing whitespaces." +find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -P ' $' | grep -P '.' && echo "^ Trailing whitespaces." From 740cf8ce820022f1ed4e0dbfbe281bf7ab53689a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 26 Oct 2020 22:20:40 +0300 Subject: [PATCH 221/325] Some aggregate functions were erroneously marked as case-insensitive --- src/AggregateFunctions/AggregateFunctionBoundingRatio.cpp | 2 +- src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp | 2 +- src/AggregateFunctions/AggregateFunctionRetention.cpp | 2 +- src/AggregateFunctions/AggregateFunctionStudentTTest.cpp | 2 +- .../AggregateFunctionTimeSeriesGroupSum.cpp | 4 ++-- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 2 +- src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionBoundingRatio.cpp b/src/AggregateFunctions/AggregateFunctionBoundingRatio.cpp index e338b060b12..6f4f254ae8f 100644 --- a/src/AggregateFunctions/AggregateFunctionBoundingRatio.cpp +++ b/src/AggregateFunctions/AggregateFunctionBoundingRatio.cpp @@ -31,7 +31,7 @@ AggregateFunctionPtr createAggregateFunctionRate(const std::string & name, const void registerAggregateFunctionRate(AggregateFunctionFactory & factory) { - factory.registerFunction("boundingRatio", createAggregateFunctionRate, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("boundingRatio", createAggregateFunctionRate); } } diff --git a/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp b/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp index 20472279dba..796ff028424 100644 --- a/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp +++ b/src/AggregateFunctions/AggregateFunctionRankCorrelation.cpp @@ -45,7 +45,7 @@ AggregateFunctionPtr createAggregateFunctionRankCorrelation(const std::string & void registerAggregateFunctionRankCorrelation(AggregateFunctionFactory & factory) { - factory.registerFunction("rankCorr", createAggregateFunctionRankCorrelation, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("rankCorr", createAggregateFunctionRankCorrelation); } } diff --git a/src/AggregateFunctions/AggregateFunctionRetention.cpp b/src/AggregateFunctions/AggregateFunctionRetention.cpp index 4497703c550..c9d475c78f3 100644 --- a/src/AggregateFunctions/AggregateFunctionRetention.cpp +++ b/src/AggregateFunctions/AggregateFunctionRetention.cpp @@ -32,7 +32,7 @@ AggregateFunctionPtr createAggregateFunctionRetention(const std::string & name, void registerAggregateFunctionRetention(AggregateFunctionFactory & factory) { - factory.registerFunction("retention", createAggregateFunctionRetention, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("retention", createAggregateFunctionRetention); } } diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp index a2c36e43488..58fc9e5b5b9 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp @@ -47,6 +47,6 @@ AggregateFunctionPtr createAggregateFunctionStudentTTest(const std::string & nam void registerAggregateFunctionStudentTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("studentTTest", createAggregateFunctionStudentTTest, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("studentTTest", createAggregateFunctionStudentTTest); } } diff --git a/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.cpp b/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.cpp index cd0599729b9..c8711c257f8 100644 --- a/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.cpp +++ b/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.cpp @@ -28,8 +28,8 @@ namespace void registerAggregateFunctionTimeSeriesGroupSum(AggregateFunctionFactory & factory) { - factory.registerFunction("timeSeriesGroupSum", createAggregateFunctionTimeSeriesGroupSum, AggregateFunctionFactory::CaseInsensitive); - factory.registerFunction("timeSeriesGroupRateSum", createAggregateFunctionTimeSeriesGroupSum, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("timeSeriesGroupSum", createAggregateFunctionTimeSeriesGroupSum); + factory.registerFunction("timeSeriesGroupRateSum", createAggregateFunctionTimeSeriesGroupSum); } } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 483c99dde9b..0dcb125305d 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -44,6 +44,6 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("welchTTest", createAggregateFunctionWelchTTest, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("welchTTest", createAggregateFunctionWelchTTest); } } diff --git a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp index 872c70c2b98..1e9f2782d95 100644 --- a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp +++ b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp @@ -58,7 +58,7 @@ AggregateFunctionPtr createAggregateFunctionWindowFunnel(const std::string & nam void registerAggregateFunctionWindowFunnel(AggregateFunctionFactory & factory) { - factory.registerFunction("windowFunnel", createAggregateFunctionWindowFunnel, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("windowFunnel", createAggregateFunctionWindowFunnel); } } From 25da82328cd8bae16eae79cc844c73dfa20a50ef Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 26 Oct 2020 22:26:01 +0300 Subject: [PATCH 222/325] Adjust tests --- tests/queries/0_stateless/01322_student_ttest.sql | 4 ++-- tests/queries/0_stateless/01322_welch_ttest.sql | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/queries/0_stateless/01322_student_ttest.sql b/tests/queries/0_stateless/01322_student_ttest.sql index babc8457bc0..b8b86384bc4 100644 --- a/tests/queries/0_stateless/01322_student_ttest.sql +++ b/tests/queries/0_stateless/01322_student_ttest.sql @@ -6,7 +6,7 @@ DROP TABLE IF EXISTS student_ttest; CREATE TABLE student_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO student_ttest VALUES (0.88854,-2.90702), (-5.76966,3.61651), (6.76618,4.27458), (3.55546,4.82133), (-9.76948,9.59483), (4.92323,1.00424), (-0.36352,2.04147), (0.97018,-3.58214), (4.61656,6.59543), (-6.78292,-1.00532), (4.02008,-3.59794), (12.41838,-2.82434), (5.14417,-3.13194), (3.86836,9.90977), (-1.26199,0.523), (12.44106,4.62779), (3.28349,-2.56872), (1.77261,2.25807), (-8.94748,1.04044), (-1.01449,-2.35744), (-1.26377,10.81531), (6.79682,-9.68469), (6.32333,3.80885), (-8.21214,12.70435), (-1.68565,-6.01112), (9.7557,1.89065), (3.66694,5.08892), (1.39967,3.45254), (-5.52035,11.58151), (-10.95601,0.85035), (0.93877,8.38397), (1.45933,1.17169), (-5.40551,4.74621), (-0.83857,-1.66614), (8.50794,4.2414), (-6.68686,1.68765), (5.03099,1.85223), (1.56251,9.10111), (4.17381,-2.38085), (-2.92644,-14.79595), (5.11068,-3.8938), (2.09617,-3.41864), (11.7787,-3.15282), (6.50336,-0.56684), (0.62098,12.87997), (-7.97121,6.89115), (3.81902,12.921), (0.33151,-7.94908), (10.68584,2.45687), (0.56007,2.14957), (-7.38621,7.55081), (5.05882,-3.71534), (2.34616,-2.41064), (11.3806,-0.80734), (5.95276,-4.75651), (-3.01429,2.05241), (5.98169,-5.44523), (0.96985,-2.75054), (-1.15932,-13.00131), (2.11547,-2.74451), (2.49668,-1.39004), (-12.49569,-3.02854), (-4.94667,7.65112), (-3.64215,1.1245), (-8.35595,6.74117), (3.211,-0.75777), (2.33805,8.93451), (2.38608,-8.85559), (-3.2862,-0.36405), (-0.80454,4.02742), (-0.53483,6.88718), (10.66445,-1.05124), (-0.37619,3.04085), (0.48246,3.32368), (7.41919,1.147), (0.42414,3.41554), (-2.32335,-3.47851), (-0.70223,-0.47684), (-5.9332,-0.55605), (-1.20561,-0.17006), (3.39865,2.26218), (9.61739,12.45494), (-0.78651,-1.84097), (-4.00256,1.64934), (-7.99646,-7.07496), (8.72923,-9.99462), (0.71859,6.09954), (-1.62726,-1.05319), (5.11234,3.04757), (-0.95625,0.93899), (-3.75573,-4.63243), (1.03141,-7.43322), (-3.33588,-7.298), (1.51804,-6.59016), (-3.30935,-6.11649), (-1.97507,0.56682), (4.06456,2.00661), (3.27195,-2.79814), (-7.81761,2.84482), (-3.81785,3.65348), (-4.18311,-4.22807), (-11.33313,-4.54336), (-0.25221,-3.63343), (7.2514,2.96878), (5.30301,6.11661), (2.46762,-1.70919), (4.22716,-4.71133), (0.33916,6.09652), (9.7638,-6.83454), (-7.58684,0.18006), (-4.09888,1.51676), (4.26617,-5.31646), (-0.56744,-3.21215), (4.65125,-5.07599), (-1.30301,-2.36591), (4.53771,3.55724), (9.96929,4.8904), (3.72939,-3.22586), (-2.29818,-1.74928), (3.09417,5.73458), (0.82251,1.41188), (5.29975,2.86255), (2.8685,2.90179), (-5.73321,-2.19949), (-1.85651,1.72727), (-1.07984,1.76939), (9.78342,-0.12848), (-13.49652,-0.52), (3.68791,3.48333), (1.9998,7.8262), (1.11674,0.09099), (9.43869,7.77017), (4.07029,9.49484), (5.32715,1.42825), (7.16504,1.99624), (6.66096,4.00419), (-5.7111,1.07925), (-0.38575,-0.09987), (4.49165,-5.48733), (-3.36489,-1.83517), (7.71814,2.38059), (-1.58966,1.42075), (-1.61063,-1.11968), (-0.91602,-6.46035), (0.73459,7.66576), (-3.24463,4.6307), (6.3947,5.55989), (-2.77845,3.16684), (4.45899,5.07671), (-8.84186,-10.20566), (2.62276,-4.73386), (1.774,1.28353), (4.3692,6.75679), (0.05942,12.09895), (-1.44042,7.0049), (-2.53594,7.16156), (-2.24752,-0.64311), (4.98874,-0.66747), (4.05434,3.99996), (-2.56483,9.07298), (-6.79286,-4.60971), (-2.06165,0.70744), (-0.26056,2.56774), (1.89567,9.32424), (-3.15145,3.95087), (-7.31321,7.11372), (0.28936,-0.89284), (-0.63111,8.6155), (0.22611,-0.14141), (-9.3377,-4.86319), (-5.76638,-6.95801), (3.87306,4.44883), (6.7011,4.6156), (9.03915,-2.3579), (-1.21835,-5.1186), (0.82892,8.12819), (2.80656,2.78392), (-1.34746,-4.30221), (-1.99912,-1.47506), (0.6036,6.8598), (-3.46117,0.47636), (5.23732,0.95383), (-1.86702,7.79779), (-5.86115,-2.61767), (6.48523,-10.5087), (-7.40158,-2.74299), (-1.38913,3.87369), (4.94613,-1.07093), (-2.07818,4.98864), (2.39808,-7.50772), (4.89238,6.41316), (4.39481,1.39061), (5.20425,-3.1747), (13.62598,-2.13621), (-2.86293,-0.02203), (-3.62396,0.89025), (-4.28695,-5.87746), (4.66425,3.60026), (2.20871,-0.23178), (1.60382,-2.1897), (-9.87024,-5.85101), (-7.37302,-1.6053), (-4.17814,3.6184), (2.5148,-8.53795), (3.21708,-0.35987), (-11.48089,2.15301), (1.19821,-6.60692), (-0.07436,9.54341), (-1.10652,1.11511), (4.03395,2.94025), (-4.35883,12.05657), (2.04013,3.75156), (0.52264,7.95597), (8.14004,-0.99449), (-8.86949,0.90597), (-0.35807,-7.90627), (-10.71113,3.50863), (-2.13755,-1.47493), (0.50715,4.11671), (6.30826,10.06325), (2.37527,-1.06059), (0.20872,-1.37737), (-5.85729,-0.42542), (-4.97217,-3.90267), (-9.78434,9.35037), (-1.53277,-7.91219), (0.14827,-4.69945), (-1.053,3.63776), (1.74558,3.46492), (11.17194,2.84518), (9.35487,-3.04301), (-9.17209,8.82764), (10.41814,7.80134), (7.41206,7.87755), (3.71775,7.01035), (-2.04674,2.43271), (6.18037,11.36418), (5.6383,-6.92659), (-0.90058,5.95541), (-1.27073,3.59436), (-2.3473,5.18429), (-8.44271,4.20225), (2.75551,0.5029), (-1.15521,4.03074), (4.08722,5.23152), (-1.70399,10.65409), (7.24114,-0.69845), (-8.43976,11.70096), (-1.53052,5.80692), (-0.00526,-8.1819), (-4.04813,4.31485), (-2.84299,5.7227), (-5.201,5.67398), (7.75774,-1.75826), (-2.85791,7.54164), (-3.86071,-1.79026), (-1.80029,-1.7395), (-5.26015,5.65042), (-3.158,0.38765), (7.71014,-4.64719), (-4.84866,-10.22048), (-8.38785,-2.05447), (7.67021,-2.43441), (4.96521,-5.38551), (-0.40919,5.47764), (-3.25711,8.26637), (3.07685,-3.6421), (2.89376,-11.66269), (-10.47331,3.972), (-3.48942,5.46642), (1.13906,-3.72304), (-8.57454,5.75251), (-3.38963,5.12841), (-2.3195,0.59067), (-1.60694,5.21138), (-5.57406,-4.58702), (-0.93075,-8.737), (-11.76579,-2.12737), (10.68283,0.22888), (8.74324,-1.46448), (7.66409,2.40311), (4.76715,-5.21814), (0.44539,13.94749), (-1.35941,-2.77448), (4.18849,-3.7867), (-6.17097,3.4954), (0.27977,3.12586), (-1.45006,-7.01485), (-4.81694,-3.20727), (-3.0297,6.31415), (0.02145,2.37521), (2.46883,8.13787), (9.60317,2.15956), (-9.93898,-0.40842), (1.05549,-7.27283), (5.55366,4.27575), (-3.80722,-2.89126), (-4.18851,6.84344), (1.00351,7.0869), (3.11385,-5.18837), (-5.17623,2.67648), (-3.18396,-6.57021), (-6.65302,0.60429), (-0.50832,-1.04921), (-4.04375,7.12873), (4.52707,1.68973), (6.63124,-2.58404), (-3.72082,-3.83114), (5.79825,-7.26546), (-2.0158,-5.07153), (-2.78369,-0.80395), (-1.91821,2.09455), (6.31714,4.33374), (-1.80869,8.54335), (8.55586,0.80566), (2.40826,-8.38085), (-8.46361,7.54812), (5.04452,8.78007), (-0.84665,1.5857), (2.30903,8.43855), (-3.71837,-1.90846), (-0.69419,-1.2434), (3.6733,7.16172), (-1.96098,-3.44129), (2.36747,-6.37542), (-12.03622,-4.99486), (4.38481,4.99033), (2.93955,-1.83734), (2.16804,-2.83289), (-0.08218,-4.13997), (-3.97934,1.40163), (-7.43985,8.57867), (0.91666,-1.87639), (7.23432,3.41667), (-6.13303,6.31762), (-10.23217,1.58473), (-6.21681,1.63625), (-0.80934,-6.93618), (0.17914,3.58046), (2.13338,-6.8097), (6.97656,4.69978), (6.90455,-1.72912), (6.25943,5.29491), (-6.04019,-1.63062), (-7.30909,5.83818), (1.4589,17.0769), (12.00208,4.54301), (2.22457,-1.33801), (-2.45912,5.64339), (-6.92213,1.26913), (4.05547,-1.01553), (0.04709,4.8316), (-7.70952,3.08635), (-1.47883,-2.27738), (1.3701,-1.13761), (-4.92928,10.08698), (-2.75872,5.33827), (-0.09178,2.84345), (2.62642,-1.51132), (-1.14623,13.46078), (2.76609,8.58965), (4.94404,-2.36683), (-7.01764,-1.8217), (-10.91568,1.96981), (-2.49738,2.31718), (0.73576,3.66493), (2.25436,1.93104), (-1.72956,5.20332), (2.41054,3.20519), (5.72149,3.34631), (-6.41371,7.0087), (3.38217,-7.96126), (1.24133,-0.62182), (10.03634,-4.65227), (-2.37303,10.6572), (-1.35543,4.50891), (-1.4387,9.74298), (-4.0976,3.85707), (-0.82501,6.41144), (-1.93498,1.48649), (5.59955,2.28076), (5.46656,2.75342), (2.43568,-5.40401), (-0.23926,7.11389), (-4.9945,5.74368), (-4.96655,6.78345), (-0.59258,3.83773), (2.02497,0.70959), (0.67583,0.57434), (3.16522,1.5888), (-1.9673,3.94889), (-6.75319,5.8234), (-6.69723,7.78366), (0.81148,9.08354), (4.44531,-7.99182), (-4.43522,-2.77033), (-5.28602,-10.29342), (-3.58829,1.76251), (-7.97395,2.09266), (-2.84891,4.20614), (-3.95112,-3.63064), (3.54945,-2.17794), (12.12376,-2.66225), (-3.12347,-2.74707), (3.65209,-1.93431), (9.34031,1.38629), (-0.26348,4.12816), (-5.23968,-1.58902), (2.22336,-5.08864), (-10.70405,-2.30491), (-4.41319,2.64605), (-5.94912,1.16158), (1.8147,2.63534), (7.69287,1.4956), (9.46125,-4.60768), (4.72497,0.60771), (-0.57565,3.29549), (-1.12303,-1.42592), (2.90272,0.8883), (-4.4584,-1.10612), (4.28819,-2.57296), (11.64512,5.88085), (-1.80395,7.40745), (2.51605,13.48116), (-3.18439,5.53539), (-0.70213,-1.46014), (-7.68383,3.73304), (-8.32268,3.5435), (-8.71115,-3.89151), (9.96933,4.16265), (0.95675,2.32663), (3.35114,5.31735), (-2.66008,6.33485), (7.75456,2.1339), (0.73568,0.82708), (0.3483,-2.95155), (-1.09203,-6.76019), (-7.76963,-4.20179), (5.81902,8.78354), (-3.41424,1.41863), (-0.39209,7.65689), (4.67608,-6.52601), (0.68753,-4.4426), (5.17179,-4.49483), (4.98983,-3.91479), (-0.12659,-2.84562), (3.25267,2.58974), (1.50184,2.24424), (2.94507,-4.65846), (-0.42333,8.4062), (-3.66227,8.20262), (8.90812,-8.63752), (4.74411,4.97966), (2.22018,-0.35563), (-2.07976,-4.72116), (4.8711,-2.95997), (0.5023,2.73959), (6.31569,-0.23956), (-4.36903,10.13915), (3.82146,11.83775), (-6.99477,-2.50332), (3.61225,-0.58181), (14.69335,-7.62836), (0.58368,2.26478), (4.65341,-3.50179), (-3.14272,-2.08023), (2.67048,4.07256), (4.64963,-1.40826), (-2.70828,-2.33644), (1.42923,3.00197), (5.84498,4.23668), (-4.76568,-2.24647), (0.19907,1.0445), (1.67486,-0.31901), (5.32145,8.62657), (-8.03477,3.92817), (3.46776,0.08462), (4.66374,10.15884), (-5.37394,0.4113), (5.39045,4.45847), (-1.44756,5.82941), (-1.64419,6.59202), (3.39699,-3.73441), (-2.94659,-5.86969), (-2.38437,-4.56543), (-0.23958,-1.32636), (6.88389,-0.17884), (-2.7172,-3.56181), (-1.53419,-0.66932), (7.38841,6.87538), (-5.44178,0.73527), (-0.89287,-0.24177), (2.93546,-0.8657), (-0.26901,-0.22977), (-4.70044,1.02095), (2.25846,6.16311), (-9.28813,-5.68027), (6.04268,-3.7619), (4.41693,4.22959), (1.75714,-1.5249); SELECT '-2.610898982580138', '0.00916587538237954'; -SELECT roundBankers(StudentTTest(left, right).1, 16) as t_stat, roundBankers(StudentTTest(left, right).2, 16) as p_value from student_ttest; +SELECT roundBankers(studentTTest(left, right).1, 16) as t_stat, roundBankers(studentTTest(left, right).2, 16) as p_value from student_ttest; DROP TABLE IF EXISTS student_ttest; /*Check t-stat and p-value and compare it with scipy.stat implementation @@ -15,5 +15,5 @@ DROP TABLE IF EXISTS student_ttest; CREATE TABLE student_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO student_ttest VALUES (4.52546,8.69444), (3.73628,3.81414), (-0.39478,12.38442), (5.15633,8.9738), (0.50539,9.19594), (-5.34036,7.21009), (0.19336,4.97743), (8.35729,4.94756), (6.95818,19.80911), (-2.93812,13.75358), (8.30807,16.56373), (-3.3517,9.72882), (4.16279,4.64509), (-3.17231,17.76854), (1.93545,4.80693), (11.06606,8.79505), (-4.22678,10.88868), (-1.99975,6.21932), (-4.51178,15.11614), (-4.50711,13.24703), (1.89786,14.76476), (-6.19638,-0.6117), (-3.70188,17.48993), (5.01334,12.11847), (1.79036,4.87439), (2.14435,18.56479), (3.0282,1.23712), (2.35528,5.41596), (-12.18535,4.54994), (5.59709,11.37668), (-12.92336,9.5982), (-0.04281,6.59822), (-0.16923,1.16703), (0.88924,8.88418), (-4.68414,10.95047), (8.01099,5.52787), (2.61686,-1.11647), (-2.76895,14.49946), (3.32165,3.27585), (-0.85135,-0.42025), (1.21368,6.37906), (4.38673,2.5242), (6.20964,8.1405), (-1.23172,6.46732), (4.65516,9.89332), (-1.87143,10.4374), (0.86429,-1.06465), (2.51184,6.84902), (-1.88822,10.96576), (-1.61802,7.83319), (1.93653,14.39823), (-3.66631,7.02594), (-1.05294,13.46629), (-10.74718,10.39531), (16.49295,11.27348), (-7.65494,9.32187), (-3.39303,12.32667), (-4.89418,8.98905), (3.2521,9.54757), (0.05831,5.98325), (-3.00409,3.47248), (5.76702,9.26966), (2.67674,5.77816), (10.52623,6.32966), (-0.54501,9.49313), (-4.89835,6.21337), (3.52457,10.00242), (-0.0451,6.25167), (-6.61226,15.64671), (9.02391,2.78968), (5.52571,6.55442), (4.54352,3.68819), (-3.8394,9.55934), (-7.75295,4.166), (5.91167,12.32471), (1.38897,7.10969), (6.24166,16.31723), (5.58536,12.99482), (4.7591,10.11585), (-2.58336,10.29455), (-1.91263,18.27524), (3.31575,12.84435), (5.3507,13.11954), (-15.22081,12.84147), (-0.84775,15.55658), (-4.538,11.45329), (6.71177,7.50912), (0.52882,8.56226), (2.0242,8.63104), (5.69146,15.68026), (4.63328,21.6361), (0.22984,6.23925), (-2.84052,8.65714), (7.91867,9.9423), (1.11001,12.28213), (-0.11251,3.11279), (-0.20905,13.58128), (0.03287,16.51407), (-1.59397,16.60476), (-5.39405,12.02022), (-7.1233,12.11035), (4.51517,9.47832), (-0.70967,6.40742), (5.67299,8.87252), (-0.33835,15.14265), (-1.83047,2.23572), (-0.62877,11.57144), (-7.23148,18.87737), (0.1802,12.1833), (11.73325,11.17519), (2.17603,16.80422), (-0.11683,6.81423), (-1.29102,12.12546), (-0.23201,8.06153), (-6.8643,10.97228), (-6.85153,7.30596), (-4.77163,15.44026), (6.11721,8.00993), (5.96406,12.60196), (3.59135,13.96832), (-0.60095,14.03207), (3.11163,4.53758), (-0.18831,8.08297), (0.67657,4.90451), (-3.16117,8.14253), (0.26957,19.88605), (2.18653,13.85254), (-5.94611,23.01839), (-4.39352,6.02084), (-3.71525,9.60319), (5.11103,1.90511), (1.33998,10.35237), (1.01629,16.27082), (-3.36917,12.52379), (-3.99661,11.37435), (8.19336,13.61823), (2.89168,15.77622), (-11.10373,15.17254), (11.68005,6.711), (3.08282,4.74205), (-6.81506,10.09812), (-2.34587,6.61722), (-2.68725,10.34164), (0.3577,8.96602), (-3.05682,12.32157), (9.08062,11.75711), (-0.77913,13.49499), (10.35215,8.57713), (6.82565,11.50313), (-1.24674,1.13097), (5.18822,7.83205), (-3.70743,5.77957), (1.40319,15.5519), (5.89432,10.82676), (1.43152,11.51218), (6.70638,9.29779), (9.76613,9.77021), (4.27604,9.94114), (-2.63141,15.54513), (-7.8133,19.10736), (-0.06668,15.04205), (1.05391,9.03114), (4.41797,24.0104), (0.09337,9.94205), (6.16075,2.5925), (7.49413,8.82726), (-3.52872,10.0209), (-2.17126,8.1635), (-3.87605,4.24074), (3.26607,7.67291), (-3.28045,5.21642), (2.1429,11.2808), (1.53386,6.88172), (0.21169,5.98743), (-0.63674,17.97249), (5.84893,6.46323), (-0.63498,15.37416), (8.29526,2.89957), (-1.08358,17.13044), (-2.306,11.06355), (2.86991,3.09625), (-0.76074,-2.33019), (5.49191,7.42675), (1.82883,15.06792), (-3.70497,8.81116), (-0.53232,19.17446), (-11.49722,18.77181), (3.44877,14.06443), (-1.8596,12.81241), (-10.34851,2.72299), (1.13093,18.67739), (-10.93389,11.63275), (-3.39703,2.23891), (0.19749,13.01195), (-3.68389,7.43402), (-4.67863,8.14599), (10.78916,16.65328), (0.37675,1.362), (3.98094,3.87957), (-3.64775,11.16134), (-4.8443,6.25357), (1.102,4.21945), (8.72112,12.50047), (-1.47361,6.45486), (6.24183,18.99924), (6.83569,18.09508), (-3.11684,13.59528), (4.91306,3.39681), (-0.03628,13.33157), (5.1282,5.8945), (-2.38558,5.61212), (2.33351,8.41149), (-0.97191,13.78608), (-0.05588,6.08609), (-4.70019,12.76962), (-5.12371,3.26206), (0.65606,0.25528), (-0.11574,11.9083), (4.4238,4.35071), (6.93399,11.19855), (3.68712,13.87404), (-0.01187,6.87986), (1.8332,8.32566), (5.81322,22.51334), (-4.04709,2.5226), (-8.26397,16.84498), (-2.11273,6.26108), (5.28396,13.84824), (0.73054,6.03262), (6.43559,14.12668), (4.35565,16.01939), (-1.05545,8.19237), (5.00087,18.01595), (-2.72239,9.45609), (7.32313,6.90459), (2.11548,12.83115), (-3.40953,10.603), (6.97051,13.70439), (-0.45567,6.1633), (1.31699,4.1151), (-1.49871,8.20499), (7.14772,11.67903), (0.79277,7.30851), (6.9698,6.50941), (2.08733,7.3949), (-3.55962,12.80075), (0.75601,5.62043), (1.21,18.2542), (-2.17877,17.9393), (1.83206,16.4569), (5.72463,8.78811), (7.42257,4.85949), (0.97829,-3.36394), (7.54238,5.38683), (9.91081,12.26083), (-4.61743,10.27907), (-4.40799,11.5144), (9.99854,11.57335), (8.53725,1.94203), (3.2905,7.78228), (0.38634,11.79385), (-2.53374,10.18415), (4.94758,14.67613), (4.79624,4.70301), (5.57664,12.72151), (-6.44871,-3.35508), (3.34431,17.63775), (0.14209,2.53883), (10.88431,14.01483), (0.31846,12.4387), (-0.54703,11.15408), (-4.67791,7.74882), (-5.68011,13.60956), (-4.93362,7.81991), (1.2271,10.90969), (5.27512,8.19828), (-3.84611,-1.18523), (6.81706,0.5916), (10.33033,0.35805), (5.13979,12.98364), (3.66534,11.38628), (-2.07219,13.94644), (10.65442,2.03781), (-3.31751,10.74447), (-1.82011,12.35656), (-0.39886,7.08701), (1.77052,2.69871), (1.29049,19.66653), (7.92344,7.88636), (-2.92595,10.36916), (-2.67107,1.632), (5.64708,11.86081), (0.34639,13.47602), (-3.04356,6.60204), (3.98828,7.01303), (-1.36695,20.19992), (-8.48462,18.88249), (-4.04669,11.34367), (9.84561,12.97305), (-6.1537,9.5776), (0.82433,17.91364), (1.92449,18.3247), (2.51288,9.9211), (0.40965,7.14257), (2.89183,6.59133), (3.84347,12.35274), (0.66829,10.57523), (-3.45094,12.12859), (1.3544,9.47177), (-9.85456,0.60659), (5.25689,4.72996), (-5.26018,4.51121), (-6.16912,13.28893), (-1.77163,8.09014), (3.96687,8.02511), (0.70893,13.85406), (-5.45342,1.75412), (-3.89706,6.00641), (3.11868,6.35554), (4.41714,7.11293), (7.64841,8.30442), (0.00489,12.63024), (3.2263,12.38966), (-5.33042,7.6801), (2.52189,11.33744), (-7.40308,4.67713), (0.67891,7.62276), (2.49343,2.14478), (5.43133,15.32988), (-0.67541,1.52299), (-0.60299,17.00017), (-6.32903,8.29701), (-3.44336,10.92961), (-0.23963,6.78449), (6.94686,7.02698), (6.59442,11.51719), (-4.18532,9.97926), (-1.8228,7.44251), (-0.29443,7.58541), (2.99821,4.76058), (2.51942,12.88959), (-3.49176,9.974), (-0.57979,17.03689), (8.69471,11.14554), (-1.19427,11.7392), (-3.17119,11.50029), (-2.99566,19.41759), (-3.34493,9.65127), (-2.33826,9.87673), (-5.04164,14.13485), (-0.48214,9.78034), (7.45097,1.57826), (3.04787,3.72091), (2.92632,9.4054), (1.39694,23.22816), (4.38686,-0.12571), (3.25753,6.97343), (7.14218,10.09049), (-4.04341,11.78393), (-9.19352,3.01909), (2.78473,16.09448), (0.33331,6.25485), (9.89238,7.13164), (6.00566,7.75879), (-1.7511,9.56834), (4.77815,6.14824), (5.07457,13.53454), (2.56132,8.26364), (2.38317,8.7095), (-1.63486,10.61607), (-1.46871,10.64418), (-5.8681,23.9106), (-2.96227,11.38978), (-1.90638,11.4383), (-13.3052,18.41498), (-2.14705,3.70959), (-9.62069,19.95918), (2.29313,9.53847), (0.22162,14.04957), (-1.83956,13.70151), (4.1853,5.45046), (6.05965,10.95061), (-0.23737,9.55156), (6.07452,17.92345), (4.34629,6.23976), (4.02922,8.71029), (3.62622,13.58736), (-3.95825,8.78527), (-1.63412,11.14213), (-1.25727,12.23717), (5.06323,16.44557), (-0.66176,0.47144), (2.36606,9.7198), (-5.77792,13.50981), (4.535,14.27806), (1.02031,13.50793), (4.49345,7.47381), (-4.99791,11.07844), (2.46716,9.89844), (3.65471,21.48548), (11.2283,6.92085), (6.69743,4.44074), (-5.60375,19.98074), (0.28683,7.92826), (-0.85737,16.6313), (4.26726,17.17618), (-3.4322,13.80807), (-2.07039,5.37083), (-2.26798,9.73962), (-0.99818,10.66273), (0.41335,8.90639), (5.18124,12.24596), (-5.01858,16.89203), (2.05561,12.69184), (-0.12117,15.59077), (0.99471,6.94287), (6.89979,-0.1801), (-4.18527,3.25318), (-6.35104,8.08804), (3.89734,13.78384), (-1.979,0.46434), (3.15404,7.78224), (3.52672,9.10987), (2.48372,-0.89391), (-6.13089,14.3696), (2.2968,3.01763), (-2.74324,8.03559), (-0.12876,7.24609), (-1.51135,11.86271), (-3.92434,6.28196), (-1.71254,8.9725), (-1.25878,14.46114), (2.03021,9.50216), (4.31726,16.30413), (-3.02908,1.02795), (9.7093,1.88717), (-3.36284,9.80106), (6.70938,4.53487), (0.42762,16.34543), (5.04726,7.71098), (2.78386,2.74639), (6.83022,6.51875), (-3.02109,10.42308), (-0.65382,13.57901), (-15.58675,0.52784), (5.89746,4.4708), (-4.11598,6.39619), (-1.37208,14.57666), (10.08082,2.71602), (5.35686,12.53905), (1.93331,11.4292), (10.47444,12.44641), (-2.36872,14.50894), (6.50752,17.64374), (2.54603,11.03218), (-0.4332,9.82789), (5.26572,10.11104), (2.09016,2.16137), (1.15513,10.24054), (14.95941,12.86909), (-3.85505,15.22845), (-2.36239,5.05411), (1.64338,10.84836), (-4.25074,11.15717), (7.29744,0.91782), (-1.18964,13.29961), (5.60612,15.11314), (-3.77011,11.54004), (6.67642,-0.94238), (-0.06862,19.32581), (5.60514,10.20744), (3.7341,6.54857), (9.59001,8.69108), (3.30093,8.2296), (-2.75658,8.4474), (4.71994,6.81178), (0.74699,5.99415), (2.91095,13.99336), (-7.36829,8.7469), (-5.29487,8.62349), (3.31079,1.84212), (1.06974,4.4762), (-1.18424,9.25421), (-7.415,10.44229), (3.40595,12.21649), (-7.63085,10.45968), (1.13336,15.34722), (-0.0096,5.50868), (0.8928,10.93609), (-0.5943,2.78631), (7.48306,11.86145), (10.11943,18.67385), (5.60459,10.64051), (4.00189,12.75565), (2.35823,6.63666), (0.33475,12.19343), (3.47072,9.08636), (-6.68867,11.67256), (3.31031,20.31392), (2.17159,11.66443); SELECT '-28.740781574102936', '7.667329672103986e-133'; -SELECT roundBankers(StudentTTest(left, right).1, 16) as t_stat, roundBankers(StudentTTest(left, right).2, 16) as p_value from student_ttest; +SELECT roundBankers(studentTTest(left, right).1, 16) as t_stat, roundBankers(studentTTest(left, right).2, 16) as p_value from student_ttest; DROP TABLE IF EXISTS student_ttest; diff --git a/tests/queries/0_stateless/01322_welch_ttest.sql b/tests/queries/0_stateless/01322_welch_ttest.sql index 5a5b52ab612..cce65c28bd8 100644 --- a/tests/queries/0_stateless/01322_welch_ttest.sql +++ b/tests/queries/0_stateless/01322_welch_ttest.sql @@ -3,19 +3,19 @@ DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (27.5,27.1), (21.0,22.0), (19.0,20.8), (23.6,23.4), (17.0,23.4), (17.9,23.5), (16.9,25.8), (20.1,22.0), (21.9,24.8), (22.6,20.2), (23.1,21.9), (19.6,22.1), (19.0,22.9), (21.7,20.5), (21.4,24.4); SELECT '0.021378001462867'; -SELECT roundBankers(WelchTTest(left, right).2, 16) from welch_ttest; +SELECT roundBankers(welchTTest(left, right).2, 16) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (30.02,29.89), (29.99,29.93), (30.11,29.72), (29.97,29.98), (30.01,30.02), (29.99,29.98); SELECT '0.090773324285671'; -SELECT roundBankers(WelchTTest(left, right).2, 16) from welch_ttest; +SELECT roundBankers(welchTTest(left, right).2, 16) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (0.010268,0.159258), (0.000167,0.136278), (0.000167,0.122389); SELECT '0.00339907162713746'; -SELECT roundBankers(WelchTTest(left, right).2, 16) from welch_ttest; +SELECT roundBankers(welchTTest(left, right).2, 16) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; /*Check t-stat and p-value and compare it with scipy.stat implementation @@ -24,7 +24,7 @@ DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (14.72789,-8.65656), (9.61661,22.98234), (13.57615,23.80821), (3.98392,13.33939), (11.98889,-4.05537), (10.99422,23.5155), (5.44792,-6.45272), (20.29346,17.7903), (7.05926,11.463), (9.22732,5.28021), (12.06847,8.39157), (13.52612,6.02464), (8.24597,14.43732), (9.35245,15.76584), (10.12297,1.54391), (15.80624,1.24897), (13.68613,27.1507), (10.72729,7.71091), (5.62078,15.71846), (6.12229,32.97808), (6.03801,-1.79334), (8.95585,-9.23439), (24.04613,11.27838), (9.04757,0.72703), (2.68263,18.51557), (15.43935,9.16619), (2.89423,17.29624), (4.01423,-1.30208), (4.30568,-3.48018), (11.99948,10.12082), (8.40574,-8.01318), (10.86642,-14.22264), (9.4266,16.58174), (-8.12752,-0.55975), (7.91634,5.61449), (7.3967,1.44626), (2.26431,7.89158), (14.20118,1.13369), (6.68233,-0.82609), (15.46221,12.23365), (7.88467,12.45443), (11.20011,14.46915), (8.92027,13.72627), (10.27926,18.41459), (5.14395,29.66702), (5.62178,1.51619), (12.84383,10.40078), (9.98009,3.33266), (-0.69789,6.12036), (11.41386,11.86553), (7.76863,6.59422), (7.21743,22.0948), (1.81176,1.79623), (9.43762,14.29513), (19.22117,19.69162), (2.97128,-7.98033), (14.32851,5.48433), (7.54959,-2.28474), (3.81545,9.91876), (10.1281,10.64097), (2.48596,0.22523), (10.0461,17.01773), (3.59714,22.37388), (9.73522,14.04215), (18.8077,23.1244), (3.15148,18.96958), (12.26062,8.42663), (5.66707,3.7165), (6.58623,14.29366), (17.30902,23.50886), (9.91391,26.33722), (5.36946,26.72396), (15.73637,13.26287), (16.96281,12.97607), (11.54063,17.41838), (18.37358,8.63875), (11.38255,17.08943), (10.53256,23.15356), (8.08833,-4.4965), (16.27556,7.58895), (2.42969,26.04074), (9.56127,6.84245), (7.32998,20.56287), (9.19511,3.84735), (9.66903,-2.76304), (4.15029,13.1615), (8.83511,8.21954), (14.60617,-3.49943), (14.06143,22.12419), (5.39556,7.08323), (10.11871,16.12937), (10.56619,-0.32672), (14.4462,16.5942), (10.42106,7.68977), (7.75551,11.39484), (11.00418,-5.11987), (4.47226,20.87404), (16.35461,8.01007), (18.55174,3.26497), (11.82044,5.61253), (7.39454,20.69182), (11.27767,0.0296), (6.83827,21.904), (7.76858,22.46572), (15.97614,3.63685), (14.53781,-5.10846), (12.99546,14.86389), (16.91151,5.47188), (9.65012,18.44095), (14.25487,16.71368), (14.03618,6.36704), (2.57382,8.82663), (2.50779,14.6727), (14.24787,7.98383), (13.34666,2.65568), (7.31102,21.45827), (10.22981,11.77948), (17.4435,4.71979), (21.2074,3.17951), (6.64191,13.90226), (18.7086,15.50578), (14.78686,10.8026), (9.85287,16.91369), (4.48263,9.90552), (14.17469,13.87322), (14.4342,4.12366), (19.2481,-3.78985), (3.47165,1.7599), (8.28712,3.43715), (8.81657,-3.45246), (0.92319,23.64571), (20.41106,-4.96877), (6.76127,3.93514), (22.00242,1.49914), (8.66129,12.71519), (10.9929,5.11521), (17.95494,4.79872), (17.20996,20.89391), (12.18888,5.363), (12.14257,8.02765), (15.81243,14.30804), (4.43362,11.49002), (1.17567,14.25281), (15.60881,7.6573), (9.34833,15.49686), (6.33513,3.29327), (-0.83095,2.27236), (12.43268,12.58104), (6.63207,19.19128), (11.96877,15.25901), (14.81029,6.5221), (21.84876,10.10965), (3.75896,12.75249), (6.91307,16.50977), (13.73015,-8.6697), (8.63753,8.28553), (15.71679,1.44315), (1.74565,4.65869), (9.16895,0.98149), (5.70685,0.16623), (5.00117,17.66332), (13.06888,4.35346), (7.51204,6.52742), (15.34885,-1.06631), (5.20264,-5.28454), (8.59043,14.25583), (6.45619,8.74058), (14.61979,1.89553), (11.7075,-0.92959), (14.04901,10.30289), (4.20525,-6.3744), (15.1733,-8.1706), (3.12934,10.95369), (8.08049,4.94384), (15.41273,28.40568), (16.90751,3.7004), (5.86893,2.52363), (7.1086,4.07997), (4.418,7.8849), (12.0614,17.95409), (7.07887,16.67021), (3.61585,11.34377), (11.73001,-0.07446), (10.80449,22.00223), (8.40311,3.31778), (9.91276,18.50719), (16.4164,-3.58655), (5.25034,6.5394), (15.20283,12.40459), (10.42909,16.59866), (9.53888,7.54176), (14.68939,-1.51044), (6.60007,12.69758), (18.31058,2.9842), (7.01885,2.49187), (18.71631,2.04113), (10.50002,-2.46544), (10.7517,15.18368), (4.23224,-0.04058), (2.28924,-0.4127), (8.56059,10.5526), (8.25095,12.03982), (9.15673,12.10923), (13.28409,11.54954), (8.4513,-1.18613), (2.83911,11.30984), (2.79676,23.54105), (9.11055,10.67321), (7.18529,24.09196), (-4.1258,7.5008), (5.28306,12.52233), (6.82757,4.30673), (10.89035,9.35793), (5.24822,4.44472), (11.935,-7.00679), (6.45675,8.56241), (10.18088,23.73891), (4.9932,15.62708), (18.09939,16.09205), (8.11738,12.52074), (5.37883,14.58927), (10.50339,-4.80187), (16.64093,8.47964), (14.77263,7.75477), (13.71385,12.6893), (6.98746,7.14147), (10.74635,12.12654), (5.49432,12.32334), (13.46078,7.98909), (10.67565,3.26652), (9.0291,20.53684), (11.51417,32.3369), (13.07118,19.74911), (9.5049,-4.62897), (8.50611,8.26483), (6.47606,20.88451), (13.06526,-2.12982), (19.08658,25.61459), (9.49741,5.32091), (10.60865,-4.1196), (2.28996,7.57937), (8.12846,21.15847), (5.62241,6.46355), (4.07712,7.74846), (17.98526,19.62636), (9.466,28.34629), (11.38904,26.73919), (5.91826,20.40427), (1.52059,3.03378), (18.79161,10.2537), (18.20669,7.47745), (-1.67829,10.79184), (18.01586,3.91962), (16.31577,19.97973), (7.88281,18.87711), (8.46179,12.56157), (10.31113,11.46033), (14.88377,3.78661), (1.31835,-9.45748), (2.53176,12.06033), (9.48625,-0.74615), (3.97936,13.2815), (11.52319,24.78052), (13.24178,5.83337), (7.58739,17.4111), (10.00959,19.70331), (9.73361,11.78446), (8.35716,-1.366), (1.65491,1.37458), (11.11521,16.31483), (6.08355,32.63464), (10.04582,-3.79736), (11.58237,19.17984), (16.40249,-0.27705), (1.9691,-3.69456), (13.22776,28.38058), (2.67059,-1.36876), (9.83651,-25.63301), (2.12539,3.58644), (9.27114,-6.85667), (9.0699,13.42225), (2.78179,12.04671), (12.49311,28.99468), (12.97662,7.87662), (15.06359,2.61119), (16.91565,-3.56022), (5.92011,1.50022), (5.81304,14.55836), (8.46425,9.35831), (9.48705,16.9366), (4.68191,29.23126), (5.70028,15.31386), (-0.78798,13.46112), (10.03442,7.39667), (15.45433,11.15599), (9.43845,9.80499), (3.05825,22.64923), (6.92126,8.67693), (14.05905,18.67335), (19.71579,-3.19127), (15.0131,22.94716), (4.50386,17.86834), (1.31061,16.98267), (10.81197,15.91653), (14.32942,11.79718), (9.26469,18.50208), (7.27679,8.90755), (22.69295,10.44843), (12.03763,4.67433), (7.34876,6.82287), (16.60689,10.82228), (7.48786,-4.18631), (15.78602,20.3872), (17.21048,11.84735), (13.93482,21.25376), (9.69911,10.55032), (12.24315,12.19023), (10.58131,0.63369), (19.57006,7.92381), (9.8856,17.90933), (11.70302,15.30781), (7.89864,10.01877), (12.24831,0.88744), (16.93707,22.20967), (9.65467,-4.23117), (4.221,21.50819), (15.45229,11.27421), (12.83088,-16.23179), (7.58313,33.43085), (12.895,5.15093), (10.02471,1.34505), (13.36059,6.027), (5.07864,-10.43035), (9.72017,27.45998), (11.05809,19.24886), (15.28528,-4.44761), (13.99834,5.453), (19.26989,12.73758), (9.41846,11.2897), (11.65425,31.032), (8.49638,7.39168), (6.38592,11.95245), (-4.69837,26.279), (12.22061,-1.0255), (9.41331,10.36675), (13.2075,11.58439), (12.97005,27.8405), (11.44352,13.1707), (9.79805,31.39133), (6.93116,27.08301), (10.07691,-2.14368), (22.05892,4.08476), (7.80353,21.5573), (-2.17276,16.69822), (0.61509,7.69955), (8.35842,8.32793), (17.77108,6.49235), (14.70841,-7.3284), (1.27992,10.58264), (15.62699,-6.17006), (9.32914,34.55782), (15.41866,10.93221), (10.82009,44.24299), (3.29902,14.6224), (9.21998,-7.42798), (7.93845,15.52351), (10.33344,11.33982), (12.06399,10.46716), (5.5308,13.0986), (8.38727,-4.25988), (18.11104,9.55316), (8.86565,0.75489), (19.41825,25.99212), (9.52376,-0.81401), (3.94552,3.49551), (9.37587,22.99402), (15.44954,10.99628), (15.90527,23.70223), (13.18927,2.71482), (7.01646,22.82309), (9.06005,31.25686), (9.06431,4.86318), (5.76006,-1.06476), (9.18705,15.10298), (-3.48446,-0.61015), (15.89817,17.81246), (12.94719,-1.55788), (23.69426,18.09709), (17.47755,9.11271), (15.61528,9.94682), (0.54832,-7.33194), (14.32916,-4.67293), (9.55305,21.81717), (13.79891,7.16318), (0.82544,13.25649), (13.34875,13.88776), (9.07614,4.95793), (5.19621,17.65303), (2.1451,14.47382), (9.87726,13.19373), (8.45439,31.86093), (-1.41842,5.73161), (7.93598,10.96492), (11.23151,6.97951), (17.84458,1.75136), (7.02237,10.96144), (10.7842,15.08137), (4.42832,9.95311), (4.45044,7.07729), (1.50938,3.08148), (21.21651,22.37954), (6.2097,8.51951), (6.84354,2.88746), (18.53804,26.73509), (12.01072,-2.88939), (4.8345,-2.82367), (20.41587,-0.35783), (14.48353,14.22076), (8.71116,11.50295), (12.42818,7.10171), (14.89244,8.28488), (8.03033,0.54178), (5.25917,13.8022), (2.30092,15.62157), (10.22504,10.79173), (15.37573,28.18946), (7.13666,30.43524), (4.45018,2.54914), (10.18405,9.89421), (3.91025,13.08631), (14.52304,4.68761), (13.14771,5.61516), (11.99219,22.88072), (9.21345,7.4735), (8.85106,11.27382), (12.91887,2.39559), (15.62308,-3.31889), (11.88034,9.61957), (15.12097,23.01381), (11.58168,-1.23467), (16.83051,9.07691), (5.25405,15.78056), (2.19976,12.28421), (4.56716,9.44888), (16.46053,13.16928), (5.61995,4.33357), (8.67704,2.21737), (5.62789,33.17833), (9.84815,13.25407), (13.05834,-2.47961), (11.74205,6.41401), (3.88393,18.8439), (16.15321,-4.63375), (4.83925,-8.2909), (13.00334,12.18221), (4.4028,-2.95356), (4.35794,19.61659), (4.47478,12.45056), (2.38713,-4.17198), (4.25235,21.9641), (10.87509,11.96416), (9.82411,12.74573), (13.61518,10.47873), (10.25507,12.73295), (4.0335,11.31373), (10.69881,9.9827), (5.70321,5.87138), (6.96244,4.24372), (9.35874,-23.72256), (6.28076,28.41337), (8.29015,4.88103), (6.88653,3.61902), (7.70687,8.93586), (8.2001,16.40759), (6.73415,27.84494), (3.82052,5.6001), (3.94469,14.51379), (15.82384,13.5576), (2.54004,12.92213), (10.74876,3.90686), (12.60517,17.07104), (17.7024,15.84268), (4.6722,17.38777), (13.67341,16.54766), (6.4565,5.94487), (12.95699,17.02804), (4.56912,7.66386), (5.58464,10.43088), (4.0638,6.16059), (13.05559,20.46178), (5.38269,20.02888), (0.16354,20.95949), (7.23962,6.50808), (7.38577,7.22366), (8.50951,8.06659), (13.72574,16.08241), (17.80421,13.83514), (3.01135,-0.33454), (8.02608,12.98848), (14.23847,12.99024); SELECT '-0.5028215369186904', '0.6152361677168877'; -SELECT roundBankers(WelchTTest(left, right).1, 16) as t_stat, roundBankers(WelchTTest(left, right).2, 16) as p_value from welch_ttest; +SELECT roundBankers(welchTTest(left, right).1, 16) as t_stat, roundBankers(welchTTest(left, right).2, 16) as p_value from welch_ttest; DROP TABLE IF EXISTS welch_ttest; /*Check t-stat and p-value and compare it with scipy.stat implementation @@ -33,5 +33,5 @@ DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (4.82025,-2.69857), (6.13896,15.80943), (15.20277,7.31555), (14.15351,3.96517), (7.21338,4.77809), (8.55506,9.6472), (13.80816,-26.41717), (11.28411,-10.85635), (7.4612,-1.4376), (7.43759,-0.96308), (12.9832,2.84315), (-5.74783,5.79467), (12.47114,-3.06091), (15.14223,-14.62902), (3.40603,22.08022), (9.27323,-2.11982), (7.88547,-4.84824), (8.56456,-10.50447), (4.59731,2.4891), (7.91213,9.90324), (7.33894,-22.66866), (21.74811,-0.97103), (11.92111,-16.57608), (0.18828,-3.78749), (10.47314,25.84511), (20.37396,5.30797), (11.04991,-18.19466), (13.30083,11.72708), (14.28065,0.2891), (2.86942,-9.83474), (24.96072,6.69942), (14.20164,18.09604), (18.28769,18.52651), (10.50949,1.38201), (9.22273,7.64615), (11.77608,17.66598), (8.56872,-2.44141), (13.74535,-9.01598), (11.65209,27.69142), (12.51894,4.06946), (17.76256,-15.0077), (13.52122,-10.49648), (8.70796,-4.88322), (6.04749,-25.09805), (16.33064,-4.64024), (8.35636,20.94434), (14.03496,24.12126), (11.05834,-14.10962), (14.49261,10.6512), (2.59383,14.50687), (8.01022,-19.88081), (4.05458,-11.55271), (13.26384,13.16921), (14.62058,16.63864), (10.52489,-24.08114), (8.46357,-9.09949), (6.4147,-10.54702), (9.70071,0.20813), (12.47581,8.19066), (4.38333,-2.70523), (17.54172,-0.23954), (10.12109,7.19398), (7.73186,-7.1618), (14.0279,-7.44322), (11.6621,-17.92031), (17.47045,-1.58146), (15.50223,9.18338), (15.46034,3.25838), (13.39964,-14.30234), (14.98025,1.84695), (15.87912,31.13794), (17.67374,-0.85067), (9.64073,19.02787), (12.84904,-3.09594), (7.70278,13.45584), (13.03156,-5.48104), (9.04512,-22.74928), (15.97014,-8.03697), (8.96389,17.31143), (11.48009,-16.65231), (9.71153,-18.58713), (13.00084,-16.52641), (12.39803,14.95261), (13.08188,12.56762), (5.82244,15.00188), (10.81871,1.85858), (8.2539,2.1926), (7.52114,-2.4095), (9.11488,21.56873), (8.37482,3.35509), (14.48652,-4.98672), (11.42152,35.08603), (16.03111,-10.01602), (13.14057,-3.85153), (-2.26351,-6.81974), (15.50394,19.56525), (14.88603,-9.35488), (13.37257,0.24268), (11.84026,-3.51488), (7.66558,-0.37066), (6.24584,24.20888), (3.6312,-11.73537), (2.7018,0.01282), (5.63656,0.03963), (5.82643,-9.65589), (10.06745,-0.37429), (-0.5831,5.61255), (14.84202,0.49984), (9.5524,-10.15066), (19.71713,-14.54314), (14.23109,16.56889), (8.69105,-7.73873), (5.33742,-3.76422), (7.30372,1.40722), (7.93342,2.28818), (15.20884,-13.12643), (7.53839,5.17082), (13.45311,4.79089), (11.04473,-17.42643), (10.76673,8.72548), (15.44145,-3.70285), (14.06596,16.77893), (9.14873,13.382), (12.88372,19.98418), (8.74994,0.00483), (10.53263,-4.75951), (16.16694,2.35391), (8.37197,21.65809), (3.43739,-9.2714), (4.72799,-18.38253), (9.08802,7.23097), (11.2531,14.97927), (5.16115,-4.02197), (10.20895,-29.8189), (18.70884,-12.8554), (15.88924,-7.60124), (3.38758,-14.90158), (6.46449,-3.31486), (10.21088,31.38144), (14.08458,-8.61288), (15.74508,15.31895), (19.31896,-10.19488), (13.19641,13.796), (11.95409,-0.32912), (10.70718,-0.0684), (1.05245,-30.06834), (10.04772,24.93912), (17.01369,-3.26506), (10.2286,-8.29751), (19.58323,-5.39189), (7.02892,-25.08603), (4.16866,-1.45318), (8.94326,16.72724), (4.99854,-3.38467), (8.88352,-26.00478), (18.65422,7.28369), (17.32328,16.96226), (9.33492,16.5858), (14.94788,10.46583), (8.05863,3.84345), (14.6737,-2.99382), (10.93801,1.42078), (0.54036,-11.0123), (-0.34242,2.09909), (5.89076,1.21064), (3.15189,15.36079), (1.94421,-21.61349), (6.38698,22.7726), (10.50654,10.50512), (8.95362,-6.95825), (6.23711,9.20036), (11.75359,15.66902), (12.42155,3.28098), (-1.55472,-9.05692), (4.6688,0.32882), (10.48087,-1.64934), (11.74615,-4.81406), (9.26822,-5.06006), (7.55517,19.97493), (12.76005,2.88646), (16.47102,-0.34552), (11.31297,7.55186), (14.37437,-22.96115), (2.38799,31.29166), (6.44577,6.18798), (5.07471,-2.52715), (11.55123,-11.58799), (7.76795,14.13596), (10.60116,13.45069), (14.40885,12.15179), (11.58158,3.44491), (8.81648,-8.78006), (12.92299,18.32087), (11.26939,11.91757), (17.95014,-2.00179), (2.95002,10.88411), (17.41959,9.09327), (11.12455,6.62484), (8.78541,8.87178), (14.36413,11.52254), (12.98554,-14.15988), (12.58505,-17.19515), (15.49789,14.03089), (11.70999,-2.4095), (0.65596,-16.83575), (11.08202,2.71469), (14.75752,4.84351), (6.84385,-1.17651), (9.27245,-3.37529), (13.78243,-19.92137), (17.4863,4.48952), (4.01777,-12.4906), (11.82861,-5.65277), (13.86551,8.50819), (6.16591,-19.61261), (8.71589,12.54156), (16.77195,11.06784), (17.23243,-12.59285), (-2.12941,3.43683), (5.66629,-3.00325), (12.45153,12.49082), (1.63971,7.20955), (13.84031,17.6547), (4.6144,15.8619), (5.26169,24.3048), (9.27769,-8.05434), (9.14288,-6.06901), (9.71953,-15.69515), (9.38446,-11.13917), (1.64788,-3.90757), (11.72922,-2.57038), (13.68926,5.14065), (9.42952,17.8497), (12.05574,-8.64665), (9.09148,-18.68331), (5.32273,5.8567), (20.25258,-20.93884), (10.14599,4.40583), (10.82156,14.35985), (5.75736,4.18134), (7.13567,4.3635), (9.29746,9.35428), (5.1618,2.8908), (10.076,16.01017), (21.65669,-1.48499), (13.35486,-9.97949), (6.79957,1.03055), (8.76243,-2.79697), (14.59294,6.85977), (16.90609,4.73213), (10.50337,2.7815), (-0.07923,-2.46866), (13.51648,18.39425), (12.0676,-0.80378), (0.86482,-0.22982), (9.03563,-16.11608), (5.38751,3.0862), (17.16866,3.20779), (2.78702,10.50146), (11.15548,-0.21305), (12.30843,11.21012), (8.04897,-0.99825), (9.95814,18.39633), (11.29308,-3.39003), (14.13032,-0.64411), (21.05877,-1.39932), (3.57386,15.45319), (7.96631,-0.66044), (3.30484,-15.2223), (18.61856,-34.39907), (16.35184,-3.57836), (7.65236,16.82828), (18.02895,1.66624), (9.79458,15.43475), (16.7274,8.17776), (8.84453,5.50486), (13.05709,10.43082), (10.91447,-6.63332), (8.40171,2.28008), (16.95211,16.37203), (11.82194,5.16313), (19.87978,-8.85281), (12.88455,13.26692), (-0.00947,-7.46842), (12.28109,8.43091), (6.96462,-13.18172), (13.75282,-0.72401), (14.39141,22.3881), (11.07193,10.65448), (12.88039,2.81289), (11.38253,10.92405), (21.02707,-8.95358), (7.51955,19.80653), (6.31984,-12.86527), (15.6543,5.38826), (14.80315,-6.83501), (8.38024,-15.7647), (21.7516,-27.67412), (14.31336,8.6499), (15.04703,-4.89542), (5.73787,16.76167), (13.16911,12.84284), (12.40695,-17.27324), (9.88968,-4.18726), (8.46703,-14.62366), (8.70637,-5.49863), (8.03551,-16.22846), (5.9757,10.60329), (12.22951,6.46781), (3.14736,1.70458), (10.51266,10.77448), (18.593,0.8463), (10.82213,13.0482), (7.14216,-4.36264), (6.81154,3.22647), (-0.6486,2.38828), (20.56136,6.7946), (11.35367,-0.25254), (11.38205,1.2497), (17.14,1.6544), (14.91215,4.1019), (15.50207,11.27839), (5.93162,-5.04127), (3.74869,18.11674), (14.11532,0.51231), (7.38954,-0.51029), (5.45764,13.52556), (18.33733,16.10171), (9.91923,5.68197), (2.38991,-2.85904), (14.16756,-8.89167), (2.39791,6.24489), (6.92586,10.85319), (5.32474,-0.39816), (2.28812,3.87079), (5.71718,-3.1867), (5.84197,1.55322), (2.76206,16.86779), (19.05928,-14.60321), (11.51788,-1.81952), (6.56648,-3.11624), (3.35735,1.24193), (7.55948,10.18179), (19.99908,4.69796), (13.00634,0.69032), (18.36886,11.7723), (11.14675,7.62896), (16.72931,9.89741), (12.50106,9.11484), (6.00605,-3.84676), (23.06653,-0.4777), (5.39694,0.95958), (9.53167,-7.95056), (12.76944,-10.97474), (7.20604,-6.54861), (13.25391,34.74933), (13.7341,27.39463), (10.85292,4.18299), (-7.75835,6.02476), (10.29728,-1.99397), (13.70099,1.26478), (10.17959,23.37106), (9.98399,10.49682), (12.69389,-11.04354), (-0.28848,-12.22284), (-2.18319,-9.87635), (13.36378,28.90511), (10.09232,6.77613), (5.49489,0.55352), (5.46156,0.37031), (0.94225,7.1418), (12.79205,3.24897), (10.09593,-1.60918), (6.06218,3.1675), (0.89463,-17.97072), (11.88986,-5.61743), (10.79733,14.1422), (1.51371,14.87695), (2.20967,-4.65961), (15.45732,-0.99174), (16.5262,-2.96623), (5.99724,-9.02263), (8.3613,-17.2088), (15.68183,2.78608), (15.32117,6.74239), (14.15674,4.8524), (6.64553,7.46731), (4.20777,1.04894), (-0.10521,-12.8023), (-0.88169,-17.18188), (1.85913,-5.08801), (9.73673,22.13942), (0.30926,-0.36384), (6.17559,17.80564), (11.76602,7.67504), (5.68385,1.59779), (14.57088,4.10942), (12.81509,0.61074), (9.85682,-14.40767), (12.06376,10.59906), (6.08874,16.57017), (11.63921,-15.17526), (14.86722,-6.98549), (10.41035,-0.64548), (2.93794,3.23756), (12.21841,14.65504), (0.23804,4.583), (3.14845,12.72378), (7.29748,5.26547), (3.06134,0.81781), (13.77684,9.38273), (16.21992,10.37636), (5.33511,10.70325), (9.68959,-0.83043), (9.44169,-7.53149), (18.08012,-9.09147), (4.04224,-19.51381), (8.77918,-28.44508), (10.18324,6.44392), (9.38914,11.10201), (11.76995,-2.86184), (14.19963,8.30673), (6.88817,8.8797), (16.56123,10.68053), (15.39885,15.62919), (5.21241,8.00579), (4.44408,6.4651), (17.87587,-4.50029), (12.53337,18.04514), (13.60916,11.12996), (6.60104,-5.14007), (7.35453,9.43857), (18.61572,3.13476), (6.10437,4.9772), (13.08682,-17.45782), (12.15404,0.05552), (4.90789,-1.90283), (2.13353,2.67908), (12.49593,-2.62243), (11.93056,-3.22767), (13.29408,-8.70222), (5.70038,-23.11605), (8.40271,21.6757), (5.19456,12.70076), (-5.51028,4.4322), (14.0329,11.69344), (10.38365,9.18052), (6.56812,-2.2549), (4.21129,-2.15615), (9.7157,20.29765), (9.88553,-0.29536), (13.45346,15.50109), (4.97752,8.79187), (12.77595,5.11533), (8.56465,-20.44436), (4.27703,-3.00909), (18.12502,-4.48291), (12.45735,21.84462), (12.42912,1.94225), (12.08125,-2.81908), (10.85779,17.19418), (4.36013,-9.33528), (11.85062,-0.17346), (8.47776,0.03958), (9.60822,-35.17786), (11.3069,8.36887), (14.25525,-9.02292), (1.55168,-10.98804), (14.57782,0.29335), (7.84786,4.29634), (9.87774,3.87718), (14.75575,-9.08532), (3.68774,7.13922), (9.37667,-7.62463), (20.28676,-10.5666), (12.10027,4.68165), (8.01819,-3.30172), (18.78158,13.04852), (20.85402,13.45616), (18.98069,2.41043), (16.1429,-0.36501), (9.24047,-15.67383), (14.12487,17.92217), (10.18841,8.42106), (-3.04478,3.22063), (5.7552,-7.31753), (9.30376,21.99596), (11.42837,-36.8273), (6.02364,-20.46391), (8.86984,5.74179), (10.91177,-15.83178), (10.04418,14.90454), (18.10774,-8.84645), (7.49384,3.72036), (9.11556,4.6877), (9.7051,16.35418), (5.23268,3.15441), (9.04647,2.39907), (8.81547,-17.58664), (2.65098,-13.18269); SELECT '14.971190998235835', '5.898143508382202e-44'; -SELECT roundBankers(WelchTTest(left, right).1, 16) as t_stat, roundBankers(WelchTTest(left, right).2, 16) as p_value from welch_ttest; +SELECT roundBankers(welchTTest(left, right).1, 16) as t_stat, roundBankers(welchTTest(left, right).2, 16) as p_value from welch_ttest; DROP TABLE IF EXISTS welch_ttest; From 73f3aac66380c8d34da999efb139b0a79811f455 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Oct 2020 00:07:24 +0300 Subject: [PATCH 223/325] Fix assert error in quantileDeterministic #15683 --- .../ReservoirSamplerDeterministic.h | 70 ++++++++----------- 1 file changed, 30 insertions(+), 40 deletions(-) diff --git a/src/AggregateFunctions/ReservoirSamplerDeterministic.h b/src/AggregateFunctions/ReservoirSamplerDeterministic.h index 3097070c651..f0f926ce31e 100644 --- a/src/AggregateFunctions/ReservoirSamplerDeterministic.h +++ b/src/AggregateFunctions/ReservoirSamplerDeterministic.h @@ -39,8 +39,8 @@ namespace ErrorCodes namespace detail { -const size_t DEFAULT_SAMPLE_COUNT = 8192; -const auto MAX_SKIP_DEGREE = sizeof(UInt32) * 8; + const size_t DEFAULT_MAX_SAMPLE_SIZE = 8192; + const auto MAX_SKIP_DEGREE = sizeof(UInt32) * 8; } /// What if there is not a single value - throw an exception, or return 0 or NaN in the case of double? @@ -50,6 +50,7 @@ enum class ReservoirSamplerDeterministicOnEmpty RETURN_NAN_OR_ZERO, }; + template class ReservoirSamplerDeterministic @@ -60,8 +61,8 @@ class ReservoirSamplerDeterministic } public: - ReservoirSamplerDeterministic(const size_t sample_count_ = DEFAULT_SAMPLE_COUNT) - : sample_count{sample_count_} + ReservoirSamplerDeterministic(const size_t max_sample_size_ = detail::DEFAULT_MAX_SAMPLE_SIZE) + : max_sample_size{max_sample_size_} { } @@ -131,8 +132,8 @@ public: void merge(const ReservoirSamplerDeterministic & b) { - if (sample_count != b.sample_count) - throw Poco::Exception("Cannot merge ReservoirSamplerDeterministic's with different sample_count"); + if (max_sample_size != b.max_sample_size) + throw Poco::Exception("Cannot merge ReservoirSamplerDeterministic's with different max sample count"); sorted = false; if (b.skip_degree > skip_degree) @@ -150,11 +151,16 @@ public: void read(DB::ReadBuffer & buf) { - DB::readIntBinary(sample_count, buf); + size_t size = 0; + DB::readIntBinary(size, buf); DB::readIntBinary(total_values, buf); - samples.resize(std::min(total_values, sample_count)); - for (size_t i = 0; i < samples.size(); ++i) + /// Compatibility with old versions. + if (size > total_values) + size = total_values; + + samples.resize(size); + for (size_t i = 0; i < size; ++i) DB::readPODBinary(samples[i], buf); sorted = false; @@ -162,10 +168,11 @@ public: void write(DB::WriteBuffer & buf) const { - DB::writeIntBinary(sample_count, buf); + size_t size = samples.size(); + DB::writeIntBinary(size, buf); DB::writeIntBinary(total_values, buf); - for (size_t i = 0; i < std::min(sample_count, total_values); ++i) + for (size_t i = 0; i < size; ++i) DB::writePODBinary(samples[i], buf); } @@ -174,18 +181,19 @@ private: using Element = std::pair; using Array = DB::PODArray; - size_t sample_count; - size_t total_values{}; - bool sorted{}; + const size_t max_sample_size; /// Maximum amount of stored values. + size_t total_values = 0; /// How many values were inserted (regardless if they remain in sample or not). + bool sorted = false; Array samples; - UInt8 skip_degree{}; + UInt8 skip_degree = 0; /// The number N determining that we save only one per 2^N elements in average. void insertImpl(const T & v, const UInt32 hash) { - /// @todo why + 1? I don't quite recall - while (samples.size() + 1 >= sample_count) + /// Make a room for plus one element. + while (samples.size() >= max_sample_size) { - if (++skip_degree > detail::MAX_SKIP_DEGREE) + ++skip_degree; + if (skip_degree > detail::MAX_SKIP_DEGREE) throw DB::Exception{"skip_degree exceeds maximum value", DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED}; thinOut(); } @@ -195,35 +203,17 @@ private: void thinOut() { - auto size = samples.size(); - for (size_t i = 0; i < size;) - { - if (!good(samples[i].second)) - { - /// swap current element with the last one - std::swap(samples[size - 1], samples[i]); - --size; - } - else - ++i; - } - - if (size != samples.size()) - { - samples.resize(size); - sorted = false; - } + samples.resize(std::distance(samples.begin(), + std::remove_if(samples.begin(), samples.end(), [this](const auto & elem){ return !good(elem.second); }))); + sorted = false; } void sortIfNeeded() { if (sorted) return; + std::sort(samples.begin(), samples.end(), [](const auto & lhs, const auto & rhs) { return lhs.first < rhs.first; }); sorted = true; - std::sort(samples.begin(), samples.end(), [] (const std::pair & lhs, const std::pair & rhs) - { - return lhs.first < rhs.first; - }); } template From f78e0d48b7833fa6f6ee2f2c22876b69755860c1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Oct 2020 00:14:49 +0300 Subject: [PATCH 224/325] Added a test --- .../0_stateless/01533_quantile_deterministic_assert.reference | 1 + .../queries/0_stateless/01533_quantile_deterministic_assert.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 tests/queries/0_stateless/01533_quantile_deterministic_assert.reference create mode 100644 tests/queries/0_stateless/01533_quantile_deterministic_assert.sql diff --git a/tests/queries/0_stateless/01533_quantile_deterministic_assert.reference b/tests/queries/0_stateless/01533_quantile_deterministic_assert.reference new file mode 100644 index 00000000000..231c72269ca --- /dev/null +++ b/tests/queries/0_stateless/01533_quantile_deterministic_assert.reference @@ -0,0 +1 @@ +3998 diff --git a/tests/queries/0_stateless/01533_quantile_deterministic_assert.sql b/tests/queries/0_stateless/01533_quantile_deterministic_assert.sql new file mode 100644 index 00000000000..c75e5dd501f --- /dev/null +++ b/tests/queries/0_stateless/01533_quantile_deterministic_assert.sql @@ -0,0 +1 @@ +SELECT quantileDeterministic(number, sipHash64(number)) FROM remote('127.0.0.{1,2}', numbers(8193)); From a34b0880c31440a59e6aca5284211a10b1699c4f Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 27 Oct 2020 00:21:56 +0300 Subject: [PATCH 225/325] Update ReservoirSamplerDeterministic.h --- src/AggregateFunctions/ReservoirSamplerDeterministic.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/ReservoirSamplerDeterministic.h b/src/AggregateFunctions/ReservoirSamplerDeterministic.h index f0f926ce31e..eae24c1f3e9 100644 --- a/src/AggregateFunctions/ReservoirSamplerDeterministic.h +++ b/src/AggregateFunctions/ReservoirSamplerDeterministic.h @@ -133,7 +133,7 @@ public: void merge(const ReservoirSamplerDeterministic & b) { if (max_sample_size != b.max_sample_size) - throw Poco::Exception("Cannot merge ReservoirSamplerDeterministic's with different max sample count"); + throw Poco::Exception("Cannot merge ReservoirSamplerDeterministic's with different max sample size"); sorted = false; if (b.skip_degree > skip_degree) From 59910991b6745e779c8492bbf41928d957bcdaa8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Oct 2020 00:33:46 +0300 Subject: [PATCH 226/325] Fix "Darwin" --- src/AggregateFunctions/AggregateFunctionStudentTTest.h | 10 +++++++++- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 10 +++++++++- src/Functions/lgamma.cpp | 1 - 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.h b/src/AggregateFunctions/AggregateFunctionStudentTTest.h index a88f8151b31..5084e34e56f 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.h +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.h @@ -20,9 +20,17 @@ namespace ErrorCodes { -extern const int BAD_ARGUMENTS; + extern const int BAD_ARGUMENTS; } +#if defined(OS_DARWIN) +extern "C" +{ + double lgamma_r(double x, int * signgamp); +} +#endif + + namespace DB { diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 5f0cc409ba9..8fd7ebeee6b 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -21,9 +21,17 @@ namespace ErrorCodes { -extern const int BAD_ARGUMENTS; + extern const int BAD_ARGUMENTS; } +#if defined(OS_DARWIN) +extern "C" +{ + double lgamma_r(double x, int * signgamp); +} +#endif + + namespace DB { diff --git a/src/Functions/lgamma.cpp b/src/Functions/lgamma.cpp index 51b3dfd97df..e4da0d8dfbd 100644 --- a/src/Functions/lgamma.cpp +++ b/src/Functions/lgamma.cpp @@ -4,7 +4,6 @@ #if defined(OS_DARWIN) extern "C" { - /// Is defined in libglibc-compatibility.a double lgamma_r(double x, int * signgamp); } #endif From d6130f13ef47100dc733fd144b2e2225365656f0 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 27 Oct 2020 10:21:49 +0300 Subject: [PATCH 227/325] Add table to system tables --- src/Storages/System/attachSystemTables.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 2b7ee363f05..012b54e3c45 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -137,6 +138,7 @@ void attachSystemTablesServer(IDatabase & system_database, bool has_zookeeper) attach(system_database, "clusters"); attach(system_database, "graphite_retentions"); attach(system_database, "macros"); + attach(system_database, "fetches"); if (has_zookeeper) attach(system_database, "zookeeper"); From 381af53d4a17f301c826e4d4384db3f037e364fa Mon Sep 17 00:00:00 2001 From: tavplubix Date: Tue, 27 Oct 2020 11:44:58 +0300 Subject: [PATCH 228/325] Update clickhouse-test --- tests/clickhouse-test | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index d5736001d60..626d7ede814 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -180,7 +180,7 @@ def need_retry(stderr): def get_processlist(client_cmd): try: - return subprocess.check_output("{} --query 'SHOW PROCESSLIST FORMAT Vertical'".format(client_cmd), shell=True) + return subprocess.check_output("{} --query 'SHOW PROCESSLIST FORMAT Vertical'".format(client_cmd), shell=True).decode('utf-8') except: return "" # server seems dead @@ -189,7 +189,7 @@ def get_processlist(client_cmd): def get_stacktraces_from_gdb(server_pid): cmd = "gdb -batch -ex 'thread apply all backtrace' -p {}".format(server_pid) try: - return subprocess.check_output(cmd, shell=True) + return subprocess.check_output(cmd, shell=True).decode('utf-8') except Exception as ex: return "Error occured while receiving stack traces from gdb: {}".format(str(ex)) @@ -198,7 +198,10 @@ def get_stacktraces_from_gdb(server_pid): # it does not work in Sandbox def get_stacktraces_from_clickhouse(client): try: - return subprocess.check_output("{} --allow_introspection_functions=1 --query \"SELECT arrayStringConcat(arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace FROM system.stack_trace format Vertical\"".format(client), shell=True) + return subprocess.check_output("{} --allow_introspection_functions=1 --query " + "\"SELECT arrayStringConcat(arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), " + "arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace " + "FROM system.stack_trace format Vertical\"".format(client), shell=True).decode('utf-8') except Exception as ex: return "Error occured while receiving stack traces from client: {}".format(str(ex)) From cd0dbcb9d8259bc061a39cd835c768ce5bb8bbe3 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 25 Oct 2020 07:44:04 +0000 Subject: [PATCH 229/325] Minimize event loop lifetime --- .../RabbitMQ/RabbitMQBlockInputStream.h | 1 + src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 97 ++++++++++--------- src/Storages/RabbitMQ/StorageRabbitMQ.h | 1 + 3 files changed, 52 insertions(+), 47 deletions(-) diff --git a/src/Storages/RabbitMQ/RabbitMQBlockInputStream.h b/src/Storages/RabbitMQ/RabbitMQBlockInputStream.h index f68b79275f6..5f2c2a62018 100644 --- a/src/Storages/RabbitMQ/RabbitMQBlockInputStream.h +++ b/src/Storages/RabbitMQ/RabbitMQBlockInputStream.h @@ -30,6 +30,7 @@ public: Block readImpl() override; void readSuffixImpl() override; + bool queueEmpty() const { return !buffer || buffer->queueEmpty(); } bool needChannelUpdate(); void updateChannel(); bool sendAck(); diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 6565afeb32d..6fa8d52764d 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -38,8 +38,9 @@ namespace DB static const auto CONNECT_SLEEP = 200; static const auto RETRIES_MAX = 20; -static const auto HEARTBEAT_RESCHEDULE_MS = 3000; static const uint32_t QUEUE_SIZE = 100000; +static const auto MAX_FAILED_READ_ATTEMPTS = 10; +static const auto MAX_THREAD_WORK_DURATION_MS = 60000; namespace ErrorCodes { @@ -122,9 +123,6 @@ StorageRabbitMQ::StorageRabbitMQ( streaming_task = global_context.getSchedulePool().createTask("RabbitMQStreamingTask", [this]{ streamingToViewsFunc(); }); streaming_task->deactivate(); - heartbeat_task = global_context.getSchedulePool().createTask("RabbitMQHeartbeatTask", [this]{ heartbeatFunc(); }); - heartbeat_task->deactivate(); - if (queue_base.empty()) { /* Make sure that local exchange name is unique for each table and is not the same as client's exchange name. It also needs to @@ -210,16 +208,6 @@ Context StorageRabbitMQ::addSettings(Context context) const } -void StorageRabbitMQ::heartbeatFunc() -{ - if (!stream_cancelled && event_handler->connectionRunning()) - { - connection->heartbeat(); - heartbeat_task->scheduleAfter(HEARTBEAT_RESCHEDULE_MS); - } -} - - void StorageRabbitMQ::loopingFunc() { if (event_handler->connectionRunning()) @@ -402,7 +390,6 @@ bool StorageRabbitMQ::restoreConnection(bool reconnecting) if (reconnecting) { - deactivateTask(heartbeat_task, false, false); connection->close(); /// Connection might be unusable, but not closed /* Connection is not closed immediately (firstly, all pending operations are completed, and then @@ -452,7 +439,6 @@ void StorageRabbitMQ::unbindExchange() */ std::call_once(flag, [&]() { - heartbeat_task->deactivate(); streaming_task->deactivate(); event_handler->updateLoopState(Loop::STOP); looping_task->deactivate(); @@ -499,8 +485,6 @@ Pipe StorageRabbitMQ::read( deactivateTask(looping_task, false, true); update_channels = restoreConnection(true); - if (update_channels) - heartbeat_task->scheduleAfter(HEARTBEAT_RESCHEDULE_MS); } Pipes pipes; @@ -521,7 +505,6 @@ Pipe StorageRabbitMQ::read( if (event_handler->loopRunning()) { deactivateTask(looping_task, false, true); - deactivateTask(heartbeat_task, false, false); } rabbit_stream->updateChannel(); @@ -568,7 +551,6 @@ void StorageRabbitMQ::startup() event_handler->updateLoopState(Loop::RUN); streaming_task->activateAndSchedule(); - heartbeat_task->activateAndSchedule(); } @@ -579,7 +561,6 @@ void StorageRabbitMQ::shutdown() deactivateTask(streaming_task, true, false); deactivateTask(looping_task, true, true); - deactivateTask(heartbeat_task, true, false); connection->close(); @@ -688,6 +669,8 @@ void StorageRabbitMQ::streamingToViewsFunc() if (dependencies_count) { + auto start_time = std::chrono::steady_clock::now(); + // Keep streaming as long as there are attached views and streaming is not cancelled while (!stream_cancelled && num_created_consumers > 0) { @@ -696,8 +679,17 @@ void StorageRabbitMQ::streamingToViewsFunc() LOG_DEBUG(log, "Started streaming to {} attached views", dependencies_count); - if (!streamToViews()) + if (streamToViews()) break; + + auto end_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast(end_time - start_time); + if (duration.count() > MAX_THREAD_WORK_DURATION_MS) + { + event_handler->updateLoopState(Loop::STOP); + LOG_TRACE(log, "Reschedule streaming. Thread work duration limit exceeded."); + break; + } } } } @@ -731,13 +723,6 @@ bool StorageRabbitMQ::streamToViews() auto column_names = block_io.out->getHeader().getNames(); auto sample_block = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); - /* event_handler->connectionRunning() does not guarantee that connnection is not closed in case loop was not running before, but - * need to anyway start the loop to activate error callbacks and update connection state, because even checking with - * connection->usable() will not give correct answer before callbacks are activated. - */ - if (!event_handler->loopRunning() && event_handler->connectionRunning()) - looping_task->activateAndSchedule(); - auto block_size = getMaxBlockSize(); // Create a stream for each consumer and join them in a union stream @@ -770,34 +755,46 @@ bool StorageRabbitMQ::streamToViews() in = streams[0]; std::atomic stub = {false}; + + /// Loop could run untill this point only if select query was made + if (!event_handler->loopRunning()) + { + event_handler->updateLoopState(Loop::RUN); + looping_task->activateAndSchedule(); + } + copyData(*in, *block_io.out, &stub); - /* Need to stop loop even if connection is ok, because sending ack() with loop running in another thread will lead to a lot of data - * races inside the library, but only in case any error occurs or connection is lost while ack is being sent + /* Note: sending ack() with loop running in another thread will lead to a lot of data races inside the library, but only in case + * error occurs or connection is lost while ack is being sent */ - if (event_handler->loopRunning()) - deactivateTask(looping_task, false, true); + deactivateTask(looping_task, false, true); + size_t queue_empty = 0; if (!event_handler->connectionRunning()) { - if (!stream_cancelled && restoreConnection(true)) + if (stream_cancelled) + return true; + + if (restoreConnection(true)) { for (auto & stream : streams) stream->as()->updateChannel(); } else { - /// Reschedule if unable to connect to rabbitmq or quit if cancelled - return false; + LOG_TRACE(log, "Reschedule streaming. Unable to restore connection."); + return true; } } else { - deactivateTask(heartbeat_task, false, false); - /// Commit for (auto & stream : streams) { + if (stream->as()->queueEmpty()) + ++queue_empty; + /* false is returned by the sendAck function in only two cases: * 1) if connection failed. In this case all channels will be closed and will be unable to send ack. Also ack is made based on * delivery tags, which are unique to channels, so if channels fail, those delivery tags will become invalid and there is @@ -828,19 +825,25 @@ bool StorageRabbitMQ::streamToViews() break; } } + + event_handler->iterateLoop(); } } - event_handler->updateLoopState(Loop::RUN); - looping_task->activateAndSchedule(); - heartbeat_task->scheduleAfter(HEARTBEAT_RESCHEDULE_MS); /// It is also deactivated in restoreConnection(), so reschedule anyway + if ((queue_empty == num_queues) && (++read_attempts == MAX_FAILED_READ_ATTEMPTS)) + { + connection->heartbeat(); + read_attempts = 0; + LOG_TRACE(log, "Reschedule streaming. Queues are empty."); + return true; + } + else + { + event_handler->updateLoopState(Loop::RUN); + looping_task->activateAndSchedule(); + } - // Check whether the limits were applied during query execution - bool limits_applied = false; - const BlockStreamProfileInfo & info = in->getProfileInfo(); - limits_applied = info.hasAppliedLimit(); - - return limits_applied; + return false; } diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index 8d9a20f9e34..9cb4a5c59a1 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -122,6 +122,7 @@ private: BackgroundSchedulePool::TaskHolder looping_task; std::atomic stream_cancelled{false}; + size_t read_attempts = 0; ConsumerBufferPtr createReadBuffer(); From 7beddceadddbf7cdab18d274cfdbe6fce1466e06 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 27 Oct 2020 07:20:19 +0000 Subject: [PATCH 230/325] Fix bug that caused awful CPU usage --- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 3 ++- src/Storages/RabbitMQ/StorageRabbitMQ.h | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 6fa8d52764d..7998bedfb8c 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -40,6 +40,7 @@ static const auto CONNECT_SLEEP = 200; static const auto RETRIES_MAX = 20; static const uint32_t QUEUE_SIZE = 100000; static const auto MAX_FAILED_READ_ATTEMPTS = 10; +static const auto RESCHEDULE_MS = 500; static const auto MAX_THREAD_WORK_DURATION_MS = 60000; namespace ErrorCodes @@ -700,7 +701,7 @@ void StorageRabbitMQ::streamingToViewsFunc() /// Wait for attached views if (!stream_cancelled) - streaming_task->schedule(); + streaming_task->scheduleAfter(RESCHEDULE_MS); } diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index 9cb4a5c59a1..cb52b6bb282 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -118,7 +118,6 @@ private: std::once_flag flag; /// remove exchange only once std::mutex task_mutex; BackgroundSchedulePool::TaskHolder streaming_task; - BackgroundSchedulePool::TaskHolder heartbeat_task; BackgroundSchedulePool::TaskHolder looping_task; std::atomic stream_cancelled{false}; From db03cd7dd7b984b5a267fa7299f6e6e44bbf1f6c Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 27 Oct 2020 07:14:38 +0000 Subject: [PATCH 231/325] Much more optimal queues setup --- .../table-engines/integrations/rabbitmq.md | 2 +- .../table-engines/integrations/rabbitmq.md | 2 +- .../ReadBufferFromRabbitMQConsumer.cpp | 71 +--------------- .../RabbitMQ/ReadBufferFromRabbitMQConsumer.h | 8 +- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 80 ++++++++++++++++++- src/Storages/RabbitMQ/StorageRabbitMQ.h | 2 + .../integration/test_storage_rabbitmq/test.py | 10 +-- 7 files changed, 89 insertions(+), 86 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/rabbitmq.md b/docs/en/engines/table-engines/integrations/rabbitmq.md index dd14ee3b4b1..0340603eaae 100644 --- a/docs/en/engines/table-engines/integrations/rabbitmq.md +++ b/docs/en/engines/table-engines/integrations/rabbitmq.md @@ -51,7 +51,7 @@ Optional parameters: - `rabbitmq_row_delimiter` – Delimiter character, which ends the message. - `rabbitmq_schema` – Parameter that must be used if the format requires a schema definition. For example, [Cap’n Proto](https://capnproto.org/) requires the path to the schema file and the name of the root `schema.capnp:Message` object. - `rabbitmq_num_consumers` – The number of consumers per table. Default: `1`. Specify more consumers if the throughput of one consumer is insufficient. -- `rabbitmq_num_queues` – The number of queues per consumer. Default: `1`. Specify more queues if the capacity of one queue per consumer is insufficient. +- `rabbitmq_num_queues` – Total number of queues. Default: `1`. Increasing this number can significantly improve performance. - `rabbitmq_queue_base` - Specify a hint for queue names. Use cases of this setting are described below. - `rabbitmq_deadletter_exchange` - Specify name for a [dead letter exchange](https://www.rabbitmq.com/dlx.html). You can create another table with this exchange name and collect messages in cases when they are republished to dead letter exchange. By default dead letter exchange is not specified. - `rabbitmq_persistent` - If set to 1 (true), in insert query delivery mode will be set to 2 (marks messages as 'persistent'). Default: `0`. diff --git a/docs/ru/engines/table-engines/integrations/rabbitmq.md b/docs/ru/engines/table-engines/integrations/rabbitmq.md index ef7b811e295..75f20827df2 100644 --- a/docs/ru/engines/table-engines/integrations/rabbitmq.md +++ b/docs/ru/engines/table-engines/integrations/rabbitmq.md @@ -45,7 +45,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - `rabbitmq_row_delimiter` – символ-разделитель, который завершает сообщение. - `rabbitmq_schema` – опциональный параметр, необходимый, если используется формат, требующий определения схемы. Например, [Cap’n Proto](https://capnproto.org/) требует путь к файлу со схемой и название корневого объекта `schema.capnp:Message`. - `rabbitmq_num_consumers` – количество потребителей на таблицу. По умолчанию: `1`. Укажите больше потребителей, если пропускная способность одного потребителя недостаточна. -- `rabbitmq_num_queues` – количество очередей на потребителя. По умолчанию: `1`. Укажите больше потребителей, если пропускная способность одной очереди на потребителя недостаточна. +- `rabbitmq_num_queues` – количество очередей. По умолчанию: `1`. Большее число очередей может сильно увеличить пропускную способность. - `rabbitmq_queue_base` - настройка для имен очередей. Сценарии использования описаны ниже. - `rabbitmq_persistent` - флаг, от которого зависит настройка 'durable' для сообщений при запросах `INSERT`. По умолчанию: `0`. - `rabbitmq_skip_broken_messages` – максимальное количество некорректных сообщений в блоке. Если `rabbitmq_skip_broken_messages = N`, то движок отбрасывает `N` сообщений, которые не получилось обработать. Одно сообщение в точности соответствует одной записи (строке). Значение по умолчанию – 0. diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp index 43a9d75d084..5644b8d7c6a 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp @@ -17,33 +17,30 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; - extern const int CANNOT_CREATE_RABBITMQ_QUEUE_BINDING; } ReadBufferFromRabbitMQConsumer::ReadBufferFromRabbitMQConsumer( ChannelPtr consumer_channel_, - ChannelPtr setup_channel_, HandlerPtr event_handler_, const String & exchange_name_, + std::vector & queues_, size_t channel_id_base_, const String & channel_base_, const String & queue_base_, Poco::Logger * log_, char row_delimiter_, - bool hash_exchange_, size_t num_queues_, const String & deadletter_exchange_, uint32_t queue_size_, const std::atomic & stopped_) : ReadBuffer(nullptr, 0) , consumer_channel(std::move(consumer_channel_)) - , setup_channel(setup_channel_) , event_handler(event_handler_) , exchange_name(exchange_name_) + , queues(queues_) , channel_base(channel_base_) , channel_id_base(channel_id_base_) , queue_base(queue_base_) - , hash_exchange(hash_exchange_) , num_queues(num_queues_) , deadletter_exchange(deadletter_exchange_) , log(log_) @@ -52,9 +49,6 @@ ReadBufferFromRabbitMQConsumer::ReadBufferFromRabbitMQConsumer( , stopped(stopped_) , received(queue_size * num_queues) { - for (size_t queue_id = 0; queue_id < num_queues; ++queue_id) - bindQueue(queue_id); - setupChannel(); } @@ -65,67 +59,6 @@ ReadBufferFromRabbitMQConsumer::~ReadBufferFromRabbitMQConsumer() } -void ReadBufferFromRabbitMQConsumer::bindQueue(size_t queue_id) -{ - std::atomic binding_created = false; - - auto success_callback = [&](const std::string & queue_name, int msgcount, int /* consumercount */) - { - queues.emplace_back(queue_name); - LOG_DEBUG(log, "Queue {} is declared", queue_name); - - if (msgcount) - LOG_INFO(log, "Queue {} is non-empty. Non-consumed messaged will also be delivered", queue_name); - - /* Here we bind either to sharding exchange (consistent-hash) or to bridge exchange (fanout). All bindings to routing keys are - * done between client's exchange and local bridge exchange. Binding key must be a string integer in case of hash exchange, for - * fanout exchange it can be arbitrary - */ - setup_channel->bindQueue(exchange_name, queue_name, std::to_string(channel_id_base)) - .onSuccess([&] { binding_created = true; }) - .onError([&](const char * message) - { - throw Exception( - ErrorCodes::CANNOT_CREATE_RABBITMQ_QUEUE_BINDING, - "Failed to create queue binding with queue {} for exchange {}. Reason: {}", std::string(message), - queue_name, exchange_name); - }); - }; - - auto error_callback([&](const char * message) - { - /* This error is most likely a result of an attempt to declare queue with different settings if it was declared before. So for a - * given queue name either deadletter_exchange parameter changed or queue_size changed, i.e. table was declared with different - * max_block_size parameter. Solution: client should specify a different queue_base parameter or manually delete previously - * declared queues via any of the various cli tools. - */ - throw Exception("Failed to declare queue. Probably queue settings are conflicting: max_block_size, deadletter_exchange. Attempt \ - specifying differently those settings or use a different queue_base or manually delete previously declared queues, \ - which were declared with the same names. ERROR reason: " - + std::string(message), ErrorCodes::BAD_ARGUMENTS); - }); - - AMQP::Table queue_settings; - - queue_settings["x-max-length"] = queue_size; - queue_settings["x-overflow"] = "reject-publish"; - - if (!deadletter_exchange.empty()) - queue_settings["x-dead-letter-exchange"] = deadletter_exchange; - - /* The first option not just simplifies queue_name, but also implements the possibility to be able to resume reading from one - * specific queue when its name is specified in queue_base setting - */ - const String queue_name = !hash_exchange ? queue_base : std::to_string(channel_id_base) + "_" + std::to_string(queue_id) + "_" + queue_base; - setup_channel->declareQueue(queue_name, AMQP::durable, queue_settings).onSuccess(success_callback).onError(error_callback); - - while (!binding_created) - { - iterateEventLoop(); - } -} - - void ReadBufferFromRabbitMQConsumer::subscribe() { for (const auto & queue_name : queues) diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h index 109770c77e9..556e069083c 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h @@ -24,15 +24,14 @@ class ReadBufferFromRabbitMQConsumer : public ReadBuffer public: ReadBufferFromRabbitMQConsumer( ChannelPtr consumer_channel_, - ChannelPtr setup_channel_, HandlerPtr event_handler_, const String & exchange_name_, + std::vector & queues_, size_t channel_id_base_, const String & channel_base_, const String & queue_base_, Poco::Logger * log_, char row_delimiter_, - bool hash_exchange_, size_t num_queues_, const String & deadletter_exchange_, uint32_t queue_size_, @@ -79,19 +78,17 @@ public: private: bool nextImpl() override; - void bindQueue(size_t queue_id); void subscribe(); void iterateEventLoop(); ChannelPtr consumer_channel; - ChannelPtr setup_channel; HandlerPtr event_handler; const String exchange_name; + std::vector queues; const String channel_base; const size_t channel_id_base; const String queue_base; - const bool hash_exchange; const size_t num_queues; const String deadletter_exchange; Poco::Logger * log; @@ -102,7 +99,6 @@ private: String channel_id; std::atomic channel_error = true, wait_subscription = false; - std::vector queues; ConcurrentBoundedQueue received; MessageData current; size_t subscribed = 0; diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 7998bedfb8c..e49b18ea391 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -52,6 +52,7 @@ namespace ErrorCodes extern const int CANNOT_BIND_RABBITMQ_EXCHANGE; extern const int CANNOT_DECLARE_RABBITMQ_EXCHANGE; extern const int CANNOT_REMOVE_RABBITMQ_EXCHANGE; + extern const int CANNOT_CREATE_RABBITMQ_QUEUE_BINDING; } namespace ExchangeType @@ -385,6 +386,67 @@ void StorageRabbitMQ::bindExchange() } +void StorageRabbitMQ::bindQueue(size_t queue_id) +{ + std::atomic binding_created = false; + + auto success_callback = [&](const std::string & queue_name, int msgcount, int /* consumercount */) + { + queues.emplace_back(queue_name); + LOG_DEBUG(log, "Queue {} is declared", queue_name); + + if (msgcount) + LOG_INFO(log, "Queue {} is non-empty. Non-consumed messaged will also be delivered", queue_name); + + /* Here we bind either to sharding exchange (consistent-hash) or to bridge exchange (fanout). All bindings to routing keys are + * done between client's exchange and local bridge exchange. Binding key must be a string integer in case of hash exchange, for + * fanout exchange it can be arbitrary + */ + setup_channel->bindQueue(consumer_exchange, queue_name, std::to_string(queue_id)) + .onSuccess([&] { binding_created = true; }) + .onError([&](const char * message) + { + throw Exception( + ErrorCodes::CANNOT_CREATE_RABBITMQ_QUEUE_BINDING, + "Failed to create queue binding for exchange {}. Reason: {}", exchange_name, std::string(message)); + }); + }; + + auto error_callback([&](const char * message) + { + /* This error is most likely a result of an attempt to declare queue with different settings if it was declared before. So for a + * given queue name either deadletter_exchange parameter changed or queue_size changed, i.e. table was declared with different + * max_block_size parameter. Solution: client should specify a different queue_base parameter or manually delete previously + * declared queues via any of the various cli tools. + */ + throw Exception("Failed to declare queue. Probably queue settings are conflicting: max_block_size, deadletter_exchange. Attempt \ + specifying differently those settings or use a different queue_base or manually delete previously declared queues, \ + which were declared with the same names. ERROR reason: " + + std::string(message), ErrorCodes::BAD_ARGUMENTS); + }); + + AMQP::Table queue_settings; + + queue_settings["x-max-length"] = queue_size; + + if (!deadletter_exchange.empty()) + queue_settings["x-dead-letter-exchange"] = deadletter_exchange; + else + queue_settings["x-overflow"] = "reject-publish"; + + /* The first option not just simplifies queue_name, but also implements the possibility to be able to resume reading from one + * specific queue when its name is specified in queue_base setting + */ + const String queue_name = !hash_exchange ? queue_base : std::to_string(queue_id) + "_" + queue_base; + setup_channel->declareQueue(queue_name, AMQP::durable, queue_settings).onSuccess(success_callback).onError(error_callback); + + while (!binding_created) + { + event_handler->iterateLoop(); + } +} + + bool StorageRabbitMQ::restoreConnection(bool reconnecting) { size_t cnt_retries = 0; @@ -444,6 +506,7 @@ void StorageRabbitMQ::unbindExchange() event_handler->updateLoopState(Loop::STOP); looping_task->deactivate(); + setup_channel = std::make_shared(connection.get()); setup_channel->removeExchange(bridge_exchange) .onSuccess([&]() { @@ -458,6 +521,8 @@ void StorageRabbitMQ::unbindExchange() { event_handler->iterateLoop(); } + + setup_channel->close(); }); } @@ -536,6 +601,13 @@ void StorageRabbitMQ::startup() initExchange(); bindExchange(); + for (size_t i = 1; i <= num_queues; ++i) + { + bindQueue(i); + } + + setup_channel->close(); + for (size_t i = 0; i < num_consumers; ++i) { try @@ -617,8 +689,8 @@ ConsumerBufferPtr StorageRabbitMQ::createReadBuffer() ChannelPtr consumer_channel = std::make_shared(connection.get()); return std::make_shared( - consumer_channel, setup_channel, event_handler, consumer_exchange, ++consumer_id, - unique_strbase, queue_base, log, row_delimiter, hash_exchange, num_queues, + consumer_channel, event_handler, consumer_exchange, queues, ++consumer_id, + unique_strbase, queue_base, log, row_delimiter, num_queues, deadletter_exchange, queue_size, stream_cancelled); } @@ -665,6 +737,7 @@ void StorageRabbitMQ::streamingToViewsFunc() try { auto table_id = getStorageID(); + // Check if at least one direct dependency is attached size_t dependencies_count = DatabaseCatalog::instance().getDependencies(table_id).size(); @@ -757,7 +830,6 @@ bool StorageRabbitMQ::streamToViews() std::atomic stub = {false}; - /// Loop could run untill this point only if select query was made if (!event_handler->loopRunning()) { event_handler->updateLoopState(Loop::RUN); @@ -831,7 +903,7 @@ bool StorageRabbitMQ::streamToViews() } } - if ((queue_empty == num_queues) && (++read_attempts == MAX_FAILED_READ_ATTEMPTS)) + if ((queue_empty == num_created_consumers) && (++read_attempts == MAX_FAILED_READ_ATTEMPTS)) { connection->heartbeat(); read_attempts = 0; diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h index cb52b6bb282..d7891aed0a7 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.h +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h @@ -114,6 +114,7 @@ private: std::atomic wait_confirm = true; /// needed to break waiting for confirmations for producer std::atomic exchange_removed = false; ChannelPtr setup_channel; + std::vector queues; std::once_flag flag; /// remove exchange only once std::mutex task_mutex; @@ -140,6 +141,7 @@ private: void initExchange(); void bindExchange(); + void bindQueue(size_t queue_id); bool restoreConnection(bool reconnecting); bool streamToViews(); diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index ab44d0ebea0..d7f98d5cb77 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -537,14 +537,14 @@ def test_rabbitmq_big_message(rabbitmq_cluster): @pytest.mark.timeout(420) def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster): NUM_CONSUMERS = 10 - NUM_QUEUES = 2 + NUM_QUEUES = 10 instance.query(''' CREATE TABLE test.rabbitmq (key UInt64, value UInt64) ENGINE = RabbitMQ SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'test_sharding', - rabbitmq_num_queues = 2, + rabbitmq_num_queues = 10, rabbitmq_num_consumers = 10, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; @@ -617,7 +617,7 @@ def test_rabbitmq_mv_combo(rabbitmq_cluster): rabbitmq_exchange_name = 'combo', rabbitmq_queue_base = 'combo', rabbitmq_num_consumers = 2, - rabbitmq_num_queues = 2, + rabbitmq_num_queues = 5, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; ''') @@ -879,7 +879,7 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): rabbitmq_queue_base = 'over', rabbitmq_exchange_type = 'direct', rabbitmq_num_consumers = 5, - rabbitmq_num_queues = 2, + rabbitmq_num_queues = 10, rabbitmq_max_block_size = 10000, rabbitmq_routing_key_list = 'over', rabbitmq_format = 'TSV', @@ -1722,7 +1722,7 @@ def test_rabbitmq_restore_failed_connection_without_losses_2(rabbitmq_cluster): SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', rabbitmq_exchange_name = 'consumer_reconnect', rabbitmq_num_consumers = 10, - rabbitmq_num_queues = 2, + rabbitmq_num_queues = 10, rabbitmq_format = 'JSONEachRow', rabbitmq_row_delimiter = '\\n'; ''') From 41971e073a6d6eb6c4f62f641d4899cc4f8d6f03 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Tue, 27 Oct 2020 12:04:03 +0100 Subject: [PATCH 232/325] Fix typos reported by codespell --- CHANGELOG.md | 4 ++-- base/common/StringRef.h | 2 +- cmake/yandex/ya.make.versions.inc | 6 +++--- contrib/libhdfs3-cmake/CMakeLists.txt | 2 +- docker/test/performance-comparison/compare.sh | 12 ++++++------ .../data-types/special-data-types/interval.md | 2 +- docs/en/sql-reference/functions/other-functions.md | 4 ++-- docs/en/sql-reference/operators/index.md | 2 +- docs/en/sql-reference/statements/create/table.md | 6 +++--- programs/local/LocalServer.cpp | 2 +- programs/server/config.xml | 2 +- src/Access/IAccessStorage.cpp | 2 +- src/AggregateFunctions/AggregateFunctionGroupArray.h | 2 +- src/Columns/IColumnUnique.h | 2 +- src/Common/FileSyncGuard.h | 6 +++--- src/Common/ThreadPool.cpp | 4 ++-- src/Common/TraceCollector.cpp | 2 +- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 4 ++-- src/Compression/CompressionFactory.cpp | 2 +- src/Compression/ICompressionCodec.cpp | 2 +- src/Core/Defines.h | 2 +- src/Core/MySQL/MySQLReplication.cpp | 2 +- src/Core/Settings.h | 2 +- src/DataTypes/NumberTraits.h | 2 +- src/Databases/DatabaseAtomic.cpp | 2 +- src/Databases/MySQL/ConnectionMySQLSettings.h | 2 +- src/Functions/FunctionBinaryArithmetic.h | 2 +- src/Functions/FunctionsAES.h | 2 +- src/Functions/formatReadableTimeDelta.cpp | 2 +- src/Functions/isDecimalOverflow.cpp | 2 +- src/Interpreters/ThreadStatusExt.cpp | 2 +- src/Interpreters/executeQuery.cpp | 6 +++--- src/Parsers/obfuscateQueries.cpp | 2 +- src/Processors/IAccumulatingTransform.h | 2 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 2 +- src/Storages/MergeTree/IMergeTreeDataPart.h | 2 +- src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp | 2 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 2 +- src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h | 2 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 2 +- .../MergeTree/ReplicatedMergeTreeRestartingThread.h | 2 +- src/Storages/MergeTree/TTLMergeSelector.h | 2 +- .../RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp | 8 ++++---- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 4 ++-- .../RabbitMQ/WriteBufferToRabbitMQProducer.h | 8 ++++---- tests/integration/test_disk_types/test.py | 4 ++-- .../test_distributed_load_balancing/test.py | 2 +- tests/queries/0_stateless/00932_geohash_support.sql | 2 +- .../01281_group_by_limit_memory_tracking.sh | 2 +- utils/db-generator/README.md | 6 +++--- 50 files changed, 77 insertions(+), 77 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e4acdc293f..09ce72d20ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -409,7 +409,7 @@ ## ClickHouse release 20.6 -### ClickHouse release v20.6.3.28-stable +### ClickHouse release v20.6.3.28-stable #### New Feature @@ -2362,7 +2362,7 @@ No changes compared to v20.4.3.16-stable. * `Live View` table engine refactoring. [#8519](https://github.com/ClickHouse/ClickHouse/pull/8519) ([vzakaznikov](https://github.com/vzakaznikov)) * Add additional checks for external dictionaries created from DDL-queries. [#8127](https://github.com/ClickHouse/ClickHouse/pull/8127) ([alesapin](https://github.com/alesapin)) * Fix error `Column ... already exists` while using `FINAL` and `SAMPLE` together, e.g. `select count() from table final sample 1/2`. Fixes [#5186](https://github.com/ClickHouse/ClickHouse/issues/5186). [#7907](https://github.com/ClickHouse/ClickHouse/pull/7907) ([Nikolai Kochetov](https://github.com/KochetovNicolai)) -* Now table the first argument of `joinGet` function can be table indentifier. [#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([Amos Bird](https://github.com/amosbird)) +* Now table the first argument of `joinGet` function can be table identifier. [#7707](https://github.com/ClickHouse/ClickHouse/pull/7707) ([Amos Bird](https://github.com/amosbird)) * Allow using `MaterializedView` with subqueries above `Kafka` tables. [#8197](https://github.com/ClickHouse/ClickHouse/pull/8197) ([filimonov](https://github.com/filimonov)) * Now background moves between disks run it the seprate thread pool. [#7670](https://github.com/ClickHouse/ClickHouse/pull/7670) ([Vladimir Chebotarev](https://github.com/excitoon)) * `SYSTEM RELOAD DICTIONARY` now executes synchronously. [#8240](https://github.com/ClickHouse/ClickHouse/pull/8240) ([Vitaly Baranov](https://github.com/vitlibar)) diff --git a/base/common/StringRef.h b/base/common/StringRef.h index 4376876c077..b51b95456cb 100644 --- a/base/common/StringRef.h +++ b/base/common/StringRef.h @@ -51,7 +51,7 @@ struct StringRef }; /// Here constexpr doesn't implicate inline, see https://www.viva64.com/en/w/v1043/ -/// nullptr can't be used because the StringRef values are used in SipHash's pointer arithmetics +/// nullptr can't be used because the StringRef values are used in SipHash's pointer arithmetic /// and the UBSan thinks that something like nullptr + 8 is UB. constexpr const inline char empty_string_ref_addr{}; constexpr const inline StringRef EMPTY_STRING_REF{&empty_string_ref_addr, 0}; diff --git a/cmake/yandex/ya.make.versions.inc b/cmake/yandex/ya.make.versions.inc index 3ac401cb108..6910164d0aa 100644 --- a/cmake/yandex/ya.make.versions.inc +++ b/cmake/yandex/ya.make.versions.inc @@ -11,11 +11,11 @@ CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=${VERSION_MAJOR}) CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=${VERSION_MINOR}) CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=${VERSION_PATCH}) CFLAGS (GLOBAL -DVERSION_FULL=\"\\\"${VERSION_FULL}\\\"\") -CFLAGS (GLOBAL -DVERSION_MAJOR=${VERSION_MAJOR}) -CFLAGS (GLOBAL -DVERSION_MINOR=${VERSION_MINOR}) +CFLAGS (GLOBAL -DVERSION_MAJOR=${VERSION_MAJOR}) +CFLAGS (GLOBAL -DVERSION_MINOR=${VERSION_MINOR}) CFLAGS (GLOBAL -DVERSION_PATCH=${VERSION_PATCH}) -# TODO: not supported yet, not sure if ya.make supports arithmetics. +# TODO: not supported yet, not sure if ya.make supports arithmetic. CFLAGS (GLOBAL -DVERSION_INTEGER=0) CFLAGS (GLOBAL -DVERSION_NAME=\"\\\"${VERSION_NAME}\\\"\") diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index ab4857f0387..3aa5cb970db 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -192,7 +192,7 @@ set(SRCS ${HDFS3_SOURCE_DIR}/common/FileWrapper.h ) -# old kernels (< 3.17) doens't have SYS_getrandom. Always use POSIX implementation to have better compatibility +# old kernels (< 3.17) doesn't have SYS_getrandom. Always use POSIX implementation to have better compatibility set_source_files_properties(${HDFS3_SOURCE_DIR}/rpc/RpcClient.cpp PROPERTIES COMPILE_FLAGS "-DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1") # target diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 30443f21fba..d0cf12baa9e 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -63,7 +63,7 @@ function configure # Make copies of the original db for both servers. Use hardlinks instead # of copying to save space. Before that, remove preprocessed configs and # system tables, because sharing them between servers with hardlinks may - # lead to weird effects. + # lead to weird effects. rm -r left/db ||: rm -r right/db ||: rm -r db0/preprocessed_configs ||: @@ -82,7 +82,7 @@ function restart export MALLOC_CONF="confirm_conf:true" set -m # Spawn servers in their own process groups - + left/clickhouse-server --config-file=left/config/config.xml \ -- --path left/db --user_files_path left/db/user_files \ &>> left-server-log.log & @@ -208,7 +208,7 @@ function run_tests echo test "$test_name" # Don't profile if we're past the time limit. - # Use awk because bash doesn't support floating point arithmetics. + # Use awk because bash doesn't support floating point arithmetic. profile_seconds=$(awk "BEGIN { print ($profile_seconds_left > 0 ? 10 : 0) }") TIMEFORMAT=$(printf "$test_name\t%%3R\t%%3U\t%%3S\n") @@ -541,10 +541,10 @@ create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv') as select abs(diff) > report_threshold and abs(diff) > stat_threshold as changed_fail, abs(diff) > report_threshold - 0.05 and abs(diff) > stat_threshold as changed_show, - + not changed_fail and stat_threshold > report_threshold + 0.10 as unstable_fail, not changed_show and stat_threshold > report_threshold - 0.05 as unstable_show, - + left, right, diff, stat_threshold, if(report_threshold > 0, report_threshold, 0.10) as report_threshold, query_metric_stats.test test, query_metric_stats.query_index query_index, @@ -767,7 +767,7 @@ create table all_tests_report engine File(TSV, 'report/all-queries.tsv') as -- The threshold for 2) is significantly larger than the threshold for 1), to -- avoid jitter. create view shortness - as select + as select (test, query_index) in (select * from file('analyze/marked-short-queries.tsv', TSV, 'test text, query_index int')) diff --git a/docs/en/sql-reference/data-types/special-data-types/interval.md b/docs/en/sql-reference/data-types/special-data-types/interval.md index 8a4b9ae7886..7c0c5b00c0d 100644 --- a/docs/en/sql-reference/data-types/special-data-types/interval.md +++ b/docs/en/sql-reference/data-types/special-data-types/interval.md @@ -80,4 +80,4 @@ Code: 43. DB::Exception: Received from localhost:9000. DB::Exception: Wrong argu ## See Also {#see-also} - [INTERVAL](../../../sql-reference/operators/index.md#operator-interval) operator -- [toInterval](../../../sql-reference/functions/type-conversion-functions.md#function-tointerval) type convertion functions +- [toInterval](../../../sql-reference/functions/type-conversion-functions.md#function-tointerval) type conversion functions diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 1145efbbc5f..518479fb728 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -551,7 +551,7 @@ formatReadableTimeDelta(column[, maximum_unit]) **Parameters** - `column` — A column with numeric time delta. -- `maximum_unit` — Optional. Maximum unit to show. Acceptable values seconds, minutes, hours, days, months, years. +- `maximum_unit` — Optional. Maximum unit to show. Acceptable values seconds, minutes, hours, days, months, years. Example: @@ -1584,7 +1584,7 @@ isDecimalOverflow(d, [p]) **Parameters** - `d` — value. [Decimal](../../sql-reference/data-types/decimal.md). -- `p` — precision. Optional. If omitted, the initial presicion of the first argument is used. Using of this paratemer could be helpful for data extraction to another DBMS or file. [UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges). +- `p` — precision. Optional. If omitted, the initial precision of the first argument is used. Using of this paratemer could be helpful for data extraction to another DBMS or file. [UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges). **Returned values** diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index e5554b58e4a..262ae00dc95 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -169,7 +169,7 @@ SELECT now() AS current_date_time, current_date_time + INTERVAL 4 DAY + INTERVAL **See Also** - [Interval](../../sql-reference/data-types/special-data-types/interval.md) data type -- [toInterval](../../sql-reference/functions/type-conversion-functions.md#function-tointerval) type convertion functions +- [toInterval](../../sql-reference/functions/type-conversion-functions.md#function-tointerval) type conversion functions ## Logical Negation Operator {#logical-negation-operator} diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 35780856ef2..82326bf51cf 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -121,7 +121,7 @@ Defines storage time for values. Can be specified only for MergeTree-family tabl ## Column Compression Codecs {#codecs} -By default, ClickHouse applies the `lz4` compression method. For `MergeTree`-engine family you can change the default compression method in the [compression](../../../operations/server-configuration-parameters/settings.md#server-settings-compression) section of a server configuration. +By default, ClickHouse applies the `lz4` compression method. For `MergeTree`-engine family you can change the default compression method in the [compression](../../../operations/server-configuration-parameters/settings.md#server-settings-compression) section of a server configuration. You can also define the compression method for each individual column in the `CREATE TABLE` query. @@ -138,7 +138,7 @@ ENGINE = ... ``` -The `Default` codec can be specified to reference default compression which may dependend on different settings (and properties of data) in runtime. +The `Default` codec can be specified to reference default compression which may depend on different settings (and properties of data) in runtime. Example: `value UInt64 CODEC(Default)` — the same as lack of codec specification. Also you can remove current CODEC from the column and use default compression from config.xml: @@ -149,7 +149,7 @@ ALTER TABLE codec_example MODIFY COLUMN float_value CODEC(Default); Codecs can be combined in a pipeline, for example, `CODEC(Delta, Default)`. -To select the best codec combination for you project, pass benchmarks similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article. One thing to note is that codec can't be applied for ALIAS column type. +To select the best codec combination for you project, pass benchmarks similar to described in the Altinity [New Encodings to Improve ClickHouse Efficiency](https://www.altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse) article. One thing to note is that codec can't be applied for ALIAS column type. !!! warning "Warning" You can’t decompress ClickHouse database files with external utilities like `lz4`. Instead, use the special [clickhouse-compressor](https://github.com/ClickHouse/ClickHouse/tree/master/programs/compressor) utility. diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 63667307876..b9dde555788 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -152,7 +152,7 @@ void LocalServer::tryInitPath() default_path = parent_folder / fmt::format("clickhouse-local-{}-{}-{}", getpid(), time(nullptr), randomSeed()); if (exists(default_path)) - throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "Unsuccessfull attempt to create working directory: {} exist!", default_path.string()); + throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "Unsuccessful attempt to create working directory: {} exist!", default_path.string()); create_directory(default_path); temporary_directory_to_delete = default_path; diff --git a/programs/server/config.xml b/programs/server/config.xml index 5bdec5377fd..679e3e9e6f4 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -270,7 +270,7 @@ This parameter is mandatory and cannot be empty. roles - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. If no roles are specified, user will not be able to perform any actions after authentication. - If any of the listed roles is not defined locally at the time of authentication, the authenthication attept + If any of the listed roles is not defined locally at the time of authentication, the authenthication attempt will fail as if the provided password was incorrect. Example: diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 8dd219e07d7..58821e7de4b 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -585,7 +585,7 @@ void IAccessStorage::throwInvalidPassword() void IAccessStorage::throwCannotAuthenticate(const String & user_name) { - /// We use the same message for all authentification failures because we don't want to give away any unnecessary information for security reasons, + /// We use the same message for all authentication failures because we don't want to give away any unnecessary information for security reasons, /// only the log will show the exact reason. throw Exception(user_name + ": Authentication failed: password is incorrect or there is no user with such name", ErrorCodes::AUTHENTICATION_FAILED); } diff --git a/src/AggregateFunctions/AggregateFunctionGroupArray.h b/src/AggregateFunctions/AggregateFunctionGroupArray.h index 83e096c797b..02b9003eb96 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArray.h +++ b/src/AggregateFunctions/AggregateFunctionGroupArray.h @@ -296,7 +296,7 @@ public: { typename ColumnVector::Container & data_to = assert_cast &>(arr_to.getData()).getData(); if constexpr (is_big_int_v) - // is data_to empty? we should probaly use std::vector::insert then + // is data_to empty? we should probably use std::vector::insert then for (auto it = this->data(place).value.begin(); it != this->data(place).value.end(); it++) data_to.push_back(*it); else diff --git a/src/Columns/IColumnUnique.h b/src/Columns/IColumnUnique.h index 99facea6055..99e134675f6 100644 --- a/src/Columns/IColumnUnique.h +++ b/src/Columns/IColumnUnique.h @@ -82,7 +82,7 @@ public: * @see DB::ColumnUnique * * The most common example uses https://clickhouse.tech/docs/en/sql-reference/data-types/lowcardinality/ columns. - * Consider data type @e LC(String). The inner type here is @e String which is more or less a contigous memory + * Consider data type @e LC(String). The inner type here is @e String which is more or less a contiguous memory * region, so it can be easily represented as a @e StringRef. So we pass that ref to this function and get its * index in the dictionary, which can be used to operate with the indices column. */ diff --git a/src/Common/FileSyncGuard.h b/src/Common/FileSyncGuard.h index 6451f6ebf36..486b02d0f24 100644 --- a/src/Common/FileSyncGuard.h +++ b/src/Common/FileSyncGuard.h @@ -5,15 +5,15 @@ namespace DB { -/// Helper class, that recieves file descriptor and does fsync for it in destructor. +/// Helper class, that receives file descriptor and does fsync for it in destructor. /// It's used to keep descriptor open, while doing some operations with it, and do fsync at the end. /// Guaranties of sequence 'close-reopen-fsync' may depend on kernel version. /// Source: linux-fsdevel mailing-list https://marc.info/?l=linux-fsdevel&m=152535409207496 class FileSyncGuard { public: - /// NOTE: If you have already opened descriptor, it's preffered to use - /// this constructor instead of construnctor with path. + /// NOTE: If you have already opened descriptor, it's preferred to use + /// this constructor instead of constructor with path. FileSyncGuard(const DiskPtr & disk_, int fd_) : disk(disk_), fd(fd_) {} FileSyncGuard(const DiskPtr & disk_, const String & path) diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index 21116e9d432..e527e97d608 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -234,13 +234,13 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ std::is_same_v ? CurrentMetrics::GlobalThreadActive : CurrentMetrics::LocalThreadActive); job(); - /// job should be reseted before decrementing scheduled_jobs to + /// job should be reset before decrementing scheduled_jobs to /// ensure that the Job destroyed before wait() returns. job = {}; } catch (...) { - /// job should be reseted before decrementing scheduled_jobs to + /// job should be reset before decrementing scheduled_jobs to /// ensure that the Job destroyed before wait() returns. job = {}; diff --git a/src/Common/TraceCollector.cpp b/src/Common/TraceCollector.cpp index fc5318b11fb..d10d5981d57 100644 --- a/src/Common/TraceCollector.cpp +++ b/src/Common/TraceCollector.cpp @@ -152,7 +152,7 @@ void TraceCollector::run() if (trace_log) { // time and time_in_microseconds are both being constructed from the same timespec so that the - // times will be equal upto the precision of a second. + // times will be equal up to the precision of a second. struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index abb8158781b..f5c57781eef 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1288,13 +1288,13 @@ void ZooKeeper::receiveEvent() response->removeRootPath(root_path); } - /// Instead of setting the watch in sendEvent, set it in receiveEvent becuase need to check the response. + /// Instead of setting the watch in sendEvent, set it in receiveEvent because need to check the response. /// The watch shouldn't be set if the node does not exist and it will never exist like sequential ephemeral nodes. /// By using getData() instead of exists(), a watch won't be set if the node doesn't exist. if (request_info.watch) { bool add_watch = false; - /// 3 indicates the ZooKeeperExistsRequest. + /// 3 indicates the ZooKeeperExistsRequest. // For exists, we set the watch on both node exist and nonexist case. // For other case like getData, we only set the watch when node exists. if (request_info.request->getOpNum() == 3) diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index fe6a5b2dacd..46d7d7dfcc4 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -87,7 +87,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr else throw Exception("Unexpected AST element for compression codec", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - /// Default codec replaced with current default codec which may dependend on different + /// Default codec replaced with current default codec which may depend on different /// settings (and properties of data) in runtime. CompressionCodecPtr result_codec; if (codec_family_name == DEFAULT_CODEC_NAME) diff --git a/src/Compression/ICompressionCodec.cpp b/src/Compression/ICompressionCodec.cpp index baf6e9b2b86..3746753df8b 100644 --- a/src/Compression/ICompressionCodec.cpp +++ b/src/Compression/ICompressionCodec.cpp @@ -26,7 +26,7 @@ void ICompressionCodec::setCodecDescription(const String & codec_name, const AST std::shared_ptr result = std::make_shared(); result->name = "CODEC"; - /// Special case for codec Multiple, which doens't have name. It's just list + /// Special case for codec Multiple, which doesn't have name. It's just list /// of other codecs. if (codec_name.empty()) { diff --git a/src/Core/Defines.h b/src/Core/Defines.h index ba3d37242fa..aee8d6ca93e 100644 --- a/src/Core/Defines.h +++ b/src/Core/Defines.h @@ -67,7 +67,7 @@ /// Minimum revision supporting SettingsBinaryFormat::STRINGS. #define DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS 54429 -/// Mininum revision supporting interserver secret. +/// Minimum revision supporting interserver secret. #define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET 54441 /// Version of ClickHouse TCP protocol. Increment it manually when you change the protocol. diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 1179c0eb46b..c09c4b3b034 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -705,7 +705,7 @@ namespace MySQLReplication break; } default: - throw ReplicationError("Position update with unsupport event", ErrorCodes::LOGICAL_ERROR); + throw ReplicationError("Position update with unsupported event", ErrorCodes::LOGICAL_ERROR); } } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b94883ca871..f6a24641bc5 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -389,7 +389,7 @@ class IColumn; M(Bool, alter_partition_verbose_result, false, "Output information about affected parts. Currently works only for FREEZE and ATTACH commands.", 0) \ M(Bool, allow_experimental_database_materialize_mysql, false, "Allow to create database with Engine=MaterializeMySQL(...).", 0) \ M(Bool, system_events_show_zero_values, false, "Include all metrics, even with zero values", 0) \ - M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precison are seen as String on ClickHouse's side.", 0) \ + M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precision are seen as String on ClickHouse's side.", 0) \ \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ diff --git a/src/DataTypes/NumberTraits.h b/src/DataTypes/NumberTraits.h index 603449150db..77bd2101f05 100644 --- a/src/DataTypes/NumberTraits.h +++ b/src/DataTypes/NumberTraits.h @@ -29,7 +29,7 @@ constexpr size_t min(size_t x, size_t y) } /// @note There's no auto scale to larger big integer, only for integral ones. -/// It's cause of (U)Int64 backward compatibilty and very big performance penalties. +/// It's cause of (U)Int64 backward compatibility and very big performance penalties. constexpr size_t nextSize(size_t size) { if (size < 8) diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index a9dbae8ec92..3784ae0961b 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -116,7 +116,7 @@ void DatabaseAtomic::dropTable(const Context &, const String & table_name, bool } tryRemoveSymlink(table_name); /// Remove the inner table (if any) to avoid deadlock - /// (due to attemp to execute DROP from the worker thread) + /// (due to attempt to execute DROP from the worker thread) if (auto * mv = dynamic_cast(table.get())) mv->dropInnerTable(no_delay); /// Notify DatabaseCatalog that table was dropped. It will remove table data in background. diff --git a/src/Databases/MySQL/ConnectionMySQLSettings.h b/src/Databases/MySQL/ConnectionMySQLSettings.h index 90279f846a4..ce2773307c5 100644 --- a/src/Databases/MySQL/ConnectionMySQLSettings.h +++ b/src/Databases/MySQL/ConnectionMySQLSettings.h @@ -11,7 +11,7 @@ class Context; class ASTStorage; #define LIST_OF_CONNECTION_MYSQL_SETTINGS(M) \ - M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precison are seen as String on ClickHouse's side.", 0) \ + M(MySQLDataTypesSupport, mysql_datatypes_support_level, 0, "Which MySQL types should be converted to corresponding ClickHouse types (rather than being represented as String). Can be empty or any combination of 'decimal' or 'datetime64'. When empty MySQL's DECIMAL and DATETIME/TIMESTAMP with non-zero precision are seen as String on ClickHouse's side.", 0) \ /// Settings that should not change after the creation of a database. #define APPLY_FOR_IMMUTABLE_CONNECTION_MYSQL_SETTINGS(M) \ diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 63df025d2b1..43ff42956cd 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -326,7 +326,7 @@ struct DecimalBinaryOperation } private: - /// there's implicit type convertion here + /// there's implicit type conversion here static NativeResultType apply(NativeResultType a, NativeResultType b) { if constexpr (can_overflow && check_overflow) diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index 10c4a27e509..68d8b41407d 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -577,7 +577,7 @@ private: auto input_value = input_column->getDataAt(r); if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) { - // empty plaintext results in empty ciphertext + tag, means there should be atleast tag_size bytes. + // empty plaintext results in empty ciphertext + tag, means there should be at least tag_size bytes. if (input_value.size < tag_size) throw Exception(fmt::format("Encrypted data is too short: only {} bytes, " "should contain at least {} bytes of a tag.", diff --git a/src/Functions/formatReadableTimeDelta.cpp b/src/Functions/formatReadableTimeDelta.cpp index e55829b1bad..2b574f672d3 100644 --- a/src/Functions/formatReadableTimeDelta.cpp +++ b/src/Functions/formatReadableTimeDelta.cpp @@ -131,7 +131,7 @@ public: for (size_t i = 0; i < input_rows_count; ++i) { - /// Virtual call is Ok (neglible comparing to the rest of calculations). + /// Virtual call is Ok (negligible comparing to the rest of calculations). Float64 value = arguments[0].column->getFloat64(i); bool is_negative = value < 0; diff --git a/src/Functions/isDecimalOverflow.cpp b/src/Functions/isDecimalOverflow.cpp index 11c413757c6..323c9951a96 100644 --- a/src/Functions/isDecimalOverflow.cpp +++ b/src/Functions/isDecimalOverflow.cpp @@ -22,7 +22,7 @@ namespace { /// Returns 1 if and Decimal value has more digits then it's Precision allow, 0 otherwise. -/// Precision could be set as second argument or omitted. If ommited function uses Decimal presicion of the first argument. +/// Precision could be set as second argument or omitted. If omitted function uses Decimal precision of the first argument. class FunctionIsDecimalOverflow : public IFunction { public: diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 2166c78ef7c..7f29cfc7e5c 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -166,7 +166,7 @@ void ThreadStatus::initPerformanceCounters() memory_tracker.setDescription("(for thread)"); // query_start_time_{microseconds, nanoseconds} are all constructed from the same time point - // to ensure that they are all equal upto the precision of a second. + // to ensure that they are all equal up to the precision of a second. const auto now = std::chrono::system_clock::now(); query_start_time_nanoseconds = time_in_nanoseconds(now); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 57c557c5658..622acf2db9f 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -209,7 +209,7 @@ static void onExceptionBeforeStart(const String & query_for_logging, Context & c // all callers to onExceptionBeforeStart method construct the timespec for event_time and // event_time_microseconds from the same time point. So, it can be assumed that both of these - // times are equal upto the precision of a second. + // times are equal up to the precision of a second. elem.event_time = current_time; elem.event_time_microseconds = current_time_microseconds; elem.query_start_time = current_time; @@ -267,7 +267,7 @@ static std::tuple executeQueryImpl( ReadBuffer * istr) { // current_time and current_time_microseconds are both constructed from the same time point - // to ensure that both the times are equal upto the precision of a second. + // to ensure that both the times are equal up to the precision of a second. const auto now = std::chrono::system_clock::now(); auto current_time = time_in_seconds(now); @@ -631,7 +631,7 @@ static std::tuple executeQueryImpl( elem.type = QueryLogElementType::EXCEPTION_WHILE_PROCESSING; // event_time and event_time_microseconds are being constructed from the same time point - // to ensure that both the times will be equal upto the precision of a second. + // to ensure that both the times will be equal up to the precision of a second. const auto time_now = std::chrono::system_clock::now(); elem.event_time = time_in_seconds(time_now); diff --git a/src/Parsers/obfuscateQueries.cpp b/src/Parsers/obfuscateQueries.cpp index 32382b70bd7..11f4a77ee0e 100644 --- a/src/Parsers/obfuscateQueries.cpp +++ b/src/Parsers/obfuscateQueries.cpp @@ -927,7 +927,7 @@ void obfuscateQueries( } else { - /// Everyting else is kept as is. + /// Everything else is kept as is. result.write(token.begin, token.size()); } } diff --git a/src/Processors/IAccumulatingTransform.h b/src/Processors/IAccumulatingTransform.h index 3e77c798ad7..b51753199c3 100644 --- a/src/Processors/IAccumulatingTransform.h +++ b/src/Processors/IAccumulatingTransform.h @@ -36,7 +36,7 @@ public: Status prepare() override; void work() override; - /// Adds additional port fo totals. + /// Adds additional port for totals. /// If added, totals will have been ready by the first generate() call (in totals chunk). InputPort * addTotalsPort(); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 319b486c2c6..ffc2dd62ce0 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -353,7 +353,7 @@ size_t IMergeTreeDataPart::getFileSizeOrZero(const String & file_name) const return checksum->second.file_size; } -String IMergeTreeDataPart::getColumnNameWithMinumumCompressedSize(const StorageMetadataPtr & metadata_snapshot) const +String IMergeTreeDataPart::getColumnNameWithMinimumCompressedSize(const StorageMetadataPtr & metadata_snapshot) const { const auto & storage_columns = metadata_snapshot->getColumns().getAllPhysical(); auto alter_conversions = storage.getAlterConversionsForPart(shared_from_this()); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 202d9494247..21932ba445c 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -145,7 +145,7 @@ public: /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). /// If no checksums are present returns the name of the first physically existing column. - String getColumnNameWithMinumumCompressedSize(const StorageMetadataPtr & metadata_snapshot) const; + String getColumnNameWithMinimumCompressedSize(const StorageMetadataPtr & metadata_snapshot) const; bool contains(const IMergeTreeDataPart & other) const { return info.contains(other.info); } diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 739dfedfde4..ad10a437b1e 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -95,7 +95,7 @@ NameSet injectRequiredColumns(const MergeTreeData & storage, const StorageMetada */ if (!have_at_least_one_physical_column) { - const auto minimum_size_column_name = part->getColumnNameWithMinumumCompressedSize(metadata_snapshot); + const auto minimum_size_column_name = part->getColumnNameWithMinimumCompressedSize(metadata_snapshot); columns.push_back(minimum_size_column_name); /// correctly report added column injected_columns.insert(columns.back()); diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index b29966751f9..141dd8002b9 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -298,7 +298,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge( if (metadata_snapshot->hasAnyTTL() && merge_with_ttl_allowed && !ttl_merges_blocker.isCancelled()) { - /// TTL delete is prefered to recompression + /// TTL delete is preferred to recompression TTLDeleteMergeSelector delete_ttl_selector( next_delete_ttl_merge_times_by_partition, current_time, diff --git a/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h b/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h index f0837f98486..17239e2618a 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h +++ b/src/Storages/MergeTree/MergeTreeDataPartTTLInfo.h @@ -49,7 +49,7 @@ struct MergeTreeDataPartTTLInfos TTLInfoMap recompression_ttl; - /// Return smalles max recompression TTL value + /// Return the smallest max recompression TTL value time_t getMinimalMaxRecompressionTTL() const; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 8c1dc845d26..44d0788901f 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1517,7 +1517,7 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( { /// In case when SELECT's predicate defines a single continuous interval of keys, /// we can use binary search algorithm to find the left and right endpoint key marks of such interval. - /// The returned value is the minumum range of marks, containing all keys for which KeyCondition holds + /// The returned value is the minimum range of marks, containing all keys for which KeyCondition holds LOG_TRACE(log, "Running binary search on index range for part {} ({} marks)", part->name, marks_count); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h index 986253a2206..824ed73c171 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h @@ -36,7 +36,7 @@ private: Poco::Logger * log; std::atomic need_stop {false}; - // We need it besides `storage.is_readonly`, bacause `shutdown()` may be called many times, that way `storage.is_readonly` will not change. + // We need it besides `storage.is_readonly`, because `shutdown()` may be called many times, that way `storage.is_readonly` will not change. bool incr_readonly = false; /// The random data we wrote into `/replicas/me/is_active`. diff --git a/src/Storages/MergeTree/TTLMergeSelector.h b/src/Storages/MergeTree/TTLMergeSelector.h index c294687cdc5..710b85b3474 100644 --- a/src/Storages/MergeTree/TTLMergeSelector.h +++ b/src/Storages/MergeTree/TTLMergeSelector.h @@ -84,7 +84,7 @@ public: time_t getTTLForPart(const IMergeSelector::Part & part) const override; /// Checks that part's codec is not already equal to required codec - /// according to recompression TTL. It doesn't make sence to assign such + /// according to recompression TTL. It doesn't make sense to assign such /// merge. bool isTTLAlreadySatisfied(const IMergeSelector::Part & part) const override; private: diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp index 43a9d75d084..685a55027ce 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp @@ -155,7 +155,7 @@ void ReadBufferFromRabbitMQConsumer::subscribe() .onError([&](const char * message) { /* End up here either if channel ends up in an error state (then there will be resubscription) or consume call error, which - * arises from queue settings mismatch or queue level error, which should not happen as noone else is supposed to touch them + * arises from queue settings mismatch or queue level error, which should not happen as no one else is supposed to touch them */ LOG_ERROR(log, "Consumer failed on channel {}. Reason: {}", channel_id, message); wait_subscription.store(false); @@ -173,16 +173,16 @@ bool ReadBufferFromRabbitMQConsumer::ackMessages() */ if (record_info.channel_id == channel_id && record_info.delivery_tag && record_info.delivery_tag > prev_tag) { - /// Commit all received messages with delivery tags from last commited to last inserted + /// Commit all received messages with delivery tags from last committed to last inserted if (!consumer_channel->ack(record_info.delivery_tag, AMQP::multiple)) { - LOG_ERROR(log, "Failed to commit messages with delivery tags from last commited to {} on channel {}", + LOG_ERROR(log, "Failed to commit messages with delivery tags from last committed to {} on channel {}", record_info.delivery_tag, channel_id); return false; } prev_tag = record_info.delivery_tag; - LOG_TRACE(log, "Consumer commited messages with deliveryTags up to {} on channel {}", record_info.delivery_tag, channel_id); + LOG_TRACE(log, "Consumer committed messages with deliveryTags up to {} on channel {}", record_info.delivery_tag, channel_id); } return true; diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 6565afeb32d..d213251e366 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -406,7 +406,7 @@ bool StorageRabbitMQ::restoreConnection(bool reconnecting) connection->close(); /// Connection might be unusable, but not closed /* Connection is not closed immediately (firstly, all pending operations are completed, and then - * an AMQP closing-handshake is performed). But cannot open a new connection untill previous one is properly closed + * an AMQP closing-handshake is performed). But cannot open a new connection until previous one is properly closed */ while (!connection->closed() && ++cnt_retries != RETRIES_MAX) event_handler->iterateLoop(); @@ -731,7 +731,7 @@ bool StorageRabbitMQ::streamToViews() auto column_names = block_io.out->getHeader().getNames(); auto sample_block = metadata_snapshot->getSampleBlockForColumns(column_names, getVirtuals(), getStorageID()); - /* event_handler->connectionRunning() does not guarantee that connnection is not closed in case loop was not running before, but + /* event_handler->connectionRunning() does not guarantee that connection is not closed in case loop was not running before, but * need to anyway start the loop to activate error callbacks and update connection state, because even checking with * connection->usable() will not give correct answer before callbacks are activated. */ diff --git a/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.h b/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.h index a8e94070dbd..28fa5df8111 100644 --- a/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.h +++ b/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.h @@ -76,20 +76,20 @@ private: String channel_id; /* payloads.queue: - * - payloads are pushed to queue in countRow and poped by another thread in writingFunc, each payload gets into queue only once + * - payloads are pushed to queue in countRow and popped by another thread in writingFunc, each payload gets into queue only once * returned.queue: * - payloads are pushed to queue: * 1) inside channel->onError() callback if channel becomes unusable and the record of pending acknowledgements from server * is non-empty. * 2) inside removeRecord() if received nack() - negative acknowledgement from the server that message failed to be written * to disk or it was unable to reach the queue. - * - payloads are poped from the queue once republished + * - payloads are popped from the queue once republished */ ConcurrentBoundedQueue> payloads, returned; /* Counter of current delivery on a current channel. Delivery tags are scoped per channel. The server attaches a delivery tag for each * published message - a serial number of delivery on current channel. Delivery tag is a way of server to notify publisher if it was - * able or unable to process delivery, i.e. it sends back a responce with a corresponding delivery tag. + * able or unable to process delivery, i.e. it sends back a response with a corresponding delivery tag. */ UInt64 delivery_tag = 0; @@ -100,7 +100,7 @@ private: */ bool wait_all = true; - /* false: untill writeSuffix is called + /* false: until writeSuffix is called * true: means payloads.queue will not grow anymore */ std::atomic wait_num = 0; diff --git a/tests/integration/test_disk_types/test.py b/tests/integration/test_disk_types/test.py index c748653bc82..ad09519a484 100644 --- a/tests/integration/test_disk_types/test.py +++ b/tests/integration/test_disk_types/test.py @@ -21,8 +21,8 @@ def cluster(): def test_different_types(cluster): node = cluster.instances["node"] - responce = node.query("SELECT * FROM system.disks") - disks = responce.split("\n") + response = node.query("SELECT * FROM system.disks") + disks = response.split("\n") for disk in disks: if disk == '': # skip empty line (after split at last position) continue diff --git a/tests/integration/test_distributed_load_balancing/test.py b/tests/integration/test_distributed_load_balancing/test.py index e7b86a210bd..0758dc38ba7 100644 --- a/tests/integration/test_distributed_load_balancing/test.py +++ b/tests/integration/test_distributed_load_balancing/test.py @@ -26,7 +26,7 @@ def bootstrap(): # just after server starts (+ 2 seconds, reload timeout). # # And on configuration reload the clusters will be re-created, so some - # internal stuff will be reseted: + # internal stuff will be reset: # - error_count # - last_used (round_robing) # diff --git a/tests/queries/0_stateless/00932_geohash_support.sql b/tests/queries/0_stateless/00932_geohash_support.sql index a477332b532..aeed72176b9 100644 --- a/tests/queries/0_stateless/00932_geohash_support.sql +++ b/tests/queries/0_stateless/00932_geohash_support.sql @@ -45,7 +45,7 @@ select 12 as p, geohashEncode(longitude, latitude, p) as actual, if(actual = enc -- Here results are floats, and hence may not be compared for equality directly. -- We select all values that are off by some reasonable value: --- each byte of encoded string provides 5 bits of precison, (roughly 2.5 for lon and lat) +-- each byte of encoded string provides 5 bits of precision, (roughly 2.5 for lon and lat) -- each bit of precision divides value range by 2. -- hence max error is roughly value range 2.5 times divided by 2 for each precision bit. -- initial value range is [-90..90] for latitude and [-180..180] for longitude. diff --git a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh index b14defd672a..5333d0b4b0b 100755 --- a/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh +++ b/tests/queries/0_stateless/01281_group_by_limit_memory_tracking.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Regression for MemoryTracker that had been incorrectly accounted -# (it was reseted before deallocation) +# (it was reset before deallocation) # # For this will be used: # - two-level group by diff --git a/utils/db-generator/README.md b/utils/db-generator/README.md index 1d276063bf9..f49c7911cb6 100644 --- a/utils/db-generator/README.md +++ b/utils/db-generator/README.md @@ -6,11 +6,11 @@ **Анализ схемы** По запросу необходимо определить возможные таблицы. Имея строку запроса можно понять, какие его части обозначают названия таблиц, таким образом можно определить их количество в нашей базе данных. -В парсере Clickhouse поддеревом запроса, отвечающее за таблицы из которых мы берем данные, является TABLES (Рисунок 1), в нем лежит основная таблица, из которой берутся колонки, а также операции JOIN, которые совершаются в запросе. Обходя все вершины в поддереве мы берем названия таблиц и баз данных в которых они лежат, а также их алиас, то есть укороченные названия, выбранные автором запроса. Эти названия могут понадобиться нам для определения принадлежности колонки в дальнейшем. +В парсере Clickhouse поддеревом запроса, отвечающее за таблицы из которых мы берем данные, является TABLES (Рисунок 1), в нем лежит основная таблица, из которой берутся колонки, а также операции JOIN, которые совершаются в запросе. Обходя все вершины в поддереве мы берем названия таблиц и баз данных в которых они лежат, а также их алиас, то есть укороченные названия, выбранные автором запроса. Эти названия могут понадобиться нам для определения принадлежности колонки в дальнейшем. Таким образом для запроса мы получаем набор баз данных, а также таблиц и их условных обозначений (алиасов), по которым делается запрос. Затем нам необходимо определить множество столбцов, которые присутствуют в запросе и таблицы, к которым они могут относиться. Во время исполнения запроса уже известно множество столбцов в каждой таблице, поэтому при исполнении программа автоматически связывает столбец и таблицу, однако в нашем случае нельзя однозначно трактовать принадлежность столбца к определенной таблице, например в следующем запросе: “SELECT column1, column2, column3 FROM table1 JOIN table2 on table1.column2 = table2.column3 ”. Здесь мы однозначно можем сказать, к какой таблице относятся колонки column2 и column3, однако column1 может принадлежать как первой, так и второй таблице. Для однозначности трактовки таких случаев, мы будем относить данную неопределенные колонки к основной таблице, по которой делается запрос, например в данном случае это будет таблица table1. -Все столбцы в дереве лежат в вершинах типа INDENTIFIER, которые находятся в поддеревьях SELECT, TABLES, WHERE, GROUP_BY, HAVING, ORDER_BY. Рекурсивно обходя поддеревья мы формируем множество всех таблиц, затем мы разделяем колонку на составляющие: таблица (если она явно указана через точку) и само название, затем, так как таблица может являться алиасом, мы заменяем алиас на оригинальное название таблицы. Теперь у нас есть список всех столбцов и таблиц, к которым они относятся, для столбцов без таблиц определяем основную таблицу запроса. +Все столбцы в дереве лежат в вершинах типа IDENTIFIER, которые находятся в поддеревьях SELECT, TABLES, WHERE, GROUP_BY, HAVING, ORDER_BY. Рекурсивно обходя поддеревья мы формируем множество всех таблиц, затем мы разделяем колонку на составляющие: таблица (если она явно указана через точку) и само название, затем, так как таблица может являться алиасом, мы заменяем алиас на оригинальное название таблицы. Теперь у нас есть список всех столбцов и таблиц, к которым они относятся, для столбцов без таблиц определяем основную таблицу запроса. **Анализ столбцов** @@ -20,7 +20,7 @@ Определить значения столбцов мы можем используя логический, арифметические и другие функции над значениями столбцов, которые указаны в запросе. Такие функции лежат в поддеревьях SELECT и WHERE. Параметром функции может быть константа, колонка либо другая функция (Рисунок 2). Таким образом для понимания типа колонки могут помочь следующие параметры: 1) Типы аргументов, которые может принимать функция, например функция TOSTARTOFMINUTE(округляет время до кратного 5 минутам вниз) может принимать только DATETIME, таким образом если аргументом данной функции является колонка, то данная колонка имеет тип DATETIME. 2) типы остальных аргументов в данной функции, например функция EQUALS(равенство), она подразумевает собой равенство типов ее аргументов, таким образом если в данной функции присутствует константа и столбец, то мы можем определить тип столбца как тип константы. Таким образом, для каждой функции мы определяем возможные типы аргументов, тип возвращаемого значения, а также параметр, являются ли аргументы функции одинакового типа. Рекурсивный обработчик функций будет определять возможные типы столбцов использующихся в данных функциях по значениям аргументов и возвращать возможные типы результата выполнения функции. -Теперь для каждого столбца мы имеем множество возможных типов его значений. Для однозначной трактовки запроса мы выберем один конкретный тип из этого множества. +Теперь для каждого столбца мы имеем множество возможных типов его значений. Для однозначной трактовки запроса мы выберем один конкретный тип из этого множества. **Определение значений столбцов** From 331545fe0df758be09b433d595fc3973141412b3 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Tue, 27 Oct 2020 14:29:10 +0300 Subject: [PATCH 233/325] Fix typo --- src/Storages/System/StorageSystemReplicas.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index 973ccfbf464..ab54d760873 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -75,7 +75,7 @@ Pipe StorageSystemReplicas::read( for (const auto & db : DatabaseCatalog::instance().getDatabases()) { /// Check if database can contain replicated tables - if (db.second->canContainMergeTreeTables()) + if (!db.second->canContainMergeTreeTables()) continue; const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) From a48bc837637ef88a2063bc1f1324f277a4b8b684 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 27 Oct 2020 12:04:07 +0000 Subject: [PATCH 234/325] Fix style --- src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp index 5644b8d7c6a..8c7b435cd2a 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp @@ -14,11 +14,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - ReadBufferFromRabbitMQConsumer::ReadBufferFromRabbitMQConsumer( ChannelPtr consumer_channel_, HandlerPtr event_handler_, From dca35b191377efa34830b3e6f72d2b95b92996d4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 27 Oct 2020 15:24:10 +0300 Subject: [PATCH 235/325] Add awesome test for fetch --- docker/test/integration/base/Dockerfile | 3 +- src/IO/ReadWriteBufferFromHTTP.h | 5 +- src/Storages/MergeTree/DataPartsExchange.cpp | 10 +-- .../MergeTree/ReplicatedFetchesList.cpp | 29 +++--- .../MergeTree/ReplicatedFetchesList.h | 23 +++-- src/Storages/System/StorageSystemFetches.cpp | 12 ++- src/Storages/System/StorageSystemFetches.h | 4 +- tests/integration/helpers/cluster.py | 1 + tests/integration/helpers/network.py | 12 +++ .../test_system_fetches/__init__.py | 1 + tests/integration/test_system_fetches/test.py | 90 +++++++++++++++++++ 11 files changed, 144 insertions(+), 46 deletions(-) create mode 100644 tests/integration/test_system_fetches/__init__.py create mode 100644 tests/integration/test_system_fetches/test.py diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index b6a46f6d934..99095de60fb 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -17,7 +17,8 @@ RUN apt-get update \ sqlite3 \ curl \ tar \ - krb5-user + krb5-user \ + iproute2 RUN rm -rf \ /var/lib/apt/lists/* \ /var/cache/debconf \ diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index c79a3bd953d..10cacf10dd9 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -205,12 +205,12 @@ namespace detail bool nextImpl() override { + if (read_callback) + read_callback(count()); if (!impl->next()) return false; internal_buffer = impl->buffer(); working_buffer = internal_buffer; - if (read_callback) - read_callback(count()); return true; } @@ -225,6 +225,7 @@ namespace detail void setNextReadCallback(std::function read_callback_) { read_callback = read_callback_; + read_callback(count()); } }; } diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index b1f1dc3f5f5..c4a7816581a 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -53,7 +53,6 @@ std::string getEndpointId(const std::string & node_id) return "DataPartsExchange:" + node_id; } - struct ReplicatedFetchReadCallback { ReplicatedFetchList::Entry & replicated_fetch_entry; @@ -65,9 +64,9 @@ struct ReplicatedFetchReadCallback void operator() (size_t bytes_count) { - replicated_fetch_entry->bytes_read_compressed = bytes_count; + replicated_fetch_entry->bytes_read_compressed.store(bytes_count, std::memory_order_relaxed); replicated_fetch_entry->progress.store( - replicated_fetch_entry->bytes_read_compressed.load(std::memory_order_relaxed) / replicated_fetch_entry->total_size_bytes_compressed, + static_cast(bytes_count) / replicated_fetch_entry->total_size_bytes_compressed, std::memory_order_relaxed); } }; @@ -307,10 +306,11 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( reservation = data.makeEmptyReservationOnLargestDisk(); } auto storage_id = data.getStorageID(); + String new_part_path = data.getFullPathOnDisk(reservation->getDisk()) + part_name + "/"; auto entry = data.global_context.getReplicatedFetchList().insert( storage_id.getDatabaseName(), storage_id.getTableName(), - part_info.partition_id, part_name, part_name, - replica_path, uri.toString(), interserver_scheme, to_detached, sum_files_size); + part_info.partition_id, part_name, new_part_path, + replica_path, uri, to_detached, sum_files_size); in.setNextReadCallback(ReplicatedFetchReadCallback(*entry)); diff --git a/src/Storages/MergeTree/ReplicatedFetchesList.cpp b/src/Storages/MergeTree/ReplicatedFetchesList.cpp index d66d0965351..92e60d9dde1 100644 --- a/src/Storages/MergeTree/ReplicatedFetchesList.cpp +++ b/src/Storages/MergeTree/ReplicatedFetchesList.cpp @@ -9,25 +9,21 @@ ReplicatedFetchListElement::ReplicatedFetchListElement( const std::string & database_, const std::string & table_, const std::string & partition_id_, const std::string & result_part_name_, const std::string & result_part_path_, const std::string & source_replica_path_, - const std::string & source_replica_address_, const std::string & interserver_scheme_, - UInt8 to_detached_, UInt64 total_size_bytes_compressed_) + const Poco::URI & uri_, UInt8 to_detached_, UInt64 total_size_bytes_compressed_) : database(database_) , table(table_) , partition_id(partition_id_) , result_part_name(result_part_name_) , result_part_path(result_part_path_) , source_replica_path(source_replica_path_) - , source_replica_address(source_replica_address_) - , interserver_scheme(interserver_scheme_) + , source_replica_hostname(uri_.getHost()) + , source_replica_port(uri_.getPort()) + , interserver_scheme(uri_.getScheme()) + , uri(uri_.toString()) , to_detached(to_detached_) , total_size_bytes_compressed(total_size_bytes_compressed_) + , thread_id(getThreadId()) { - background_thread_memory_tracker = CurrentThread::getMemoryTracker(); - if (background_thread_memory_tracker) - { - background_thread_memory_tracker_prev_parent = background_thread_memory_tracker->getParent(); - background_thread_memory_tracker->setParent(&memory_tracker); - } } @@ -40,23 +36,18 @@ ReplicatedFetchInfo ReplicatedFetchListElement::getInfo() const res.result_part_name = result_part_name; res.result_part_path = result_part_path; res.source_replica_path = source_replica_path; - res.source_replica_address = source_replica_address; + res.source_replica_hostname = source_replica_hostname; + res.source_replica_port = source_replica_port; + res.interserver_scheme = interserver_scheme; + res.uri = uri; res.interserver_scheme = interserver_scheme; res.to_detached = to_detached; res.elapsed = watch.elapsedSeconds(); res.progress = progress.load(std::memory_order_relaxed); res.bytes_read_compressed = bytes_read_compressed.load(std::memory_order_relaxed); res.total_size_bytes_compressed = total_size_bytes_compressed; - res.memory_usage = memory_tracker.get(); res.thread_id = thread_id; return res; } -ReplicatedFetchListElement::~ReplicatedFetchListElement() -{ - /// Unplug memory_tracker from current background processing pool thread - if (background_thread_memory_tracker) - background_thread_memory_tracker->setParent(background_thread_memory_tracker_prev_parent); -} - } diff --git a/src/Storages/MergeTree/ReplicatedFetchesList.h b/src/Storages/MergeTree/ReplicatedFetchesList.h index ad8edb6ad59..670e1fb984a 100644 --- a/src/Storages/MergeTree/ReplicatedFetchesList.h +++ b/src/Storages/MergeTree/ReplicatedFetchesList.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace CurrentMetrics { @@ -23,8 +24,10 @@ struct ReplicatedFetchInfo std::string result_part_path; std::string source_replica_path; - std::string source_replica_address; + std::string source_replica_hostname; + UInt16 source_replica_port; std::string interserver_scheme; + std::string uri; UInt8 to_detached; @@ -34,7 +37,6 @@ struct ReplicatedFetchInfo UInt64 total_size_bytes_compressed; UInt64 bytes_read_compressed; - UInt64 memory_usage; UInt64 thread_id; }; @@ -48,9 +50,11 @@ struct ReplicatedFetchListElement : private boost::noncopyable const std::string result_part_name; const std::string result_part_path; - const std::string source_replica_path; - const std::string source_replica_address; - const std::string interserver_scheme; + std::string source_replica_path; + std::string source_replica_hostname; + UInt16 source_replica_port; + std::string interserver_scheme; + std::string uri; const UInt8 to_detached; @@ -60,22 +64,15 @@ struct ReplicatedFetchListElement : private boost::noncopyable std::atomic bytes_read_compressed{}; UInt64 total_size_bytes_compressed{}; - MemoryTracker memory_tracker{VariableContext::Process}; - MemoryTracker * background_thread_memory_tracker; - MemoryTracker * background_thread_memory_tracker_prev_parent = nullptr; - UInt64 thread_id; ReplicatedFetchListElement( const std::string & database_, const std::string & table_, const std::string & partition_id_, const std::string & result_part_name_, const std::string & result_part_path_, const std::string & source_replica_path_, - const std::string & source_replica_address_, const std::string & interserver_scheme_, - UInt8 to_detached_, UInt64 total_size_bytes_compressed_); + const Poco::URI & uri, UInt8 to_detached_, UInt64 total_size_bytes_compressed_); ReplicatedFetchInfo getInfo() const; - - ~ReplicatedFetchListElement(); }; diff --git a/src/Storages/System/StorageSystemFetches.cpp b/src/Storages/System/StorageSystemFetches.cpp index 27d4eeddbfc..0c992457c6c 100644 --- a/src/Storages/System/StorageSystemFetches.cpp +++ b/src/Storages/System/StorageSystemFetches.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include #include #include @@ -19,10 +21,11 @@ NamesAndTypesList StorageSystemFetches::getNamesAndTypes() {"total_size_bytes_compressed", std::make_shared()}, {"bytes_read_compressed", std::make_shared()}, {"source_replica_path", std::make_shared()}, - {"source_replica_address", std::make_shared()}, + {"source_replica_hostname", std::make_shared()}, + {"source_replica_port", std::make_shared()}, {"interserver_scheme", std::make_shared()}, + {"URI", std::make_shared()}, {"to_detached", std::make_shared()}, - {"memory_usage", std::make_shared()}, {"thread_id", std::make_shared()}, }; } @@ -48,10 +51,11 @@ void StorageSystemFetches::fillData(MutableColumns & res_columns, const Context res_columns[i++]->insert(fetch.total_size_bytes_compressed); res_columns[i++]->insert(fetch.bytes_read_compressed); res_columns[i++]->insert(fetch.source_replica_path); - res_columns[i++]->insert(fetch.source_replica_address); + res_columns[i++]->insert(fetch.source_replica_hostname); + res_columns[i++]->insert(fetch.source_replica_port); res_columns[i++]->insert(fetch.interserver_scheme); + res_columns[i++]->insert(fetch.uri); res_columns[i++]->insert(fetch.to_detached); - res_columns[i++]->insert(fetch.memory_usage); res_columns[i++]->insert(fetch.thread_id); } } diff --git a/src/Storages/System/StorageSystemFetches.h b/src/Storages/System/StorageSystemFetches.h index be1b66193bf..7c93bce7ee7 100644 --- a/src/Storages/System/StorageSystemFetches.h +++ b/src/Storages/System/StorageSystemFetches.h @@ -1,7 +1,6 @@ #pragma once -#include -#include + #include #include @@ -11,6 +10,7 @@ namespace DB class Context; +/// system.fetches table. Takes data from context.getReplicatedFetchesList() class StorageSystemFetches final : public ext::shared_ptr_helper, public IStorageSystemOneBlock { friend struct ext::shared_ptr_helper; diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 6b24bc30460..7c44065320b 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -814,6 +814,7 @@ services: tmpfs: {tmpfs} cap_add: - SYS_PTRACE + - NET_ADMIN depends_on: {depends_on} user: '{user}' env_file: diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index 194903efff4..add812bea58 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -19,6 +19,7 @@ class PartitionManager: def __init__(self): self._iptables_rules = [] + self._netem_delayed_instances = [] _NetworkManager.get() def drop_instance_zk_connections(self, instance, action='DROP'): @@ -46,11 +47,18 @@ class PartitionManager: self._add_rule(create_rule(left, right)) self._add_rule(create_rule(right, left)) + def add_network_delay(self, instance, delay_ms): + self._add_tc_netem_delay(instance, delay_ms) + def heal_all(self): while self._iptables_rules: rule = self._iptables_rules.pop() _NetworkManager.get().delete_iptables_rule(**rule) + while self._netem_delayed_instances: + instance = self._netem_delayed_instances.pop() + instance.exec_in_container(["bash", "-c", "tc qdisc del dev eth0 root netem"], user="root") + def pop_rules(self): res = self._iptables_rules[:] self.heal_all() @@ -73,6 +81,10 @@ class PartitionManager: _NetworkManager.get().delete_iptables_rule(**rule) self._iptables_rules.remove(rule) + def _add_tc_netem_delay(self, instance, delay_ms): + instance.exec_in_container(["bash", "-c", "tc qdisc add dev eth0 root netem delay {}ms".format(delay_ms)], user="root") + self._netem_delayed_instances.append(instance) + def __enter__(self): return self diff --git a/tests/integration/test_system_fetches/__init__.py b/tests/integration/test_system_fetches/__init__.py new file mode 100644 index 00000000000..e5a0d9b4834 --- /dev/null +++ b/tests/integration/test_system_fetches/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tests/integration/test_system_fetches/test.py b/tests/integration/test_system_fetches/test.py new file mode 100644 index 00000000000..2746ced300c --- /dev/null +++ b/tests/integration/test_system_fetches/test.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 + + +import pytest +import time +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager +from helpers.test_tools import assert_eq_with_retry +import random +import string +import json + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', with_zookeeper=True) +node2 = cluster.add_instance('node2', with_zookeeper=True) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + yield cluster + + finally: + cluster.shutdown() + +def get_random_string(length): + return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) + +def test_system_fetches(started_cluster): + node1.query("CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '1') ORDER BY tuple()") + node2.query("CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '2') ORDER BY tuple()") + + with PartitionManager() as pm: + node2.query("SYSTEM STOP FETCHES t") + node1.query("INSERT INTO t SELECT number, '{}' FROM numbers(10000)".format(get_random_string(104857))) + pm.add_network_delay(node1, 80) + node2.query("SYSTEM START FETCHES t") + fetches_result = [] + for _ in range(1000): + result = json.loads(node2.query("SELECT * FROM system.fetches FORMAT JSON")) + if not result["data"]: + if fetches_result: + break + time.sleep(0.1) + else: + fetches_result.append(result["data"][0]) + print(fetches_result[-1]) + time.sleep(0.1) + + node2.query("SYSTEM SYNC REPLICA t", timeout=10) + assert node2.query("SELECT COUNT() FROM t") == "10000\n" + + for elem in fetches_result: + elem['bytes_read_compressed'] = float(elem['bytes_read_compressed']) + elem['total_size_bytes_compressed'] = float(elem['total_size_bytes_compressed']) + elem['progress'] = float(elem['progress']) + elem['elapsed'] = float(elem['elapsed']) + + assert len(fetches_result) > 0 + first_non_empty = fetches_result[0] + + assert first_non_empty['database'] == "default" + assert first_non_empty['table'] == "t" + assert first_non_empty['source_replica_hostname'] == 'node1' + assert first_non_empty['source_replica_port'] == 9009 + assert first_non_empty['source_replica_path'] == '/clickhouse/test/t/replicas/1' + assert first_non_empty['interserver_scheme'] == 'http' + assert first_non_empty['partition_id'] == 'all' + assert first_non_empty['URI'].startswith('http://node1:9009/?endpoint=DataPartsExchange') + + for elem in fetches_result: + assert elem['bytes_read_compressed'] <= elem['total_size_bytes_compressed'], "Bytes read ({}) more than total bytes ({}). It's a bug".format(elem['bytes_read_compressed'], elem['total_size_bytes_compressed']) + assert 0.0 <= elem['progress'] <= 1.0, "Progress shouldn't less than 0 and bigger than 1, got {}".format(elem['progress']) + assert 0.0 <= elem['elapsed'], "Elapsed time must be greater than 0, got {}".format(elem['elapsed']) + + prev_progress = first_non_empty['progress'] + for elem in fetches_result: + assert elem['progress'] >= prev_progress, "Progress decreasing prev{}, next {}? It's a bug".format(prev_progress, elem['progress']) + prev_progress = elem['progress'] + + prev_bytes = first_non_empty['bytes_read_compressed'] + for elem in fetches_result: + assert elem['bytes_read_compressed'] >= prev_bytes, "Bytes read decreasing prev {}, next {}? It's a bug".format(prev_bytes, elem['bytes_read_compressed']) + prev_bytes = elem['bytes_read_compressed'] + + prev_elapsed = first_non_empty['elapsed'] + for elem in fetches_result: + assert elem['elapsed'] >= prev_elapsed, "Elapsed time decreasing prev {}, next {}? It's a bug".format(prev_elapsed, elem['elapsed']) + prev_elapsed = elem['elapsed'] From 31d63efdac7d888de000b6a3b428181f2fc81d37 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 27 Oct 2020 15:26:10 +0300 Subject: [PATCH 236/325] Fix style --- src/Storages/MergeTree/BackgroundProcessList.h | 2 +- src/Storages/ya.make | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/BackgroundProcessList.h b/src/Storages/MergeTree/BackgroundProcessList.h index f90aa1c88a8..11516ad2ce7 100644 --- a/src/Storages/MergeTree/BackgroundProcessList.h +++ b/src/Storages/MergeTree/BackgroundProcessList.h @@ -85,5 +85,5 @@ public: virtual void onEntryDestroy(const Entry & /* entry */) {} virtual inline ~BackgroundProcessList() {} }; - + } diff --git a/src/Storages/ya.make b/src/Storages/ya.make index 107433b5e73..289a226ce7d 100644 --- a/src/Storages/ya.make +++ b/src/Storages/ya.make @@ -91,6 +91,7 @@ SRCS( MergeTree/MergeTreeWriteAheadLog.cpp MergeTree/MergeType.cpp MergeTree/registerStorageMergeTree.cpp + MergeTree/ReplicatedFetchesList.cpp MergeTree/ReplicatedMergeTreeAddress.cpp MergeTree/ReplicatedMergeTreeAltersSequence.cpp MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -155,6 +156,7 @@ SRCS( System/StorageSystemDistributionQueue.cpp System/StorageSystemEnabledRoles.cpp System/StorageSystemEvents.cpp + System/StorageSystemFetches.cpp System/StorageSystemFormats.cpp System/StorageSystemFunctions.cpp System/StorageSystemGrants.cpp From 3824fcb0762878cff9c927e63700912b63a850af Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 27 Oct 2020 15:47:42 +0300 Subject: [PATCH 237/325] Rename file and some tweaks --- src/Interpreters/Context.cpp | 2 +- .../MergeTree/BackgroundProcessList.h | 2 ++ src/Storages/MergeTree/DataPartsExchange.cpp | 19 ++++++++++--------- ...etchesList.cpp => ReplicatedFetchList.cpp} | 2 +- ...tedFetchesList.h => ReplicatedFetchList.h} | 0 src/Storages/System/StorageSystemFetches.cpp | 2 +- src/Storages/System/StorageSystemFetches.h | 2 +- src/Storages/ya.make | 2 +- tests/integration/test_system_fetches/test.py | 3 +++ 9 files changed, 20 insertions(+), 14 deletions(-) rename src/Storages/MergeTree/{ReplicatedFetchesList.cpp => ReplicatedFetchList.cpp} (97%) rename src/Storages/MergeTree/{ReplicatedFetchesList.h => ReplicatedFetchList.h} (100%) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 38e8ffdf76a..293ac4ab9e7 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Storages/MergeTree/BackgroundProcessList.h b/src/Storages/MergeTree/BackgroundProcessList.h index 11516ad2ce7..2f4cc020796 100644 --- a/src/Storages/MergeTree/BackgroundProcessList.h +++ b/src/Storages/MergeTree/BackgroundProcessList.h @@ -9,6 +9,8 @@ namespace DB { +/// Common code for background processes lists, like system.merges and system.fetches +/// Look at examples in MergeList and ReplicatedFetchList template class BackgroundProcessList; diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index c4a7816581a..bb92202b004 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include namespace CurrentMetrics @@ -305,14 +305,6 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( /// We don't know real size of part because sender server version is too old reservation = data.makeEmptyReservationOnLargestDisk(); } - auto storage_id = data.getStorageID(); - String new_part_path = data.getFullPathOnDisk(reservation->getDisk()) + part_name + "/"; - auto entry = data.global_context.getReplicatedFetchList().insert( - storage_id.getDatabaseName(), storage_id.getTableName(), - part_info.partition_id, part_name, new_part_path, - replica_path, uri, to_detached, sum_files_size); - - in.setNextReadCallback(ReplicatedFetchReadCallback(*entry)); bool sync = (data_settings->min_compressed_bytes_to_fsync_after_fetch && sum_files_size >= data_settings->min_compressed_bytes_to_fsync_after_fetch); @@ -321,6 +313,15 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_TYPE) readStringBinary(part_type, in); + auto storage_id = data.getStorageID(); + String new_part_path = part_type == "InMemory" ? "memory" : data.getFullPathOnDisk(reservation->getDisk()) + part_name + "/"; + auto entry = data.global_context.getReplicatedFetchList().insert( + storage_id.getDatabaseName(), storage_id.getTableName(), + part_info.partition_id, part_name, new_part_path, + replica_path, uri, to_detached, sum_files_size); + + in.setNextReadCallback(ReplicatedFetchReadCallback(*entry)); + return part_type == "InMemory" ? downloadPartToMemory(part_name, metadata_snapshot, std::move(reservation), in) : downloadPartToDisk(part_name, replica_path, to_detached, tmp_prefix_, sync, std::move(reservation), in); } diff --git a/src/Storages/MergeTree/ReplicatedFetchesList.cpp b/src/Storages/MergeTree/ReplicatedFetchList.cpp similarity index 97% rename from src/Storages/MergeTree/ReplicatedFetchesList.cpp rename to src/Storages/MergeTree/ReplicatedFetchList.cpp index 92e60d9dde1..63971a7b2e1 100644 --- a/src/Storages/MergeTree/ReplicatedFetchesList.cpp +++ b/src/Storages/MergeTree/ReplicatedFetchList.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Storages/MergeTree/ReplicatedFetchesList.h b/src/Storages/MergeTree/ReplicatedFetchList.h similarity index 100% rename from src/Storages/MergeTree/ReplicatedFetchesList.h rename to src/Storages/MergeTree/ReplicatedFetchList.h diff --git a/src/Storages/System/StorageSystemFetches.cpp b/src/Storages/System/StorageSystemFetches.cpp index 0c992457c6c..080915db514 100644 --- a/src/Storages/System/StorageSystemFetches.cpp +++ b/src/Storages/System/StorageSystemFetches.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemFetches.h b/src/Storages/System/StorageSystemFetches.h index 7c93bce7ee7..3e422c5bc01 100644 --- a/src/Storages/System/StorageSystemFetches.h +++ b/src/Storages/System/StorageSystemFetches.h @@ -10,7 +10,7 @@ namespace DB class Context; -/// system.fetches table. Takes data from context.getReplicatedFetchesList() +/// system.fetches table. Takes data from context.getReplicatedFetchList() class StorageSystemFetches final : public ext::shared_ptr_helper, public IStorageSystemOneBlock { friend struct ext::shared_ptr_helper; diff --git a/src/Storages/ya.make b/src/Storages/ya.make index 289a226ce7d..063b0d3296b 100644 --- a/src/Storages/ya.make +++ b/src/Storages/ya.make @@ -91,7 +91,7 @@ SRCS( MergeTree/MergeTreeWriteAheadLog.cpp MergeTree/MergeType.cpp MergeTree/registerStorageMergeTree.cpp - MergeTree/ReplicatedFetchesList.cpp + MergeTree/ReplicatedFetchList.cpp MergeTree/ReplicatedMergeTreeAddress.cpp MergeTree/ReplicatedMergeTreeAltersSequence.cpp MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp diff --git a/tests/integration/test_system_fetches/test.py b/tests/integration/test_system_fetches/test.py index 2746ced300c..bd2345984ea 100644 --- a/tests/integration/test_system_fetches/test.py +++ b/tests/integration/test_system_fetches/test.py @@ -66,6 +66,9 @@ def test_system_fetches(started_cluster): assert first_non_empty['source_replica_port'] == 9009 assert first_non_empty['source_replica_path'] == '/clickhouse/test/t/replicas/1' assert first_non_empty['interserver_scheme'] == 'http' + assert first_non_empty['result_part_name'] == 'all_0_0_0' + assert first_non_empty['result_part_path'].startswith('/var/lib/clickhouse/') + assert first_non_empty['result_part_path'].endswith('all_0_0_0/') assert first_non_empty['partition_id'] == 'all' assert first_non_empty['URI'].startswith('http://node1:9009/?endpoint=DataPartsExchange') From db16942716a625df8884dfbf10c39586c1d8e6b2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 27 Oct 2020 15:50:43 +0300 Subject: [PATCH 238/325] Better comment --- src/IO/ReadWriteBufferFromHTTP.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 10cacf10dd9..0a3b4d374b5 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -155,6 +155,7 @@ namespace detail } public: + using NextReadCallback = std::function; using OutStreamCallback = std::function; explicit ReadWriteBufferFromHTTPBase( @@ -222,9 +223,14 @@ namespace detail return def; } - void setNextReadCallback(std::function read_callback_) + /// Set function to call on each nextImpl, useful when you need to track + /// progress. + /// NOTE: parameter on each call is not incremental -- it's all bytes count + /// passed through the buffer + void setNextReadCallback(NextReadCallback read_callback_) { read_callback = read_callback_; + /// Some data maybe already read read_callback(count()); } }; From ee4e6caf70857c5e4b57a66fb50899eee416930b Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 27 Oct 2020 15:52:49 +0300 Subject: [PATCH 239/325] Remove redundant metric increment --- src/Storages/MergeTree/DataPartsExchange.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index bb92202b004..e87ae599521 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -18,7 +18,6 @@ namespace CurrentMetrics { extern const Metric ReplicatedSend; - extern const Metric ReplicatedFetch; } namespace DB @@ -380,8 +379,6 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( if (disk->exists(part_download_path)) throw Exception("Directory " + fullPath(disk, part_download_path) + " already exists.", ErrorCodes::DIRECTORY_ALREADY_EXISTS); - CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedFetch}; - disk->createDirectories(part_download_path); std::optional sync_guard; From f35edfb2a59ea7d413d13322cb60149f33e7951d Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 27 Oct 2020 15:57:55 +0300 Subject: [PATCH 240/325] Remove some unused fields --- src/Storages/MergeTree/ReplicatedFetchList.cpp | 3 +-- src/Storages/MergeTree/ReplicatedFetchList.h | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedFetchList.cpp b/src/Storages/MergeTree/ReplicatedFetchList.cpp index 63971a7b2e1..82bc8ae21e0 100644 --- a/src/Storages/MergeTree/ReplicatedFetchList.cpp +++ b/src/Storages/MergeTree/ReplicatedFetchList.cpp @@ -1,10 +1,10 @@ #include #include #include -#include namespace DB { + ReplicatedFetchListElement::ReplicatedFetchListElement( const std::string & database_, const std::string & table_, const std::string & partition_id_, const std::string & result_part_name_, @@ -26,7 +26,6 @@ ReplicatedFetchListElement::ReplicatedFetchListElement( { } - ReplicatedFetchInfo ReplicatedFetchListElement::getInfo() const { ReplicatedFetchInfo res; diff --git a/src/Storages/MergeTree/ReplicatedFetchList.h b/src/Storages/MergeTree/ReplicatedFetchList.h index 670e1fb984a..ed134de3f2c 100644 --- a/src/Storages/MergeTree/ReplicatedFetchList.h +++ b/src/Storages/MergeTree/ReplicatedFetchList.h @@ -50,21 +50,22 @@ struct ReplicatedFetchListElement : private boost::noncopyable const std::string result_part_name; const std::string result_part_path; - std::string source_replica_path; - std::string source_replica_hostname; - UInt16 source_replica_port; - std::string interserver_scheme; - std::string uri; + const std::string source_replica_path; + const std::string source_replica_hostname; + const UInt16 source_replica_port; + const std::string interserver_scheme; + const std::string uri; const UInt8 to_detached; Stopwatch watch; std::atomic progress{}; - std::atomic is_cancelled{}; + /// How many bytes already read std::atomic bytes_read_compressed{}; - UInt64 total_size_bytes_compressed{}; + /// Total bytes to read + const UInt64 total_size_bytes_compressed{}; - UInt64 thread_id; + const UInt64 thread_id; ReplicatedFetchListElement( const std::string & database_, const std::string & table_, @@ -78,6 +79,7 @@ struct ReplicatedFetchListElement : private boost::noncopyable using ReplicatedFetchListEntry = BackgroundProcessListEntry; +/// List of currently processing replicated fetches class ReplicatedFetchList final : public BackgroundProcessList { private: From e40c509392ce7a8634f67837dd1c608dce78f5d3 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 27 Oct 2020 16:00:40 +0300 Subject: [PATCH 241/325] More clear callback name --- src/IO/ReadWriteBufferFromHTTP.h | 14 +++++++------- src/Storages/MergeTree/DataPartsExchange.cpp | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 0a3b4d374b5..6ff99690bb4 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -106,7 +106,7 @@ namespace detail std::vector cookies; HTTPHeaderEntries http_header_entries; RemoteHostFilter remote_host_filter; - std::function read_callback; + std::function next_callback; std::istream * call(const Poco::URI uri_, Poco::Net::HTTPResponse & response) { @@ -155,7 +155,7 @@ namespace detail } public: - using NextReadCallback = std::function; + using NextCallback = std::function; using OutStreamCallback = std::function; explicit ReadWriteBufferFromHTTPBase( @@ -206,8 +206,8 @@ namespace detail bool nextImpl() override { - if (read_callback) - read_callback(count()); + if (next_callback) + next_callback(count()); if (!impl->next()) return false; internal_buffer = impl->buffer(); @@ -227,11 +227,11 @@ namespace detail /// progress. /// NOTE: parameter on each call is not incremental -- it's all bytes count /// passed through the buffer - void setNextReadCallback(NextReadCallback read_callback_) + void setNextCallback(NextCallback next_callback_) { - read_callback = read_callback_; + next_callback = next_callback_; /// Some data maybe already read - read_callback(count()); + next_callback(count()); } }; } diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index e87ae599521..a6c3c562d91 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -319,7 +319,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( part_info.partition_id, part_name, new_part_path, replica_path, uri, to_detached, sum_files_size); - in.setNextReadCallback(ReplicatedFetchReadCallback(*entry)); + in.setNextCallback(ReplicatedFetchReadCallback(*entry)); return part_type == "InMemory" ? downloadPartToMemory(part_name, metadata_snapshot, std::move(reservation), in) : downloadPartToDisk(part_name, replica_path, to_detached, tmp_prefix_, sync, std::move(reservation), in); From 3d3d246bdcd1140c272df9676868bac33283e2e0 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 27 Oct 2020 16:07:50 +0300 Subject: [PATCH 242/325] Rename table --- src/Storages/MergeTree/DataPartsExchange.cpp | 1 + ...SystemFetches.cpp => StorageSystemReplicatedFetches.cpp} | 6 +++--- ...rageSystemFetches.h => StorageSystemReplicatedFetches.h} | 6 +++--- src/Storages/System/attachSystemTables.cpp | 4 ++-- src/Storages/ya.make | 2 +- .../__init__.py | 0 .../test.py | 4 ++-- 7 files changed, 12 insertions(+), 11 deletions(-) rename src/Storages/System/{StorageSystemFetches.cpp => StorageSystemReplicatedFetches.cpp} (90%) rename src/Storages/System/{StorageSystemFetches.h => StorageSystemReplicatedFetches.h} (59%) rename tests/integration/{test_system_fetches => test_system_replicated_fetches}/__init__.py (100%) rename tests/integration/{test_system_fetches => test_system_replicated_fetches}/test.py (97%) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index a6c3c562d91..b9c4c6b625e 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -52,6 +52,7 @@ std::string getEndpointId(const std::string & node_id) return "DataPartsExchange:" + node_id; } +/// Simple functor for tracking fetch progress in system.fetches table. struct ReplicatedFetchReadCallback { ReplicatedFetchList::Entry & replicated_fetch_entry; diff --git a/src/Storages/System/StorageSystemFetches.cpp b/src/Storages/System/StorageSystemReplicatedFetches.cpp similarity index 90% rename from src/Storages/System/StorageSystemFetches.cpp rename to src/Storages/System/StorageSystemReplicatedFetches.cpp index 080915db514..53bec5aa42f 100644 --- a/src/Storages/System/StorageSystemFetches.cpp +++ b/src/Storages/System/StorageSystemReplicatedFetches.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -8,7 +8,7 @@ namespace DB { -NamesAndTypesList StorageSystemFetches::getNamesAndTypes() +NamesAndTypesList StorageSystemReplicatedFetches::getNamesAndTypes() { return { {"database", std::make_shared()}, @@ -30,7 +30,7 @@ NamesAndTypesList StorageSystemFetches::getNamesAndTypes() }; } -void StorageSystemFetches::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const +void StorageSystemReplicatedFetches::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { const auto access = context.getAccess(); const bool check_access_for_tables = !access->isGranted(AccessType::SHOW_TABLES); diff --git a/src/Storages/System/StorageSystemFetches.h b/src/Storages/System/StorageSystemReplicatedFetches.h similarity index 59% rename from src/Storages/System/StorageSystemFetches.h rename to src/Storages/System/StorageSystemReplicatedFetches.h index 3e422c5bc01..4ec1f9b9f6c 100644 --- a/src/Storages/System/StorageSystemFetches.h +++ b/src/Storages/System/StorageSystemReplicatedFetches.h @@ -11,11 +11,11 @@ namespace DB class Context; /// system.fetches table. Takes data from context.getReplicatedFetchList() -class StorageSystemFetches final : public ext::shared_ptr_helper, public IStorageSystemOneBlock +class StorageSystemReplicatedFetches final : public ext::shared_ptr_helper, public IStorageSystemOneBlock { - friend struct ext::shared_ptr_helper; + friend struct ext::shared_ptr_helper; public: - std::string getName() const override { return "SystemFetches"; } + std::string getName() const override { return "SystemReplicatedFetches"; } static NamesAndTypesList getNamesAndTypes(); diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 012b54e3c45..a4158f256c1 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include #include @@ -138,7 +138,7 @@ void attachSystemTablesServer(IDatabase & system_database, bool has_zookeeper) attach(system_database, "clusters"); attach(system_database, "graphite_retentions"); attach(system_database, "macros"); - attach(system_database, "fetches"); + attach(system_database, "replicated_fetches"); if (has_zookeeper) attach(system_database, "zookeeper"); diff --git a/src/Storages/ya.make b/src/Storages/ya.make index 063b0d3296b..bf1a6982976 100644 --- a/src/Storages/ya.make +++ b/src/Storages/ya.make @@ -156,7 +156,6 @@ SRCS( System/StorageSystemDistributionQueue.cpp System/StorageSystemEnabledRoles.cpp System/StorageSystemEvents.cpp - System/StorageSystemFetches.cpp System/StorageSystemFormats.cpp System/StorageSystemFunctions.cpp System/StorageSystemGrants.cpp @@ -179,6 +178,7 @@ SRCS( System/StorageSystemQuotasUsage.cpp System/StorageSystemQuotaUsage.cpp System/StorageSystemReplicas.cpp + System/StorageSystemReplicatedFetches.cpp System/StorageSystemReplicationQueue.cpp System/StorageSystemRoleGrants.cpp System/StorageSystemRoles.cpp diff --git a/tests/integration/test_system_fetches/__init__.py b/tests/integration/test_system_replicated_fetches/__init__.py similarity index 100% rename from tests/integration/test_system_fetches/__init__.py rename to tests/integration/test_system_replicated_fetches/__init__.py diff --git a/tests/integration/test_system_fetches/test.py b/tests/integration/test_system_replicated_fetches/test.py similarity index 97% rename from tests/integration/test_system_fetches/test.py rename to tests/integration/test_system_replicated_fetches/test.py index bd2345984ea..cefb3256893 100644 --- a/tests/integration/test_system_fetches/test.py +++ b/tests/integration/test_system_replicated_fetches/test.py @@ -27,7 +27,7 @@ def started_cluster(): def get_random_string(length): return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) -def test_system_fetches(started_cluster): +def test_system_replicated_fetches(started_cluster): node1.query("CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '1') ORDER BY tuple()") node2.query("CREATE TABLE t (key UInt64, data String) ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '2') ORDER BY tuple()") @@ -38,7 +38,7 @@ def test_system_fetches(started_cluster): node2.query("SYSTEM START FETCHES t") fetches_result = [] for _ in range(1000): - result = json.loads(node2.query("SELECT * FROM system.fetches FORMAT JSON")) + result = json.loads(node2.query("SELECT * FROM system.replicated_fetches FORMAT JSON")) if not result["data"]: if fetches_result: break From afc28c84a03d22af4c378af0e48dbd40f7307bc9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 27 Oct 2020 16:07:58 +0300 Subject: [PATCH 243/325] Fix double free for shared exception message in case of dictGet from not loaded dictionary. --- src/Functions/FunctionsExternalDictionaries.h | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index 0fae3de1fb2..a0ef6c893ed 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -57,6 +57,7 @@ namespace ErrorCodes extern const int TYPE_MISMATCH; extern const int ILLEGAL_COLUMN; extern const int BAD_ARGUMENTS; + extern const int DICTIONARIES_WAS_NOT_LOADED; } @@ -82,14 +83,29 @@ public: std::shared_ptr getDictionary(const String & dictionary_name) { - String resolved_name = DatabaseCatalog::instance().resolveDictionaryName(dictionary_name); - auto dict = external_loader.getDictionary(resolved_name); - if (!access_checked) + // Exception from external_loader may be shared for dictGet call from multiple threads. + // Don't just rethrow it, because sharing the same exception object + // between multiple threads can lead to weird effects if they decide to + // modify it, for example, by adding some error context. + try { - context.checkAccess(AccessType::dictGet, dict->getDatabaseOrNoDatabaseTag(), dict->getDictionaryID().getTableName()); - access_checked = true; + String resolved_name = DatabaseCatalog::instance().resolveDictionaryName(dictionary_name); + auto dict = external_loader.getDictionary(resolved_name); + if (!access_checked) + { + context.checkAccess(AccessType::dictGet, dict->getDatabaseOrNoDatabaseTag(), dict->getDictionaryID().getTableName()); + access_checked = true; + } + return dict; + } + catch (...) + { + throw DB::Exception(ErrorCodes::DICTIONARIES_WAS_NOT_LOADED, + "Failed to load dictionary '{}': {}", + dictionary_name, + getCurrentExceptionMessage(true /*with stack trace*/, + true /*check embedded stack trace*/)); } - return dict; } std::shared_ptr getDictionary(const ColumnWithTypeAndName & column) From 6253d1466abd894ccf3b678cb64368ce922e9862 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 27 Oct 2020 16:09:14 +0300 Subject: [PATCH 244/325] Fix comments --- src/Storages/MergeTree/BackgroundProcessList.h | 2 +- src/Storages/MergeTree/DataPartsExchange.cpp | 2 +- src/Storages/System/StorageSystemReplicatedFetches.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/BackgroundProcessList.h b/src/Storages/MergeTree/BackgroundProcessList.h index 2f4cc020796..b447753ca74 100644 --- a/src/Storages/MergeTree/BackgroundProcessList.h +++ b/src/Storages/MergeTree/BackgroundProcessList.h @@ -9,7 +9,7 @@ namespace DB { -/// Common code for background processes lists, like system.merges and system.fetches +/// Common code for background processes lists, like system.merges and system.replicated_fetches /// Look at examples in MergeList and ReplicatedFetchList template diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index b9c4c6b625e..2e8705e761d 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -52,7 +52,7 @@ std::string getEndpointId(const std::string & node_id) return "DataPartsExchange:" + node_id; } -/// Simple functor for tracking fetch progress in system.fetches table. +/// Simple functor for tracking fetch progress in system.replicated_fetches table. struct ReplicatedFetchReadCallback { ReplicatedFetchList::Entry & replicated_fetch_entry; diff --git a/src/Storages/System/StorageSystemReplicatedFetches.h b/src/Storages/System/StorageSystemReplicatedFetches.h index 4ec1f9b9f6c..34081923e4f 100644 --- a/src/Storages/System/StorageSystemReplicatedFetches.h +++ b/src/Storages/System/StorageSystemReplicatedFetches.h @@ -10,7 +10,7 @@ namespace DB class Context; -/// system.fetches table. Takes data from context.getReplicatedFetchList() +/// system.replicated_fetches table. Takes data from context.getReplicatedFetchList() class StorageSystemReplicatedFetches final : public ext::shared_ptr_helper, public IStorageSystemOneBlock { friend struct ext::shared_ptr_helper; From b39b2932bc269cc92fbceb713f5417c523af17fc Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 27 Oct 2020 16:49:08 +0300 Subject: [PATCH 245/325] Added test. --- ...2_dictionary_load_exception_race.reference | 0 .../01542_dictionary_load_exception_race.sh | 44 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 tests/queries/0_stateless/01542_dictionary_load_exception_race.reference create mode 100755 tests/queries/0_stateless/01542_dictionary_load_exception_race.sh diff --git a/tests/queries/0_stateless/01542_dictionary_load_exception_race.reference b/tests/queries/0_stateless/01542_dictionary_load_exception_race.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh b/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh new file mode 100755 index 00000000000..8782faac003 --- /dev/null +++ b/tests/queries/0_stateless/01542_dictionary_load_exception_race.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS database_for_dict" +$CLICKHOUSE_CLIENT --query "CREATE DATABASE database_for_dict" +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS database_for_dict.table_for_dict" +$CLICKHOUSE_CLIENT --query "CREATE TABLE database_for_dict.table_for_dict (key_column UInt64, second_column UInt64, third_column String) ENGINE = MergeTree() ORDER BY key_column" +$CLICKHOUSE_CLIENT --query "INSERT INTO database_for_dict.table_for_dict VALUES (100500, 10000000, 'Hello world')" + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ordinary_db" +$CLICKHOUSE_CLIENT --query "CREATE DATABASE ordinary_db" +$CLICKHOUSE_CLIENT --query "CREATE DICTIONARY ordinary_db.dict1 ( key_column UInt64 DEFAULT 0, second_column UInt64 DEFAULT 1, third_column String DEFAULT 'qqq' ) PRIMARY KEY key_column SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT()) SETTINGS(max_result_bytes=1)" + +function dict_get_thread() +{ + while true; do + $CLICKHOUSE_CLIENT --query "SELECT dictGetString('ordinary_db.dict1', 'third_column', toUInt64(rand() % 1000)) from numbers(2)" &>/dev/null + done +} + +export -f dict_get_thread; + +TIMEOUT=10 + +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & +timeout $TIMEOUT bash -c dict_get_thread 2> /dev/null & + +wait + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ordinary_db" +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS database_for_dict" From 71298ba496344e3b6875c9925b01cacf1fbb535e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 27 Oct 2020 17:21:51 +0300 Subject: [PATCH 246/325] Move fix to ExternalLoader::checkLoaded. --- src/Functions/FunctionsExternalDictionaries.h | 28 ++++--------------- src/Interpreters/ExternalLoader.cpp | 20 ++++++++++++- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index a0ef6c893ed..0fae3de1fb2 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -57,7 +57,6 @@ namespace ErrorCodes extern const int TYPE_MISMATCH; extern const int ILLEGAL_COLUMN; extern const int BAD_ARGUMENTS; - extern const int DICTIONARIES_WAS_NOT_LOADED; } @@ -83,29 +82,14 @@ public: std::shared_ptr getDictionary(const String & dictionary_name) { - // Exception from external_loader may be shared for dictGet call from multiple threads. - // Don't just rethrow it, because sharing the same exception object - // between multiple threads can lead to weird effects if they decide to - // modify it, for example, by adding some error context. - try + String resolved_name = DatabaseCatalog::instance().resolveDictionaryName(dictionary_name); + auto dict = external_loader.getDictionary(resolved_name); + if (!access_checked) { - String resolved_name = DatabaseCatalog::instance().resolveDictionaryName(dictionary_name); - auto dict = external_loader.getDictionary(resolved_name); - if (!access_checked) - { - context.checkAccess(AccessType::dictGet, dict->getDatabaseOrNoDatabaseTag(), dict->getDictionaryID().getTableName()); - access_checked = true; - } - return dict; - } - catch (...) - { - throw DB::Exception(ErrorCodes::DICTIONARIES_WAS_NOT_LOADED, - "Failed to load dictionary '{}': {}", - dictionary_name, - getCurrentExceptionMessage(true /*with stack trace*/, - true /*check embedded stack trace*/)); + context.checkAccess(AccessType::dictGet, dict->getDatabaseOrNoDatabaseTag(), dict->getDictionaryID().getTableName()); + access_checked = true; } + return dict; } std::shared_ptr getDictionary(const ColumnWithTypeAndName & column) diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index dcef36de175..b5f802aa4bc 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -28,6 +28,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; + extern const int DICTIONARIES_WAS_NOT_LOADED; } @@ -1404,7 +1405,24 @@ void ExternalLoader::checkLoaded(const ExternalLoader::LoadResult & result, if (result.status == ExternalLoader::Status::LOADING) throw Exception(type_name + " '" + result.name + "' is still loading", ErrorCodes::BAD_ARGUMENTS); if (result.exception) - std::rethrow_exception(result.exception); + { + // Exception is shared for multiple threads. + // Don't just rethrow it, because sharing the same exception object + // between multiple threads can lead to weird effects if they decide to + // modify it, for example, by adding some error context. + try + { + std::rethrow_exception(result.exception); + } + catch (...) + { + throw DB::Exception(ErrorCodes::DICTIONARIES_WAS_NOT_LOADED, + "Failed to load dictionary '{}': {}", + result.name, + getCurrentExceptionMessage(true /*with stack trace*/, + true /*check embedded stack trace*/)); + } + } if (result.status == ExternalLoader::Status::NOT_EXIST) throw Exception(type_name + " '" + result.name + "' not found", ErrorCodes::BAD_ARGUMENTS); if (result.status == ExternalLoader::Status::NOT_LOADED) From 382a6e62ecbe38d6c4a0b2f1d7ef3d6f9abca226 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 27 Oct 2020 17:55:03 +0300 Subject: [PATCH 247/325] Update AggregateFunctionStudentTTest.h --- src/AggregateFunctions/AggregateFunctionStudentTTest.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.h b/src/AggregateFunctions/AggregateFunctionStudentTTest.h index 5084e34e56f..0aef8f3ee2a 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.h +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.h @@ -26,7 +26,7 @@ namespace ErrorCodes #if defined(OS_DARWIN) extern "C" { - double lgamma_r(double x, int * signgamp); + double lgammal_r(double x, int * signgamp); } #endif From 42eb784a26937c7c21ac668a80b8a95cfd5332fa Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 27 Oct 2020 17:55:19 +0300 Subject: [PATCH 248/325] Update AggregateFunctionWelchTTest.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 8fd7ebeee6b..b598f25162e 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -27,7 +27,7 @@ namespace ErrorCodes #if defined(OS_DARWIN) extern "C" { - double lgamma_r(double x, int * signgamp); + double lgammal_r(double x, int * signgamp); } #endif From 4bbaf3e99e4870f02756e26546c92c9134d1acc9 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 27 Oct 2020 18:03:35 +0300 Subject: [PATCH 249/325] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0c07021630b..e2a128eee25 100644 --- a/README.md +++ b/README.md @@ -17,4 +17,6 @@ ClickHouse is an open-source column-oriented database management system that all ## Upcoming Events -* [ClickHouse virtual office hours](https://www.eventbrite.com/e/clickhouse-october-virtual-meetup-office-hours-tickets-123129500651) on October 22, 2020. +* [The Second ClickHouse Meetup East](https://www.eventbrite.com/e/the-second-clickhouse-meetup-east-tickets-126787955187) on October 31, 2020. +* [ClickHouse for Enterprise Meetup (in Russian)](https://arenadata-events.timepad.ru/event/1465249/) on November 10, 2020. + From 2a26136c9ef96d5f01a22b8ab12ec5d05fb586d3 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Tue, 27 Oct 2020 18:18:05 +0300 Subject: [PATCH 250/325] add __init__.py --- tests/integration/test_disabled_mysql_server/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/integration/test_disabled_mysql_server/__init__.py diff --git a/tests/integration/test_disabled_mysql_server/__init__.py b/tests/integration/test_disabled_mysql_server/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 60f2d822d73e19649453e4db87855993610ea526 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 27 Oct 2020 18:27:12 +0300 Subject: [PATCH 251/325] Fix fake race condition on system.merges merge_algorithm --- src/Storages/MergeTree/MergeList.cpp | 2 +- src/Storages/MergeTree/MergeList.h | 3 ++- .../MergeTree/MergeTreeDataMergerMutator.cpp | 16 ++++++++-------- .../test_concurrent_ttl_merges/test.py | 4 ++-- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/Storages/MergeTree/MergeList.cpp b/src/Storages/MergeTree/MergeList.cpp index 5b044622b36..ba6c2a3d462 100644 --- a/src/Storages/MergeTree/MergeList.cpp +++ b/src/Storages/MergeTree/MergeList.cpp @@ -68,7 +68,7 @@ MergeInfo MergeListElement::getInfo() const res.memory_usage = memory_tracker.get(); res.thread_id = thread_id; res.merge_type = toString(merge_type); - res.merge_algorithm = toString(merge_algorithm); + res.merge_algorithm = toString(merge_algorithm.load(std::memory_order_relaxed)); for (const auto & source_part_name : source_part_names) res.source_part_names.emplace_back(source_part_name); diff --git a/src/Storages/MergeTree/MergeList.h b/src/Storages/MergeTree/MergeList.h index c1166c55703..09c61250afd 100644 --- a/src/Storages/MergeTree/MergeList.h +++ b/src/Storages/MergeTree/MergeList.h @@ -92,7 +92,8 @@ struct MergeListElement : boost::noncopyable UInt64 thread_id; MergeType merge_type; - MergeAlgorithm merge_algorithm; + /// Detected after merge already started + std::atomic merge_algorithm; MergeListElement(const std::string & database, const std::string & table, const FutureMergedMutatedPart & future_part); diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index b29966751f9..b7581907fd3 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -710,10 +710,10 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor size_t sum_input_rows_upper_bound = merge_entry->total_rows_count; size_t sum_compressed_bytes_upper_bound = merge_entry->total_size_bytes_compressed; - MergeAlgorithm merge_alg = chooseMergeAlgorithm(parts, sum_input_rows_upper_bound, gathering_columns, deduplicate, need_remove_expired_values); - merge_entry->merge_algorithm = merge_alg; + MergeAlgorithm chosen_merge_algorithm = chooseMergeAlgorithm(parts, sum_input_rows_upper_bound, gathering_columns, deduplicate, need_remove_expired_values); + merge_entry->merge_algorithm.store(chosen_merge_algorithm, std::memory_order_relaxed); - LOG_DEBUG(log, "Selected MergeAlgorithm: {}", toString(merge_alg)); + LOG_DEBUG(log, "Selected MergeAlgorithm: {}", toString(chosen_merge_algorithm)); /// Note: this is done before creating input streams, because otherwise data.data_parts_mutex /// (which is locked in data.getTotalActiveSizeInBytes()) @@ -728,7 +728,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor std::unique_ptr rows_sources_write_buf; std::optional column_sizes; - if (merge_alg == MergeAlgorithm::Vertical) + if (chosen_merge_algorithm == MergeAlgorithm::Vertical) { tmp_disk->createDirectories(new_part_tmp_path); rows_sources_file_path = new_part_tmp_path + "rows_sources"; @@ -818,7 +818,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor ProcessorPtr merged_transform; /// If merge is vertical we cannot calculate it - bool blocks_are_granules_size = (merge_alg == MergeAlgorithm::Vertical); + bool blocks_are_granules_size = (chosen_merge_algorithm == MergeAlgorithm::Vertical); UInt64 merge_block_size = data_settings->merge_max_block_size; switch (data.merging_params.mode) @@ -917,7 +917,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor { /// The same progress from merge_entry could be used for both algorithms (it should be more accurate) /// But now we are using inaccurate row-based estimation in Horizontal case for backward compatibility - Float64 progress = (merge_alg == MergeAlgorithm::Horizontal) + Float64 progress = (chosen_merge_algorithm == MergeAlgorithm::Horizontal) ? std::min(1., 1. * rows_written / sum_input_rows_upper_bound) : std::min(1., merge_entry->progress.load(std::memory_order_relaxed)); @@ -938,7 +938,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor MergeTreeData::DataPart::Checksums checksums_gathered_columns; /// Gather ordinary columns - if (merge_alg == MergeAlgorithm::Vertical) + if (chosen_merge_algorithm == MergeAlgorithm::Vertical) { size_t sum_input_rows_exact = merge_entry->rows_read; merge_entry->columns_written = merging_column_names.size(); @@ -1054,7 +1054,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor ReadableSize(merge_entry->bytes_read_uncompressed / elapsed_seconds)); } - if (merge_alg != MergeAlgorithm::Vertical) + if (chosen_merge_algorithm != MergeAlgorithm::Vertical) to.writeSuffixAndFinalizePart(new_data_part, need_sync); else to.writeSuffixAndFinalizePart(new_data_part, need_sync, &storage_columns, &checksums_gathered_columns); diff --git a/tests/integration/test_concurrent_ttl_merges/test.py b/tests/integration/test_concurrent_ttl_merges/test.py index f067e65f58a..65bc3828b38 100644 --- a/tests/integration/test_concurrent_ttl_merges/test.py +++ b/tests/integration/test_concurrent_ttl_merges/test.py @@ -5,8 +5,8 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True) -node2 = cluster.add_instance('node2', main_configs=['configs/fast_background_pool.xml'], with_zookeeper=True) +node1 = cluster.add_instance('node1', main_configs=['configs/fast_background_pool.xml', 'configs/log_conf.xml'], with_zookeeper=True) +node2 = cluster.add_instance('node2', main_configs=['configs/fast_background_pool.xml', 'configs/log_conf.xml'], with_zookeeper=True) @pytest.fixture(scope="module") From 3ec5fc7a4c59411bb0b479d20fba02a66697469b Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 27 Oct 2020 18:29:06 +0300 Subject: [PATCH 252/325] Add explicit --- src/Storages/MergeTree/DataPartsExchange.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 2e8705e761d..eaf32385908 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -57,11 +57,10 @@ struct ReplicatedFetchReadCallback { ReplicatedFetchList::Entry & replicated_fetch_entry; - ReplicatedFetchReadCallback(ReplicatedFetchList::Entry & replicated_fetch_entry_) + explicit ReplicatedFetchReadCallback(ReplicatedFetchList::Entry & replicated_fetch_entry_) : replicated_fetch_entry(replicated_fetch_entry_) {} - void operator() (size_t bytes_count) { replicated_fetch_entry->bytes_read_compressed.store(bytes_count, std::memory_order_relaxed); From c7236a93596937e0875a5748b5f757df58406398 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 27 Oct 2020 19:12:53 +0300 Subject: [PATCH 253/325] Try fix tests. --- src/Interpreters/ExternalLoader.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index b5f802aa4bc..e1713c7cbbb 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -1414,6 +1414,11 @@ void ExternalLoader::checkLoaded(const ExternalLoader::LoadResult & result, { std::rethrow_exception(result.exception); } + catch (const Poco::Exception & e) + { + /// This will create a copy for Poco::Exception and DB::Exception + e.rethrow(); + } catch (...) { throw DB::Exception(ErrorCodes::DICTIONARIES_WAS_NOT_LOADED, From 2f752b0db5f2dbb67b1ab7937dee3c457e41e6cd Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 27 Oct 2020 20:40:54 +0300 Subject: [PATCH 254/325] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e2a128eee25..03b5c988586 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,6 @@ ClickHouse is an open-source column-oriented database management system that all ## Upcoming Events -* [The Second ClickHouse Meetup East](https://www.eventbrite.com/e/the-second-clickhouse-meetup-east-tickets-126787955187) on October 31, 2020. -* [ClickHouse for Enterprise Meetup (in Russian)](https://arenadata-events.timepad.ru/event/1465249/) on November 10, 2020. +* [The Second ClickHouse Meetup East (online)](https://www.eventbrite.com/e/the-second-clickhouse-meetup-east-tickets-126787955187) on October 31, 2020. +* [ClickHouse for Enterprise Meetup (online in Russian)](https://arenadata-events.timepad.ru/event/1465249/) on November 10, 2020. From fc14fde24acd7e06547c266959fd14614b597999 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 26 Oct 2020 23:01:06 +0300 Subject: [PATCH 255/325] Fix DROP TABLE for Distributed (racy with INSERT)
``` drop() on T1275: 0 DB::StorageDistributed::drop (this=0x7f9ed34f0000) at ../contrib/libcxx/include/__hash_table:966 1 0x000000000d557242 in DB::DatabaseOnDisk::dropTable (this=0x7f9fc22706d8, context=..., table_name=...) at ../contrib/libcxx/include/new:340 2 0x000000000d6fcf7c in DB::InterpreterDropQuery::executeToTable (this=this@entry=0x7f9e42560dc0, query=...) at ../contrib/libcxx/include/memory:3826 3 0x000000000d6ff5ee in DB::InterpreterDropQuery::execute (this=0x7f9e42560dc0) at ../src/Interpreters/InterpreterDropQuery.cpp:50 4 0x000000000daa40c0 in DB::executeQueryImpl (begin=, end=, context=..., internal=, stage=DB::QueryProcessingStage::Complete, has_query_tail=false, istr=0x0) at ../src/Interpreters/executeQuery.cpp:420 5 0x000000000daa59df in DB::executeQuery (query=..., context=..., internal=internal@entry=false, stage=, may_have_embedded_data=) at ../contrib/libcxx/include/string:1487 6 0x000000000e1369e6 in DB::TCPHandler::runImpl (this=this@entry=0x7f9ddf3a9000) at ../src/Server/TCPHandler.cpp:254 7 0x000000000e1379c9 in DB::TCPHandler::run (this=0x7f9ddf3a9000) at ../src/Server/TCPHandler.cpp:1326 8 0x000000001086fac7 in Poco::Net::TCPServerConnection::start (this=this@entry=0x7f9ddf3a9000) at ../contrib/poco/Net/src/TCPServerConnection.cpp:43 9 0x000000001086ff2b in Poco::Net::TCPServerDispatcher::run (this=0x7f9e4eba5c00) at ../contrib/poco/Net/src/TCPServerDispatcher.cpp:114 10 0x00000000109dbe8e in Poco::PooledThread::run (this=0x7f9e4a2d2f80) at ../contrib/poco/Foundation/src/ThreadPool.cpp:199 11 0x00000000109d78f9 in Poco::ThreadImpl::runnableEntry (pThread=) at ../contrib/poco/Foundation/include/Poco/SharedPtr.h:401 12 0x00007f9fc3cccea7 in start_thread (arg=) at pthread_create.c:477 13 0x00007f9fc3bebeaf in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 StorageDistributedDirectoryMonitor on T166: 0 DB::StorageDistributedDirectoryMonitor::StorageDistributedDirectoryMonitor (this=0x7f9ea7ab1400, storage_=..., path_=..., pool_=..., monitor_blocker_=..., bg_pool_=...) at ../src/Storages/Distributed/DirectoryMonitor.cpp:81 1 0x000000000dbf684e in std::__1::make_unique<> () at ../contrib/libcxx/include/memory:3474 2 DB::StorageDistributed::requireDirectoryMonitor (this=0x7f9ed34f0000, disk=..., name=...) at ../src/Storages/StorageDistributed.cpp:682 3 0x000000000de3d5fa in DB::DistributedBlockOutputStream::writeToShard (this=this@entry=0x7f9ed39c7418, block=..., dir_names=...) at ../src/Storages/Distributed/DistributedBlockOutputStream.cpp:634 4 0x000000000de3e214 in DB::DistributedBlockOutputStream::writeAsyncImpl (this=this@entry=0x7f9ed39c7418, block=..., shard_id=shard_id@entry=79) at ../src/Storages/Distributed/DistributedBlockOutputStream.cpp:539 5 0x000000000de3e47b in DB::DistributedBlockOutputStream::writeSplitAsync (this=this@entry=0x7f9ed39c7418, block=...) at ../contrib/libcxx/include/vector:1546 6 0x000000000de3eab0 in DB::DistributedBlockOutputStream::writeAsync (block=..., this=0x7f9ed39c7418) at ../src/Storages/Distributed/DistributedBlockOutputStream.cpp:141 7 DB::DistributedBlockOutputStream::write (this=0x7f9ed39c7418, block=...) at ../src/Storages/Distributed/DistributedBlockOutputStream.cpp:135 8 0x000000000d73b376 in DB::PushingToViewsBlockOutputStream::write (this=this@entry=0x7f9ea7a8cf58, block=...) at ../src/DataStreams/PushingToViewsBlockOutputStream.cpp:157 9 0x000000000d7853eb in DB::AddingDefaultBlockOutputStream::write (this=0x7f9ed383d118, block=...) at ../contrib/libcxx/include/memory:3826 10 0x000000000d740790 in DB::SquashingBlockOutputStream::write (this=0x7f9ed383de18, block=...) at ../contrib/libcxx/include/memory:3826 11 0x000000000d68c308 in DB::CountingBlockOutputStream::write (this=0x7f9ea7ac6d60, block=...) at ../contrib/libcxx/include/memory:3826 12 0x000000000ddab449 in DB::StorageBuffer::writeBlockToDestination (this=this@entry=0x7f9fbd56a000, block=..., table=...) at ../src/Storages/StorageBuffer.cpp:747 13 0x000000000ddabfa6 in DB::StorageBuffer::flushBuffer (this=this@entry=0x7f9fbd56a000, buffer=..., check_thresholds=check_thresholds@entry=true, locked=locked@entry=false, reset_block_structure=reset_block_structure@entry=false) at ../src/Storages/StorageBuffer.cpp:661 14 0x000000000ddac415 in DB::StorageBuffer::flushAllBuffers (reset_blocks_structure=false, check_thresholds=true, this=0x7f9fbd56a000) at ../src/Storages/StorageBuffer.cpp:605 shutdown() on T1275: 0 DB::StorageDistributed::shutdown (this=0x7f9ed34f0000) at ../contrib/libcxx/include/atomic:1612 1 0x000000000d6fd938 in DB::InterpreterDropQuery::executeToTable (this=this@entry=0x7f98530c79a0, query=...) at ../src/Storages/TableLockHolder.h:12 2 0x000000000d6ff5ee in DB::InterpreterDropQuery::execute (this=0x7f98530c79a0) at ../src/Interpreters/InterpreterDropQuery.cpp:50 3 0x000000000daa40c0 in DB::executeQueryImpl (begin=, end=, context=..., internal=, stage=DB::QueryProcessingStage::Complete, has_query_tail=false, istr=0x0) at ../src/Interpreters/executeQuery.cpp:420 4 0x000000000daa59df in DB::executeQuery (query=..., context=..., internal=internal@entry=false, stage=, may_have_embedded_data=) at ../contrib/libcxx/include/string:1487 5 0x000000000e1369e6 in DB::TCPHandler::runImpl (this=this@entry=0x7f9ddf3a9000) at ../src/Server/TCPHandler.cpp:254 6 0x000000000e1379c9 in DB::TCPHandler::run (this=0x7f9ddf3a9000) at ../src/Server/TCPHandler.cpp:1326 7 0x000000001086fac7 in Poco::Net::TCPServerConnection::start (this=this@entry=0x7f9ddf3a9000) at ../contrib/poco/Net/src/TCPServerConnection.cpp:43 8 0x000000001086ff2b in Poco::Net::TCPServerDispatcher::run (this=0x7f9e4eba5c00) at ../contrib/poco/Net/src/TCPServerDispatcher.cpp:114 9 0x00000000109dbe8e in Poco::PooledThread::run (this=0x7f9e4a2d2f80) at ../contrib/poco/Foundation/src/ThreadPool.cpp:199 10 0x00000000109d78f9 in Poco::ThreadImpl::runnableEntry (pThread=) at ../contrib/poco/Foundation/include/Poco/SharedPtr.h:401 11 0x00007f9fc3cccea7 in start_thread (arg=) at pthread_create.c:477 12 0x00007f9fc3bebeaf in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 ```
--- src/Storages/StorageDistributed.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 9046940b3f7..0c1561fca9b 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -80,7 +80,6 @@ namespace ErrorCodes extern const int TYPE_MISMATCH; extern const int TOO_MANY_ROWS; extern const int UNABLE_TO_SKIP_UNUSED_SHARDS; - extern const int LOGICAL_ERROR; } namespace ActionLocks @@ -600,15 +599,22 @@ void StorageDistributed::shutdown() monitors_blocker.cancelForever(); std::lock_guard lock(cluster_nodes_mutex); + + LOG_DEBUG(log, "Joining background threads for async INSERT"); cluster_nodes_data.clear(); + LOG_DEBUG(log, "Background threads for async INSERT joined"); } void StorageDistributed::drop() { - // shutdown() should be already called - // and by the same reason we cannot use truncate() here, since - // cluster_nodes_data already cleaned - if (!cluster_nodes_data.empty()) - throw Exception("drop called before shutdown", ErrorCodes::LOGICAL_ERROR); + // Some INSERT in-between shutdown() and drop() can call + // requireDirectoryMonitor() again, so call shutdown() to clear them, but + // when the drop() (this function) executed none of INSERT is allowed in + // parallel. + // + // And second time shutdown() should be fast, since none of + // DirectoryMonitor should do anything, because ActionBlocker is canceled + // (in shutdown()). + shutdown(); // Distributed table w/o sharding_key does not allows INSERTs if (relative_data_path.empty()) From 2352c144e84791405de28b8f903114445140bbd7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Oct 2020 22:28:40 +0300 Subject: [PATCH 256/325] Update reference --- .../queries/0_stateless/00273_quantiles.reference | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/queries/0_stateless/00273_quantiles.reference b/tests/queries/0_stateless/00273_quantiles.reference index d75d11191f4..616e06841e4 100644 --- a/tests/queries/0_stateless/00273_quantiles.reference +++ b/tests/queries/0_stateless/00273_quantiles.reference @@ -6,17 +6,17 @@ [0,1,10,50,100,200,300,400,500,600,700,800,900,950,990,999,1000] [0,0.50100005,9.51,49.55,99.6,199.7,299.8,399.9,500,600.1,700.2,800.3,900.4,950.45,990.49,999.499,1000] [0,1,10,50,100,200,300,400,500,600,700,800,900,950,990,999,1000] -1 333334 [699140.3,835642,967430.8] [699999,833333,966666] +1 333334 [699144.2,835663,967429.2] [699999,833333,966666] 2 266667 [426549.5,536255.5,638957.6] [426665,533332,639999] -3 114285 [296938.5,342335,388777.5] [297142,342856,388570] +3 114285 [296938,342324,388778] [297142,342856,388570] 4 63492 [228370.2,254019.5,279351.4] [228571,253968,279364] 5 40404 [185603.4,202009,218107] [185858,202020,218181] -6 27972 [156598.6,167864,179118.40000000002] [156643,167832,179020] -7 20513 [135401,143553.5,151792.5] [135384,143589,151794] +6 27972 [156598.7,167866,179118.3] [156643,167832,179020] +7 20513 [135400.8,143550,151792.6] [135384,143589,151794] 8 15686 [119239.20000000001,125463,131772.40000000002] [119215,125490,131764] -9 12384 [106509.79999999999,111538,116415.8] [106501,111455,116408] -10 10025 [96223.6,100347,104288.6] [96240,100250,104260] -11 8282 [87732.70000000001,91035,94408.6] [87784,91097,94409] +9 12384 [106510.20000000001,111539,116415.7] [106501,111455,116408] +10 10025 [96223.2,100346,104288.7] [96240,100250,104260] +11 8282 [87732.8,91036,94410.20000000001] [87784,91097,94409] 12 6957 [80694.6,83477,86259.4] [80694,83477,86260] 13 5925 [74666.40000000001,77036,79405.6] [74666,77036,79406] 14 5109 [69475.8,71519,73562.2] [69475,71519,73563] From e1b8d0da7ba72a1f5bb2ef8ad528c8701c4824fd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 27 Oct 2020 22:41:36 +0300 Subject: [PATCH 257/325] Trailing whitespace --- utils/simple-backport/changelog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/simple-backport/changelog.sh b/utils/simple-backport/changelog.sh index 92957987cab..d3d9714cb04 100755 --- a/utils/simple-backport/changelog.sh +++ b/utils/simple-backport/changelog.sh @@ -66,7 +66,7 @@ do # Filter out PRs by bots. user_login=$(jq -r .user.login "$file") - + filter_bot=$(echo "$user_login" | grep -q "\[bot\]$" && echo "Skip." || echo "Ok." ||:) filter_robot=$(echo "$user_login" | grep -q "robot-clickhouse" && echo "Skip." || echo "Ok." ||:) From dba8c85ae4f8ebc0371eef8965e6e87c05cd284d Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 27 Oct 2020 22:56:54 +0300 Subject: [PATCH 258/325] do not wait while holding lock --- src/Databases/DatabaseLazy.cpp | 6 -- src/Databases/DatabaseLazy.h | 1 - src/Databases/IDatabase.h | 4 - .../MySQL/DatabaseMaterializeTablesIterator.h | 5 - src/Interpreters/InterpreterDropQuery.cpp | 100 +++++++++++++----- src/Interpreters/InterpreterDropQuery.h | 6 +- 6 files changed, 77 insertions(+), 45 deletions(-) diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index 81414902a33..0119f17f843 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -329,10 +329,4 @@ const StoragePtr & DatabaseLazyIterator::table() const return current_storage; } -void DatabaseLazyIterator::reset() -{ - if (current_storage) - current_storage.reset(); -} - } diff --git a/src/Databases/DatabaseLazy.h b/src/Databases/DatabaseLazy.h index 58e5e465eef..13c14863efb 100644 --- a/src/Databases/DatabaseLazy.h +++ b/src/Databases/DatabaseLazy.h @@ -122,7 +122,6 @@ public: bool isValid() const override; const String & name() const override; const StoragePtr & table() const override; - void reset() override; private: const DatabaseLazy & database; diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 9b744259406..b28bd5fd599 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -44,8 +44,6 @@ public: /// (a database with support for lazy tables loading /// - it maintains a list of tables but tables are loaded lazily). virtual const StoragePtr & table() const = 0; - /// Reset reference counter to the StoragePtr. - virtual void reset() = 0; virtual ~IDatabaseTablesIterator() = default; @@ -95,8 +93,6 @@ public: const String & name() const override { return it->first; } const StoragePtr & table() const override { return it->second; } - - void reset() override { it->second.reset(); } }; /// Copies list of dictionaries and iterates through such snapshot. diff --git a/src/Databases/MySQL/DatabaseMaterializeTablesIterator.h b/src/Databases/MySQL/DatabaseMaterializeTablesIterator.h index 5a0ec242c2f..86a5cbf8206 100644 --- a/src/Databases/MySQL/DatabaseMaterializeTablesIterator.h +++ b/src/Databases/MySQL/DatabaseMaterializeTablesIterator.h @@ -28,11 +28,6 @@ public: return tables.emplace_back(storage); } - void reset() override - { - tables.clear(); - } - UUID uuid() const override { return nested_iterator->uuid(); } DatabaseMaterializeTablesIterator(DatabaseTablesIteratorPtr nested_iterator_, DatabaseMaterializeMySQL * database_) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index c4ebe596649..8bf8675b15d 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -52,13 +52,37 @@ BlockIO InterpreterDropQuery::execute() return executeToDictionary(drop.database, drop.table, drop.kind, drop.if_exists, drop.temporary, drop.no_ddl_lock); } else if (!drop.database.empty()) - return executeToDatabase(drop.database, drop.kind, drop.if_exists, drop.no_delay); + return executeToDatabase(drop); else throw Exception("Nothing to drop, both names are empty", ErrorCodes::LOGICAL_ERROR); } +void InterpreterDropQuery::waitForTableToBeActuallyDroppedOrDetached(const ASTDropQuery & query, const DatabasePtr & db, const UUID & uuid_to_wait) +{ + if (uuid_to_wait == UUIDHelpers::Nil) + return; + + if (query.kind == ASTDropQuery::Kind::Drop) + DatabaseCatalog::instance().waitTableFinallyDropped(uuid_to_wait); + else if (query.kind == ASTDropQuery::Kind::Detach) + { + if (auto * atomic = typeid_cast(db.get())) + atomic->waitDetachedTableNotInUse(uuid_to_wait); + } +} + BlockIO InterpreterDropQuery::executeToTable(const ASTDropQuery & query) +{ + DatabasePtr database; + UUID table_to_wait_on = UUIDHelpers::Nil; + auto res = executeToTableImpl(query, database, table_to_wait_on); + if (query.no_delay) + waitForTableToBeActuallyDroppedOrDetached(query, database, table_to_wait_on); + return res; +} + +BlockIO InterpreterDropQuery::executeToTableImpl(const ASTDropQuery & query, DatabasePtr & db, UUID & uuid_to_wait) { /// NOTE: it does not contain UUID, we will resolve it with locked DDLGuard auto table_id = StorageID(query); @@ -125,19 +149,9 @@ BlockIO InterpreterDropQuery::executeToTable(const ASTDropQuery & query) database->dropTable(context, table_id.table_name, query.no_delay); } - } - table.reset(); - ddl_guard = {}; - if (query.no_delay) - { - if (query.kind == ASTDropQuery::Kind::Drop) - DatabaseCatalog::instance().waitTableFinallyDropped(table_id.uuid); - else if (query.kind == ASTDropQuery::Kind::Detach) - { - if (auto * atomic = typeid_cast(database.get())) - atomic->waitDetachedTableNotInUse(table_id.uuid); - } + db = database; + uuid_to_wait = table_id.uuid; } return {}; @@ -223,19 +237,48 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, } -BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists, bool no_delay) +BlockIO InterpreterDropQuery::executeToDatabase(const ASTDropQuery & query) { + DatabasePtr database; + std::vector tables_to_wait; + BlockIO res; + try + { + res = executeToDatabaseImpl(query, database, tables_to_wait); + } + catch (...) + { + if (query.no_delay) + { + for (const auto table_uuid : tables_to_wait) + waitForTableToBeActuallyDroppedOrDetached(query, database, table_uuid); + } + throw; + } + + if (query.no_delay) + { + for (const auto table_uuid : tables_to_wait) + waitForTableToBeActuallyDroppedOrDetached(query, database, table_uuid); + } + return res; +} + +BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query, DatabasePtr & database, std::vector & uuids_to_wait) +{ + const auto & database_name = query.database; auto ddl_guard = DatabaseCatalog::instance().getDDLGuard(database_name, ""); - if (auto database = tryGetDatabase(database_name, if_exists)) + database = tryGetDatabase(database_name, query.if_exists); + if (database) { - if (kind == ASTDropQuery::Kind::Truncate) + if (query.kind == ASTDropQuery::Kind::Truncate) { throw Exception("Unable to truncate database", ErrorCodes::SYNTAX_ERROR); } - else if (kind == ASTDropQuery::Kind::Detach || kind == ASTDropQuery::Kind::Drop) + else if (query.kind == ASTDropQuery::Kind::Detach || query.kind == ASTDropQuery::Kind::Drop) { - bool drop = kind == ASTDropQuery::Kind::Drop; + bool drop = query.kind == ASTDropQuery::Kind::Drop; context.checkAccess(AccessType::DROP_DATABASE, database_name); if (database->shouldBeEmptyOnDetach()) @@ -246,21 +289,22 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS for (auto iterator = database->getDictionariesIterator(); iterator->isValid(); iterator->next()) { String current_dictionary = iterator->name(); - executeToDictionary(database_name, current_dictionary, kind, false, false, false); + executeToDictionary(database_name, current_dictionary, query.kind, false, false, false); } - ASTDropQuery query; - query.kind = kind; - query.if_exists = true; - query.database = database_name; - query.no_delay = no_delay; + ASTDropQuery query_for_table; + query_for_table.kind = query.kind; + query_for_table.if_exists = true; + query_for_table.database = database_name; + query_for_table.no_delay = query.no_delay; for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next()) { - /// Reset reference counter of the StoragePtr to allow synchronous drop. - iterator->reset(); - query.table = iterator->name(); - executeToTable(query); + DatabasePtr db; + UUID table_to_wait = UUIDHelpers::Nil; + query_for_table.table = iterator->name(); + executeToTableImpl(query_for_table, db, table_to_wait); + uuids_to_wait.push_back(table_to_wait); } } diff --git a/src/Interpreters/InterpreterDropQuery.h b/src/Interpreters/InterpreterDropQuery.h index 0e1fd47b079..1fc4acffa04 100644 --- a/src/Interpreters/InterpreterDropQuery.h +++ b/src/Interpreters/InterpreterDropQuery.h @@ -29,9 +29,13 @@ private: ASTPtr query_ptr; Context & context; - BlockIO executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists, bool no_delay); + BlockIO executeToDatabase(const ASTDropQuery & query); + BlockIO executeToDatabaseImpl(const ASTDropQuery & query, DatabasePtr & db, std::vector & uuids_to_wait); BlockIO executeToTable(const ASTDropQuery & query); + BlockIO executeToTableImpl(const ASTDropQuery & query, DatabasePtr & db, UUID & uuid_to_wait); + + static void waitForTableToBeActuallyDroppedOrDetached(const ASTDropQuery & query, const DatabasePtr & db, const UUID & uuid_to_wait); BlockIO executeToDictionary(const String & database_name, const String & dictionary_name, ASTDropQuery::Kind kind, bool if_exists, bool is_temporary, bool no_ddl_lock); From 4b59882c30674cb1e7a39523db83970fa908840e Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 27 Oct 2020 20:28:52 +0000 Subject: [PATCH 259/325] Remove redundant, add virt col --- .../table-engines/integrations/rabbitmq.md | 3 ++- .../table-engines/integrations/rabbitmq.md | 3 ++- .../RabbitMQ/RabbitMQBlockInputStream.cpp | 4 +++- .../ReadBufferFromRabbitMQConsumer.cpp | 18 ++++-------------- .../RabbitMQ/ReadBufferFromRabbitMQConsumer.h | 12 ++---------- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 8 ++++---- 6 files changed, 17 insertions(+), 31 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/rabbitmq.md b/docs/en/engines/table-engines/integrations/rabbitmq.md index 0340603eaae..b0901ee6f6e 100644 --- a/docs/en/engines/table-engines/integrations/rabbitmq.md +++ b/docs/en/engines/table-engines/integrations/rabbitmq.md @@ -148,4 +148,5 @@ Example: - `_channel_id` - ChannelID, on which consumer, who received the message, was declared. - `_delivery_tag` - DeliveryTag of the received message. Scoped per channel. - `_redelivered` - `redelivered` flag of the message. -- `_message_id` - MessageID of the received message; non-empty if was set, when message was published. +- `_message_id` - messageID of the received message; non-empty if was set, when message was published. +- `_timestamp` - timestamp of the received message; non-empty if was set, when message was published. diff --git a/docs/ru/engines/table-engines/integrations/rabbitmq.md b/docs/ru/engines/table-engines/integrations/rabbitmq.md index 75f20827df2..dedb5842d68 100644 --- a/docs/ru/engines/table-engines/integrations/rabbitmq.md +++ b/docs/ru/engines/table-engines/integrations/rabbitmq.md @@ -140,4 +140,5 @@ Example: - `_channel_id` - идентификатор канала `ChannelID`, на котором было получено сообщение. - `_delivery_tag` - значение `DeliveryTag` полученного сообщения. Уникально в рамках одного канала. - `_redelivered` - флаг `redelivered`. (Не равно нулю, если есть возможность, что сообщение было получено более, чем одним каналом.) -- `_message_id` - значение `MessageID` полученного сообщения. Данное поле непусто, если указано в параметрах при отправке сообщения. +- `_message_id` - значение поля `messageID` полученного сообщения. Данное поле непусто, если указано в параметрах при отправке сообщения. +- `_timestamp` - значение поля `timestamp` полученного сообщения. Данное поле непусто, если указано в параметрах при отправке сообщения. diff --git a/src/Storages/RabbitMQ/RabbitMQBlockInputStream.cpp b/src/Storages/RabbitMQ/RabbitMQBlockInputStream.cpp index c74081d8802..830c6224b9e 100644 --- a/src/Storages/RabbitMQ/RabbitMQBlockInputStream.cpp +++ b/src/Storages/RabbitMQ/RabbitMQBlockInputStream.cpp @@ -27,7 +27,7 @@ RabbitMQBlockInputStream::RabbitMQBlockInputStream( , non_virtual_header(metadata_snapshot->getSampleBlockNonMaterialized()) , sample_block(non_virtual_header) , virtual_header(metadata_snapshot->getSampleBlockForColumns( - {"_exchange_name", "_channel_id", "_delivery_tag", "_redelivered", "_message_id"}, + {"_exchange_name", "_channel_id", "_delivery_tag", "_redelivered", "_message_id", "_timestamp"}, storage.getVirtuals(), storage.getStorageID())) { for (const auto & column : virtual_header) @@ -158,6 +158,7 @@ Block RabbitMQBlockInputStream::readImpl() auto delivery_tag = buffer->getDeliveryTag(); auto redelivered = buffer->getRedelivered(); auto message_id = buffer->getMessageID(); + auto timestamp = buffer->getTimestamp(); buffer->updateAckTracker({delivery_tag, channel_id}); @@ -168,6 +169,7 @@ Block RabbitMQBlockInputStream::readImpl() virtual_columns[2]->insert(delivery_tag); virtual_columns[3]->insert(redelivered); virtual_columns[4]->insert(message_id); + virtual_columns[5]->insert(timestamp); } total_rows = total_rows + new_rows; diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp index 8c7b435cd2a..661e645cc0c 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.cpp @@ -17,32 +17,23 @@ namespace DB ReadBufferFromRabbitMQConsumer::ReadBufferFromRabbitMQConsumer( ChannelPtr consumer_channel_, HandlerPtr event_handler_, - const String & exchange_name_, std::vector & queues_, size_t channel_id_base_, const String & channel_base_, - const String & queue_base_, Poco::Logger * log_, char row_delimiter_, - size_t num_queues_, - const String & deadletter_exchange_, uint32_t queue_size_, const std::atomic & stopped_) : ReadBuffer(nullptr, 0) , consumer_channel(std::move(consumer_channel_)) , event_handler(event_handler_) - , exchange_name(exchange_name_) , queues(queues_) , channel_base(channel_base_) , channel_id_base(channel_id_base_) - , queue_base(queue_base_) - , num_queues(num_queues_) - , deadletter_exchange(deadletter_exchange_) , log(log_) , row_delimiter(row_delimiter_) - , queue_size(queue_size_) , stopped(stopped_) - , received(queue_size * num_queues) + , received(queue_size_) { setupChannel(); } @@ -74,10 +65,9 @@ void ReadBufferFromRabbitMQConsumer::subscribe() if (row_delimiter != '\0') message_received += row_delimiter; - if (message.hasMessageID()) - received.push({message_received, message.messageID(), redelivered, AckTracker(delivery_tag, channel_id)}); - else - received.push({message_received, "", redelivered, AckTracker(delivery_tag, channel_id)}); + received.push({message_received, message.hasMessageID() ? message.messageID() : "", + message.hasTimestamp() ? message.timestamp() : 0, + redelivered, AckTracker(delivery_tag, channel_id)}); } }) .onError([&](const char * message) diff --git a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h index 556e069083c..476db3f5e94 100644 --- a/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h +++ b/src/Storages/RabbitMQ/ReadBufferFromRabbitMQConsumer.h @@ -25,15 +25,11 @@ public: ReadBufferFromRabbitMQConsumer( ChannelPtr consumer_channel_, HandlerPtr event_handler_, - const String & exchange_name_, std::vector & queues_, size_t channel_id_base_, const String & channel_base_, - const String & queue_base_, Poco::Logger * log_, char row_delimiter_, - size_t num_queues_, - const String & deadletter_exchange_, uint32_t queue_size_, const std::atomic & stopped_); @@ -52,6 +48,7 @@ public: { String message; String message_id; + uint64_t timestamp; bool redelivered; AckTracker track; }; @@ -74,6 +71,7 @@ public: auto getDeliveryTag() const { return current.track.delivery_tag; } auto getRedelivered() const { return current.redelivered; } auto getMessageID() const { return current.message_id; } + auto getTimestamp() const { return current.timestamp; } private: bool nextImpl() override; @@ -83,18 +81,12 @@ private: ChannelPtr consumer_channel; HandlerPtr event_handler; - - const String exchange_name; std::vector queues; const String channel_base; const size_t channel_id_base; - const String queue_base; - const size_t num_queues; - const String deadletter_exchange; Poco::Logger * log; char row_delimiter; bool allowed = true; - uint32_t queue_size; const std::atomic & stopped; String channel_id; diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index e49b18ea391..c32590ca0ba 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -689,9 +689,8 @@ ConsumerBufferPtr StorageRabbitMQ::createReadBuffer() ChannelPtr consumer_channel = std::make_shared(connection.get()); return std::make_shared( - consumer_channel, event_handler, consumer_exchange, queues, ++consumer_id, - unique_strbase, queue_base, log, row_delimiter, num_queues, - deadletter_exchange, queue_size, stream_cancelled); + consumer_channel, event_handler, queues, ++consumer_id, + unique_strbase, log, row_delimiter, queue_size, stream_cancelled); } @@ -983,7 +982,8 @@ NamesAndTypesList StorageRabbitMQ::getVirtuals() const {"_channel_id", std::make_shared()}, {"_delivery_tag", std::make_shared()}, {"_redelivered", std::make_shared()}, - {"_message_id", std::make_shared()} + {"_message_id", std::make_shared()}, + {"_timestamp", std::make_shared()} }; } From b5ccb5ed5b47b43c87b54bfbeb6ac723c6d751e1 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 27 Oct 2020 23:52:49 +0300 Subject: [PATCH 260/325] review suggestions --- src/Databases/DatabaseAtomic.cpp | 11 +++++++++-- src/Interpreters/DatabaseCatalog.cpp | 16 +++++++++++++--- src/Interpreters/DatabaseCatalog.h | 8 +++++++- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index 6c1ca1e8fce..b50bdb8d894 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -261,22 +261,29 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora { DetachedTables not_in_use; auto table_data_path = getTableDataPath(query); + bool locked_uuid = false; try { std::unique_lock lock{mutex}; if (query.database != database_name) throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database was renamed to `{}`, cannot create table in `{}`", database_name, query.database); + /// Do some checks before renaming file from .tmp to .sql not_in_use = cleanupDetachedTables(); assertDetachedTableNotInUse(query.uuid); - DatabaseCatalog::instance().addUUIDMapping(query.uuid, {}, {}); - renameNoReplace(table_metadata_tmp_path, table_metadata_path); + /// We will get en exception if some table with the same UUID exists (even if it's detached table or table from another database) + DatabaseCatalog::instance().addUUIDMapping(query.uuid); + locked_uuid = true; + /// It throws if `table_metadata_path` already exists (it's possible if table was detached) + renameNoReplace(table_metadata_tmp_path, table_metadata_path); /// Commit point (a sort of) attachTableUnlocked(query.table, table, lock); /// Should never throw table_name_to_path.emplace(query.table, table_data_path); } catch (...) { Poco::File(table_metadata_tmp_path).remove(); + if (locked_uuid) + DatabaseCatalog::instance().removeUUIDMappingFinally(query.uuid); throw; } tryCreateSymlink(query.table, table_data_path); diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 90e1a32827f..906863f3f44 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -164,7 +164,12 @@ void DatabaseCatalog::shutdownImpl() std::lock_guard lock(databases_mutex); assert(std::find_if(uuid_map.begin(), uuid_map.end(), [](const auto & elem) { - const auto & not_empty_mapping = [] (const auto & mapping) { return mapping.second.second; }; + /// Ensure that all UUID mappings are emtpy (i.e. all mappings contain nullptr instead of a pointer to storage) + const auto & not_empty_mapping = [] (const auto & mapping) + { + auto & table = mapping.second.second; + return table; + }; auto it = std::find_if(elem.map.begin(), elem.map.end(), not_empty_mapping); return it != elem.map.end(); }) == uuid_map.end()); @@ -423,6 +428,11 @@ DatabasePtr DatabaseCatalog::getSystemDatabase() const return getDatabase(SYSTEM_DATABASE); } +void DatabaseCatalog::addUUIDMapping(const UUID & uuid) +{ + addUUIDMapping(uuid, nullptr, nullptr); +} + void DatabaseCatalog::addUUIDMapping(const UUID & uuid, const DatabasePtr & database, const StoragePtr & table) { assert(uuid != UUIDHelpers::Nil && getFirstLevelIdx(uuid) < uuid_map.size()); @@ -744,7 +754,7 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr LOG_WARNING(log, "Cannot parse metadata of partially dropped table {} from {}. Will remove metadata file and data directory. Garbage may be left in /store directory and ZooKeeper.", table_id.getNameForLogs(), dropped_metadata_path); } - addUUIDMapping(table_id.uuid, {}, {}); + addUUIDMapping(table_id.uuid); drop_time = Poco::File(dropped_metadata_path).getLastModified().epochTime(); } @@ -786,7 +796,7 @@ void DatabaseCatalog::dropTableDataTask() } else { - LOG_TRACE(log, "No tables to drop. Queue size: {}", tables_marked_dropped.size()); + LOG_TRACE(log, "Not found any suitable tables to drop, still have {} tables in drop queue", tables_marked_dropped.size()); } need_reschedule = !tables_marked_dropped.empty(); } diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 46646f2ff05..d26307a3bc3 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -167,13 +167,19 @@ public: /// If table has UUID, addUUIDMapping(...) must be called when table attached to some database /// removeUUIDMapping(...) must be called when it detached, /// and removeUUIDMappingFinally(...) must be called when table is dropped and its data removed from disk. - /// To "lock" some UUID and prevent collision, addUUIDMapping(...) may be called with nullptr arguments. /// Such tables can be accessed by persistent UUID instead of database and table name. void addUUIDMapping(const UUID & uuid, const DatabasePtr & database, const StoragePtr & table); void removeUUIDMapping(const UUID & uuid); void removeUUIDMappingFinally(const UUID & uuid); /// For moving table between databases void updateUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table); + /// This method adds empty mapping (with database and storage equal to nullptr). + /// It's required to "lock" some UUIDs and protect us from collision. + /// Collisions of random 122-bit integers are very unlikely to happen, + /// but we allow to explicitly specify UUID in CREATE query (in particular for testing). + /// If some UUID was already added and we are trying to add it again, + /// this method will throw an exception. + void addUUIDMapping(const UUID & uuid); static String getPathForUUID(const UUID & uuid); From 722e5fe73f947c6fca7f3e09f96060b677f2b670 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 28 Oct 2020 00:53:21 +0300 Subject: [PATCH 261/325] Rename check --- tests/ci/ci_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/ci_config.json b/tests/ci/ci_config.json index 418b8ce0356..4e5f36a5f5d 100644 --- a/tests/ci/ci_config.json +++ b/tests/ci/ci_config.json @@ -333,7 +333,7 @@ "with_coverage": false } }, - "Functional stateless tests (release, polymorphic parts enabled)": { + "Functional stateless tests (release, wide parts enabled)": { "required_build_properties": { "compiler": "clang-11", "package_type": "deb", From ff6499cdc48ecac7882612f9fb0341a562c73b77 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Tue, 27 Oct 2020 19:02:56 -0300 Subject: [PATCH 262/325] test for bug_13492 --- ...533_distinct_depends_on_max_threads.reference | 2 ++ .../01533_distinct_depends_on_max_threads.sql | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 tests/queries/0_stateless/01533_distinct_depends_on_max_threads.reference create mode 100644 tests/queries/0_stateless/01533_distinct_depends_on_max_threads.sql diff --git a/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.reference b/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.reference new file mode 100644 index 00000000000..6ed281c757a --- /dev/null +++ b/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.sql b/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.sql new file mode 100644 index 00000000000..4f32576e86b --- /dev/null +++ b/tests/queries/0_stateless/01533_distinct_depends_on_max_threads.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS bug_13492; + +CREATE TABLE bug_13492 (`d` DateTime) ENGINE = MergeTree +PARTITION BY toYYYYMMDD(d) ORDER BY tuple(); + +INSERT INTO bug_13492 SELECT addDays(now(), number) FROM numbers(100); + +SET max_threads = 5; + +SELECT DISTINCT 1 FROM bug_13492, numbers(1) n; + +SET max_threads = 2; + +SELECT DISTINCT 1 FROM bug_13492, numbers(1) n; + +DROP TABLE bug_13492; From 2953795330f40c5e0ef0b52078461526526a9183 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Tue, 27 Oct 2020 18:45:04 -0400 Subject: [PATCH 263/325] Fixing bug in LDAP add_config method where log size was grabbed before container was stopped. Updating ldap/external_user_directory/tests/authentications.py to reduce test time. --- tests/testflows/helpers/cluster.py | 28 ++ .../ldap/authentication/regression.py | 7 +- .../ldap/authentication/tests/common.py | 18 +- .../external_user_directory/regression.py | 7 +- .../tests/authentications.py | 372 ++++++++---------- 5 files changed, 218 insertions(+), 214 deletions(-) diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py index 27e06a7c17f..d173547a916 100755 --- a/tests/testflows/helpers/cluster.py +++ b/tests/testflows/helpers/cluster.py @@ -54,6 +54,34 @@ class ClickHouseNode(Node): continue assert False, "container is not healthy" + def stop(self, timeout=300, safe=True): + """Stop node. + """ + if safe: + self.query("SYSTEM STOP MOVES") + self.query("SYSTEM STOP MERGES") + self.query("SYSTEM FLUSH LOGS") + with By("waiting for 5 sec for moves and merges to stop"): + time.sleep(5) + with And("forcing to sync everything to disk"): + self.command("sync", timeout=30) + + with self.cluster.lock: + for key in list(self.cluster._bash.keys()): + if key.endswith(f"-{self.name}"): + shell = self.cluster._bash.pop(key) + shell.__exit__(None, None, None) + + self.cluster.command(None, f'{self.cluster.docker_compose} stop {self.name}', timeout=timeout) + + def start(self, timeout=300, wait_healthy=True): + """Start node. + """ + self.cluster.command(None, f'{self.cluster.docker_compose} start {self.name}', timeout=timeout) + + if wait_healthy: + self.wait_healthy(timeout) + def restart(self, timeout=300, safe=True, wait_healthy=True): """Restart node. """ diff --git a/tests/testflows/ldap/authentication/regression.py b/tests/testflows/ldap/authentication/regression.py index 9d0a5ca743f..ed75ce4fe75 100755 --- a/tests/testflows/ldap/authentication/regression.py +++ b/tests/testflows/ldap/authentication/regression.py @@ -33,7 +33,7 @@ xfails = { RQ_SRS_007_LDAP_Authentication("1.0") ) @XFails(xfails) -def regression(self, local, clickhouse_binary_path): +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """ClickHouse integration with LDAP regression module. """ nodes = { @@ -43,6 +43,11 @@ def regression(self, local, clickhouse_binary_path): with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: self.context.cluster = cluster + if stress is not None or not hasattr(self.context, "stress"): + self.context.stress = stress + if parallel is not None or not hasattr(self.context, "parallel"): + self.context.parallel = parallel + Scenario(run=load("ldap.authentication.tests.sanity", "scenario")) Scenario(run=load("ldap.authentication.tests.multiple_servers", "scenario")) Feature(run=load("ldap.authentication.tests.connections", "feature")) diff --git a/tests/testflows/ldap/authentication/tests/common.py b/tests/testflows/ldap/authentication/tests/common.py index 0fd9670fae0..ed8d46df92b 100644 --- a/tests/testflows/ldap/authentication/tests/common.py +++ b/tests/testflows/ldap/authentication/tests/common.py @@ -85,6 +85,8 @@ def add_config(config, timeout=60, restart=False): :param config: configuration file description :param timeout: timeout, default: 20 sec """ + node = current().context.node + def check_preprocessed_config_is_updated(after_removal=False): """Check that preprocessed config is updated. """ @@ -116,13 +118,16 @@ def add_config(config, timeout=60, restart=False): with When("I close terminal to the node to be restarted"): bash.close() - with And("I get the current log size"): - logsize = \ - node.command("stat --format=%s /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[ - 0].strip() + with And("I stop ClickHouse to apply the config changes"): + node.stop(safe=False) - with And("I restart ClickHouse to apply the config changes"): - node.restart(safe=False) + with And("I get the current log size"): + cmd = node.cluster.command(None, + f"stat --format=%s {os.environ['CLICKHOUSE_TESTS_DIR']}/_instances/{node.name}/logs/clickhouse-server.log") + logsize = cmd.output.split(" ")[0].strip() + + with And("I start ClickHouse back up"): + node.start() with Then("I tail the log file from using previous log size as the offset"): bash.prompt = bash.__class__.prompt @@ -139,7 +144,6 @@ def add_config(config, timeout=60, restart=False): f"ConfigReloader: Loaded config '/etc/clickhouse-server/{config.preprocessed_name}', performed update on configuration", timeout=timeout) - node = current().context.node try: with Given(f"{config.name}"): if settings.debug: diff --git a/tests/testflows/ldap/external_user_directory/regression.py b/tests/testflows/ldap/external_user_directory/regression.py index 6ce860a6fd2..bd404d54438 100755 --- a/tests/testflows/ldap/external_user_directory/regression.py +++ b/tests/testflows/ldap/external_user_directory/regression.py @@ -33,7 +33,7 @@ xfails = { RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication("1.0") ) @XFails(xfails) -def regression(self, local, clickhouse_binary_path): +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """ClickHouse LDAP external user directory regression module. """ nodes = { @@ -42,6 +42,11 @@ def regression(self, local, clickhouse_binary_path): with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: self.context.cluster = cluster + + if stress is not None or not hasattr(self.context, "stress"): + self.context.stress = stress + if parallel is not None or not hasattr(self.context, "parallel"): + self.context.parallel = parallel Scenario(run=load("ldap.authentication.tests.sanity", "scenario")) Scenario(run=load("ldap.external_user_directory.tests.simple", "scenario")) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index 9b216e7dd30..47c10121b68 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -92,25 +92,23 @@ def parallel_login(self, server, user_count=10, timeout=200): with Given("a group of LDAP users"): users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*users): - tasks = [] - try: - with When("users try to login in parallel", description=""" - * with valid username and password - * with invalid username and valid password - * with valid username and invalid password - """): - p = Pool(15) - for i in range(25): - tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + with ldap_users(*users): + tasks = [] + try: + with When("users try to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) - finally: - with Then("it should work"): - join(tasks, timeout) + finally: + with Then("it should work"): + join(tasks, timeout) @TestScenario @Requirements( @@ -127,25 +125,23 @@ def parallel_login_with_the_same_user(self, server, timeout=200): with Given("only one LDAP user"): users = [{"cn": f"parallel_user1", "userpassword": randomword(20)}] - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*users): - tasks = [] - try: - with When("the same user tries to login in parallel", description=""" - * with valid username and password - * with invalid username and valid password - * with valid username and invalid password - """): - p = Pool(15) - for i in range(25): - tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + with ldap_users(*users): + tasks = [] + try: + with When("the same user tries to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) - finally: - with Then("it should work"): - join(tasks, timeout) + finally: + with Then("it should work"): + join(tasks, timeout) @TestScenario def login_after_ldap_external_user_directory_is_removed(self, server): @@ -162,6 +158,7 @@ def login_after_ldap_external_user_directory_is_removed(self, server): login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) @TestScenario +@Tags("custom config") @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_SameUser("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") @@ -204,6 +201,7 @@ def parallel_login_with_the_same_user_multiple_servers(self, server, timeout=200 join(tasks, timeout) @TestScenario +@Tags("custom config") @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_MultipleServers("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") @@ -256,6 +254,7 @@ def parallel_login_with_multiple_servers(self, server, user_count=10, timeout=20 join(tasks, timeout) @TestScenario +@Tags("custom config") @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalAndMultipleLDAP("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") @@ -323,20 +322,18 @@ def parallel_login_with_rbac_users(self, server, user_count=10, timeout=200): users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with rbac_users(*users): - tasks = [] - try: - with When("I login in parallel"): - p = Pool(15) - for i in range(25): - tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) - finally: - with Then("it should work"): - join(tasks, timeout) + with rbac_users(*users): + tasks = [] + try: + with When("I login in parallel"): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + finally: + with Then("it should work"): + join(tasks, timeout) @TestScenario @Requirements( @@ -347,10 +344,8 @@ def login_after_user_is_added_to_ldap(self, server): """ user = {"cn": "myuser", "userpassword": "myuser"} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with When(f"I add user to LDAP and try to login"): - add_user_to_ldap_and_login(user=user, server=server) + with When(f"I add user to LDAP and try to login"): + add_user_to_ldap_and_login(user=user, server=server) @TestScenario @Requirements( @@ -363,27 +358,25 @@ def login_after_user_is_deleted_from_ldap(self, server): self.context.ldap_node = self.context.cluster.node(server) user = None - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": "myuser"} - user = add_user_to_ldap(**user) + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with When("I delete this user from LDAP"): - delete_user_from_ldap(user) + with When("I delete this user from LDAP"): + delete_user_from_ldap(user) - with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - ) - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) @TestScenario @Requirements( @@ -396,31 +389,29 @@ def login_after_user_password_changed_in_ldap(self, server): self.context.ldap_node = self.context.cluster.node(server) user = None - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": "myuser"} - user = add_user_to_ldap(**user) + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with When("I change user password in LDAP"): - change_user_password_in_ldap(user, "newpassword") + with When("I change user password in LDAP"): + change_user_password_in_ldap(user, "newpassword") - with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - ) + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) - with And("when I try to login with the new password it should work"): - login_and_execute_query(username=user["cn"], password="newpassword") + with And("when I try to login with the new password it should work"): + login_and_execute_query(username=user["cn"], password="newpassword") - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) @TestScenario @Requirements( @@ -434,27 +425,25 @@ def login_after_user_cn_changed_in_ldap(self, server): user = None new_user = None - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": "myuser"} - user = add_user_to_ldap(**user) + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with When("I change user password in LDAP"): - new_user = change_user_cn_in_ldap(user, "myuser2") + with When("I change user password in LDAP"): + new_user = change_user_cn_in_ldap(user, "myuser2") - with Then("when I try to login again it should fail"): - login_and_execute_query(username=user["cn"], password=user["userpassword"], - exitcode=4, - message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - ) - finally: - with Finally("I make sure LDAP user is deleted"): - if new_user is not None: - delete_user_from_ldap(new_user, exitcode=None) + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) + finally: + with Finally("I make sure LDAP user is deleted"): + if new_user is not None: + delete_user_from_ldap(new_user, exitcode=None) @TestScenario @Requirements( @@ -467,31 +456,29 @@ def login_after_ldap_server_is_restarted(self, server, timeout=60): self.context.ldap_node = self.context.cluster.node(server) user = None - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": getuid()} - user = add_user_to_ldap(**user) + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": getuid()} + user = add_user_to_ldap(**user) - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with When("I restart LDAP server"): - self.context.ldap_node.restart() + with When("I restart LDAP server"): + self.context.ldap_node.restart() - with Then("I try to login until it works", description=f"timeout {timeout} sec"): - started = time.time() - while True: - r = self.context.node.query("SELECT 1", - settings=[("user", user["cn"]), ("password", user["userpassword"])], - no_checks=True) - if r.exitcode == 0: - break - assert time.time() - started < timeout, error(r.output) - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) + with Then("I try to login until it works", description=f"timeout {timeout} sec"): + started = time.time() + while True: + r = self.context.node.query("SELECT 1", + settings=[("user", user["cn"]), ("password", user["userpassword"])], + no_checks=True) + if r.exitcode == 0: + break + assert time.time() - started < timeout, error(r.output) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) @TestScenario @Requirements( @@ -504,31 +491,29 @@ def login_after_clickhouse_server_is_restarted(self, server, timeout=60): self.context.ldap_node = self.context.cluster.node(server) user = None - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - try: - with Given(f"I add user to LDAP"): - user = {"cn": "myuser", "userpassword": getuid()} - user = add_user_to_ldap(**user) + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": getuid()} + user = add_user_to_ldap(**user) - login_and_execute_query(username=user["cn"], password=user["userpassword"]) + login_and_execute_query(username=user["cn"], password=user["userpassword"]) - with When("I restart ClickHouse server"): - self.context.node.restart() + with When("I restart ClickHouse server"): + self.context.node.restart() - with Then("I try to login until it works", description=f"timeout {timeout} sec"): - started = time.time() - while True: - r = self.context.node.query("SELECT 1", - settings=[("user", user["cn"]), ("password", user["userpassword"])], - no_checks=True) - if r.exitcode == 0: - break - assert time.time() - started < timeout, error(r.output) - finally: - with Finally("I make sure LDAP user is deleted"): - if user is not None: - delete_user_from_ldap(user, exitcode=None) + with Then("I try to login until it works", description=f"timeout {timeout} sec"): + started = time.time() + while True: + r = self.context.node.query("SELECT 1", + settings=[("user", user["cn"]), ("password", user["userpassword"])], + no_checks=True) + if r.exitcode == 0: + break + assert time.time() - started < timeout, error(r.output) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) @TestScenario @Requirements( @@ -542,9 +527,7 @@ def valid_username_with_valid_empty_password(self, server): exitcode = 4 message = f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -561,9 +544,7 @@ def valid_username_and_invalid_empty_password(self, server): exitcode = 4 message = f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -575,10 +556,8 @@ def valid_username_and_password(self, server): username = "valid_username_and_password" user = {"cn": username, "userpassword": username} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with When(f"I add user {username} to LDAP and try to login"): - add_user_to_ldap_and_login(user=user, server=server) + with When(f"I add user {username} to LDAP and try to login"): + add_user_to_ldap_and_login(user=user, server=server) @TestScenario @Requirements( @@ -593,9 +572,7 @@ def valid_username_and_password_invalid_server(self, server=None): exitcode = 4 message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) + login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) @TestScenario @Requirements( @@ -608,9 +585,7 @@ def valid_long_username_and_short_password(self, server): username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890" user = {"cn": username, "userpassword": "long_username"} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, server=server) + add_user_to_ldap_and_login(user=user, server=server) @TestScenario @Requirements( @@ -626,9 +601,7 @@ def invalid_long_username_and_valid_short_password(self, server): exitcode = 4 message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -641,9 +614,7 @@ def valid_short_username_and_long_password(self, server): username = "long_password" user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, server=server) + add_user_to_ldap_and_login(user=user, server=server) @TestScenario @Requirements( @@ -659,9 +630,7 @@ def valid_short_username_and_invalid_long_password(self, server): exitcode = 4 message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -677,9 +646,7 @@ def valid_username_and_invalid_password(self, server): exitcode = 4 message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -695,9 +662,7 @@ def invalid_username_and_valid_password(self, server): exitcode = 4 message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) @TestScenario @Requirements( @@ -710,9 +675,7 @@ def valid_utf8_username_and_ascii_password(self, server): username = "utf8_username_Gãńdåłf_Thê_Gręât" user = {"cn": username, "userpassword": "utf8_username"} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, server=server) + add_user_to_ldap_and_login(user=user, server=server) @TestScenario @Requirements( @@ -725,18 +688,14 @@ def valid_ascii_username_and_utf8_password(self, server): username = "utf8_password" user = {"cn": username, "userpassword": "utf8_password_Gãńdåłf_Thê_Gręât"} - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - add_user_to_ldap_and_login(user=user, server=server) + add_user_to_ldap_and_login(user=user, server=server) @TestScenario def empty_username_and_empty_password(self, server=None): """Check that we can login using empty username and empty password as it will use the default user and that has an empty password. """ - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - login_and_execute_query(username="", password="") + login_and_execute_query(username="", password="") @TestScenario @Requirements( @@ -763,18 +722,16 @@ def user_lookup_priority(self, server): "ldap": {"username": "ldap", "password": "userldap"} } - with rbac_roles("ldap_role") as roles: - with ldap_external_user_directory(server=server, roles=roles, restart=True): - with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users.values()]): - with rbac_users({"cn": "local", "userpassword": "local"}): - with When("I try to login as 'default' user which is also defined in users.xml it should fail"): - login_and_execute_query(**users["default"], exitcode=exitcode, message=message.format(username="default")) + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users.values()]): + with rbac_users({"cn": "local", "userpassword": "local"}): + with When("I try to login as 'default' user which is also defined in users.xml it should fail"): + login_and_execute_query(**users["default"], exitcode=exitcode, message=message.format(username="default")) - with When("I try to login as 'local' user which is also defined in local storage it should fail"): - login_and_execute_query(**users["local"], exitcode=exitcode, message=message.format(username="local")) + with When("I try to login as 'local' user which is also defined in local storage it should fail"): + login_and_execute_query(**users["local"], exitcode=exitcode, message=message.format(username="local")) - with When("I try to login as 'ldap' user defined only in LDAP it should work"): - login_and_execute_query(**users["ldap"]) + with When("I try to login as 'ldap' user defined only in LDAP it should work"): + login_and_execute_query(**users["ldap"]) @TestOutline(Feature) @@ -795,5 +752,10 @@ def feature(self, servers=None, server=None, node="clickhouse1"): server = "openldap1" with ldap_servers(servers): - for scenario in loads(current_module(), Scenario): + with rbac_roles("ldap_role") as roles: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + for scenario in loads(current_module(), Scenario, filter=~has.tag("custom config")): + Scenario(test=scenario, flags=TE)(server=server) + + for scenario in loads(current_module(), Scenario, filter=has.tag("custom config")): Scenario(test=scenario, flags=TE)(server=server) From d4e640daaeda7824dbda3182367b4bc56655ab80 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Tue, 27 Oct 2020 18:50:59 -0400 Subject: [PATCH 264/325] Updating TestFlows to 1.6.59 Updating AES encryption requirements.py to include Specification Updating AES regression.py to link Specification Fixing typos in LDAP authentication SRS --- docker/test/testflows/runner/Dockerfile | 2 +- tests/testflows/aes_encryption/regression.py | 1 + .../requirements/requirements.py | 2980 ++++++++++++++--- .../requirements/requirements.md | 4 +- .../requirements/requirements.py | 827 ++++- 5 files changed, 3161 insertions(+), 653 deletions(-) diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index 9565e39598c..06241d6d497 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN pip3 install urllib3 testflows==1.6.57 docker-compose docker dicttoxml kazoo tzlocal +RUN pip3 install urllib3 testflows==1.6.59 docker-compose docker dicttoxml kazoo tzlocal ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 17.09.1-ce diff --git a/tests/testflows/aes_encryption/regression.py b/tests/testflows/aes_encryption/regression.py index 6c4598dae28..06610f6e42b 100755 --- a/tests/testflows/aes_encryption/regression.py +++ b/tests/testflows/aes_encryption/regression.py @@ -48,6 +48,7 @@ xfails = { @TestFeature @Name("aes encryption") @ArgumentParser(argparser) +@Specifications(SRS_008_ClickHouse_AES_Encryption_Functions) @Requirements( RQ_SRS008_AES_Functions("1.0"), RQ_SRS008_AES_Functions_DifferentModes("1.0") diff --git a/tests/testflows/aes_encryption/requirements/requirements.py b/tests/testflows/aes_encryption/requirements/requirements.py index bae8b5cc3c1..d92e159547d 100644 --- a/tests/testflows/aes_encryption/requirements/requirements.py +++ b/tests/testflows/aes_encryption/requirements/requirements.py @@ -1,10 +1,1956 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.200731.1222107. +# document by TestFlows v1.6.201026.1232822. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. +from testflows.core import Specification from testflows.core import Requirement +SRS_008_ClickHouse_AES_Encryption_Functions = Specification( + name='SRS-008 ClickHouse AES Encryption Functions', + description=None, + author=None, + date=None, + status=None, + approved_by=None, + approved_date=None, + approved_version=None, + version=None, + group=None, + type=None, + link=None, + uid=None, + parent=None, + children=None, + content=''' +# SRS-008 ClickHouse AES Encryption Functions +# Software Requirements Specification + +## Table of Contents +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Terminology](#terminology) +* 4 [Requirements](#requirements) + * 4.1 [Generic](#generic) + * 4.1.1 [RQ.SRS008.AES.Functions](#rqsrs008aesfunctions) + * 4.1.2 [RQ.SRS008.AES.Functions.Compatability.MySQL](#rqsrs008aesfunctionscompatabilitymysql) + * 4.1.3 [RQ.SRS008.AES.Functions.Compatability.Dictionaries](#rqsrs008aesfunctionscompatabilitydictionaries) + * 4.1.4 [RQ.SRS008.AES.Functions.Compatability.Engine.Database.MySQL](#rqsrs008aesfunctionscompatabilityenginedatabasemysql) + * 4.1.5 [RQ.SRS008.AES.Functions.Compatability.Engine.Table.MySQL](#rqsrs008aesfunctionscompatabilityenginetablemysql) + * 4.1.6 [RQ.SRS008.AES.Functions.Compatability.TableFunction.MySQL](#rqsrs008aesfunctionscompatabilitytablefunctionmysql) + * 4.1.7 [RQ.SRS008.AES.Functions.DifferentModes](#rqsrs008aesfunctionsdifferentmodes) + * 4.1.8 [RQ.SRS008.AES.Functions.DataFromMultipleSources](#rqsrs008aesfunctionsdatafrommultiplesources) + * 4.1.9 [RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues](#rqsrs008aesfunctionssuppressoutputofsensitivevalues) + * 4.1.10 [RQ.SRS008.AES.Functions.InvalidParameters](#rqsrs008aesfunctionsinvalidparameters) + * 4.1.11 [RQ.SRS008.AES.Functions.MismatchedKey](#rqsrs008aesfunctionsmismatchedkey) + * 4.1.12 [RQ.SRS008.AES.Functions.Check.Performance](#rqsrs008aesfunctionscheckperformance) + * 4.1.13 [RQ.SRS008.AES.Function.Check.Performance.BestCase](#rqsrs008aesfunctioncheckperformancebestcase) + * 4.1.14 [RQ.SRS008.AES.Function.Check.Performance.WorstCase](#rqsrs008aesfunctioncheckperformanceworstcase) + * 4.1.15 [RQ.SRS008.AES.Functions.Check.Compression](#rqsrs008aesfunctionscheckcompression) + * 4.1.16 [RQ.SRS008.AES.Functions.Check.Compression.LowCardinality](#rqsrs008aesfunctionscheckcompressionlowcardinality) + * 4.2 [Specific](#specific) + * 4.2.1 [RQ.SRS008.AES.Encrypt.Function](#rqsrs008aesencryptfunction) + * 4.2.2 [RQ.SRS008.AES.Encrypt.Function.Syntax](#rqsrs008aesencryptfunctionsyntax) + * 4.2.3 [RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors](#rqsrs008aesencryptfunctionnisttestvectors) + * 4.2.4 [RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText](#rqsrs008aesencryptfunctionparametersplaintext) + * 4.2.5 [RQ.SRS008.AES.Encrypt.Function.Parameters.Key](#rqsrs008aesencryptfunctionparameterskey) + * 4.2.6 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode](#rqsrs008aesencryptfunctionparametersmode) + * 4.2.7 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesencryptfunctionparametersmodevaluesformat) + * 4.2.8 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesencryptfunctionparametersmodevalueinvalid) + * 4.2.9 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-ecb) + * 4.2.10 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-ecb) + * 4.2.11 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-ecb) + * 4.2.12 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cbc) + * 4.2.13 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cbc) + * 4.2.14 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cbc) + * 4.2.15 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cfb1) + * 4.2.16 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cfb1) + * 4.2.17 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cfb1) + * 4.2.18 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cfb8) + * 4.2.19 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cfb8) + * 4.2.20 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cfb8) + * 4.2.21 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-cfb128) + * 4.2.22 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-cfb128) + * 4.2.23 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-cfb128) + * 4.2.24 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-ofb) + * 4.2.25 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-ofb) + * 4.2.26 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-ofb) + * 4.2.27 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-gcm) + * 4.2.28 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-gcm) + * 4.2.29 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-gcm) + * 4.2.30 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR](#rqsrs008aesencryptfunctionparametersmodevalueaes-128-ctr) + * 4.2.31 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR](#rqsrs008aesencryptfunctionparametersmodevalueaes-192-ctr) + * 4.2.32 [RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR](#rqsrs008aesencryptfunctionparametersmodevalueaes-256-ctr) + * 4.2.33 [RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector](#rqsrs008aesencryptfunctionparametersinitializationvector) + * 4.2.34 [RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData](#rqsrs008aesencryptfunctionparametersadditionalauthenticateddata) + * 4.2.35 [RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue](#rqsrs008aesencryptfunctionparametersreturnvalue) + * 4.2.36 [RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError](#rqsrs008aesencryptfunctionkeylengthinvalidlengtherror) + * 4.2.37 [RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError](#rqsrs008aesencryptfunctioninitializationvectorlengthinvalidlengtherror) + * 4.2.38 [RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesencryptfunctioninitializationvectornotvalidformode) + * 4.2.39 [RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode](#rqsrs008aesencryptfunctionadditionalauthenticationdatanotvalidformode) + * 4.2.40 [RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length](#rqsrs008aesencryptfunctionadditionalauthenticationdatalength) + * 4.2.41 [RQ.SRS008.AES.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.2.42 [RQ.SRS008.AES.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.2.43 [RQ.SRS008.AES.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.2.44 [RQ.SRS008.AES.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.2.45 [RQ.SRS008.AES.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.2.46 [RQ.SRS008.AES.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.2.47 [RQ.SRS008.AES.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.2.48 [RQ.SRS008.AES.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.2.49 [RQ.SRS008.AES.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.2.50 [RQ.SRS008.AES.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.2.51 [RQ.SRS008.AES.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.2.52 [RQ.SRS008.AES.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.2.53 [RQ.SRS008.AES.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.2.54 [RQ.SRS008.AES.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.2.55 [RQ.SRS008.AES.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.2.56 [RQ.SRS008.AES.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.2.57 [RQ.SRS008.AES.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.2.58 [RQ.SRS008.AES.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-ofbkeyandinitializationvectorlength) + * 4.2.59 [RQ.SRS008.AES.Encrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-gcmkeyandinitializationvectorlength) + * 4.2.60 [RQ.SRS008.AES.Encrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-gcmkeyandinitializationvectorlength) + * 4.2.61 [RQ.SRS008.AES.Encrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-gcmkeyandinitializationvectorlength) + * 4.2.62 [RQ.SRS008.AES.Encrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-128-ctrkeyandinitializationvectorlength) + * 4.2.63 [RQ.SRS008.AES.Encrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-192-ctrkeyandinitializationvectorlength) + * 4.2.64 [RQ.SRS008.AES.Encrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length](#rqsrs008aesencryptfunctionaes-256-ctrkeyandinitializationvectorlength) + * 4.2.65 [RQ.SRS008.AES.Decrypt.Function](#rqsrs008aesdecryptfunction) + * 4.2.66 [RQ.SRS008.AES.Decrypt.Function.Syntax](#rqsrs008aesdecryptfunctionsyntax) + * 4.2.67 [RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText](#rqsrs008aesdecryptfunctionparametersciphertext) + * 4.2.68 [RQ.SRS008.AES.Decrypt.Function.Parameters.Key](#rqsrs008aesdecryptfunctionparameterskey) + * 4.2.69 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode](#rqsrs008aesdecryptfunctionparametersmode) + * 4.2.70 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesdecryptfunctionparametersmodevaluesformat) + * 4.2.71 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesdecryptfunctionparametersmodevalueinvalid) + * 4.2.72 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-ecb) + * 4.2.73 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-ecb) + * 4.2.74 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-ecb) + * 4.2.75 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cbc) + * 4.2.76 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cbc) + * 4.2.77 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cbc) + * 4.2.78 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cfb1) + * 4.2.79 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cfb1) + * 4.2.80 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cfb1) + * 4.2.81 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cfb8) + * 4.2.82 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cfb8) + * 4.2.83 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cfb8) + * 4.2.84 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-cfb128) + * 4.2.85 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-cfb128) + * 4.2.86 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-cfb128) + * 4.2.87 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-ofb) + * 4.2.88 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-ofb) + * 4.2.89 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-ofb) + * 4.2.90 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-gcm) + * 4.2.91 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-gcm) + * 4.2.92 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-gcm) + * 4.2.93 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR](#rqsrs008aesdecryptfunctionparametersmodevalueaes-128-ctr) + * 4.2.94 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR](#rqsrs008aesdecryptfunctionparametersmodevalueaes-192-ctr) + * 4.2.95 [RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR](#rqsrs008aesdecryptfunctionparametersmodevalueaes-256-ctr) + * 4.2.96 [RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector](#rqsrs008aesdecryptfunctionparametersinitializationvector) + * 4.2.97 [RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData](#rqsrs008aesdecryptfunctionparametersadditionalauthenticateddata) + * 4.2.98 [RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue](#rqsrs008aesdecryptfunctionparametersreturnvalue) + * 4.2.99 [RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError](#rqsrs008aesdecryptfunctionkeylengthinvalidlengtherror) + * 4.2.100 [RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError](#rqsrs008aesdecryptfunctioninitializationvectorlengthinvalidlengtherror) + * 4.2.101 [RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesdecryptfunctioninitializationvectornotvalidformode) + * 4.2.102 [RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode](#rqsrs008aesdecryptfunctionadditionalauthenticationdatanotvalidformode) + * 4.2.103 [RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length](#rqsrs008aesdecryptfunctionadditionalauthenticationdatalength) + * 4.2.104 [RQ.SRS008.AES.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.2.105 [RQ.SRS008.AES.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.2.106 [RQ.SRS008.AES.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.2.107 [RQ.SRS008.AES.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.2.108 [RQ.SRS008.AES.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.2.109 [RQ.SRS008.AES.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.2.110 [RQ.SRS008.AES.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.2.111 [RQ.SRS008.AES.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.2.112 [RQ.SRS008.AES.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.2.113 [RQ.SRS008.AES.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.2.114 [RQ.SRS008.AES.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.2.115 [RQ.SRS008.AES.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.2.116 [RQ.SRS008.AES.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.2.117 [RQ.SRS008.AES.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.2.118 [RQ.SRS008.AES.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.2.119 [RQ.SRS008.AES.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.2.120 [RQ.SRS008.AES.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.2.121 [RQ.SRS008.AES.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-ofbkeyandinitializationvectorlength) + * 4.2.122 [RQ.SRS008.AES.Decrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-gcmkeyandinitializationvectorlength) + * 4.2.123 [RQ.SRS008.AES.Decrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-gcmkeyandinitializationvectorlength) + * 4.2.124 [RQ.SRS008.AES.Decrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-gcmkeyandinitializationvectorlength) + * 4.2.125 [RQ.SRS008.AES.Decrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-128-ctrkeyandinitializationvectorlength) + * 4.2.126 [RQ.SRS008.AES.Decrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-192-ctrkeyandinitializationvectorlength) + * 4.2.127 [RQ.SRS008.AES.Decrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length](#rqsrs008aesdecryptfunctionaes-256-ctrkeyandinitializationvectorlength) + * 4.3 [MySQL Specific Functions](#mysql-specific-functions) + * 4.3.1 [RQ.SRS008.AES.MySQL.Encrypt.Function](#rqsrs008aesmysqlencryptfunction) + * 4.3.2 [RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax](#rqsrs008aesmysqlencryptfunctionsyntax) + * 4.3.3 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText](#rqsrs008aesmysqlencryptfunctionparametersplaintext) + * 4.3.4 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key](#rqsrs008aesmysqlencryptfunctionparameterskey) + * 4.3.5 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode](#rqsrs008aesmysqlencryptfunctionparametersmode) + * 4.3.6 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesmysqlencryptfunctionparametersmodevaluesformat) + * 4.3.7 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesmysqlencryptfunctionparametersmodevalueinvalid) + * 4.3.8 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-ecb) + * 4.3.9 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-ecb) + * 4.3.10 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-ecb) + * 4.3.11 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cbc) + * 4.3.12 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cbc) + * 4.3.13 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cbc) + * 4.3.14 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cfb1) + * 4.3.15 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cfb1) + * 4.3.16 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cfb1) + * 4.3.17 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cfb8) + * 4.3.18 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cfb8) + * 4.3.19 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cfb8) + * 4.3.20 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-cfb128) + * 4.3.21 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-cfb128) + * 4.3.22 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-cfb128) + * 4.3.23 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-ofb) + * 4.3.24 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-ofb) + * 4.3.25 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-ofb) + * 4.3.26 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-gcmerror) + * 4.3.27 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-gcmerror) + * 4.3.28 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-gcmerror) + * 4.3.29 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-128-ctrerror) + * 4.3.30 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-192-ctrerror) + * 4.3.31 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error](#rqsrs008aesmysqlencryptfunctionparametersmodevalueaes-256-ctrerror) + * 4.3.32 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector](#rqsrs008aesmysqlencryptfunctionparametersinitializationvector) + * 4.3.33 [RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue](#rqsrs008aesmysqlencryptfunctionparametersreturnvalue) + * 4.3.34 [RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError](#rqsrs008aesmysqlencryptfunctionkeylengthtooshorterror) + * 4.3.35 [RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong](#rqsrs008aesmysqlencryptfunctionkeylengthtoolong) + * 4.3.36 [RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError](#rqsrs008aesmysqlencryptfunctioninitializationvectorlengthtooshorterror) + * 4.3.37 [RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong](#rqsrs008aesmysqlencryptfunctioninitializationvectorlengthtoolong) + * 4.3.38 [RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesmysqlencryptfunctioninitializationvectornotvalidformode) + * 4.3.39 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.3.40 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.3.41 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.3.42 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.3.43 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.3.44 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.3.45 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.3.46 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.3.47 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.3.48 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.3.49 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.3.50 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.3.51 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.3.52 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.3.53 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.3.54 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.3.55 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.3.56 [RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqlencryptfunctionaes-256-ofbkeyandinitializationvectorlength) + * 4.3.57 [RQ.SRS008.AES.MySQL.Decrypt.Function](#rqsrs008aesmysqldecryptfunction) + * 4.3.58 [RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax](#rqsrs008aesmysqldecryptfunctionsyntax) + * 4.3.59 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText](#rqsrs008aesmysqldecryptfunctionparametersciphertext) + * 4.3.60 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key](#rqsrs008aesmysqldecryptfunctionparameterskey) + * 4.3.61 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode](#rqsrs008aesmysqldecryptfunctionparametersmode) + * 4.3.62 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat](#rqsrs008aesmysqldecryptfunctionparametersmodevaluesformat) + * 4.3.63 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid](#rqsrs008aesmysqldecryptfunctionparametersmodevalueinvalid) + * 4.3.64 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-ecb) + * 4.3.65 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-ecb) + * 4.3.66 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-ecb) + * 4.3.67 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cbc) + * 4.3.68 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cbc) + * 4.3.69 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cbc) + * 4.3.70 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cfb1) + * 4.3.71 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cfb1) + * 4.3.72 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cfb1) + * 4.3.73 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cfb8) + * 4.3.74 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cfb8) + * 4.3.75 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cfb8) + * 4.3.76 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-cfb128) + * 4.3.77 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-cfb128) + * 4.3.78 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-cfb128) + * 4.3.79 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-ofb) + * 4.3.80 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-ofb) + * 4.3.81 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-ofb) + * 4.3.82 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-gcmerror) + * 4.3.83 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-gcmerror) + * 4.3.84 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-gcmerror) + * 4.3.85 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-128-ctrerror) + * 4.3.86 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-192-ctrerror) + * 4.3.87 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error](#rqsrs008aesmysqldecryptfunctionparametersmodevalueaes-256-ctrerror) + * 4.3.88 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector](#rqsrs008aesmysqldecryptfunctionparametersinitializationvector) + * 4.3.89 [RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue](#rqsrs008aesmysqldecryptfunctionparametersreturnvalue) + * 4.3.90 [RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError](#rqsrs008aesmysqldecryptfunctionkeylengthtooshorterror) + * 4.3.91 [RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong](#rqsrs008aesmysqldecryptfunctionkeylengthtoolong) + * 4.3.92 [RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError](#rqsrs008aesmysqldecryptfunctioninitializationvectorlengthtooshorterror) + * 4.3.93 [RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong](#rqsrs008aesmysqldecryptfunctioninitializationvectorlengthtoolong) + * 4.3.94 [RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode](#rqsrs008aesmysqldecryptfunctioninitializationvectornotvalidformode) + * 4.3.95 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-ecbkeyandinitializationvectorlength) + * 4.3.96 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-ecbkeyandinitializationvectorlength) + * 4.3.97 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-ecbkeyandinitializationvectorlength) + * 4.3.98 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cbckeyandinitializationvectorlength) + * 4.3.99 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cbckeyandinitializationvectorlength) + * 4.3.100 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cbckeyandinitializationvectorlength) + * 4.3.101 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cfb1keyandinitializationvectorlength) + * 4.3.102 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cfb1keyandinitializationvectorlength) + * 4.3.103 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cfb1keyandinitializationvectorlength) + * 4.3.104 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cfb8keyandinitializationvectorlength) + * 4.3.105 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cfb8keyandinitializationvectorlength) + * 4.3.106 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cfb8keyandinitializationvectorlength) + * 4.3.107 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-cfb128keyandinitializationvectorlength) + * 4.3.108 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-cfb128keyandinitializationvectorlength) + * 4.3.109 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-cfb128keyandinitializationvectorlength) + * 4.3.110 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-128-ofbkeyandinitializationvectorlength) + * 4.3.111 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-192-ofbkeyandinitializationvectorlength) + * 4.3.112 [RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length](#rqsrs008aesmysqldecryptfunctionaes-256-ofbkeyandinitializationvectorlength) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Revision History]. + +## Introduction + +Users need an ability to encrypt and decrypt column data with tenant specific keys. +Use cases include protection of sensitive column values and [GDPR] right to forget policies. +The implementation will support capabilities of the [MySQL aes_encrypt] and [MySQL aes_decrypt] +functions which encrypt and decrypt values using the [AES] (Advanced Encryption Standard) +algorithm. This functionality will enable encryption and decryption of data +accessed on remote [MySQL] servers via [MySQL Dictionary] or [MySQL Database Engine], +[MySQL Table Engine], or [MySQL Table Function]. + +## Terminology + +* **AES** - + Advanced Encryption Standard ([AES]) + +## Requirements + +### Generic + +#### RQ.SRS008.AES.Functions +version: 1.0 + +[ClickHouse] SHALL support [AES] encryption functions to encrypt and decrypt data. + +#### RQ.SRS008.AES.Functions.Compatability.MySQL +version: 1.0 + +[ClickHouse] SHALL support [AES] encryption functions compatible with [MySQL 5.7]. + +#### RQ.SRS008.AES.Functions.Compatability.Dictionaries +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed on remote +[MySQL] servers using [MySQL Dictionary]. + +#### RQ.SRS008.AES.Functions.Compatability.Engine.Database.MySQL +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Database Engine], + +#### RQ.SRS008.AES.Functions.Compatability.Engine.Table.MySQL +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Engine]. + +#### RQ.SRS008.AES.Functions.Compatability.TableFunction.MySQL +version: 1.0 + +[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Function]. + +#### RQ.SRS008.AES.Functions.DifferentModes +version: 1.0 + +[ClickHouse] SHALL allow different modes to be supported in a single SQL statement +using explicit function parameters. + +#### RQ.SRS008.AES.Functions.DataFromMultipleSources +version: 1.0 + +[ClickHouse] SHALL support handling encryption and decryption of data from multiple sources +in the `SELECT` statement, including [ClickHouse] [MergeTree] table as well as [MySQL Dictionary], +[MySQL Database Engine], [MySQL Table Engine], and [MySQL Table Function] +with possibly different encryption schemes. + +#### RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues +version: 1.0 + +[ClickHouse] SHALL suppress output of [AES] `string` and `key` parameters to the system log, +error log, and `query_log` table to prevent leakage of sensitive values. + +#### RQ.SRS008.AES.Functions.InvalidParameters +version: 1.0 + +[ClickHouse] SHALL return an error when parameters are invalid. + +#### RQ.SRS008.AES.Functions.Mismatched.Key +version: 1.0 + +[ClickHouse] SHALL return garbage for mismatched keys. + +#### RQ.SRS008.AES.Functions.Mismatched.IV +version: 1.0 + +[ClickHouse] SHALL return garbage for mismatched initialization vector for the modes that use it. + +#### RQ.SRS008.AES.Functions.Mismatched.AAD +version: 1.0 + +[ClickHouse] SHALL return garbage for mismatched additional authentication data for the modes that use it. + +#### RQ.SRS008.AES.Functions.Mismatched.Mode +version: 1.0 + +[ClickHouse] SHALL return an error or garbage for mismatched mode. + +#### RQ.SRS008.AES.Functions.Check.Performance +version: 1.0 + +Performance of [AES] encryption functions SHALL be measured. + +#### RQ.SRS008.AES.Function.Check.Performance.BestCase +version: 1.0 + +Performance of [AES] encryption functions SHALL be checked for the best case +scenario where there is one key, one initialization vector, and one large stream of data. + +#### RQ.SRS008.AES.Function.Check.Performance.WorstCase +version: 1.0 + +Performance of [AES] encryption functions SHALL be checked for the worst case +where there are `N` keys, `N` initialization vectors and `N` very small streams of data. + +#### RQ.SRS008.AES.Functions.Check.Compression +version: 1.0 + +Effect of [AES] encryption on column compression SHALL be measured. + +#### RQ.SRS008.AES.Functions.Check.Compression.LowCardinality +version: 1.0 + +Effect of [AES] encryption on the compression of a column with [LowCardinality] data type +SHALL be measured. + +### Specific + +#### RQ.SRS008.AES.Encrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_encrypt` function to encrypt data using [AES]. + +#### RQ.SRS008.AES.Encrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_encrypt` function + +```sql +aes_encrypt(plaintext, key, mode, [iv, aad]) +``` + +#### RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors +version: 1.0 + +[ClickHouse] `aes_encrypt` function output SHALL produce output that matches [NIST test vectors]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText +version: 1.0 + +[ClickHouse] SHALL support `plaintext` accepting any data type as +the first parameter to the `aes_encrypt` function that SHALL specify the data to be encrypted. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_encrypt` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_encrypt` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_encrypt` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as +[CTR] and [GCM] as the values. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-128-gcm` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key. +An `AEAD` 16-byte tag is appended to the resulting ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-192-gcm` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key. +An `AEAD` 16-byte tag is appended to the resulting ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-256-gcm` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key. +An `AEAD` 16-byte tag is appended to the resulting ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_encrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_encrypt` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData +version: 1.0 + +[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth +parameter to the `aes_encrypt` function that SHALL specify the additional authenticated data +for block modes that require it. + +#### RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the encrypted value of the data +using `String` data type as the result of `aes_encrypt` function. + +#### RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_encrypt` function for a given block mode. + +#### RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` length is specified and not of the exact size for the `aes_encrypt` function for a given block mode. + +#### RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt` function for a mode that does not need it. + +#### RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_encrypt` function for a mode that does not need it. + +#### RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length +version: 1.0 + +[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_encrypt` function. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes +and if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Encrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_decrypt` function to decrypt data using [AES]. + +#### RQ.SRS008.AES.Decrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_decrypt` function + +```sql +aes_decrypt(ciphertext, key, mode, [iv, aad]) +``` + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText +version: 1.0 + +[ClickHouse] SHALL support `ciphertext` accepting `FixedString` or `String` data types as +the first parameter to the `aes_decrypt` function that SHALL specify the data to be decrypted. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_decrypt` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_decrypt` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_decrypt` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as +[CTR] and [GCM] as the values. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-128-gcm` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key. +An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-192-gcm` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key. +An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM +version: 1.0 + +[ClickHouse] SHALL support `aes-256-gcm` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key. +An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to +the [RFC5116]. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_decrypt` function +and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_decrypt` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData +version: 1.0 + +[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth +parameter to the `aes_decrypt` function that SHALL specify the additional authenticated data +for block modes that require it. + +#### RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the decrypted value of the data +using `String` data type as the result of `aes_decrypt` function. + +#### RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_decrypt` function for a given block mode. + +#### RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is speficified and the length is not exact for the `aes_decrypt` function for a given block mode. + +#### RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt` function +for a mode that does not need it. + +#### RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_decrypt` function +for a mode that does not need it. + +#### RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length +version: 1.0 + +[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_decrypt` function. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes +or `iv` or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes +and if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes +or `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes or `aad` is specified. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes +or `iv` is not specified or is less than 8 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes +or if specified `iv` is not 16 bytes. + +#### RQ.SRS008.AES.Decrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes +or if specified `iv` is not 16 bytes. + +### MySQL Specific Functions + +#### RQ.SRS008.AES.MySQL.Encrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_encrypt_mysql` function to encrypt data using [AES]. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_encrypt_mysql` function + +```sql +aes_encrypt_mysql(plaintext, key, mode, [iv]) +``` + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText +version: 1.0 + +[ClickHouse] SHALL support `plaintext` accepting any data type as +the first parameter to the `aes_encrypt_mysql` function that SHALL specify the data to be encrypted. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_encrypt_mysql` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_encrypt_mysql` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_encrypt_mysql` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt_mysql` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the +`aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_encrypt_mysql` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the encrypted value of the data +using `String` data type as the result of `aes_encrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_encrypt_mysql` +function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required +for the `aes_encrypt_mysql` function for a given block mode. + +```python +def fold_key(key, cipher_key_size): + key = list(key) if not isinstance(key, (list, tuple)) else key + folded_key = key[:cipher_key_size] + for i in range(cipher_key_size, len(key)): + print(i % cipher_key_size, i) + folded_key[i % cipher_key_size] ^= key[i] + return folded_key +``` + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum +that is required for the `aes_encrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and +its length is longer than required for the `aes_encrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt_mysql` +function for a mode that does not need it. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes +and if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function +version: 1.0 + +[ClickHouse] SHALL support `aes_decrypt_mysql` function to decrypt data using [AES]. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following syntax for the `aes_decrypt_mysql` function + +```sql +aes_decrypt_mysql(ciphertext, key, mode, [iv]) +``` + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText +version: 1.0 + +[ClickHouse] SHALL support `ciphertext` accepting any data type as +the first parameter to the `aes_decrypt_mysql` function that SHALL specify the data to be decrypted. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key +version: 1.0 + +[ClickHouse] SHALL support `key` with `String` or `FixedString` data types +as the second parameter to the `aes_decrypt_mysql` function that SHALL specify the encryption key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode +version: 1.0 + +[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter +to the `aes_decrypt_mysql` function that SHALL specify encryption key length and block encryption mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat +version: 1.0 + +[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter +of the `aes_decrypt_mysql` function where +the `key_length` SHALL specifies the length of the key and SHALL accept +`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption +mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt_mysql` +function is not valid with the exception where such a mode is supported by the underlying +[OpenSSL] implementation. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128 +version: 1.0 + +[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB +version: 1.0 + +[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function +and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error +version: 1.0 + +[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the +`aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector +version: 1.0 + +[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth +parameter to the `aes_decrypt_mysql` function that SHALL specify the initialization vector for block modes that require +it. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue +version: 1.0 + +[ClickHouse] SHALL return the decrypted value of the data +using `String` data type as the result of `aes_decrypt_mysql` function. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_decrypt_mysql` +function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required +for the `aes_decrypt_mysql` function for a given block mode. + +```python +def fold_key(key, cipher_key_size): + key = list(key) if not isinstance(key, (list, tuple)) else key + folded_key = key[:cipher_key_size] + for i in range(cipher_key_size, len(key)): + print(i % cipher_key_size, i) + folded_key[i % cipher_key_size] ^= key[i] + return folded_key +``` + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum +that is required for the `aes_decrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong +version: 1.0 + +[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and +its length is longer than required for the `aes_decrypt_mysql` function for a given block mode. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode +version: 1.0 + +[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt_mysql` +function for a mode that does not need it. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes +or `iv` is specified. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes +and if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes +or if specified `iv` is less than 16 bytes. + +#### RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length +version: 1.0 + +[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes +or if specified `iv` is less than 16 bytes. + +## References + +* **GDPR:** https://en.wikipedia.org/wiki/General_Data_Protection_Regulation +* **MySQL:** https://www.mysql.com/ +* **AES:** https://en.wikipedia.org/wiki/Advanced_Encryption_Standard +* **ClickHouse:** https://clickhouse.tech +* **Git:** https://git-scm.com/ + +[OpenSSL]: https://www.openssl.org/ +[LowCardinality]: https://clickhouse.tech/docs/en/sql-reference/data-types/lowcardinality/ +[MergeTree]: https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/ +[MySQL Database Engine]: https://clickhouse.tech/docs/en/engines/database-engines/mysql/ +[MySQL Table Engine]: https://clickhouse.tech/docs/en/engines/table-engines/integrations/mysql/ +[MySQL Table Function]: https://clickhouse.tech/docs/en/sql-reference/table-functions/mysql/ +[MySQL Dictionary]: https://clickhouse.tech/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources/#dicts-external_dicts_dict_sources-mysql +[GCM]: https://en.wikipedia.org/wiki/Galois/Counter_Mode +[CTR]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_(CTR) +[CBC]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_block_chaining_(CBC) +[ECB]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Electronic_codebook_(ECB) +[CFB]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[CFB1]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[CFB8]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[CFB128]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_feedback_(CFB) +[OFB]: https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Output_feedback_(OFB) +[GDPR]: https://en.wikipedia.org/wiki/General_Data_Protection_Regulation +[RFC5116]: https://tools.ietf.org/html/rfc5116#section-5.1 +[MySQL]: https://www.mysql.com/ +[MySQL 5.7]: https://dev.mysql.com/doc/refman/5.7/en/ +[MySQL aes_encrypt]: https://dev.mysql.com/doc/refman/5.7/en/encryption-functions.html#function_aes-encrypt +[MySQL aes_decrypt]: https://dev.mysql.com/doc/refman/5.7/en/encryption-functions.html#function_aes-decrypt +[AES]: https://en.wikipedia.org/wiki/Advanced_Encryption_Standard +[ClickHouse]: https://clickhouse.tech +[GitHub repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/aes_encryption/requirements/requirements.md +[Revision history]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/aes_encryption/requirements/requirements.md +[Git]: https://git-scm.com/ +[NIST test vectors]: https://csrc.nist.gov/Projects/Cryptographic-Algorithm-Validation-Program +''') + RQ_SRS008_AES_Functions = Requirement( name='RQ.SRS008.AES.Functions', version='1.0', @@ -14,9 +1960,9 @@ RQ_SRS008_AES_Functions = Requirement( uid=None, description=( '[ClickHouse] SHALL support [AES] encryption functions to encrypt and decrypt data.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Compatability_MySQL = Requirement( name='RQ.SRS008.AES.Functions.Compatability.MySQL', @@ -27,9 +1973,9 @@ RQ_SRS008_AES_Functions_Compatability_MySQL = Requirement( uid=None, description=( '[ClickHouse] SHALL support [AES] encryption functions compatible with [MySQL 5.7].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Compatability_Dictionaries = Requirement( name='RQ.SRS008.AES.Functions.Compatability.Dictionaries', @@ -41,9 +1987,9 @@ RQ_SRS008_AES_Functions_Compatability_Dictionaries = Requirement( description=( '[ClickHouse] SHALL support encryption and decryption of data accessed on remote\n' '[MySQL] servers using [MySQL Dictionary].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Compatability_Engine_Database_MySQL = Requirement( name='RQ.SRS008.AES.Functions.Compatability.Engine.Database.MySQL', @@ -54,9 +2000,9 @@ RQ_SRS008_AES_Functions_Compatability_Engine_Database_MySQL = Requirement( uid=None, description=( '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Database Engine],\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Compatability_Engine_Table_MySQL = Requirement( name='RQ.SRS008.AES.Functions.Compatability.Engine.Table.MySQL', @@ -67,9 +2013,9 @@ RQ_SRS008_AES_Functions_Compatability_Engine_Table_MySQL = Requirement( uid=None, description=( '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Engine].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Compatability_TableFunction_MySQL = Requirement( name='RQ.SRS008.AES.Functions.Compatability.TableFunction.MySQL', @@ -80,9 +2026,9 @@ RQ_SRS008_AES_Functions_Compatability_TableFunction_MySQL = Requirement( uid=None, description=( '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Function].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_DifferentModes = Requirement( name='RQ.SRS008.AES.Functions.DifferentModes', @@ -94,9 +2040,9 @@ RQ_SRS008_AES_Functions_DifferentModes = Requirement( description=( '[ClickHouse] SHALL allow different modes to be supported in a single SQL statement\n' 'using explicit function parameters.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_DataFromMultipleSources = Requirement( name='RQ.SRS008.AES.Functions.DataFromMultipleSources', @@ -110,9 +2056,9 @@ RQ_SRS008_AES_Functions_DataFromMultipleSources = Requirement( 'in the `SELECT` statement, including [ClickHouse] [MergeTree] table as well as [MySQL Dictionary],\n' '[MySQL Database Engine], [MySQL Table Engine], and [MySQL Table Function]\n' 'with possibly different encryption schemes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_SuppressOutputOfSensitiveValues = Requirement( name='RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues', @@ -124,9 +2070,9 @@ RQ_SRS008_AES_Functions_SuppressOutputOfSensitiveValues = Requirement( description=( '[ClickHouse] SHALL suppress output of [AES] `string` and `key` parameters to the system log,\n' 'error log, and `query_log` table to prevent leakage of sensitive values.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_InvalidParameters = Requirement( name='RQ.SRS008.AES.Functions.InvalidParameters', @@ -137,9 +2083,9 @@ RQ_SRS008_AES_Functions_InvalidParameters = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error when parameters are invalid.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Mismatched_Key = Requirement( name='RQ.SRS008.AES.Functions.Mismatched.Key', @@ -150,9 +2096,9 @@ RQ_SRS008_AES_Functions_Mismatched_Key = Requirement( uid=None, description=( '[ClickHouse] SHALL return garbage for mismatched keys.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Mismatched_IV = Requirement( name='RQ.SRS008.AES.Functions.Mismatched.IV', @@ -163,9 +2109,9 @@ RQ_SRS008_AES_Functions_Mismatched_IV = Requirement( uid=None, description=( '[ClickHouse] SHALL return garbage for mismatched initialization vector for the modes that use it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Mismatched_AAD = Requirement( name='RQ.SRS008.AES.Functions.Mismatched.AAD', @@ -176,9 +2122,9 @@ RQ_SRS008_AES_Functions_Mismatched_AAD = Requirement( uid=None, description=( '[ClickHouse] SHALL return garbage for mismatched additional authentication data for the modes that use it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Mismatched_Mode = Requirement( name='RQ.SRS008.AES.Functions.Mismatched.Mode', @@ -189,9 +2135,9 @@ RQ_SRS008_AES_Functions_Mismatched_Mode = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error or garbage for mismatched mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Check_Performance = Requirement( name='RQ.SRS008.AES.Functions.Check.Performance', @@ -202,9 +2148,9 @@ RQ_SRS008_AES_Functions_Check_Performance = Requirement( uid=None, description=( 'Performance of [AES] encryption functions SHALL be measured.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Function_Check_Performance_BestCase = Requirement( name='RQ.SRS008.AES.Function.Check.Performance.BestCase', @@ -216,9 +2162,9 @@ RQ_SRS008_AES_Function_Check_Performance_BestCase = Requirement( description=( 'Performance of [AES] encryption functions SHALL be checked for the best case\n' 'scenario where there is one key, one initialization vector, and one large stream of data.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Function_Check_Performance_WorstCase = Requirement( name='RQ.SRS008.AES.Function.Check.Performance.WorstCase', @@ -230,9 +2176,9 @@ RQ_SRS008_AES_Function_Check_Performance_WorstCase = Requirement( description=( 'Performance of [AES] encryption functions SHALL be checked for the worst case\n' 'where there are `N` keys, `N` initialization vectors and `N` very small streams of data.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Check_Compression = Requirement( name='RQ.SRS008.AES.Functions.Check.Compression', @@ -243,9 +2189,9 @@ RQ_SRS008_AES_Functions_Check_Compression = Requirement( uid=None, description=( 'Effect of [AES] encryption on column compression SHALL be measured.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Functions_Check_Compression_LowCardinality = Requirement( name='RQ.SRS008.AES.Functions.Check.Compression.LowCardinality', @@ -257,9 +2203,9 @@ RQ_SRS008_AES_Functions_Check_Compression_LowCardinality = Requirement( description=( 'Effect of [AES] encryption on the compression of a column with [LowCardinality] data type\n' 'SHALL be measured.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function = Requirement( name='RQ.SRS008.AES.Encrypt.Function', @@ -270,9 +2216,9 @@ RQ_SRS008_AES_Encrypt_Function = Requirement( uid=None, description=( '[ClickHouse] SHALL support `aes_encrypt` function to encrypt data using [AES].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Syntax = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Syntax', @@ -287,9 +2233,9 @@ RQ_SRS008_AES_Encrypt_Function_Syntax = Requirement( '```sql\n' 'aes_encrypt(plaintext, key, mode, [iv, aad])\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_NIST_TestVectors = Requirement( name='RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors', @@ -300,9 +2246,9 @@ RQ_SRS008_AES_Encrypt_Function_NIST_TestVectors = Requirement( uid=None, description=( '[ClickHouse] `aes_encrypt` function output SHALL produce output that matches [NIST test vectors].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_PlainText = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText', @@ -314,9 +2260,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_PlainText = Requirement( description=( '[ClickHouse] SHALL support `plaintext` accepting any data type as\n' 'the first parameter to the `aes_encrypt` function that SHALL specify the data to be encrypted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Key = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Key', @@ -328,9 +2274,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Key = Requirement( description=( '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' 'as the second parameter to the `aes_encrypt` function that SHALL specify the encryption key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode', @@ -342,9 +2288,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode = Requirement( description=( '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' 'to the `aes_encrypt` function that SHALL specify encryption key length and block encryption mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat', @@ -360,9 +2306,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as\n' '[CTR] and [GCM] as the values. For example, `aes-256-ofb`.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid', @@ -375,9 +2321,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement( '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt`\n' 'function is not valid with the exception where such a mode is supported by the underlying\n' '[OpenSSL] implementation.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB', @@ -389,9 +2335,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB', @@ -403,9 +2349,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB', @@ -417,9 +2363,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC', @@ -431,9 +2377,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC', @@ -445,9 +2391,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC', @@ -459,9 +2405,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1', @@ -473,9 +2419,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1', @@ -487,9 +2433,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1', @@ -501,9 +2447,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8', @@ -515,9 +2461,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8', @@ -529,9 +2475,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8', @@ -543,9 +2489,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128', @@ -557,9 +2503,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128', @@ -571,9 +2517,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128', @@ -585,9 +2531,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB', @@ -599,9 +2545,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB', @@ -613,9 +2559,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB', @@ -627,9 +2573,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM', @@ -643,9 +2589,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key.\n' 'An `AEAD` 16-byte tag is appended to the resulting ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM', @@ -659,9 +2605,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key.\n' 'An `AEAD` 16-byte tag is appended to the resulting ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM', @@ -675,9 +2621,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key.\n' 'An `AEAD` 16-byte tag is appended to the resulting ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR', @@ -689,9 +2635,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR', @@ -703,9 +2649,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR', @@ -717,9 +2663,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_encrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_InitializationVector = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector', @@ -732,9 +2678,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_InitializationVector = Requirement( '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' 'parameter to the `aes_encrypt` function that SHALL specify the initialization vector for block modes that require\n' 'it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_AdditionalAuthenticatedData = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData', @@ -747,9 +2693,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_AdditionalAuthenticatedData = Requirem '[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth\n' 'parameter to the `aes_encrypt` function that SHALL specify the additional authenticated data\n' 'for block modes that require it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Parameters_ReturnValue = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue', @@ -761,9 +2707,9 @@ RQ_SRS008_AES_Encrypt_Function_Parameters_ReturnValue = Requirement( description=( '[ClickHouse] SHALL return the encrypted value of the data\n' 'using `String` data type as the result of `aes_encrypt` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError = Requirement( name='RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError', @@ -774,9 +2720,9 @@ RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_encrypt` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError = Requirement( name='RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError', @@ -787,9 +2733,9 @@ RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError = uid=None, description=( '[ClickHouse] SHALL return an error if the `iv` length is specified and not of the exact size for the `aes_encrypt` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode = Requirement( name='RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode', @@ -800,9 +2746,9 @@ RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode = Requiremen uid=None, description=( '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt` function for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode', @@ -813,9 +2759,9 @@ RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode = Re uid=None, description=( '[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_encrypt` function for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length', @@ -826,9 +2772,9 @@ RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_Length = Requirement uid=None, description=( '[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_encrypt` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', @@ -840,9 +2786,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', @@ -854,9 +2800,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', @@ -868,9 +2814,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', @@ -882,9 +2828,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', @@ -896,9 +2842,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', @@ -910,9 +2856,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', @@ -924,9 +2870,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', @@ -938,9 +2884,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', @@ -952,9 +2898,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', @@ -966,9 +2912,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes\n' 'and if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', @@ -980,9 +2926,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', @@ -994,9 +2940,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', @@ -1008,9 +2954,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', @@ -1022,9 +2968,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', @@ -1036,9 +2982,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', @@ -1050,9 +2996,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', @@ -1064,9 +3010,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', @@ -1078,9 +3024,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length', @@ -1092,9 +3038,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length', @@ -1106,9 +3052,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length', @@ -1120,9 +3066,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length', @@ -1134,9 +3080,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length', @@ -1148,9 +3094,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Encrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Encrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length', @@ -1162,9 +3108,9 @@ RQ_SRS008_AES_Encrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function = Requirement( name='RQ.SRS008.AES.Decrypt.Function', @@ -1175,9 +3121,9 @@ RQ_SRS008_AES_Decrypt_Function = Requirement( uid=None, description=( '[ClickHouse] SHALL support `aes_decrypt` function to decrypt data using [AES].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Syntax = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Syntax', @@ -1192,9 +3138,9 @@ RQ_SRS008_AES_Decrypt_Function_Syntax = Requirement( '```sql\n' 'aes_decrypt(ciphertext, key, mode, [iv, aad])\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText', @@ -1206,9 +3152,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText = Requirement( description=( '[ClickHouse] SHALL support `ciphertext` accepting `FixedString` or `String` data types as\n' 'the first parameter to the `aes_decrypt` function that SHALL specify the data to be decrypted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Key = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Key', @@ -1220,9 +3166,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Key = Requirement( description=( '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' 'as the second parameter to the `aes_decrypt` function that SHALL specify the encryption key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode', @@ -1234,9 +3180,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode = Requirement( description=( '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' 'to the `aes_decrypt` function that SHALL specify encryption key length and block encryption mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat', @@ -1252,9 +3198,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as\n' '[CTR] and [GCM] as the values. For example, `aes-256-ofb`.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid', @@ -1267,9 +3213,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement( '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt`\n' 'function is not valid with the exception where such a mode is supported by the underlying\n' '[OpenSSL] implementation.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB', @@ -1281,9 +3227,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB', @@ -1295,9 +3241,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB', @@ -1309,9 +3255,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC', @@ -1323,9 +3269,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC', @@ -1337,9 +3283,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC', @@ -1351,9 +3297,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1', @@ -1365,9 +3311,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1', @@ -1379,9 +3325,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1', @@ -1393,9 +3339,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8', @@ -1407,9 +3353,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8', @@ -1421,9 +3367,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8', @@ -1435,9 +3381,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( description=( '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128', @@ -1449,9 +3395,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128', @@ -1463,9 +3409,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128', @@ -1477,9 +3423,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requiremen description=( '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB', @@ -1491,9 +3437,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB', @@ -1505,9 +3451,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB', @@ -1519,9 +3465,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM', @@ -1535,9 +3481,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key.\n' 'An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM', @@ -1551,9 +3497,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key.\n' 'An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM', @@ -1567,9 +3513,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key.\n' 'An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' 'the [RFC5116].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR', @@ -1581,9 +3527,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR', @@ -1595,9 +3541,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR', @@ -1609,9 +3555,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( description=( '[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_decrypt` function\n' 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_InitializationVector = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector', @@ -1624,9 +3570,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_InitializationVector = Requirement( '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' 'parameter to the `aes_decrypt` function that SHALL specify the initialization vector for block modes that require\n' 'it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_AdditionalAuthenticatedData = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData', @@ -1639,9 +3585,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_AdditionalAuthenticatedData = Requirem '[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth\n' 'parameter to the `aes_decrypt` function that SHALL specify the additional authenticated data\n' 'for block modes that require it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Parameters_ReturnValue = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue', @@ -1653,9 +3599,9 @@ RQ_SRS008_AES_Decrypt_Function_Parameters_ReturnValue = Requirement( description=( '[ClickHouse] SHALL return the decrypted value of the data\n' 'using `String` data type as the result of `aes_decrypt` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError = Requirement( name='RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError', @@ -1666,9 +3612,9 @@ RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_decrypt` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError = Requirement( name='RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError', @@ -1679,9 +3625,9 @@ RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError = uid=None, description=( '[ClickHouse] SHALL return an error if the `iv` is speficified and the length is not exact for the `aes_decrypt` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode = Requirement( name='RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode', @@ -1693,9 +3639,9 @@ RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode = Requiremen description=( '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt` function\n' 'for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode', @@ -1707,9 +3653,9 @@ RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode = Re description=( '[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_decrypt` function\n' 'for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length', @@ -1720,9 +3666,9 @@ RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_Length = Requirement uid=None, description=( '[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_decrypt` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', @@ -1734,9 +3680,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', @@ -1748,9 +3694,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', @@ -1762,9 +3708,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes\n' 'or `iv` or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', @@ -1776,9 +3722,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', @@ -1790,9 +3736,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', @@ -1804,9 +3750,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', @@ -1818,9 +3764,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', @@ -1832,9 +3778,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', @@ -1846,9 +3792,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', @@ -1860,9 +3806,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes\n' 'and if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', @@ -1874,9 +3820,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes\n' 'or `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', @@ -1888,9 +3834,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', @@ -1902,9 +3848,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', @@ -1916,9 +3862,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', @@ -1930,9 +3876,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', @@ -1944,9 +3890,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', @@ -1958,9 +3904,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', @@ -1972,9 +3918,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length', @@ -1986,9 +3932,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length', @@ -2000,9 +3946,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length', @@ -2014,9 +3960,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes\n' 'or `iv` is not specified or is less than 8 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length', @@ -2028,9 +3974,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length', @@ -2042,9 +3988,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_Decrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.Decrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length', @@ -2056,9 +4002,9 @@ RQ_SRS008_AES_Decrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = R description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes\n' 'or if specified `iv` is not 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function', @@ -2069,9 +4015,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function = Requirement( uid=None, description=( '[ClickHouse] SHALL support `aes_encrypt_mysql` function to encrypt data using [AES].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Syntax = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax', @@ -2086,9 +4032,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Syntax = Requirement( '```sql\n' 'aes_encrypt_mysql(plaintext, key, mode, [iv])\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_PlainText = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText', @@ -2100,9 +4046,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_PlainText = Requirement( description=( '[ClickHouse] SHALL support `plaintext` accepting any data type as\n' 'the first parameter to the `aes_encrypt_mysql` function that SHALL specify the data to be encrypted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Key = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key', @@ -2114,9 +4060,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Key = Requirement( description=( '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' 'as the second parameter to the `aes_encrypt_mysql` function that SHALL specify the encryption key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode', @@ -2128,9 +4074,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode = Requirement( description=( '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' 'to the `aes_encrypt_mysql` function that SHALL specify encryption key length and block encryption mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat', @@ -2145,9 +4091,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid', @@ -2160,9 +4106,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt_mysql`\n' 'function is not valid with the exception where such a mode is supported by the underlying\n' '[OpenSSL] implementation.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB', @@ -2174,9 +4120,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Require description=( '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB', @@ -2188,9 +4134,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Require description=( '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB', @@ -2202,9 +4148,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Require description=( '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC', @@ -2216,9 +4162,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Require description=( '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC', @@ -2230,9 +4176,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Require description=( '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC', @@ -2244,9 +4190,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Require description=( '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1', @@ -2258,9 +4204,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1', @@ -2272,9 +4218,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1', @@ -2286,9 +4232,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8', @@ -2300,9 +4246,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8', @@ -2314,9 +4260,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8', @@ -2328,9 +4274,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128', @@ -2342,9 +4288,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128', @@ -2356,9 +4302,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128', @@ -2370,9 +4316,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB', @@ -2384,9 +4330,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Require description=( '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB', @@ -2398,9 +4344,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Require description=( '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB', @@ -2412,9 +4358,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Require description=( '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error', @@ -2426,9 +4372,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error', @@ -2440,9 +4386,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error', @@ -2454,9 +4400,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error', @@ -2468,9 +4414,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error', @@ -2482,9 +4428,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error', @@ -2496,9 +4442,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_InitializationVector = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector', @@ -2511,9 +4457,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_InitializationVector = Requireme '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' 'parameter to the `aes_encrypt_mysql` function that SHALL specify the initialization vector for block modes that require\n' 'it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_ReturnValue = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue', @@ -2525,9 +4471,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_ReturnValue = Requirement( description=( '[ClickHouse] SHALL return the encrypted value of the data\n' 'using `String` data type as the result of `aes_encrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooShortError = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError', @@ -2539,9 +4485,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooShortError = Requirement( description=( '[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_encrypt_mysql`\n' 'function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooLong = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong', @@ -2563,9 +4509,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooLong = Requirement( '\t\t folded_key[i % cipher_key_size] ^= key[i]\n' '\t return folded_key\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooShortError = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError', @@ -2577,9 +4523,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooShortError = description=( '[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum\n' 'that is required for the `aes_encrypt_mysql` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooLong = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong', @@ -2591,9 +4537,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooLong = Requi description=( '[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and\n' 'its length is longer than required for the `aes_encrypt_mysql` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode', @@ -2605,9 +4551,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode = Requ description=( '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt_mysql`\n' 'function for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', @@ -2619,9 +4565,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', @@ -2633,9 +4579,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', @@ -2647,9 +4593,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', @@ -2661,9 +4607,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', @@ -2675,9 +4621,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', @@ -2689,9 +4635,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', @@ -2703,9 +4649,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', @@ -2717,9 +4663,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', @@ -2731,9 +4677,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', @@ -2745,9 +4691,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes\n' 'and if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', @@ -2759,9 +4705,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', @@ -2773,9 +4719,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', @@ -2787,9 +4733,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', @@ -2801,9 +4747,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', @@ -2815,9 +4761,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', @@ -2829,9 +4775,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', @@ -2843,9 +4789,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', @@ -2857,9 +4803,9 @@ RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function', @@ -2870,9 +4816,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function = Requirement( uid=None, description=( '[ClickHouse] SHALL support `aes_decrypt_mysql` function to decrypt data using [AES].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Syntax = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax', @@ -2887,9 +4833,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Syntax = Requirement( '```sql\n' 'aes_decrypt_mysql(ciphertext, key, mode, [iv])\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText', @@ -2901,9 +4847,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText = Requirement( description=( '[ClickHouse] SHALL support `ciphertext` accepting any data type as\n' 'the first parameter to the `aes_decrypt_mysql` function that SHALL specify the data to be decrypted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Key = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key', @@ -2915,9 +4861,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Key = Requirement( description=( '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' 'as the second parameter to the `aes_decrypt_mysql` function that SHALL specify the encryption key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode', @@ -2929,9 +4875,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode = Requirement( description=( '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' 'to the `aes_decrypt_mysql` function that SHALL specify encryption key length and block encryption mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat', @@ -2946,9 +4892,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid', @@ -2961,9 +4907,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt_mysql`\n' 'function is not valid with the exception where such a mode is supported by the underlying\n' '[OpenSSL] implementation.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB', @@ -2975,9 +4921,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Require description=( '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB', @@ -2989,9 +4935,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Require description=( '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB', @@ -3003,9 +4949,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Require description=( '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC', @@ -3017,9 +4963,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Require description=( '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC', @@ -3031,9 +4977,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Require description=( '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC', @@ -3045,9 +4991,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Require description=( '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1', @@ -3059,9 +5005,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1', @@ -3073,9 +5019,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1', @@ -3087,9 +5033,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requir description=( '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8', @@ -3101,9 +5047,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8', @@ -3115,9 +5061,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8', @@ -3129,9 +5075,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requir description=( '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128', @@ -3143,9 +5089,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128', @@ -3157,9 +5103,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128', @@ -3171,9 +5117,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requ description=( '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB', @@ -3185,9 +5131,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Require description=( '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB', @@ -3199,9 +5145,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Require description=( '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB', @@ -3213,9 +5159,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Require description=( '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error', @@ -3227,9 +5173,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error', @@ -3241,9 +5187,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error', @@ -3255,9 +5201,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = R description=( '[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error', @@ -3269,9 +5215,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error', @@ -3283,9 +5229,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error', @@ -3297,9 +5243,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = R description=( '[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the\n' '`aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_InitializationVector = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector', @@ -3312,9 +5258,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_InitializationVector = Requireme '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' 'parameter to the `aes_decrypt_mysql` function that SHALL specify the initialization vector for block modes that require\n' 'it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_ReturnValue = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue', @@ -3326,9 +5272,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_ReturnValue = Requirement( description=( '[ClickHouse] SHALL return the decrypted value of the data\n' 'using `String` data type as the result of `aes_decrypt_mysql` function.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooShortError = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError', @@ -3340,9 +5286,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooShortError = Requirement( description=( '[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_decrypt_mysql`\n' 'function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooLong = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong', @@ -3364,9 +5310,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooLong = Requirement( '\t\t folded_key[i % cipher_key_size] ^= key[i]\n' '\t return folded_key\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooShortError = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError', @@ -3378,9 +5324,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooShortError = description=( '[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum\n' 'that is required for the `aes_decrypt_mysql` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooLong = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong', @@ -3392,9 +5338,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooLong = Requi description=( '[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and\n' 'its length is longer than required for the `aes_decrypt_mysql` function for a given block mode.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode', @@ -3406,9 +5352,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode = Requ description=( '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt_mysql`\n' 'function for a mode that does not need it.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', @@ -3420,9 +5366,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', @@ -3434,9 +5380,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', @@ -3448,9 +5394,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes\n' 'or `iv` is specified.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', @@ -3462,9 +5408,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', @@ -3476,9 +5422,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', @@ -3490,9 +5436,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', @@ -3504,9 +5450,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', @@ -3518,9 +5464,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', @@ -3532,9 +5478,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', @@ -3546,9 +5492,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes\n' 'and if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', @@ -3560,9 +5506,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', @@ -3574,9 +5520,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Len description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', @@ -3588,9 +5534,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', @@ -3602,9 +5548,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', @@ -3616,9 +5562,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_L description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', @@ -3630,9 +5576,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', @@ -3644,9 +5590,9 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', @@ -3658,6 +5604,6 @@ RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Leng description=( '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes\n' 'or if specified `iv` is less than 16 bytes.\n' + '\n' ), - link=None - ) + link=None) diff --git a/tests/testflows/ldap/authentication/requirements/requirements.md b/tests/testflows/ldap/authentication/requirements/requirements.md index d322db70330..1c65a29fef4 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.md +++ b/tests/testflows/ldap/authentication/requirements/requirements.md @@ -460,14 +460,14 @@ time user configuration contains any of the `` entries. #### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined version: 1.0 -[ClickHouse] SHALL throw an error during any authentification attempt +[ClickHouse] SHALL throw an error during any authentication attempt if the name of the [LDAP] server used inside the `` entry is not defined in the `` section. #### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty version: 1.0 -[ClickHouse] SHALL throw an error during any authentification attempt +[ClickHouse] SHALL throw an error during any authentication attempt if the name of the [LDAP] server used inside the `` entry is empty. diff --git a/tests/testflows/ldap/authentication/requirements/requirements.py b/tests/testflows/ldap/authentication/requirements/requirements.py index 967e097d1fa..60fbef9b8cd 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.py +++ b/tests/testflows/ldap/authentication/requirements/requirements.py @@ -1,10 +1,571 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.200811.1124123. +# document by TestFlows v1.6.201026.1232822. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. +from testflows.core import Specification from testflows.core import Requirement +SRS_007_ClickHouse_Authentication_of_Users_via_LDAP = Specification( + name='SRS-007 ClickHouse Authentication of Users via LDAP', + description=None, + author=None, + date=None, + status=None, + approved_by=None, + approved_date=None, + approved_version=None, + version=None, + group=None, + type=None, + link=None, + uid=None, + parent=None, + children=None, + content=''' +# SRS-007 ClickHouse Authentication of Users via LDAP + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Terminology](#terminology) +* 4 [Requirements](#requirements) + * 4.1 [Generic](#generic) + * 4.1.1 [RQ.SRS-007.LDAP.Authentication](#rqsrs-007ldapauthentication) + * 4.1.2 [RQ.SRS-007.LDAP.Authentication.MultipleServers](#rqsrs-007ldapauthenticationmultipleservers) + * 4.1.3 [RQ.SRS-007.LDAP.Authentication.Protocol.PlainText](#rqsrs-007ldapauthenticationprotocolplaintext) + * 4.1.4 [RQ.SRS-007.LDAP.Authentication.Protocol.TLS](#rqsrs-007ldapauthenticationprotocoltls) + * 4.1.5 [RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS](#rqsrs-007ldapauthenticationprotocolstarttls) + * 4.1.6 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation](#rqsrs-007ldapauthenticationtlscertificatevalidation) + * 4.1.7 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned](#rqsrs-007ldapauthenticationtlscertificateselfsigned) + * 4.1.8 [RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority](#rqsrs-007ldapauthenticationtlscertificatespecificcertificationauthority) + * 4.1.9 [RQ.SRS-007.LDAP.Server.Configuration.Invalid](#rqsrs-007ldapserverconfigurationinvalid) + * 4.1.10 [RQ.SRS-007.LDAP.User.Configuration.Invalid](#rqsrs-007ldapuserconfigurationinvalid) + * 4.1.11 [RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous](#rqsrs-007ldapauthenticationmechanismanonymous) + * 4.1.12 [RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated](#rqsrs-007ldapauthenticationmechanismunauthenticated) + * 4.1.13 [RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword](#rqsrs-007ldapauthenticationmechanismnamepassword) + * 4.1.14 [RQ.SRS-007.LDAP.Authentication.Valid](#rqsrs-007ldapauthenticationvalid) + * 4.1.15 [RQ.SRS-007.LDAP.Authentication.Invalid](#rqsrs-007ldapauthenticationinvalid) + * 4.1.16 [RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser](#rqsrs-007ldapauthenticationinvaliddeleteduser) + * 4.1.17 [RQ.SRS-007.LDAP.Authentication.UsernameChanged](#rqsrs-007ldapauthenticationusernamechanged) + * 4.1.18 [RQ.SRS-007.LDAP.Authentication.PasswordChanged](#rqsrs-007ldapauthenticationpasswordchanged) + * 4.1.19 [RQ.SRS-007.LDAP.Authentication.LDAPServerRestart](#rqsrs-007ldapauthenticationldapserverrestart) + * 4.1.20 [RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart](#rqsrs-007ldapauthenticationclickhouseserverrestart) + * 4.1.21 [RQ.SRS-007.LDAP.Authentication.Parallel](#rqsrs-007ldapauthenticationparallel) + * 4.1.22 [RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid](#rqsrs-007ldapauthenticationparallelvalidandinvalid) + * 4.2 [Specific](#specific) + * 4.2.1 [RQ.SRS-007.LDAP.UnreachableServer](#rqsrs-007ldapunreachableserver) + * 4.2.2 [RQ.SRS-007.LDAP.Configuration.Server.Name](#rqsrs-007ldapconfigurationservername) + * 4.2.3 [RQ.SRS-007.LDAP.Configuration.Server.Host](#rqsrs-007ldapconfigurationserverhost) + * 4.2.4 [RQ.SRS-007.LDAP.Configuration.Server.Port](#rqsrs-007ldapconfigurationserverport) + * 4.2.5 [RQ.SRS-007.LDAP.Configuration.Server.Port.Default](#rqsrs-007ldapconfigurationserverportdefault) + * 4.2.6 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix](#rqsrs-007ldapconfigurationserverauthdnprefix) + * 4.2.7 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix](#rqsrs-007ldapconfigurationserverauthdnsuffix) + * 4.2.8 [RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value](#rqsrs-007ldapconfigurationserverauthdnvalue) + * 4.2.9 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS](#rqsrs-007ldapconfigurationserverenabletls) + * 4.2.10 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default](#rqsrs-007ldapconfigurationserverenabletlsoptionsdefault) + * 4.2.11 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No](#rqsrs-007ldapconfigurationserverenabletlsoptionsno) + * 4.2.12 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes](#rqsrs-007ldapconfigurationserverenabletlsoptionsyes) + * 4.2.13 [RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS](#rqsrs-007ldapconfigurationserverenabletlsoptionsstarttls) + * 4.2.14 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion](#rqsrs-007ldapconfigurationservertlsminimumprotocolversion) + * 4.2.15 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values](#rqsrs-007ldapconfigurationservertlsminimumprotocolversionvalues) + * 4.2.16 [RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default](#rqsrs-007ldapconfigurationservertlsminimumprotocolversiondefault) + * 4.2.17 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert](#rqsrs-007ldapconfigurationservertlsrequirecert) + * 4.2.18 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdefault) + * 4.2.19 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsdemand) + * 4.2.20 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsallow) + * 4.2.21 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try](#rqsrs-007ldapconfigurationservertlsrequirecertoptionstry) + * 4.2.22 [RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never](#rqsrs-007ldapconfigurationservertlsrequirecertoptionsnever) + * 4.2.23 [RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile](#rqsrs-007ldapconfigurationservertlscertfile) + * 4.2.24 [RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile](#rqsrs-007ldapconfigurationservertlskeyfile) + * 4.2.25 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir](#rqsrs-007ldapconfigurationservertlscacertdir) + * 4.2.26 [RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile](#rqsrs-007ldapconfigurationservertlscacertfile) + * 4.2.27 [RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite](#rqsrs-007ldapconfigurationservertlsciphersuite) + * 4.2.28 [RQ.SRS-007.LDAP.Configuration.Server.Syntax](#rqsrs-007ldapconfigurationserversyntax) + * 4.2.29 [RQ.SRS-007.LDAP.Configuration.User.RBAC](#rqsrs-007ldapconfigurationuserrbac) + * 4.2.30 [RQ.SRS-007.LDAP.Configuration.User.Syntax](#rqsrs-007ldapconfigurationusersyntax) + * 4.2.31 [RQ.SRS-007.LDAP.Configuration.User.Name.Empty](#rqsrs-007ldapconfigurationusernameempty) + * 4.2.32 [RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP](#rqsrs-007ldapconfigurationuserbothpasswordandldap) + * 4.2.33 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined](#rqsrs-007ldapconfigurationuserldapinvalidservernamenotdefined) + * 4.2.34 [RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty](#rqsrs-007ldapconfigurationuserldapinvalidservernameempty) + * 4.2.35 [RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer](#rqsrs-007ldapconfigurationuseronlyoneserver) + * 4.2.36 [RQ.SRS-007.LDAP.Configuration.User.Name.Long](#rqsrs-007ldapconfigurationusernamelong) + * 4.2.37 [RQ.SRS-007.LDAP.Configuration.User.Name.UTF8](#rqsrs-007ldapconfigurationusernameutf8) + * 4.2.38 [RQ.SRS-007.LDAP.Authentication.Username.Empty](#rqsrs-007ldapauthenticationusernameempty) + * 4.2.39 [RQ.SRS-007.LDAP.Authentication.Username.Long](#rqsrs-007ldapauthenticationusernamelong) + * 4.2.40 [RQ.SRS-007.LDAP.Authentication.Username.UTF8](#rqsrs-007ldapauthenticationusernameutf8) + * 4.2.41 [RQ.SRS-007.LDAP.Authentication.Password.Empty](#rqsrs-007ldapauthenticationpasswordempty) + * 4.2.42 [RQ.SRS-007.LDAP.Authentication.Password.Long](#rqsrs-007ldapauthenticationpasswordlong) + * 4.2.43 [RQ.SRS-007.LDAP.Authentication.Password.UTF8](#rqsrs-007ldapauthenticationpasswordutf8) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Git]'s [Revision History]. + +## Introduction + +[ClickHouse] currently does not have any integration with [LDAP]. +As the initial step in integrating with [LDAP] this software requirements specification covers +only the requirements to enable authentication of users using an [LDAP] server. + +## Terminology + +* **CA** - + Certificate Authority ([CA]) + +* **LDAP** - + Lightweight Directory Access Protocol ([LDAP]) + +## Requirements + +### Generic + +#### RQ.SRS-007.LDAP.Authentication +version: 1.0 + +[ClickHouse] SHALL support user authentication via an [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.MultipleServers +version: 1.0 + +[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate +users. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.PlainText +version: 1.0 + +[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.TLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol. + +#### RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a +plain text `ldap://` protocol that is upgraded to [TLS]. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation +version: 1.0 + +[ClickHouse] SHALL support certificate validation used for [TLS] connections. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned +version: 1.0 + +[ClickHouse] SHALL support self-signed certificates for [TLS] connections. + +#### RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority +version: 1.0 + +[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections. + +#### RQ.SRS-007.LDAP.Server.Configuration.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid. + +#### RQ.SRS-007.LDAP.User.Configuration.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword +version: 1.0 + +[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind] +authentication mechanism. + +#### RQ.SRS-007.LDAP.Authentication.Valid +version: 1.0 + +[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if +user name and password match [LDAP] server records for the user. + +#### RQ.SRS-007.LDAP.Authentication.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if either user name or password +do not match [LDAP] server records for the user. + +#### RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the user +has been deleted from the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.UsernameChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the username is changed +on the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.PasswordChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the password +for the user is changed on the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.LDAPServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted. + +#### RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after server is restarted. + +#### RQ.SRS-007.LDAP.Authentication.Parallel +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of users using [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid +version: 1.0 + +[ClickHouse] SHALL support authentication of valid users and +prohibit authentication of invalid users using [LDAP] server +in parallel without having invalid attempts affecting valid authentications. + +### Specific + +#### RQ.SRS-007.LDAP.UnreachableServer +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable. + +#### RQ.SRS-007.LDAP.Configuration.Server.Name +version: 1.0 + +[ClickHouse] SHALL not support empty string as a server name. + +#### RQ.SRS-007.LDAP.Configuration.Server.Host +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] +server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty. + +#### RQ.SRS-007.LDAP.Configuration.Server.Port +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] server port. + +#### RQ.SRS-007.LDAP.Configuration.Server.Port.Default +version: 1.0 + +[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the prefix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the suffix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value +version: 1.0 + +[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string. + +> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS +version: 1.0 + +[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `yes` value as the default for `` parameter +to enable SSL/TLS `ldaps://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No +version: 1.0 + +[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable +plain text `ldap://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes +version: 1.0 + +[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable +SSL/TLS `ldaps://` protocol. + +#### RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS +version: 1.0 + +[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable +legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS]. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify +the minimum protocol version of SSL/TLS. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values +version: 1.0 + +[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2` +as a value of the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default +version: 1.0 + +[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [TLS] peer +certificate verification behavior. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `demand` value as the default for the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand +version: 1.0 + +[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, or a bad certificate is +provided, the session SHALL be immediately terminated. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow +version: 1.0 + +[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to +enable requesting of client certificate. If no +certificate is provided, the session SHALL proceed normally. +If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try +version: 1.0 + +[ClickHouse] SHALL support specifying `try` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, the session +SHALL proceed normally. If a bad certificate is provided, the session SHALL be +immediately terminated. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never +version: 1.0 + +[ClickHouse] SHALL support specifying `never` as the value of `` parameter to +disable requesting of client certificate. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to certificate file used by +[ClickHouse] to establish connection with the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to key file for the certificate +specified by the `` parameter. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify to a path to +the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify a path to a specific +[CA] certificate file used to verify certificates provided by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite +version: 1.0 + +[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites. +The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers]. + +For example, + +```xml +ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 +``` + +The available suites SHALL depend on the [OpenSSL] library version and variant used to build +[ClickHouse] and therefore might change. + +#### RQ.SRS-007.LDAP.Configuration.Server.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml` +configuration file or of any configuration file inside the `config.d` directory. + +```xml + + + localhost + 636 + cn= + , ou=users, dc=example, dc=com + yes + tls1.2 + demand + /path/to/tls_cert_file + /path/to/tls_key_file + /path/to/tls_ca_cert_file + /path/to/tls_ca_cert_dir + ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + + +``` + +#### RQ.SRS-007.LDAP.Configuration.User.RBAC +version: 1.0 + +[ClickHouse] SHALL support creating users identified using an [LDAP] server using +the following RBAC command + +```sql +CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name' +``` + +#### RQ.SRS-007.LDAP.Configuration.User.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following example syntax to create a user that is authenticated using +an [LDAP] server inside the `users.xml` file or any configuration file inside the `users.d` directory. + +```xml + + + + + my_ldap_server + + + + +``` + +#### RQ.SRS-007.LDAP.Configuration.User.Name.Empty +version: 1.0 + +[ClickHouse] SHALL not support empty string as a user name. + +#### RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP +version: 1.0 + +[ClickHouse] SHALL throw an error if `` is specified for the user and at the same +time user configuration contains any of the `` entries. + +#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined +version: 1.0 + +[ClickHouse] SHALL throw an error during any authentication attempt +if the name of the [LDAP] server used inside the `` entry +is not defined in the `` section. + +#### RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty +version: 1.0 + +[ClickHouse] SHALL throw an error during any authentication attempt +if the name of the [LDAP] server used inside the `` entry +is empty. + +#### RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer +version: 1.0 + +[ClickHouse] SHALL support specifying only one [LDAP] server for a given user. + +#### RQ.SRS-007.LDAP.Configuration.User.Name.Long +version: 1.0 + +[ClickHouse] SHALL support long user names of at least 256 bytes +to specify users that can be authenticated using an [LDAP] server. + +#### RQ.SRS-007.LDAP.Configuration.User.Name.UTF8 +version: 1.0 + +[ClickHouse] SHALL support user names that contain [UTF-8] characters. + +#### RQ.SRS-007.LDAP.Authentication.Username.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty username. + +#### RQ.SRS-007.LDAP.Authentication.Username.Long +version: 1.0 + +[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes. + +#### RQ.SRS-007.LDAP.Authentication.Username.UTF8 +version: 1.0 + +[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters. + +#### RQ.SRS-007.LDAP.Authentication.Password.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty passwords +even if an empty password is valid for the user and +is allowed by the [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Password.Long +version: 1.0 + +[ClickHouse] SHALL support long password of at least 256 bytes +that can be used to authenticate users using an [LDAP] server. + +#### RQ.SRS-007.LDAP.Authentication.Password.UTF8 +version: 1.0 + +[ClickHouse] SHALL support [UTF-8] characters in passwords +used to authenticate users using an [LDAP] server. + +## References + +* **ClickHouse:** https://clickhouse.tech + +[Anonymous Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-AnonymousAuthenticationMechanismOfSimpleBind +[Unauthenticated Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-UnauthenticatedAuthenticationMechanismOfSimpleBind +[Name/Password Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-NamePasswordAuthenticationMechanismOfSimpleBind +[UTF-8]: https://en.wikipedia.org/wiki/UTF-8 +[OpenSSL]: https://www.openssl.org/ +[OpenSSL Ciphers]: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html +[CA]: https://en.wikipedia.org/wiki/Certificate_authority +[TLS]: https://en.wikipedia.org/wiki/Transport_Layer_Security +[LDAP]: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol +[ClickHouse]: https://clickhouse.tech +[GitHub]: https://github.com +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/authentication/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/authentication/requirements/requirements.md +[Git]: https://git-scm.com/ +''') + RQ_SRS_007_LDAP_Authentication = Requirement( name='RQ.SRS-007.LDAP.Authentication', version='1.0', @@ -14,9 +575,9 @@ RQ_SRS_007_LDAP_Authentication = Requirement( uid=None, description=( '[ClickHouse] SHALL support user authentication via an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_MultipleServers = Requirement( name='RQ.SRS-007.LDAP.Authentication.MultipleServers', @@ -28,9 +589,9 @@ RQ_SRS_007_LDAP_Authentication_MultipleServers = Requirement( description=( '[ClickHouse] SHALL support specifying multiple [LDAP] servers that can be used to authenticate\n' 'users.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Protocol_PlainText = Requirement( name='RQ.SRS-007.LDAP.Authentication.Protocol.PlainText', @@ -41,9 +602,9 @@ RQ_SRS_007_LDAP_Authentication_Protocol_PlainText = Requirement( uid=None, description=( '[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Protocol_TLS = Requirement( name='RQ.SRS-007.LDAP.Authentication.Protocol.TLS', @@ -54,9 +615,9 @@ RQ_SRS_007_LDAP_Authentication_Protocol_TLS = Requirement( uid=None, description=( '[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Protocol_StartTLS = Requirement( name='RQ.SRS-007.LDAP.Authentication.Protocol.StartTLS', @@ -68,9 +629,9 @@ RQ_SRS_007_LDAP_Authentication_Protocol_StartTLS = Requirement( description=( '[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a\n' 'plain text `ldap://` protocol that is upgraded to [TLS].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_TLS_Certificate_Validation = Requirement( name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.Validation', @@ -81,9 +642,9 @@ RQ_SRS_007_LDAP_Authentication_TLS_Certificate_Validation = Requirement( uid=None, description=( '[ClickHouse] SHALL support certificate validation used for [TLS] connections.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SelfSigned = Requirement( name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SelfSigned', @@ -94,9 +655,9 @@ RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SelfSigned = Requirement( uid=None, description=( '[ClickHouse] SHALL support self-signed certificates for [TLS] connections.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SpecificCertificationAuthority = Requirement( name='RQ.SRS-007.LDAP.Authentication.TLS.Certificate.SpecificCertificationAuthority', @@ -107,9 +668,9 @@ RQ_SRS_007_LDAP_Authentication_TLS_Certificate_SpecificCertificationAuthority = uid=None, description=( '[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Server_Configuration_Invalid = Requirement( name='RQ.SRS-007.LDAP.Server.Configuration.Invalid', @@ -120,9 +681,9 @@ RQ_SRS_007_LDAP_Server_Configuration_Invalid = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_User_Configuration_Invalid = Requirement( name='RQ.SRS-007.LDAP.User.Configuration.Invalid', @@ -133,9 +694,9 @@ RQ_SRS_007_LDAP_User_Configuration_Invalid = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error and prohibit user login if user configuration is not valid.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Mechanism_Anonymous = Requirement( name='RQ.SRS-007.LDAP.Authentication.Mechanism.Anonymous', @@ -147,9 +708,9 @@ RQ_SRS_007_LDAP_Authentication_Mechanism_Anonymous = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind]\n' 'authentication mechanism.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Mechanism_Unauthenticated = Requirement( name='RQ.SRS-007.LDAP.Authentication.Mechanism.Unauthenticated', @@ -161,9 +722,9 @@ RQ_SRS_007_LDAP_Authentication_Mechanism_Unauthenticated = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind]\n' 'authentication mechanism.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Mechanism_NamePassword = Requirement( name='RQ.SRS-007.LDAP.Authentication.Mechanism.NamePassword', @@ -175,9 +736,9 @@ RQ_SRS_007_LDAP_Authentication_Mechanism_NamePassword = Requirement( description=( '[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind]\n' 'authentication mechanism.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Valid = Requirement( name='RQ.SRS-007.LDAP.Authentication.Valid', @@ -189,9 +750,9 @@ RQ_SRS_007_LDAP_Authentication_Valid = Requirement( description=( '[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if\n' 'user name and password match [LDAP] server records for the user.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Invalid = Requirement( name='RQ.SRS-007.LDAP.Authentication.Invalid', @@ -203,9 +764,9 @@ RQ_SRS_007_LDAP_Authentication_Invalid = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if either user name or password\n' 'do not match [LDAP] server records for the user.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser = Requirement( name='RQ.SRS-007.LDAP.Authentication.Invalid.DeletedUser', @@ -217,9 +778,9 @@ RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if the user\n' 'has been deleted from the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_UsernameChanged = Requirement( name='RQ.SRS-007.LDAP.Authentication.UsernameChanged', @@ -231,9 +792,9 @@ RQ_SRS_007_LDAP_Authentication_UsernameChanged = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if the username is changed\n' 'on the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_PasswordChanged = Requirement( name='RQ.SRS-007.LDAP.Authentication.PasswordChanged', @@ -245,9 +806,9 @@ RQ_SRS_007_LDAP_Authentication_PasswordChanged = Requirement( description=( '[ClickHouse] SHALL return an error and prohibit authentication if the password\n' 'for the user is changed on the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_LDAPServerRestart = Requirement( name='RQ.SRS-007.LDAP.Authentication.LDAPServerRestart', @@ -258,9 +819,9 @@ RQ_SRS_007_LDAP_Authentication_LDAPServerRestart = Requirement( uid=None, description=( '[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart = Requirement( name='RQ.SRS-007.LDAP.Authentication.ClickHouseServerRestart', @@ -271,9 +832,9 @@ RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart = Requirement( uid=None, description=( '[ClickHouse] SHALL support authenticating users after server is restarted.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Parallel = Requirement( name='RQ.SRS-007.LDAP.Authentication.Parallel', @@ -284,9 +845,9 @@ RQ_SRS_007_LDAP_Authentication_Parallel = Requirement( uid=None, description=( '[ClickHouse] SHALL support parallel authentication of users using [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid = Requirement( name='RQ.SRS-007.LDAP.Authentication.Parallel.ValidAndInvalid', @@ -299,9 +860,9 @@ RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid = Requirement( '[ClickHouse] SHALL support authentication of valid users and\n' 'prohibit authentication of invalid users using [LDAP] server\n' 'in parallel without having invalid attempts affecting valid authentications.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_UnreachableServer = Requirement( name='RQ.SRS-007.LDAP.UnreachableServer', @@ -312,9 +873,9 @@ RQ_SRS_007_LDAP_UnreachableServer = Requirement( uid=None, description=( '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Name = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Name', @@ -325,9 +886,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Name = Requirement( uid=None, description=( '[ClickHouse] SHALL not support empty string as a server name.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Host = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Host', @@ -339,9 +900,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Host = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify [LDAP]\n' 'server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Port = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Port', @@ -352,9 +913,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Port = Requirement( uid=None, description=( '[ClickHouse] SHALL support `` parameter to specify [LDAP] server port.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Port_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Port.Default', @@ -365,9 +926,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Port_Default = Requirement( uid=None, description=( '[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Prefix = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Prefix', @@ -379,9 +940,9 @@ RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Prefix = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify the prefix\n' 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Suffix = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Suffix', @@ -393,9 +954,9 @@ RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Suffix = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify the suffix\n' 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Value = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.AuthDN.Value', @@ -408,9 +969,9 @@ RQ_SRS_007_LDAP_Configuration_Server_AuthDN_Value = Requirement( '[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string.\n' '\n' "> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character.\n" + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS', @@ -421,9 +982,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS = Requirement( uid=None, description=( '[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Default', @@ -435,9 +996,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Default = Requirement( description=( '[ClickHouse] SHALL use `yes` value as the default for `` parameter\n' 'to enable SSL/TLS `ldaps://` protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_No = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.No', @@ -449,9 +1010,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_No = Requirement( description=( '[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable\n' 'plain text `ldap://` protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Yes = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.Yes', @@ -463,9 +1024,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Yes = Requirement( description=( '[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable\n' 'SSL/TLS `ldaps://` protocol.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_StartTLS = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.EnableTLS.Options.StartTLS', @@ -477,9 +1038,9 @@ RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_StartTLS = Requirement( description=( '[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable\n' 'legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS].\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion', @@ -491,9 +1052,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify\n' 'the minimum protocol version of SSL/TLS.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Values = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Values', @@ -505,9 +1066,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Values = Requirem description=( '[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2`\n' 'as a value of the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSMinimumProtocolVersion.Default', @@ -518,9 +1079,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSMinimumProtocolVersion_Default = Require uid=None, description=( '[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert', @@ -532,9 +1093,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify [TLS] peer\n' 'certificate verification behavior.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Default = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Default', @@ -545,9 +1106,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Default = Requiremen uid=None, description=( '[ClickHouse] SHALL use `demand` value as the default for the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Demand = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Demand', @@ -560,9 +1121,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Demand = Requirement '[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to\n' 'enable requesting of client certificate. If no certificate is provided, or a bad certificate is\n' 'provided, the session SHALL be immediately terminated.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Allow = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Allow', @@ -576,9 +1137,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Allow = Requirement( 'enable requesting of client certificate. If no\n' 'certificate is provided, the session SHALL proceed normally.\n' 'If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Try = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Try', @@ -592,9 +1153,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Try = Requirement( 'enable requesting of client certificate. If no certificate is provided, the session\n' 'SHALL proceed normally. If a bad certificate is provided, the session SHALL be\n' 'immediately terminated.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Never = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSRequireCert.Options.Never', @@ -606,9 +1167,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Never = Requirement( description=( '[ClickHouse] SHALL support specifying `never` as the value of `` parameter to\n' 'disable requesting of client certificate.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCertFile = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCertFile', @@ -620,9 +1181,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCertFile = Requirement( description=( '[ClickHouse] SHALL support `` to specify the path to certificate file used by\n' '[ClickHouse] to establish connection with the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSKeyFile = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSKeyFile', @@ -634,9 +1195,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSKeyFile = Requirement( description=( '[ClickHouse] SHALL support `` to specify the path to key file for the certificate\n' 'specified by the `` parameter.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCACertDir = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertDir', @@ -648,9 +1209,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCACertDir = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify to a path to\n' 'the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCACertFile = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCACertFile', @@ -662,9 +1223,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCACertFile = Requirement( description=( '[ClickHouse] SHALL support `` parameter to specify a path to a specific\n' '[CA] certificate file used to verify certificates provided by the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.TLSCipherSuite', @@ -685,9 +1246,9 @@ RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite = Requirement( '\n' 'The available suites SHALL depend on the [OpenSSL] library version and variant used to build\n' '[ClickHouse] and therefore might change.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( name='RQ.SRS-007.LDAP.Configuration.Server.Syntax', @@ -718,9 +1279,9 @@ RQ_SRS_007_LDAP_Configuration_Server_Syntax = Requirement( ' \n' '\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_RBAC = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.RBAC', @@ -736,9 +1297,9 @@ RQ_SRS_007_LDAP_Configuration_User_RBAC = Requirement( '```sql\n' "CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name'\n" '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Syntax = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Syntax', @@ -762,9 +1323,9 @@ RQ_SRS_007_LDAP_Configuration_User_Syntax = Requirement( ' \n' '\n' '```\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Name_Empty = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Name.Empty', @@ -775,9 +1336,9 @@ RQ_SRS_007_LDAP_Configuration_User_Name_Empty = Requirement( uid=None, description=( '[ClickHouse] SHALL not support empty string as a user name.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_BothPasswordAndLDAP = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.BothPasswordAndLDAP', @@ -789,9 +1350,9 @@ RQ_SRS_007_LDAP_Configuration_User_BothPasswordAndLDAP = Requirement( description=( '[ClickHouse] SHALL throw an error if `` is specified for the user and at the same\n' 'time user configuration contains any of the `` entries.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_NotDefined = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.NotDefined', @@ -801,12 +1362,12 @@ RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_NotDefined = Requireme type=None, uid=None, description=( - '[ClickHouse] SHALL throw an error during any authentification attempt\n' + '[ClickHouse] SHALL throw an error during any authentication attempt\n' 'if the name of the [LDAP] server used inside the `` entry\n' 'is not defined in the `` section.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_Empty = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.LDAP.InvalidServerName.Empty', @@ -816,12 +1377,12 @@ RQ_SRS_007_LDAP_Configuration_User_LDAP_InvalidServerName_Empty = Requirement( type=None, uid=None, description=( - '[ClickHouse] SHALL throw an error during any authentification attempt\n' + '[ClickHouse] SHALL throw an error during any authentication attempt\n' 'if the name of the [LDAP] server used inside the `` entry\n' 'is empty.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_OnlyOneServer = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.OnlyOneServer', @@ -832,9 +1393,9 @@ RQ_SRS_007_LDAP_Configuration_User_OnlyOneServer = Requirement( uid=None, description=( '[ClickHouse] SHALL support specifying only one [LDAP] server for a given user.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Name_Long = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Name.Long', @@ -846,9 +1407,9 @@ RQ_SRS_007_LDAP_Configuration_User_Name_Long = Requirement( description=( '[ClickHouse] SHALL support long user names of at least 256 bytes\n' 'to specify users that can be authenticated using an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Configuration_User_Name_UTF8 = Requirement( name='RQ.SRS-007.LDAP.Configuration.User.Name.UTF8', @@ -859,9 +1420,9 @@ RQ_SRS_007_LDAP_Configuration_User_Name_UTF8 = Requirement( uid=None, description=( '[ClickHouse] SHALL support user names that contain [UTF-8] characters.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Username_Empty = Requirement( name='RQ.SRS-007.LDAP.Authentication.Username.Empty', @@ -872,9 +1433,9 @@ RQ_SRS_007_LDAP_Authentication_Username_Empty = Requirement( uid=None, description=( '[ClickHouse] SHALL not support authenticating users with empty username.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Username_Long = Requirement( name='RQ.SRS-007.LDAP.Authentication.Username.Long', @@ -885,9 +1446,9 @@ RQ_SRS_007_LDAP_Authentication_Username_Long = Requirement( uid=None, description=( '[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Username_UTF8 = Requirement( name='RQ.SRS-007.LDAP.Authentication.Username.UTF8', @@ -898,9 +1459,9 @@ RQ_SRS_007_LDAP_Authentication_Username_UTF8 = Requirement( uid=None, description=( '[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Password_Empty = Requirement( name='RQ.SRS-007.LDAP.Authentication.Password.Empty', @@ -913,9 +1474,9 @@ RQ_SRS_007_LDAP_Authentication_Password_Empty = Requirement( '[ClickHouse] SHALL not support authenticating users with empty passwords\n' 'even if an empty password is valid for the user and\n' 'is allowed by the [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Password_Long = Requirement( name='RQ.SRS-007.LDAP.Authentication.Password.Long', @@ -927,9 +1488,9 @@ RQ_SRS_007_LDAP_Authentication_Password_Long = Requirement( description=( '[ClickHouse] SHALL support long password of at least 256 bytes\n' 'that can be used to authenticate users using an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) RQ_SRS_007_LDAP_Authentication_Password_UTF8 = Requirement( name='RQ.SRS-007.LDAP.Authentication.Password.UTF8', @@ -941,6 +1502,6 @@ RQ_SRS_007_LDAP_Authentication_Password_UTF8 = Requirement( description=( '[ClickHouse] SHALL support [UTF-8] characters in passwords\n' 'used to authenticate users using an [LDAP] server.\n' + '\n' ), - link=None - ) + link=None) From eee81eef43610da23127c14fbf0d2a28bc936df2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Oct 2020 01:58:53 +0300 Subject: [PATCH 265/325] Remove dependency on "services" database --- contrib/mariadb-connector-c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/mariadb-connector-c b/contrib/mariadb-connector-c index f5638e954a7..1485b0de3ea 160000 --- a/contrib/mariadb-connector-c +++ b/contrib/mariadb-connector-c @@ -1 +1 @@ -Subproject commit f5638e954a79f50bac7c7a5deaa5a241e0ce8b5f +Subproject commit 1485b0de3eaa1508dfe49a5ba1e4aa2a71fd8335 From 309e1c622f2a6dfcf797d0fa369d6416c804dc15 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Oct 2020 03:13:59 +0300 Subject: [PATCH 266/325] Added a test for #11364 --- .../0_stateless/01534_lambda_array_join.reference | 3 +++ .../queries/0_stateless/01534_lambda_array_join.sql | 12 ++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 tests/queries/0_stateless/01534_lambda_array_join.reference create mode 100644 tests/queries/0_stateless/01534_lambda_array_join.sql diff --git a/tests/queries/0_stateless/01534_lambda_array_join.reference b/tests/queries/0_stateless/01534_lambda_array_join.reference new file mode 100644 index 00000000000..78e098be306 --- /dev/null +++ b/tests/queries/0_stateless/01534_lambda_array_join.reference @@ -0,0 +1,3 @@ +[NULL] +[1,1] +\N 70 diff --git a/tests/queries/0_stateless/01534_lambda_array_join.sql b/tests/queries/0_stateless/01534_lambda_array_join.sql new file mode 100644 index 00000000000..aee9dd1411a --- /dev/null +++ b/tests/queries/0_stateless/01534_lambda_array_join.sql @@ -0,0 +1,12 @@ +SELECT arrayMap(x -> concat(x, concat(arrayJoin([1]), x, NULL), ''), [1]); +SELECT arrayMap(x -> arrayJoin([1]), [1, 2]); + +SELECT + arrayJoin(arrayMap(x -> reinterpretAsUInt8(substring(randomString(range(randomString(1048577), NULL), arrayJoin(arrayMap(x -> reinterpretAsUInt8(substring(randomString(range(NULL), 65537), 255)), range(1))), substring(randomString(NULL), x + 7), '257'), 1025)), range(7))) AS byte, + count() AS c + FROM numbers(10) + GROUP BY + arrayMap(x -> reinterpretAsUInt8(substring(randomString(randomString(range(randomString(255), NULL)), NULL))), range(3)), + randomString(range(randomString(1048577), NULL), NULL), + byte + ORDER BY byte ASC; From 05099bebcf77a6c5ea958ad552ac11d07b2955e1 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 28 Oct 2020 02:50:27 +0300 Subject: [PATCH 267/325] fix --- src/Interpreters/InterpreterDropQuery.h | 2 +- tests/queries/0_stateless/01193_metadata_loading.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterDropQuery.h b/src/Interpreters/InterpreterDropQuery.h index 1fc4acffa04..fe5362985de 100644 --- a/src/Interpreters/InterpreterDropQuery.h +++ b/src/Interpreters/InterpreterDropQuery.h @@ -30,7 +30,7 @@ private: Context & context; BlockIO executeToDatabase(const ASTDropQuery & query); - BlockIO executeToDatabaseImpl(const ASTDropQuery & query, DatabasePtr & db, std::vector & uuids_to_wait); + BlockIO executeToDatabaseImpl(const ASTDropQuery & query, DatabasePtr & database, std::vector & uuids_to_wait); BlockIO executeToTable(const ASTDropQuery & query); BlockIO executeToTableImpl(const ASTDropQuery & query, DatabasePtr & db, UUID & uuid_to_wait); diff --git a/tests/queries/0_stateless/01193_metadata_loading.sh b/tests/queries/0_stateless/01193_metadata_loading.sh index 0ee583a7265..319b537e84b 100755 --- a/tests/queries/0_stateless/01193_metadata_loading.sh +++ b/tests/queries/0_stateless/01193_metadata_loading.sh @@ -49,4 +49,4 @@ $CLICKHOUSE_CLIENT -q "SELECT if(quantile(0.5)(query_duration_ms) < $max_time_ms $CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM $db.table_merge GROUP BY i, d, s, n.i, n.f ORDER BY i" -$CLICKHOUSE_CLIENT -q "DROP DATABASE $db" +$CLICKHOUSE_CLIENT -q "DROP DATABASE $db" --database_atomic_wait_for_drop_and_detach_synchronously=0 From f573aeb97e4373956355c82fe35d12111dc2e7fe Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Oct 2020 03:32:31 +0300 Subject: [PATCH 268/325] Whitespace --- src/Interpreters/DDLWorker.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 32d0e25bde5..fc7f5c2f765 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -1246,7 +1246,6 @@ public: size_t num_unfinished_hosts = waiting_hosts.size() - num_hosts_finished; size_t num_active_hosts = current_active_hosts.size(); - throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Watching task {} is executing longer than distributed_ddl_task_timeout (={}) seconds. " "There are {} unfinished hosts ({} of them are currently active), they are going to execute the query in background", From b11229c1b6c37bb11489c1df72e850d5421d7af6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Oct 2020 04:15:41 +0300 Subject: [PATCH 269/325] Add a test for #13338 --- .../01535_decimal_round_scale_overflow_check.reference | 0 .../0_stateless/01535_decimal_round_scale_overflow_check.sql | 1 + 2 files changed, 1 insertion(+) create mode 100644 tests/queries/0_stateless/01535_decimal_round_scale_overflow_check.reference create mode 100644 tests/queries/0_stateless/01535_decimal_round_scale_overflow_check.sql diff --git a/tests/queries/0_stateless/01535_decimal_round_scale_overflow_check.reference b/tests/queries/0_stateless/01535_decimal_round_scale_overflow_check.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01535_decimal_round_scale_overflow_check.sql b/tests/queries/0_stateless/01535_decimal_round_scale_overflow_check.sql new file mode 100644 index 00000000000..18509221203 --- /dev/null +++ b/tests/queries/0_stateless/01535_decimal_round_scale_overflow_check.sql @@ -0,0 +1 @@ +SELECT round(toDecimal32(1, 0), -9223372036854775806); -- { serverError 69 } From 7254f40ba89bea76242bac643a2ec2140e5b7c8b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Oct 2020 04:16:52 +0300 Subject: [PATCH 270/325] Add range check for "scale" argument of decimal rounding function to prevent overflow --- src/Functions/FunctionsRound.h | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/Functions/FunctionsRound.h b/src/Functions/FunctionsRound.h index 7a8304dbfa9..542463255d3 100644 --- a/src/Functions/FunctionsRound.h +++ b/src/Functions/FunctionsRound.h @@ -31,6 +31,7 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ARGUMENT_OUT_OF_BOUND; extern const int ILLEGAL_COLUMN; extern const int BAD_ARGUMENTS; } @@ -84,6 +85,9 @@ enum class TieBreakingMode Bankers, // use banker's rounding }; +/// For N, no more than the number of digits in the largest type. +using Scale = Int16; + /** Rounding functions for integer values. */ @@ -416,7 +420,7 @@ private: using Container = typename ColumnDecimal::Container; public: - static NO_INLINE void apply(const Container & in, Container & out, Int64 scale_arg) + static NO_INLINE void apply(const Container & in, Container & out, Scale scale_arg) { scale_arg = in.getScale() - scale_arg; if (scale_arg > 0) @@ -458,7 +462,7 @@ class Dispatcher FloatRoundingImpl, IntegerRoundingImpl>; - static ColumnPtr apply(const ColumnVector * col, Int64 scale_arg) + static ColumnPtr apply(const ColumnVector * col, Scale scale_arg) { auto col_res = ColumnVector::create(); @@ -487,7 +491,7 @@ class Dispatcher return col_res; } - static ColumnPtr apply(const ColumnDecimal * col, Int64 scale_arg) + static ColumnPtr apply(const ColumnDecimal * col, Scale scale_arg) { const typename ColumnDecimal::Container & vec_src = col->getData(); @@ -501,7 +505,7 @@ class Dispatcher } public: - static ColumnPtr apply(const IColumn * column, Int64 scale_arg) + static ColumnPtr apply(const IColumn * column, Scale scale_arg) { if constexpr (IsNumber) return apply(checkAndGetColumn>(column), scale_arg); @@ -544,20 +548,25 @@ public: return arguments[0]; } - static Int64 getScaleArg(ColumnsWithTypeAndName & arguments) + static Scale getScaleArg(ColumnsWithTypeAndName & arguments) { if (arguments.size() == 2) { const IColumn & scale_column = *arguments[1].column; if (!isColumnConst(scale_column)) - throw Exception("Scale argument for rounding functions must be constant.", ErrorCodes::ILLEGAL_COLUMN); + throw Exception("Scale argument for rounding functions must be constant", ErrorCodes::ILLEGAL_COLUMN); Field scale_field = assert_cast(scale_column).getField(); if (scale_field.getType() != Field::Types::UInt64 && scale_field.getType() != Field::Types::Int64) - throw Exception("Scale argument for rounding functions must have integer type.", ErrorCodes::ILLEGAL_COLUMN); + throw Exception("Scale argument for rounding functions must have integer type", ErrorCodes::ILLEGAL_COLUMN); - return scale_field.get(); + Int64 scale64 = scale_field.get(); + if (scale64 > std::numeric_limits::max() + || scale64 < std::numeric_limits::min()) + throw Exception("Scale argument for rounding function is too large", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + return scale64; } return 0; } @@ -568,7 +577,7 @@ public: ColumnPtr executeImpl(ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override { const ColumnWithTypeAndName & column = arguments[0]; - Int64 scale_arg = getScaleArg(arguments); + Scale scale_arg = getScaleArg(arguments); ColumnPtr res; auto call = [&](const auto & types) -> bool From 690a3b431075f1676c67c661633708451e0908cd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Oct 2020 04:24:30 +0300 Subject: [PATCH 271/325] Fix test --- tests/queries/0_stateless/01526_client_start_and_exit.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01526_client_start_and_exit.sh b/tests/queries/0_stateless/01526_client_start_and_exit.sh index 0b3a2ee6b4f..c179be79d03 100755 --- a/tests/queries/0_stateless/01526_client_start_and_exit.sh +++ b/tests/queries/0_stateless/01526_client_start_and_exit.sh @@ -9,10 +9,11 @@ ${CLICKHOUSE_CLIENT} -q "SELECT 'CREATE TABLE test_' || hex(randomPrintableASCII function stress() { while true; do - "$CURDIR"/01526_client_start_and_exit.expect | grep -v -P 'ClickHouse client|Connecting|Connected|:\) Bye\.|^\s*$|spawn bash|^0\s*$' + "${CURDIR}"/01526_client_start_and_exit.expect | grep -v -P 'ClickHouse client|Connecting|Connected|:\) Bye\.|^\s*$|spawn bash|^0\s*$' done } +export CURDIR export -f stress for _ in {1..10}; do From dec51cdb0b1664660bbb9fea633b245e6df9b3e8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Oct 2020 04:27:10 +0300 Subject: [PATCH 272/325] Add a test for #13342 --- tests/queries/0_stateless/01536_fuzz_cast.reference | 0 tests/queries/0_stateless/01536_fuzz_cast.sql | 1 + 2 files changed, 1 insertion(+) create mode 100644 tests/queries/0_stateless/01536_fuzz_cast.reference create mode 100644 tests/queries/0_stateless/01536_fuzz_cast.sql diff --git a/tests/queries/0_stateless/01536_fuzz_cast.reference b/tests/queries/0_stateless/01536_fuzz_cast.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01536_fuzz_cast.sql b/tests/queries/0_stateless/01536_fuzz_cast.sql new file mode 100644 index 00000000000..436d76b5c4c --- /dev/null +++ b/tests/queries/0_stateless/01536_fuzz_cast.sql @@ -0,0 +1 @@ +SELECT CAST(arrayJoin([NULL, '', '', NULL, '', NULL, '01.02.2017 03:04\005GMT', '', NULL, '01/02/2017 03:04:05 MSK01/02/\0017 03:04:05 MSK', '', NULL, '03/04/201903/04/201903/04/\001903/04/2019']), 'Enum8(\'a\' = 1, \'b\' = 2)') AS x; -- { serverError 349 } From 81cd172af113911021c207dab81737081afd8e40 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Oct 2020 04:31:47 +0300 Subject: [PATCH 273/325] Add a test for #13893 --- tests/queries/0_stateless/01537_fuzz_count_equal.reference | 1 + tests/queries/0_stateless/01537_fuzz_count_equal.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 tests/queries/0_stateless/01537_fuzz_count_equal.reference create mode 100644 tests/queries/0_stateless/01537_fuzz_count_equal.sql diff --git a/tests/queries/0_stateless/01537_fuzz_count_equal.reference b/tests/queries/0_stateless/01537_fuzz_count_equal.reference new file mode 100644 index 00000000000..dec7d2fabd2 --- /dev/null +++ b/tests/queries/0_stateless/01537_fuzz_count_equal.reference @@ -0,0 +1 @@ +\N diff --git a/tests/queries/0_stateless/01537_fuzz_count_equal.sql b/tests/queries/0_stateless/01537_fuzz_count_equal.sql new file mode 100644 index 00000000000..fde3fe19124 --- /dev/null +++ b/tests/queries/0_stateless/01537_fuzz_count_equal.sql @@ -0,0 +1 @@ +SELECT DISTINCT NULL = countEqual(materialize([arrayJoin([NULL, NULL, NULL]), NULL AS x, arrayJoin([255, 1025, NULL, NULL]), arrayJoin([2, 1048576, NULL, NULL])]), materialize(x)); From b767efd889667b6a677dc05686c4dec87d052639 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Oct 2020 04:35:40 +0300 Subject: [PATCH 274/325] Add a test for #15540 --- .../queries/0_stateless/01538_fuzz_aggregate.reference | 0 tests/queries/0_stateless/01538_fuzz_aggregate.sql | 10 ++++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/01538_fuzz_aggregate.reference create mode 100644 tests/queries/0_stateless/01538_fuzz_aggregate.sql diff --git a/tests/queries/0_stateless/01538_fuzz_aggregate.reference b/tests/queries/0_stateless/01538_fuzz_aggregate.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01538_fuzz_aggregate.sql b/tests/queries/0_stateless/01538_fuzz_aggregate.sql new file mode 100644 index 00000000000..13dadabda63 --- /dev/null +++ b/tests/queries/0_stateless/01538_fuzz_aggregate.sql @@ -0,0 +1,10 @@ +SELECT + count(), + sum(ns) +FROM +( + SELECT intDiv(number, NULL) AS k + FROM system.numbers_mt + GROUP BY k +) +ARRAY JOIN ns; -- { serverError 47 } From 62e7e0637f6efda6f474bc451e3528be64b92d13 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Tue, 27 Oct 2020 23:34:19 -0300 Subject: [PATCH 275/325] test for #14144 --- .../01533_distinct_nullable_uuid.reference | 4 ++ .../01533_distinct_nullable_uuid.sql | 38 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 tests/queries/0_stateless/01533_distinct_nullable_uuid.reference create mode 100644 tests/queries/0_stateless/01533_distinct_nullable_uuid.sql diff --git a/tests/queries/0_stateless/01533_distinct_nullable_uuid.reference b/tests/queries/0_stateless/01533_distinct_nullable_uuid.reference new file mode 100644 index 00000000000..e02acad09d6 --- /dev/null +++ b/tests/queries/0_stateless/01533_distinct_nullable_uuid.reference @@ -0,0 +1,4 @@ +442d3ff4-842a-45bb-8b02-b616122c0dc6 +05fe40cb-1d0c-45b0-8e60-8e311c2463f1 +2fc89389-4728-4b30-9e51-b5bc3ad215f6 +10000 diff --git a/tests/queries/0_stateless/01533_distinct_nullable_uuid.sql b/tests/queries/0_stateless/01533_distinct_nullable_uuid.sql new file mode 100644 index 00000000000..926739d3f58 --- /dev/null +++ b/tests/queries/0_stateless/01533_distinct_nullable_uuid.sql @@ -0,0 +1,38 @@ +DROP TABLE IF EXISTS bug_14144; + +CREATE TABLE bug_14144 +( meta_source_req_uuid Nullable(UUID), + a Int64, + meta_source_type String +) +ENGINE = MergeTree +ORDER BY a; + +INSERT INTO bug_14144 SELECT cast(toUUID('442d3ff4-842a-45bb-8b02-b616122c0dc6'), 'Nullable(UUID)'), number, 'missing' FROM numbers(1000); + +INSERT INTO bug_14144 SELECT cast(toUUIDOrZero('2fc89389-4728-4b30-9e51-b5bc3ad215f6'), 'Nullable(UUID)'), number, 'missing' FROM numbers(1000); + +INSERT INTO bug_14144 SELECT cast(toUUIDOrNull('05fe40cb-1d0c-45b0-8e60-8e311c2463f1'), 'Nullable(UUID)'), number, 'missing' FROM numbers(1000); + +SELECT DISTINCT meta_source_req_uuid +FROM bug_14144 +WHERE meta_source_type = 'missing' +ORDER BY meta_source_req_uuid ASC; + +TRUNCATE TABLE bug_14144; + +INSERT INTO bug_14144 SELECT generateUUIDv4(), number, 'missing' FROM numbers(10000); + +SELECT COUNT() FROM ( + SELECT DISTINCT meta_source_req_uuid + FROM bug_14144 + WHERE meta_source_type = 'missing' + ORDER BY meta_source_req_uuid ASC + LIMIT 100000 +); + +DROP TABLE bug_14144; + + + + From 2b26cbbc97affb8b5678d2dc9dd004b813cf60cf Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 28 Oct 2020 10:01:57 +0300 Subject: [PATCH 276/325] Add missed file --- .../test_concurrent_ttl_merges/configs/log_conf.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tests/integration/test_concurrent_ttl_merges/configs/log_conf.xml diff --git a/tests/integration/test_concurrent_ttl_merges/configs/log_conf.xml b/tests/integration/test_concurrent_ttl_merges/configs/log_conf.xml new file mode 100644 index 00000000000..318a6bca95d --- /dev/null +++ b/tests/integration/test_concurrent_ttl_merges/configs/log_conf.xml @@ -0,0 +1,12 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + From ff686700291f9765db614c3ba1a01d3042415c67 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 22 Oct 2020 19:43:36 +0300 Subject: [PATCH 277/325] add missing __init__.py --- tests/integration/test_system_flush_logs/__init__.py | 0 .../{test_SYSTEM_FLUSH_LOGS => test_system_flush_logs}/test.py | 0 tests/integration/test_system_queries/test.py | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 tests/integration/test_system_flush_logs/__init__.py rename tests/integration/{test_SYSTEM_FLUSH_LOGS => test_system_flush_logs}/test.py (100%) diff --git a/tests/integration/test_system_flush_logs/__init__.py b/tests/integration/test_system_flush_logs/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_SYSTEM_FLUSH_LOGS/test.py b/tests/integration/test_system_flush_logs/test.py similarity index 100% rename from tests/integration/test_SYSTEM_FLUSH_LOGS/test.py rename to tests/integration/test_system_flush_logs/test.py diff --git a/tests/integration/test_system_queries/test.py b/tests/integration/test_system_queries/test.py index 7f5bce97805..b159e8b4cf3 100644 --- a/tests/integration/test_system_queries/test.py +++ b/tests/integration/test_system_queries/test.py @@ -107,7 +107,7 @@ def test_RELOAD_CONFIG_AND_MACROS(started_cluster): assert TSV(instance.query("select * from system.macros")) == TSV("instance\tch1\nmac\tro\n") -def test_SYSTEM_FLUSH_LOGS(started_cluster): +def test_system_flush_logs(started_cluster): instance = cluster.instances['ch1'] instance.query(''' SET log_queries = 0; From 3c31a5134e158d8155ea2a9e0c97c4e9e2585dbf Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 28 Oct 2020 14:03:26 +0300 Subject: [PATCH 278/325] Fix bug --- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index a1f20016cb3..880ad4dd0d3 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -490,7 +490,7 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper { std::sort(log_entries.begin(), log_entries.end()); - for (size_t entry_idx = 0, num_entries = log_entries.size(); entry_idx < num_entries; entry_idx += current_multi_batch_size) + for (size_t entry_idx = 0, num_entries = log_entries.size(); entry_idx < num_entries;) { auto begin = log_entries.begin() + entry_idx; auto end = entry_idx + current_multi_batch_size >= log_entries.size() @@ -498,6 +498,9 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper : (begin + current_multi_batch_size); auto last = end - 1; + /// Increment entry_idx before batch size increase (we copied at most current_multi_batch_size entries) + entry_idx += current_multi_batch_size; + /// Increase the batch size exponentially, so it will saturate to MAX_MULTI_OPS. if (current_multi_batch_size < MAX_MULTI_OPS) current_multi_batch_size = std::min(MAX_MULTI_OPS, current_multi_batch_size * 2); From d126024248f0bd50aa1ce9e1149e5a66a9bcc9e7 Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Wed, 28 Oct 2020 13:58:38 +0100 Subject: [PATCH 279/325] Docker alpine build --- docker/server/.dockerignore | 8 ++ docker/server/.gitignore | 1 + docker/server/Dockerfile.alpine | 26 +++++ docker/server/alpine-build.sh | 59 +++++++++++ docker/server/entrypoint.alpine.sh | 152 +++++++++++++++++++++++++++++ 5 files changed, 246 insertions(+) create mode 100644 docker/server/.dockerignore create mode 100644 docker/server/.gitignore create mode 100644 docker/server/Dockerfile.alpine create mode 100755 docker/server/alpine-build.sh create mode 100755 docker/server/entrypoint.alpine.sh diff --git a/docker/server/.dockerignore b/docker/server/.dockerignore new file mode 100644 index 00000000000..468a8cafb00 --- /dev/null +++ b/docker/server/.dockerignore @@ -0,0 +1,8 @@ +# post / preinstall scripts (not needed, we do it in Dockerfile) +alpine-root/install/* + +# docs (looks useless) +alpine-root/usr/share/doc/* + +# packages, etc. (used by prepare.sh) +alpine-root/tgz-packages/* \ No newline at end of file diff --git a/docker/server/.gitignore b/docker/server/.gitignore new file mode 100644 index 00000000000..4081b5f124c --- /dev/null +++ b/docker/server/.gitignore @@ -0,0 +1 @@ +alpine-root/* \ No newline at end of file diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine new file mode 100644 index 00000000000..fc2756eac8c --- /dev/null +++ b/docker/server/Dockerfile.alpine @@ -0,0 +1,26 @@ +FROM alpine + +ENV LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 \ + TZ=UTC \ + CLICKHOUSE_CONFIG=/etc/clickhouse-server/config.xml + +COPY alpine-root/ / + +# from https://github.com/ClickHouse/ClickHouse/blob/master/debian/clickhouse-server.postinst +RUN addgroup clickhouse \ + && adduser -S -H -h /nonexistent -s /bin/false -G clickhouse -g "ClickHouse server" clickhouse \ + && chown clickhouse:clickhouse /var/lib/clickhouse \ + && chmod 700 /var/lib/clickhouse \ + && chown root:clickhouse /var/log/clickhouse-server \ + && chmod 775 /var/log/clickhouse-server \ + && chmod +x /entrypoint.sh \ + && apk add --no-cache su-exec + +EXPOSE 9000 8123 9009 + +VOLUME /var/lib/clickhouse \ + /var/log/clickhouse-server + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/server/alpine-build.sh b/docker/server/alpine-build.sh new file mode 100755 index 00000000000..30101225b3e --- /dev/null +++ b/docker/server/alpine-build.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -x + +REPO_CHANNEL="${REPO_CHANNEL:-stable}" # lts / testing / prestable / etc +REPO_URL="${REPO_URL:-"https://repo.yandex.ru/clickhouse/tgz/${REPO_CHANNEL}"}" +VERSION="${VERSION:-20.9.3.45}" + +# where original files live +DOCKER_BUILD_FOLDER="${BASH_SOURCE%/*}" + +# we will create root for our image here +CONTAINER_ROOT_FOLDER="${DOCKER_BUILD_FOLDER}/alpine-root" + +# where to put downloaded tgz +TGZ_PACKAGES_FOLDER="${CONTAINER_ROOT_FOLDER}/tgz-packages" + +# clean up the root from old runs +rm -rf "$CONTAINER_ROOT_FOLDER" + +mkdir -p "$TGZ_PACKAGES_FOLDER" + +PACKAGES=( "clickhouse-client" "clickhouse-server" "clickhouse-common-static" ) + +# download tars from the repo +for package in "${PACKAGES[@]}" +do + wget -q --show-progress "${REPO_URL}/${package}-${VERSION}.tgz" -O "${TGZ_PACKAGES_FOLDER}/${package}-${VERSION}.tgz" +done + +# unpack tars +for package in "${PACKAGES[@]}" +do + tar xvzf "${TGZ_PACKAGES_FOLDER}/${package}-${VERSION}.tgz" --strip-components=2 -C "$CONTAINER_ROOT_FOLDER" +done + +# prepare few more folders +mkdir -p "${CONTAINER_ROOT_FOLDER}/etc/clickhouse-server/users.d" \ + "${CONTAINER_ROOT_FOLDER}/etc/clickhouse-server/config.d" \ + "${CONTAINER_ROOT_FOLDER}/var/log/clickhouse-server" \ + "${CONTAINER_ROOT_FOLDER}/var/lib/clickhouse" \ + "${CONTAINER_ROOT_FOLDER}/docker-entrypoint-initdb.d" \ + "${CONTAINER_ROOT_FOLDER}/lib64" + +cp "${DOCKER_BUILD_FOLDER}/docker_related_config.xml" "${CONTAINER_ROOT_FOLDER}/etc/clickhouse-server/config.d/" +cp "${DOCKER_BUILD_FOLDER}/entrypoint.alpine.sh" "${CONTAINER_ROOT_FOLDER}/entrypoint.sh" + +## get glibc components from ubuntu 20.04 and put them to expected place +docker pull ubuntu:20.04 +ubuntu20image=$(docker create --rm ubuntu:20.04) +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libc.so.6 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libdl.so.2 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libm.so.6 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libpthread.so.0 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/librt.so.1 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libnss_dns.so.2 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib/x86_64-linux-gnu/libresolv.so.2 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L ${ubuntu20image}:/lib64/ld-linux-x86-64.so.2 "${CONTAINER_ROOT_FOLDER}/lib64" + +docker build "$DOCKER_BUILD_FOLDER" -f Dockerfile.alpine -t "yandex/clickhouse-server:${VERSION}-alpine" --pull \ No newline at end of file diff --git a/docker/server/entrypoint.alpine.sh b/docker/server/entrypoint.alpine.sh new file mode 100755 index 00000000000..e2edda9ca26 --- /dev/null +++ b/docker/server/entrypoint.alpine.sh @@ -0,0 +1,152 @@ +#!/bin/sh +#set -x + +DO_CHOWN=1 +if [ "$CLICKHOUSE_DO_NOT_CHOWN" = 1 ]; then + DO_CHOWN=0 +fi + +CLICKHOUSE_UID="${CLICKHOUSE_UID:-"$(id -u clickhouse)"}" +CLICKHOUSE_GID="${CLICKHOUSE_GID:-"$(id -g clickhouse)"}" + +# support --user +if [ "$(id -u)" = "0" ]; then + USER=$CLICKHOUSE_UID + GROUP=$CLICKHOUSE_GID + # busybox has setuidgid & chpst buildin + gosu="su-exec $USER:$GROUP" +else + USER="$(id -u)" + GROUP="$(id -g)" + gosu="" + DO_CHOWN=0 +fi + +# set some vars +CLICKHOUSE_CONFIG="${CLICKHOUSE_CONFIG:-/etc/clickhouse-server/config.xml}" + +# port is needed to check if clickhouse-server is ready for connections +HTTP_PORT="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=http_port)" + +# get CH directories locations +DATA_DIR="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=path || true)" +TMP_DIR="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=tmp_path || true)" +USER_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=user_files_path || true)" +LOG_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=logger.log || true)" +LOG_DIR="$(dirname $LOG_PATH || true)" +ERROR_LOG_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=logger.errorlog || true)" +ERROR_LOG_DIR="$(dirname $ERROR_LOG_PATH || true)" +FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file $CLICKHOUSE_CONFIG --key=format_schema_path || true)" + +CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}" +CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}" +CLICKHOUSE_DB="${CLICKHOUSE_DB:-}" + +for dir in "$DATA_DIR" \ + "$ERROR_LOG_DIR" \ + "$LOG_DIR" \ + "$TMP_DIR" \ + "$USER_PATH" \ + "$FORMAT_SCHEMA_PATH" +do + # check if variable not empty + [ -z "$dir" ] && continue + # ensure directories exist + if ! mkdir -p "$dir"; then + echo "Couldn't create necessary directory: $dir" + exit 1 + fi + + if [ "$DO_CHOWN" = "1" ]; then + # ensure proper directories permissions + chown -R "$USER:$GROUP" "$dir" + elif [ "$(stat -c %u "$dir")" != "$USER" ]; then + echo "Necessary directory '$dir' isn't owned by user with id '$USER'" + exit 1 + fi +done + +# if clickhouse user is defined - create it (user "default" already exists out of box) +if [ -n "$CLICKHOUSE_USER" ] && [ "$CLICKHOUSE_USER" != "default" ] || [ -n "$CLICKHOUSE_PASSWORD" ]; then + echo "$0: create new user '$CLICKHOUSE_USER' instead 'default'" + cat < /etc/clickhouse-server/users.d/default-user.xml + + + + + + + + <${CLICKHOUSE_USER}> + default + + ::/0 + + ${CLICKHOUSE_PASSWORD} + default + + + +EOT +fi + +if [ -n "$(ls /docker-entrypoint-initdb.d/)" ] || [ -n "$CLICKHOUSE_DB" ]; then + # Listen only on localhost until the initialization is done + $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG -- --listen_host=127.0.0.1 & + pid="$!" + + # check if clickhouse is ready to accept connections + # will try to send ping clickhouse via http_port (max 6 retries, with 1 sec timeout and 1 sec delay between retries) + tries=6 + while ! wget --spider -T 1 -q "http://localhost:$HTTP_PORT/ping" 2>/dev/null; do + if [ "$tries" -le "0" ]; then + echo >&2 'ClickHouse init process failed.' + exit 1 + fi + tries=$(( tries-1 )) + sleep 1 + done + + if [ ! -z "$CLICKHOUSE_PASSWORD" ]; then + printf -v WITH_PASSWORD '%s %q' "--password" "$CLICKHOUSE_PASSWORD" + fi + + clickhouseclient="clickhouse-client --multiquery -u $CLICKHOUSE_USER $WITH_PASSWORD " + + # create default database, if defined + if [ -n "$CLICKHOUSE_DB" ]; then + echo "$0: create database '$CLICKHOUSE_DB'" + "$clickhouseclient" -q "CREATE DATABASE IF NOT EXISTS $CLICKHOUSE_DB"; + fi + + for f in /docker-entrypoint-initdb.d/*; do + case "$f" in + *.sh) + if [ -x "$f" ]; then + echo "$0: running $f" + "$f" + else + echo "$0: sourcing $f" + . "$f" + fi + ;; + *.sql) echo "$0: running $f"; cat "$f" | "$clickhouseclient" ; echo ;; + *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "$clickhouseclient"; echo ;; + *) echo "$0: ignoring $f" ;; + esac + echo + done + + if ! kill -s TERM "$pid" || ! wait "$pid"; then + echo >&2 'Finishing of ClickHouse init process failed.' + exit 1 + fi +fi + +# if no args passed to `docker run` or first argument start with `--`, then the user is passing clickhouse-server arguments +if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then + exec $gosu /usr/bin/clickhouse-server --config-file=$CLICKHOUSE_CONFIG "$@" +fi + +# Otherwise, we assume the user want to run his own process, for example a `bash` shell to explore this image +exec "$@" From 2d28d97233622beb8d7a97299e8c7a437ee52dfc Mon Sep 17 00:00:00 2001 From: filimonov <1549571+filimonov@users.noreply.github.com> Date: Wed, 28 Oct 2020 15:13:27 +0100 Subject: [PATCH 280/325] Update other-functions.md --- docs/en/sql-reference/functions/other-functions.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 518479fb728..2cc80dcffc1 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -626,7 +626,12 @@ neighbor(column, offset[, default_value]) ``` The result of the function depends on the affected data blocks and the order of data in the block. -If you make a subquery with ORDER BY and call the function from outside the subquery, you can get the expected result. + +!!! warning "Warning" + It can reach the neighbor rows only inside the currently processed data block. + +The rows order used during the calculation of `neighbor` can differ from the order of rows returned to the user. +To prevent that you can make a subquery with ORDER BY and call the function from outside the subquery. **Parameters** @@ -731,8 +736,13 @@ Result: Calculates the difference between successive row values ​​in the data block. Returns 0 for the first row and the difference from the previous row for each subsequent row. +!!! warning "Warning" + It can reach the previos row only inside the currently processed data block. + The result of the function depends on the affected data blocks and the order of data in the block. -If you make a subquery with ORDER BY and call the function from outside the subquery, you can get the expected result. + +The rows order used during the calculation of `runningDifference` can differ from the order of rows returned to the user. +To prevent that you can make a subquery with ORDER BY and call the function from outside the subquery. Example: From a1f1db753b7705177f9472181f816e2a036289f8 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Wed, 28 Oct 2020 18:23:10 +0300 Subject: [PATCH 281/325] Update CompressedReadBufferBase.cpp --- src/Compression/CompressedReadBufferBase.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index be2f697e1b3..7a6b605d015 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -185,9 +185,9 @@ void CompressedReadBufferBase::decompress(char * to, size_t size_decompressed, s } else { - throw Exception("Data compressed with different methods, given method byte " + throw Exception("Data compressed with different methods, given method byte 0x" + getHexUIntLowercase(method) - + ", previous method byte " + + ", previous method byte 0x" + getHexUIntLowercase(codec->getMethodByte()), ErrorCodes::CANNOT_DECOMPRESS); } From 885bd847201befc61f0063a113ab2eb8ad575741 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 28 Oct 2020 20:48:02 +0300 Subject: [PATCH 282/325] Remove trash from CMakeLists --- CMakeLists.txt | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 21cc74bbd2b..783a9f80b66 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,25 +59,6 @@ set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a pos # For more info see https://cmake.org/cmake/help/latest/prop_gbl/USE_FOLDERS.html set_property(GLOBAL PROPERTY USE_FOLDERS ON) -# cmake 3.9+ needed. -# Usually impractical. -# See also ${ENABLE_THINLTO} -option(ENABLE_IPO "Full link time optimization") - -if(ENABLE_IPO) - cmake_policy(SET CMP0069 NEW) - include(CheckIPOSupported) - check_ipo_supported(RESULT IPO_SUPPORTED OUTPUT IPO_NOT_SUPPORTED) - if(IPO_SUPPORTED) - message(STATUS "IPO/LTO is supported, enabling") - set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) - else() - message (${RECONFIGURE_MESSAGE_LEVEL} "IPO/LTO is not supported: <${IPO_NOT_SUPPORTED}>") - endif() -else() - message(STATUS "IPO/LTO not enabled.") -endif() - # Check that submodules are present only if source was downloaded with git if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boost/boost") message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive") From 6eaba28e012310ec89827529d054f58bfd538955 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 28 Oct 2020 22:44:37 +0300 Subject: [PATCH 283/325] Trigger CI --- tests/integration/test_disabled_mysql_server/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_disabled_mysql_server/test.py b/tests/integration/test_disabled_mysql_server/test.py index df5f123c6bd..a2cbcb17534 100644 --- a/tests/integration/test_disabled_mysql_server/test.py +++ b/tests/integration/test_disabled_mysql_server/test.py @@ -27,7 +27,7 @@ class MySQLNodeInstance: self.port = port self.hostname = hostname self.password = password - self.mysql_connection = None # lazy init + self.mysql_connection = None # lazy init def alloc_connection(self): if self.mysql_connection is None: From f995ef9797875354d9d0c3d01756a395b685303f Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 27 Oct 2020 16:55:57 +0800 Subject: [PATCH 284/325] Balanced reading from JBOD --- src/Core/Settings.h | 2 + .../MergeTree/MergeTreeDataSelectExecutor.cpp | 2 +- src/Storages/MergeTree/MergeTreeReadPool.cpp | 90 +++++++++++++++---- src/Storages/MergeTree/MergeTreeReadPool.h | 11 ++- 4 files changed, 84 insertions(+), 21 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b94883ca871..0c5d5957dac 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -169,6 +169,8 @@ class IColumn; M(Milliseconds, read_backoff_min_interval_between_events_ms, 1000, "Settings to reduce the number of threads in case of slow reads. Do not pay attention to the event, if the previous one has passed less than a certain amount of time.", 0) \ M(UInt64, read_backoff_min_events, 2, "Settings to reduce the number of threads in case of slow reads. The number of events after which the number of threads will be reduced.", 0) \ \ + M(UInt64, read_backoff_min_concurrency, 1, "Settings to try keeping the minimal number of threads in case of slow reads.", 0) \ + \ M(Float, memory_tracker_fault_probability, 0., "For testing of `exception safety` - throw an exception every time you allocate memory with the specified probability.", 0) \ \ M(Bool, enable_http_compression, 0, "Compress the result if the client over HTTP said that it understands data compressed by gzip or deflate.", 0) \ diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 8c1dc845d26..e08ae40a13d 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -898,7 +898,7 @@ Pipe MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( num_streams, sum_marks, min_marks_for_concurrent_read, - parts, + std::move(parts), data, metadata_snapshot, query_info.prewhere_info, diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index d78f72d1dd0..5ab2cbf2b88 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -21,7 +21,7 @@ MergeTreeReadPool::MergeTreeReadPool( const size_t threads_, const size_t sum_marks_, const size_t min_marks_for_concurrent_read_, - RangesInDataParts parts_, + RangesInDataParts && parts_, const MergeTreeData & data_, const StorageMetadataPtr & metadata_snapshot_, const PrewhereInfoPtr & prewhere_info_, @@ -38,11 +38,11 @@ MergeTreeReadPool::MergeTreeReadPool( , do_not_steal_tasks{do_not_steal_tasks_} , predict_block_size_bytes{preferred_block_size_bytes_ > 0} , prewhere_info{prewhere_info_} - , parts_ranges{parts_} + , parts_ranges{std::move(parts_)} { /// parts don't contain duplicate MergeTreeDataPart's. - const auto per_part_sum_marks = fillPerPartInfo(parts_, check_columns_); - fillPerThreadInfo(threads_, sum_marks_, per_part_sum_marks, parts_, min_marks_for_concurrent_read_); + const auto per_part_sum_marks = fillPerPartInfo(parts_ranges, check_columns_); + fillPerThreadInfo(threads_, sum_marks_, per_part_sum_marks, parts_ranges, min_marks_for_concurrent_read_); } @@ -62,7 +62,24 @@ MergeTreeReadTaskPtr MergeTreeReadPool::getTask(const size_t min_marks_to_read, return nullptr; /// Steal task if nothing to do and it's not prohibited - const auto thread_idx = tasks_remaining_for_this_thread ? thread : *std::begin(remaining_thread_tasks); + auto thread_idx = thread; + if (!tasks_remaining_for_this_thread) + { + auto it = remaining_thread_tasks.lower_bound(backoff_state.current_threads); + // Grab the entire tasks of a thread which is killed by backoff + if (it != remaining_thread_tasks.end()) + { + threads_tasks[thread] = std::move(threads_tasks[*it]); + remaining_thread_tasks.erase(it); + } + else // Try steal tasks from the next thread + { + it = remaining_thread_tasks.upper_bound(thread); + if (it == remaining_thread_tasks.end()) + it = remaining_thread_tasks.begin(); + thread_idx = *it; + } + } auto & thread_tasks = threads_tasks[thread_idx]; auto & thread_task = thread_tasks.parts_and_ranges.back(); @@ -163,7 +180,7 @@ void MergeTreeReadPool::profileFeedback(const ReadBufferFromFileBase::ProfileInf std::lock_guard lock(mutex); - if (backoff_state.current_threads <= 1) + if (backoff_state.current_threads <= backoff_settings.min_concurrency) return; size_t throughput = info.bytes_read * 1000000000 / info.nanoseconds; @@ -194,14 +211,14 @@ void MergeTreeReadPool::profileFeedback(const ReadBufferFromFileBase::ProfileInf std::vector MergeTreeReadPool::fillPerPartInfo( - RangesInDataParts & parts, const bool check_columns) + const RangesInDataParts & parts, const bool check_columns) { std::vector per_part_sum_marks; Block sample_block = metadata_snapshot->getSampleBlock(); for (const auto i : ext::range(0, parts.size())) { - auto & part = parts[i]; + const auto & part = parts[i]; /// Read marks for every data part. size_t sum_marks = 0; @@ -238,21 +255,63 @@ std::vector MergeTreeReadPool::fillPerPartInfo( void MergeTreeReadPool::fillPerThreadInfo( const size_t threads, const size_t sum_marks, std::vector per_part_sum_marks, - RangesInDataParts & parts, const size_t min_marks_for_concurrent_read) + const RangesInDataParts & parts, const size_t min_marks_for_concurrent_read) { threads_tasks.resize(threads); + if (parts.empty()) + return; + + struct PartInfo + { + RangesInDataPart part; + size_t sum_marks; + size_t part_idx; + }; + std::map> parts_per_disk; + + for (size_t i = 0; i < parts.size(); ++i) + { + PartInfo part_info{parts[i], per_part_sum_marks[i], i}; + if (parts[i].data_part->isStoredOnDisk()) + parts_per_disk[parts[i].data_part->volume->getDisk()->getName()].push_back(std::move(part_info)); + else + parts_per_disk[""].push_back(std::move(part_info)); + } const size_t min_marks_per_thread = (sum_marks - 1) / threads + 1; - for (size_t i = 0; i < threads && !parts.empty(); ++i) + auto it = std::prev(parts_per_disk.end()); + auto * current_parts = &it->second; + + auto get_next_parts = [&]() + { + size_t n = parts_per_disk.size(); + do + { + ++it; + if (it == parts_per_disk.end()) + it = parts_per_disk.begin(); + current_parts = &it->second; + } while (current_parts->empty() && --n); + return !current_parts->empty(); + }; + + auto get_current_parts = [&]() + { + if (!current_parts->empty()) + return true; + return get_next_parts(); + }; + + for (size_t i = 0; i < threads && get_next_parts(); ++i) { auto need_marks = min_marks_per_thread; - while (need_marks > 0 && !parts.empty()) + while (need_marks > 0 && get_current_parts()) { - const auto part_idx = parts.size() - 1; - RangesInDataPart & part = parts.back(); - size_t & marks_in_part = per_part_sum_marks.back(); + RangesInDataPart & part = current_parts->back().part; + size_t & marks_in_part = current_parts->back().sum_marks; + const auto part_idx = current_parts->back().part_idx; /// Do not get too few rows from part. if (marks_in_part >= min_marks_for_concurrent_read && @@ -274,8 +333,7 @@ void MergeTreeReadPool::fillPerThreadInfo( marks_in_ranges = marks_in_part; need_marks -= marks_in_part; - parts.pop_back(); - per_part_sum_marks.pop_back(); + current_parts->pop_back(); } else { diff --git a/src/Storages/MergeTree/MergeTreeReadPool.h b/src/Storages/MergeTree/MergeTreeReadPool.h index c0b04c6a228..aa6811661e6 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.h +++ b/src/Storages/MergeTree/MergeTreeReadPool.h @@ -36,13 +36,16 @@ public: size_t min_interval_between_events_ms = 1000; /// Number of events to do backoff - to lower number of threads in pool. size_t min_events = 2; + /// Try keeping the minimal number of threads in pool. + size_t min_concurrency = 1; /// Constants above is just an example. BackoffSettings(const Settings & settings) : min_read_latency_ms(settings.read_backoff_min_latency_ms.totalMilliseconds()), max_throughput(settings.read_backoff_max_throughput), min_interval_between_events_ms(settings.read_backoff_min_interval_between_events_ms.totalMilliseconds()), - min_events(settings.read_backoff_min_events) + min_events(settings.read_backoff_min_events), + min_concurrency(settings.read_backoff_min_concurrency) { } @@ -68,7 +71,7 @@ private: public: MergeTreeReadPool( const size_t threads_, const size_t sum_marks_, const size_t min_marks_for_concurrent_read_, - RangesInDataParts parts_, const MergeTreeData & data_, const StorageMetadataPtr & metadata_snapshot_, const PrewhereInfoPtr & prewhere_info_, + RangesInDataParts && parts_, const MergeTreeData & data_, const StorageMetadataPtr & metadata_snapshot_, const PrewhereInfoPtr & prewhere_info_, const bool check_columns_, const Names & column_names_, const BackoffSettings & backoff_settings_, size_t preferred_block_size_bytes_, const bool do_not_steal_tasks_ = false); @@ -88,11 +91,11 @@ public: private: std::vector fillPerPartInfo( - RangesInDataParts & parts, const bool check_columns); + const RangesInDataParts & parts, const bool check_columns); void fillPerThreadInfo( const size_t threads, const size_t sum_marks, std::vector per_part_sum_marks, - RangesInDataParts & parts, const size_t min_marks_for_concurrent_read); + const RangesInDataParts & parts, const size_t min_marks_for_concurrent_read); const MergeTreeData & data; StorageMetadataPtr metadata_snapshot; From 10bad32fb405cdf9c073875235c678c83e0bcfd8 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 29 Oct 2020 00:52:26 +0300 Subject: [PATCH 285/325] Refactor code a little bit. Add comment. --- src/Storages/MergeTree/MergeTreeReadPool.cpp | 75 ++++++++++---------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index 5ab2cbf2b88..6ce47af87c5 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -267,51 +267,41 @@ void MergeTreeReadPool::fillPerThreadInfo( size_t sum_marks; size_t part_idx; }; - std::map> parts_per_disk; - for (size_t i = 0; i < parts.size(); ++i) + using PartsInfo = std::vector; + std::queue parts_queue; + { - PartInfo part_info{parts[i], per_part_sum_marks[i], i}; - if (parts[i].data_part->isStoredOnDisk()) - parts_per_disk[parts[i].data_part->volume->getDisk()->getName()].push_back(std::move(part_info)); - else - parts_per_disk[""].push_back(std::move(part_info)); + /// Group parts by volume name. + /// We try minimize the number of threads concurrently read from the same volume. + /// It improves the performance for JBOD architecture. + std::map> parts_per_disk; + + for (size_t i = 0; i < parts.size(); ++i) + { + PartInfo part_info{parts[i], per_part_sum_marks[i], i}; + if (parts[i].data_part->isStoredOnDisk()) + parts_per_disk[parts[i].data_part->volume->getDisk()->getName()].push_back(std::move(part_info)); + else + parts_per_disk[""].push_back(std::move(part_info)); + } + + for (auto & info : parts_per_disk) + parts_queue.push(std::move(info.second)); } const size_t min_marks_per_thread = (sum_marks - 1) / threads + 1; - auto it = std::prev(parts_per_disk.end()); - auto * current_parts = &it->second; - - auto get_next_parts = [&]() - { - size_t n = parts_per_disk.size(); - do - { - ++it; - if (it == parts_per_disk.end()) - it = parts_per_disk.begin(); - current_parts = &it->second; - } while (current_parts->empty() && --n); - return !current_parts->empty(); - }; - - auto get_current_parts = [&]() - { - if (!current_parts->empty()) - return true; - return get_next_parts(); - }; - - for (size_t i = 0; i < threads && get_next_parts(); ++i) + for (size_t i = 0; i < threads && !parts_queue.empty(); ++i) { auto need_marks = min_marks_per_thread; - while (need_marks > 0 && get_current_parts()) + while (need_marks > 0 && !parts_queue.empty()) { - RangesInDataPart & part = current_parts->back().part; - size_t & marks_in_part = current_parts->back().sum_marks; - const auto part_idx = current_parts->back().part_idx; + auto & current_parts = parts_queue.front(); + RangesInDataPart & part = current_parts.back().part; + size_t & marks_in_part = current_parts.back().sum_marks; + const auto part_idx = current_parts.back().part_idx; /// Do not get too few rows from part. if (marks_in_part >= min_marks_for_concurrent_read && @@ -333,7 +323,9 @@ void MergeTreeReadPool::fillPerThreadInfo( marks_in_ranges = marks_in_part; need_marks -= marks_in_part; - current_parts->pop_back(); + current_parts.pop_back(); + if (current_parts.empty()) + parts_queue.pop(); } else { @@ -362,6 +354,17 @@ void MergeTreeReadPool::fillPerThreadInfo( if (marks_in_ranges != 0) remaining_thread_tasks.insert(i); } + + /// Before processing next thread, change volume if possible. + /// Different threads will likely start reading from different volumes, + /// which may improve read parallelism for JBOD. + /// It also may be helpful in case we have backoff threads. + /// Backoff threads will likely to reduce load for different disks, not the same one. + if (parts_queue.size() > 1) + { + parts_queue.push(std::move(parts_queue.front())); + parts_queue.pop(); + } } } From 5531bbdc980b32e49bb28f903de7dfad1cbd7f10 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 29 Oct 2020 01:21:32 +0300 Subject: [PATCH 286/325] Update version_date.tsv after release 20.10.3.30 --- utils/list-versions/version_date.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index f7d6536a890..3b2681a4ec2 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v20.10.3.30-stable 2020-10-29 v20.10.2.20-stable 2020-10-23 v20.9.3.45-stable 2020-10-09 v20.9.2.20-stable 2020-09-22 From e094b001d309a38bc887873923949dd9e72c35e0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 29 Oct 2020 02:01:30 +0300 Subject: [PATCH 287/325] Update version_date.tsv after release 20.9.4.76 --- utils/list-versions/version_date.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 3b2681a4ec2..c58c64e249c 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,5 +1,6 @@ v20.10.3.30-stable 2020-10-29 v20.10.2.20-stable 2020-10-23 +v20.9.4.76-stable 2020-10-29 v20.9.3.45-stable 2020-10-09 v20.9.2.20-stable 2020-09-22 v20.8.4.11-lts 2020-10-09 From 3e8f399f0bf5884e49506bd266c5f1f346b49dbc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 29 Oct 2020 05:15:01 +0300 Subject: [PATCH 288/325] Update test --- tests/queries/0_stateless/00161_rounding_functions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00161_rounding_functions.sql b/tests/queries/0_stateless/00161_rounding_functions.sql index 460129d2e9d..cc3542338bb 100644 --- a/tests/queries/0_stateless/00161_rounding_functions.sql +++ b/tests/queries/0_stateless/00161_rounding_functions.sql @@ -44,4 +44,4 @@ SELECT 12345.6789 AS x, floor(x, -1), floor(x, -2), floor(x, -3), floor(x, -4), SELECT roundToExp2(100), roundToExp2(64), roundToExp2(3), roundToExp2(0), roundToExp2(-1); SELECT roundToExp2(0.9), roundToExp2(0), roundToExp2(-0.5), roundToExp2(-0.6), roundToExp2(-0.2); -SELECT ceil(29375422, -54212) --{serverError 36} +SELECT ceil(29375422, -54212) --{serverError 69} From c2ca5d29a85b343d1e2bf558db3b4d07c3d4c2ea Mon Sep 17 00:00:00 2001 From: feng lv Date: Thu, 29 Oct 2020 02:40:39 +0000 Subject: [PATCH 289/325] fix build --- src/Interpreters/InterpreterDropQuery.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 8bf8675b15d..144e045ecee 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -250,7 +250,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const ASTDropQuery & query) { if (query.no_delay) { - for (const auto table_uuid : tables_to_wait) + for (const auto & table_uuid : tables_to_wait) waitForTableToBeActuallyDroppedOrDetached(query, database, table_uuid); } throw; @@ -258,7 +258,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const ASTDropQuery & query) if (query.no_delay) { - for (const auto table_uuid : tables_to_wait) + for (const auto & table_uuid : tables_to_wait) waitForTableToBeActuallyDroppedOrDetached(query, database, table_uuid); } return res; From 745cb4ab2f8d5b04e2b2481a9f9bc585cb583807 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 29 Oct 2020 09:57:42 +0300 Subject: [PATCH 290/325] Update version_date.tsv after release 20.8.5.45 --- utils/list-versions/version_date.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index c58c64e249c..37e3e412a63 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -3,6 +3,7 @@ v20.10.2.20-stable 2020-10-23 v20.9.4.76-stable 2020-10-29 v20.9.3.45-stable 2020-10-09 v20.9.2.20-stable 2020-09-22 +v20.8.5.45-lts 2020-10-29 v20.8.4.11-lts 2020-10-09 v20.8.3.18-stable 2020-09-18 v20.8.2.3-stable 2020-09-08 From 671d2b7f1b793b8f233a9685e469277c2a7079dc Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 29 Oct 2020 10:04:23 +0300 Subject: [PATCH 291/325] Update MergeTreeReadPool.cpp Fix comment --- src/Storages/MergeTree/MergeTreeReadPool.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index 6ce47af87c5..e44ff500c88 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -272,8 +272,8 @@ void MergeTreeReadPool::fillPerThreadInfo( std::queue parts_queue; { - /// Group parts by volume name. - /// We try minimize the number of threads concurrently read from the same volume. + /// Group parts by disk name. + /// We try minimize the number of threads concurrently read from the same disk. /// It improves the performance for JBOD architecture. std::map> parts_per_disk; @@ -355,8 +355,8 @@ void MergeTreeReadPool::fillPerThreadInfo( remaining_thread_tasks.insert(i); } - /// Before processing next thread, change volume if possible. - /// Different threads will likely start reading from different volumes, + /// Before processing next thread, change disk if possible. + /// Different threads will likely start reading from different disk, /// which may improve read parallelism for JBOD. /// It also may be helpful in case we have backoff threads. /// Backoff threads will likely to reduce load for different disks, not the same one. From bb1ac2af8218b606a96f730eb7a1419b06a3c8ab Mon Sep 17 00:00:00 2001 From: taichong Date: Thu, 29 Oct 2020 14:53:32 +0800 Subject: [PATCH 292/325] Shrink sequence gtid set when use MySQL Master -> MySQL Slave -> ClickHouse MaterializeMySQL Engine and MySQL Slave enable slave_parallel_worker the gtid in .metadata won't shrink. Like this: https://github.com/ClickHouse/ClickHouse/issues/15951 --- src/Core/MySQL/MySQLGtid.cpp | 12 ++++++++++++ src/Core/MySQL/MySQLGtid.h | 2 ++ src/Core/tests/mysql_protocol.cpp | 11 +++++++++++ 3 files changed, 25 insertions(+) diff --git a/src/Core/MySQL/MySQLGtid.cpp b/src/Core/MySQL/MySQLGtid.cpp index df26eb7aa08..6e50998da62 100644 --- a/src/Core/MySQL/MySQLGtid.cpp +++ b/src/Core/MySQL/MySQLGtid.cpp @@ -85,6 +85,9 @@ void GTIDSets::update(const GTID & other) ErrorCodes::LOGICAL_ERROR); } + /// Try to shirnk Sequence interval. + GTIDSet::tryShirnk(set, i, current); + /// Sequence, extend the interval. if (other.seq_no == current.end) { @@ -116,6 +119,15 @@ void GTIDSets::update(const GTID & other) sets.emplace_back(set); } +void GTIDSet::tryShirnk(GTIDSet & set, unsigned int i, GTIDSet::Interval & current) +{ + if (i != set.intervals.size() -1) { + auto & next = set.intervals[i+1]; + if (current.end == next.start) + set.tryMerge(i); + } +} + String GTIDSets::toString() const { WriteBufferFromOwnString buffer; diff --git a/src/Core/MySQL/MySQLGtid.h b/src/Core/MySQL/MySQLGtid.h index d228e269872..cd8cd0a2e98 100644 --- a/src/Core/MySQL/MySQLGtid.h +++ b/src/Core/MySQL/MySQLGtid.h @@ -26,6 +26,8 @@ public: std::vector intervals; void tryMerge(size_t i); + + static void tryShirnk(GTIDSet & set, unsigned int i, Interval & current) ; }; class GTIDSets diff --git a/src/Core/tests/mysql_protocol.cpp b/src/Core/tests/mysql_protocol.cpp index 6cad095fc85..7e6aae5da23 100644 --- a/src/Core/tests/mysql_protocol.cpp +++ b/src/Core/tests/mysql_protocol.cpp @@ -260,6 +260,17 @@ int main(int argc, char ** argv) "10662d71-9d91-11ea-bbc2-0242ac110003:6-7", "20662d71-9d91-11ea-bbc2-0242ac110003:9", "10662d71-9d91-11ea-bbc2-0242ac110003:6-7,20662d71-9d91-11ea-bbc2-0242ac110003:9"}, + + {"shirnk-sequence", + "10662d71-9d91-11ea-bbc2-0242ac110003:1-3:4-5:7", + "10662d71-9d91-11ea-bbc2-0242ac110003:6", + "10662d71-9d91-11ea-bbc2-0242ac110003:1-7"}, + + {"shirnk-sequence", + "10662d71-9d91-11ea-bbc2-0242ac110003:1-3:4-5:10", + "10662d71-9d91-11ea-bbc2-0242ac110003:8", + "10662d71-9d91-11ea-bbc2-0242ac110003:1-5:8:10" + } }; for (auto & tc : cases) From d3f08b21de47065de5687401fe4e28fd72ad46f9 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 29 Oct 2020 11:28:40 +0300 Subject: [PATCH 293/325] CMake moved power pc check in tools --- cmake/arch.cmake | 4 ---- cmake/tools.cmake | 6 ++++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cmake/arch.cmake b/cmake/arch.cmake index 57ed42295bb..9604ef62b31 100644 --- a/cmake/arch.cmake +++ b/cmake/arch.cmake @@ -16,8 +16,4 @@ endif () if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le.*|PPC64LE.*)") set (ARCH_PPC64LE 1) - # FIXME: move this check into tools.cmake - if (COMPILER_CLANG OR (COMPILER_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)) - message(FATAL_ERROR "Only gcc-8 or higher is supported for powerpc architecture") - endif () endif () diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 6f07cc2439c..12ffa8eb5fd 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -80,3 +80,9 @@ if (LINKER_NAME) message(STATUS "Using custom linker by name: ${LINKER_NAME}") endif () + +if (ARCH_PPC64LE) + if (COMPILER_CLANG OR (COMPILER_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)) + message(FATAL_ERROR "Only gcc-8 or higher is supported for powerpc architecture") + endif () +endif () \ No newline at end of file From 2a66c17472387c09afc3ac0556b1c4b4a8c90564 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Thu, 29 Oct 2020 16:37:59 +0300 Subject: [PATCH 294/325] Update InterpreterDropQuery.cpp --- src/Interpreters/InterpreterDropQuery.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 8bf8675b15d..144e045ecee 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -250,7 +250,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const ASTDropQuery & query) { if (query.no_delay) { - for (const auto table_uuid : tables_to_wait) + for (const auto & table_uuid : tables_to_wait) waitForTableToBeActuallyDroppedOrDetached(query, database, table_uuid); } throw; @@ -258,7 +258,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const ASTDropQuery & query) if (query.no_delay) { - for (const auto table_uuid : tables_to_wait) + for (const auto & table_uuid : tables_to_wait) waitForTableToBeActuallyDroppedOrDetached(query, database, table_uuid); } return res; From 3627fabfb9e2445a32fbed81eef9e79a00de76ef Mon Sep 17 00:00:00 2001 From: Maxim Akhmedov Date: Thu, 29 Oct 2020 17:37:23 +0300 Subject: [PATCH 295/325] Remove -g0 form Arcadia build settings. --- src/Access/ya.make | 1 - src/Access/ya.make.in | 1 - src/AggregateFunctions/ya.make | 1 - src/AggregateFunctions/ya.make.in | 1 - src/Client/ya.make | 1 - src/Client/ya.make.in | 1 - src/Columns/ya.make | 1 - src/Common/ya.make | 1 - src/Common/ya.make.in | 1 - src/Compression/ya.make | 1 - src/Compression/ya.make.in | 1 - src/Core/ya.make | 1 - src/Core/ya.make.in | 1 - src/DataStreams/ya.make | 1 - src/DataStreams/ya.make.in | 1 - src/DataTypes/ya.make | 1 - src/DataTypes/ya.make.in | 1 - src/Databases/ya.make | 1 - src/Databases/ya.make.in | 1 - src/Dictionaries/ya.make | 1 - src/Dictionaries/ya.make.in | 1 - src/Disks/S3/ya.make | 1 - src/Disks/ya.make | 1 - src/Disks/ya.make.in | 1 - src/Formats/ya.make | 1 - src/Formats/ya.make.in | 1 - src/Functions/ya.make | 1 - src/Functions/ya.make.in | 1 - src/IO/ya.make | 1 - src/IO/ya.make.in | 1 - src/Interpreters/ya.make | 1 - src/Interpreters/ya.make.in | 1 - src/Parsers/ya.make | 1 - src/Parsers/ya.make.in | 1 - src/Processors/ya.make | 1 - src/Processors/ya.make.in | 1 - src/Server/ya.make | 1 - src/Server/ya.make.in | 1 - src/Storages/ya.make | 1 - src/Storages/ya.make.in | 1 - src/TableFunctions/ya.make | 1 - src/TableFunctions/ya.make.in | 1 - 42 files changed, 42 deletions(-) diff --git a/src/Access/ya.make b/src/Access/ya.make index b945c5a192a..1ec8cb32c97 100644 --- a/src/Access/ya.make +++ b/src/Access/ya.make @@ -5,7 +5,6 @@ PEERDIR( clickhouse/src/Common ) -CFLAGS(-g0) SRCS( AccessControlManager.cpp diff --git a/src/Access/ya.make.in b/src/Access/ya.make.in index e48d0d1bda7..ce7cd88b272 100644 --- a/src/Access/ya.make.in +++ b/src/Access/ya.make.in @@ -4,7 +4,6 @@ PEERDIR( clickhouse/src/Common ) -CFLAGS(-g0) SRCS( diff --git a/src/AggregateFunctions/ya.make b/src/AggregateFunctions/ya.make index 1578e0c80ea..8d27cf986d0 100644 --- a/src/AggregateFunctions/ya.make +++ b/src/AggregateFunctions/ya.make @@ -5,7 +5,6 @@ PEERDIR( clickhouse/src/Common ) -CFLAGS(-g0) SRCS( AggregateFunctionAggThrow.cpp diff --git a/src/AggregateFunctions/ya.make.in b/src/AggregateFunctions/ya.make.in index 4c2943b0539..dd49b679d28 100644 --- a/src/AggregateFunctions/ya.make.in +++ b/src/AggregateFunctions/ya.make.in @@ -4,7 +4,6 @@ PEERDIR( clickhouse/src/Common ) -CFLAGS(-g0) SRCS( diff --git a/src/Client/ya.make b/src/Client/ya.make index b07e5afc0fb..07cc6725308 100644 --- a/src/Client/ya.make +++ b/src/Client/ya.make @@ -6,7 +6,6 @@ PEERDIR( contrib/libs/poco/NetSSL_OpenSSL ) -CFLAGS(-g0) SRCS( Connection.cpp diff --git a/src/Client/ya.make.in b/src/Client/ya.make.in index 704a05d8f3b..d8faff9ae1a 100644 --- a/src/Client/ya.make.in +++ b/src/Client/ya.make.in @@ -5,7 +5,6 @@ PEERDIR( contrib/libs/poco/NetSSL_OpenSSL ) -CFLAGS(-g0) SRCS( diff --git a/src/Columns/ya.make b/src/Columns/ya.make index 78c0e1b992d..1463bbc69e2 100644 --- a/src/Columns/ya.make +++ b/src/Columns/ya.make @@ -13,7 +13,6 @@ PEERDIR( contrib/libs/pdqsort ) -CFLAGS(-g0) SRCS( Collator.cpp diff --git a/src/Common/ya.make b/src/Common/ya.make index fb04ecaa141..b19a5183201 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -21,7 +21,6 @@ PEERDIR( INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/yandex/ya.make.versions.inc) -CFLAGS(-g0) SRCS( ActionLock.cpp diff --git a/src/Common/ya.make.in b/src/Common/ya.make.in index f8b7601e215..49c8baa5eec 100644 --- a/src/Common/ya.make.in +++ b/src/Common/ya.make.in @@ -20,7 +20,6 @@ PEERDIR( INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/yandex/ya.make.versions.inc) -CFLAGS(-g0) SRCS( diff --git a/src/Compression/ya.make b/src/Compression/ya.make index a17e2029b8f..8ffcb6be547 100644 --- a/src/Compression/ya.make +++ b/src/Compression/ya.make @@ -12,7 +12,6 @@ PEERDIR( contrib/libs/zstd ) -CFLAGS(-g0) SRCS( CachedCompressedReadBuffer.cpp diff --git a/src/Compression/ya.make.in b/src/Compression/ya.make.in index 780ea72b3ec..3c46b036aa0 100644 --- a/src/Compression/ya.make.in +++ b/src/Compression/ya.make.in @@ -11,7 +11,6 @@ PEERDIR( contrib/libs/zstd ) -CFLAGS(-g0) SRCS( diff --git a/src/Core/ya.make b/src/Core/ya.make index 424566d212a..626662e992c 100644 --- a/src/Core/ya.make +++ b/src/Core/ya.make @@ -7,7 +7,6 @@ PEERDIR( contrib/restricted/boost/libs ) -CFLAGS(-g0) SRCS( BackgroundSchedulePool.cpp diff --git a/src/Core/ya.make.in b/src/Core/ya.make.in index 95c4e32995d..b2e82663c1e 100644 --- a/src/Core/ya.make.in +++ b/src/Core/ya.make.in @@ -6,7 +6,6 @@ PEERDIR( contrib/restricted/boost/libs ) -CFLAGS(-g0) SRCS( diff --git a/src/DataStreams/ya.make b/src/DataStreams/ya.make index 0c46e42d456..adef8246f33 100644 --- a/src/DataStreams/ya.make +++ b/src/DataStreams/ya.make @@ -8,7 +8,6 @@ PEERDIR( NO_COMPILER_WARNINGS() -CFLAGS(-g0) SRCS( AddingDefaultBlockOutputStream.cpp diff --git a/src/DataStreams/ya.make.in b/src/DataStreams/ya.make.in index 268719112ac..7aa2fe4874e 100644 --- a/src/DataStreams/ya.make.in +++ b/src/DataStreams/ya.make.in @@ -7,7 +7,6 @@ PEERDIR( NO_COMPILER_WARNINGS() -CFLAGS(-g0) SRCS( diff --git a/src/DataTypes/ya.make b/src/DataTypes/ya.make index 20a63bb7727..97b600f70ba 100644 --- a/src/DataTypes/ya.make +++ b/src/DataTypes/ya.make @@ -6,7 +6,6 @@ PEERDIR( clickhouse/src/Formats ) -CFLAGS(-g0) SRCS( convertMySQLDataType.cpp diff --git a/src/DataTypes/ya.make.in b/src/DataTypes/ya.make.in index f1983be1032..05170178925 100644 --- a/src/DataTypes/ya.make.in +++ b/src/DataTypes/ya.make.in @@ -5,7 +5,6 @@ PEERDIR( clickhouse/src/Formats ) -CFLAGS(-g0) SRCS( diff --git a/src/Databases/ya.make b/src/Databases/ya.make index b4173057e03..e3c5daeb6bc 100644 --- a/src/Databases/ya.make +++ b/src/Databases/ya.make @@ -5,7 +5,6 @@ PEERDIR( clickhouse/src/Common ) -CFLAGS(-g0) SRCS( DatabaseAtomic.cpp diff --git a/src/Databases/ya.make.in b/src/Databases/ya.make.in index e48d0d1bda7..ce7cd88b272 100644 --- a/src/Databases/ya.make.in +++ b/src/Databases/ya.make.in @@ -4,7 +4,6 @@ PEERDIR( clickhouse/src/Common ) -CFLAGS(-g0) SRCS( diff --git a/src/Dictionaries/ya.make b/src/Dictionaries/ya.make index 485d8b0a16d..9edf156c015 100644 --- a/src/Dictionaries/ya.make +++ b/src/Dictionaries/ya.make @@ -12,7 +12,6 @@ PEERDIR( NO_COMPILER_WARNINGS() -CFLAGS(-g0) SRCS( CacheDictionary.cpp diff --git a/src/Dictionaries/ya.make.in b/src/Dictionaries/ya.make.in index 3eb8e728643..2c0735d38a4 100644 --- a/src/Dictionaries/ya.make.in +++ b/src/Dictionaries/ya.make.in @@ -11,7 +11,6 @@ PEERDIR( NO_COMPILER_WARNINGS() -CFLAGS(-g0) SRCS( diff --git a/src/Disks/S3/ya.make b/src/Disks/S3/ya.make index b32adee0d26..17425f6e69a 100644 --- a/src/Disks/S3/ya.make +++ b/src/Disks/S3/ya.make @@ -4,7 +4,6 @@ PEERDIR( clickhouse/src/Common ) -CFLAGS(-g0) SRCS( DiskS3.cpp diff --git a/src/Disks/ya.make b/src/Disks/ya.make index f01348ff945..d14bc0d05c8 100644 --- a/src/Disks/ya.make +++ b/src/Disks/ya.make @@ -5,7 +5,6 @@ PEERDIR( clickhouse/src/Common ) -CFLAGS(-g0) SRCS( createVolume.cpp diff --git a/src/Disks/ya.make.in b/src/Disks/ya.make.in index 9ed04e23f83..ee13bb272cd 100644 --- a/src/Disks/ya.make.in +++ b/src/Disks/ya.make.in @@ -4,7 +4,6 @@ PEERDIR( clickhouse/src/Common ) -CFLAGS(-g0) SRCS( diff --git a/src/Formats/ya.make b/src/Formats/ya.make index b4f7b073e21..8e797de39f8 100644 --- a/src/Formats/ya.make +++ b/src/Formats/ya.make @@ -7,7 +7,6 @@ PEERDIR( contrib/libs/protoc ) -CFLAGS(-g0) SRCS( FormatFactory.cpp diff --git a/src/Formats/ya.make.in b/src/Formats/ya.make.in index 51c8bfde2f8..f7d03e7b00f 100644 --- a/src/Formats/ya.make.in +++ b/src/Formats/ya.make.in @@ -6,7 +6,6 @@ PEERDIR( contrib/libs/protoc ) -CFLAGS(-g0) SRCS( diff --git a/src/Functions/ya.make b/src/Functions/ya.make index ed03f5175ab..4c2cbaf5b1f 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -32,7 +32,6 @@ PEERDIR( ) # "Arcadia" build is slightly deficient. It lacks many libraries that we need. -CFLAGS(-g0) SRCS( abs.cpp diff --git a/src/Functions/ya.make.in b/src/Functions/ya.make.in index 2a66aa5553e..be90a8b6c7a 100644 --- a/src/Functions/ya.make.in +++ b/src/Functions/ya.make.in @@ -31,7 +31,6 @@ PEERDIR( ) # "Arcadia" build is slightly deficient. It lacks many libraries that we need. -CFLAGS(-g0) SRCS( diff --git a/src/IO/ya.make b/src/IO/ya.make index 28099818b46..3796494ff33 100644 --- a/src/IO/ya.make +++ b/src/IO/ya.make @@ -8,7 +8,6 @@ PEERDIR( contrib/libs/poco/NetSSL_OpenSSL ) -CFLAGS(-g0) SRCS( AIOContextPool.cpp diff --git a/src/IO/ya.make.in b/src/IO/ya.make.in index 1b3ca7d6d35..b566644f78b 100644 --- a/src/IO/ya.make.in +++ b/src/IO/ya.make.in @@ -7,7 +7,6 @@ PEERDIR( contrib/libs/poco/NetSSL_OpenSSL ) -CFLAGS(-g0) SRCS( diff --git a/src/Interpreters/ya.make b/src/Interpreters/ya.make index 563b7374fdc..1c463eff7e4 100644 --- a/src/Interpreters/ya.make +++ b/src/Interpreters/ya.make @@ -14,7 +14,6 @@ PEERDIR( NO_COMPILER_WARNINGS() -CFLAGS(-g0) SRCS( ActionLocksManager.cpp diff --git a/src/Interpreters/ya.make.in b/src/Interpreters/ya.make.in index da34c1e3680..2445a9ba850 100644 --- a/src/Interpreters/ya.make.in +++ b/src/Interpreters/ya.make.in @@ -13,7 +13,6 @@ PEERDIR( NO_COMPILER_WARNINGS() -CFLAGS(-g0) SRCS( diff --git a/src/Parsers/ya.make b/src/Parsers/ya.make index 4ec97b8b55b..4f8b8a82210 100644 --- a/src/Parsers/ya.make +++ b/src/Parsers/ya.make @@ -5,7 +5,6 @@ PEERDIR( clickhouse/src/Common ) -CFLAGS(-g0) SRCS( ASTAlterQuery.cpp diff --git a/src/Parsers/ya.make.in b/src/Parsers/ya.make.in index 8b214f90a03..5ee7f637941 100644 --- a/src/Parsers/ya.make.in +++ b/src/Parsers/ya.make.in @@ -4,7 +4,6 @@ PEERDIR( clickhouse/src/Common ) -CFLAGS(-g0) SRCS( diff --git a/src/Processors/ya.make b/src/Processors/ya.make index b2f8b9ba7c2..7898576ad2d 100644 --- a/src/Processors/ya.make +++ b/src/Processors/ya.make @@ -7,7 +7,6 @@ PEERDIR( contrib/libs/protobuf ) -CFLAGS(-g0) SRCS( Chunk.cpp diff --git a/src/Processors/ya.make.in b/src/Processors/ya.make.in index 3dc63479238..d1aa7d43b6a 100644 --- a/src/Processors/ya.make.in +++ b/src/Processors/ya.make.in @@ -6,7 +6,6 @@ PEERDIR( contrib/libs/protobuf ) -CFLAGS(-g0) SRCS( diff --git a/src/Server/ya.make b/src/Server/ya.make index cab114871c5..8a9bbd3bbc2 100644 --- a/src/Server/ya.make +++ b/src/Server/ya.make @@ -6,7 +6,6 @@ PEERDIR( contrib/libs/poco/Util ) -CFLAGS(-g0) SRCS( HTTPHandler.cpp diff --git a/src/Server/ya.make.in b/src/Server/ya.make.in index 44a2531208f..9adec7e3685 100644 --- a/src/Server/ya.make.in +++ b/src/Server/ya.make.in @@ -5,7 +5,6 @@ PEERDIR( contrib/libs/poco/Util ) -CFLAGS(-g0) SRCS( diff --git a/src/Storages/ya.make b/src/Storages/ya.make index 107433b5e73..29d9e87bfa0 100644 --- a/src/Storages/ya.make +++ b/src/Storages/ya.make @@ -7,7 +7,6 @@ PEERDIR( contrib/libs/poco/MongoDB ) -CFLAGS(-g0) SRCS( AlterCommands.cpp diff --git a/src/Storages/ya.make.in b/src/Storages/ya.make.in index a2fb50a6d61..dbae43aa3fc 100644 --- a/src/Storages/ya.make.in +++ b/src/Storages/ya.make.in @@ -6,7 +6,6 @@ PEERDIR( contrib/libs/poco/MongoDB ) -CFLAGS(-g0) SRCS( diff --git a/src/TableFunctions/ya.make b/src/TableFunctions/ya.make index 03432e2bbbc..50f685fef3e 100644 --- a/src/TableFunctions/ya.make +++ b/src/TableFunctions/ya.make @@ -5,7 +5,6 @@ PEERDIR( clickhouse/src/Common ) -CFLAGS(-g0) SRCS( ITableFunction.cpp diff --git a/src/TableFunctions/ya.make.in b/src/TableFunctions/ya.make.in index 289c831f11a..aedb6209ef8 100644 --- a/src/TableFunctions/ya.make.in +++ b/src/TableFunctions/ya.make.in @@ -4,7 +4,6 @@ PEERDIR( clickhouse/src/Common ) -CFLAGS(-g0) SRCS( From dbfef325e6a4fdb081f6b99a6d4d068237f92135 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Thu, 29 Oct 2020 20:22:48 +0300 Subject: [PATCH 296/325] Rename NullSink to NullOutput --- src/IO/ReadHelpers.cpp | 10 +++++----- src/IO/ReadHelpers.h | 2 +- src/Processors/Formats/Impl/TSKVRowInputFormat.cpp | 2 +- .../Formats/Impl/TabSeparatedRowInputFormat.cpp | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index 900e9c7b535..bf41de3959a 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -480,7 +480,7 @@ void readEscapedString(String & s, ReadBuffer & buf) } template void readEscapedStringInto>(PaddedPODArray & s, ReadBuffer & buf); -template void readEscapedStringInto(NullSink & s, ReadBuffer & buf); +template void readEscapedStringInto(NullOutput & s, ReadBuffer & buf); /** If enable_sql_style_quoting == true, @@ -562,7 +562,7 @@ void readQuotedStringWithSQLStyle(String & s, ReadBuffer & buf) template void readQuotedStringInto(PaddedPODArray & s, ReadBuffer & buf); -template void readDoubleQuotedStringInto(NullSink & s, ReadBuffer & buf); +template void readDoubleQuotedStringInto(NullOutput & s, ReadBuffer & buf); void readDoubleQuotedString(String & s, ReadBuffer & buf) { @@ -742,7 +742,7 @@ void readJSONString(String & s, ReadBuffer & buf) template void readJSONStringInto, void>(PaddedPODArray & s, ReadBuffer & buf); template bool readJSONStringInto, bool>(PaddedPODArray & s, ReadBuffer & buf); -template void readJSONStringInto(NullSink & s, ReadBuffer & buf); +template void readJSONStringInto(NullOutput & s, ReadBuffer & buf); template void readJSONStringInto(String & s, ReadBuffer & buf); @@ -891,7 +891,7 @@ void skipJSONField(ReadBuffer & buf, const StringRef & name_of_field) throw Exception("Unexpected EOF for key '" + name_of_field.toString() + "'", ErrorCodes::INCORRECT_DATA); else if (*buf.position() == '"') /// skip double-quoted string { - NullSink sink; + NullOutput sink; readJSONStringInto(sink, buf); } else if (isNumericASCII(*buf.position()) || *buf.position() == '-' || *buf.position() == '+' || *buf.position() == '.') /// skip number @@ -955,7 +955,7 @@ void skipJSONField(ReadBuffer & buf, const StringRef & name_of_field) // field name if (*buf.position() == '"') { - NullSink sink; + NullOutput sink; readJSONStringInto(sink, buf); } else diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index d79328889f1..9ff1858c723 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -527,7 +527,7 @@ bool tryReadJSONStringInto(Vector & s, ReadBuffer & buf) } /// This could be used as template parameter for functions above, if you want to just skip data. -struct NullSink +struct NullOutput { void append(const char *, size_t) {} void push_back(char) {} diff --git a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp index 93cd0a623c7..abb468741c5 100644 --- a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp @@ -130,7 +130,7 @@ bool TSKVRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & ex throw Exception("Unknown field found while parsing TSKV format: " + name_ref.toString(), ErrorCodes::INCORRECT_DATA); /// If the key is not found, skip the value. - NullSink sink; + NullOutput sink; readEscapedStringInto(sink, in); } else diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index c7da0e7383e..529b70e4e09 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -20,7 +20,7 @@ namespace ErrorCodes static void skipTSVRow(ReadBuffer & in, const size_t num_columns) { - NullSink null_sink; + NullOutput null_sink; for (size_t i = 0; i < num_columns; ++i) { @@ -196,7 +196,7 @@ bool TabSeparatedRowInputFormat::readRow(MutableColumns & columns, RowReadExtens } else { - NullSink null_sink; + NullOutput null_sink; readEscapedStringInto(null_sink, in); } @@ -353,7 +353,7 @@ void TabSeparatedRowInputFormat::tryDeserializeField(const DataTypePtr & type, I } else { - NullSink null_sink; + NullOutput null_sink; readEscapedStringInto(null_sink, in); } } From 68b47f5f617b89ab7b53001b8d430b8488da7f2d Mon Sep 17 00:00:00 2001 From: myrrc Date: Thu, 29 Oct 2020 20:24:54 +0300 Subject: [PATCH 297/325] removing LC in AVRO input format if needed --- src/Processors/Formats/Impl/AvroRowInputFormat.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 812bed23451..d756b8191bd 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -1,4 +1,5 @@ #include "AvroRowInputFormat.h" +#include "DataTypes/DataTypeLowCardinality.h" #if USE_AVRO #include @@ -174,7 +175,10 @@ static std::string nodeName(avro::NodePtr node) AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::NodePtr root_node, DataTypePtr target_type) { - WhichDataType target(target_type); + const WhichDataType target = target_type->getTypeId() == TypeIndex::LowCardinality + ? removeLowCardinality(target_type) + : target_type; + switch (root_node->type()) { case avro::AVRO_STRING: [[fallthrough]]; @@ -384,7 +388,8 @@ AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::Node } throw Exception( - "Type " + target_type->getName() + " is not compatible with Avro " + avro::toString(root_node->type()) + ":\n" + nodeToJson(root_node), + "Type " + target_type->getName() + " is not compatible with Avro " + avro::toString(root_node->type()) + ":\n" + + nodeToJson(root_node), ErrorCodes::ILLEGAL_COLUMN); } From 9564a7f6d1c940c8355ef3dd4edebeb0d620b41c Mon Sep 17 00:00:00 2001 From: myrrc Date: Thu, 29 Oct 2020 20:30:24 +0300 Subject: [PATCH 298/325] simplified the cast --- src/Processors/Formats/Impl/AvroRowInputFormat.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index d756b8191bd..cf7a020ee0b 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -175,9 +175,7 @@ static std::string nodeName(avro::NodePtr node) AvroDeserializer::DeserializeFn AvroDeserializer::createDeserializeFn(avro::NodePtr root_node, DataTypePtr target_type) { - const WhichDataType target = target_type->getTypeId() == TypeIndex::LowCardinality - ? removeLowCardinality(target_type) - : target_type; + const WhichDataType target = removeLowCardinality(target_type); switch (root_node->type()) { From 4e5bfc97848f6290bf14f652b253c8bda6fb2eed Mon Sep 17 00:00:00 2001 From: myrrc Date: Thu, 29 Oct 2020 21:02:48 +0300 Subject: [PATCH 299/325] added the simple test --- .../01543_avro_deserialization_with_lc.reference | 0 .../01543_avro_deserialization_with_lc.sh | 12 ++++++++++++ 2 files changed, 12 insertions(+) create mode 100644 tests/queries/0_stateless/01543_avro_deserialization_with_lc.reference create mode 100755 tests/queries/0_stateless/01543_avro_deserialization_with_lc.sh diff --git a/tests/queries/0_stateless/01543_avro_deserialization_with_lc.reference b/tests/queries/0_stateless/01543_avro_deserialization_with_lc.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01543_avro_deserialization_with_lc.sh b/tests/queries/0_stateless/01543_avro_deserialization_with_lc.sh new file mode 100755 index 00000000000..0971396ec9c --- /dev/null +++ b/tests/queries/0_stateless/01543_avro_deserialization_with_lc.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS test_01543 (value LowCardinality(String)) ENGINE=Memory()" +$CLICKHOUSE_CLIENT --query "INSERT INTO test_01543 SELECT toString(number) FROM numbers(1000)" + +$CLICKHOUSE_CLIENT -q "SELECT * FROM test_01543 FORMAT Avro" | + $CLICKHOUSE_CLIENT -q "INSERT INTO test_01543 FORMAT Avro"; + +$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_01543" From 0f92fe9ad2003b9c091c3a10db0190ff84d9a10d Mon Sep 17 00:00:00 2001 From: Anna <42538400+adevyatova@users.noreply.github.com> Date: Thu, 29 Oct 2020 21:22:28 +0300 Subject: [PATCH 300/325] DOCSUP-3118: Document the named subqueries WITH AS (#16339) * Update WITH desc * Update docs/en/sql-reference/statements/select/with.md Co-authored-by: BayoNet * Update docs/en/sql-reference/statements/select/with.md Co-authored-by: BayoNet * Update docs/en/sql-reference/statements/select/with.md Co-authored-by: BayoNet * Fixed, ru translated * Update docs/en/sql-reference/statements/select/with.md Co-authored-by: BayoNet * Fixed syntax description * Update docs/ru/sql-reference/statements/select/with.md Co-authored-by: BayoNet * Update Co-authored-by: BayoNet --- .../sql-reference/statements/select/with.md | 45 +++++++---------- .../sql-reference/statements/select/with.md | 48 ++++++++----------- 2 files changed, 37 insertions(+), 56 deletions(-) diff --git a/docs/en/sql-reference/statements/select/with.md b/docs/en/sql-reference/statements/select/with.md index a507d5224aa..6a0564a8ede 100644 --- a/docs/en/sql-reference/statements/select/with.md +++ b/docs/en/sql-reference/statements/select/with.md @@ -4,13 +4,17 @@ toc_title: WITH # WITH Clause {#with-clause} -This section provides support for Common Table Expressions ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)), so the results of `WITH` clause can be used in the rest of `SELECT` query. +Clickhouse supports Common Table Expressions ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)), that is provides to use results of `WITH` clause in the rest of `SELECT` query. Named subqueries can be included to the current and child query context in places where table objects are allowed. Recursion is prevented by hiding the current level CTEs from the WITH expression. -## Limitations {#limitations} +## Syntax -1. Recursive queries are not supported. -2. When subquery is used inside WITH section, it’s result should be scalar with exactly one row. -3. Expression’s results are not available in subqueries. +``` sql +WITH AS +``` +or +``` sql +WITH AS +``` ## Examples {#examples} @@ -22,10 +26,10 @@ SELECT * FROM hits WHERE EventDate = toDate(ts_upper_bound) AND - EventTime <= ts_upper_bound + EventTime <= ts_upper_bound; ``` -**Example 2:** Evicting sum(bytes) expression result from SELECT clause column list +**Example 2:** Evicting a sum(bytes) expression result from the SELECT clause column list ``` sql WITH sum(bytes) as s @@ -34,10 +38,10 @@ SELECT table FROM system.parts GROUP BY table -ORDER BY s +ORDER BY s; ``` -**Example 3:** Using results of scalar subquery +**Example 3:** Using results of a scalar subquery ``` sql /* this example would return TOP 10 of most huge tables */ @@ -53,27 +57,14 @@ SELECT FROM system.parts GROUP BY table ORDER BY table_disk_usage DESC -LIMIT 10 +LIMIT 10; ``` -**Example 4:** Re-using expression in subquery - -As a workaround for current limitation for expression usage in subqueries, you may duplicate it. +**Example 4:** Reusing expression in a subquery ``` sql -WITH ['hello'] AS hello -SELECT - hello, - * -FROM -( - WITH ['hello'] AS hello - SELECT hello -) +WITH test1 AS (SELECT i + 1, j + 1 FROM test1) +SELECT * FROM test1; ``` -``` text -┌─hello─────┬─hello─────┐ -│ ['hello'] │ ['hello'] │ -└───────────┴───────────┘ -``` +[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/select/with/) diff --git a/docs/ru/sql-reference/statements/select/with.md b/docs/ru/sql-reference/statements/select/with.md index 4feae232bd7..328b28c27ef 100644 --- a/docs/ru/sql-reference/statements/select/with.md +++ b/docs/ru/sql-reference/statements/select/with.md @@ -2,18 +2,21 @@ toc_title: WITH --- -# Секция WITH {#sektsiia-with} +# Секция WITH {#with-clause} -Данная секция представляет собой [Common Table Expressions](https://ru.wikipedia.org/wiki/Иерархические_и_рекурсивные_запросы_в_SQL), то есть позволяет использовать результаты выражений из секции `WITH` в остальной части `SELECT` запроса. +Clickhouse поддерживает [Общие табличные выражения](https://ru.wikipedia.org/wiki/Иерархические_и_рекурсивные_запросы_в_SQL), то есть позволяет использовать результаты выражений из секции `WITH` в остальной части `SELECT` запроса. Именованные подзапросы могут быть включены в текущий и дочерний контекст запроса в тех местах, где разрешены табличные объекты. Рекурсия предотвращается путем скрытия общего табличного выражения текущего уровня из выражения `WITH`. + +## Синтаксис + +``` sql +WITH AS +``` +или +``` sql +WITH AS +``` -### Ограничения - -1. Рекурсивные запросы не поддерживаются -2. Если в качестве выражения используется подзапрос, то результат должен содержать ровно одну строку -3. Результаты выражений нельзя переиспользовать во вложенных запросах -В дальнейшем, результаты выражений можно использовать в секции SELECT. - -### Примеры +## Примеры **Пример 1:** Использование константного выражения как «переменной» @@ -23,7 +26,7 @@ SELECT * FROM hits WHERE EventDate = toDate(ts_upper_bound) AND - EventTime <= ts_upper_bound + EventTime <= ts_upper_bound; ``` **Пример 2:** Выкидывание выражения sum(bytes) из списка колонок в SELECT @@ -35,7 +38,7 @@ SELECT table FROM system.parts GROUP BY table -ORDER BY s +ORDER BY s; ``` **Пример 3:** Использование результатов скалярного подзапроса @@ -54,27 +57,14 @@ SELECT FROM system.parts GROUP BY table ORDER BY table_disk_usage DESC -LIMIT 10 +LIMIT 10; ``` **Пример 4:** Переиспользование выражения -В настоящий момент, переиспользование выражения из секции WITH внутри подзапроса возможно только через дублирование. - ``` sql -WITH ['hello'] AS hello -SELECT - hello, - * -FROM -( - WITH ['hello'] AS hello - SELECT hello -) +WITH test1 AS (SELECT i + 1, j + 1 FROM test1) +SELECT * FROM test1; ``` -``` text -┌─hello─────┬─hello─────┐ -│ ['hello'] │ ['hello'] │ -└───────────┴───────────┘ -``` +[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/statements/select/with/) From 19794bee077ed23261930b37c003ad4ae8eedcbd Mon Sep 17 00:00:00 2001 From: Ekaterina <72217280+antarctictardigrade@users.noreply.github.com> Date: Thu, 29 Oct 2020 21:27:37 +0300 Subject: [PATCH 301/325] DOCSUP-2966: Update the INSERT INTO statement (#16404) * Revert "Update version_date.tsv after release 20.7.4.11" This reverts commit 9aff247afed1479f95a4714d699c1ddaefc75499. * Added translation for feature. * Fixed. * Fixed. * Fixed. * Update docs/en/sql-reference/statements/insert-into.md * Update docs/en/sql-reference/statements/insert-into.md * Update docs/en/sql-reference/statements/insert-into.md * Update docs/en/sql-reference/statements/insert-into.md * Update docs/en/sql-reference/statements/insert-into.md * Update docs/en/sql-reference/statements/insert-into.md * Update docs/ru/sql-reference/statements/insert-into.md * Update docs/ru/sql-reference/statements/insert-into.md * Update docs/ru/sql-reference/statements/insert-into.md * Update version_date.tsv Co-authored-by: BayoNet --- .../sql-reference/statements/insert-into.md | 53 ++++++++++++++++++- .../sql-reference/statements/insert-into.md | 50 ++++++++++++++++- 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/statements/insert-into.md b/docs/en/sql-reference/statements/insert-into.md index b49314a1785..ae5e074fd15 100644 --- a/docs/en/sql-reference/statements/insert-into.md +++ b/docs/en/sql-reference/statements/insert-into.md @@ -13,12 +13,61 @@ Basic query format: INSERT INTO [db.]table [(c1, c2, c3)] VALUES (v11, v12, v13), (v21, v22, v23), ... ``` -The query can specify a list of columns to insert `[(c1, c2, c3)]`. In this case, the rest of the columns are filled with: +You can specify a list of columns to insert using the `(c1, c2, c3)` or `COLUMNS(c1,c2,c3)` syntax. + +Instead of listing all the required columns you can use the `(* EXCEPT(column_list))` syntax. + +For example, consider the table: + +``` sql +SHOW CREATE insert_select_testtable; +``` + +``` +┌─statement────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ CREATE TABLE insert_select_testtable +( + `a` Int8, + `b` String, + `c` Int8 +) +ENGINE = MergeTree() +ORDER BY a +SETTINGS index_granularity = 8192 │ +└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +``` sql +INSERT INTO insert_select_testtable (*) VALUES (1, 'a', 1) ; +``` + +If you want to insert data in all the columns, except 'b', you need to pass so many values how many columns you chose in parenthesis then: + +``` sql +INSERT INTO insert_select_testtable (* EXCEPT(b)) Values (2, 2); +``` + +``` sql +SELECT * FROM insert_select_testtable; +``` + +``` +┌─a─┬─b─┬─c─┐ +│ 2 │ │ 2 │ +└───┴───┴───┘ +┌─a─┬─b─┬─c─┐ +│ 1 │ a │ 1 │ +└───┴───┴───┘ +``` + +In this example, we see that the second inserted row has `a` and `c` columns filled by the passed values, and `b` filled with value by default. + +If a list of columns doesn't include all existing columns, the rest of the columns are filled with: - The values calculated from the `DEFAULT` expressions specified in the table definition. - Zeros and empty strings, if `DEFAULT` expressions are not defined. -If [strict_insert_defaults=1](../../operations/settings/settings.md), columns that do not have `DEFAULT` defined must be listed in the query. +If [strict\_insert\_defaults=1](../../operations/settings/settings.md), columns that do not have `DEFAULT` defined must be listed in the query. Data can be passed to the INSERT in any [format](../../interfaces/formats.md#formats) supported by ClickHouse. The format must be specified explicitly in the query: diff --git a/docs/ru/sql-reference/statements/insert-into.md b/docs/ru/sql-reference/statements/insert-into.md index 8ea7c83bec8..0d38be81ac6 100644 --- a/docs/ru/sql-reference/statements/insert-into.md +++ b/docs/ru/sql-reference/statements/insert-into.md @@ -13,7 +13,55 @@ toc_title: INSERT INTO INSERT INTO [db.]table [(c1, c2, c3)] VALUES (v11, v12, v13), (v21, v22, v23), ... ``` -В запросе можно указать список столбцов для вставки `[(c1, c2, c3)]`. В этом случае, в остальные столбцы записываются: +Вы можете указать список столбцов для вставки, используя следующий синтаксис: `(c1, c2, c3)` или `COLUMNS(c1,c2,c3)`. + +Можно не перечислять все необходимые столбцы, а использовать синтаксис `(* EXCEPT(column_list))`. + +В качестве примера рассмотрим таблицу: + +``` sql +SHOW CREATE insert_select_testtable +``` + +``` +┌─statement────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ CREATE TABLE insert_select_testtable +( + `a` Int8, + `b` String, + `c` Int8 +) +ENGINE = MergeTree() +ORDER BY a +SETTINGS index_granularity = 8192 │ +└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +``` sql +INSERT INTO insert_select_testtable (*) VALUES (1, 'a', 1) +``` + +Если вы хотите вставить данные во все столбцы, кроме 'b', вам нужно передать столько значений, сколько столбцов вы указали в скобках: + +``` sql +INSERT INTO insert_select_testtable (* EXCEPT(b)) Values (2, 2) +``` + +``` sql +SELECT * FROM insert_select_testtable +``` + +``` +┌─a─┬─b─┬─c─┐ +│ 2 │ │ 2 │ +└───┴───┴───┘ +┌─a─┬─b─┬─c─┐ +│ 1 │ a │ 1 │ +└───┴───┴───┘ +``` + +В этом примере мы видим, что вторая строка содержит столбцы `a` и `c`, заполненные переданными значениями и `b`, заполненный значением по умолчанию. +Если список столбцов не включает все существующие столбцы, то все остальные столбцы заполняются следующим образом: - Значения, вычисляемые из `DEFAULT` выражений, указанных в определении таблицы. - Нули и пустые строки, если `DEFAULT` не определены. From ba4bb18663d868bee059f844ec871da6e8cc1eee Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 29 Oct 2020 21:57:16 +0300 Subject: [PATCH 302/325] Remove useless sleep in test --- tests/queries/0_stateless/00933_ttl_with_default.sql | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/queries/0_stateless/00933_ttl_with_default.sql b/tests/queries/0_stateless/00933_ttl_with_default.sql index d3f3b62126c..e6c0a6e700c 100644 --- a/tests/queries/0_stateless/00933_ttl_with_default.sql +++ b/tests/queries/0_stateless/00933_ttl_with_default.sql @@ -5,7 +5,6 @@ insert into ttl_00933_2 values (toDateTime('2000-10-10 00:00:00'), 1); insert into ttl_00933_2 values (toDateTime('2000-10-10 00:00:00'), 2); insert into ttl_00933_2 values (toDateTime('2100-10-10 00:00:00'), 3); insert into ttl_00933_2 values (toDateTime('2100-10-10 00:00:00'), 4); -select sleep(0.7) format Null; -- wait if very fast merge happen optimize table ttl_00933_2 final; select a from ttl_00933_2 order by a; @@ -16,7 +15,6 @@ insert into ttl_00933_2 values (toDateTime('2000-10-10 00:00:00'), 1, 100); insert into ttl_00933_2 values (toDateTime('2000-10-10 00:00:00'), 2, 200); insert into ttl_00933_2 values (toDateTime('2100-10-10 00:00:00'), 3, 300); insert into ttl_00933_2 values (toDateTime('2100-10-10 00:00:00'), 4, 400); -select sleep(0.7) format Null; -- wait if very fast merge happen optimize table ttl_00933_2 final; select a, b from ttl_00933_2 order by a; @@ -27,7 +25,6 @@ insert into ttl_00933_2 values (toDateTime('2000-10-10 00:00:00'), 1, 5); insert into ttl_00933_2 values (toDateTime('2000-10-10 00:00:00'), 2, 10); insert into ttl_00933_2 values (toDateTime('2100-10-10 00:00:00'), 3, 15); insert into ttl_00933_2 values (toDateTime('2100-10-10 00:00:00'), 4, 20); -select sleep(0.7) format Null; -- wait if very fast merge happen optimize table ttl_00933_2 final; select a, b from ttl_00933_2 order by a; From f428675b6e2465f4837e71932b50a4093199a40a Mon Sep 17 00:00:00 2001 From: feng lv Date: Fri, 30 Oct 2020 05:42:10 +0000 Subject: [PATCH 303/325] fix parseDateTimeBestEffortOrNull with empty string --- src/IO/parseDateTimeBestEffort.cpp | 4 ++++ ...rse_datetime_besteffort_or_null_empty_string.reference | 8 ++++++++ ...543_parse_datetime_besteffort_or_null_empty_string.sql | 8 ++++++++ 3 files changed, 20 insertions(+) create mode 100644 tests/queries/0_stateless/01543_parse_datetime_besteffort_or_null_empty_string.reference create mode 100644 tests/queries/0_stateless/01543_parse_datetime_besteffort_or_null_empty_string.sql diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index d84ac76b164..a5e23e7f697 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -533,6 +533,10 @@ ReturnType parseDateTimeBestEffortImpl( } } + /// If neigher Date nor Time is parsed successfully, it should fail + if (!year && !month && !day_of_month && !has_time) + return on_error("Cannot read DateTime: neither Date nor Time was parsed successfully", ErrorCodes::CANNOT_PARSE_DATETIME); + if (!year) year = 2000; if (!month) diff --git a/tests/queries/0_stateless/01543_parse_datetime_besteffort_or_null_empty_string.reference b/tests/queries/0_stateless/01543_parse_datetime_besteffort_or_null_empty_string.reference new file mode 100644 index 00000000000..14be7c77e23 --- /dev/null +++ b/tests/queries/0_stateless/01543_parse_datetime_besteffort_or_null_empty_string.reference @@ -0,0 +1,8 @@ +2010-01-01 00:00:00 +2010-01-01 01:01:01 +2000-01-01 01:01:01 +\N +\N +\N +\N +\N diff --git a/tests/queries/0_stateless/01543_parse_datetime_besteffort_or_null_empty_string.sql b/tests/queries/0_stateless/01543_parse_datetime_besteffort_or_null_empty_string.sql new file mode 100644 index 00000000000..66d28534b30 --- /dev/null +++ b/tests/queries/0_stateless/01543_parse_datetime_besteffort_or_null_empty_string.sql @@ -0,0 +1,8 @@ +SELECT parseDateTimeBestEffortOrNull('2010-01-01'); +SELECT parseDateTimeBestEffortOrNull('2010-01-01 01:01:01'); +SELECT parseDateTimeBestEffortOrNull('01:01:01'); +SELECT parseDateTimeBestEffortOrNull('20100'); +SELECT parseDateTimeBestEffortOrNull('0100:0100:0000'); +SELECT parseDateTimeBestEffortOrNull('x'); +SELECT parseDateTimeBestEffortOrNull(''); +SELECT parseDateTimeBestEffortOrNull(' '); From 21fcd8aece26fe63e373f6e56d3f1fcfcbd6056e Mon Sep 17 00:00:00 2001 From: feng lv Date: Fri, 30 Oct 2020 06:45:09 +0000 Subject: [PATCH 304/325] fix fix --- .../00569_parse_date_time_best_effort.reference | 4 ++-- .../01313_parse_date_time_best_effort_null_zero.sql | 4 ++-- .../0_stateless/01442_date_time_with_params.reference | 4 ++-- tests/queries/0_stateless/01442_date_time_with_params.sql | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference b/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference index 6c10720d0b1..ad7c17b7717 100644 --- a/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference +++ b/tests/queries/0_stateless/00569_parse_date_time_best_effort.reference @@ -1,7 +1,7 @@ s a b - 0 2000-01-01 00:00:00 2000-01-01 00:00:00 - 0000 2000-01-01 00:00:00 2000-01-01 00:00:00 + 0 ᴺᵁᴸᴸ 1970-01-01 00:00:00 + 0000 ᴺᵁᴸᴸ 1970-01-01 00:00:00 00:00:00 2000-01-01 00:00:00 2000-01-01 00:00:00 01:00:00 2000-01-01 01:00:00 2000-01-01 01:00:00 02/01/17 010203 MSK 2017-01-01 22:02:03 2017-01-01 22:02:03 diff --git a/tests/queries/0_stateless/01313_parse_date_time_best_effort_null_zero.sql b/tests/queries/0_stateless/01313_parse_date_time_best_effort_null_zero.sql index c58eb722327..ed56aec3fb0 100644 --- a/tests/queries/0_stateless/01313_parse_date_time_best_effort_null_zero.sql +++ b/tests/queries/0_stateless/01313_parse_date_time_best_effort_null_zero.sql @@ -1,8 +1,8 @@ -SELECT parseDateTimeBestEffort(''); -- { serverError 6 } +SELECT parseDateTimeBestEffort(''); -- { serverError 41 } SELECT parseDateTimeBestEffortOrNull(''); SELECT parseDateTimeBestEffortOrZero('', 'UTC'); -SELECT parseDateTime64BestEffort(''); -- { serverError 6 } +SELECT parseDateTime64BestEffort(''); -- { serverError 41 } SELECT parseDateTime64BestEffortOrNull(''); SELECT parseDateTime64BestEffortOrZero('', 0, 'UTC'); diff --git a/tests/queries/0_stateless/01442_date_time_with_params.reference b/tests/queries/0_stateless/01442_date_time_with_params.reference index f55d095d164..94ffcffdd37 100644 --- a/tests/queries/0_stateless/01442_date_time_with_params.reference +++ b/tests/queries/0_stateless/01442_date_time_with_params.reference @@ -15,7 +15,7 @@ 2020-05-14 03:37:03.253 Nullable(DateTime64(3, \'UTC\')) 2020-05-14 06:37:03.253 Nullable(DateTime64(3, \'Europe/Minsk\')) 2020-05-14 03:37:03.253 Nullable(DateTime64(3, \'UTC\')) -1970-01-01 03:00:00.000 DateTime64(3) +1970-01-01 00:00:00.000 DateTime64(3, \'UTC\') 2020-05-14 03:37:03.000 DateTime64(3, \'UTC\') 2020-05-14 03:37:03.000 DateTime64(3, \'UTC\') 2020-05-14 03:37:03.253 DateTime64(3, \'UTC\') @@ -35,7 +35,7 @@ 2020-05-14 03:37:03 Nullable(DateTime(\'UTC\')) 2020-05-14 06:37:03 Nullable(DateTime(\'Europe/Minsk\')) 2020-05-14 03:37:03 Nullable(DateTime(\'UTC\')) -1970-01-01 03:00:00 DateTime +1970-01-01 00:00:00 DateTime(\'UTC\') 2020-05-14 03:37:03 DateTime(\'UTC\') 2020-05-14 03:37:03 DateTime(\'UTC\') 2020-05-14 03:37:03 DateTime(\'UTC\') diff --git a/tests/queries/0_stateless/01442_date_time_with_params.sql b/tests/queries/0_stateless/01442_date_time_with_params.sql index 5ae7fe22699..2eefa47ba8c 100644 --- a/tests/queries/0_stateless/01442_date_time_with_params.sql +++ b/tests/queries/0_stateless/01442_date_time_with_params.sql @@ -12,7 +12,7 @@ SELECT CAST('2020-01-01 00:00:00', 'DateTime') AS a, toTypeName(a), CAST('2020-0 SELECT toDateTime32('2020-01-01 00:00:00') AS a, toTypeName(a); -SELECT parseDateTimeBestEffort('', 3) AS a, toTypeName(a); -- {serverError 6} +SELECT parseDateTimeBestEffort('', 3) AS a, toTypeName(a); -- {serverError 41} SELECT parseDateTimeBestEffort('2020-05-14T03:37:03', 3, 'UTC') AS a, toTypeName(a); SELECT parseDateTimeBestEffort('2020-05-14 03:37:03', 3, 'UTC') AS a, toTypeName(a); SELECT parseDateTimeBestEffort('2020-05-14T03:37:03.253184', 3, 'UTC') AS a, toTypeName(a); @@ -28,7 +28,7 @@ SELECT parseDateTimeBestEffortOrNull('2020-05-14T03:37:03.253184Z', 3, 'UTC') AS SELECT parseDateTimeBestEffortOrNull('2020-05-14T03:37:03.253184Z', 3, 'Europe/Minsk') AS a, toTypeName(a); SELECT parseDateTimeBestEffortOrNull(materialize('2020-05-14T03:37:03.253184Z'), 3, 'UTC') AS a, toTypeName(a); -SELECT parseDateTimeBestEffortOrZero('', 3) AS a, toTypeName(a); +SELECT parseDateTimeBestEffortOrZero('', 3, 'UTC') AS a, toTypeName(a); SELECT parseDateTimeBestEffortOrZero('2020-05-14T03:37:03', 3, 'UTC') AS a, toTypeName(a); SELECT parseDateTimeBestEffortOrZero('2020-05-14 03:37:03', 3, 'UTC') AS a, toTypeName(a); SELECT parseDateTimeBestEffortOrZero('2020-05-14T03:37:03.253184', 3, 'UTC') AS a, toTypeName(a); @@ -37,7 +37,7 @@ SELECT parseDateTimeBestEffortOrZero('2020-05-14T03:37:03.253184Z', 3, 'Europe/M SELECT parseDateTimeBestEffortOrZero(materialize('2020-05-14T03:37:03.253184Z'), 3, 'UTC') AS a, toTypeName(a); -SELECT parseDateTime32BestEffort('') AS a, toTypeName(a); -- {serverError 6} +SELECT parseDateTime32BestEffort('') AS a, toTypeName(a); -- {serverError 41} SELECT parseDateTime32BestEffort('2020-05-14T03:37:03', 'UTC') AS a, toTypeName(a); SELECT parseDateTime32BestEffort('2020-05-14 03:37:03', 'UTC') AS a, toTypeName(a); SELECT parseDateTime32BestEffort('2020-05-14T03:37:03.253184', 'UTC') AS a, toTypeName(a); @@ -53,7 +53,7 @@ SELECT parseDateTime32BestEffortOrNull('2020-05-14T03:37:03.253184Z', 'UTC') AS SELECT parseDateTime32BestEffortOrNull('2020-05-14T03:37:03.253184Z', 'Europe/Minsk') AS a, toTypeName(a); SELECT parseDateTime32BestEffortOrNull(materialize('2020-05-14T03:37:03.253184Z'), 'UTC') AS a, toTypeName(a); -SELECT parseDateTime32BestEffortOrZero('') AS a, toTypeName(a); +SELECT parseDateTime32BestEffortOrZero('', 'UTC') AS a, toTypeName(a); SELECT parseDateTime32BestEffortOrZero('2020-05-14T03:37:03', 'UTC') AS a, toTypeName(a); SELECT parseDateTime32BestEffortOrZero('2020-05-14 03:37:03', 'UTC') AS a, toTypeName(a); SELECT parseDateTime32BestEffortOrZero('2020-05-14T03:37:03.253184', 'UTC') AS a, toTypeName(a); From 15ba4dd85134290b519cf9bf1c9bb6a223e931f9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 30 Oct 2020 11:52:11 +0300 Subject: [PATCH 305/325] Review fix --- src/Storages/MergeTree/DataPartsExchange.cpp | 12 +++++++++--- src/Storages/MergeTree/ReplicatedFetchList.h | 2 ++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index eaf32385908..5ab0cc4612f 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -64,9 +64,15 @@ struct ReplicatedFetchReadCallback void operator() (size_t bytes_count) { replicated_fetch_entry->bytes_read_compressed.store(bytes_count, std::memory_order_relaxed); - replicated_fetch_entry->progress.store( - static_cast(bytes_count) / replicated_fetch_entry->total_size_bytes_compressed, - std::memory_order_relaxed); + + /// It's possible when we fetch part from very old clickhouse version + /// which doesn't send total size. + if (replicated_fetch_entry->total_size_bytes_compressed != 0) + { + replicated_fetch_entry->progress.store( + static_cast(bytes_count) / replicated_fetch_entry->total_size_bytes_compressed, + std::memory_order_relaxed); + } } }; diff --git a/src/Storages/MergeTree/ReplicatedFetchList.h b/src/Storages/MergeTree/ReplicatedFetchList.h index ed134de3f2c..81d538abf9c 100644 --- a/src/Storages/MergeTree/ReplicatedFetchList.h +++ b/src/Storages/MergeTree/ReplicatedFetchList.h @@ -63,6 +63,8 @@ struct ReplicatedFetchListElement : private boost::noncopyable /// How many bytes already read std::atomic bytes_read_compressed{}; /// Total bytes to read + /// NOTE: can be zero if we fetching data from old server. + /// In this case progress is not tracked. const UInt64 total_size_bytes_compressed{}; const UInt64 thread_id; From f52d0919f06eb5448f9c136705c0dcf6a08e66ce Mon Sep 17 00:00:00 2001 From: filimonov <1549571+filimonov@users.noreply.github.com> Date: Fri, 30 Oct 2020 10:31:05 +0100 Subject: [PATCH 306/325] Update date-time-functions.md --- docs/en/sql-reference/functions/date-time-functions.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 15214cc4c00..a5a347e553a 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -23,8 +23,6 @@ SELECT └─────────────────────┴────────────┴────────────┴─────────────────────┘ ``` -Only time zones that differ from UTC by a whole number of hours are supported. - ## toTimeZone {#totimezone} Convert time or date and time to the specified time zone. From 182f9fbdca5e7401490e2e5e3dcd5a174d3ec771 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 30 Oct 2020 12:36:46 +0300 Subject: [PATCH 307/325] Remove wrong assertion from SSDComplexKeyCacheDictionary --- src/Dictionaries/SSDComplexKeyCacheDictionary.cpp | 1 - tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Dictionaries/SSDComplexKeyCacheDictionary.cpp b/src/Dictionaries/SSDComplexKeyCacheDictionary.cpp index 44847df48ff..5ac821e5eda 100644 --- a/src/Dictionaries/SSDComplexKeyCacheDictionary.cpp +++ b/src/Dictionaries/SSDComplexKeyCacheDictionary.cpp @@ -1467,7 +1467,6 @@ void SSDComplexKeyCacheDictionary::getItemsNumberImpl( { assert(dict_struct.key); assert(key_columns.size() == key_types.size()); - assert(key_columns.size() == dict_struct.key->size()); dict_struct.validateKeyTypes(key_types); diff --git a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql index 9faafb6c0c7..7f2da983525 100644 --- a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql +++ b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql @@ -62,6 +62,8 @@ SELECT dictGetUInt64('database_for_dict.ssd_dict', 'a', tuple('10', toInt32(-20) SELECT dictGetInt32('database_for_dict.ssd_dict', 'b', tuple('10', toInt32(-20))); SELECT dictGetString('database_for_dict.ssd_dict', 'c', tuple('10', toInt32(-20))); +SELECT dictGetUInt64('database_for_dict.ssd_dict', 'a', tuple(toInt32(3))); --{serverError 53} + DROP DICTIONARY database_for_dict.ssd_dict; DROP TABLE IF EXISTS database_for_dict.keys_table; From cf8202d36ebff92e2104859664a24d7547d334d2 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 30 Oct 2020 15:41:39 +0300 Subject: [PATCH 308/325] better logging, fix flacky test --- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 84 +++++++++++-------- .../MergeTree/ReplicatedMergeTreeQueue.h | 1 + ...tem_parts_race_condition_drop_zookeeper.sh | 5 +- 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 880ad4dd0d3..bf9a94747b6 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -965,13 +965,16 @@ void ReplicatedMergeTreeQueue::checkThereAreNoConflictsInRange(const MergeTreePa } -bool ReplicatedMergeTreeQueue::isNotCoveredByFuturePartsImpl(const String & new_part_name, String & out_reason, std::lock_guard & /* queue_lock */) const +bool ReplicatedMergeTreeQueue::isNotCoveredByFuturePartsImpl(const String & log_entry_name, const String & new_part_name, + String & out_reason, std::lock_guard & /* queue_lock */) const { /// Let's check if the same part is now being created by another action. if (future_parts.count(new_part_name)) { - out_reason = "Not executing log entry for part " + new_part_name - + " because another log entry for the same part is being processed. This shouldn't happen often."; + const char * format_str = "Not executing log entry {} for part {} " + "because another log entry for the same part is being processed. This shouldn't happen often."; + LOG_INFO(log, format_str, log_entry_name, new_part_name); + out_reason = fmt::format(format_str, log_entry_name, new_part_name); return false; /** When the corresponding action is completed, then `isNotCoveredByFuturePart` next time, will succeed, @@ -992,8 +995,10 @@ bool ReplicatedMergeTreeQueue::isNotCoveredByFuturePartsImpl(const String & new_ if (future_part.contains(result_part)) { - out_reason = "Not executing log entry for part " + new_part_name + " because it is covered by part " - + future_part_elem.first + " that is currently executing"; + const char * format_str = "Not executing log entry {} for part {} " + "because it is covered by part {} that is currently executing."; + LOG_TRACE(log, format_str, log_entry_name, new_part_name, future_part_elem.first); + out_reason = fmt::format(format_str, log_entry_name, new_part_name, future_part_elem.first); return false; } } @@ -1005,7 +1010,7 @@ bool ReplicatedMergeTreeQueue::addFuturePartIfNotCoveredByThem(const String & pa { std::lock_guard lock(state_mutex); - if (isNotCoveredByFuturePartsImpl(part_name, reject_reason, lock)) + if (isNotCoveredByFuturePartsImpl(entry.znode_name, part_name, reject_reason, lock)) { CurrentlyExecuting::setActualPartName(entry, part_name, *this); return true; @@ -1030,12 +1035,8 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { for (const String & new_part_name : entry.getBlockingPartNames()) { - if (!isNotCoveredByFuturePartsImpl(new_part_name, out_postpone_reason, state_lock)) - { - if (!out_postpone_reason.empty()) - LOG_DEBUG(log, out_postpone_reason); + if (!isNotCoveredByFuturePartsImpl(entry.znode_name, new_part_name, out_postpone_reason, state_lock)) return false; - } } } @@ -1051,10 +1052,11 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { if (future_parts.count(name)) { - String reason = "Not merging into part " + entry.new_part_name - + " because part " + name + " is not ready yet (log entry for that part is being processed)."; - LOG_TRACE(log, reason); - out_postpone_reason = reason; + const char * format_str = "Not executing log entry {} of type {} for part {} " + "because part {} is not ready yet (log entry for that part is being processed)."; + LOG_TRACE(log, format_str, entry.znode_name, entry.typeToString(), entry.new_part_name, name); + /// Copy-paste of above because we need structured logging (instead of already formatted message). + out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name, name); return false; } @@ -1070,9 +1072,9 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (merger_mutator.merges_blocker.isCancelled()) { - String reason = "Not executing log entry for part " + entry.new_part_name + " because merges and mutations are cancelled now."; - LOG_DEBUG(log, reason); - out_postpone_reason = reason; + const char * format_str = "Not executing log entry {} of type {} for part {} because merges and mutations are cancelled now."; + LOG_DEBUG(log, format_str, entry.znode_name, entry.typeToString(), entry.new_part_name); + out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name); return false; } @@ -1094,17 +1096,19 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { if (merger_mutator.ttl_merges_blocker.isCancelled()) { - String reason = "Not executing log entry for part " + entry.new_part_name + " because merges with TTL are cancelled now."; - LOG_DEBUG(log, reason); - out_postpone_reason = reason; + const char * format_str = "Not executing log entry {} for part {} because merges with TTL are cancelled now."; + LOG_DEBUG(log, format_str, + entry.znode_name, entry.new_part_name); + out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.new_part_name); return false; } size_t total_merges_with_ttl = data.getTotalMergesWithTTLInMergeList(); if (total_merges_with_ttl >= data_settings->max_number_of_merges_with_ttl_in_pool) { - const char * format_str = "Not executing log entry for part {}" + const char * format_str = "Not executing log entry {} for part {}" " because {} merges with TTL already executing, maximum {}."; - LOG_DEBUG(log, format_str, entry.new_part_name, total_merges_with_ttl, + LOG_DEBUG(log, format_str, entry.znode_name, + entry.new_part_name, total_merges_with_ttl, data_settings->max_number_of_merges_with_ttl_in_pool); out_postpone_reason = fmt::format(format_str, entry.new_part_name, total_merges_with_ttl, @@ -1116,15 +1120,14 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (!ignore_max_size && sum_parts_size_in_bytes > max_source_parts_size) { - const char * format_str = "Not executing log entry {} for part {}" + const char * format_str = "Not executing log entry {} of type {} for part {}" " because source parts size ({}) is greater than the current maximum ({})."; - LOG_DEBUG(log, format_str, + LOG_DEBUG(log, format_str, entry.znode_name, entry.typeToString(), entry.new_part_name, ReadableSize(sum_parts_size_in_bytes), ReadableSize(max_source_parts_size)); - /// Copy-paste of above because we need structured logging (instead of already formatted message). - out_postpone_reason = fmt::format(format_str, + out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name, ReadableSize(sum_parts_size_in_bytes), ReadableSize(max_source_parts_size)); @@ -1139,9 +1142,9 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (!alter_sequence.canExecuteMetaAlter(entry.alter_version, state_lock)) { int head_alter = alter_sequence.getHeadAlterVersion(state_lock); - out_postpone_reason = "Cannot execute alter metadata with version: " + std::to_string(entry.alter_version) - + " because another alter " + std::to_string(head_alter) - + " must be executed before"; + const char * format_str = "Cannot execute alter metadata {} with version {} because another alter {} must be executed before"; + LOG_TRACE(log, format_str, entry.znode_name, entry.alter_version, head_alter); + out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.alter_version); return false; } } @@ -1153,11 +1156,17 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { int head_alter = alter_sequence.getHeadAlterVersion(state_lock); if (head_alter == entry.alter_version) - out_postpone_reason = "Cannot execute alter data with version: " - + std::to_string(entry.alter_version) + " because metadata still not altered"; + { + const char * format_str = "Cannot execute alter data {} with version {} because metadata still not altered"; + LOG_TRACE(log, format_str, entry.znode_name, entry.alter_version); + out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.alter_version); + } else - out_postpone_reason = "Cannot execute alter data with version: " + std::to_string(entry.alter_version) - + " because another alter " + std::to_string(head_alter) + " must be executed before"; + { + const char * format_str = "Cannot execute alter data {} with version {} because another alter {} must be executed before"; + LOG_TRACE(log, format_str, entry.znode_name, entry.alter_version, head_alter); + out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.alter_version, head_alter); + } return false; } @@ -1170,7 +1179,14 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( /// Deadlock is possible if multiple DROP/REPLACE RANGE entries are executing in parallel and wait each other. /// See also removePartProducingOpsInRange(...) and ReplicatedMergeTreeQueue::CurrentlyExecuting. if (currently_executing_drop_or_replace_range) + { + + const char * format_str = "Not executing log entry {} of type {} for part {} " + "because another DROP_RANGE or REPLACE_RANGE entry are currently executing."; + LOG_TRACE(log, format_str, entry.znode_name, entry.typeToString(), entry.new_part_name); + out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name); return false; + } } return true; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 93b79c8336c..8036e66b86b 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -205,6 +205,7 @@ private: * Should be called under state_mutex. */ bool isNotCoveredByFuturePartsImpl( + const String & log_entry_name, const String & new_part_name, String & out_reason, std::lock_guard & state_lock) const; diff --git a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh index d86631513a1..d4344e6e8bd 100755 --- a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh +++ b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh @@ -97,4 +97,7 @@ timeout $TIMEOUT bash -c thread6 2>&1 | grep "was not completely removed from Zo wait -for i in {0..9}; do $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS alter_table_$i"; done +for i in {0..9}; do + $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS alter_table_$i" & +done +wait From 83220b09566737d7e8e4acc868c1cdf68cfe19e5 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 30 Oct 2020 16:00:49 +0300 Subject: [PATCH 309/325] Update parseDateTimeBestEffort.cpp --- src/IO/parseDateTimeBestEffort.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index a5e23e7f697..8a188d22236 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -533,7 +533,7 @@ ReturnType parseDateTimeBestEffortImpl( } } - /// If neigher Date nor Time is parsed successfully, it should fail + /// If neither Date nor Time is parsed successfully, it should fail if (!year && !month && !day_of_month && !has_time) return on_error("Cannot read DateTime: neither Date nor Time was parsed successfully", ErrorCodes::CANNOT_PARSE_DATETIME); From 3529100da9a3351a3482151d65044704e9e09d8e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 30 Oct 2020 18:12:15 +0300 Subject: [PATCH 310/325] Adjust ignored perf test changes after NUMA binding --- tests/performance/agg_functions_min_max_any.xml | 2 +- tests/performance/arithmetic.xml | 2 +- tests/performance/array_index_low_cardinality_strings.xml | 2 +- tests/performance/codecs_float_insert.xml | 2 +- tests/performance/codecs_int_insert.xml | 2 +- tests/performance/collations.xml | 2 +- tests/performance/column_column_comparison.xml | 2 +- tests/performance/columns_hashing.xml | 2 +- tests/performance/count.xml | 2 +- tests/performance/cpu_synthetic.xml | 2 +- tests/performance/cryptographic_hashes.xml | 2 +- tests/performance/date_parsing.xml | 2 +- tests/performance/decimal_casts.xml | 2 +- tests/performance/empty_string_serialization.xml | 2 +- tests/performance/entropy.xml | 2 +- tests/performance/extract.xml | 2 +- tests/performance/float_formatting.xml | 2 +- tests/performance/functions_with_hash_tables.xml | 2 +- tests/performance/general_purpose_hashes.xml | 2 +- tests/performance/general_purpose_hashes_on_UUID.xml | 2 +- tests/performance/great_circle_dist.xml | 2 +- tests/performance/insert_parallel.xml | 2 +- tests/performance/jit_large_requests.xml | 2 +- tests/performance/jit_small_requests.xml | 2 +- tests/performance/joins_in_memory.xml | 2 +- tests/performance/joins_in_memory_pmj.xml | 2 +- tests/performance/local_replica.xml | 2 +- tests/performance/logical_functions_large.xml | 2 +- tests/performance/logical_functions_medium.xml | 2 +- tests/performance/logical_functions_small.xml | 2 +- tests/performance/materialized_view_parallel_insert.xml | 2 +- tests/performance/math.xml | 2 +- tests/performance/merge_tree_huge_pk.xml | 2 +- tests/performance/merge_tree_many_partitions.xml | 2 +- tests/performance/merge_tree_many_partitions_2.xml | 2 +- tests/performance/merge_tree_simple_select.xml | 2 +- tests/performance/mingroupby-orderbylimit1.xml | 2 +- tests/performance/number_formatting_formats.xml | 2 +- tests/performance/order_by_single_column.xml | 2 +- tests/performance/parallel_insert.xml | 2 +- tests/performance/push_down_limit.xml | 2 +- tests/performance/random_string.xml | 2 +- tests/performance/range.xml | 2 +- tests/performance/read_in_order_many_parts.xml | 2 +- tests/performance/redundant_functions_in_order_by.xml | 2 +- tests/performance/removing_group_by_keys.xml | 2 +- tests/performance/select_format.xml | 2 +- tests/performance/set.xml | 2 +- tests/performance/set_index.xml | 2 +- tests/performance/single_fixed_string_groupby.xml | 2 +- tests/performance/string_set.xml | 2 +- tests/performance/string_sort.xml | 2 +- tests/performance/sum_map.xml | 2 +- tests/performance/uniq.xml | 2 +- tests/performance/website.xml | 2 +- 55 files changed, 55 insertions(+), 55 deletions(-) diff --git a/tests/performance/agg_functions_min_max_any.xml b/tests/performance/agg_functions_min_max_any.xml index e564cdcc609..79c9e2c6976 100644 --- a/tests/performance/agg_functions_min_max_any.xml +++ b/tests/performance/agg_functions_min_max_any.xml @@ -1,4 +1,4 @@ - + hits_100m_single diff --git a/tests/performance/arithmetic.xml b/tests/performance/arithmetic.xml index 45f0d62f227..0be61eb5823 100644 --- a/tests/performance/arithmetic.xml +++ b/tests/performance/arithmetic.xml @@ -1,4 +1,4 @@ - + 30000000000 diff --git a/tests/performance/array_index_low_cardinality_strings.xml b/tests/performance/array_index_low_cardinality_strings.xml index 896a5923a9e..bbfea083f0a 100644 --- a/tests/performance/array_index_low_cardinality_strings.xml +++ b/tests/performance/array_index_low_cardinality_strings.xml @@ -1,4 +1,4 @@ - + DROP TABLE IF EXISTS perf_lc_str CREATE TABLE perf_lc_str( str LowCardinality(String), diff --git a/tests/performance/codecs_float_insert.xml b/tests/performance/codecs_float_insert.xml index 8470df7e35c..a7cb5152c09 100644 --- a/tests/performance/codecs_float_insert.xml +++ b/tests/performance/codecs_float_insert.xml @@ -1,5 +1,5 @@ - + 1 diff --git a/tests/performance/codecs_int_insert.xml b/tests/performance/codecs_int_insert.xml index 662df80ae70..caefaba3725 100644 --- a/tests/performance/codecs_int_insert.xml +++ b/tests/performance/codecs_int_insert.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/collations.xml b/tests/performance/collations.xml index 40153a48d07..17b2d36b7e3 100644 --- a/tests/performance/collations.xml +++ b/tests/performance/collations.xml @@ -1,4 +1,4 @@ - + diff --git a/tests/performance/column_column_comparison.xml b/tests/performance/column_column_comparison.xml index dd77ba24043..2b59a65a54b 100644 --- a/tests/performance/column_column_comparison.xml +++ b/tests/performance/column_column_comparison.xml @@ -1,4 +1,4 @@ - + comparison diff --git a/tests/performance/columns_hashing.xml b/tests/performance/columns_hashing.xml index 3ea2e013acc..147bee93a17 100644 --- a/tests/performance/columns_hashing.xml +++ b/tests/performance/columns_hashing.xml @@ -1,4 +1,4 @@ - + hits_10m_single hits_100m_single diff --git a/tests/performance/count.xml b/tests/performance/count.xml index 4b8b00f48db..b75fd4e4df5 100644 --- a/tests/performance/count.xml +++ b/tests/performance/count.xml @@ -1,4 +1,4 @@ - + CREATE TABLE data(k UInt64, v UInt64) ENGINE = MergeTree ORDER BY k INSERT INTO data SELECT number, 1 from numbers(10000000) diff --git a/tests/performance/cpu_synthetic.xml b/tests/performance/cpu_synthetic.xml index 00e8b4e86d8..1a3c0737046 100644 --- a/tests/performance/cpu_synthetic.xml +++ b/tests/performance/cpu_synthetic.xml @@ -1,4 +1,4 @@ - + hits_100m_single hits_10m_single diff --git a/tests/performance/cryptographic_hashes.xml b/tests/performance/cryptographic_hashes.xml index 03d275a7bb7..fbe0babd43c 100644 --- a/tests/performance/cryptographic_hashes.xml +++ b/tests/performance/cryptographic_hashes.xml @@ -1,4 +1,4 @@ - + hash_slow diff --git a/tests/performance/date_parsing.xml b/tests/performance/date_parsing.xml index 15d267dbde5..ffe4ffb9799 100644 --- a/tests/performance/date_parsing.xml +++ b/tests/performance/date_parsing.xml @@ -1,4 +1,4 @@ - + hits_100m_single diff --git a/tests/performance/decimal_casts.xml b/tests/performance/decimal_casts.xml index 582672fa30e..d08aec7a0e4 100644 --- a/tests/performance/decimal_casts.xml +++ b/tests/performance/decimal_casts.xml @@ -1,4 +1,4 @@ - + 15G diff --git a/tests/performance/empty_string_serialization.xml b/tests/performance/empty_string_serialization.xml index d82bcf998aa..303283f08c7 100644 --- a/tests/performance/empty_string_serialization.xml +++ b/tests/performance/empty_string_serialization.xml @@ -1,4 +1,4 @@ - + - + expr diff --git a/tests/performance/functions_with_hash_tables.xml b/tests/performance/functions_with_hash_tables.xml index 2e9a88107bd..18697d7fb07 100644 --- a/tests/performance/functions_with_hash_tables.xml +++ b/tests/performance/functions_with_hash_tables.xml @@ -1,4 +1,4 @@ - + select arrayUniq(range(1 + (number % 100) * 10)) from numbers(100000) format Null select arrayDistinct(range(1 + (number % 100) * 10)) from numbers(100000) format Null select arrayEnumerateUniq(range(1 + (number % 100) * 10)) from numbers(100000) format Null diff --git a/tests/performance/general_purpose_hashes.xml b/tests/performance/general_purpose_hashes.xml index cb99b564f17..e37e1c34496 100644 --- a/tests/performance/general_purpose_hashes.xml +++ b/tests/performance/general_purpose_hashes.xml @@ -1,4 +1,4 @@ - + gp_hash_func diff --git a/tests/performance/general_purpose_hashes_on_UUID.xml b/tests/performance/general_purpose_hashes_on_UUID.xml index 9e8de81c1b1..3694e0a38bd 100644 --- a/tests/performance/general_purpose_hashes_on_UUID.xml +++ b/tests/performance/general_purpose_hashes_on_UUID.xml @@ -1,4 +1,4 @@ - + hash_func diff --git a/tests/performance/great_circle_dist.xml b/tests/performance/great_circle_dist.xml index 13f9e6fde56..b5e271ddfa8 100644 --- a/tests/performance/great_circle_dist.xml +++ b/tests/performance/great_circle_dist.xml @@ -1,4 +1,4 @@ - + SELECT count() FROM numbers(1000000) WHERE NOT ignore(greatCircleDistance((rand(1) % 360) * 1. - 180, (number % 150) * 1.2 - 90, (number % 360) + toFloat64(rand(2)) / 4294967296 - 180, (rand(3) % 180) * 1. - 90)) diff --git a/tests/performance/insert_parallel.xml b/tests/performance/insert_parallel.xml index 4c0e3f03e58..4ae50dce29a 100644 --- a/tests/performance/insert_parallel.xml +++ b/tests/performance/insert_parallel.xml @@ -1,4 +1,4 @@ - + 4 diff --git a/tests/performance/jit_large_requests.xml b/tests/performance/jit_large_requests.xml index 46328b64af2..6aed7bea544 100644 --- a/tests/performance/jit_large_requests.xml +++ b/tests/performance/jit_large_requests.xml @@ -1,4 +1,4 @@ - + CREATE TABLE jit_test ( a UInt64, diff --git a/tests/performance/jit_small_requests.xml b/tests/performance/jit_small_requests.xml index c9abec0926b..d8f917fb9af 100644 --- a/tests/performance/jit_small_requests.xml +++ b/tests/performance/jit_small_requests.xml @@ -1,4 +1,4 @@ - + WITH bitXor(number, 0x4CF2D2BAAE6DA887) AS x0, diff --git a/tests/performance/joins_in_memory.xml b/tests/performance/joins_in_memory.xml index ccb4f5d0e20..bac7679930f 100644 --- a/tests/performance/joins_in_memory.xml +++ b/tests/performance/joins_in_memory.xml @@ -1,4 +1,4 @@ - + CREATE TABLE ints (i64 Int64, i32 Int32, i16 Int16, i8 Int8) ENGINE = Memory INSERT INTO ints SELECT number AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000) diff --git a/tests/performance/joins_in_memory_pmj.xml b/tests/performance/joins_in_memory_pmj.xml index e8d1d80a12b..2b9596e1591 100644 --- a/tests/performance/joins_in_memory_pmj.xml +++ b/tests/performance/joins_in_memory_pmj.xml @@ -1,4 +1,4 @@ - + CREATE TABLE ints (i64 Int64, i32 Int32, i16 Int16, i8 Int8) ENGINE = Memory SET join_algorithm = 'partial_merge' diff --git a/tests/performance/local_replica.xml b/tests/performance/local_replica.xml index 5a979e82449..4162cb631dd 100644 --- a/tests/performance/local_replica.xml +++ b/tests/performance/local_replica.xml @@ -1,3 +1,3 @@ - + select sum(number) from remote('127.0.0.{{1|2}}', numbers_mt(1000000000)) group by bitAnd(number, 1) diff --git a/tests/performance/logical_functions_large.xml b/tests/performance/logical_functions_large.xml index 51478455c45..e696b506ca5 100644 --- a/tests/performance/logical_functions_large.xml +++ b/tests/performance/logical_functions_large.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/logical_functions_medium.xml b/tests/performance/logical_functions_medium.xml index 5e0dbd9b852..be474894b54 100644 --- a/tests/performance/logical_functions_medium.xml +++ b/tests/performance/logical_functions_medium.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/logical_functions_small.xml b/tests/performance/logical_functions_small.xml index 9876e5188d5..3d70ef6811d 100644 --- a/tests/performance/logical_functions_small.xml +++ b/tests/performance/logical_functions_small.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/materialized_view_parallel_insert.xml b/tests/performance/materialized_view_parallel_insert.xml index 1a34f71e7e6..4b71354dec3 100644 --- a/tests/performance/materialized_view_parallel_insert.xml +++ b/tests/performance/materialized_view_parallel_insert.xml @@ -1,4 +1,4 @@ - + hits_10m_single diff --git a/tests/performance/math.xml b/tests/performance/math.xml index afc20bad927..006e33548c9 100644 --- a/tests/performance/math.xml +++ b/tests/performance/math.xml @@ -1,4 +1,4 @@ - + func_slow diff --git a/tests/performance/merge_tree_huge_pk.xml b/tests/performance/merge_tree_huge_pk.xml index f4012315dbe..84e9ef2e41b 100644 --- a/tests/performance/merge_tree_huge_pk.xml +++ b/tests/performance/merge_tree_huge_pk.xml @@ -1,4 +1,4 @@ - + CREATE TABLE huge_pk ENGINE = MergeTree ORDER BY ( c001, c002, c003, c004, c005, c006, c007, c008, c009, c010, c011, c012, c013, c014, c015, c016, c017, c018, c019, c020, diff --git a/tests/performance/merge_tree_many_partitions.xml b/tests/performance/merge_tree_many_partitions.xml index 0d640d242ae..2a8a52943a3 100644 --- a/tests/performance/merge_tree_many_partitions.xml +++ b/tests/performance/merge_tree_many_partitions.xml @@ -1,4 +1,4 @@ - + CREATE TABLE bad_partitions (x UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x INSERT INTO bad_partitions SELECT * FROM numbers(10000) diff --git a/tests/performance/merge_tree_many_partitions_2.xml b/tests/performance/merge_tree_many_partitions_2.xml index 6799153ed65..0b8a4650835 100644 --- a/tests/performance/merge_tree_many_partitions_2.xml +++ b/tests/performance/merge_tree_many_partitions_2.xml @@ -1,4 +1,4 @@ - + CREATE TABLE bad_partitions (a UInt64, b UInt64, c UInt64, d UInt64, e UInt64, f UInt64, g UInt64, h UInt64, i UInt64, j UInt64, k UInt64, l UInt64, m UInt64, n UInt64, o UInt64, p UInt64, q UInt64, r UInt64, s UInt64, t UInt64, u UInt64, v UInt64, w UInt64, x UInt64, y UInt64, z UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x INSERT INTO bad_partitions (x) SELECT * FROM numbers_mt(3000) diff --git a/tests/performance/merge_tree_simple_select.xml b/tests/performance/merge_tree_simple_select.xml index 93c1e5a3f33..624e924f59a 100644 --- a/tests/performance/merge_tree_simple_select.xml +++ b/tests/performance/merge_tree_simple_select.xml @@ -1,4 +1,4 @@ - + CREATE TABLE simple_mergetree (EventDate Date, x UInt64) ENGINE = MergeTree ORDER BY x INSERT INTO simple_mergetree SELECT number, today() + intDiv(number, 10000000) FROM numbers_mt(100000000) OPTIMIZE TABLE simple_mergetree FINAL diff --git a/tests/performance/mingroupby-orderbylimit1.xml b/tests/performance/mingroupby-orderbylimit1.xml index 8cf401235a9..ba86f2e3368 100644 --- a/tests/performance/mingroupby-orderbylimit1.xml +++ b/tests/performance/mingroupby-orderbylimit1.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/number_formatting_formats.xml b/tests/performance/number_formatting_formats.xml index c4a17b1f133..92e04a62024 100644 --- a/tests/performance/number_formatting_formats.xml +++ b/tests/performance/number_formatting_formats.xml @@ -1,4 +1,4 @@ - + format diff --git a/tests/performance/order_by_single_column.xml b/tests/performance/order_by_single_column.xml index d58f3b5fff1..9b708ea393c 100644 --- a/tests/performance/order_by_single_column.xml +++ b/tests/performance/order_by_single_column.xml @@ -1,4 +1,4 @@ - + sorting comparison diff --git a/tests/performance/parallel_insert.xml b/tests/performance/parallel_insert.xml index b17d0f8bd64..4050e771cb8 100644 --- a/tests/performance/parallel_insert.xml +++ b/tests/performance/parallel_insert.xml @@ -1,4 +1,4 @@ - + hits_10m_single diff --git a/tests/performance/push_down_limit.xml b/tests/performance/push_down_limit.xml index 02cce9539e9..0dcd9335a52 100644 --- a/tests/performance/push_down_limit.xml +++ b/tests/performance/push_down_limit.xml @@ -1,4 +1,4 @@ - + select number from (select number from numbers(10000000) order by -number) limit 10 select number from (select number from numbers_mt(100000000) order by -number) limit 10 diff --git a/tests/performance/random_string.xml b/tests/performance/random_string.xml index 3b714187be3..13ea35ebaba 100644 --- a/tests/performance/random_string.xml +++ b/tests/performance/random_string.xml @@ -1,4 +1,4 @@ - + SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomString(10)) SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomString(100)) SELECT count() FROM zeros(100000) WHERE NOT ignore(randomString(1000)) diff --git a/tests/performance/range.xml b/tests/performance/range.xml index 97c7d4c9c3f..48115db875b 100644 --- a/tests/performance/range.xml +++ b/tests/performance/range.xml @@ -1,4 +1,4 @@ - + SELECT range(number % 100) FROM numbers(10000000) FORMAT Null SELECT range(0, number % 100, 1) FROM numbers(10000000) FORMAT Null diff --git a/tests/performance/read_in_order_many_parts.xml b/tests/performance/read_in_order_many_parts.xml index 5329c45bfdd..b45655a6f21 100644 --- a/tests/performance/read_in_order_many_parts.xml +++ b/tests/performance/read_in_order_many_parts.xml @@ -1,4 +1,4 @@ - + 1 1 diff --git a/tests/performance/redundant_functions_in_order_by.xml b/tests/performance/redundant_functions_in_order_by.xml index b259e08c973..894e8dc1917 100644 --- a/tests/performance/redundant_functions_in_order_by.xml +++ b/tests/performance/redundant_functions_in_order_by.xml @@ -1,4 +1,4 @@ - + hits_100m_single diff --git a/tests/performance/removing_group_by_keys.xml b/tests/performance/removing_group_by_keys.xml index 6db641966ec..8f792c7ccc2 100644 --- a/tests/performance/removing_group_by_keys.xml +++ b/tests/performance/removing_group_by_keys.xml @@ -1,4 +1,4 @@ - + hits_10m_single hits_100m_single diff --git a/tests/performance/select_format.xml b/tests/performance/select_format.xml index 5f9c2e3f73b..bbe489c06c6 100644 --- a/tests/performance/select_format.xml +++ b/tests/performance/select_format.xml @@ -1,4 +1,4 @@ - + 1000000 1 diff --git a/tests/performance/set.xml b/tests/performance/set.xml index be39cac77e1..cbbff1f5bb2 100644 --- a/tests/performance/set.xml +++ b/tests/performance/set.xml @@ -1,4 +1,4 @@ - + table_small diff --git a/tests/performance/set_index.xml b/tests/performance/set_index.xml index 657d831f3ee..76f1087a1bf 100644 --- a/tests/performance/set_index.xml +++ b/tests/performance/set_index.xml @@ -1,4 +1,4 @@ - + CREATE TABLE test_in (`a` UInt32) ENGINE = MergeTree() ORDER BY a INSERT INTO test_in SELECT number FROM numbers(500000000) diff --git a/tests/performance/single_fixed_string_groupby.xml b/tests/performance/single_fixed_string_groupby.xml index 013de70c11a..6bf5821707f 100644 --- a/tests/performance/single_fixed_string_groupby.xml +++ b/tests/performance/single_fixed_string_groupby.xml @@ -1,4 +1,4 @@ - + DROP TABLE IF EXISTS perf_lc_fixed_str_groupby CREATE TABLE perf_lc_fixed_str_groupby( a LowCardinality(FixedString(14)), diff --git a/tests/performance/string_set.xml b/tests/performance/string_set.xml index bbbfe2d3c2b..4d128787d1f 100644 --- a/tests/performance/string_set.xml +++ b/tests/performance/string_set.xml @@ -1,4 +1,4 @@ - + diff --git a/tests/performance/string_sort.xml b/tests/performance/string_sort.xml index e7d06d930be..5d859398ece 100644 --- a/tests/performance/string_sort.xml +++ b/tests/performance/string_sort.xml @@ -1,4 +1,4 @@ - + hits_10m_single hits_100m_single diff --git a/tests/performance/sum_map.xml b/tests/performance/sum_map.xml index b732c150220..bc9f9be2a18 100644 --- a/tests/performance/sum_map.xml +++ b/tests/performance/sum_map.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/uniq.xml b/tests/performance/uniq.xml index 334a513ab1d..7a35c6fb704 100644 --- a/tests/performance/uniq.xml +++ b/tests/performance/uniq.xml @@ -1,4 +1,4 @@ - + hits_100m_single diff --git a/tests/performance/website.xml b/tests/performance/website.xml index 2127a71c55c..9e7c8cdc536 100644 --- a/tests/performance/website.xml +++ b/tests/performance/website.xml @@ -1,4 +1,4 @@ - + hits_10m_single From 38726847fcd1a7da049790cab65c76f9f42ed370 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 30 Oct 2020 18:16:30 +0300 Subject: [PATCH 311/325] make the test longer --- tests/performance/round_down.xml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/performance/round_down.xml b/tests/performance/round_down.xml index c309a767843..32263d69fc7 100644 --- a/tests/performance/round_down.xml +++ b/tests/performance/round_down.xml @@ -1,10 +1,7 @@ - - - - SELECT count() FROM zeros(10000000) WHERE NOT ignore(roundDuration(rand() % 65536)) - SELECT count() FROM zeros(10000000) WHERE NOT ignore(roundDown(rand() % 65536, [0, 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000])) - SELECT count() FROM zeros(10000000) WHERE NOT ignore(roundAge(rand() % 100)) - SELECT count() FROM zeros(10000000) WHERE NOT ignore(roundDown(rand() % 100, [0, 1, 18, 25, 35, 45, 55])) - SELECT count() FROM numbers(10000000) WHERE NOT ignore(roundDown(rand() % 65536, (SELECT groupArray(number) FROM numbers(65536)))) + SELECT count() FROM zeros (100000000) WHERE NOT ignore(roundDuration(rand() % 65536)) + SELECT count() FROM zeros (100000000) WHERE NOT ignore(roundDown(rand() % 65536, [0, 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000])) + SELECT count() FROM zeros (100000000) WHERE NOT ignore(roundAge(rand() % 100)) + SELECT count() FROM zeros (100000000) WHERE NOT ignore(roundDown(rand() % 100, [0, 1, 18, 25, 35, 45, 55])) + SELECT count() FROM numbers(10000000) WHERE NOT ignore(roundDown(rand() % 65536, (SELECT groupArray(number) FROM numbers(65536)))) From ce6c44eb9fdcdde5c6ed77ea30e1d1bd83fb4aa6 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Fri, 30 Oct 2020 18:47:58 +0300 Subject: [PATCH 312/325] Update ReplicatedMergeTreeQueue.cpp --- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index bf9a94747b6..ef4d7ebc9c8 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1144,7 +1144,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( int head_alter = alter_sequence.getHeadAlterVersion(state_lock); const char * format_str = "Cannot execute alter metadata {} with version {} because another alter {} must be executed before"; LOG_TRACE(log, format_str, entry.znode_name, entry.alter_version, head_alter); - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.alter_version); + out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.alter_version, head_alter); return false; } } From 7e76b7b51090c43e3cbd3459688007db2aeb0e35 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 30 Oct 2020 18:54:38 +0300 Subject: [PATCH 313/325] more fixes --- tests/performance/any_anyLast.xml | 21 ++++--------------- tests/performance/cpu_synthetic.xml | 2 +- tests/performance/decimal_casts.xml | 2 +- tests/performance/format_readable.xml | 6 +++--- .../functions_with_hash_tables.xml | 2 +- .../inserts_arrays_lowcardinality.xml | 2 +- tests/performance/jit_large_requests.xml | 2 +- tests/performance/order_by_read_in_order.xml | 6 +----- tests/performance/parallel_mv.xml | 3 +-- tests/performance/rand.xml | 12 +---------- tests/performance/random_string.xml | 14 ++++++------- tests/performance/random_string_utf8.xml | 17 +++++++-------- 12 files changed, 29 insertions(+), 60 deletions(-) diff --git a/tests/performance/any_anyLast.xml b/tests/performance/any_anyLast.xml index 8d75d7556be..659df4a22ac 100644 --- a/tests/performance/any_anyLast.xml +++ b/tests/performance/any_anyLast.xml @@ -1,19 +1,6 @@ - - - - 10 - - - - - SELECT any(-1 * (((-2 * (number * -3)) * -4) * -5)) FROM numbers(120000000) - - SELECT anyLast(-1 * (((-2 * (number * -3)) * -4) * -5)) FROM numbers(120000000) - - SELECT any(number * 2) as n, n * 3 FROM numbers(120000000) - - SELECT any(number * round(toInt64(number), -2)) FROM numbers(120000000) - - + SELECT any(-1 * (((-2 * (number * -3)) * -4) * -5)) FROM numbers(500000000) + SELECT anyLast(-1 * (((-2 * (number * -3)) * -4) * -5)) FROM numbers(500000000) + SELECT any(number * 2) as n, n * 3 FROM numbers(500000000) + SELECT any(number * round(toInt64(number), -2)) FROM numbers(500000000) diff --git a/tests/performance/cpu_synthetic.xml b/tests/performance/cpu_synthetic.xml index 1a3c0737046..85a4e54c752 100644 --- a/tests/performance/cpu_synthetic.xml +++ b/tests/performance/cpu_synthetic.xml @@ -67,7 +67,7 @@ PageCharset тоже почти всегда непуст, но его сред SELECT uniqCombined(17)(UserID) FROM hits_100m_single SELECT uniqExact(UserID) FROM hits_10m_single SETTINGS max_threads = 1 -SELECT uniqExact(UserID) FROM hits_10m_single +SELECT uniqExact(UserID) FROM hits_100m_single SELECT RegionID, uniq(UserID) FROM hits_10m_single GROUP BY RegionID SETTINGS max_threads = 1 SELECT RegionID, uniq(UserID) FROM hits_100m_single GROUP BY RegionID diff --git a/tests/performance/decimal_casts.xml b/tests/performance/decimal_casts.xml index d08aec7a0e4..f087d0938c1 100644 --- a/tests/performance/decimal_casts.xml +++ b/tests/performance/decimal_casts.xml @@ -5,7 +5,7 @@ CREATE TABLE t (x UInt64, d32 Decimal32(3), d64 Decimal64(4), d128 Decimal128(5)) ENGINE = Memory - INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(100000000) SETTINGS max_threads = 8 + INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(200000000) SETTINGS max_threads = 8 DROP TABLE IF EXISTS t SELECT toUInt32(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t FORMAT Null diff --git a/tests/performance/format_readable.xml b/tests/performance/format_readable.xml index 7040b35da2b..5a76ac67333 100644 --- a/tests/performance/format_readable.xml +++ b/tests/performance/format_readable.xml @@ -1,5 +1,5 @@ -SELECT count() FROM numbers(1000000) WHERE NOT ignore(formatReadableSize(number)) -SELECT count() FROM numbers(1000000) WHERE NOT ignore(formatReadableQuantity(number)) -SELECT count() FROM numbers(1000000) WHERE NOT ignore(formatReadableTimeDelta(number)) +SELECT count() FROM numbers(10000000) WHERE NOT ignore(formatReadableSize(number)) +SELECT count() FROM numbers(10000000) WHERE NOT ignore(formatReadableQuantity(number)) +SELECT count() FROM numbers(10000000) WHERE NOT ignore(formatReadableTimeDelta(number)) diff --git a/tests/performance/functions_with_hash_tables.xml b/tests/performance/functions_with_hash_tables.xml index 18697d7fb07..18f73b54e30 100644 --- a/tests/performance/functions_with_hash_tables.xml +++ b/tests/performance/functions_with_hash_tables.xml @@ -2,7 +2,7 @@ select arrayUniq(range(1 + (number % 100) * 10)) from numbers(100000) format Null select arrayDistinct(range(1 + (number % 100) * 10)) from numbers(100000) format Null select arrayEnumerateUniq(range(1 + (number % 100) * 10)) from numbers(100000) format Null - select arrayIntersect(range((1 + number % 100)), range(1, (1 + number % 100) + 1)) from numbers(100000) format Null + select arrayIntersect(range((1 + number % 100)), range(1, (1 + number % 100) + 1)) from numbers(1000000) format Null select groupUniqArray(rand() % 100) from numbers(1000 * 1000) group by number / 1000 format Null select entropy(number / 10) from numbers(1000 * 1000) group by number / 1000 format Null diff --git a/tests/performance/inserts_arrays_lowcardinality.xml b/tests/performance/inserts_arrays_lowcardinality.xml index 40617fb9593..505156d4fd5 100644 --- a/tests/performance/inserts_arrays_lowcardinality.xml +++ b/tests/performance/inserts_arrays_lowcardinality.xml @@ -61,7 +61,7 @@ intDiv( number, 4 ) as rowid, groupArray( base64Encode( left( reinterpretAsString( rand64() ), 6) ) ) as arr FROM - numbers(1000000) + numbers(2000000) GROUP BY rowid ) GROUP BY id optimize table lot_of_string_arrays_src diff --git a/tests/performance/jit_large_requests.xml b/tests/performance/jit_large_requests.xml index 6aed7bea544..fe7d4346152 100644 --- a/tests/performance/jit_large_requests.xml +++ b/tests/performance/jit_large_requests.xml @@ -29,7 +29,7 @@ number FROM system.numbers - LIMIT 10000000 + LIMIT 100000000 SELECT diff --git a/tests/performance/order_by_read_in_order.xml b/tests/performance/order_by_read_in_order.xml index bb3a921c862..b91cd14baf4 100644 --- a/tests/performance/order_by_read_in_order.xml +++ b/tests/performance/order_by_read_in_order.xml @@ -1,13 +1,9 @@ - - - - hits_100m_single -SELECT * FROM hits_100m_single ORDER BY CounterID, EventDate LIMIT 100 +SELECT * FROM hits_100m_single ORDER BY CounterID, EventDate LIMIT 1000 SELECT * FROM hits_100m_single ORDER BY CounterID DESC, toStartOfWeek(EventDate) DESC LIMIT 100 SELECT * FROM hits_100m_single ORDER BY CounterID, EventDate, URL LIMIT 100 SELECT * FROM hits_100m_single WHERE CounterID IN (152220, 168777, 149234, 149234) ORDER BY CounterID DESC, EventDate DESC LIMIT 100 diff --git a/tests/performance/parallel_mv.xml b/tests/performance/parallel_mv.xml index ef50d506708..1e9b92c54df 100644 --- a/tests/performance/parallel_mv.xml +++ b/tests/performance/parallel_mv.xml @@ -13,8 +13,7 @@ create materialized view mv_4 engine = MergeTree order by tuple() as select number, toString(number) from main_table where number % 13 != 4; - - insert into main_table select number from numbers(1000000) + insert into main_table select number from numbers(10000000) drop table if exists main_table; drop table if exists mv_1; diff --git a/tests/performance/rand.xml b/tests/performance/rand.xml index ed629e5a2a7..3f46d152206 100644 --- a/tests/performance/rand.xml +++ b/tests/performance/rand.xml @@ -1,19 +1,9 @@ - - - 10000 - - - 20000 - - - - table - zeros(100000000) + zeros(1000000000) diff --git a/tests/performance/random_string.xml b/tests/performance/random_string.xml index 13ea35ebaba..1a740ae077a 100644 --- a/tests/performance/random_string.xml +++ b/tests/performance/random_string.xml @@ -1,9 +1,9 @@ - SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomString(10)) - SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomString(100)) - SELECT count() FROM zeros(100000) WHERE NOT ignore(randomString(1000)) - SELECT count() FROM zeros(10000) WHERE NOT ignore(randomString(10000)) - SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomString(rand() % 10)) - SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomString(rand() % 100)) - SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomString(rand() % 1000)) + SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomString(10)) + SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomString(100)) + SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomString(1000)) + SELECT count() FROM zeros(100000) WHERE NOT ignore(randomString(10000)) + SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomString(rand() % 10)) + SELECT count() FROM zeros(100000000) WHERE NOT ignore(randomString(rand() % 100)) + SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomString(rand() % 1000)) diff --git a/tests/performance/random_string_utf8.xml b/tests/performance/random_string_utf8.xml index 0185f519c31..f501cc31c24 100644 --- a/tests/performance/random_string_utf8.xml +++ b/tests/performance/random_string_utf8.xml @@ -1,12 +1,9 @@ - - - - SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomStringUTF8(10)) - SELECT count() FROM zeros(100000) WHERE NOT ignore(randomStringUTF8(100)) - SELECT count() FROM zeros(10000) WHERE NOT ignore(randomStringUTF8(1000)) - SELECT count() FROM zeros(1000) WHERE NOT ignore(randomStringUTF8(10000)) - SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomStringUTF8(rand() % 10)) - SELECT count() FROM zeros(100000) WHERE NOT ignore(randomStringUTF8(rand() % 100)) - SELECT count() FROM zeros(10000) WHERE NOT ignore(randomStringUTF8(rand() % 1000)) + SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomStringUTF8(10)) + SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomStringUTF8(100)) + SELECT count() FROM zeros(100000) WHERE NOT ignore(randomStringUTF8(1000)) + SELECT count() FROM zeros(10000) WHERE NOT ignore(randomStringUTF8(10000)) + SELECT count() FROM zeros(10000000) WHERE NOT ignore(randomStringUTF8(rand() % 10)) + SELECT count() FROM zeros(1000000) WHERE NOT ignore(randomStringUTF8(rand() % 100)) + SELECT count() FROM zeros(100000) WHERE NOT ignore(randomStringUTF8(rand() % 1000)) From 49d3b65cc276aa743d6964bd420d1eba6bc6cfb0 Mon Sep 17 00:00:00 2001 From: olgarev <56617294+olgarev@users.noreply.github.com> Date: Fri, 30 Oct 2020 20:25:05 +0300 Subject: [PATCH 314/325] DOCSUP-3537: Documented the output_format_pretty_row_numbers setting (#16446) * Docs en, ru * Update docs/en/operations/settings/settings.md Co-authored-by: BayoNet * Docs for allow_experimental_bigint_types translated to Russian. Co-authored-by: Olga Revyakina Co-authored-by: BayoNet --- docs/en/operations/settings/settings.md | 31 +++++++++++++++++- docs/ru/operations/settings/settings.md | 42 +++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 1efb7cfa8d6..a4bd7d77bfc 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -2148,7 +2148,34 @@ Result: └───────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) +## output_format_pretty_row_numbers {#output_format_pretty_row_numbers} + +Adds row numbers to output in the [Pretty](../../interfaces/formats.md#pretty) format. + +Possible values: + +- 0 — Output without row numbers. +- 1 — Output with row numbers. + +Default value: `0`. + +**Example** + +Query: + +```sql +SET output_format_pretty_row_numbers = 1; +SELECT TOP 3 name, value FROM system.settings; +``` + +Result: +```text + ┌─name────────────────────┬─value───┐ +1. │ min_compress_block_size │ 65536 │ +2. │ max_compress_block_size │ 1048576 │ +3. │ max_block_size │ 65505 │ + └─────────────────────────┴─────────┘ +``` ## allow_experimental_bigint_types {#allow_experimental_bigint_types} @@ -2160,3 +2187,5 @@ Possible values: - 0 — The bigint data type is disabled. Default value: `0`. + +[Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 3192b523d48..700203e36e1 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1977,6 +1977,48 @@ SELECT range(number) FROM system.numbers LIMIT 5 FORMAT PrettyCompactNoEscapes; └───────────────┘ ``` +## output_format_pretty_row_numbers {#output_format_pretty_row_numbers} + +Включает режим отображения номеров строк для запросов, выводимых в формате [Pretty](../../interfaces/formats.md#pretty). + +Возможные значения: + +- 0 — номера строк не выводятся. +- 1 — номера строк выводятся. + +Значение по умолчанию: `0`. + +**Пример** + +Запрос: + +```sql +SET output_format_pretty_row_numbers = 1; +SELECT TOP 3 name, value FROM system.settings; +``` + +Результат: + +```text + ┌─name────────────────────┬─value───┐ +1. │ min_compress_block_size │ 65536 │ +2. │ max_compress_block_size │ 1048576 │ +3. │ max_block_size │ 65505 │ + └─────────────────────────┴─────────┘ +``` + +## allow_experimental_bigint_types {#allow_experimental_bigint_types} + +Включает или отключает поддержку целочисленных значений, превышающих максимальное значение, допустимое для типа `int`. + +Возможные значения: + +- 1 — большие целочисленные значения поддерживаются. +- 0 — большие целочисленные значения не поддерживаются. + +Значение по умолчанию: `0`. + + ## lock_acquire_timeout {#lock_acquire_timeout} Устанавливает, сколько секунд сервер ожидает возможности выполнить блокировку таблицы. From e15a8d5ac0a509f52f7d3d74e2f219cea05fdf17 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 30 Oct 2020 21:03:07 +0300 Subject: [PATCH 315/325] fixup --- tests/performance/parse_engine_file.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/parse_engine_file.xml b/tests/performance/parse_engine_file.xml index dacd73f5364..f876f314a90 100644 --- a/tests/performance/parse_engine_file.xml +++ b/tests/performance/parse_engine_file.xml @@ -2,7 +2,7 @@ CREATE TABLE IF NOT EXISTS table_{format} ENGINE = File({format}) AS test.hits - INSERT INTO table_{format} SELECT * FROM test.hits LIMIT 100000 + INSERT INTO table_{format} SELECT * FROM test.hits LIMIT 200000 From 5bab7a5bc72ee8bfe7d79f283f265aa0e1d87bb1 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 30 Oct 2020 21:16:10 +0300 Subject: [PATCH 316/325] Add current_database into query_thread_log --- src/Common/ThreadStatus.h | 2 +- src/Interpreters/QueryThreadLog.cpp | 2 + src/Interpreters/QueryThreadLog.h | 2 + src/Interpreters/ThreadStatusExt.cpp | 5 ++- ...01547_query_log_current_database.reference | 2 + .../01547_query_log_current_database.sql | 37 +++++++++++++++++++ 6 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/01547_query_log_current_database.reference create mode 100644 tests/queries/0_stateless/01547_query_log_current_database.sql diff --git a/src/Common/ThreadStatus.h b/src/Common/ThreadStatus.h index 7d85059f23e..820ea449d66 100644 --- a/src/Common/ThreadStatus.h +++ b/src/Common/ThreadStatus.h @@ -172,7 +172,7 @@ protected: void finalizeQueryProfiler(); - void logToQueryThreadLog(QueryThreadLog & thread_log); + void logToQueryThreadLog(QueryThreadLog & thread_log, const String & current_database); void assertState(const std::initializer_list & permitted_states, const char * description = nullptr) const; diff --git a/src/Interpreters/QueryThreadLog.cpp b/src/Interpreters/QueryThreadLog.cpp index 2ecb03d622a..8fea360085b 100644 --- a/src/Interpreters/QueryThreadLog.cpp +++ b/src/Interpreters/QueryThreadLog.cpp @@ -38,6 +38,7 @@ Block QueryThreadLogElement::createBlock() {std::make_shared(), "thread_name"}, {std::make_shared(), "thread_id"}, {std::make_shared(), "master_thread_id"}, + {std::make_shared(), "current_database"}, {std::make_shared(), "query"}, {std::make_shared(), "is_initial_query"}, @@ -91,6 +92,7 @@ void QueryThreadLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(thread_id); columns[i++]->insert(master_thread_id); + columns[i++]->insertData(current_database.data(), current_database.size()); columns[i++]->insertData(query.data(), query.size()); QueryLogElement::appendClientInfo(client_info, columns, i); diff --git a/src/Interpreters/QueryThreadLog.h b/src/Interpreters/QueryThreadLog.h index 715902b29ad..5080bfe6919 100644 --- a/src/Interpreters/QueryThreadLog.h +++ b/src/Interpreters/QueryThreadLog.h @@ -39,7 +39,9 @@ struct QueryThreadLogElement UInt64 thread_id{}; UInt64 master_thread_id{}; + String current_database; String query; + ClientInfo client_info; std::shared_ptr profile_counters; diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp index 7f29cfc7e5c..2ce98819a44 100644 --- a/src/Interpreters/ThreadStatusExt.cpp +++ b/src/Interpreters/ThreadStatusExt.cpp @@ -243,7 +243,7 @@ void ThreadStatus::finalizePerformanceCounters() const auto & settings = query_context->getSettingsRef(); if (settings.log_queries && settings.log_query_threads) if (auto thread_log = global_context->getQueryThreadLog()) - logToQueryThreadLog(*thread_log); + logToQueryThreadLog(*thread_log, query_context->getCurrentDatabase()); } } catch (...) @@ -322,7 +322,7 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits) #endif } -void ThreadStatus::logToQueryThreadLog(QueryThreadLog & thread_log) +void ThreadStatus::logToQueryThreadLog(QueryThreadLog & thread_log, const String & current_database) { QueryThreadLogElement elem; @@ -350,6 +350,7 @@ void ThreadStatus::logToQueryThreadLog(QueryThreadLog & thread_log) elem.thread_name = getThreadName(); elem.thread_id = thread_id; + elem.current_database = current_database; if (thread_group) { { diff --git a/tests/queries/0_stateless/01547_query_log_current_database.reference b/tests/queries/0_stateless/01547_query_log_current_database.reference new file mode 100644 index 00000000000..6ed281c757a --- /dev/null +++ b/tests/queries/0_stateless/01547_query_log_current_database.reference @@ -0,0 +1,2 @@ +1 +1 diff --git a/tests/queries/0_stateless/01547_query_log_current_database.sql b/tests/queries/0_stateless/01547_query_log_current_database.sql new file mode 100644 index 00000000000..c0ad22163ba --- /dev/null +++ b/tests/queries/0_stateless/01547_query_log_current_database.sql @@ -0,0 +1,37 @@ +-- +-- This is more cleaner approach for writing a test that relies on system.query_log/query_thread_log. +-- +-- It uses current database, and since clickhouse-test will generate random for +-- each run you can run the test multiple times without worrying about +-- overlaps. +-- +-- There is still event_date/event_time filter for better performance +-- (even though this is not relevant for runs on CI) +-- + +set log_query_threads=1; +set log_queries_min_type='QUERY_FINISH'; +set log_queries=1; +select '01547_query_log_current_database' from system.one format Null; +set log_queries=0; +set log_query_threads=0; + +system flush logs; + +select count() +from system.query_log +where + query like '%01547_query_log_current_database%' + and current_database = currentDatabase() + and event_date = today() + and event_time >= now() - interval 1 minute; + +-- at least two threads for processing +-- (but one just waits for another, sigh) +select count() == 2 +from system.query_thread_log +where + query like '%01547_query_log_current_database%' + and current_database = currentDatabase() + and event_date = today() + and event_time >= now() - interval 1 minute; From ccddf5e9cfcdb977c2c777b4bbd25d584b612e03 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 30 Oct 2020 22:56:11 +0300 Subject: [PATCH 317/325] Add 01547_query_log_current_database into arcadia_skip_list In attempt to fix "Yandex synchronization check (only for Yandex employees)" --- tests/queries/0_stateless/arcadia_skip_list.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 65cdca80485..928ddb4cf97 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -155,3 +155,4 @@ 01509_dictionary_preallocate 01526_max_untracked_memory 01530_drop_database_atomic_sync +01547_query_log_current_database From 067a3f9814186b62c3cd60b97adbee582c32b1b5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 31 Oct 2020 01:23:28 +0300 Subject: [PATCH 318/325] Fix 01541_max_memory_usage_for_user flackiness --- .../queries/0_stateless/01541_max_memory_usage_for_user.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01541_max_memory_usage_for_user.sh b/tests/queries/0_stateless/01541_max_memory_usage_for_user.sh index 945f202a803..ecae442b134 100755 --- a/tests/queries/0_stateless/01541_max_memory_usage_for_user.sh +++ b/tests/queries/0_stateless/01541_max_memory_usage_for_user.sh @@ -35,7 +35,8 @@ function execute_tcp_one_session() # one users query in background (to avoid reseting max_memory_usage_for_user) # --max_block_size=1 to make it killable (check the state each 1 second, 1 row) # (the test takes ~40 seconds in debug build, so 60 seconds is ok) -${CLICKHOUSE_CLIENT} --max_block_size=1 --format Null -q 'SELECT sleepEachRow(1) FROM numbers(600)' & +query_id=$$-$RANDOM-$SECONDS +${CLICKHOUSE_CLIENT} --max_block_size=1 --format Null --query_id $query_id -q 'SELECT sleepEachRow(1) FROM numbers(600)' & # trap sleep_query_pid=$! function cleanup() @@ -43,6 +44,10 @@ function cleanup() echo 'KILL sleep' # if the timeout will not be enough, it will trigger "No such process" error/message kill $sleep_query_pid + # waiting for a query to finish + while ${CLICKHOUSE_CLIENT} -q "SELECT query_id FROM system.processes WHERE query_id = '$query_id'" | grep -xq "$query_id"; do + sleep 0.1 + done } trap cleanup EXIT From 8631b31ae0156602fbc21a751908afbf058ea707 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Sat, 31 Oct 2020 14:43:24 +0300 Subject: [PATCH 319/325] Update MySQLGtid.cpp --- src/Core/MySQL/MySQLGtid.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/MySQL/MySQLGtid.cpp b/src/Core/MySQL/MySQLGtid.cpp index 6e50998da62..aac3e01369a 100644 --- a/src/Core/MySQL/MySQLGtid.cpp +++ b/src/Core/MySQL/MySQLGtid.cpp @@ -121,7 +121,8 @@ void GTIDSets::update(const GTID & other) void GTIDSet::tryShirnk(GTIDSet & set, unsigned int i, GTIDSet::Interval & current) { - if (i != set.intervals.size() -1) { + if (i != set.intervals.size() -1) + { auto & next = set.intervals[i+1]; if (current.end == next.start) set.tryMerge(i); From 17ecafe452b237e29a0ec7039da0fa77f2fc36c9 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Sat, 31 Oct 2020 14:43:46 +0300 Subject: [PATCH 320/325] Update MySQLGtid.h --- src/Core/MySQL/MySQLGtid.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/MySQL/MySQLGtid.h b/src/Core/MySQL/MySQLGtid.h index cd8cd0a2e98..27aabdafc11 100644 --- a/src/Core/MySQL/MySQLGtid.h +++ b/src/Core/MySQL/MySQLGtid.h @@ -27,7 +27,7 @@ public: void tryMerge(size_t i); - static void tryShirnk(GTIDSet & set, unsigned int i, Interval & current) ; + static void tryShirnk(GTIDSet & set, unsigned int i, Interval & current); }; class GTIDSets From 90dcc135b4043ecca726adc2dc9470ef48a0fba2 Mon Sep 17 00:00:00 2001 From: Anna Shakhova <72564598+annvsh@users.noreply.github.com> Date: Sun, 1 Nov 2020 03:26:14 +0700 Subject: [PATCH 321/325] DOCSUP-3199: Support default arguments for Replicated engine (#16282) * Init commit * Fixed * Fixed * Fixed --- .../mergetree-family/replication.md | 25 +++++++++++++++++++ .../mergetree-family/replication.md | 25 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 26f79484d67..932facc9ddc 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -148,6 +148,31 @@ You can define the parameters explicitly instead of using substitutions. This mi When working with large clusters, we recommend using substitutions because they reduce the probability of error. +You can specify default arguments for `Replicated` table engine in the server configuration file. For instance: + +```xml +/clickhouse/tables/{shard}/{database}/{table} +{replica} +``` + +In this case, you can omit arguments when creating tables: + +``` sql +CREATE TABLE table_name ( + x UInt32 +) ENGINE = ReplicatedMergeTree +ORDER BY x; +``` + +It is equivalent to: + +``` sql +CREATE TABLE table_name ( + x UInt32 +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/table_name', '{replica}') +ORDER BY x; +``` + Run the `CREATE TABLE` query on each replica. This query creates a new replicated table, or adds a new replica to an existing one. If you add a new replica after the table already contains some data on other replicas, the data will be copied from the other replicas to the new one after running the query. In other words, the new replica syncs itself with the others. diff --git a/docs/ru/engines/table-engines/mergetree-family/replication.md b/docs/ru/engines/table-engines/mergetree-family/replication.md index 426c456ba3a..6d3930e33ce 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replication.md +++ b/docs/ru/engines/table-engines/mergetree-family/replication.md @@ -149,6 +149,31 @@ CREATE TABLE table_name При работе с большими кластерами мы рекомендуем использовать подстановки, они уменьшают вероятность ошибки. +Можно указать аргументы по умолчанию для движка реплицируемых таблиц в файле конфигурации сервера. + +```xml +/clickhouse/tables/{shard}/{database}/{table} +{replica} +``` + +В этом случае можно опустить аргументы при создании таблиц: + +``` sql +CREATE TABLE table_name ( + x UInt32 +) ENGINE = ReplicatedMergeTree +ORDER BY x; +``` + +Это будет эквивалентно следующему запросу: + +``` sql +CREATE TABLE table_name ( + x UInt32 +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/{database}/table_name', '{replica}') +ORDER BY x; +``` + Выполните запрос `CREATE TABLE` на каждой реплике. Запрос создаёт новую реплицируемую таблицу, или добавляет новую реплику к имеющимся. Если вы добавляете новую реплику после того, как таблица на других репликах уже содержит некоторые данные, то после выполнения запроса, данные на новую реплику будут скачаны с других реплик. То есть, новая реплика синхронизирует себя с остальными. From 4adef402e6ae82823518712f4901d93093df027a Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Sun, 1 Nov 2020 12:14:25 -0400 Subject: [PATCH 322/325] Doc/multiple_leaders --- docs/en/development/architecture.md | 2 +- docs/en/operations/system-tables/replicas.md | 4 ++-- docs/ru/development/architecture.md | 2 +- docs/ru/operations/system-tables/replicas.md | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/development/architecture.md b/docs/en/development/architecture.md index bea75140606..b4f53207a41 100644 --- a/docs/en/development/architecture.md +++ b/docs/en/development/architecture.md @@ -189,7 +189,7 @@ Replication is implemented in the `ReplicatedMergeTree` storage engine. The path Replication uses an asynchronous multi-master scheme. You can insert data into any replica that has a session with `ZooKeeper`, and data is replicated to all other replicas asynchronously. Because ClickHouse doesn’t support UPDATEs, replication is conflict-free. As there is no quorum acknowledgment of inserts, just-inserted data might be lost if one node fails. -Metadata for replication is stored in ZooKeeper. There is a replication log that lists what actions to do. Actions are: get part; merge parts; drop a partition, and so on. Each replica copies the replication log to its queue and then executes the actions from the queue. For example, on insertion, the “get the part” action is created in the log, and every replica downloads that part. Merges are coordinated between replicas to get byte-identical results. All parts are merged in the same way on all replicas. It is achieved by electing one replica as the leader, and that replica initiates merges and writes “merge parts” actions to the log. +Metadata for replication is stored in ZooKeeper. There is a replication log that lists what actions to do. Actions are: get part; merge parts; drop a partition, and so on. Each replica copies the replication log to its queue and then executes the actions from the queue. For example, on insertion, the “get the part” action is created in the log, and every replica downloads that part. Merges are coordinated between replicas to get byte-identical results. All parts are merged in the same way on all replicas. One of the leaders initiates a new merge first and writes “merge parts” actions to the log. Multiple replicas can be leaders at the same time. A replica can be prevented from becoming a leader using the `merge_tree` setting `replicated_can_become_leader`. The leaders are responsible for scheduling background merges. Replication is physical: only compressed parts are transferred between nodes, not queries. Merges are processed on each replica independently in most cases to lower the network costs by avoiding network amplification. Large merged parts are sent over the network only in cases of significant replication lag. diff --git a/docs/en/operations/system-tables/replicas.md b/docs/en/operations/system-tables/replicas.md index f31f7fd27ea..8da68d2d2ab 100644 --- a/docs/en/operations/system-tables/replicas.md +++ b/docs/en/operations/system-tables/replicas.md @@ -53,9 +53,9 @@ Columns: - `table` (`String`) - Table name - `engine` (`String`) - Table engine name - `is_leader` (`UInt8`) - Whether the replica is the leader. - Only one replica at a time can be the leader. The leader is responsible for selecting background merges to perform. + Multiple replicas can be leaders at the same time. A replica can be prevented from becoming a leader using the `merge_tree` setting `replicated_can_become_leader`. The leaders are responsible for scheduling background merges. Note that writes can be performed to any replica that is available and has a session in ZK, regardless of whether it is a leader. -- `can_become_leader` (`UInt8`) - Whether the replica can be elected as a leader. +- `can_become_leader` (`UInt8`) - Whether the replica can be a leader. - `is_readonly` (`UInt8`) - Whether the replica is in read-only mode. This mode is turned on if the config doesn’t have sections with ZooKeeper, if an unknown error occurred when reinitializing sessions in ZooKeeper, and during session reinitialization in ZooKeeper. - `is_session_expired` (`UInt8`) - the session with ZooKeeper has expired. Basically the same as `is_readonly`. diff --git a/docs/ru/development/architecture.md b/docs/ru/development/architecture.md index 0d8ddb6f795..53c007e078f 100644 --- a/docs/ru/development/architecture.md +++ b/docs/ru/development/architecture.md @@ -190,7 +190,7 @@ ClickHouse имеет сильную типизацию, поэтому нет Репликация использует асинхронную multi-master схему. Вы можете вставить данные в любую реплику, которая имеет открытую сессию в `ZooKeeper`, и данные реплицируются на все другие реплики асинхронно. Поскольку ClickHouse не поддерживает UPDATE, репликация исключает конфликты (conflict-free replication). Поскольку подтверждение вставок кворумом не реализовано, только что вставленные данные могут быть потеряны в случае сбоя одного узла. -Метаданные для репликации хранятся в `ZooKeeper`. Существует журнал репликации, в котором перечислены действия, которые необходимо выполнить. Среди этих действий: получить часть (get the part); объединить части (merge parts); удалить партицию (drop a partition) и так далее. Каждая реплика копирует журнал репликации в свою очередь, а затем выполняет действия из очереди. Например, при вставке в журнале создается действие «получить часть» (get the part), и каждая реплика загружает эту часть. Слияния координируются между репликами, чтобы получить идентичные до байта результаты. Все части объединяются одинаково на всех репликах. Это достигается путем выбора одной реплики в качестве лидера, и эта реплика инициирует слияния и записывает действия «слияния частей» в журнал. +Метаданные для репликации хранятся в `ZooKeeper`. Существует журнал репликации, в котором перечислены действия, которые необходимо выполнить. Среди этих действий: получить часть (get the part); объединить части (merge parts); удалить партицию (drop a partition) и так далее. Каждая реплика копирует журнал репликации в свою очередь, а затем выполняет действия из очереди. Например, при вставке в журнале создается действие «получить часть» (get the part), и каждая реплика загружает эту часть. Слияния координируются между репликами, чтобы получить идентичные до байта результаты. Все части объединяются одинаково на всех репликах. Одна из реплик-лидеров инициирует новое слияние кусков первой и записывает действия «слияния частей» в журнал. Несколько реплик (или все) могут быть лидерами одновременно. Реплике можно запретить быть лидером с помощью `merge_tree` настройки `replicated_can_become_leader`. Репликация является физической: между узлами передаются только сжатые части, а не запросы. Слияния обрабатываются на каждой реплике независимо, в большинстве случаев, чтобы снизить затраты на сеть, во избежание усиления роли сети. Крупные объединенные части отправляются по сети только в случае значительной задержки репликации. diff --git a/docs/ru/operations/system-tables/replicas.md b/docs/ru/operations/system-tables/replicas.md index bb2e6a550a0..8d4eb60c56a 100644 --- a/docs/ru/operations/system-tables/replicas.md +++ b/docs/ru/operations/system-tables/replicas.md @@ -53,9 +53,9 @@ active_replicas: 2 - `table` (`String`) - имя таблицы. - `engine` (`String`) - имя движка таблицы. - `is_leader` (`UInt8`) - является ли реплика лидером. - В один момент времени, не более одной из реплик является лидером. Лидер отвечает за выбор фоновых слияний, которые следует произвести. + Несколько реплик могут быть лидерами одновременно. Реплике можно запретить быть лидером с помощью `merge_tree` настройки `replicated_can_become_leader`. Лидеры назначают фоновые слияния, которые следует произвести. Замечу, что запись можно осуществлять на любую реплику (доступную и имеющую сессию в ZK), независимо от лидерства. -- `can_become_leader` (`UInt8`) - может ли реплика быть выбрана лидером. +- `can_become_leader` (`UInt8`) - может ли реплика быть лидером. - `is_readonly` (`UInt8`) - находится ли реплика в режиме «только для чтения» Этот режим включается, если в конфиге нет секции с ZK; если при переинициализации сессии в ZK произошла неизвестная ошибка; во время переинициализации сессии с ZK. - `is_session_expired` (`UInt8`) - истекла ли сессия с ZK. В основном, то же самое, что и `is_readonly`. From de3f88c3fd77150eea64c0a81f018a201137972e Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Sun, 1 Nov 2020 12:17:17 -0400 Subject: [PATCH 323/325] Update architecture.md --- docs/en/development/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/architecture.md b/docs/en/development/architecture.md index b4f53207a41..19caa5241b0 100644 --- a/docs/en/development/architecture.md +++ b/docs/en/development/architecture.md @@ -189,7 +189,7 @@ Replication is implemented in the `ReplicatedMergeTree` storage engine. The path Replication uses an asynchronous multi-master scheme. You can insert data into any replica that has a session with `ZooKeeper`, and data is replicated to all other replicas asynchronously. Because ClickHouse doesn’t support UPDATEs, replication is conflict-free. As there is no quorum acknowledgment of inserts, just-inserted data might be lost if one node fails. -Metadata for replication is stored in ZooKeeper. There is a replication log that lists what actions to do. Actions are: get part; merge parts; drop a partition, and so on. Each replica copies the replication log to its queue and then executes the actions from the queue. For example, on insertion, the “get the part” action is created in the log, and every replica downloads that part. Merges are coordinated between replicas to get byte-identical results. All parts are merged in the same way on all replicas. One of the leaders initiates a new merge first and writes “merge parts” actions to the log. Multiple replicas can be leaders at the same time. A replica can be prevented from becoming a leader using the `merge_tree` setting `replicated_can_become_leader`. The leaders are responsible for scheduling background merges. +Metadata for replication is stored in ZooKeeper. There is a replication log that lists what actions to do. Actions are: get part; merge parts; drop a partition, and so on. Each replica copies the replication log to its queue and then executes the actions from the queue. For example, on insertion, the “get the part” action is created in the log, and every replica downloads that part. Merges are coordinated between replicas to get byte-identical results. All parts are merged in the same way on all replicas. One of the leaders initiates a new merge first and writes “merge parts” actions to the log. Multiple replicas (or all) can be leaders at the same time. A replica can be prevented from becoming a leader using the `merge_tree` setting `replicated_can_become_leader`. The leaders are responsible for scheduling background merges. Replication is physical: only compressed parts are transferred between nodes, not queries. Merges are processed on each replica independently in most cases to lower the network costs by avoiding network amplification. Large merged parts are sent over the network only in cases of significant replication lag. From af76657855d3e5cb85f038d98452ab885a0ebcfd Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 06:10:24 +0000 Subject: [PATCH 324/325] Bump tornado from 5.1.1 to 6.1 in /docs/tools Bumps [tornado](https://github.com/tornadoweb/tornado) from 5.1.1 to 6.1. - [Release notes](https://github.com/tornadoweb/tornado/releases) - [Changelog](https://github.com/tornadoweb/tornado/blob/master/docs/releases.rst) - [Commits](https://github.com/tornadoweb/tornado/compare/v5.1.1...v6.1.0) Signed-off-by: dependabot-preview[bot] --- docs/tools/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index c4559696b16..3c59a601ff2 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -33,6 +33,6 @@ singledispatch==3.4.0.3 six==1.15.0 soupsieve==2.0.1 termcolor==1.1.0 -tornado==5.1.1 +tornado==6.1 Unidecode==1.1.1 urllib3==1.25.10 From 8a68ee2547173cd90827659d2bdc300fa3075eb3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 2 Nov 2020 13:45:37 +0300 Subject: [PATCH 325/325] Whitespaces --- src/IO/ReadWriteBufferFromHTTP.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 6ff99690bb4..ee6fcc58ab0 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -187,7 +187,7 @@ namespace detail session->updateSession(uri_redirect); - istr = call(uri_redirect,response); + istr = call(uri_redirect, response); } try @@ -241,7 +241,8 @@ class UpdatableSession : public UpdatableSessionBase using Parent = UpdatableSessionBase; public: - explicit UpdatableSession(const Poco::URI uri, + explicit UpdatableSession( + const Poco::URI uri, const ConnectionTimeouts & timeouts_, const UInt64 max_redirects_) : Parent(uri, timeouts_, max_redirects_) @@ -260,7 +261,8 @@ class ReadWriteBufferFromHTTP : public detail::ReadWriteBufferFromHTTPBase>; public: - explicit ReadWriteBufferFromHTTP(Poco::URI uri_, + explicit ReadWriteBufferFromHTTP( + Poco::URI uri_, const std::string & method_, OutStreamCallback out_stream_callback_, const ConnectionTimeouts & timeouts, @@ -269,7 +271,8 @@ public: size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, const HTTPHeaderEntries & http_header_entries_ = {}, const RemoteHostFilter & remote_host_filter_ = {}) - : Parent(std::make_shared(uri_, timeouts, max_redirects), uri_, method_, out_stream_callback_, credentials_, buffer_size_, http_header_entries_, remote_host_filter_) + : Parent(std::make_shared(uri_, timeouts, max_redirects), + uri_, method_, out_stream_callback_, credentials_, buffer_size_, http_header_entries_, remote_host_filter_) { } };