From 665fabde9e60eea5f6364200cdad56d00ed39298 Mon Sep 17 00:00:00 2001 From: antikvist Date: Sat, 18 Apr 2020 19:20:34 +0300 Subject: [PATCH 001/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- .../AggregateFunctionWelchTTest.cpp | 28 ++ .../AggregateFunctionWelchTTest.h | 251 ++++++++++++++++++ .../registerAggregateFunctions.cpp | 1 + .../registerAggregateFunctions.h | 1 + 4 files changed, 281 insertions(+) create mode 100644 src/AggregateFunctions/AggregateFunctionWelchTTest.cpp create mode 100644 src/AggregateFunctions/AggregateFunctionWelchTTest.h diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp new file mode 100644 index 00000000000..46b533a2c0b --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -0,0 +1,28 @@ +#include +#include +#include + + +namespace DB +{ + +namespace +{ + +AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, const DataTypes & argument_types, const Array & parameters) +{ + + + return std::make_shared(argument_types, parameters); + +} + +} + +void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) +{ + + factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest, AggregateFunctionFactory::CaseInsensitive); +} + +} diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h new file mode 100644 index 00000000000..a52528df431 --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -0,0 +1,251 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +// hard-codded values - part of the algorithm + +#define SIGN_LVL_CNT 6 + +Float64 CriticalValuesTable[SIGN_LVL_CNT][102] = { + // for significance level = 0.2 + {0.2, 3.078, 1.886, 1.638, 1.533, 1.476, 1.44, 1.415, 1.397, 1.383, 1.372, 1.363, 1.356, 1.35, 1.345, 1.341, 1.337, 1.333, 1.33, 1.328, 1.325, 1.323, 1.321, 1.319, 1.318, 1.316, 1.315, 1.314, 1.313, 1.311, 1.31, 1.309, 1.309, 1.308, 1.307, 1.306, 1.306, 1.305, 1.304, 1.304, 1.303, 1.303, 1.302, 1.302, 1.301, 1.301, 1.3, 1.3, 1.299, 1.299, 1.299, 1.298, 1.298, 1.298, 1.297, 1.297, 1.297, 1.297, 1.296, 1.296, 1.296, 1.296, 1.295, 1.295, 1.295, 1.295, 1.295, 1.294, 1.294, 1.294, 1.294, 1.294, 1.293, 1.293, 1.293, 1.293, 1.293, 1.293, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.29, 1.29, 1.29, 1.29, 1.29, 1.282} + + // for significance level = 0.1 + {0.1, 6.314, 2.92, 2.353, 2.132, 2.015, 1.943, 1.895, 1.86, 1.833, 1.812, 1.796, 1.782, 1.771, 1.761, 1.753, 1.746, 1.74, 1.734, 1.729, 1.725, 1.721, 1.717, 1.714, 1.711, 1.708, 1.706, 1.703, 1.701, 1.699, 1.697, 1.696, 1.694, 1.692, 1.691, 1.69, 1.688, 1.687, 1.686, 1.685, 1.684, 1.683, 1.682, 1.681, 1.68, 1.679, 1.679, 1.678, 1.677, 1.677, 1.676, 1.675, 1.675, 1.674, 1.674, 1.673, 1.673, 1.672, 1.672, 1.671, 1.671, 1.67, 1.67, 1.669, 1.669, 1.669, 1.668, 1.668, 1.668, 1.667, 1.667, 1.667, 1.666, 1.666, 1.666, 1.665, 1.665, 1.665, 1.665, 1.664, 1.664, 1.664, 1.664, 1.663, 1.663, 1.663, 1.663, 1.663, 1.662, 1.662, 1.662, 1.662, 1.662, 1.661, 1.661, 1.661, 1.661, 1.661, 1.661, 1.66, 1.66, 1.645} + + // for significance level = 0.05 + {0.05, 12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145, 2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, 2.045, 2.042, 2.04, 2.037, 2.035, 2.032, 2.03, 2.028, 2.026, 2.024, 2.023, 2.021, 2.02, 2.018, 2.017, 2.015, 2.014, 2.013, 2.012, 2.011, 2.01, 2.009, 2.008, 2.007, 2.006, 2.005, 2.004, 2.003, 2.002, 2.002, 2.001, 2.0, 2.0, 1.999, 1.998, 1.998, 1.997, 1.997, 1.996, 1.995, 1.995, 1.994, 1.994, 1.993, 1.993, 1.993, 1.992, 1.992, 1.991, 1.991, 1.99, 1.99, 1.99, 1.989, 1.989, 1.989, 1.988, 1.988, 1.988, 1.987, 1.987, 1.987, 1.986, 1.986, 1.986, 1.986, 1.985, 1.985, 1.985, 1.984, 1.984, 1.984, 1.96} + + // for significance level = 0.02 + {0.02, 31.821, 6.965, 4.541, 3.747, 3.365, 3.143, 2.998, 2.896, 2.821, 2.764, 2.718, 2.681, 2.65, 2.624, 2.602, 2.583, 2.567, 2.552, 2.539, 2.528, 2.518, 2.508, 2.5, 2.492, 2.485, 2.479, 2.473, 2.467, 2.462, 2.457, 2.453, 2.449, 2.445, 2.441, 2.438, 2.434, 2.431, 2.429, 2.426, 2.423, 2.421, 2.418, 2.416, 2.414, 2.412, 2.41, 2.408, 2.407, 2.405, 2.403, 2.402, 2.4, 2.399, 2.397, 2.396, 2.395, 2.394, 2.392, 2.391, 2.39, 2.389, 2.388, 2.387, 2.386, 2.385, 2.384, 2.383, 2.382, 2.382, 2.381, 2.38, 2.379, 2.379, 2.378, 2.377, 2.376, 2.376, 2.375, 2.374, 2.374, 2.373, 2.373, 2.372, 2.372, 2.371, 2.37, 2.37, 2.369, 2.369, 2.368, 2.368, 2.368, 2.367, 2.367, 2.366, 2.366, 2.365, 2.365, 2.365, 2.364, 2.326} + + // for significance level = 0.01 + {0.01, 63.657, 9.925, 5.841, 4.604, 4.032, 3.707, 3.499, 3.355, 3.25, 3.169, 3.106, 3.055, 3.012, 2.977, 2.947, 2.921, 2.898, 2.878, 2.861, 2.845, 2.831, 2.819, 2.807, 2.797, 2.787, 2.779, 2.771, 2.763, 2.756, 2.75, 2.744, 2.738, 2.733, 2.728, 2.724, 2.719, 2.715, 2.712, 2.708, 2.704, 2.701, 2.698, 2.695, 2.692, 2.69, 2.687, 2.685, 2.682, 2.68, 2.678, 2.676, 2.674, 2.672, 2.67, 2.668, 2.667, 2.665, 2.663, 2.662, 2.66, 2.659, 2.657, 2.656, 2.655, 2.654, 2.652, 2.651, 2.65, 2.649, 2.648, 2.647, 2.646, 2.645, 2.644, 2.643, 2.642, 2.641, 2.64, 2.64, 2.639, 2.638, 2.637, 2.636, 2.636, 2.635, 2.634, 2.634, 2.633, 2.632, 2.632, 2.631, 2.63, 2.63, 2.629, 2.629, 2.628, 2.627, 2.627, 2.626, 2.626, 2.576} + + // for significance level = 0.002 + {0.002, 318.313, 22.327, 10.215, 7.173, 5.893, 5.208, 4.782, 4.499, 4.296, 4.143, 4.024, 3.929, 3.852, 3.787, 3.733, 3.686, 3.646, 3.61, 3.579, 3.552, 3.527, 3.505, 3.485, 3.467, 3.45, 3.435, 3.421, 3.408, 3.396, 3.385, 3.375, 3.365, 3.356, 3.348, 3.34, 3.333, 3.326, 3.319, 3.313, 3.307, 3.301, 3.296, 3.291, 3.286, 3.281, 3.277, 3.273, 3.269, 3.265, 3.261, 3.258, 3.255, 3.251, 3.248, 3.245, 3.242, 3.239, 3.237, 3.234, 3.232, 3.229, 3.227, 3.225, 3.223, 3.22, 3.218, 3.216, 3.214, 3.213, 3.211, 3.209, 3.207, 3.206, 3.204, 3.202, 3.201, 3.199, 3.198, 3.197, 3.195, 3.194, 3.193, 3.191, 3.19, 3.189, 3.188, 3.187, 3.185, 3.184, 3.183, 3.182, 3.181, 3.18, 3.179, 3.178, 3.177, 3.176, 3.175, 3.175, 3.174, 3.09} +} + +// our algorithm implementation via vectors: +// https://gist.github.com/ltybc-coder/792748cfdb2f7cadef424ffb7b011c71 +// col, col, bool +template +struct AggregateFunctionWelchTTestData final { + + size_t size_x = 0; + size_t size_y = 0; + X sum_x = 0; + Y sum_y = 0; + X square_sum_x = 0; + Y square_sum_y = 0; + Float64 mean_x = 0; + Float64 mean_y = 0; + + /* + not yet sure how to use them + void add_x(X x) { + mean_x = (Float64)(sum_x + x) / (size_x + 1); + size_x ++; + sum_x += x; + square_sum_x += x * x; + } + + void add_y(Y y) { + mean_y = (sum_y + y) / (size_y + 1); + size_y ++; + sum_y += y; + square_sum_y += y * y; + } + */ + + void add(X x, Y y) { + sum_x += x; + sum_y += y; + size_x++; + size_y++; + mean_x = (Float64) sum_x / size_x; + mean_y = (Float64) sum_y / size_y; + square_sum_x += x * x; + square_sum_y += y * y; + } + + void merge(const AggregateFunctionWelchTTestData &other) { + sum_x += other.sum_x; + sum_y += other.sum_y; + size_x += other.size_x; + size_y += other.size_y; + mean_x = (Float64) sum_x / size_x; + mean_y = (Float64) sum_y / size_y; + square_sum_x += other.square_sum_x; + square_sum_y += other.square_sum_y; + } + + void serialize(WriteBuffer &buf) const { + writeBinary(mean_x, buf); + writeBinary(mean_y, buf); + writeBinary(sum_x, buf); + writeBinary(sum_y, buf); + writeBinary(square_sum_x, buf); + writeBinary(square_sum_y, buf); + writeBinary(size_x, buf); + writeBinary(size_y, buf); + } + + void deserialize(ReadBuffer &buf) { + readBinary(mean_x, buf); + readBinary(mean_y, buf); + readBinary(sum_x, buf); + readBinary(sum_y, buf); + readBinary(square_sum_x, buf); + readBinary(square_sum_y, buf); + readBinary(size_x, buf); + readBinary(size_y, buf); + } + + Float64 get_sx() const { + return (Float64)(square_sum_x + size_x * mean_x * mean_x - 2 * mean_x * sum_x) / (size_x - 1); + } + + Float64 get_sy() const { + return (Float64)(square_sum_y + size_y * mean_y * mean_y - 2 * mean_y * sum_y) / (size_y - 1); + } + + Float64 get_T(Float64 sx, Float64 sy) const { + return (Float64)(mean_x - mean_y) / std::sqrt(sx / size_x + sy / size_y); + } + + Float64 get_degrees_of_freed(Float64 sx, Float64 sy) const { + return (Float64)(sx / size_x + sy / size_y) * (sx / size_x + sy / size_y) / + ((sx * sx / (size_x * size_x * (size_x - 1))) + (sy * sy / (size_y * size_y * (size_y - 1)))); + } + + Ret get_result(Float64 t, Float64 dof, Float64 parametr) const + { + //find our table + int table; + for (int i = 0; i < SIGN_LVL_CNT; ++i) + { + if (CriticalValuesTable[i][0] == parametr) + { + table = i; + } + } + + //round or make infinity dof + dof = (int) dof; + if (dof > 100) + { + dof = 101; + } + //check if abs of t is greater than table[dof] + t = abs(t) + if(t > CriticalValuesTable[table][dof]) { + return (UInt8) 1; + //in this case we reject the null hypothesis + } + else { + return (UInt8) 0; + } + } +}; + +template +class AggregateFunctionWelchTTest final : public + IAggregateFunctionDataHelper< + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest + > +{ +public: + AggregateFunctionWelchTTest( + const DataTypes & arguments, + const Array & params + ): + IAggregateFunctionDataHelper< + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest + > {arguments, params} + { + // notice: arguments has been in factory + } + + String getName() const override + { + return "WelchTTest"; + } + + void add( + AggregateDataPtr place, + const IColumn ** columns, + size_t row_num, + Arena * + ) const override + { + auto col_x = assert_cast *>(columns[0]); + auto col_y = assert_cast *>(columns[1]); + + X x = col_x->getData()[row_num]; + Y y = col_y->getData()[row_num]; + + this->data(place).add(x, y); + } + + void merge( + AggregateDataPtr place, + ConstAggregateDataPtr rhs, Arena * + ) const override + { + this->data(place).merge(this->data(rhs)); + } + + void serialize( + ConstAggregateDataPtr place, + WriteBuffer & buf + ) const override + { + this->data(place).serialize(buf); + } + + void deserialize( + AggregateDataPtr place, + ReadBuffer & buf, Arena * + ) const override + { + this->data(place).deserialize(buf); + } + + void insertResultInto( + ConstAggregateDataPtr place, + IColumn & to + ) const override + { + Float64 significance_level = applyVisitor(FieldVisitorConvertToNumber(), params[0]); + + Float64 sx = this->data(place).get_sx(); + Float64 sy = this->data(place).get_sy(); + Float64 t_value = this->data(place).get_T(sx, sy); + Float64 dof = this->data(place).get_degrees_of_freed(sx, sy); + Ret result = this->data(place).get_result(t_value, dof, significance_level); + + auto & column = static_cast(to); + column.getData().push_back(result); + } + + +} +}; + +}; \ No newline at end of file diff --git a/src/AggregateFunctions/registerAggregateFunctions.cpp b/src/AggregateFunctions/registerAggregateFunctions.cpp index a9ab1d4f8ea..adc72ec9169 100644 --- a/src/AggregateFunctions/registerAggregateFunctions.cpp +++ b/src/AggregateFunctions/registerAggregateFunctions.cpp @@ -45,6 +45,7 @@ void registerAggregateFunctions() registerAggregateFunctionMoving(factory); registerAggregateFunctionCategoricalIV(factory); registerAggregateFunctionAggThrow(factory); + registerAggregateFunctionWelchTTest(factory); } { diff --git a/src/AggregateFunctions/registerAggregateFunctions.h b/src/AggregateFunctions/registerAggregateFunctions.h index 88cdf4a504d..046b125dec5 100644 --- a/src/AggregateFunctions/registerAggregateFunctions.h +++ b/src/AggregateFunctions/registerAggregateFunctions.h @@ -35,6 +35,7 @@ void registerAggregateFunctionSimpleLinearRegression(AggregateFunctionFactory &) void registerAggregateFunctionMoving(AggregateFunctionFactory &); void registerAggregateFunctionCategoricalIV(AggregateFunctionFactory &); void registerAggregateFunctionAggThrow(AggregateFunctionFactory &); +void registerAggregateFunctionWelchTTest(AggregateFunctionFactory &); class AggregateFunctionCombinatorFactory; void registerAggregateFunctionCombinatorIf(AggregateFunctionCombinatorFactory &); From 69c1f33b0b22bd103cbce8f0ce747dc12937d1a1 Mon Sep 17 00:00:00 2001 From: antikvist Date: Mon, 27 Apr 2020 00:09:56 +0300 Subject: [PATCH 002/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- .../AggregateFunctionWelchTTest.h | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index a52528df431..be1e176d540 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -4,13 +4,17 @@ #include #include #include +#include #include #include #include #include #include #include -#include + +#include +#include +#include namespace DB @@ -21,19 +25,19 @@ namespace DB Float64 CriticalValuesTable[SIGN_LVL_CNT][102] = { // for significance level = 0.2 - {0.2, 3.078, 1.886, 1.638, 1.533, 1.476, 1.44, 1.415, 1.397, 1.383, 1.372, 1.363, 1.356, 1.35, 1.345, 1.341, 1.337, 1.333, 1.33, 1.328, 1.325, 1.323, 1.321, 1.319, 1.318, 1.316, 1.315, 1.314, 1.313, 1.311, 1.31, 1.309, 1.309, 1.308, 1.307, 1.306, 1.306, 1.305, 1.304, 1.304, 1.303, 1.303, 1.302, 1.302, 1.301, 1.301, 1.3, 1.3, 1.299, 1.299, 1.299, 1.298, 1.298, 1.298, 1.297, 1.297, 1.297, 1.297, 1.296, 1.296, 1.296, 1.296, 1.295, 1.295, 1.295, 1.295, 1.295, 1.294, 1.294, 1.294, 1.294, 1.294, 1.293, 1.293, 1.293, 1.293, 1.293, 1.293, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.29, 1.29, 1.29, 1.29, 1.29, 1.282} + {0.2, 3.078, 1.886, 1.638, 1.533, 1.476, 1.44, 1.415, 1.397, 1.383, 1.372, 1.363, 1.356, 1.35, 1.345, 1.341, 1.337, 1.333, 1.33, 1.328, 1.325, 1.323, 1.321, 1.319, 1.318, 1.316, 1.315, 1.314, 1.313, 1.311, 1.31, 1.309, 1.309, 1.308, 1.307, 1.306, 1.306, 1.305, 1.304, 1.304, 1.303, 1.303, 1.302, 1.302, 1.301, 1.301, 1.3, 1.3, 1.299, 1.299, 1.299, 1.298, 1.298, 1.298, 1.297, 1.297, 1.297, 1.297, 1.296, 1.296, 1.296, 1.296, 1.295, 1.295, 1.295, 1.295, 1.295, 1.294, 1.294, 1.294, 1.294, 1.294, 1.293, 1.293, 1.293, 1.293, 1.293, 1.293, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.292, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.291, 1.29, 1.29, 1.29, 1.29, 1.29, 1.282}, // for significance level = 0.1 - {0.1, 6.314, 2.92, 2.353, 2.132, 2.015, 1.943, 1.895, 1.86, 1.833, 1.812, 1.796, 1.782, 1.771, 1.761, 1.753, 1.746, 1.74, 1.734, 1.729, 1.725, 1.721, 1.717, 1.714, 1.711, 1.708, 1.706, 1.703, 1.701, 1.699, 1.697, 1.696, 1.694, 1.692, 1.691, 1.69, 1.688, 1.687, 1.686, 1.685, 1.684, 1.683, 1.682, 1.681, 1.68, 1.679, 1.679, 1.678, 1.677, 1.677, 1.676, 1.675, 1.675, 1.674, 1.674, 1.673, 1.673, 1.672, 1.672, 1.671, 1.671, 1.67, 1.67, 1.669, 1.669, 1.669, 1.668, 1.668, 1.668, 1.667, 1.667, 1.667, 1.666, 1.666, 1.666, 1.665, 1.665, 1.665, 1.665, 1.664, 1.664, 1.664, 1.664, 1.663, 1.663, 1.663, 1.663, 1.663, 1.662, 1.662, 1.662, 1.662, 1.662, 1.661, 1.661, 1.661, 1.661, 1.661, 1.661, 1.66, 1.66, 1.645} + {0.1, 6.314, 2.92, 2.353, 2.132, 2.015, 1.943, 1.895, 1.86, 1.833, 1.812, 1.796, 1.782, 1.771, 1.761, 1.753, 1.746, 1.74, 1.734, 1.729, 1.725, 1.721, 1.717, 1.714, 1.711, 1.708, 1.706, 1.703, 1.701, 1.699, 1.697, 1.696, 1.694, 1.692, 1.691, 1.69, 1.688, 1.687, 1.686, 1.685, 1.684, 1.683, 1.682, 1.681, 1.68, 1.679, 1.679, 1.678, 1.677, 1.677, 1.676, 1.675, 1.675, 1.674, 1.674, 1.673, 1.673, 1.672, 1.672, 1.671, 1.671, 1.67, 1.67, 1.669, 1.669, 1.669, 1.668, 1.668, 1.668, 1.667, 1.667, 1.667, 1.666, 1.666, 1.666, 1.665, 1.665, 1.665, 1.665, 1.664, 1.664, 1.664, 1.664, 1.663, 1.663, 1.663, 1.663, 1.663, 1.662, 1.662, 1.662, 1.662, 1.662, 1.661, 1.661, 1.661, 1.661, 1.661, 1.661, 1.66, 1.66, 1.645}, // for significance level = 0.05 - {0.05, 12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145, 2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, 2.045, 2.042, 2.04, 2.037, 2.035, 2.032, 2.03, 2.028, 2.026, 2.024, 2.023, 2.021, 2.02, 2.018, 2.017, 2.015, 2.014, 2.013, 2.012, 2.011, 2.01, 2.009, 2.008, 2.007, 2.006, 2.005, 2.004, 2.003, 2.002, 2.002, 2.001, 2.0, 2.0, 1.999, 1.998, 1.998, 1.997, 1.997, 1.996, 1.995, 1.995, 1.994, 1.994, 1.993, 1.993, 1.993, 1.992, 1.992, 1.991, 1.991, 1.99, 1.99, 1.99, 1.989, 1.989, 1.989, 1.988, 1.988, 1.988, 1.987, 1.987, 1.987, 1.986, 1.986, 1.986, 1.986, 1.985, 1.985, 1.985, 1.984, 1.984, 1.984, 1.96} + {0.05, 12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145, 2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, 2.045, 2.042, 2.04, 2.037, 2.035, 2.032, 2.03, 2.028, 2.026, 2.024, 2.023, 2.021, 2.02, 2.018, 2.017, 2.015, 2.014, 2.013, 2.012, 2.011, 2.01, 2.009, 2.008, 2.007, 2.006, 2.005, 2.004, 2.003, 2.002, 2.002, 2.001, 2.0, 2.0, 1.999, 1.998, 1.998, 1.997, 1.997, 1.996, 1.995, 1.995, 1.994, 1.994, 1.993, 1.993, 1.993, 1.992, 1.992, 1.991, 1.991, 1.99, 1.99, 1.99, 1.989, 1.989, 1.989, 1.988, 1.988, 1.988, 1.987, 1.987, 1.987, 1.986, 1.986, 1.986, 1.986, 1.985, 1.985, 1.985, 1.984, 1.984, 1.984, 1.96}, // for significance level = 0.02 - {0.02, 31.821, 6.965, 4.541, 3.747, 3.365, 3.143, 2.998, 2.896, 2.821, 2.764, 2.718, 2.681, 2.65, 2.624, 2.602, 2.583, 2.567, 2.552, 2.539, 2.528, 2.518, 2.508, 2.5, 2.492, 2.485, 2.479, 2.473, 2.467, 2.462, 2.457, 2.453, 2.449, 2.445, 2.441, 2.438, 2.434, 2.431, 2.429, 2.426, 2.423, 2.421, 2.418, 2.416, 2.414, 2.412, 2.41, 2.408, 2.407, 2.405, 2.403, 2.402, 2.4, 2.399, 2.397, 2.396, 2.395, 2.394, 2.392, 2.391, 2.39, 2.389, 2.388, 2.387, 2.386, 2.385, 2.384, 2.383, 2.382, 2.382, 2.381, 2.38, 2.379, 2.379, 2.378, 2.377, 2.376, 2.376, 2.375, 2.374, 2.374, 2.373, 2.373, 2.372, 2.372, 2.371, 2.37, 2.37, 2.369, 2.369, 2.368, 2.368, 2.368, 2.367, 2.367, 2.366, 2.366, 2.365, 2.365, 2.365, 2.364, 2.326} + {0.02, 31.821, 6.965, 4.541, 3.747, 3.365, 3.143, 2.998, 2.896, 2.821, 2.764, 2.718, 2.681, 2.65, 2.624, 2.602, 2.583, 2.567, 2.552, 2.539, 2.528, 2.518, 2.508, 2.5, 2.492, 2.485, 2.479, 2.473, 2.467, 2.462, 2.457, 2.453, 2.449, 2.445, 2.441, 2.438, 2.434, 2.431, 2.429, 2.426, 2.423, 2.421, 2.418, 2.416, 2.414, 2.412, 2.41, 2.408, 2.407, 2.405, 2.403, 2.402, 2.4, 2.399, 2.397, 2.396, 2.395, 2.394, 2.392, 2.391, 2.39, 2.389, 2.388, 2.387, 2.386, 2.385, 2.384, 2.383, 2.382, 2.382, 2.381, 2.38, 2.379, 2.379, 2.378, 2.377, 2.376, 2.376, 2.375, 2.374, 2.374, 2.373, 2.373, 2.372, 2.372, 2.371, 2.37, 2.37, 2.369, 2.369, 2.368, 2.368, 2.368, 2.367, 2.367, 2.366, 2.366, 2.365, 2.365, 2.365, 2.364, 2.326}, // for significance level = 0.01 - {0.01, 63.657, 9.925, 5.841, 4.604, 4.032, 3.707, 3.499, 3.355, 3.25, 3.169, 3.106, 3.055, 3.012, 2.977, 2.947, 2.921, 2.898, 2.878, 2.861, 2.845, 2.831, 2.819, 2.807, 2.797, 2.787, 2.779, 2.771, 2.763, 2.756, 2.75, 2.744, 2.738, 2.733, 2.728, 2.724, 2.719, 2.715, 2.712, 2.708, 2.704, 2.701, 2.698, 2.695, 2.692, 2.69, 2.687, 2.685, 2.682, 2.68, 2.678, 2.676, 2.674, 2.672, 2.67, 2.668, 2.667, 2.665, 2.663, 2.662, 2.66, 2.659, 2.657, 2.656, 2.655, 2.654, 2.652, 2.651, 2.65, 2.649, 2.648, 2.647, 2.646, 2.645, 2.644, 2.643, 2.642, 2.641, 2.64, 2.64, 2.639, 2.638, 2.637, 2.636, 2.636, 2.635, 2.634, 2.634, 2.633, 2.632, 2.632, 2.631, 2.63, 2.63, 2.629, 2.629, 2.628, 2.627, 2.627, 2.626, 2.626, 2.576} + {0.01, 63.657, 9.925, 5.841, 4.604, 4.032, 3.707, 3.499, 3.355, 3.25, 3.169, 3.106, 3.055, 3.012, 2.977, 2.947, 2.921, 2.898, 2.878, 2.861, 2.845, 2.831, 2.819, 2.807, 2.797, 2.787, 2.779, 2.771, 2.763, 2.756, 2.75, 2.744, 2.738, 2.733, 2.728, 2.724, 2.719, 2.715, 2.712, 2.708, 2.704, 2.701, 2.698, 2.695, 2.692, 2.69, 2.687, 2.685, 2.682, 2.68, 2.678, 2.676, 2.674, 2.672, 2.67, 2.668, 2.667, 2.665, 2.663, 2.662, 2.66, 2.659, 2.657, 2.656, 2.655, 2.654, 2.652, 2.651, 2.65, 2.649, 2.648, 2.647, 2.646, 2.645, 2.644, 2.643, 2.642, 2.641, 2.64, 2.64, 2.639, 2.638, 2.637, 2.636, 2.636, 2.635, 2.634, 2.634, 2.633, 2.632, 2.632, 2.631, 2.63, 2.63, 2.629, 2.629, 2.628, 2.627, 2.627, 2.626, 2.626, 2.576}, // for significance level = 0.002 {0.002, 318.313, 22.327, 10.215, 7.173, 5.893, 5.208, 4.782, 4.499, 4.296, 4.143, 4.024, 3.929, 3.852, 3.787, 3.733, 3.686, 3.646, 3.61, 3.579, 3.552, 3.527, 3.505, 3.485, 3.467, 3.45, 3.435, 3.421, 3.408, 3.396, 3.385, 3.375, 3.365, 3.356, 3.348, 3.34, 3.333, 3.326, 3.319, 3.313, 3.307, 3.301, 3.296, 3.291, 3.286, 3.281, 3.277, 3.273, 3.269, 3.265, 3.261, 3.258, 3.255, 3.251, 3.248, 3.245, 3.242, 3.239, 3.237, 3.234, 3.232, 3.229, 3.227, 3.225, 3.223, 3.22, 3.218, 3.216, 3.214, 3.213, 3.211, 3.209, 3.207, 3.206, 3.204, 3.202, 3.201, 3.199, 3.198, 3.197, 3.195, 3.194, 3.193, 3.191, 3.19, 3.189, 3.188, 3.187, 3.185, 3.184, 3.183, 3.182, 3.181, 3.18, 3.179, 3.178, 3.177, 3.176, 3.175, 3.175, 3.174, 3.09} @@ -145,19 +149,19 @@ struct AggregateFunctionWelchTTestData final { } //round or make infinity dof - dof = (int) dof; + dof = static_cast(dof); if (dof > 100) { dof = 101; } //check if abs of t is greater than table[dof] - t = abs(t) + t = abs(t); if(t > CriticalValuesTable[table][dof]) { - return (UInt8) 1; + return static_cast(1); //in this case we reject the null hypothesis } else { - return (UInt8) 0; + return static_cast(0); } } }; From 62460faf97e2a10d3245d7b11d139240fcafa84b Mon Sep 17 00:00:00 2001 From: antikvist Date: Mon, 27 Apr 2020 01:59:41 +0300 Subject: [PATCH 003/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 2 +- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 46b533a2c0b..a4e0a54775b 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -9,7 +9,7 @@ namespace DB namespace { -AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, const DataTypes & argument_types, const Array & parameters) +AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argument_types, const Array & parameters) { diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index be1e176d540..0279849be2c 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -41,7 +41,7 @@ Float64 CriticalValuesTable[SIGN_LVL_CNT][102] = { // for significance level = 0.002 {0.002, 318.313, 22.327, 10.215, 7.173, 5.893, 5.208, 4.782, 4.499, 4.296, 4.143, 4.024, 3.929, 3.852, 3.787, 3.733, 3.686, 3.646, 3.61, 3.579, 3.552, 3.527, 3.505, 3.485, 3.467, 3.45, 3.435, 3.421, 3.408, 3.396, 3.385, 3.375, 3.365, 3.356, 3.348, 3.34, 3.333, 3.326, 3.319, 3.313, 3.307, 3.301, 3.296, 3.291, 3.286, 3.281, 3.277, 3.273, 3.269, 3.265, 3.261, 3.258, 3.255, 3.251, 3.248, 3.245, 3.242, 3.239, 3.237, 3.234, 3.232, 3.229, 3.227, 3.225, 3.223, 3.22, 3.218, 3.216, 3.214, 3.213, 3.211, 3.209, 3.207, 3.206, 3.204, 3.202, 3.201, 3.199, 3.198, 3.197, 3.195, 3.194, 3.193, 3.191, 3.19, 3.189, 3.188, 3.187, 3.185, 3.184, 3.183, 3.182, 3.181, 3.18, 3.179, 3.178, 3.177, 3.176, 3.175, 3.175, 3.174, 3.09} -} +}; // our algorithm implementation via vectors: // https://gist.github.com/ltybc-coder/792748cfdb2f7cadef424ffb7b011c71 From dcfb99b9877528fe09db02a1afff6346de2c8e57 Mon Sep 17 00:00:00 2001 From: antikvist Date: Mon, 27 Apr 2020 19:16:02 +0300 Subject: [PATCH 004/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- .../AggregateFunctionWelchTTest.h | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 0279849be2c..ecc397cf731 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -17,6 +17,11 @@ #include +#include + +#include + + namespace DB { // hard-codded values - part of the algorithm @@ -80,8 +85,8 @@ struct AggregateFunctionWelchTTestData final { sum_y += y; size_x++; size_y++; - mean_x = (Float64) sum_x / size_x; - mean_y = (Float64) sum_y / size_y; + mean_x = static_cast(sum_x) / size_x; + mean_y = static_cast(sum_y) / size_y; square_sum_x += x * x; square_sum_y += y * y; } @@ -91,8 +96,8 @@ struct AggregateFunctionWelchTTestData final { sum_y += other.sum_y; size_x += other.size_x; size_y += other.size_y; - mean_x = (Float64) sum_x / size_x; - mean_y = (Float64) sum_y / size_y; + mean_x = static_cast(sum_x) / size_x; + mean_y = static_cast(sum_y) / size_y; square_sum_x += other.square_sum_x; square_sum_y += other.square_sum_y; } @@ -120,19 +125,19 @@ struct AggregateFunctionWelchTTestData final { } Float64 get_sx() const { - return (Float64)(square_sum_x + size_x * mean_x * mean_x - 2 * mean_x * sum_x) / (size_x - 1); + return static_cast(square_sum_x + size_x * mean_x * mean_x - 2 * mean_x * sum_x) / (size_x - 1); } Float64 get_sy() const { - return (Float64)(square_sum_y + size_y * mean_y * mean_y - 2 * mean_y * sum_y) / (size_y - 1); + return static_cast(square_sum_y + size_y * mean_y * mean_y - 2 * mean_y * sum_y) / (size_y - 1); } Float64 get_T(Float64 sx, Float64 sy) const { - return (Float64)(mean_x - mean_y) / std::sqrt(sx / size_x + sy / size_y); + return static_cast(mean_x - mean_y) / std::sqrt(sx / size_x + sy / size_y); } Float64 get_degrees_of_freed(Float64 sx, Float64 sy) const { - return (Float64)(sx / size_x + sy / size_y) * (sx / size_x + sy / size_y) / + return static_cast(sx / size_x + sy / size_y) * (sx / size_x + sy / size_y) / ((sx * sx / (size_x * size_x * (size_x - 1))) + (sy * sy / (size_y * size_y * (size_y - 1)))); } From 7cfe5ef42b80a45243e70d3b73cf459054e46890 Mon Sep 17 00:00:00 2001 From: antikvist Date: Fri, 1 May 2020 00:36:37 +0300 Subject: [PATCH 005/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index ecc397cf731..e716553e065 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -144,7 +144,7 @@ struct AggregateFunctionWelchTTestData final { Ret get_result(Float64 t, Float64 dof, Float64 parametr) const { //find our table - int table; + int table = 0; for (int i = 0; i < SIGN_LVL_CNT; ++i) { if (CriticalValuesTable[i][0] == parametr) @@ -249,12 +249,11 @@ public: Float64 dof = this->data(place).get_degrees_of_freed(sx, sy); Ret result = this->data(place).get_result(t_value, dof, significance_level); - auto & column = static_cast(to); + //check the type + auto & column = static_cast &>(to); column.getData().push_back(result); } -} }; - -}; \ No newline at end of file +}; From 4f56cc32ae388c5dc002afe071b0b24a9d1adafb Mon Sep 17 00:00:00 2001 From: antikvist Date: Wed, 6 May 2020 00:48:05 +0300 Subject: [PATCH 006/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 6 +++--- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index a4e0a54775b..c4a0c6d4e2b 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -9,11 +9,11 @@ namespace DB namespace { +template AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argument_types, const Array & parameters) { - - - return std::make_shared(argument_types, parameters); + Float64 significance_level = applyVisitor(FieldVisitorConvertToNumber(), params[0]); + return std::make_shared>(argument_types, parameters); } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index e716553e065..a1416f9ab41 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -154,14 +154,14 @@ struct AggregateFunctionWelchTTestData final { } //round or make infinity dof - dof = static_cast(dof); - if (dof > 100) + i_dof = static_cast(dof); + if (i_dof > 100) { - dof = 101; + i_dof = 101; } //check if abs of t is greater than table[dof] t = abs(t); - if(t > CriticalValuesTable[table][dof]) { + if(t > CriticalValuesTable[table][i_dof]) { return static_cast(1); //in this case we reject the null hypothesis } From f0ac5b441f7dccbdfe31512c90ae02da1d9921fa Mon Sep 17 00:00:00 2001 From: antikvist Date: Wed, 6 May 2020 02:07:51 +0300 Subject: [PATCH 007/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 1 - src/AggregateFunctions/AggregateFunctionWelchTTest.h | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index c4a0c6d4e2b..2c8d1e0aed8 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -12,7 +12,6 @@ namespace template AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argument_types, const Array & parameters) { - Float64 significance_level = applyVisitor(FieldVisitorConvertToNumber(), params[0]); return std::make_shared>(argument_types, parameters); } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index a1416f9ab41..9445ccc506d 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -154,7 +154,7 @@ struct AggregateFunctionWelchTTestData final { } //round or make infinity dof - i_dof = static_cast(dof); + int i_dof = static_cast(dof); if (i_dof > 100) { i_dof = 101; From 3dde788146bcef2697cd2701b60d82f603d9c887 Mon Sep 17 00:00:00 2001 From: antikvist Date: Thu, 7 May 2020 14:17:58 +0300 Subject: [PATCH 008/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 9445ccc506d..5203ba1f988 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -186,7 +186,7 @@ public: IAggregateFunctionDataHelper< AggregateFunctionWelchTTestData, AggregateFunctionWelchTTest - > {arguments, params} + > ({argument}, params) { // notice: arguments has been in factory } From 92afa6c0decda463505263f2956005c58e831862 Mon Sep 17 00:00:00 2001 From: antikvist Date: Thu, 7 May 2020 17:14:29 +0300 Subject: [PATCH 009/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 5203ba1f988..22dda8ea244 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -186,7 +186,7 @@ public: IAggregateFunctionDataHelper< AggregateFunctionWelchTTestData, AggregateFunctionWelchTTest - > ({argument}, params) + > ({arguments}, params) { // notice: arguments has been in factory } From 7975d8d5b0c9c4534f108344deb1fdd279cf1eaf Mon Sep 17 00:00:00 2001 From: antikvist Date: Thu, 7 May 2020 23:11:25 +0300 Subject: [PATCH 010/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- .../AggregateFunctionCount.cpp | 2 +- .../AggregateFunctionWelchTTest.h | 47 +++++++++++-------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionCount.cpp b/src/AggregateFunctions/AggregateFunctionCount.cpp index 6c22fec87a2..7ede78e720f 100644 --- a/src/AggregateFunctions/AggregateFunctionCount.cpp +++ b/src/AggregateFunctions/AggregateFunctionCount.cpp @@ -22,7 +22,7 @@ AggregateFunctionPtr createAggregateFunctionCount(const std::string & name, cons void registerAggregateFunctionCount(AggregateFunctionFactory & factory) { - factory.registerFunction("count", createAggregateFunctionCount, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("count", createAggregateFunctionCount); } } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 22dda8ea244..e2e720a12ef 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include #include @@ -12,14 +14,8 @@ #include #include -#include -#include -#include - - #include -#include namespace DB @@ -52,7 +48,8 @@ Float64 CriticalValuesTable[SIGN_LVL_CNT][102] = { // https://gist.github.com/ltybc-coder/792748cfdb2f7cadef424ffb7b011c71 // col, col, bool template -struct AggregateFunctionWelchTTestData final { +struct AggregateFunctionWelchTTestData final +{ size_t size_x = 0; size_t size_y = 0; @@ -65,14 +62,16 @@ struct AggregateFunctionWelchTTestData final { /* not yet sure how to use them - void add_x(X x) { + void add_x(X x) + { mean_x = (Float64)(sum_x + x) / (size_x + 1); size_x ++; sum_x += x; square_sum_x += x * x; } - void add_y(Y y) { + void add_y(Y y) + { mean_y = (sum_y + y) / (size_y + 1); size_y ++; sum_y += y; @@ -80,7 +79,8 @@ struct AggregateFunctionWelchTTestData final { } */ - void add(X x, Y y) { + void add(X x, Y y) + { sum_x += x; sum_y += y; size_x++; @@ -91,7 +91,8 @@ struct AggregateFunctionWelchTTestData final { square_sum_y += y * y; } - void merge(const AggregateFunctionWelchTTestData &other) { + void merge(const AggregateFunctionWelchTTestData &other) + { sum_x += other.sum_x; sum_y += other.sum_y; size_x += other.size_x; @@ -102,7 +103,8 @@ struct AggregateFunctionWelchTTestData final { square_sum_y += other.square_sum_y; } - void serialize(WriteBuffer &buf) const { + void serialize(WriteBuffer &buf) const + { writeBinary(mean_x, buf); writeBinary(mean_y, buf); writeBinary(sum_x, buf); @@ -113,7 +115,8 @@ struct AggregateFunctionWelchTTestData final { writeBinary(size_y, buf); } - void deserialize(ReadBuffer &buf) { + void deserialize(ReadBuffer &buf) + { readBinary(mean_x, buf); readBinary(mean_y, buf); readBinary(sum_x, buf); @@ -124,19 +127,23 @@ struct AggregateFunctionWelchTTestData final { readBinary(size_y, buf); } - Float64 get_sx() const { + Float64 get_sx() const + { return static_cast(square_sum_x + size_x * mean_x * mean_x - 2 * mean_x * sum_x) / (size_x - 1); } - Float64 get_sy() const { + Float64 get_sy() const + { return static_cast(square_sum_y + size_y * mean_y * mean_y - 2 * mean_y * sum_y) / (size_y - 1); } - Float64 get_T(Float64 sx, Float64 sy) const { + Float64 get_T(Float64 sx, Float64 sy) const + { return static_cast(mean_x - mean_y) / std::sqrt(sx / size_x + sy / size_y); } - Float64 get_degrees_of_freed(Float64 sx, Float64 sy) const { + Float64 get_degrees_of_freed(Float64 sx, Float64 sy) const + { return static_cast(sx / size_x + sy / size_y) * (sx / size_x + sy / size_y) / ((sx * sx / (size_x * size_x * (size_x - 1))) + (sy * sy / (size_y * size_y * (size_y - 1)))); } @@ -161,11 +168,13 @@ struct AggregateFunctionWelchTTestData final { } //check if abs of t is greater than table[dof] t = abs(t); - if(t > CriticalValuesTable[table][i_dof]) { + if(t > CriticalValuesTable[table][i_dof]) + { return static_cast(1); //in this case we reject the null hypothesis } - else { + else + { return static_cast(0); } } From b390043f31fe9a4023e9a00b248560155f114b72 Mon Sep 17 00:00:00 2001 From: antikvist Date: Fri, 8 May 2020 00:39:51 +0300 Subject: [PATCH 011/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- .../AggregateFunctionWelchTTest.cpp | 12 ++++++++++-- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 12 +++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 2c8d1e0aed8..c8f2a46b1e4 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -12,7 +12,15 @@ namespace template AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argument_types, const Array & parameters) { - return std::make_shared>(argument_types, parameters); + // default value + Float64 significance_level = 0.1; + if (!params.empty()) + { + significance_level = applyVisitor(FieldVisitorConvertToNumber(), params[0]); + } + + + return std::make_shared>(significance_level, argument_types, parameters); } @@ -21,7 +29,7 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argumen void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest, AggregateFunctionFactory::CaseInsensitive); + factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); } } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index e2e720a12ef..367970fa4e0 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -168,7 +168,7 @@ struct AggregateFunctionWelchTTestData final } //check if abs of t is greater than table[dof] t = abs(t); - if(t > CriticalValuesTable[table][i_dof]) + if (t > CriticalValuesTable[table][i_dof]) { return static_cast(1); //in this case we reject the null hypothesis @@ -187,15 +187,22 @@ class AggregateFunctionWelchTTest final : public AggregateFunctionWelchTTest > { + + +private: + Float64 significance_level; + + public: AggregateFunctionWelchTTest( + Float64 sglvl_, const DataTypes & arguments, const Array & params ): IAggregateFunctionDataHelper< AggregateFunctionWelchTTestData, AggregateFunctionWelchTTest - > ({arguments}, params) + > ({arguments}, params), significance_level(sglvl_) { // notice: arguments has been in factory } @@ -250,7 +257,6 @@ public: IColumn & to ) const override { - Float64 significance_level = applyVisitor(FieldVisitorConvertToNumber(), params[0]); Float64 sx = this->data(place).get_sx(); Float64 sy = this->data(place).get_sy(); From ab7d1fb86fe2645bb7d3ed43cb79b41517f64b08 Mon Sep 17 00:00:00 2001 From: antikvist Date: Fri, 8 May 2020 00:44:31 +0300 Subject: [PATCH 012/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 367970fa4e0..4b199e799f5 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -264,7 +264,7 @@ public: Float64 dof = this->data(place).get_degrees_of_freed(sx, sy); Ret result = this->data(place).get_result(t_value, dof, significance_level); - //check the type + auto & column = static_cast &>(to); column.getData().push_back(result); } From 4b4ff06cab3642cd061f80a8d0eddb74ff0d79db Mon Sep 17 00:00:00 2001 From: antikvist Date: Fri, 8 May 2020 02:22:12 +0300 Subject: [PATCH 013/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 5 +++-- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index c8f2a46b1e4..151071091e4 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -1,6 +1,7 @@ #include #include #include +#include "registerAggregateFunctions.h" namespace DB @@ -14,9 +15,9 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argumen { // default value Float64 significance_level = 0.1; - if (!params.empty()) + if (!parameters.empty()) { - significance_level = applyVisitor(FieldVisitorConvertToNumber(), params[0]); + significance_level = applyVisitor(FieldVisitorConvertToNumber(), parameters[0]); } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 4b199e799f5..855b3fcd917 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -17,7 +17,6 @@ #include - namespace DB { // hard-codded values - part of the algorithm @@ -269,6 +268,5 @@ public: column.getData().push_back(result); } - }; }; From df4c312e141f938099ab5e9d00d17f40c442389d Mon Sep 17 00:00:00 2001 From: antikvist Date: Fri, 8 May 2020 13:17:59 +0300 Subject: [PATCH 014/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 151071091e4..bc759b9f8f1 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -30,7 +30,7 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argumen void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); + factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); } } From 6d6f0b00ba532c3bb637d5a8f6d2dcaf845f257e Mon Sep 17 00:00:00 2001 From: antikvist Date: Sat, 9 May 2020 01:55:09 +0300 Subject: [PATCH 015/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- .../AggregateFunctionWelchTTest.cpp | 25 ++++++++--- .../AggregateFunctionWelchTTest.h | 43 +++++++++++-------- 2 files changed, 43 insertions(+), 25 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index bc759b9f8f1..1634d0149da 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -3,26 +3,39 @@ #include #include "registerAggregateFunctions.h" +#include +#include +#include + namespace DB { namespace { +//template +static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) +{ + return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); +} -template -AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argument_types, const Array & parameters) +//template +AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, + const DataTypes & argument_types, + const Array & parameters) { // default value Float64 significance_level = 0.1; + if (parameters.size() > 1) + throw Exception("Aggregate function " + name + " requires two parameters or less.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); if (!parameters.empty()) { significance_level = applyVisitor(FieldVisitorConvertToNumber(), parameters[0]); } - - return std::make_shared>(significance_level, argument_types, parameters); - + AggregateFunctionPtr res (createWithExtraTypes(significance_level, argument_types, parameters)); + return res; } } @@ -30,7 +43,7 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const DataTypes & argumen void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); + factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); } } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 855b3fcd917..bbf02200745 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -46,16 +46,16 @@ Float64 CriticalValuesTable[SIGN_LVL_CNT][102] = { // our algorithm implementation via vectors: // https://gist.github.com/ltybc-coder/792748cfdb2f7cadef424ffb7b011c71 // col, col, bool -template +//template struct AggregateFunctionWelchTTestData final { size_t size_x = 0; size_t size_y = 0; - X sum_x = 0; - Y sum_y = 0; - X square_sum_x = 0; - Y square_sum_y = 0; + Float64 sum_x = static_cast(0); + Float64 sum_y = static_cast(0); + Float64 square_sum_x = static_cast(0); + Float64 square_sum_y = static_cast(0); Float64 mean_x = 0; Float64 mean_y = 0; @@ -78,7 +78,7 @@ struct AggregateFunctionWelchTTestData final } */ - void add(X x, Y y) + void add(Float64 x, Float64 y) { sum_x += x; sum_y += y; @@ -147,7 +147,7 @@ struct AggregateFunctionWelchTTestData final ((sx * sx / (size_x * size_x * (size_x - 1))) + (sy * sy / (size_y * size_y * (size_y - 1)))); } - Ret get_result(Float64 t, Float64 dof, Float64 parametr) const + UInt8 get_result(Float64 t, Float64 dof, Float64 parametr) const { //find our table int table = 0; @@ -179,11 +179,11 @@ struct AggregateFunctionWelchTTestData final } }; -template -class AggregateFunctionWelchTTest final : public +//template +class AggregateFunctionWelchTTest : public IAggregateFunctionDataHelper< - AggregateFunctionWelchTTestData, - AggregateFunctionWelchTTest + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest > { @@ -199,8 +199,8 @@ public: const Array & params ): IAggregateFunctionDataHelper< - AggregateFunctionWelchTTestData, - AggregateFunctionWelchTTest + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest > ({arguments}, params), significance_level(sglvl_) { // notice: arguments has been in factory @@ -211,6 +211,11 @@ public: return "WelchTTest"; } + DataTypePtr getReturnType() const override + { + return std::make_shared(); + } + void add( AggregateDataPtr place, const IColumn ** columns, @@ -218,11 +223,11 @@ public: Arena * ) const override { - auto col_x = assert_cast *>(columns[0]); - auto col_y = assert_cast *>(columns[1]); + auto col_x = assert_cast *>(columns[0]); + auto col_y = assert_cast *>(columns[1]); - X x = col_x->getData()[row_num]; - Y y = col_y->getData()[row_num]; + Float64 x = col_x->getData()[row_num]; + Float64 y = col_y->getData()[row_num]; this->data(place).add(x, y); } @@ -261,10 +266,10 @@ public: Float64 sy = this->data(place).get_sy(); Float64 t_value = this->data(place).get_T(sx, sy); Float64 dof = this->data(place).get_degrees_of_freed(sx, sy); - Ret result = this->data(place).get_result(t_value, dof, significance_level); + UInt8 result = this->data(place).get_result(t_value, dof, significance_level); - auto & column = static_cast &>(to); + auto & column = static_cast &>(to); column.getData().push_back(result); } From f6e6c48d9c477f92f7669996e4742d1125061ea5 Mon Sep 17 00:00:00 2001 From: antikvist Date: Sun, 17 May 2020 12:25:41 +0300 Subject: [PATCH 016/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionCount.cpp | 2 +- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionCount.cpp b/src/AggregateFunctions/AggregateFunctionCount.cpp index 7ede78e720f..6c22fec87a2 100644 --- a/src/AggregateFunctions/AggregateFunctionCount.cpp +++ b/src/AggregateFunctions/AggregateFunctionCount.cpp @@ -22,7 +22,7 @@ AggregateFunctionPtr createAggregateFunctionCount(const std::string & name, cons void registerAggregateFunctionCount(AggregateFunctionFactory & factory) { - factory.registerFunction("count", createAggregateFunctionCount); + factory.registerFunction("count", createAggregateFunctionCount, AggregateFunctionFactory::CaseInsensitive); } } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 1634d0149da..90b1c445a14 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -14,7 +14,7 @@ namespace DB namespace { //template -static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) +IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) { return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); } From 6069750b0e6bf95ab180d033c0fe376102d540fd Mon Sep 17 00:00:00 2001 From: antikvist Date: Sun, 17 May 2020 14:15:49 +0300 Subject: [PATCH 017/411] #WelchTTest aggregate function implementation What's new: -Main classes for aggreagate function added. -Data class with needed mathods added. -Registered function in registerAggregateFunctions.h --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 90b1c445a14..1634d0149da 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -14,7 +14,7 @@ namespace DB namespace { //template -IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) +static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) { return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); } From 1903e6cec227f216159b23656213587beaea4981 Mon Sep 17 00:00:00 2001 From: antikvist Date: Fri, 12 Jun 2020 02:43:55 +0300 Subject: [PATCH 018/411] pray to ClickHouse gods --- .../AggregateFunctionWelchTTest.cpp | 21 +++++--- .../AggregateFunctionWelchTTest.h | 50 +++++++++---------- 2 files changed, 37 insertions(+), 34 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 1634d0149da..3a2bc831cc1 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -4,30 +4,34 @@ #include "registerAggregateFunctions.h" #include -#include #include +namespace ErrorCodes +{ +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + namespace DB { namespace { -//template +template static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) { return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); } -//template +template AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, - const DataTypes & argument_types, - const Array & parameters) + const DataTypes & argument_types, + const Array & parameters) { // default value Float64 significance_level = 0.1; if (parameters.size() > 1) - throw Exception("Aggregate function " + name + " requires two parameters or less.", + throw Exception("Aggregate function " + name + " requires one parameter or less.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); if (!parameters.empty()) { @@ -40,10 +44,11 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, } +template void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); + factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); } -} +} \ No newline at end of file diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index bbf02200745..d625cc908ec 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -46,16 +46,17 @@ Float64 CriticalValuesTable[SIGN_LVL_CNT][102] = { // our algorithm implementation via vectors: // https://gist.github.com/ltybc-coder/792748cfdb2f7cadef424ffb7b011c71 // col, col, bool -//template +template +//template struct AggregateFunctionWelchTTestData final { size_t size_x = 0; size_t size_y = 0; - Float64 sum_x = static_cast(0); - Float64 sum_y = static_cast(0); - Float64 square_sum_x = static_cast(0); - Float64 square_sum_y = static_cast(0); + X sum_x = static_cast(0); + Y sum_y = static_cast(0); + X square_sum_x = static_cast(0); + Y square_sum_y = static_cast(0); Float64 mean_x = 0; Float64 mean_y = 0; @@ -78,7 +79,7 @@ struct AggregateFunctionWelchTTestData final } */ - void add(Float64 x, Float64 y) + void add(X x, Y y) { sum_x += x; sum_y += y; @@ -147,7 +148,7 @@ struct AggregateFunctionWelchTTestData final ((sx * sx / (size_x * size_x * (size_x - 1))) + (sy * sy / (size_y * size_y * (size_y - 1)))); } - UInt8 get_result(Float64 t, Float64 dof, Float64 parametr) const + Ret get_result(Float64 t, Float64 dof, Float64 parametr) const { //find our table int table = 0; @@ -169,29 +170,27 @@ struct AggregateFunctionWelchTTestData final t = abs(t); if (t > CriticalValuesTable[table][i_dof]) { - return static_cast(1); + return static_cast(1); //in this case we reject the null hypothesis } else { - return static_cast(0); + return static_cast(0); } } }; -//template +template class AggregateFunctionWelchTTest : public - IAggregateFunctionDataHelper< - AggregateFunctionWelchTTestData, - AggregateFunctionWelchTTest - > + IAggregateFunctionDataHelper< + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest + > { - private: Float64 significance_level; - public: AggregateFunctionWelchTTest( Float64 sglvl_, @@ -213,7 +212,7 @@ public: DataTypePtr getReturnType() const override { - return std::make_shared(); + return std::make_shared>(); } void add( @@ -223,11 +222,11 @@ public: Arena * ) const override { - auto col_x = assert_cast *>(columns[0]); - auto col_y = assert_cast *>(columns[1]); + auto col_x = assert_cast *>(columns[0]); + auto col_y = assert_cast *>(columns[1]); - Float64 x = col_x->getData()[row_num]; - Float64 y = col_y->getData()[row_num]; + X x = col_x->getData()[row_num]; + Y y = col_y->getData()[row_num]; this->data(place).add(x, y); } @@ -257,7 +256,7 @@ public: } void insertResultInto( - ConstAggregateDataPtr place, + AggregateDataPtr place, IColumn & to ) const override { @@ -266,12 +265,11 @@ public: Float64 sy = this->data(place).get_sy(); Float64 t_value = this->data(place).get_T(sx, sy); Float64 dof = this->data(place).get_degrees_of_freed(sx, sy); - UInt8 result = this->data(place).get_result(t_value, dof, significance_level); + Ret result = this->data(place).get_result(t_value, dof, significance_level); - - auto & column = static_cast &>(to); + auto & column = static_cast &>(to); column.getData().push_back(result); } }; -}; +}; \ No newline at end of file From bbfccd491ece148818b5d0b09ffdd6abaedae908 Mon Sep 17 00:00:00 2001 From: antikvist Date: Fri, 12 Jun 2020 16:51:33 +0300 Subject: [PATCH 019/411] welch t-test --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 2 +- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 3a2bc831cc1..853a1182340 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -20,7 +20,7 @@ namespace template static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) { - return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); + return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); } template diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index d625cc908ec..bbd02d844c4 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -198,8 +198,8 @@ public: const Array & params ): IAggregateFunctionDataHelper< - AggregateFunctionWelchTTestData, - AggregateFunctionWelchTTest + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest > ({arguments}, params), significance_level(sglvl_) { // notice: arguments has been in factory From 13faa3d83f8e50a2e1ea1c23b0026a11c9d72163 Mon Sep 17 00:00:00 2001 From: antikvist Date: Sat, 13 Jun 2020 19:23:17 +0300 Subject: [PATCH 020/411] welch t-test --- .../AggregateFunctionWelchTTest.cpp | 26 ++++++++++------- .../AggregateFunctionWelchTTest.h | 28 +++++++++---------- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 853a1182340..9f451fd5d88 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -7,6 +7,8 @@ #include +// the return type is boolean (we use UInt8 as we do not have boolean in clickhouse) + namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -17,13 +19,11 @@ namespace DB namespace { -template -static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) -{ - return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); -} +//static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) +//{ +// return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); +//} -template AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, const DataTypes & argument_types, const Array & parameters) @@ -38,17 +38,23 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, significance_level = applyVisitor(FieldVisitorConvertToNumber(), parameters[0]); } - AggregateFunctionPtr res (createWithExtraTypes(significance_level, argument_types, parameters)); + AggregateFunctionPtr res; + DataTypePtr data_type = argument_types[0]; +// if (isDecimal(data_type)) +// res.reset(createWithDecimalType(*data_type, significance_level, argument_types, parameters)); +// else + res.reset(createWithNumericType(*data_type, significance_level, argument_types, parameters)); + + //AggregateFunctionPtr res (createWithExtraTypes(significance_level, argument_types, parameters)); return res; } } -template + void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - - factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); + factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); } } \ No newline at end of file diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index bbd02d844c4..13b9c992162 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -46,8 +46,8 @@ Float64 CriticalValuesTable[SIGN_LVL_CNT][102] = { // our algorithm implementation via vectors: // https://gist.github.com/ltybc-coder/792748cfdb2f7cadef424ffb7b011c71 // col, col, bool -template -//template +template +//template struct AggregateFunctionWelchTTestData final { @@ -148,7 +148,7 @@ struct AggregateFunctionWelchTTestData final ((sx * sx / (size_x * size_x * (size_x - 1))) + (sy * sy / (size_y * size_y * (size_y - 1)))); } - Ret get_result(Float64 t, Float64 dof, Float64 parametr) const + UInt8 get_result(Float64 t, Float64 dof, Float64 parametr) const { //find our table int table = 0; @@ -170,21 +170,21 @@ struct AggregateFunctionWelchTTestData final t = abs(t); if (t > CriticalValuesTable[table][i_dof]) { - return static_cast(1); + return static_cast(1); //in this case we reject the null hypothesis } else { - return static_cast(0); + return static_cast(0); } } }; -template +template class AggregateFunctionWelchTTest : public IAggregateFunctionDataHelper< - AggregateFunctionWelchTTestData, - AggregateFunctionWelchTTest + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest > { @@ -198,8 +198,8 @@ public: const Array & params ): IAggregateFunctionDataHelper< - AggregateFunctionWelchTTestData, - AggregateFunctionWelchTTest + AggregateFunctionWelchTTestData, + AggregateFunctionWelchTTest > ({arguments}, params), significance_level(sglvl_) { // notice: arguments has been in factory @@ -212,7 +212,7 @@ public: DataTypePtr getReturnType() const override { - return std::make_shared>(); + return std::make_shared>(); } void add( @@ -265,11 +265,11 @@ public: Float64 sy = this->data(place).get_sy(); Float64 t_value = this->data(place).get_T(sx, sy); Float64 dof = this->data(place).get_degrees_of_freed(sx, sy); - Ret result = this->data(place).get_result(t_value, dof, significance_level); + UInt8 result = this->data(place).get_result(t_value, dof, significance_level); - auto & column = static_cast &>(to); + auto & column = static_cast &>(to); column.getData().push_back(result); } }; -}; \ No newline at end of file +}; \ No newline at end of file From cb8eec8def65f817773f623e57d8e02518d3c2bd Mon Sep 17 00:00:00 2001 From: antikvist Date: Sun, 14 Jun 2020 00:55:01 +0300 Subject: [PATCH 021/411] welch t-test --- .../AggregateFunctionWelchTTest.cpp | 24 +++++++++++-------- .../AggregateFunctionWelchTTest.h | 2 +- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 9f451fd5d88..8d2963aba74 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -12,6 +12,7 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int NOT_IMPLEMENTED; } namespace DB @@ -31,21 +32,24 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, // default value Float64 significance_level = 0.1; if (parameters.size() > 1) - throw Exception("Aggregate function " + name + " requires one parameter or less.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + { + throw Exception("Aggregate function " + name + " requires one parameter or less.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + } + if (!parameters.empty()) { significance_level = applyVisitor(FieldVisitorConvertToNumber(), parameters[0]); } AggregateFunctionPtr res; - DataTypePtr data_type = argument_types[0]; -// if (isDecimal(data_type)) -// res.reset(createWithDecimalType(*data_type, significance_level, argument_types, parameters)); -// else - res.reset(createWithNumericType(*data_type, significance_level, argument_types, parameters)); - //AggregateFunctionPtr res (createWithExtraTypes(significance_level, argument_types, parameters)); + if (isDecimal(argument_types[0]) || isDecimal(argument_types[1])) + { + throw Exception("Aggregate function " + name + " does not support decimal types.", ErrorCodes::NOT_IMPLEMENTED); + } + + res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, argument_types, parameters)); + return res; } @@ -54,7 +58,7 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, void registerAggregateFunctionWelchTTest(AggregateFunctionFactory & factory) { - factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest); + factory.registerFunction("WelchTTest", createAggregateFunctionWelchTTest, AggregateFunctionFactory::CaseInsensitive); } -} \ No newline at end of file +} diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 13b9c992162..29f8e17b6be 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -272,4 +272,4 @@ public: } }; -}; \ No newline at end of file +}; From 9638eb7490bd8cfad234353d791115cd9988cbf6 Mon Sep 17 00:00:00 2001 From: antikvist Date: Sun, 14 Jun 2020 19:18:04 +0300 Subject: [PATCH 022/411] welch t-test --- .../AggregateFunctionWelchTTest.cpp | 15 ++++++--------- .../AggregateFunctionWelchTTest.h | 12 ++++++------ 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 8d2963aba74..c7349f28d90 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -20,10 +20,6 @@ namespace DB namespace { -//static IAggregateFunction * createWithExtraTypes(Float64 significance_level, const DataTypes & argument_types, const Array & parameters) -//{ -// return new AggregateFunctionWelchTTest(significance_level, argument_types, parameters); -//} AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, const DataTypes & argument_types, @@ -43,12 +39,13 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, AggregateFunctionPtr res; - if (isDecimal(argument_types[0]) || isDecimal(argument_types[1])) - { - throw Exception("Aggregate function " + name + " does not support decimal types.", ErrorCodes::NOT_IMPLEMENTED); - } +// if (isDecimal(argument_types[0]) || isDecimal(argument_types[1])) +// { +// throw Exception("Aggregate function " + name + " does not support decimal types.", ErrorCodes::NOT_IMPLEMENTED); +// } - res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, argument_types, parameters)); + res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, + argument_types, parameters)); return res; } diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 29f8e17b6be..210b8990693 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -53,12 +53,12 @@ struct AggregateFunctionWelchTTestData final size_t size_x = 0; size_t size_y = 0; - X sum_x = static_cast(0); - Y sum_y = static_cast(0); - X square_sum_x = static_cast(0); - Y square_sum_y = static_cast(0); - Float64 mean_x = 0; - Float64 mean_y = 0; + X sum_x = static_cast(0); + Y sum_y = static_cast(0); + X square_sum_x = static_cast(0); + Y square_sum_y = static_cast(0); + Float64 mean_x = static_cast(0); + Float64 mean_y = static_cast(0); /* not yet sure how to use them From 3b30ea2f373d841f936b6b2bd5be25e335b82d70 Mon Sep 17 00:00:00 2001 From: antikvist Date: Sun, 14 Jun 2020 23:00:02 +0300 Subject: [PATCH 023/411] welch t-test --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index c7349f28d90..e7ba204046e 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -25,6 +25,8 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, const DataTypes & argument_types, const Array & parameters) { + assertBinary(name, argument_types); + // default value Float64 significance_level = 0.1; if (parameters.size() > 1) @@ -39,10 +41,10 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, AggregateFunctionPtr res; -// if (isDecimal(argument_types[0]) || isDecimal(argument_types[1])) -// { -// throw Exception("Aggregate function " + name + " does not support decimal types.", ErrorCodes::NOT_IMPLEMENTED); -// } + if (isDecimal(argument_types[0]) || isDecimal(argument_types[1])) + { + throw Exception("Aggregate function " + name + " does not support decimal types.", ErrorCodes::NOT_IMPLEMENTED); + } res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, argument_types, parameters)); From 2f073ab3f785fc267715940d685a1b7743c588f4 Mon Sep 17 00:00:00 2001 From: antikvist Date: Mon, 15 Jun 2020 21:18:22 +0300 Subject: [PATCH 024/411] welch t-test --- .../AggregateFunctionWelchTTest.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index e7ba204046e..b1c8f73d2e9 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -43,11 +43,17 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, if (isDecimal(argument_types[0]) || isDecimal(argument_types[1])) { - throw Exception("Aggregate function " + name + " does not support decimal types.", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Aggregate function " + name + " only supports numerical types.", ErrorCodes::NOT_IMPLEMENTED); + } + + else{ + res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, + argument_types, parameters)); } - res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, - argument_types, parameters)); + if(!res){ + throw Exception("Aggregate function " + name + " only supports numerical types.", ErrorCodes::NOT_IMPLEMENTED); + } return res; } From ccce2537f686152e09c7143517707e5dcec8fe95 Mon Sep 17 00:00:00 2001 From: antikvist Date: Mon, 15 Jun 2020 21:21:48 +0300 Subject: [PATCH 025/411] welch t-test --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index b1c8f73d2e9..28ad414146e 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -45,12 +45,13 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, { throw Exception("Aggregate function " + name + " only supports numerical types.", ErrorCodes::NOT_IMPLEMENTED); } - + else{ res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, argument_types, parameters)); } + if(!res){ throw Exception("Aggregate function " + name + " only supports numerical types.", ErrorCodes::NOT_IMPLEMENTED); } From 965bf4bd6535676f99b667b8630fcdce7f4f9e5b Mon Sep 17 00:00:00 2001 From: antikvist Date: Tue, 16 Jun 2020 01:38:35 +0300 Subject: [PATCH 026/411] welch t-test --- src/AggregateFunctions/AggregateFunctionWelchTTest.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 28ad414146e..3d2e98e2a0e 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -46,13 +46,15 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest(const std::string & name, throw Exception("Aggregate function " + name + " only supports numerical types.", ErrorCodes::NOT_IMPLEMENTED); } - else{ + else + { res.reset(createWithTwoNumericTypes(*argument_types[0], *argument_types[1], significance_level, argument_types, parameters)); } - if(!res){ + if (!res) + { throw Exception("Aggregate function " + name + " only supports numerical types.", ErrorCodes::NOT_IMPLEMENTED); } From 51ff2f4e92d163593c192437e98a5a03bf67011e Mon Sep 17 00:00:00 2001 From: antikvist Date: Tue, 16 Jun 2020 09:14:54 +0300 Subject: [PATCH 027/411] welch t-test --- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 210b8990693..897c583d913 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -272,4 +272,5 @@ public: } }; + }; From a7f8d6b380f290f25725fef6ab97c10dba0c37b1 Mon Sep 17 00:00:00 2001 From: antikvist Date: Tue, 16 Jun 2020 12:45:46 +0300 Subject: [PATCH 028/411] welch t-test --- src/AggregateFunctions/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/src/AggregateFunctions/ya.make b/src/AggregateFunctions/ya.make index bfa32b6dd78..edae91cc745 100644 --- a/src/AggregateFunctions/ya.make +++ b/src/AggregateFunctions/ya.make @@ -49,6 +49,7 @@ SRCS( registerAggregateFunctions.cpp UniqCombinedBiasData.cpp UniqVariadicHash.cpp + AggregateFunctionWelchTTest.cpp ) END() From 21c5ecb597f3c3e4e65cbf4d48b3e604f9428931 Mon Sep 17 00:00:00 2001 From: antikvist Date: Tue, 16 Jun 2020 22:58:06 +0300 Subject: [PATCH 029/411] welch t-test --- .../AggregateFunctionWelchTTest.h | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 897c583d913..266da9fde5b 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -16,6 +16,10 @@ #include +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +} namespace DB { @@ -127,6 +131,16 @@ struct AggregateFunctionWelchTTestData final readBinary(size_y, buf); } + size_t get_size_y() const + { + return size_y; + } + + size_t get_size_x() const + { + return size_x; + } + Float64 get_sx() const { return static_cast(square_sum_x + size_x * mean_x * mean_x - 2 * mean_x * sum_x) / (size_x - 1); @@ -162,10 +176,17 @@ struct AggregateFunctionWelchTTestData final //round or make infinity dof int i_dof = static_cast(dof); + if (i_dof > 100) { i_dof = 101; } + + if(i_dof < 100) + { + i_dof = 1; + } + //check if abs of t is greater than table[dof] t = abs(t); if (t > CriticalValuesTable[table][i_dof]) @@ -260,6 +281,13 @@ public: IColumn & to ) const override { + size_t size_x = this->data(place).get_size_x(); + size_t size_y = this->data(place).get_size_y(); + + if(size_x < 2 || size_y < 2) + { + throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS); + } Float64 sx = this->data(place).get_sx(); Float64 sy = this->data(place).get_sy(); From d0f92b5492b635881b72f32d2b4462b4f44b907c Mon Sep 17 00:00:00 2001 From: antikvist Date: Wed, 17 Jun 2020 18:43:22 +0300 Subject: [PATCH 030/411] welch t-test --- .../AggregateFunctionWelchTTest.h | 5 +++-- .../0_stateless/01319_welch_ttest.reference | 3 +++ tests/queries/0_stateless/01319_welch_ttest.sql | 17 +++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/01319_welch_ttest.reference create mode 100644 tests/queries/0_stateless/01319_welch_ttest.sql diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 266da9fde5b..e2ae1761c51 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -13,6 +13,7 @@ #include #include #include +#include #include @@ -189,7 +190,7 @@ struct AggregateFunctionWelchTTestData final //check if abs of t is greater than table[dof] t = abs(t); - if (t > CriticalValuesTable[table][i_dof]) + if (t >= CriticalValuesTable[table][i_dof]) { return static_cast(1); //in this case we reject the null hypothesis @@ -284,7 +285,7 @@ public: size_t size_x = this->data(place).get_size_x(); size_t size_y = this->data(place).get_size_y(); - if(size_x < 2 || size_y < 2) + if( size_x < 2 || size_y < 2) { throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS); } diff --git a/tests/queries/0_stateless/01319_welch_ttest.reference b/tests/queries/0_stateless/01319_welch_ttest.reference new file mode 100644 index 00000000000..e22493782f0 --- /dev/null +++ b/tests/queries/0_stateless/01319_welch_ttest.reference @@ -0,0 +1,3 @@ +1 +0 +0 diff --git a/tests/queries/0_stateless/01319_welch_ttest.sql b/tests/queries/0_stateless/01319_welch_ttest.sql new file mode 100644 index 00000000000..ea103cc433b --- /dev/null +++ b/tests/queries/0_stateless/01319_welch_ttest.sql @@ -0,0 +1,17 @@ +DROP TABLE IF EXISTS welch_ttest; +CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; + +INSERT INTO welch_ttest VALUES (2224.779, 2465.0984), (2588.11, 1909.0328), (1979.625, 1175.8747), (2137.442, 2171.378), (2565.818, 2193.2821), (1754.023, 2854.9475), (1654.947, 2060.1777), (1789.256, 2258.2366), (2320.659, 1856.0535), (2039.532, 1501.8126), (1983.497, 2987.6542), (2232.903, 1681.9778), (2513.93, 2479.6776), (2066.382, 1259.8584), (2492.715, 1120.9043), (1988.287, 1982.1213), (1840.036, 3012.3949), (2249.749, 2252.373), (1766.982, 2591.3122), (1724.84, 1940.589), (0, 1995.185), (0, 2535.1344), (0, 597.3155), (0, 2343.2192), (0, 3154.84), (0, 1125.1966), (0, 1227.8842), (0, 1692.805), (0, 2539.6772), (0, 1936.1927), (0, 1783.7795), (0, 1703.4384), (0, 2077.194), (0, 1614.4071), (0, 2360.0365), (0, 1619.2781), (0, 2033.5109), (0, 2333.7834), (0, 2144.0485), (0, 2583.8709), (0, 1116.7213), (0, 1601.9383), (0, 1570.0431), (0, 1963.0777), (0, 1639.2533), (0, 2277.5223), (0, 1991.9286), (0, 2044.3338), (0, 1794.4781), (0, 1597.9119) +SELECT WelchTTest(0.1)(left, right) from welch_ttest; + +DROP TABLE IF EXISTS welch_ttest; +CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; + +INSERT INTO welch_ttest VALUES (2224.779, 2465.0984), (2588.11, 1909.0328), (1979.625, 1175.8747), (2137.442, 2171.378), (2565.818, 2193.2821), (1754.023, 2854.9475), (1654.947, 2060.1777), (1789.256, 2258.2366), (2320.659, 1856.0535), (2039.532, 1501.8126), (1983.497, 2987.6542), (2232.903, 1681.9778), (2513.93, 2479.6776), (2066.382, 1259.8584), (2492.715, 1120.9043), (1988.287, 1982.1213), (1840.036, 3012.3949), (2249.749, 2252.373), (1766.982, 2591.3122), (1724.84, 1940.589), (0, 1995.185), (0, 2535.1344), (0, 597.3155), (0, 2343.2192), (0, 3154.84), (0, 1125.1966), (0, 1227.8842), (0, 1692.805), (0, 2539.6772), (0, 1936.1927), (0, 1783.7795), (0, 1703.4384), (0, 2077.194), (0, 1614.4071), (0, 2360.0365), (0, 1619.2781), (0, 2033.5109), (0, 2333.7834), (0, 2144.0485), (0, 2583.8709), (0, 1116.7213), (0, 1601.9383), (0, 1570.0431), (0, 1963.0777), (0, 1639.2533), (0, 2277.5223), (0, 1991.9286), (0, 2044.3338), (0, 1794.4781), (0, 1597.9119) +SELECT WelchTTest(0.02)(left, right) from welch_ttest; + +DROP TABLE IF EXISTS welch_ttest; +CREATE TABLE welch_ttest (left Int64, right Int64) ENGINE = Memory; + +INSERT INTO welch_ttest VALUES (1, 1), (1, 1), (1, 1); +SELECT WelchTTest(0.1)(left, right) from welch_ttest; \ No newline at end of file From d92160e734fe71b03e66b330512f2f261274becf Mon Sep 17 00:00:00 2001 From: antikvist Date: Wed, 17 Jun 2020 18:59:58 +0300 Subject: [PATCH 031/411] welch t-test --- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index e2ae1761c51..456effc53b8 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -183,7 +183,7 @@ struct AggregateFunctionWelchTTestData final i_dof = 101; } - if(i_dof < 100) + if (i_dof < 100) { i_dof = 1; } @@ -285,7 +285,7 @@ public: size_t size_x = this->data(place).get_size_x(); size_t size_y = this->data(place).get_size_y(); - if( size_x < 2 || size_y < 2) + if (size_x < 2 || size_y < 2) { throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS); } From e4792df9a96be8a99a68292affb7c406e40d15a5 Mon Sep 17 00:00:00 2001 From: antikvist Date: Wed, 17 Jun 2020 21:56:39 +0300 Subject: [PATCH 032/411] welch t-test --- tests/queries/0_stateless/01319_welch_ttest.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01319_welch_ttest.sql b/tests/queries/0_stateless/01319_welch_ttest.sql index ea103cc433b..26bd686efab 100644 --- a/tests/queries/0_stateless/01319_welch_ttest.sql +++ b/tests/queries/0_stateless/01319_welch_ttest.sql @@ -14,4 +14,5 @@ DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Int64, right Int64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (1, 1), (1, 1), (1, 1); -SELECT WelchTTest(0.1)(left, right) from welch_ttest; \ No newline at end of file +SELECT WelchTTest(0.1)(left, right) from welch_ttest; +DROP TABLE IF EXISTS welch_ttest; \ No newline at end of file From ae8ee1cbfaba57dcee2210d98ccf2ac7f054e6a4 Mon Sep 17 00:00:00 2001 From: antikvist Date: Wed, 17 Jun 2020 22:22:26 +0300 Subject: [PATCH 033/411] welch t-test --- src/AggregateFunctions/AggregateFunctionWelchTTest.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index 456effc53b8..cc1417e6659 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -154,6 +154,16 @@ struct AggregateFunctionWelchTTestData final Float64 get_T(Float64 sx, Float64 sy) const { + if (sx == 0 && sy == 0) + { + throw Exception("division by zero encountered in Aggregate function WelchTTest", ErrorCodes::BAD_ARGUMENTS); + } + + if (sx == -sy && size_x == size_y) + { + throw Exception("division by zero encountered in Aggregate function WelchTTest", ErrorCodes::BAD_ARGUMENTS); + } + return static_cast(mean_x - mean_y) / std::sqrt(sx / size_x + sy / size_y); } From 08f9444842dfa6c0782237b839d491a54200c21e Mon Sep 17 00:00:00 2001 From: antikvist Date: Thu, 18 Jun 2020 00:02:50 +0300 Subject: [PATCH 034/411] welch t-test --- tests/queries/0_stateless/01319_welch_ttest.reference | 1 - tests/queries/0_stateless/01319_welch_ttest.sql | 5 ----- 2 files changed, 6 deletions(-) diff --git a/tests/queries/0_stateless/01319_welch_ttest.reference b/tests/queries/0_stateless/01319_welch_ttest.reference index e22493782f0..b261da18d51 100644 --- a/tests/queries/0_stateless/01319_welch_ttest.reference +++ b/tests/queries/0_stateless/01319_welch_ttest.reference @@ -1,3 +1,2 @@ 1 0 -0 diff --git a/tests/queries/0_stateless/01319_welch_ttest.sql b/tests/queries/0_stateless/01319_welch_ttest.sql index 26bd686efab..b8e881a069b 100644 --- a/tests/queries/0_stateless/01319_welch_ttest.sql +++ b/tests/queries/0_stateless/01319_welch_ttest.sql @@ -10,9 +10,4 @@ CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (2224.779, 2465.0984), (2588.11, 1909.0328), (1979.625, 1175.8747), (2137.442, 2171.378), (2565.818, 2193.2821), (1754.023, 2854.9475), (1654.947, 2060.1777), (1789.256, 2258.2366), (2320.659, 1856.0535), (2039.532, 1501.8126), (1983.497, 2987.6542), (2232.903, 1681.9778), (2513.93, 2479.6776), (2066.382, 1259.8584), (2492.715, 1120.9043), (1988.287, 1982.1213), (1840.036, 3012.3949), (2249.749, 2252.373), (1766.982, 2591.3122), (1724.84, 1940.589), (0, 1995.185), (0, 2535.1344), (0, 597.3155), (0, 2343.2192), (0, 3154.84), (0, 1125.1966), (0, 1227.8842), (0, 1692.805), (0, 2539.6772), (0, 1936.1927), (0, 1783.7795), (0, 1703.4384), (0, 2077.194), (0, 1614.4071), (0, 2360.0365), (0, 1619.2781), (0, 2033.5109), (0, 2333.7834), (0, 2144.0485), (0, 2583.8709), (0, 1116.7213), (0, 1601.9383), (0, 1570.0431), (0, 1963.0777), (0, 1639.2533), (0, 2277.5223), (0, 1991.9286), (0, 2044.3338), (0, 1794.4781), (0, 1597.9119) SELECT WelchTTest(0.02)(left, right) from welch_ttest; -DROP TABLE IF EXISTS welch_ttest; -CREATE TABLE welch_ttest (left Int64, right Int64) ENGINE = Memory; - -INSERT INTO welch_ttest VALUES (1, 1), (1, 1), (1, 1); -SELECT WelchTTest(0.1)(left, right) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; \ No newline at end of file From 4660da3e5e509bd2abd8674fc2babb4dcbeee1c4 Mon Sep 17 00:00:00 2001 From: antikvist Date: Thu, 18 Jun 2020 22:18:52 +0300 Subject: [PATCH 035/411] welch t-test --- .../AggregateFunctionWelchTTest.h | 34 +++++++++---------- .../0_stateless/01319_welch_ttest.reference | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index cc1417e6659..2f56e5e6b6c 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -132,27 +132,27 @@ struct AggregateFunctionWelchTTestData final readBinary(size_y, buf); } - size_t get_size_y() const + size_t getSizeY() const { return size_y; } - size_t get_size_x() const + size_t getSizeX() const { return size_x; } - Float64 get_sx() const + Float64 getSx() const { return static_cast(square_sum_x + size_x * mean_x * mean_x - 2 * mean_x * sum_x) / (size_x - 1); } - Float64 get_sy() const + Float64 getSy() const { return static_cast(square_sum_y + size_y * mean_y * mean_y - 2 * mean_y * sum_y) / (size_y - 1); } - Float64 get_T(Float64 sx, Float64 sy) const + Float64 getT(Float64 sx, Float64 sy) const { if (sx == 0 && sy == 0) { @@ -167,13 +167,13 @@ struct AggregateFunctionWelchTTestData final return static_cast(mean_x - mean_y) / std::sqrt(sx / size_x + sy / size_y); } - Float64 get_degrees_of_freed(Float64 sx, Float64 sy) const + Float64 getDegreesOfFreedom(Float64 sx, Float64 sy) const { return static_cast(sx / size_x + sy / size_y) * (sx / size_x + sy / size_y) / ((sx * sx / (size_x * size_x * (size_x - 1))) + (sy * sy / (size_y * size_y * (size_y - 1)))); } - UInt8 get_result(Float64 t, Float64 dof, Float64 parametr) const + UInt8 getResult(Float64 t, Float64 dof, Float64 parametr) const { //find our table int table = 0; @@ -193,7 +193,7 @@ struct AggregateFunctionWelchTTestData final i_dof = 101; } - if (i_dof < 100) + if (i_dof < 1) { i_dof = 1; } @@ -202,12 +202,12 @@ struct AggregateFunctionWelchTTestData final t = abs(t); if (t >= CriticalValuesTable[table][i_dof]) { - return static_cast(1); + return static_cast(0); //in this case we reject the null hypothesis } else { - return static_cast(0); + return static_cast(1); } } }; @@ -292,19 +292,19 @@ public: IColumn & to ) const override { - size_t size_x = this->data(place).get_size_x(); - size_t size_y = this->data(place).get_size_y(); + size_t size_x = this->data(place).getSizeX(); + size_t size_y = this->data(place).getSizeY(); if (size_x < 2 || size_y < 2) { throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS); } - Float64 sx = this->data(place).get_sx(); - Float64 sy = this->data(place).get_sy(); - Float64 t_value = this->data(place).get_T(sx, sy); - Float64 dof = this->data(place).get_degrees_of_freed(sx, sy); - UInt8 result = this->data(place).get_result(t_value, dof, significance_level); + Float64 sx = this->data(place).getSx(); + Float64 sy = this->data(place).getSy(); + Float64 t_value = this->data(place).getT(sx, sy); + Float64 dof = this->data(place).getDegreesOfFreedom(sx, sy); + UInt8 result = this->data(place).getResult(t_value, dof, significance_level); auto & column = static_cast &>(to); column.getData().push_back(result); diff --git a/tests/queries/0_stateless/01319_welch_ttest.reference b/tests/queries/0_stateless/01319_welch_ttest.reference index b261da18d51..aa47d0d46d4 100644 --- a/tests/queries/0_stateless/01319_welch_ttest.reference +++ b/tests/queries/0_stateless/01319_welch_ttest.reference @@ -1,2 +1,2 @@ -1 +0 0 From 839ee63294515a2899d5a5855bfe824231d17319 Mon Sep 17 00:00:00 2001 From: antikvist Date: Sat, 20 Jun 2020 19:31:00 +0300 Subject: [PATCH 036/411] welch --- .../{01319_welch_ttest.reference => 01322_welch_ttest.reference} | 0 .../0_stateless/{01319_welch_ttest.sql => 01322_welch_ttest.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{01319_welch_ttest.reference => 01322_welch_ttest.reference} (100%) rename tests/queries/0_stateless/{01319_welch_ttest.sql => 01322_welch_ttest.sql} (100%) diff --git a/tests/queries/0_stateless/01319_welch_ttest.reference b/tests/queries/0_stateless/01322_welch_ttest.reference similarity index 100% rename from tests/queries/0_stateless/01319_welch_ttest.reference rename to tests/queries/0_stateless/01322_welch_ttest.reference diff --git a/tests/queries/0_stateless/01319_welch_ttest.sql b/tests/queries/0_stateless/01322_welch_ttest.sql similarity index 100% rename from tests/queries/0_stateless/01319_welch_ttest.sql rename to tests/queries/0_stateless/01322_welch_ttest.sql From 3d89f0e9df325b6830153735263b5efaa829cf0a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 21 Jul 2020 15:41:14 +0300 Subject: [PATCH 037/411] Perf test: bind server to one NUMA node --- docker/test/performance-comparison/compare.sh | 12 ++++++++++-- tests/performance/decimal_aggregates.xml | 2 +- tests/performance/jit_large_requests.xml | 4 +--- tests/performance/string_sort.xml | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 378e87f443b..4bde4f945a6 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -55,12 +55,18 @@ function restart set -m # Spawn servers in their own process groups - left/clickhouse-server --config-file=left/config/config.xml -- --path left/db --user_files_path left/db/user_files &>> left-server-log.log & + numactl --membind=0 --cpunodebind=0 --localalloc \ + left/clickhouse-server --config-file=left/config/config.xml \ + -- --path left/db --user_files_path left/db/user_files \ + &>> left-server-log.log & left_pid=$! kill -0 $left_pid disown $left_pid - right/clickhouse-server --config-file=right/config/config.xml -- --path right/db --user_files_path right/db/user_files &>> right-server-log.log & + numactl --membind=0 --cpunodebind=0 --localalloc \ + right/clickhouse-server --config-file=right/config/config.xml \ + -- --path right/db --user_files_path right/db/user_files \ + &>> right-server-log.log & right_pid=$! kill -0 $right_pid disown $right_pid @@ -909,6 +915,8 @@ case "$stage" in time configure ;& "restart") + numactl --hardware ||: + lscpu ||: time restart ;& "run_tests") diff --git a/tests/performance/decimal_aggregates.xml b/tests/performance/decimal_aggregates.xml index 0c8df88c73c..142d9388404 100644 --- a/tests/performance/decimal_aggregates.xml +++ b/tests/performance/decimal_aggregates.xml @@ -1,6 +1,6 @@ - 30G + 35G CREATE TABLE t (x UInt64, d32 Decimal32(3), d64 Decimal64(4), d128 Decimal128(5)) ENGINE = Memory diff --git a/tests/performance/jit_large_requests.xml b/tests/performance/jit_large_requests.xml index 805b7f2edb1..6aed7bea544 100644 --- a/tests/performance/jit_large_requests.xml +++ b/tests/performance/jit_large_requests.xml @@ -1,6 +1,4 @@ - - CREATE TABLE jit_test ( a UInt64, @@ -43,7 +41,7 @@ SETTINGS compile_expressions = 0; - + SELECT COUNT() FROM diff --git a/tests/performance/string_sort.xml b/tests/performance/string_sort.xml index ce5a54e2680..5d859398ece 100644 --- a/tests/performance/string_sort.xml +++ b/tests/performance/string_sort.xml @@ -43,7 +43,7 @@ - + From 9f49bf2d82154db6417585505bfab0897d3d1d4e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 22 Jul 2020 01:09:54 +0300 Subject: [PATCH 038/411] fixup --- docker/test/performance-comparison/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index 912a8bd12cd..df666af8e8e 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -17,6 +17,7 @@ RUN apt-get update \ libc6-dbg \ moreutils \ ncdu \ + numactl \ p7zip-full \ parallel \ psmisc \ From 2b7c0167cb0fda79b59eecf752a56611eae36ab0 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 22 Jul 2020 21:02:56 +0300 Subject: [PATCH 039/411] Update compare.sh --- docker/test/performance-comparison/compare.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 4bde4f945a6..caaff129cfb 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -55,7 +55,7 @@ function restart set -m # Spawn servers in their own process groups - numactl --membind=0 --cpunodebind=0 --localalloc \ + numactl --cpunodebind=0 --localalloc \ left/clickhouse-server --config-file=left/config/config.xml \ -- --path left/db --user_files_path left/db/user_files \ &>> left-server-log.log & @@ -63,7 +63,7 @@ function restart kill -0 $left_pid disown $left_pid - numactl --membind=0 --cpunodebind=0 --localalloc \ + numactl --cpunodebind=0 --localalloc \ right/clickhouse-server --config-file=right/config/config.xml \ -- --path right/db --user_files_path right/db/user_files \ &>> right-server-log.log & From 15cd448afaf98c57f5300674e451eebc5b1f427e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 23 Jul 2020 17:59:32 +0300 Subject: [PATCH 040/411] Update compare.sh --- docker/test/performance-comparison/compare.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index caaff129cfb..84aa31fb76b 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -55,7 +55,7 @@ function restart set -m # Spawn servers in their own process groups - numactl --cpunodebind=0 --localalloc \ + numactl --cpunodebind=0 --membind=0 \ left/clickhouse-server --config-file=left/config/config.xml \ -- --path left/db --user_files_path left/db/user_files \ &>> left-server-log.log & @@ -63,7 +63,7 @@ function restart kill -0 $left_pid disown $left_pid - numactl --cpunodebind=0 --localalloc \ + numactl --cpunodebind=0 --membind=0 \ right/clickhouse-server --config-file=right/config/config.xml \ -- --path right/db --user_files_path right/db/user_files \ &>> right-server-log.log & From 5d6b5101fe2049ce2704e0c022d45a487a84ccec Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Thu, 23 Jul 2020 21:55:24 +0400 Subject: [PATCH 041/411] Implement LDAPAccessStorage and integrate it into AccessControlManager Rename ExternalAuthenticators::setConfig to setConfiguration Revisit LDAP servers config section comments Add user_directories config section with comments (only for ldap) Fix bug in MemoryAccessStorage::insertImpl --- programs/server/Server.cpp | 8 ++ programs/server/config.xml | 23 +++- src/Access/AccessControlManager.cpp | 23 +++- src/Access/AccessControlManager.h | 4 +- src/Access/ExternalAuthenticators.cpp | 2 +- src/Access/ExternalAuthenticators.h | 2 +- src/Access/LDAPAccessStorage.cpp | 175 ++++++++++++++++++++++++++ src/Access/LDAPAccessStorage.h | 54 ++++++++ src/Access/MemoryAccessStorage.cpp | 2 +- src/Access/ya.make | 1 + 10 files changed, 281 insertions(+), 13 deletions(-) create mode 100644 src/Access/LDAPAccessStorage.cpp create mode 100644 src/Access/LDAPAccessStorage.h diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index c3b17824151..eb03deec072 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -595,6 +595,14 @@ int Server::main(const std::vector & /*args*/) if (!access_control_local_path.empty()) global_context->getAccessControlManager().setLocalDirectory(access_control_local_path); + /// Set LDAP user directory config. + const bool has_ldap_directory_config = config().has("user_directories.ldap"); + if (has_ldap_directory_config) { + auto ldap_directory_config = config().createView("user_directories.ldap"); + if (ldap_directory_config) + global_context->getAccessControlManager().setLDAPConfig(*ldap_directory_config); + } + /// Limit on total number of concurrently executed queries. global_context->getProcessList().setMaxSize(config().getInt("max_concurrent_queries", 0)); diff --git a/programs/server/config.xml b/programs/server/config.xml index 3e01964f0ff..b09a53ccf98 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -215,10 +215,10 @@ /var/lib/clickhouse/access/ - + + + + + + users.xml diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index 5966c1aff75..3f457009a5b 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -28,11 +29,14 @@ namespace #if 0 /// Memory access storage is disabled. list.emplace_back(std::make_unique()); #endif + + list.emplace_back(std::make_unique()); return list; } constexpr size_t DISK_ACCESS_STORAGE_INDEX = 0; constexpr size_t USERS_CONFIG_ACCESS_STORAGE_INDEX = 1; + constexpr size_t LDAP_ACCESS_STORAGE_INDEX = 2; } @@ -81,12 +85,6 @@ void AccessControlManager::setLocalDirectory(const String & directory_path) } -void AccessControlManager::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config) -{ - external_authenticators->setConfig(config, getLogger()); -} - - void AccessControlManager::setUsersConfig(const Poco::Util::AbstractConfiguration & users_config) { auto & users_config_access_storage = dynamic_cast(getStorageByIndex(USERS_CONFIG_ACCESS_STORAGE_INDEX)); @@ -94,6 +92,19 @@ void AccessControlManager::setUsersConfig(const Poco::Util::AbstractConfiguratio } +void AccessControlManager::setLDAPConfig(const Poco::Util::AbstractConfiguration & users_config) +{ + auto & ldap_access_storage = dynamic_cast(getStorageByIndex(LDAP_ACCESS_STORAGE_INDEX)); + ldap_access_storage.setConfiguration(users_config, this); +} + + +void AccessControlManager::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config) +{ + external_authenticators->setConfiguration(config, getLogger()); +} + + void AccessControlManager::setDefaultProfileName(const String & default_profile_name) { settings_profiles_cache->setDefaultProfileName(default_profile_name); diff --git a/src/Access/AccessControlManager.h b/src/Access/AccessControlManager.h index 467b7471423..08aa447275b 100644 --- a/src/Access/AccessControlManager.h +++ b/src/Access/AccessControlManager.h @@ -49,8 +49,10 @@ public: ~AccessControlManager(); void setLocalDirectory(const String & directory); - void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config); void setUsersConfig(const Poco::Util::AbstractConfiguration & users_config); + void setLDAPConfig(const Poco::Util::AbstractConfiguration & users_config); + + void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config); void setDefaultProfileName(const String & default_profile_name); std::shared_ptr getContextAccess( diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index a0c5fbf1a79..3ed1b21c3c2 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -156,7 +156,7 @@ void ExternalAuthenticators::reset() ldap_server_params.clear(); } -void ExternalAuthenticators::setConfig(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log) +void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log) { std::scoped_lock lock(mutex); reset(); diff --git a/src/Access/ExternalAuthenticators.h b/src/Access/ExternalAuthenticators.h index 54af87604a6..5dde82b68df 100644 --- a/src/Access/ExternalAuthenticators.h +++ b/src/Access/ExternalAuthenticators.h @@ -26,7 +26,7 @@ class ExternalAuthenticators { public: void reset(); - void setConfig(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log); + void setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log); void setLDAPServerParams(const String & server, const LDAPServerParams & params); LDAPServerParams getLDAPServerParams(const String & server) const; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp new file mode 100644 index 00000000000..314beca4dcc --- /dev/null +++ b/src/Access/LDAPAccessStorage.cpp @@ -0,0 +1,175 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int UNKNOWN_ADDRESS_PATTERN_TYPE; + extern const int NOT_IMPLEMENTED; +} + + +LDAPAccessStorage::LDAPAccessStorage() : IAccessStorage("ldap") +{ +} + + +void LDAPAccessStorage::setConfiguration(const Poco::Util::AbstractConfiguration & config, IAccessStorage * top_enclosing_storage_) +{ + std::scoped_lock lock(mutex); + + const bool has_server = config.has("server"); + const bool has_user_template = config.has("user_template"); + + if (!has_server) + throw Exception("Missing 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); + + const auto ldap_server_ = config.getString("server"); + const auto user_template_ = (has_user_template ? config.getString("user_template") : "default"); + + if (ldap_server_.empty()) + throw Exception("Empty 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); + + if (user_template_.empty()) + throw Exception("Empty 'user_template' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); + + ldap_server = ldap_server_; + user_template = user_template_; + top_enclosing_storage = top_enclosing_storage_; +} + + +bool LDAPAccessStorage::isConfiguredNoLock() const +{ + return !ldap_server.empty() && !user_template.empty() && top_enclosing_storage; +} + + +std::optional LDAPAccessStorage::findImpl(EntityType type, const String & name) const +{ + if (type == EntityType::USER) + { + std::scoped_lock lock(mutex); + + // Detect and avoid loops/duplicate creations. + if (helper_lookup_in_progress) + return {}; + + helper_lookup_in_progress = true; + SCOPE_EXIT({ helper_lookup_in_progress = false; }); + + // Return the id immediately if we already have it. + const auto id = memory_storage.find(type, name); + if (id.has_value()) + return id; + + if (!isConfiguredNoLock()) + { + LOG_WARNING(getLogger(), "Access storage instance is not configured."); + return {}; + } + + // Stop if entity exists anywhere else, to avoid duplicates. + if (top_enclosing_storage->find(type, name)) + return {}; + + // Entity doesn't exist. We are going to create it. Here we retrieve the template first. + const auto user_tmp = top_enclosing_storage->read(user_template); + if (!user_tmp) + { + LOG_WARNING(getLogger(), "Unable to retrieve user template '{}': user does not exist in access storage '{}'.", user_template, top_enclosing_storage->getStorageName()); + return {}; + } + + // Build the new entity based on the existing template. + const auto user = std::make_shared(*user_tmp); + user->setName(name); + user->authentication = Authentication(Authentication::Type::LDAP_SERVER); + user->authentication.setServerName(ldap_server); + + return memory_storage.insert(user); + } + + return memory_storage.find(type, name); +} + + +std::vector LDAPAccessStorage::findAllImpl(EntityType type) const +{ + return memory_storage.findAll(type); +} + + +bool LDAPAccessStorage::existsImpl(const UUID & id) const +{ + return memory_storage.exists(id); +} + + +AccessEntityPtr LDAPAccessStorage::readImpl(const UUID & id) const +{ + return memory_storage.read(id); +} + + +String LDAPAccessStorage::readNameImpl(const UUID & id) const +{ + return memory_storage.readName(id); +} + + +bool LDAPAccessStorage::canInsertImpl(const AccessEntityPtr &) const +{ + return false; +} + + +UUID LDAPAccessStorage::insertImpl(const AccessEntityPtr & entity, bool) +{ + throwReadonlyCannotInsert(entity->getType(), entity->getName()); +} + + +void LDAPAccessStorage::removeImpl(const UUID & id) +{ + auto entity = read(id); + throwReadonlyCannotRemove(entity->getType(), entity->getName()); +} + + +void LDAPAccessStorage::updateImpl(const UUID & id, const UpdateFunc &) +{ + auto entity = read(id); + throwReadonlyCannotUpdate(entity->getType(), entity->getName()); +} + + +ext::scope_guard LDAPAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const +{ + return memory_storage.subscribeForChanges(id, handler); +} + + +ext::scope_guard LDAPAccessStorage::subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const +{ + return memory_storage.subscribeForChanges(type, handler); +} + + +bool LDAPAccessStorage::hasSubscriptionImpl(const UUID & id) const +{ + return memory_storage.hasSubscription(id); +} + + +bool LDAPAccessStorage::hasSubscriptionImpl(EntityType type) const +{ + return memory_storage.hasSubscription(type); +} +} diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h new file mode 100644 index 00000000000..2978724aff7 --- /dev/null +++ b/src/Access/LDAPAccessStorage.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include + + +namespace Poco +{ + namespace Util + { + class AbstractConfiguration; + } +} + + +namespace DB +{ +/// Implementation of IAccessStorage which allows attaching users from a remote LDAP server. +/// Currently, any user name will be treated as a name of an existing remote user, +/// a user info entity will be created, with LDAP_SERVER authentication type. +class LDAPAccessStorage : public IAccessStorage +{ +public: + LDAPAccessStorage(); + + void setConfiguration(const Poco::Util::AbstractConfiguration & config, IAccessStorage * top_enclosing_storage_); + +private: // IAccessStorage implementations. + std::optional findImpl(EntityType type, const String & name) const override; + std::vector findAllImpl(EntityType type) const override; + bool existsImpl(const UUID & id) const override; + AccessEntityPtr readImpl(const UUID & id) const override; + String readNameImpl(const UUID & id) const override; + bool canInsertImpl(const AccessEntityPtr &) const override; + UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override; + void removeImpl(const UUID & id) override; + void updateImpl(const UUID & id, const UpdateFunc & update_func) override; + ext::scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; + ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; + bool hasSubscriptionImpl(const UUID & id) const override; + bool hasSubscriptionImpl(EntityType type) const override; + +private: + bool isConfiguredNoLock() const; + + mutable std::recursive_mutex mutex; + String ldap_server; + String user_template; + IAccessStorage * top_enclosing_storage = nullptr; + mutable bool helper_lookup_in_progress = false; + mutable MemoryAccessStorage memory_storage; +}; +} diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index 720b82796b7..20bea4e472c 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -69,7 +69,7 @@ UUID MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool re UUID id = generateRandomID(); std::lock_guard lock{mutex}; - insertNoLock(generateRandomID(), new_entity, replace_if_exists, notifications); + insertNoLock(id, new_entity, replace_if_exists, notifications); return id; } diff --git a/src/Access/ya.make b/src/Access/ya.make index 175bb86d737..b74642fb95b 100644 --- a/src/Access/ya.make +++ b/src/Access/ya.make @@ -21,6 +21,7 @@ SRCS( GrantedRoles.cpp IAccessEntity.cpp IAccessStorage.cpp + LDAPAccessStorage.cpp LDAPClient.cpp MemoryAccessStorage.cpp MultipleAccessStorage.cpp From 3b3404c326bd80314f2e3e1525903f08f6ee033e Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Thu, 23 Jul 2020 22:10:57 +0400 Subject: [PATCH 042/411] Style fix Remove unused declarations --- programs/server/Server.cpp | 3 ++- src/Access/LDAPAccessStorage.cpp | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index eb03deec072..bd04dc963be 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -597,7 +597,8 @@ int Server::main(const std::vector & /*args*/) /// Set LDAP user directory config. const bool has_ldap_directory_config = config().has("user_directories.ldap"); - if (has_ldap_directory_config) { + if (has_ldap_directory_config) + { auto ldap_directory_config = config().createView("user_directories.ldap"); if (ldap_directory_config) global_context->getAccessControlManager().setLDAPConfig(*ldap_directory_config); diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 314beca4dcc..ab705ab303c 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -10,8 +10,6 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; - extern const int UNKNOWN_ADDRESS_PATTERN_TYPE; - extern const int NOT_IMPLEMENTED; } From 79332c561e6c9b7d2a499d5de9f73749778393da Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 24 Jul 2020 13:52:03 +0400 Subject: [PATCH 043/411] Fix local variable naming --- src/Access/LDAPAccessStorage.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index ab705ab303c..b67459aa322 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -28,17 +28,17 @@ void LDAPAccessStorage::setConfiguration(const Poco::Util::AbstractConfiguration if (!has_server) throw Exception("Missing 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); - const auto ldap_server_ = config.getString("server"); - const auto user_template_ = (has_user_template ? config.getString("user_template") : "default"); + const auto ldap_server_cfg = config.getString("server"); + const auto user_template_cfg = (has_user_template ? config.getString("user_template") : "default"); - if (ldap_server_.empty()) + if (ldap_server_cfg.empty()) throw Exception("Empty 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); - if (user_template_.empty()) + if (user_template_cfg.empty()) throw Exception("Empty 'user_template' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); - ldap_server = ldap_server_; - user_template = user_template_; + ldap_server = ldap_server_cfg; + user_template = user_template_cfg; top_enclosing_storage = top_enclosing_storage_; } From 6c1643d3c54e903d9e5c400ccfbc4dbf1b87bba1 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 24 Jul 2020 14:11:00 +0400 Subject: [PATCH 044/411] Remove unneeded logging --- src/Access/LDAPAccessStorage.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index b67459aa322..16d985064a7 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -68,16 +68,15 @@ std::optional LDAPAccessStorage::findImpl(EntityType type, const String & return id; if (!isConfiguredNoLock()) - { - LOG_WARNING(getLogger(), "Access storage instance is not configured."); return {}; - } // Stop if entity exists anywhere else, to avoid duplicates. if (top_enclosing_storage->find(type, name)) return {}; - // Entity doesn't exist. We are going to create it. Here we retrieve the template first. + // Entity doesn't exist. We are going to create one. + + // Retrieve the template first. const auto user_tmp = top_enclosing_storage->read(user_template); if (!user_tmp) { From bda2eeef17bbf4c1750484b133bf562dccf05efb Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 24 Jul 2020 19:38:33 +0400 Subject: [PATCH 045/411] Fixing timeouts, courtesy of @vzakaznikov --- tests/testflows/helpers/cluster.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py index 9f86d44124c..6d3ae97e000 100644 --- a/tests/testflows/helpers/cluster.py +++ b/tests/testflows/helpers/cluster.py @@ -167,17 +167,20 @@ class Cluster(object): self.docker_compose += f" --project-directory \"{docker_compose_project_dir}\" --file \"{docker_compose_file_path}\"" self.lock = threading.Lock() - def shell(self, node): + def shell(self, node, timeout=120): """Returns unique shell terminal to be used. """ if node is None: return Shell() - return Shell(command=[ + shell = Shell(command=[ "/bin/bash", "--noediting", "-c", f"{self.docker_compose} exec {node} bash --noediting" ], name=node) - def bash(self, node, timeout=60): + shell.timeout = timeout + return shell + + def bash(self, node, timeout=120): """Returns thread-local bash terminal to a specific node. From 90a064c7a62a0c6ddb0a114a11e5f8803d898398 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 24 Jul 2020 19:39:18 +0400 Subject: [PATCH 046/411] Fix compilation --- programs/server/Server.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index bd04dc963be..d579ea40316 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -599,7 +599,7 @@ int Server::main(const std::vector & /*args*/) const bool has_ldap_directory_config = config().has("user_directories.ldap"); if (has_ldap_directory_config) { - auto ldap_directory_config = config().createView("user_directories.ldap"); + auto * ldap_directory_config = config().createView("user_directories.ldap"); if (ldap_directory_config) global_context->getAccessControlManager().setLDAPConfig(*ldap_directory_config); } From 479fa4c3254eb445c85092de4df238453f4abe01 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 27 Jul 2020 14:24:56 +0400 Subject: [PATCH 047/411] Improve LDAP-related comments --- programs/server/config.xml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index b09a53ccf98..d82e3b6a72c 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -217,8 +217,8 @@ - - diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index 06347412da7..9dcb1cdd4f9 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -157,7 +157,6 @@ void AccessControlManager::addUsersConfigStorage(const Poco::Util::AbstractConfi addUsersConfigStorage(UsersConfigAccessStorage::STORAGE_TYPE, users_config_); } - void AccessControlManager::addUsersConfigStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & users_config_) { auto check_setting_name_function = [this](const std::string_view & setting_name) { checkSettingNameIsAllowed(setting_name); }; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index a98eab5a2b0..8285239abb0 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -1,5 +1,7 @@ #include +#include #include +#include #include #include #include @@ -20,39 +22,42 @@ LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_) } -void LDAPAccessStorage::setConfiguration(IAccessStorage * top_enclosing_storage_, const Poco::Util::AbstractConfiguration & config, const String & prefix) +void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix) { + // TODO: switch to passing config as a ConfigurationView and remove this extra prefix once a version of Poco with proper implementation is available. const String prefix_str = (prefix.empty() ? "" : prefix + "."); std::scoped_lock lock(mutex); const bool has_server = config.has(prefix_str + "server"); - const bool has_user_template = config.has(prefix_str + "user_template"); + const bool has_roles = config.has(prefix_str + "roles"); if (!has_server) throw Exception("Missing 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); const auto ldap_server_cfg = config.getString(prefix_str + "server"); - String user_template_cfg; - if (ldap_server_cfg.empty()) throw Exception("Empty 'server' field for LDAP user directory.", ErrorCodes::BAD_ARGUMENTS); - if (has_user_template) - user_template_cfg = config.getString(prefix_str + "user_template"); + std::set roles_cfg; + if (has_roles) + { + Poco::Util::AbstractConfiguration::Keys role_names; + config.keys(prefix_str + "roles", role_names); - if (user_template_cfg.empty()) - user_template_cfg = "default"; + // Currently, we only extract names of roles from the section names and assign them directly and unconditionally. + roles_cfg.insert(role_names.begin(), role_names.end()); + } ldap_server = ldap_server_cfg; - user_template = user_template_cfg; - top_enclosing_storage = top_enclosing_storage_; + roles.swap(roles_cfg); + access_control_manager = access_control_manager_; } bool LDAPAccessStorage::isConfiguredNoLock() const { - return !ldap_server.empty() && !user_template.empty() && top_enclosing_storage; + return !ldap_server.empty() &&/* !roles.empty() &&*/ access_control_manager; } @@ -74,13 +79,6 @@ std::optional LDAPAccessStorage::findImpl(EntityType type, const String & { std::scoped_lock lock(mutex); - // Detect and avoid loops/duplicate creations. - if (helper_lookup_in_progress) - return {}; - - helper_lookup_in_progress = true; - SCOPE_EXIT({ helper_lookup_in_progress = false; }); - // Return the id immediately if we already have it. const auto id = memory_storage.find(type, name); if (id.has_value()) @@ -89,32 +87,39 @@ std::optional LDAPAccessStorage::findImpl(EntityType type, const String & if (!isConfiguredNoLock()) return {}; - // Stop if entity exists anywhere else, to avoid duplicates. - if (top_enclosing_storage->find(type, name)) - return {}; + // Stop if entity exists anywhere else, to avoid generating duplicates. + auto * this_base = dynamic_cast(this); + const auto storages = access_control_manager->getStoragesPtr(); + for (const auto & storage : *storages) + { + if (storage.get() != this_base && storage->find(type, name)) + return {}; + } // Entity doesn't exist. We are going to create one. - - // Retrieve the template first. - std::shared_ptr user_tmp; - try - { - user_tmp = top_enclosing_storage->read(user_template); - if (!user_tmp) - throw Exception("Retrieved user is empty", IAccessEntity::TypeInfo::get(IAccessEntity::Type::USER).not_found_error_code); - } - catch (...) - { - tryLogCurrentException(getLogger(), "Unable to retrieve user template '" + user_template + "' from access storage '" + top_enclosing_storage->getStorageName() + "'"); - return {}; - } - - // Build the new entity based on the existing template. - const auto user = std::make_shared(*user_tmp); + const auto user = std::make_shared(); user->setName(name); user->authentication = Authentication(Authentication::Type::LDAP_SERVER); user->authentication.setServerName(ldap_server); + for (const auto& role_name : roles) { + std::optional role_id; + + try + { + role_id = access_control_manager->find(role_name); + if (!role_id) + throw Exception("Retrieved role info is empty", IAccessEntity::TypeInfo::get(IAccessEntity::Type::ROLE).not_found_error_code); + } + catch (...) + { + tryLogCurrentException(getLogger(), "Unable to retrieve role '" + role_name + "' info from access storage '" + access_control_manager->getStorageName() + "'"); + return {}; + } + + user->granted_roles.grant(role_id.value()); + } + return memory_storage.insert(user); } diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index 90a87eca292..eb8b5cac1bc 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace Poco @@ -16,6 +17,8 @@ namespace Poco namespace DB { +class AccessControlManager; + /// Implementation of IAccessStorage which allows attaching users from a remote LDAP server. /// Currently, any user name will be treated as a name of an existing remote user, /// a user info entity will be created, with LDAP_SERVER authentication type. @@ -27,7 +30,7 @@ public: explicit LDAPAccessStorage(const String & storage_name_ = STORAGE_TYPE); virtual ~LDAPAccessStorage() override = default; - void setConfiguration(IAccessStorage * top_enclosing_storage_, const Poco::Util::AbstractConfiguration & config, const String & prefix = ""); + void setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix = ""); public: // IAccessStorage implementations. virtual const char * getStorageType() const override; @@ -53,9 +56,8 @@ private: mutable std::recursive_mutex mutex; String ldap_server; - String user_template; - IAccessStorage * top_enclosing_storage = nullptr; - mutable bool helper_lookup_in_progress = false; + std::set roles; + AccessControlManager * access_control_manager = nullptr; mutable MemoryAccessStorage memory_storage; }; } From ec52a165af469c0d71fdc9d54773060b74cad21c Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Thu, 20 Aug 2020 12:46:42 +0400 Subject: [PATCH 055/411] Style fixes --- programs/server/config.xml | 2 +- src/Access/LDAPAccessStorage.cpp | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index cdae1c5409e..26ca26087ba 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -262,7 +262,7 @@ with the following parameters: server - one of LDAP server names defined in 'ldap_servers' config section above. This parameter is mandatory and cannot be empty. - roles - section with a list of locally defined roles that will be granted to each user retrieved from the LDAP server. + roles - section with a list of locally defined roles that will be assigned to each user retrieved from the LDAP server. If no roles are specified, user will not be able to perform any actions after authentication. If any of the listed roles is not defined locally at the time of authentication, the authenthication attept will fail as if the provided password was incorrect. diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 8285239abb0..92aef53af64 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -88,7 +88,7 @@ std::optional LDAPAccessStorage::findImpl(EntityType type, const String & return {}; // Stop if entity exists anywhere else, to avoid generating duplicates. - auto * this_base = dynamic_cast(this); + const auto * this_base = dynamic_cast(this); const auto storages = access_control_manager->getStoragesPtr(); for (const auto & storage : *storages) { @@ -102,7 +102,8 @@ std::optional LDAPAccessStorage::findImpl(EntityType type, const String & user->authentication = Authentication(Authentication::Type::LDAP_SERVER); user->authentication.setServerName(ldap_server); - for (const auto& role_name : roles) { + for (const auto& role_name : roles) + { std::optional role_id; try From 1cc9b81cddd3d0ba2cc0a205678e9f858b829652 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 24 Aug 2020 16:02:47 +0400 Subject: [PATCH 056/411] Disallow multiple ldap sections in user_directories section --- src/Access/AccessControlManager.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index 9dcb1cdd4f9..c1a6baf4972 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -24,6 +24,7 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_ELEMENT_IN_CONFIG; + extern const int EXCESSIVE_ELEMENT_IN_CONFIG; extern const int UNKNOWN_SETTING; } @@ -250,7 +251,8 @@ void AccessControlManager::addStoragesFromUserDirectoriesConfig( String prefix = key + "." + key_in_user_directories; String type = key_in_user_directories; - if (size_t bracket_pos = type.find('['); bracket_pos != String::npos) + const size_t bracket_pos = type.find('['); + if (bracket_pos != String::npos) type.resize(bracket_pos); if ((type == "users_xml") || (type == "users_config")) type = UsersConfigAccessStorage::STORAGE_TYPE; @@ -280,6 +282,8 @@ void AccessControlManager::addStoragesFromUserDirectoriesConfig( } else if (type == LDAPAccessStorage::STORAGE_TYPE) { + if (bracket_pos != String::npos) + throw Exception("Duplicate storage type '" + type + "' at " + prefix + " in config", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); addLDAPStorage(name, config, prefix); } else From 5d31442fa88271072a5478b95f4903977cef4fb1 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 26 Aug 2020 03:26:54 +0300 Subject: [PATCH 057/411] Delete perf.py.orig --- .../test/performance-comparison/perf.py.orig | 190 ------------------ 1 file changed, 190 deletions(-) delete mode 100755 docker/test/performance-comparison/perf.py.orig diff --git a/docker/test/performance-comparison/perf.py.orig b/docker/test/performance-comparison/perf.py.orig deleted file mode 100755 index c25a3041a67..00000000000 --- a/docker/test/performance-comparison/perf.py.orig +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/python3 - -import os -import sys -import itertools -import clickhouse_driver -import xml.etree.ElementTree as et -import argparse -import pprint -import string -import time -import traceback - -stage_start_seconds = time.perf_counter() - -def report_stage_end(stage_name): - global stage_start_seconds - print('{}\t{}'.format(stage_name, time.perf_counter() - stage_start_seconds)) - stage_start_seconds = time.perf_counter() - -report_stage_end('start') - -parser = argparse.ArgumentParser(description='Run performance test.') -# Explicitly decode files as UTF-8 because sometimes we have Russian characters in queries, and LANG=C is set. -parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs=1, help='test description file') -parser.add_argument('--host', nargs='*', default=['127.0.0.1', '127.0.0.1'], help="Server hostname. Parallel to '--port'.") -parser.add_argument('--port', nargs='*', default=[9001, 9002], help="Server port. Parallel to '--host'.") -parser.add_argument('--runs', type=int, default=int(os.environ.get('CHPC_RUNS', 7)), help='Number of query runs per server. Defaults to CHPC_RUNS environment variable.') -parser.add_argument('--no-long', type=bool, default=True, help='Skip the tests tagged as long.') -args = parser.parse_args() - -tree = et.parse(args.file[0]) -root = tree.getroot() - -# Skip long tests -for tag in root.findall('.//tag'): - if tag.text == 'long': - print('skipped\tTest is tagged as long.') - sys.exit(0) - -# Check main metric -main_metric_element = root.find('main_metric/*') -if main_metric_element is not None and main_metric_element.tag != 'min_time': - raise Exception('Only the min_time main metric is supported. This test uses \'{}\''.format(main_metric_element.tag)) - -# FIXME another way to detect infinite tests. They should have an appropriate main_metric but sometimes they don't. -infinite_sign = root.find('.//average_speed_not_changing_for_ms') -if infinite_sign is not None: - raise Exception('Looks like the test is infinite (sign 1)') - -# Open connections -servers = [{'host': host, 'port': port} for (host, port) in zip(args.host, args.port)] -connections = [clickhouse_driver.Client(**server) for server in servers] - -for s in servers: - print('server\t{}\t{}'.format(s['host'], s['port'])) - -report_stage_end('connect') - -# Process query parameters -subst_elems = root.findall('substitutions/substitution') -available_parameters = {} # { 'table': ['hits_10m', 'hits_100m'], ... } -for e in subst_elems: - available_parameters[e.find('name').text] = [v.text for v in e.findall('values/value')] - -# Take care to keep the order of queries -- sometimes we have DROP IF EXISTS -# followed by CREATE in create queries section, so the order matters. -def substitute_parameters(query_templates): - result = [] - for q in query_templates: - keys = set(n for _, n, _, _ in string.Formatter().parse(q) if n) - values = [available_parameters[k] for k in keys] - result.extend([ - q.format(**dict(zip(keys, values_combo))) - for values_combo in itertools.product(*values)]) - return result - -report_stage_end('substitute') - -# Run drop queries, ignoring errors. Do this before all other activity, because -# clickhouse_driver disconnects on error (this is not configurable), and the new -# connection loses the changes in settings. -drop_query_templates = [q.text for q in root.findall('drop_query')] -drop_queries = substitute_parameters(drop_query_templates) -for c in connections: - for q in drop_queries: - try: - c.execute(q) - except: - traceback.print_exc() - pass - -report_stage_end('drop1') - -# Apply settings -settings = root.findall('settings/*') -for c in connections: - for s in settings: - c.execute("set {} = '{}'".format(s.tag, s.text)) - -report_stage_end('settings') - -# Check tables that should exist. If they don't exist, just skip this test. -tables = [e.text for e in root.findall('preconditions/table_exists')] -for t in tables: - for c in connections: - try: - res = c.execute("show create table {}".format(t)) - except: - print('skipped\t' + traceback.format_exception_only(*sys.exc_info()[:2])[-1]) - traceback.print_exc() - sys.exit(0) - -report_stage_end('preconditions') - -# Run create queries -create_query_templates = [q.text for q in root.findall('create_query')] -create_queries = substitute_parameters(create_query_templates) -for c in connections: - for q in create_queries: - c.execute(q) - -# Run fill queries -fill_query_templates = [q.text for q in root.findall('fill_query')] -fill_queries = substitute_parameters(fill_query_templates) -for c in connections: - for q in fill_queries: - c.execute(q) - -report_stage_end('fill') - -# Run test queries -def tsv_escape(s): - return s.replace('\\', '\\\\').replace('\t', '\\t').replace('\n', '\\n').replace('\r','') - -test_query_templates = [q.text for q in root.findall('query')] -test_queries = substitute_parameters(test_query_templates) - -report_stage_end('substitute2') - -for q in test_queries: - # Prewarm: run once on both servers. Helps to bring the data into memory, - # precompile the queries, etc. -<<<<<<< HEAD - for conn_index, c in enumerate(connections): - res = c.execute(q, query_id = 'prewarm {} {}'.format(0, q)) - print('prewarm\t' + tsv_escape(q) + '\t' + str(conn_index) + '\t' + str(c.last_query.elapsed)) -======= - try: - for conn_index, c in enumerate(connections): - prewarm_id = f'{query_prefix}.prewarm0' - res = c.execute(q, query_id = prewarm_id) - print(f'prewarm\t{query_index}\t{prewarm_id}\t{conn_index}\t{c.last_query.elapsed}') - except KeyboardInterrupt: - raise - except: - # If prewarm fails for some query -- skip it, and try to test the others. - # This might happen if the new test introduces some function that the - # old server doesn't support. Still, report it as an error. - # FIXME the driver reconnects on error and we lose settings, so this might - # lead to further errors or unexpected behavior. - print(traceback.format_exc(), file=sys.stderr) - continue ->>>>>>> 4b1bb43543... Merge pull request #11076 from ClickHouse/aku/join-error-messages - - # Now, perform measured runs. - # Track the time spent by the client to process this query, so that we can notice - # out the queries that take long to process on the client side, e.g. by sending - # excessive data. - start_seconds = time.perf_counter() - server_seconds = 0 - for run in range(0, args.runs): - for conn_index, c in enumerate(connections): - res = c.execute(q) - print('query\t' + tsv_escape(q) + '\t' + str(run) + '\t' + str(conn_index) + '\t' + str(c.last_query.elapsed)) - server_seconds += c.last_query.elapsed - - client_seconds = time.perf_counter() - start_seconds - print('client-time\t{}\t{}\t{}'.format(tsv_escape(q), client_seconds, server_seconds)) - -report_stage_end('benchmark') - -# Run drop queries -drop_query_templates = [q.text for q in root.findall('drop_query')] -drop_queries = substitute_parameters(drop_query_templates) -for c in connections: - for q in drop_queries: - c.execute(q) - -report_stage_end('drop2') From 1768db2c2f1c3f3342526f38399c058500823144 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Wed, 26 Aug 2020 22:09:26 +0400 Subject: [PATCH 058/411] Add findOrGenerate() to IAccessStorage interface (falls back to find() by default) Use findOrGenerate() when setting user in Context (authentication path) Implement findOrGenerateImpl() and findImpl() in LDAPAccessStorage --- src/Access/IAccessStorage.cpp | 12 ++++++++++++ src/Access/IAccessStorage.h | 7 +++++++ src/Access/LDAPAccessStorage.cpp | 6 ++++++ src/Access/LDAPAccessStorage.h | 1 + src/Interpreters/Context.cpp | 2 +- 5 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 874ae612034..f60a50dc9c3 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -151,6 +151,12 @@ std::vector IAccessStorage::find(EntityType type, const Strings & names) c } +std::optional IAccessStorage::findOrGenerate(EntityType type, const String & name) const +{ + return findOrGenerateImpl(type, name); +} + + UUID IAccessStorage::getID(EntityType type, const String & name) const { auto id = findImpl(type, name); @@ -430,6 +436,12 @@ Poco::Logger * IAccessStorage::getLogger() const } +std::optional IAccessStorage::findOrGenerateImpl(EntityType type, const String & name) const +{ + return findImpl(type, name); +} + + void IAccessStorage::throwNotFound(const UUID & id) const { throw Exception(outputID(id) + " not found in " + getStorageName(), ErrorCodes::ACCESS_ENTITY_NOT_FOUND); diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 7851f8c9b6b..f6f80dce000 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -48,6 +48,12 @@ public: template std::vector find(const Strings & names) const { return find(EntityClassT::TYPE, names); } + /// Searches for an entity with specified type and name. Returns std::nullopt if not found and cannot be generated. + std::optional findOrGenerate(EntityType type, const String & name) const; + + template + std::optional findOrGenerate(const String & name) const { return findOrGenerate(EntityClassT::TYPE, name); } + /// Searches for an entity with specified name and type. Throws an exception if not found. UUID getID(EntityType type, const String & name) const; @@ -139,6 +145,7 @@ public: protected: virtual std::optional findImpl(EntityType type, const String & name) const = 0; + virtual std::optional findOrGenerateImpl(EntityType type, const String & name) const; virtual std::vector findAllImpl(EntityType type) const = 0; virtual bool existsImpl(const UUID & id) const = 0; virtual AccessEntityPtr readImpl(const UUID & id) const = 0; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 92aef53af64..0df0e7d852b 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -74,6 +74,12 @@ bool LDAPAccessStorage::isStorageReadOnly() const std::optional LDAPAccessStorage::findImpl(EntityType type, const String & name) const +{ + return memory_storage.find(type, name); +} + + +std::optional LDAPAccessStorage::findOrGenerateImpl(EntityType type, const String & name) const { if (type == EntityType::USER) { diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index eb8b5cac1bc..e570d7a84a8 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -38,6 +38,7 @@ public: // IAccessStorage implementations. private: // IAccessStorage implementations. virtual std::optional findImpl(EntityType type, const String & name) const override; + virtual std::optional findOrGenerateImpl(EntityType type, const String & name) const override; virtual std::vector findAllImpl(EntityType type) const override; virtual bool existsImpl(const UUID & id) const override; virtual AccessEntityPtr readImpl(const UUID & id) const override; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 70cf41a679c..9c4167e6caf 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -673,7 +673,7 @@ void Context::setUser(const String & name, const String & password, const Poco:: client_info.current_password = password; #endif - auto new_user_id = getAccessControlManager().find(name); + auto new_user_id = getAccessControlManager().findOrGenerate(name); std::shared_ptr new_access; if (new_user_id) { From f7e93531f840eb1a044c40851290df20e0765f57 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 26 Aug 2020 22:53:49 +0300 Subject: [PATCH 059/411] Simplify init script --- debian/clickhouse-server.init | 69 ++--------------------------------- 1 file changed, 3 insertions(+), 66 deletions(-) diff --git a/debian/clickhouse-server.init b/debian/clickhouse-server.init index b82c70bd6e0..978feb10337 100755 --- a/debian/clickhouse-server.init +++ b/debian/clickhouse-server.init @@ -160,82 +160,19 @@ initdb() start() { - [ -x $CLICKHOUSE_BINDIR/$PROGRAM ] || exit 0 - local EXIT_STATUS - EXIT_STATUS=0 - - echo -n "Start $PROGRAM service: " - - if is_running; then - echo -n "already running " - EXIT_STATUS=1 - else - ulimit -n 262144 - mkdir -p $CLICKHOUSE_PIDDIR - chown -R $CLICKHOUSE_USER:$CLICKHOUSE_GROUP $CLICKHOUSE_PIDDIR - initdb - if ! is_running; then - # Lock should not be held while running child process, so we release the lock. Note: obviously, there is race condition. - # But clickhouse-server has protection from simultaneous runs with same data directory. - su -s $SHELL ${CLICKHOUSE_USER} -c "$FLOCK -u 9; $CLICKHOUSE_PROGRAM_ENV exec -a \"$PROGRAM\" \"$CLICKHOUSE_BINDIR/$PROGRAM\" --daemon --pid-file=\"$CLICKHOUSE_PIDFILE\" --config-file=\"$CLICKHOUSE_CONFIG\"" - EXIT_STATUS=$? - if [ $EXIT_STATUS -ne 0 ]; then - return $EXIT_STATUS - fi - fi - fi - - if [ $EXIT_STATUS -eq 0 ]; then - attempts=0 - while ! is_running && [ $attempts -le ${CLICKHOUSE_START_TIMEOUT:=10} ]; do - attempts=$(($attempts + 1)) - sleep 1 - done - if is_running; then - echo "DONE" - else - echo "UNKNOWN" - fi - else - echo "FAILED" - fi - - return $EXIT_STATUS + ${CLICKHOUSE_GENERIC_PROGRAM} start --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}/${PROGRAM}" } stop() { - #local EXIT_STATUS - EXIT_STATUS=0 - - if [ -f $CLICKHOUSE_PIDFILE ]; then - - echo -n "Stop $PROGRAM service: " - - kill -TERM $(cat "$CLICKHOUSE_PIDFILE") - - if ! wait_for_done ${CLICKHOUSE_STOP_TIMEOUT}; then - EXIT_STATUS=2 - echo "TIMEOUT" - else - echo "DONE" - fi - - fi - return $EXIT_STATUS + ${CLICKHOUSE_GENERIC_PROGRAM} stop --pid-path "${CLICKHOUSE_PIDDIR}" } restart() { - check_config - if stop; then - if start; then - return 0 - fi - fi - return 1 + ${CLICKHOUSE_GENERIC_PROGRAM} restart --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}/${PROGRAM}" } From 3d6e56cd613229f635539fefaff7bf29591a4ade Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Thu, 27 Aug 2020 00:34:33 +0400 Subject: [PATCH 060/411] Maintain the list and update role changes in cached users --- src/Access/LDAPAccessStorage.cpp | 56 ++++++++++++++++++++++++++++++++ src/Access/LDAPAccessStorage.h | 4 +++ 2 files changed, 60 insertions(+) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 0df0e7d852b..7ec838c56f9 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -52,6 +52,13 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m ldap_server = ldap_server_cfg; roles.swap(roles_cfg); access_control_manager = access_control_manager_; + role_change_subscription = access_control_manager->subscribeForChanges( + [this] (const UUID & id, const AccessEntityPtr & entity) + { + return this->processRoleChange(id, entity); + } + ); + roles_of_interest.clear(); } @@ -61,6 +68,54 @@ bool LDAPAccessStorage::isConfiguredNoLock() const } +void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr & entity) +{ + auto role_ptr = typeid_cast>(entity); + if (role_ptr) + { + if (roles.find(role_ptr->getName()) != roles.end()) + { + auto update_func = [&id](const AccessEntityPtr & cached_entity) -> AccessEntityPtr + { + auto user_ptr = typeid_cast>(cached_entity); + if (user_ptr && !user_ptr->granted_roles.roles.contains(id)) + { + auto clone = user_ptr->clone(); + auto user_clone_ptr = typeid_cast>(clone); + user_clone_ptr->granted_roles.grant(id); + return user_clone_ptr; + } + return cached_entity; + }; + + memory_storage.update(memory_storage.findAll(), update_func); + roles_of_interest.insert(id); + } + } + else + { + if (roles_of_interest.find(id) != roles_of_interest.end()) + { + auto update_func = [&id](const AccessEntityPtr & cached_entity) -> AccessEntityPtr + { + auto user_ptr = typeid_cast>(cached_entity); + if (user_ptr && user_ptr->granted_roles.roles.contains(id)) + { + auto clone = user_ptr->clone(); + auto user_clone_ptr = typeid_cast>(clone); + user_clone_ptr->granted_roles.revoke(id); + return user_clone_ptr; + } + return cached_entity; + }; + + memory_storage.update(memory_storage.findAll(), update_func); + roles_of_interest.erase(id); + } + } +} + + const char * LDAPAccessStorage::getStorageType() const { return STORAGE_TYPE; @@ -124,6 +179,7 @@ std::optional LDAPAccessStorage::findOrGenerateImpl(EntityType type, const return {}; } + roles_of_interest.insert(role_id.value()); user->granted_roles.grant(role_id.value()); } diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index e570d7a84a8..434033bc3c6 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -54,11 +55,14 @@ private: // IAccessStorage implementations. private: bool isConfiguredNoLock() const; + void processRoleChange(const UUID & id, const AccessEntityPtr & entity); mutable std::recursive_mutex mutex; String ldap_server; std::set roles; AccessControlManager * access_control_manager = nullptr; + ext::scope_guard role_change_subscription; + mutable std::set roles_of_interest; mutable MemoryAccessStorage memory_storage; }; } From a4a612843160840b73e4ab5c546ee11a69e52eca Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 26 Aug 2020 23:45:24 +0300 Subject: [PATCH 061/411] Trigger CI --- src/Dictionaries/FileDictionarySource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dictionaries/FileDictionarySource.cpp b/src/Dictionaries/FileDictionarySource.cpp index 18893a99f4e..402fb876c09 100644 --- a/src/Dictionaries/FileDictionarySource.cpp +++ b/src/Dictionaries/FileDictionarySource.cpp @@ -32,7 +32,7 @@ FileDictionarySource::FileDictionarySource( { const String user_files_path = context.getUserFilesPath(); if (!startsWith(filepath, user_files_path)) - throw Exception("File path " + filepath + " is not inside " + user_files_path, ErrorCodes::PATH_ACCESS_DENIED); + throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", filepath, user_files_path); } } From c72765187b5d9f154fcffd4f3f254a75f176a5a0 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Thu, 27 Aug 2020 12:36:31 +0400 Subject: [PATCH 062/411] GCC 9 compilation fix --- src/Access/LDAPAccessStorage.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 7ec838c56f9..afd3d6a2eb0 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -78,7 +78,7 @@ void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr auto update_func = [&id](const AccessEntityPtr & cached_entity) -> AccessEntityPtr { auto user_ptr = typeid_cast>(cached_entity); - if (user_ptr && !user_ptr->granted_roles.roles.contains(id)) + if (user_ptr && user_ptr->granted_roles.roles.find(id) == user_ptr->granted_roles.roles.end()) { auto clone = user_ptr->clone(); auto user_clone_ptr = typeid_cast>(clone); @@ -99,7 +99,7 @@ void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr auto update_func = [&id](const AccessEntityPtr & cached_entity) -> AccessEntityPtr { auto user_ptr = typeid_cast>(cached_entity); - if (user_ptr && user_ptr->granted_roles.roles.contains(id)) + if (user_ptr && user_ptr->granted_roles.roles.find(id) != user_ptr->granted_roles.roles.end()) { auto clone = user_ptr->clone(); auto user_clone_ptr = typeid_cast>(clone); From fc1b112e2b3a4cf03af7206c58831fec582b37d2 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 27 Aug 2020 21:50:13 +0300 Subject: [PATCH 063/411] Update clickhouse-server.init --- debian/clickhouse-server.init | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/clickhouse-server.init b/debian/clickhouse-server.init index 978feb10337..4764bca768b 100755 --- a/debian/clickhouse-server.init +++ b/debian/clickhouse-server.init @@ -160,7 +160,7 @@ initdb() start() { - ${CLICKHOUSE_GENERIC_PROGRAM} start --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}/${PROGRAM}" + ${CLICKHOUSE_GENERIC_PROGRAM} start --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" } @@ -172,7 +172,7 @@ stop() restart() { - ${CLICKHOUSE_GENERIC_PROGRAM} restart --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}/${PROGRAM}" + ${CLICKHOUSE_GENERIC_PROGRAM} restart --user "${CLICKHOUSE_USER}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" } From bce8166420a289feebd14adc8fc3fa831d47da45 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 27 Aug 2020 21:51:19 +0300 Subject: [PATCH 064/411] Update FileDictionarySource.cpp --- src/Dictionaries/FileDictionarySource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dictionaries/FileDictionarySource.cpp b/src/Dictionaries/FileDictionarySource.cpp index 402fb876c09..82aea4cbb98 100644 --- a/src/Dictionaries/FileDictionarySource.cpp +++ b/src/Dictionaries/FileDictionarySource.cpp @@ -60,7 +60,7 @@ BlockInputStreamPtr FileDictionarySource::loadAll() std::string FileDictionarySource::toString() const { - return "File: " + filepath + ' ' + format; + return fmt::format("File: {}, {}", filepath, format); } From f91d57adacf7827b53f56593a83b69e3fae0b92c Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 28 Aug 2020 12:06:06 +0400 Subject: [PATCH 065/411] Adjust naming --- src/Access/LDAPAccessStorage.cpp | 10 +++++----- src/Access/LDAPAccessStorage.h | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index afd3d6a2eb0..59f061c51a5 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -49,16 +49,16 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m roles_cfg.insert(role_names.begin(), role_names.end()); } - ldap_server = ldap_server_cfg; - roles.swap(roles_cfg); access_control_manager = access_control_manager_; + ldap_server = ldap_server_cfg; + default_role_names.swap(roles_cfg); + roles_of_interest.clear(); role_change_subscription = access_control_manager->subscribeForChanges( [this] (const UUID & id, const AccessEntityPtr & entity) { return this->processRoleChange(id, entity); } ); - roles_of_interest.clear(); } @@ -73,7 +73,7 @@ void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr auto role_ptr = typeid_cast>(entity); if (role_ptr) { - if (roles.find(role_ptr->getName()) != roles.end()) + if (default_role_names.find(role_ptr->getName()) != default_role_names.end()) { auto update_func = [&id](const AccessEntityPtr & cached_entity) -> AccessEntityPtr { @@ -163,7 +163,7 @@ std::optional LDAPAccessStorage::findOrGenerateImpl(EntityType type, const user->authentication = Authentication(Authentication::Type::LDAP_SERVER); user->authentication.setServerName(ldap_server); - for (const auto& role_name : roles) + for (const auto& role_name : default_role_names) { std::optional role_id; diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index 434033bc3c6..d52056fe947 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -58,11 +58,11 @@ private: void processRoleChange(const UUID & id, const AccessEntityPtr & entity); mutable std::recursive_mutex mutex; - String ldap_server; - std::set roles; AccessControlManager * access_control_manager = nullptr; - ext::scope_guard role_change_subscription; + String ldap_server; + std::set default_role_names; mutable std::set roles_of_interest; + ext::scope_guard role_change_subscription; mutable MemoryAccessStorage memory_storage; }; } From 7ffb618f6eb487ef282a9acf0595c3b7eda4b653 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 28 Aug 2020 16:05:08 +0400 Subject: [PATCH 066/411] Add missing proper findOrGenerateImpl() implementation to MultipleAccessStorage class --- src/Access/MultipleAccessStorage.cpp | 17 +++++++++++++++++ src/Access/MultipleAccessStorage.h | 1 + 2 files changed, 18 insertions(+) diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index bf711b54d54..7557649601b 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -104,6 +104,23 @@ std::optional MultipleAccessStorage::findImpl(EntityType type, const Strin } +std::optional MultipleAccessStorage::findOrGenerateImpl(EntityType type, const String & name) const +{ + auto storages = getStoragesInternal(); + for (const auto & storage : *storages) + { + auto id = storage->findOrGenerate(type, name); + if (id) + { + std::lock_guard lock{mutex}; + ids_cache.set(*id, storage); + return id; + } + } + return {}; +} + + std::vector MultipleAccessStorage::findAllImpl(EntityType type) const { std::vector all_ids; diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index 5d01894621f..cf56d71899e 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -35,6 +35,7 @@ public: protected: std::optional findImpl(EntityType type, const String & name) const override; + std::optional findOrGenerateImpl(EntityType type, const String & name) const override; std::vector findAllImpl(EntityType type) const override; bool existsImpl(const UUID & id) const override; AccessEntityPtr readImpl(const UUID & id) const override; From b147ffcd437d7eaa66567fd4f77358447dded8db Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 16 Jun 2020 12:22:55 +0300 Subject: [PATCH 067/411] encrypt, aes_encrypt_mysql, decrypt, aes_decrypt_mysql functions Functions to encrypt/decrypt any input data with OpenSSL's ciphers with custom key, iv, and add (-gcm mode only). _mysql versions are 100% compatitable with corresponding MySQL functions Supported modes depend on OpenSSL version, but generally are: aes-{128,192,56}-{ecb,cbc,cfb1,cfb8,cfb128,ofb,gcm} Please note that in a -gcm mode a 16-byte tag is appended to the ciphertext on encryption and is expected to be found at the end of ciphertext on decryption. Added tests that verify compatibility with MySQL functions, and test vectors for GCM mode from OpenSSL. Added masking rules for aes_X funtions Rules are installed by default to config.d/query_masking_rules.xml --- base/common/defines.h | 2 + cmake/sanitize.cmake | 4 +- debian/clickhouse-server.install | 1 + programs/server/CMakeLists.txt | 2 + .../server/config.d/query_masking_rules.xml | 9 + src/Columns/ColumnString.cpp | 6 + src/Columns/ColumnString.h | 3 + src/Functions/FunctionHelpers.cpp | 2 +- src/Functions/FunctionsAES.cpp | 56 ++ src/Functions/FunctionsAES.h | 656 ++++++++++++++++++ src/Functions/aes_decrypt_mysql.cpp | 29 + src/Functions/aes_encrypt_mysql.cpp | 29 + src/Functions/decrypt.cpp | 29 + src/Functions/encrypt.cpp | 29 + src/Functions/registerFunctions.cpp | 19 + src/Functions/ya.make | 5 + .../0_stateless/01318_decrypt.reference | 80 +++ tests/queries/0_stateless/01318_decrypt.sql | 123 ++++ .../0_stateless/01318_encrypt.reference | 86 +++ tests/queries/0_stateless/01318_encrypt.sql | 125 ++++ tests/ubsan_suppressions.txt | 3 + 21 files changed, 1295 insertions(+), 3 deletions(-) create mode 100644 src/Functions/FunctionsAES.cpp create mode 100644 src/Functions/FunctionsAES.h create mode 100644 src/Functions/aes_decrypt_mysql.cpp create mode 100644 src/Functions/aes_encrypt_mysql.cpp create mode 100644 src/Functions/decrypt.cpp create mode 100644 src/Functions/encrypt.cpp create mode 100644 tests/queries/0_stateless/01318_decrypt.reference create mode 100644 tests/queries/0_stateless/01318_decrypt.sql create mode 100644 tests/queries/0_stateless/01318_encrypt.reference create mode 100644 tests/queries/0_stateless/01318_encrypt.sql create mode 100644 tests/ubsan_suppressions.txt diff --git a/base/common/defines.h b/base/common/defines.h index af5981023ff..722a1f206bf 100644 --- a/base/common/defines.h +++ b/base/common/defines.h @@ -70,10 +70,12 @@ # define NO_SANITIZE_UNDEFINED __attribute__((__no_sanitize__("undefined"))) # define NO_SANITIZE_ADDRESS __attribute__((__no_sanitize__("address"))) # define NO_SANITIZE_THREAD __attribute__((__no_sanitize__("thread"))) +# define NO_SANITIZE_MEMORY __attribute__((__no_sanitize__("memory"))) #else /// It does not work in GCC. GCC 7 cannot recognize this attribute and GCC 8 simply ignores it. # define NO_SANITIZE_UNDEFINED # define NO_SANITIZE_ADDRESS # define NO_SANITIZE_THREAD +# define NO_SANITIZE_MEMORY #endif #if defined __GNUC__ && !defined __clang__ diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index 32443ed78c3..043268ceba5 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -48,8 +48,8 @@ if (SANITIZE) endif () elseif (SANITIZE STREQUAL "undefined") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/ubsan_suppressions.txt") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/ubsan_suppressions.txt") if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined") endif() diff --git a/debian/clickhouse-server.install b/debian/clickhouse-server.install index b1475fdf162..57f858407af 100644 --- a/debian/clickhouse-server.install +++ b/debian/clickhouse-server.install @@ -2,5 +2,6 @@ usr/bin/clickhouse-server usr/bin/clickhouse-copier usr/bin/clickhouse-report etc/clickhouse-server/config.xml +etc/clickhouse-server/config.d/*.xml etc/clickhouse-server/users.xml etc/systemd/system/clickhouse-server.service diff --git a/programs/server/CMakeLists.txt b/programs/server/CMakeLists.txt index 5500a4680b7..adaddcf1762 100644 --- a/programs/server/CMakeLists.txt +++ b/programs/server/CMakeLists.txt @@ -29,6 +29,8 @@ set (CLICKHOUSE_SERVER_LINK clickhouse_program_add(server) install(FILES config.xml users.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server COMPONENT clickhouse) +install(FILES config.xml users.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server COMPONENT clickhouse) +install(FILES config.d/query_masking_rules.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server/config.d COMPONENT clickhouse) # TODO We actually need this on Mac, FreeBSD. if (OS_LINUX) diff --git a/programs/server/config.d/query_masking_rules.xml b/programs/server/config.d/query_masking_rules.xml index f919523472c..9ba77198d88 100644 --- a/programs/server/config.d/query_masking_rules.xml +++ b/programs/server/config.d/query_masking_rules.xml @@ -15,5 +15,14 @@ TOPSECRET.TOPSECRET [hidden] + + + hide encrypt/decrypt arguments + ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) + + \1(???) + diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 6c84107caae..d38008c44c7 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -612,4 +612,10 @@ void ColumnString::protect() getOffsets().protect(); } +void ColumnString::validate() const +{ + if (!offsets.empty() && offsets.back() != chars.size()) + throw Exception(fmt::format("ColumnString validation failed: size mismatch (internal logical error) {} != {}", offsets.back(), chars.size()), ErrorCodes::LOGICAL_ERROR); +} + } diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index 128e1efe146..19398e07b83 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -267,6 +267,9 @@ public: Offsets & getOffsets() { return offsets; } const Offsets & getOffsets() const { return offsets; } + + // Throws an exception if offsets/chars are messed up + void validate() const; }; diff --git a/src/Functions/FunctionHelpers.cpp b/src/Functions/FunctionHelpers.cpp index 18e5fde5462..91c9d67ea2b 100644 --- a/src/Functions/FunctionHelpers.cpp +++ b/src/Functions/FunctionHelpers.cpp @@ -137,7 +137,7 @@ void validateArgumentsImpl(const IFunction & func, const auto & arg = arguments[i + argument_offset]; const auto descriptor = descriptors[i]; if (int error_code = descriptor.isValid(arg.type, arg.column); error_code != 0) - throw Exception("Illegal type of argument #" + std::to_string(i) + throw Exception("Illegal type of argument #" + std::to_string(argument_offset + i + 1) // +1 is for human-friendly 1-based indexing + (descriptor.argument_name ? " '" + std::string(descriptor.argument_name) + "'" : String{}) + " of function " + func.getName() + (descriptor.expected_type_description ? String(", expected ") + descriptor.expected_type_description : String{}) diff --git a/src/Functions/FunctionsAES.cpp b/src/Functions/FunctionsAES.cpp new file mode 100644 index 00000000000..48533be054a --- /dev/null +++ b/src/Functions/FunctionsAES.cpp @@ -0,0 +1,56 @@ +#include + +#if USE_SSL + +#include + +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int OPENSSL_ERROR; +} +} + +namespace OpenSSLDetails +{ +void onError(std::string error_message) +{ + error_message += ". OpenSSL error code: " + std::to_string(ERR_get_error()); + throw DB::Exception(error_message, DB::ErrorCodes::OPENSSL_ERROR); +} + +StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array & folded_key) +{ + memcpy(folded_key.data(), key.data, cipher_key_size); + + for (size_t i = cipher_key_size; i < key.size; ++i) + { + folded_key[i % cipher_key_size] ^= key.data[i]; + } + + return StringRef(folded_key.data(), cipher_key_size); +} + +const EVP_CIPHER * getCipherByName(const StringRef & cipher_name) +{ + const auto *evp_cipher = EVP_get_cipherbyname(cipher_name.data); + if (evp_cipher == nullptr) + { + // For some reasons following ciphers can't be found by name. + if (cipher_name == "aes-128-cfb128") + evp_cipher = EVP_aes_128_cfb128(); + else if (cipher_name == "aes-192-cfb128") + evp_cipher = EVP_aes_192_cfb128(); + else if (cipher_name == "aes-256-cfb128") + evp_cipher = EVP_aes_256_cfb128(); + } + + return evp_cipher; +} + +} + +#endif diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h new file mode 100644 index 00000000000..8d062e9b12d --- /dev/null +++ b/src/Functions/FunctionsAES.h @@ -0,0 +1,656 @@ +#pragma once + +#include + +#if USE_SSL +#include +#include +#include +#include +#include + +#include + +#include + +#include + +#include +#include +#include +#include + +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} +} + +namespace OpenSSLDetails +{ +[[noreturn]] void onError(std::string error_message); +StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array & folded_key); +const EVP_CIPHER * getCipherByName(const StringRef & name); + +enum class CompatibilityMode +{ + MySQL, + OpenSSL +}; + +enum class CipherMode +{ + MySQLCompatibility, // with key folding + OpenSSLCompatibility, // just as regular openssl's enc application does (AEAD modes, like GCM and CCM are not supported) + RFC5116_AEAD_AES_GCM // AEAD GCM with custom IV length and tag (HMAC) appended to the ciphertext, see https://tools.ietf.org/html/rfc5116#section-5.1 +}; + + +template +struct KeyHolder +{ + inline StringRef setKey(size_t cipher_key_size, const StringRef & key) const + { + if (key.size != cipher_key_size) + throw DB::Exception(fmt::format("Invalid key size: {} expected {}", key.size, cipher_key_size), + DB::ErrorCodes::BAD_ARGUMENTS); + + return key; + } +}; + +template <> +struct KeyHolder +{ + inline StringRef setKey(size_t cipher_key_size, const StringRef & key) + { + if (key.size < cipher_key_size) + throw DB::Exception(fmt::format("Invalid key size: {} expected {}", key.size, cipher_key_size), + DB::ErrorCodes::BAD_ARGUMENTS); + + // MySQL does something fancy with the keys that are too long, + // ruining compatibility with OpenSSL and not improving security. + // But we have to do the same to be compatitable with MySQL. + // see https://github.com/mysql/mysql-server/blob/8.0/router/src/harness/src/my_aes_openssl.cc#L71 + // (my_aes_create_key function) + return foldEncryptionKeyInMySQLCompatitableMode(cipher_key_size, key, folded_key); + } + + ~KeyHolder() + { + OPENSSL_cleanse(folded_key.data(), folded_key.size()); + } + +private: + std::array folded_key; +}; + +template +inline void validateCipherMode(const EVP_CIPHER * evp_cipher) +{ + if constexpr (compatibility_mode == CompatibilityMode::MySQL) + { + switch (EVP_CIPHER_mode(evp_cipher)) + { + case EVP_CIPH_ECB_MODE: [[fallthrough]]; + case EVP_CIPH_CBC_MODE: [[fallthrough]]; + case EVP_CIPH_CFB_MODE: [[fallthrough]]; + case EVP_CIPH_OFB_MODE: + return; + } + } + else if constexpr (compatibility_mode == CompatibilityMode::OpenSSL) + { + switch (EVP_CIPHER_mode(evp_cipher)) + { + case EVP_CIPH_ECB_MODE: [[fallthrough]]; + case EVP_CIPH_CBC_MODE: [[fallthrough]]; + case EVP_CIPH_CFB_MODE: [[fallthrough]]; + case EVP_CIPH_OFB_MODE: [[fallthrough]]; + case EVP_CIPH_CTR_MODE: [[fallthrough]]; + case EVP_CIPH_GCM_MODE: + return; + } + } + + throw DB::Exception("Unsupported cipher mode " + std::string(EVP_CIPHER_name(evp_cipher)), DB::ErrorCodes::BAD_ARGUMENTS); +} + +template +inline void validateIV(const StringRef & iv_value, const size_t cipher_iv_size) +{ + // In MySQL mode we don't care if IV is longer than expected, only if shorter. + if ((mode == CipherMode::MySQLCompatibility && iv_value.size != 0 && iv_value.size < cipher_iv_size) + || (mode == CipherMode::OpenSSLCompatibility && iv_value.size != 0 && iv_value.size != cipher_iv_size)) + throw DB::Exception(fmt::format("Invalid IV size: {} expected {}", iv_value.size, cipher_iv_size), + DB::ErrorCodes::BAD_ARGUMENTS); +} + +} + +namespace DB +{ +template +class FunctionEncrypt : public IFunction +{ +public: + static constexpr OpenSSLDetails::CompatibilityMode compatibility_mode = Impl::compatibility_mode; + static constexpr auto name = Impl::name; + static FunctionPtr create(const Context &) { return std::make_shared(); } + +private: + using CipherMode = OpenSSLDetails::CipherMode; + + String getName() const override { return name; } + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + auto optional_args = FunctionArgumentDescriptors{ + {"IV", isStringOrFixedString, nullptr, "Initialization vector binary string"}, + }; + + if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::OpenSSL) + { + optional_args.emplace_back(FunctionArgumentDescriptor{ + "AAD", isStringOrFixedString, nullptr, "Additional authenticated data binary string for GCM mode" + }); + } + + validateFunctionArgumentTypes(*this, arguments, + FunctionArgumentDescriptors{ + {"mode", isStringOrFixedString, isColumnConst, "encryption mode string"}, + {"input", nullptr, nullptr, "plaintext"}, + {"key", isStringOrFixedString, nullptr, "encryption key binary string"}, + }, + optional_args + ); + + return std::make_shared(); + } + + void executeImplDryRun(Block & block, const ColumnNumbers & /*arguments*/, size_t result, size_t /*input_rows_count*/) const override + { + block.getByPosition(result).column = block.getByPosition(result).type->createColumn(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + { + using namespace OpenSSLDetails; + + const auto mode = block.getByPosition(arguments[0]).column->getDataAt(0); + + if (mode.size == 0 || !std::string_view(mode).starts_with("aes-")) + throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); + + const auto * evp_cipher = getCipherByName(mode); + if (evp_cipher == nullptr) + throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); + + const auto cipher_mode = EVP_CIPHER_mode(evp_cipher); + + const auto input_column = block.getByPosition(arguments[1]).column; + const auto key_column = block.getByPosition(arguments[2]).column; + + OpenSSLDetails::validateCipherMode(evp_cipher); + + ColumnPtr result_column; + if (arguments.size() <= 3) + result_column = doEncrypt(evp_cipher, input_rows_count, input_column, key_column, nullptr, nullptr); + else + { + const auto iv_column = block.getByPosition(arguments[3]).column; + if (compatibility_mode != OpenSSLDetails::CompatibilityMode::MySQL && EVP_CIPHER_iv_length(evp_cipher) == 0) + throw Exception(mode.toString() + " does not support IV", ErrorCodes::BAD_ARGUMENTS); + + if (arguments.size() <= 4) + { + result_column = doEncrypt(evp_cipher, input_rows_count, input_column, key_column, iv_column, nullptr); + } + else + { + if (cipher_mode != EVP_CIPH_GCM_MODE) + throw Exception("AAD can be only set for GCM-mode", ErrorCodes::BAD_ARGUMENTS); + + const auto aad_column = block.getByPosition(arguments[4]).column; + result_column = doEncrypt(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + } + + block.getByPosition(result).column = std::move(result_column); + } + + template + static ColumnPtr doEncrypt(const EVP_CIPHER * evp_cipher, + size_t input_rows_count, + const InputColumnType & input_column, + const KeyColumnType & key_column, + const IvColumnType & iv_column, + const AadColumnType & aad_column) + { + if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::MySQL) + { + return doEncryptImpl(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + else + { + if (EVP_CIPHER_mode(evp_cipher) == EVP_CIPH_GCM_MODE) + { + return doEncryptImpl(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + else + { + return doEncryptImpl(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + } + + return nullptr; + } + + template + static ColumnPtr doEncryptImpl(const EVP_CIPHER * evp_cipher, + size_t input_rows_count, + const InputColumnType & input_column, + const KeyColumnType & key_column, + [[maybe_unused]] const IvColumnType & iv_column, + [[maybe_unused]] const AadColumnType & aad_column) + { + using namespace OpenSSLDetails; + + auto evp_ctx_ptr = std::unique_ptr(EVP_CIPHER_CTX_new(), &EVP_CIPHER_CTX_free); + auto evp_ctx = evp_ctx_ptr.get(); + + const auto block_size = static_cast(EVP_CIPHER_block_size(evp_cipher)); + const auto key_size = static_cast(EVP_CIPHER_key_length(evp_cipher)); + [[maybe_unused]] const auto iv_size = static_cast(EVP_CIPHER_iv_length(evp_cipher)); + const auto tag_size = 16; // https://tools.ietf.org/html/rfc5116#section-5.1 + + auto encrypted_result_column = ColumnString::create(); + auto & encrypted_result_column_data = encrypted_result_column->getChars(); + auto & encrypted_result_column_offsets = encrypted_result_column->getOffsets(); + + { + size_t resulting_size = 0; + // for modes with block_size > 1, plaintext is padded up to a block_size, + // which may result in allocating to much for block_size = 1. + // That may lead later to reading unallocated data from underlying PaddedPODArray + // due to assumption that it is safe to read up to 15 bytes past end. + const auto pad_to_next_block = block_size == 1 ? 0 : 1; + for (size_t r = 0; r < input_rows_count; ++r) + { + resulting_size += (input_column->getDataAt(r).size / block_size + pad_to_next_block) * block_size + 1; + if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) + resulting_size += tag_size; + } +#if defined(MEMORY_SANITIZER) + encrypted_result_column_data.resize_fill(resulting_size, 0xFF); +#else + encrypted_result_column_data.resize(resulting_size); +#endif + } + + auto encrypted = encrypted_result_column_data.data(); + + KeyHolder key_holder; + + for (size_t r = 0; r < input_rows_count; ++r) + { + const auto key_value = key_holder.setKey(key_size, key_column->getDataAt(r)); + auto iv_value = StringRef{}; + if constexpr (!std::is_same_v>) + { + iv_value = iv_column->getDataAt(r); + } + + const auto input_value = input_column->getDataAt(r); + + // 1: Init CTX + if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) + { + // 1.a.1: Init CTX with custom IV length and optionally with AAD + if (EVP_EncryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr) != 1) + onError("Failed to initialize encryption context with cipher"); + + if (EVP_CIPHER_CTX_ctrl(evp_ctx, EVP_CTRL_AEAD_SET_IVLEN, iv_value.size, nullptr) != 1) + onError("Failed to set custom IV length to " + std::to_string(iv_value.size)); + + if (EVP_EncryptInit_ex(evp_ctx, nullptr, nullptr, + reinterpret_cast(key_value.data), + reinterpret_cast(iv_value.data)) != 1) + onError("Failed to set key and IV"); + + // 1.a.2 Set AAD + if constexpr (!std::is_same_v>) + { + const auto aad_data = aad_column->getDataAt(r); + int tmp_len = 0; + if (aad_data.size != 0 && EVP_EncryptUpdate(evp_ctx, nullptr, &tmp_len, + reinterpret_cast(aad_data.data), aad_data.size) != 1) + onError("Failed to set AAD data"); + } + } + else + { + // 1.b: Init CTX + validateIV(iv_value, iv_size); + + if (EVP_EncryptInit_ex(evp_ctx, evp_cipher, nullptr, + reinterpret_cast(key_value.data), + reinterpret_cast(iv_value.data)) != 1) + onError("Failed to initialize cipher context"); + } + + int output_len = 0; + // 2: Feed the data to the cipher + if (EVP_EncryptUpdate(evp_ctx, + reinterpret_cast(encrypted), &output_len, + reinterpret_cast(input_value.data), static_cast(input_value.size)) != 1) + onError("Failed to encrypt"); + encrypted += output_len; + + // 3: retrieve encrypted data (ciphertext) + if (EVP_EncryptFinal_ex(evp_ctx, + reinterpret_cast(encrypted), &output_len) != 1) + onError("Failed to fetch ciphertext"); + encrypted += output_len; + + // 4: optionally retrieve a tag and append it to the ciphertext (RFC5116): + // https://tools.ietf.org/html/rfc5116#section-5.1 + if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) + { + if (EVP_CIPHER_CTX_ctrl(evp_ctx, EVP_CTRL_AEAD_GET_TAG, tag_size, encrypted) != 1) + onError("Failed to retrieve GCM tag"); + encrypted += tag_size; + } + *encrypted = '\0'; + ++encrypted; + + encrypted_result_column_offsets.push_back(encrypted - encrypted_result_column_data.data()); + + if (EVP_CIPHER_CTX_reset(evp_ctx) != 1) + onError("Failed to reset context"); + } + + // in case of block size of 1, we overestimate buffer required for encrypted data, fix it up. + if (!encrypted_result_column_offsets.empty() && encrypted_result_column_data.size() > encrypted_result_column_offsets.back()) + { + encrypted_result_column_data.resize(encrypted_result_column_offsets.back()); + } + + encrypted_result_column->validate(); + return encrypted_result_column; + } +}; + + +/// AES_decrypt(string, key, block_mode[, init_vector]) +template +class FunctionDecrypt : public IFunction +{ +public: + static constexpr OpenSSLDetails::CompatibilityMode compatibility_mode = Impl::compatibility_mode; + static constexpr auto name = Impl::name; + static FunctionPtr create(const Context &) { return std::make_shared(); } + +private: + using CipherMode = OpenSSLDetails::CipherMode; + + String getName() const override { return name; } + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + auto optional_args = FunctionArgumentDescriptors{ + {"IV", isStringOrFixedString, nullptr, "Initialization vector binary string"}, + }; + + if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::OpenSSL) + { + optional_args.emplace_back(FunctionArgumentDescriptor{ + "AAD", isStringOrFixedString, nullptr, "Additional authenticated data binary string for GCM mode" + }); + } + + validateFunctionArgumentTypes(*this, arguments, + FunctionArgumentDescriptors{ + {"mode", isStringOrFixedString, isColumnConst, "decryption mode string"}, + {"input", nullptr, nullptr, "ciphertext"}, + {"key", isStringOrFixedString, nullptr, "decryption key binary string"}, + }, + optional_args + ); + + return std::make_shared(); + } + + void executeImplDryRun(Block & block, const ColumnNumbers & /*arguments*/, size_t result, size_t /*input_rows_count*/) const override + { + block.getByPosition(result).column = block.getByPosition(result).type->createColumn(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + { + using namespace OpenSSLDetails; + + const auto mode = block.getByPosition(arguments[0]).column->getDataAt(0); + + if (mode.size == 0 || !std::string_view(mode).starts_with("aes-")) + throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); + + const auto * evp_cipher = getCipherByName(mode); + if (evp_cipher == nullptr) + throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); + + OpenSSLDetails::validateCipherMode(evp_cipher); + + const auto input_column = block.getByPosition(arguments[1]).column; + const auto key_column = block.getByPosition(arguments[2]).column; + + ColumnPtr result_column; + if (arguments.size() <= 3) + result_column = doDecrypt(evp_cipher, input_rows_count, input_column, key_column, nullptr, nullptr); + else + { + const auto iv_column = block.getByPosition(arguments[3]).column; + if (compatibility_mode != OpenSSLDetails::CompatibilityMode::MySQL && EVP_CIPHER_iv_length(evp_cipher) == 0) + throw Exception(mode.toString() + " does not support IV", ErrorCodes::BAD_ARGUMENTS); + + if (arguments.size() <= 4) + { + result_column = doDecrypt(evp_cipher, input_rows_count, input_column, key_column, iv_column, nullptr); + } + else + { + if (EVP_CIPHER_mode(evp_cipher) != EVP_CIPH_GCM_MODE) + throw Exception("AAD can be only set for GCM-mode", ErrorCodes::BAD_ARGUMENTS); + + const auto aad_column = block.getByPosition(arguments[4]).column; + result_column = doDecrypt(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + } + + block.getByPosition(result).column = std::move(result_column); + } + + template + static ColumnPtr doDecrypt(const EVP_CIPHER * evp_cipher, + size_t input_rows_count, + const InputColumnType & input_column, + const KeyColumnType & key_column, + const IvColumnType & iv_column, + const AadColumnType & aad_column) + { + if constexpr (compatibility_mode == OpenSSLDetails::CompatibilityMode::MySQL) + { + return doDecryptImpl(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + else + { + const auto cipher_mode = EVP_CIPHER_mode(evp_cipher); + if (cipher_mode == EVP_CIPH_GCM_MODE) + { + return doDecryptImpl(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + else + { + return doDecryptImpl(evp_cipher, input_rows_count, input_column, key_column, iv_column, aad_column); + } + } + + return nullptr; + } + + template + static ColumnPtr doDecryptImpl(const EVP_CIPHER * evp_cipher, + size_t input_rows_count, + const InputColumnType & input_column, + const KeyColumnType & key_column, + [[maybe_unused]] const IvColumnType & iv_column, + [[maybe_unused]] const AadColumnType & aad_column) + { + using namespace OpenSSLDetails; + + auto evp_ctx_ptr = std::unique_ptr(EVP_CIPHER_CTX_new(), &EVP_CIPHER_CTX_free); + auto evp_ctx = evp_ctx_ptr.get(); + + [[maybe_unused]] const auto block_size = static_cast(EVP_CIPHER_block_size(evp_cipher)); + [[maybe_unused]] const auto iv_size = static_cast(EVP_CIPHER_iv_length(evp_cipher)); + const auto key_size = static_cast(EVP_CIPHER_key_length(evp_cipher)); + const auto tag_size = 16; // https://tools.ietf.org/html/rfc5116#section-5.1 + + auto decrypted_result_column = ColumnString::create(); + auto & decrypted_result_column_data = decrypted_result_column->getChars(); + auto & decrypted_result_column_offsets = decrypted_result_column->getOffsets(); + + { + size_t resulting_size = 0; + for (size_t r = 0; r < input_rows_count; ++r) + { + resulting_size += input_column->getDataAt(r).size + 1; + if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) + resulting_size -= tag_size; + } + +#if defined(MEMORY_SANITIZER) + // Pre-fill result column with values to prevent MSAN from dropping dead on + // aes-X-ecb mode with "WARNING: MemorySanitizer: use-of-uninitialized-value". + // This is most likely to be caused by the underlying assembler implementation: + // see crypto/aes/aesni-x86_64.s, function aesni_ecb_encrypt + // which msan seems to fail instrument correctly. + decrypted_result_column_data.resize_fill(resulting_size, 0xFF); +#else + decrypted_result_column_data.resize(resulting_size); +#endif + } + auto decrypted = decrypted_result_column_data.data(); + + KeyHolder key_holder; + for (size_t r = 0; r < input_rows_count; ++r) + { + // 0: prepare key if required + auto key_value = key_holder.setKey(key_size, key_column->getDataAt(r)); + auto iv_value = StringRef{}; + if constexpr (!std::is_same_v>) + { + iv_value = iv_column->getDataAt(r); + } + + auto input_value = input_column->getDataAt(r); + if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) + { + // empty plaintext results in empty ciphertext + tag, means there should be atleast tag_size bytes. + if (input_value.size < tag_size) + throw Exception(fmt::format("Encrypted data is too short: only {} bytes, " + "should contain at least {} bytes of a tag.", + input_value.size, block_size, tag_size), ErrorCodes::BAD_ARGUMENTS); + input_value.size -= tag_size; + } + + // 1: Init CTX + if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) + { + // 1.a.1 : Init CTX with custom IV length and optionally with AAD + if (EVP_DecryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr) != 1) + onError("Failed to initialize cipher context"); + + if (EVP_CIPHER_CTX_ctrl(evp_ctx, EVP_CTRL_AEAD_SET_IVLEN, iv_value.size, nullptr) != 1) + onError("Failed to set custom IV length to " + std::to_string(iv_value.size)); + + if (EVP_DecryptInit_ex(evp_ctx, nullptr, nullptr, + reinterpret_cast(key_value.data), + reinterpret_cast(iv_value.data)) != 1) + onError("Failed to set key and IV"); + + // 1.a.2: Set AAD if present + if constexpr (!std::is_same_v>) + { + const auto aad_data = aad_column->getDataAt(r); + int tmp_len = 0; + if (aad_data.size != 0 && EVP_DecryptUpdate(evp_ctx, nullptr, &tmp_len, + reinterpret_cast(aad_data.data), aad_data.size) != 1) + onError("Failed to sed AAD data"); + } + } + else + { + // 1.b: Init CTX + validateIV(iv_value, iv_size); + + if (EVP_DecryptInit_ex(evp_ctx, evp_cipher, nullptr, + reinterpret_cast(key_value.data), + reinterpret_cast(iv_value.data)) != 1) + onError("Failed to initialize cipher context"); + } + + // 2: Feed the data to the cipher + int output_len = 0; + if (EVP_DecryptUpdate(evp_ctx, + reinterpret_cast(decrypted), &output_len, + reinterpret_cast(input_value.data), static_cast(input_value.size)) != 1) + onError("Failed to decrypt"); + decrypted += output_len; + + // 3: optionally get tag from the ciphertext (RFC5116) and feed it to the context + if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) + { + void * tag = const_cast(reinterpret_cast(input_value.data + input_value.size)); + if (EVP_CIPHER_CTX_ctrl(evp_ctx, EVP_CTRL_AEAD_SET_TAG, tag_size, tag) != 1) + onError("Failed to set tag"); + } + + // 4: retrieve encrypted data (ciphertext) + if (EVP_DecryptFinal_ex(evp_ctx, + reinterpret_cast(decrypted), &output_len) != 1) + onError("Failed to decrypt"); + decrypted += output_len; + + *decrypted = '\0'; + ++decrypted; + + decrypted_result_column_offsets.push_back(decrypted - decrypted_result_column_data.data()); + + if (EVP_CIPHER_CTX_reset(evp_ctx) != 1) + onError("Failed to reset context"); + } + + // in case we overestimate buffer required for decrypted data, fix it up. + if (!decrypted_result_column_offsets.empty() && decrypted_result_column_data.size() > decrypted_result_column_offsets.back()) + { + decrypted_result_column_data.resize(decrypted_result_column_offsets.back()); + } + + decrypted_result_column->validate(); + return decrypted_result_column; + } +}; + +} + + +#endif diff --git a/src/Functions/aes_decrypt_mysql.cpp b/src/Functions/aes_decrypt_mysql.cpp new file mode 100644 index 00000000000..764fcf06c1a --- /dev/null +++ b/src/Functions/aes_decrypt_mysql.cpp @@ -0,0 +1,29 @@ +#include + +#if USE_SSL + +#include +#include + +namespace +{ + +struct DecryptMySQLModeImpl +{ + static constexpr auto name = "aes_decrypt_mysql"; + static constexpr auto compatibility_mode = OpenSSLDetails::CompatibilityMode::MySQL; +}; + +} + +namespace DB +{ + +void registerFunctionAESDecryptMysql(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +} + +#endif diff --git a/src/Functions/aes_encrypt_mysql.cpp b/src/Functions/aes_encrypt_mysql.cpp new file mode 100644 index 00000000000..1d84824d9d6 --- /dev/null +++ b/src/Functions/aes_encrypt_mysql.cpp @@ -0,0 +1,29 @@ +#include + +#if USE_SSL + +#include +#include + +namespace +{ + +struct EncryptMySQLModeImpl +{ + static constexpr auto name = "aes_encrypt_mysql"; + static constexpr auto compatibility_mode = OpenSSLDetails::CompatibilityMode::MySQL; +}; + +} + +namespace DB +{ + +void registerFunctionAESEncryptMysql(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +} + +#endif diff --git a/src/Functions/decrypt.cpp b/src/Functions/decrypt.cpp new file mode 100644 index 00000000000..1cbda0dba99 --- /dev/null +++ b/src/Functions/decrypt.cpp @@ -0,0 +1,29 @@ +#include + +#if USE_SSL + +#include +#include + +namespace +{ + +struct DecryptImpl +{ + static constexpr auto name = "decrypt"; + static constexpr auto compatibility_mode = OpenSSLDetails::CompatibilityMode::OpenSSL; +}; + +} + +namespace DB +{ + +void registerFunctionDecrypt(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +} + +#endif diff --git a/src/Functions/encrypt.cpp b/src/Functions/encrypt.cpp new file mode 100644 index 00000000000..807443a2fb8 --- /dev/null +++ b/src/Functions/encrypt.cpp @@ -0,0 +1,29 @@ +#include + +#if USE_SSL + +#include +#include + +namespace +{ + +struct EncryptImpl +{ + static constexpr auto name = "encrypt"; + static constexpr auto compatibility_mode = OpenSSLDetails::CompatibilityMode::OpenSSL; +}; + +} + +namespace DB +{ + +void registerFunctionEncrypt(FunctionFactory & factory) +{ + factory.registerFunction>(); +} + +} + +#endif diff --git a/src/Functions/registerFunctions.cpp b/src/Functions/registerFunctions.cpp index 804ceb4d309..b7fb3cecd66 100644 --- a/src/Functions/registerFunctions.cpp +++ b/src/Functions/registerFunctions.cpp @@ -1,3 +1,5 @@ +#include + #include @@ -38,10 +40,19 @@ void registerFunctionsNull(FunctionFactory &); void registerFunctionsJSON(FunctionFactory &); void registerFunctionsConsistentHashing(FunctionFactory & factory); void registerFunctionsUnixTimestamp64(FunctionFactory & factory); + #if !defined(ARCADIA_BUILD) void registerFunctionBayesAB(FunctionFactory &); #endif +#if USE_SSL +void registerFunctionEncrypt(FunctionFactory & factory); +void registerFunctionDecrypt(FunctionFactory & factory); +void registerFunctionAESEncryptMysql(FunctionFactory & factory); +void registerFunctionAESDecryptMysql(FunctionFactory & factory); + +#endif + void registerFunctions() { @@ -83,9 +94,17 @@ void registerFunctions() registerFunctionsIntrospection(factory); registerFunctionsConsistentHashing(factory); registerFunctionsUnixTimestamp64(factory); + #if !defined(ARCADIA_BUILD) registerFunctionBayesAB(factory); #endif + +#if USE_SSL + registerFunctionEncrypt(factory); + registerFunctionDecrypt(factory); + registerFunctionAESEncryptMysql(factory); + registerFunctionAESDecryptMysql(factory); +#endif } } diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 31d5dfa9fd3..1806d762d33 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -46,6 +46,8 @@ SRCS( addSeconds.cpp addWeeks.cpp addYears.cpp + aes_decrypt_mysql.cpp + aes_encrypt_mysql.cpp appendTrailingCharIfAbsent.cpp array/arrayAll.cpp array/arrayAUC.cpp @@ -138,6 +140,7 @@ SRCS( currentUser.cpp dateDiff.cpp date_trunc.cpp + decrypt.cpp defaultValueOfArgumentType.cpp defaultValueOfTypeName.cpp demange.cpp @@ -145,6 +148,7 @@ SRCS( dumpColumnStructure.cpp e.cpp empty.cpp + encrypt.cpp endsWith.cpp equals.cpp erfc.cpp @@ -170,6 +174,7 @@ SRCS( FunctionFQDN.cpp FunctionHelpers.cpp FunctionJoinGet.cpp + FunctionsAES.cpp FunctionsCoding.cpp FunctionsConversion.cpp FunctionsEmbeddedDictionaries.cpp diff --git a/tests/queries/0_stateless/01318_decrypt.reference b/tests/queries/0_stateless/01318_decrypt.reference new file mode 100644 index 00000000000..2241501f564 --- /dev/null +++ b/tests/queries/0_stateless/01318_decrypt.reference @@ -0,0 +1,80 @@ +0 +0 +0 +1 +MySQL-specific key folding and decrpyting +aes-128-ecb 1 +aes-128-ecb 1 +aes-128-ecb 1 +aes-192-ecb 1 +aes-192-ecb 1 +aes-192-ecb 1 +aes-256-ecb 1 +aes-256-ecb 1 +aes-256-ecb 1 +aes-128-cbc 1 +aes-128-cbc 1 +aes-128-cbc 1 +aes-192-cbc 1 +aes-192-cbc 1 +aes-192-cbc 1 +aes-256-cbc 1 +aes-256-cbc 1 +aes-256-cbc 1 +aes-128-cfb1 1 +aes-128-cfb1 1 +aes-128-cfb1 1 +aes-192-cfb1 1 +aes-192-cfb1 1 +aes-192-cfb1 1 +aes-256-cfb1 1 +aes-256-cfb1 1 +aes-256-cfb1 1 +aes-128-cfb8 1 +aes-128-cfb8 1 +aes-128-cfb8 1 +aes-192-cfb8 1 +aes-192-cfb8 1 +aes-192-cfb8 1 +aes-256-cfb8 1 +aes-256-cfb8 1 +aes-256-cfb8 1 +aes-128-cfb128 1 +aes-128-cfb128 1 +aes-128-cfb128 1 +aes-192-cfb128 1 +aes-192-cfb128 1 +aes-192-cfb128 1 +aes-256-cfb128 1 +aes-256-cfb128 1 +aes-256-cfb128 1 +aes-128-ofb 1 +aes-128-ofb 1 +aes-128-ofb 1 +aes-192-ofb 1 +aes-192-ofb 1 +aes-192-ofb 1 +aes-256-ofb 1 +aes-256-ofb 1 +aes-256-ofb 1 +GCM mode with IV +aes-128-gcm FB9958E2E897EF3FDB49067B51A24AF6 1 +aes-128-gcm FB9958E2E897EF3FDB49067B51A24AF6 1 +aes-128-gcm FB9958E2E897EF3FDB49067B51A24AF6 1 +aes-192-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1 1 +aes-192-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1 1 +aes-192-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1 1 +aes-256-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1DC7FD4DD71B7E38F 1 +aes-256-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1DC7FD4DD71B7E38F 1 +aes-256-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1DC7FD4DD71B7E38F 1 +GCM mode with IV and AAD +aes-128-gcm FB9958E2E897EF3FDB49067B51A24AF6 1 +aes-128-gcm FB9958E2E897EF3FDB49067B51A24AF6 1 +aes-128-gcm FB9958E2E897EF3FDB49067B51A24AF6 1 +aes-192-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1 1 +aes-192-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1 1 +aes-192-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1 1 +aes-256-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1DC7FD4DD71B7E38F 1 +aes-256-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1DC7FD4DD71B7E38F 1 +aes-256-gcm FB9958E2E897EF3FDB49067B51A24AF645B3626EED2F9EA1DC7FD4DD71B7E38F 1 +F56E87055BC32D0EEB31B2EACC2BF2A5 1 diff --git a/tests/queries/0_stateless/01318_decrypt.sql b/tests/queries/0_stateless/01318_decrypt.sql new file mode 100644 index 00000000000..5e148b90724 --- /dev/null +++ b/tests/queries/0_stateless/01318_decrypt.sql @@ -0,0 +1,123 @@ +--- aes_decrypt_mysql(string, key, block_mode[, init_vector, AAD]) +-- The MySQL-compatitable encryption, only ecb, cbc, cfb1, cfb8, cfb128 and ofb modes are supported, +-- just like for MySQL +-- https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_aes-encrypt +-- https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_block_encryption_mode +-- Please note that for keys that exceed mode-specific length, keys are folded in a MySQL-specific way, +-- meaning that whole key is used, but effective key length is still determined by mode. +-- when key doesn't exceed the default mode length, ecryption result equals with AES_encypt() + +----------------------------------------------------------------------------------------- +-- error cases +----------------------------------------------------------------------------------------- +SELECT aes_decrypt_mysql(); --{serverError 42} not enough arguments +SELECT aes_decrypt_mysql('aes-128-ecb'); --{serverError 42} not enough arguments +SELECT aes_decrypt_mysql('aes-128-ecb', 'text'); --{serverError 42} not enough arguments + + +-- Mode +SELECT aes_decrypt_mysql(789, 'text', 'key'); --{serverError 43} bad mode type +SELECT aes_decrypt_mysql('blah blah blah', 'text', 'key'); -- {serverError 36} garbage mode value +SELECT aes_decrypt_mysql('des-ede3-ecb', 'text', 'key'); -- {serverError 36} bad mode value of valid cipher name +SELECT aes_decrypt_mysql('aes-128-gcm', 'text', 'key'); -- {serverError 36} mode is not supported by _mysql-functions + +SELECT decrypt(789, 'text', 'key'); --{serverError 43} bad mode type +SELECT decrypt('blah blah blah', 'text', 'key'); -- {serverError 36} garbage mode value +SELECT decrypt('des-ede3-ecb', 'text', 'key'); -- {serverError 36} bad mode value of valid cipher name + + +-- Key +SELECT aes_decrypt_mysql('aes-128-ecb', 'text', 456); --{serverError 43} bad key type +SELECT aes_decrypt_mysql('aes-128-ecb', 'text', 'key'); -- {serverError 36} key is too short + +SELECT decrypt('aes-128-ecb', 'text'); --{serverError 42} key is missing +SELECT decrypt('aes-128-ecb', 'text', 456); --{serverError 43} bad key type +SELECT decrypt('aes-128-ecb', 'text', 'key'); -- {serverError 36} key is too short +SELECT decrypt('aes-128-ecb', 'text', 'keykeykeykeykeykeykeykeykeykeykeykey'); -- {serverError 36} key is to long + + +-- IV +SELECT aes_decrypt_mysql('aes-128-ecb', 'text', 'key', 1011); --{serverError 43} bad IV type 6 +SELECT aes_decrypt_mysql('aes-128-ecb', 'text', 'key', 'iv'); --{serverError 36} IV is too short 4 + +SELECT decrypt('aes-128-cbc', 'text', 'keykeykeykeykeyk', 1011); --{serverError 43} bad IV type 1 +SELECT decrypt('aes-128-cbc', 'text', 'keykeykeykeykeyk', 'iviviviviviviviviviviviviviviviviviviviviv'); --{serverError 36} IV is too long 3 +SELECT decrypt('aes-128-cbc', 'text', 'keykeykeykeykeyk', 'iv'); --{serverError 36} IV is too short 2 + + +--AAD +SELECT aes_decrypt_mysql('aes-128-ecb', 'text', 'key', 'IV', 1213); --{serverError 42} too many arguments + +SELECT decrypt('aes-128-ecb', 'text', 'key', 'IV', 1213); --{serverError 43} bad AAD type +SELECT decrypt('aes-128-gcm', 'text', 'key', 'IV', 1213); --{serverError 43} bad AAD type + + +-- decrypting invalid cipher text, should cause an error or produce garbage +SELECT ignore(decrypt('aes-128-ecb', 'hello there', '1111111111111111')); -- {serverError 454} 1 +SELECT ignore(decrypt('aes-128-cbc', 'hello there', '1111111111111111')); -- {serverError 454} 2 +SELECT ignore(decrypt('aes-128-cfb1', 'hello there', '1111111111111111')); -- GIGO +SELECT ignore(decrypt('aes-128-ofb', 'hello there', '1111111111111111')); -- GIGO +SELECT ignore(decrypt('aes-128-ctr', 'hello there', '1111111111111111')); -- GIGO +SELECT decrypt('aes-128-ctr', '', '1111111111111111') == ''; + + +----------------------------------------------------------------------------------------- +-- Validate against predefined ciphertext,plaintext,key and IV for MySQL compatibility mode +----------------------------------------------------------------------------------------- +CREATE TABLE encryption_test +( + input String, + key String DEFAULT unhex('fb9958e2e897ef3fdb49067b51a24af645b3626eed2f9ea1dc7fd4dd71b7e38f9a68db2a3184f952382c783785f9d77bf923577108a88adaacae5c141b1576b0'), + iv String DEFAULT unhex('8CA3554377DFF8A369BC50A89780DD85') +) Engine = Memory; + +INSERT INTO encryption_test (input) +VALUES (''), ('text'), ('What Is ClickHouse? ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).'); + +SELECT 'MySQL-specific key folding and decrpyting'; +SELECT 'aes-128-ecb' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-192-ecb' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-256-ecb' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; + +SELECT 'aes-128-cbc' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-192-cbc' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-256-cbc' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; + +SELECT 'aes-128-cfb1' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-192-cfb1' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-256-cfb1' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; + +SELECT 'aes-128-cfb8' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-192-cfb8' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-256-cfb8' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; + +SELECT 'aes-128-cfb128' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-192-cfb128' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-256-cfb128' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; + +SELECT 'aes-128-ofb' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-192-ofb' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; +SELECT 'aes-256-ofb' as mode, aes_decrypt_mysql(mode, aes_encrypt_mysql(mode, input, key, iv), key, iv) == input FROM encryption_test; + +SELECT 'GCM mode with IV'; +SELECT 'aes-128-gcm' as mode, hex(substr(key, 1, 16)) as key, decrypt(mode, encrypt(mode, input, unhex(key), iv), unhex(key), iv) == input FROM encryption_test; +SELECT 'aes-192-gcm' as mode, hex(substr(key, 1, 24)) as key, decrypt(mode, encrypt(mode, input, unhex(key), iv), unhex(key), iv) == input FROM encryption_test; +SELECT 'aes-256-gcm' as mode, hex(substr(key, 1, 32)) as key, decrypt(mode, encrypt(mode, input, unhex(key), iv), unhex(key), iv) == input FROM encryption_test; + +SELECT 'GCM mode with IV and AAD'; +SELECT 'aes-128-gcm' as mode, hex(substr(key, 1, 16)) AS key, decrypt(mode, encrypt(mode, input, unhex(key), iv, 'AAD'), unhex(key), iv, 'AAD') == input FROM encryption_test; +SELECT 'aes-192-gcm' as mode, hex(substr(key, 1, 24)) AS key, decrypt(mode, encrypt(mode, input, unhex(key), iv, 'AAD'), unhex(key), iv, 'AAD') == input FROM encryption_test; +SELECT 'aes-256-gcm' as mode, hex(substr(key, 1, 32)) AS key, decrypt(mode, encrypt(mode, input, unhex(key), iv, 'AAD'), unhex(key), iv, 'AAD') == input FROM encryption_test; + + +-- based on https://github.com/openssl/openssl/blob/master/demos/evp/aesgcm.c#L20 +WITH + unhex('eebc1f57487f51921c0465665f8ae6d1658bb26de6f8a069a3520293a572078f') as key, + unhex('67ba0510262ae487d737ee6298f77e0c') as tag, + unhex('99aa3e68ed8173a0eed06684') as iv, + unhex('f56e87055bc32d0eeb31b2eacc2bf2a5') as plaintext, + unhex('4d23c3cec334b49bdb370c437fec78de') as aad, + unhex('f7264413a84c0e7cd536867eb9f21736') as ciphertext +SELECT + hex(decrypt('aes-256-gcm', concat(ciphertext, tag), key, iv, aad)) as plaintext_actual, + plaintext_actual = hex(plaintext); diff --git a/tests/queries/0_stateless/01318_encrypt.reference b/tests/queries/0_stateless/01318_encrypt.reference new file mode 100644 index 00000000000..130a95ab178 --- /dev/null +++ b/tests/queries/0_stateless/01318_encrypt.reference @@ -0,0 +1,86 @@ +UInt64 +5417DEA8D67A1A03FD561809C62402FF +Float64 +9B66D0AA685DC0F1EFFA2E385F7EA2F2 +Decimal64 +5417DEA8D67A1A03FD561809C62402FF +MySQL-specific key folding +aes-128-ecb 861BA71B647390651F75F0EB4A18DCA1 +aes-128-ecb 557F8E81CBBBB2515A33500768018C3C +aes-128-ecb C508CC3350317F61AB1793DB6D93AEFAB000F51C8651ABB578F5EEF362F8560FB3655364CEC9B4D2758C71BC03E4D72FBC54385094A20E20949F70D91462442C5ABA90EF581BC3309C7F2E9E468E34D83C73346C05627D4E1634615F6F5B01E1B388664AECCAD26E4508B537082CEA572222DDBFC9BD0CB5D1D6FEE26A8CCD57BAE76655DCAF0952B80C7F1903990B60 +aes-192-ecb 04793D6184FFAD00C6457B54D30FED91 +aes-192-ecb 0028EDF20F6C08FD7097653CE4DB9C39 +aes-192-ecb 733ECCEEBD7C760CA38FC8ED736A053DCCA5C8DE06035689C765BE53DBFEB9BA9B98C9169C884278E227A3BAFA38F53E837857DF96D6A9B09C58BD196553FFDF1B6F191EAF5A82950EDCEDBE46B91BB8FDC8DDAA6566481B807FA5FCA40D687CF14E2F1A318E0B4CE5C2305FB43A411B4B65EC5B525FD4AB08CDDE49212FC2E99B1096EA5B5F4957594654CA3B369145 +aes-256-ecb 3FEBF71206304655B6451E02EBFDB965 +aes-256-ecb EBB295D0F05E820370629399AD7B04DB +aes-256-ecb 54D9B7BF0FEE21A848051927FB29D9F621FDD4DEA6D320D80D3102E46A92F17D7E2B9B7AB3C0C4B34B1A7ABABDF98E7ACC4066DFCC385AC799A8D686655311886152E49D3AF8F0D4EF321E05E22E3CE19D0CDCA1C05035C86C6EA4D2D2C7B31AA0D496E03CEB7661905F9463A140E5F8875E876CBD1A72A2B4DE0F98533E1C87D06FE4A68ADF572DD9A74A562DE9A45F +aes-128-cbc 9617622E3D3A2BB45B3D0922D63F7C1E +aes-128-cbc 700AED2DCC265D7E8D98D0DBBD7653C4 +aes-128-cbc 63A26A3E2FC9DD48E3BA6CD6BF94D3196181BF85D43301DD7E129C95C90A6760F762317A5E868FECB48CCC9F789B2552A40D12D8B8D8DF062115968751BFD36281D47D63EA4A1629337A0EC5051128FECFE573D6EA055175C17C39C79AF5ECAEB4764ED2AF89784C5BF9F43C75AA4781E5DD483DDCD529D8D30C22710CA930F79BBACBDA51086B227F7A3B087D4EBD7F +aes-192-cbc ABF263E5090CC373E7D92CAE91A9136C +aes-192-cbc B3DBB188BC9DEF8AF436D68A23FEAA99 +aes-192-cbc 99E393B279CB125B11F830262800D00A2E4CEFD59A2D1441AAEC11334DDD2AD2CCE75900BA42BE1A78F95C79CEEA85CB0FA401463A53229F8B520E6883011445AE90C2A3A1ECBC2589E1429E631474B5B269AA7617CB818384E93B51C1B3A73F9183CA27899B41BE3E9BB95A45F70C0BA94331E3B7E4849808F83195979079DAC69953C7208D00D6C2C4E6304CDA6B9E +aes-256-cbc 221820CEBE1F8B472AC06C8C0DE52BA7 +aes-256-cbc ADC9060184FE5A23B6CE35327CE5309A +aes-256-cbc 09E8AE34C02AB3A50BF4FC49A70344C6B956FCA52D764287B3E935D192D80D7F3E22DDA0E42D4911796A3E4D47A4CD884C511D7AEEF89AD739F4A8A519F03C70C5FE0C0C65A8916B3BA1A64D6964693999C002B96CDE2D53327D2614D8D8D473D6F7B537DC5B459F694196ECF0F034156DBB0A91E98735531E5E756908390F262456D104393F099934E1F8E5406D537E +aes-128-cfb1 +aes-128-cfb1 6B939C8E +aes-128-cfb1 50CACD46A945BD61615CFA5638EB313236AE7611B3EA6653E844D7D9F3D4D05442492390DD33D1DA753AD94F6C49D43EB40CF096734AC36D95D1FB6FEDB1AC9D1C0A59C7688C5B5C3D992F6F3FF06418E4E4F5372E51B22F9B7C155AB2B763CD47F5EA5C5F1454D912589DB8E24D46BFE1D429D6F57EEFAFCA91D9C2513705913B3205BFFDA952F65C +aes-192-cfb1 +aes-192-cfb1 B90F814C +aes-192-cfb1 8A3503A6DA7547D161B5F778492910321B570C795450FDF2B67DD571467745397769B919CF00ADA1FFBF9DEEFBA4170E625F2E0F3B371DABF5D9439FB154E09890AB951D18F7C1D2D041E697E6AB1779112C31F068AD59A4E4D9ABF30CA5504174EE62BCA04C092C29C60B29BB633DB31F111F905E33D63418A5048256711EF6A276C8C2417EF94C45 +aes-256-cfb1 +aes-256-cfb1 88876A00 +aes-256-cfb1 A31D52BADDE4E79FA12A3C6E145C86E0DDA653EACFDC934541E4D4E2166A06E801A5BC7D30C430B65BE5F8AF5E8BE0A900F03A54BD16D8CD27BBA3411BA00B10A53FEEF7691398BCE0FFB548A4F856219364DD44C4BD9E5515809018045CBC5CFA5308B25498B9F16E437F10EF6E38F08FDBE8424481B18147690A7A63BE674DB566356A1A6FCD642F +aes-128-cfb8 +aes-128-cfb8 76698980 +aes-128-cfb8 5505B55E6BD7BB69C38FFC9953E1729111D99EB2A6F6392BC1B211141B455923D25FC9387A92C95D91448AA46FD08B0C8977A0CF0123D4503681760D2BAC1203EABB6D6BCD52A9DD99ECD69FA28648229E8F1BC7D194DB817BF39CEC2880722915A542B098FBDE142206A3418B6024B7961BB42358FDB9CB7FC742E387353C961AEB2240F7ABA6EC29 +aes-192-cfb8 +aes-192-cfb8 FB065B88 +aes-192-cfb8 D8015BD1B2CBA6EA04D8A299E17E1D7D0DEE7D31B20711EDF9CEACB158CDDFE4ED569B5C6425DAF6FB0B2B0CA1C87D18C867D87AC760FDD3CF4E94A9FDF1325593F6C36600F4105FEFF5E9B733AB7B28908376DCF4C6FA5F90B720071958C8C69FCABCE9D4D97B33E4A902A41B6F5C72387ADC9F2FD9B787FC0E09FB603CD74BE25DE276D140A00C28 +aes-256-cfb8 +aes-256-cfb8 A62542C4 +aes-256-cfb8 85406B1C10E1F8D7208BD274FD166558C541FF6B761B59BB64BB74A253FE8FE9C6F59BAB49314B2D286270CCC05E6C6C569EB279558E9218162B5CC7D11ECE21CE8CD5805B85F0879EE277AB742C0D07C08A3CA037EAA8D5C643E2E9116D574E9E2284FE7D1EE123640E5F698ACEB407BD8855741472E0BECE67B3760CA2C9F3EB656A6A884AB09C8C +aes-128-cfb128 +aes-128-cfb128 761CC7B1 +aes-128-cfb128 5511DEB1CD27BED6062453F99912A690AA3628279677D02473DDA04CB6763AF2C5DD191F388E16AC7E122D1A070DB9BEE68B403FAB2FBEF09B224C646EDE9A773E2671D8D8B89B263FDD218FE5E8E6FB37CCA8AEC51C8B4BE0BA5FBC1496B95731E405D14B269CEFCAF7244CE893F9F8B950A66BD30540F0F1E1BF6ECB3ABB71F6470B4875A38C5372 +aes-192-cfb128 +aes-192-cfb128 FBD2EB05 +aes-192-cfb128 D8DFF205334E1E67A0FBC960069219C7D75497A6FCD47E98AB572823BCB1CC472FB94A502AD597B565021F78EAFF3BD7D8B4E4C175A76F97B039FB53E73F35B997EBB693A0AB42AA1003946F84FFBEB32F4E1AC57312C05C8F4E315F6275C097862F347CD3B20B56BFFD73A95FC441AEA0DCFB9E3EABE231114C013A769ADA1D0E46A76D4979A27B08 +aes-256-cfb128 +aes-256-cfb128 A6DE9438 +aes-256-cfb128 85D38D383998E41817837668832DA0068BB5B64BB612FFF9C87D8E5773375C6503B50105A6F96F462A62E1E6C5C7FD7081C629818C10E91A23DA38F1421247B83AE2DEBE317C2F221122DB2E6222DC4C274EEFDE5E425D1FCAD20A3B6C82EF3363335F423F9C1E88AE69F017D53F4ADE0FD748B0BDFF7F15400E27E641FC674EBC40B38EF44D911C8B +aes-128-ofb +aes-128-ofb 761CC7B1 +aes-128-ofb 5511DEB1CD27BED6062453F99912A6905E263DE7ABC54E8CF73A9D9FB2F05F643D7E15B468FFF70EB7EFF3A4DD69A60725852D39D4C389231FDD8B82AC16374F101D34D231113E8ACA9762529B38B49313D8700C4650933C3A4E2CE624C0253AEE0ADC8BCB9E6042D1EE5BA75E2327A5D5B039EA4DA20076E8CDFE0E597AD18710DAFC4B261BC02E32 +aes-192-ofb +aes-192-ofb FBD2EB05 +aes-192-ofb D8DFF205334E1E67A0FBC960069219C7F891621DCBF5B36DE32C2BDC0F37DF021B9106560DBEC7FDE953CC5DAA05C5FD2E37EF1ABD955DD77407CF631BFCBE33873E4F1365E316C707F63DE258C0077E403BD2E8F567601F0D5A5B01920E790B358F2B6B73D9BCBFFBCF718C4223A004B46CBB4BA70E31B542263C0902E03A8EF8FA00ACA70770050A +aes-256-ofb +aes-256-ofb A6DE9438 +aes-256-ofb 85D38D383998E41817837668832DA00685DA2D37602CB00AD4DAFB8EB0B85840BCC0CDAD6E229ED356EB2D1E8819E9951D4F6D0F6EA4894F3FD5401938529F388077AC5373CA5D383510769C08256551E6718B236FE29D27C284BB1A3B4E0F6AC5A7625607668F05ED92E7FF54B02771D5ED2AA1381D427E64010EDF21E11CDCDB9B8C35B542720430 +Nullable and LowCardinality +Nullable(String) \N +Nullable(String) A6DE9438 +LowCardinality(String) A6DE9438 +GCM mode with IV +aes-128-gcm 3D67D2B8D8F49A24C482085FEC494231 +aes-128-gcm C08B1CF60C5A2C92C55DAC62223CBA22C736446C +aes-128-gcm E38605F61AE032292F25A35A6CDF6E827980625B9A50BB3C77CD2AD54DB9BE5578322CC55569D1B0C5B82577060C0053388F511DB7BF9C898BF4B05FB6C8C0D0F50449C06A2E89E086610CB2BAEF25B206312836884DCBCC6CD8329B2A43E2BA751183B1696AB3F070BE94FA823F1E1A5E2372A06E1AD2719AF37983D23FCD199820B7769E72EDC20AF48466DAEB6550DC7FDBA041F77D5231 +aes-192-gcm FC2C8C63A570E584AB71F19BA6E79A8F +aes-192-gcm 9A6CF0FDA93F1614F982B5570CC1216D84E356BD +aes-192-gcm B961E9FD9B940EBAD7ADDA75C9F198A40797A598AC7FA183AC58705EF6E4E295504D71B85D81978B4CE196AFFFA019B941F44B14DF06375688FCA986F2F3088C24E955392F0DB378F033052822D560CD8DDFF5472C66029E997AE2D63935DAAA10D6703E5AB627E8168F16CF5CDB1112DD2D49F10E087BA20831DCCE592465C95AAA5AF8F766BAEDC3FD3949EDD2E667333C83E58786504137 +aes-256-gcm E99DBEBC01F021758352D7FBD9039EFA +aes-256-gcm 8742CE3A7B0595B281C712600D274CA881F47414 +aes-256-gcm A44FD73ACEB1A64BDE2D03808A2576EDBB6076F61614CC84A960CCBE55FBABF365671B7017BC89C8A2E0A633E0A05E40B2681B33AD3E7A0AC4925DBD9735C4D1C1E33726B1D6A83CBD337A65C50D7CC33CC4E64369D54C1B6AF3A82D206DF0698BEB61EF9AB2DF81B03DF3829A2EC42D667D87376B8A1351C69BB7A11CCBE50DA88ABA991E98D3BD71129682F35422AD73B05EC624357E77FC +GCM mode with IV and AAD +aes-128-gcm 5AB059BB98F087E8134B19E7EB5CD9C7 +aes-128-gcm C08B1CF67AD5D38FE0F3508689794961B8D1FAB8 +aes-128-gcm E38605F61AE032292F25A35A6CDF6E827980625B9A50BB3C77CD2AD54DB9BE5578322CC55569D1B0C5B82577060C0053388F511DB7BF9C898BF4B05FB6C8C0D0F50449C06A2E89E086610CB2BAEF25B206312836884DCBCC6CD8329B2A43E2BA751183B1696AB3F070BE94FA823F1E1A5E2372A06E1AD2719AF37983D23FCD199820B7769E72EDC20A0826DB2A479DB59F7216A9BDCBD0C989 +aes-192-gcm 04C13E4B1D62481ED22B3644595CB5DB +aes-192-gcm 9A6CF0FD2B329B04EAD18301818F016DF8F77447 +aes-192-gcm B961E9FD9B940EBAD7ADDA75C9F198A40797A598AC7FA183AC58705EF6E4E295504D71B85D81978B4CE196AFFFA019B941F44B14DF06375688FCA986F2F3088C24E955392F0DB378F033052822D560CD8DDFF5472C66029E997AE2D63935DAAA10D6703E5AB627E8168F16CF5CDB1112DD2D49F10E087BA20831DCCE592465C95AAA5AF8F766BAEDC3668E035498D8C46FB662833CCC12C9D6 +aes-256-gcm E94F5F6ED4A99B741D492D7EA10B7147 +aes-256-gcm 8742CE3A3EA5153952DB4C0D94B501FE878FF9A7 +aes-256-gcm A44FD73ACEB1A64BDE2D03808A2576EDBB6076F61614CC84A960CCBE55FBABF365671B7017BC89C8A2E0A633E0A05E40B2681B33AD3E7A0AC4925DBD9735C4D1C1E33726B1D6A83CBD337A65C50D7CC33CC4E64369D54C1B6AF3A82D206DF0698BEB61EF9AB2DF81B03DF3829A2EC42D667D87376B8A1351C69BB7A11CCBE50DA88ABA991E98D3BD712F56268961DDAB59FA4D2B50578602C4 +F7264413A84C0E7CD536867EB9F2173667BA0510262AE487D737EE6298F77E0C 1 diff --git a/tests/queries/0_stateless/01318_encrypt.sql b/tests/queries/0_stateless/01318_encrypt.sql new file mode 100644 index 00000000000..dd5561efda6 --- /dev/null +++ b/tests/queries/0_stateless/01318_encrypt.sql @@ -0,0 +1,125 @@ +--- aes_encrypt_mysql(string, key, block_mode[, init_vector, AAD]) +-- The MySQL-compatitable encryption, only ecb, cbc, cfb1, cfb8, cfb128 and ofb modes are supported, +-- just like for MySQL +-- https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_aes-encrypt +-- https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_block_encryption_mode +-- Please note that for keys that exceed mode-specific length, keys are folded in a MySQL-specific way, +-- meaning that whole key is used, but effective key length is still determined by mode. +-- when key doesn't exceed the default mode length, ecryption result equals with AES_encypt() + +----------------------------------------------------------------------------------------- +-- error cases +----------------------------------------------------------------------------------------- +SELECT aes_encrypt_mysql(); --{serverError 42} not enough arguments +SELECT aes_encrypt_mysql('aes-128-ecb'); --{serverError 42} not enough arguments +SELECT aes_encrypt_mysql('aes-128-ecb', 'text'); --{serverError 42} not enough arguments + +-- Mode +SELECT aes_encrypt_mysql(789, 'text', 'key'); --{serverError 43} bad mode type +SELECT aes_encrypt_mysql('blah blah blah', 'text', 'key'); -- {serverError 36} garbage mode value +SELECT aes_encrypt_mysql('des-ede3-ecb', 'text', 'key'); -- {serverError 36} bad mode value of valid cipher name +SELECT aes_encrypt_mysql('aes-128-gcm', 'text', 'key'); -- {serverError 36} mode is not supported by _mysql-functions + +SELECT encrypt(789, 'text', 'key'); --{serverError 43} bad mode type +SELECT encrypt('blah blah blah', 'text', 'key'); -- {serverError 36} garbage mode value +SELECT encrypt('des-ede3-ecb', 'text', 'key'); -- {serverError 36} bad mode value of valid cipher name + + +-- Key +SELECT aes_encrypt_mysql('aes-128-ecb', 'text', 456); --{serverError 43} bad key type +SELECT aes_encrypt_mysql('aes-128-ecb', 'text', 'key'); -- {serverError 36} key is too short + +SELECT encrypt('aes-128-ecb', 'text'); --{serverError 42} key is missing +SELECT encrypt('aes-128-ecb', 'text', 456); --{serverError 43} bad key type +SELECT encrypt('aes-128-ecb', 'text', 'key'); -- {serverError 36} key is too short +SELECT encrypt('aes-128-ecb', 'text', 'keykeykeykeykeykeykeykeykeykeykeykey'); -- {serverError 36} key is to long + +-- IV +SELECT aes_encrypt_mysql('aes-128-ecb', 'text', 'key', 1011); --{serverError 43} bad IV type 6 +SELECT aes_encrypt_mysql('aes-128-ecb', 'text', 'key', 'iv'); --{serverError 36} IV is too short 4 + +SELECT encrypt('aes-128-cbc', 'text', 'keykeykeykeykeyk', 1011); --{serverError 43} bad IV type 1 +SELECT encrypt('aes-128-cbc', 'text', 'keykeykeykeykeyk', 'iviviviviviviviviviviviviviviviviviviviviv'); --{serverError 36} IV is too long 3 +SELECT encrypt('aes-128-cbc', 'text', 'keykeykeykeykeyk', 'iv'); --{serverError 36} IV is too short 2 + +--AAD +SELECT aes_encrypt_mysql('aes-128-ecb', 'text', 'key', 'IV', 1213); --{serverError 42} too many arguments + +SELECT encrypt('aes-128-ecb', 'text', 'key', 'IV', 1213); --{serverError 43} bad AAD type +SELECT encrypt('aes-128-gcm', 'text', 'key', 'IV', 1213); --{serverError 43} bad AAD type + +----------------------------------------------------------------------------------------- +-- Valid cases +----------------------------------------------------------------------------------------- + +SELECT 'UInt64'; +SELECT hex(aes_encrypt_mysql('aes-128-ecb', 123456789101112, 'keykeykeykeykeykeykeykeykeykeyke')); +SELECT 'Float64'; +SELECT hex(aes_encrypt_mysql('aes-128-ecb', 1234567891011.12, 'keykeykeykeykeykeykeykeykeykeyke')); +SELECT 'Decimal64'; +SELECT hex(aes_encrypt_mysql('aes-128-ecb', toDecimal64(1234567891011.12, 2), 'keykeykeykeykeykeykeykeykeykeyke')); + +----------------------------------------------------------------------------------------- +-- Validate against predefined ciphertext,plaintext,key and IV for MySQL compatibility mode +----------------------------------------------------------------------------------------- +CREATE TABLE encryption_test +( + input String, + key String DEFAULT unhex('fb9958e2e897ef3fdb49067b51a24af645b3626eed2f9ea1dc7fd4dd71b7e38f9a68db2a3184f952382c783785f9d77bf923577108a88adaacae5c141b1576b0'), + iv String DEFAULT unhex('8CA3554377DFF8A369BC50A89780DD85') +) Engine = Memory; + +INSERT INTO encryption_test (input) +VALUES (''), ('text'), ('What Is ClickHouse? ClickHouse is a column-oriented database management system (DBMS) for online analytical processing of queries (OLAP).'); + +SELECT 'MySQL-specific key folding'; +SELECT 'aes-128-ecb' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-192-ecb' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-256-ecb' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; + +SELECT 'aes-128-cbc' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-192-cbc' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-256-cbc' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; + +SELECT 'aes-128-cfb1' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-192-cfb1' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-256-cfb1' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; + +SELECT 'aes-128-cfb8' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-192-cfb8' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-256-cfb8' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; + +SELECT 'aes-128-cfb128' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-192-cfb128' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-256-cfb128' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; + +SELECT 'aes-128-ofb' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-192-ofb' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; +SELECT 'aes-256-ofb' as mode, hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test; + +SELECT 'Nullable and LowCardinality'; +WITH CAST(NULL as Nullable(String)) as input, 'aes-256-ofb' as mode SELECT toTypeName(input), hex(aes_encrypt_mysql(mode, input, key,iv)) FROM encryption_test LIMIT 1; +WITH CAST('text' as Nullable(String)) as input, 'aes-256-ofb' as mode SELECT toTypeName(input), hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test LIMIT 1; +WITH CAST('text' as LowCardinality(String)) as input, 'aes-256-ofb' as mode SELECT toTypeName(input), hex(aes_encrypt_mysql(mode, input, key, iv)) FROM encryption_test LIMIT 1; + +SELECT 'GCM mode with IV'; +SELECT 'aes-128-gcm' as mode, hex(encrypt(mode, input, substr(key, 1, 16), iv)) FROM encryption_test; +SELECT 'aes-192-gcm' as mode, hex(encrypt(mode, input, substr(key, 1, 24), iv)) FROM encryption_test; +SELECT 'aes-256-gcm' as mode, hex(encrypt(mode, input, substr(key, 1, 32), iv)) FROM encryption_test; + +SELECT 'GCM mode with IV and AAD'; +SELECT 'aes-128-gcm' as mode, hex(encrypt(mode, input, substr(key, 1, 16), iv, 'AAD')) FROM encryption_test; +SELECT 'aes-192-gcm' as mode, hex(encrypt(mode, input, substr(key, 1, 24), iv, 'AAD')) FROM encryption_test; +SELECT 'aes-256-gcm' as mode, hex(encrypt(mode, input, substr(key, 1, 32), iv, 'AAD')) FROM encryption_test; + +-- based on https://github.com/openssl/openssl/blob/master/demos/evp/aesgcm.c#L20 +WITH + unhex('eebc1f57487f51921c0465665f8ae6d1658bb26de6f8a069a3520293a572078f') as key, + unhex('67ba0510262ae487d737ee6298f77e0c') as tag, + unhex('99aa3e68ed8173a0eed06684') as iv, + unhex('f56e87055bc32d0eeb31b2eacc2bf2a5') as plaintext, + unhex('4d23c3cec334b49bdb370c437fec78de') as aad, + unhex('f7264413a84c0e7cd536867eb9f21736') as ciphertext +SELECT + hex(encrypt('aes-256-gcm', plaintext, key, iv, aad)) as ciphertext_actual, + ciphertext_actual = concat(hex(ciphertext), hex(tag)); diff --git a/tests/ubsan_suppressions.txt b/tests/ubsan_suppressions.txt new file mode 100644 index 00000000000..c29da437a11 --- /dev/null +++ b/tests/ubsan_suppressions.txt @@ -0,0 +1,3 @@ +# Suppress some failures in contrib so that we can enable UBSan in CI. +# Ideally, we should report these upstream. +src:*/contrib/openssl/* \ No newline at end of file From 266b231a3ce300bc1b7b6ea27d69b1084f283596 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Tue, 4 Aug 2020 20:06:20 +0200 Subject: [PATCH 068/411] Adding AES encryption tests in TestFlows. --- .../configs/clickhouse/common.xml | 6 + .../configs/clickhouse/config.d/logs.xml | 17 + .../configs/clickhouse/config.d/ports.xml | 5 + .../configs/clickhouse/config.d/remote.xml | 107 + .../configs/clickhouse/config.d/ssl.xml | 17 + .../configs/clickhouse/config.d/storage.xml | 20 + .../configs/clickhouse/config.d/zookeeper.xml | 10 + .../configs/clickhouse/config.xml | 436 ++ .../configs/clickhouse/ssl/dhparam.pem | 8 + .../configs/clickhouse/ssl/server.crt | 19 + .../configs/clickhouse/ssl/server.key | 28 + .../configs/clickhouse/users.xml | 133 + .../configs/clickhouse1/config.d/macros.xml | 8 + .../configs/clickhouse2/config.d/macros.xml | 8 + .../configs/clickhouse3/config.d/macros.xml | 8 + .../docker-compose/clickhouse-service.yml | 28 + .../docker-compose/docker-compose.yml | 73 + .../docker-compose/mysql-service.yml | 19 + .../docker-compose/zookeeper-service.yml | 18 + tests/testflows/aes_encryption/regression.py | 73 + .../aes_encryption/requirements/__init__.py | 1 + .../requirements/requirements.py | 3663 +++++++++++++++++ .../testflows/aes_encryption/tests/common.py | 162 + .../tests/compatibility/__init__.py | 0 .../tests/compatibility/feature.py | 17 + .../tests/compatibility/insert.py | 414 ++ .../tests/compatibility/mysql/__init__.py | 0 .../compatibility/mysql/database_engine.py | 196 + .../tests/compatibility/mysql/dictionary.py | 251 ++ .../tests/compatibility/mysql/feature.py | 18 + .../tests/compatibility/mysql/table_engine.py | 202 + .../compatibility/mysql/table_function.py | 183 + .../tests/compatibility/select.py | 186 + .../snapshots/insert.py.insert.snapshot | 192 + .../testflows/aes_encryption/tests/decrypt.py | 634 +++ .../aes_encryption/tests/decrypt_mysql.py | 521 +++ .../testflows/aes_encryption/tests/encrypt.py | 408 ++ .../aes_encryption/tests/encrypt_mysql.py | 326 ++ .../snapshots/encrypt.py.encrypt.snapshot | 2700 ++++++++++++ .../encrypt_mysql.py.encrypt_mysql.snapshot | 3060 ++++++++++++++ tests/testflows/regression.py | 1 + 41 files changed, 14176 insertions(+) create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/common.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/config.d/logs.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/config.d/ports.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/config.d/remote.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/config.d/ssl.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/config.d/storage.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/config.d/zookeeper.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/config.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/ssl/dhparam.pem create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/ssl/server.crt create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/ssl/server.key create mode 100644 tests/testflows/aes_encryption/configs/clickhouse/users.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse1/config.d/macros.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse2/config.d/macros.xml create mode 100644 tests/testflows/aes_encryption/configs/clickhouse3/config.d/macros.xml create mode 100644 tests/testflows/aes_encryption/docker-compose/clickhouse-service.yml create mode 100644 tests/testflows/aes_encryption/docker-compose/docker-compose.yml create mode 100644 tests/testflows/aes_encryption/docker-compose/mysql-service.yml create mode 100644 tests/testflows/aes_encryption/docker-compose/zookeeper-service.yml create mode 100755 tests/testflows/aes_encryption/regression.py create mode 100644 tests/testflows/aes_encryption/requirements/__init__.py create mode 100644 tests/testflows/aes_encryption/requirements/requirements.py create mode 100644 tests/testflows/aes_encryption/tests/common.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/__init__.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/feature.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/insert.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/mysql/__init__.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/mysql/database_engine.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/mysql/dictionary.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/mysql/feature.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/mysql/table_engine.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/mysql/table_function.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/select.py create mode 100644 tests/testflows/aes_encryption/tests/compatibility/snapshots/insert.py.insert.snapshot create mode 100644 tests/testflows/aes_encryption/tests/decrypt.py create mode 100644 tests/testflows/aes_encryption/tests/decrypt_mysql.py create mode 100644 tests/testflows/aes_encryption/tests/encrypt.py create mode 100644 tests/testflows/aes_encryption/tests/encrypt_mysql.py create mode 100644 tests/testflows/aes_encryption/tests/snapshots/encrypt.py.encrypt.snapshot create mode 100644 tests/testflows/aes_encryption/tests/snapshots/encrypt_mysql.py.encrypt_mysql.snapshot diff --git a/tests/testflows/aes_encryption/configs/clickhouse/common.xml b/tests/testflows/aes_encryption/configs/clickhouse/common.xml new file mode 100644 index 00000000000..df952b28c82 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/common.xml @@ -0,0 +1,6 @@ + + Europe/Moscow + 0.0.0.0 + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + diff --git a/tests/testflows/aes_encryption/configs/clickhouse/config.d/logs.xml b/tests/testflows/aes_encryption/configs/clickhouse/config.d/logs.xml new file mode 100644 index 00000000000..bdf1bbc11c1 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/config.d/logs.xml @@ -0,0 +1,17 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + system + part_log
+ 500 + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse/config.d/ports.xml b/tests/testflows/aes_encryption/configs/clickhouse/config.d/ports.xml new file mode 100644 index 00000000000..fbc6cea74c0 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/config.d/ports.xml @@ -0,0 +1,5 @@ + + + 8443 + 9440 + \ No newline at end of file diff --git a/tests/testflows/aes_encryption/configs/clickhouse/config.d/remote.xml b/tests/testflows/aes_encryption/configs/clickhouse/config.d/remote.xml new file mode 100644 index 00000000000..51be2a6e8e3 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/config.d/remote.xml @@ -0,0 +1,107 @@ + + + + + + true + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + + + + true + + clickhouse1 + 9440 + 1 + + + clickhouse2 + 9440 + 1 + + + clickhouse3 + 9440 + 1 + + + + + + + clickhouse1 + 9000 + + + + + clickhouse2 + 9000 + + + + + clickhouse3 + 9000 + + + + + + + clickhouse1 + 9440 + 1 + + + + + clickhouse2 + 9440 + 1 + + + + + clickhouse3 + 9440 + 1 + + + + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse/config.d/ssl.xml b/tests/testflows/aes_encryption/configs/clickhouse/config.d/ssl.xml new file mode 100644 index 00000000000..ca65ffd5e04 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/config.d/ssl.xml @@ -0,0 +1,17 @@ + + + + /etc/clickhouse-server/ssl/server.crt + /etc/clickhouse-server/ssl/server.key + none + true + + + true + none + + AcceptCertificateHandler + + + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse/config.d/storage.xml b/tests/testflows/aes_encryption/configs/clickhouse/config.d/storage.xml new file mode 100644 index 00000000000..618fd6b6d24 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/config.d/storage.xml @@ -0,0 +1,20 @@ + + + + + + 1024 + + + + + + + default + + + + + + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse/config.d/zookeeper.xml b/tests/testflows/aes_encryption/configs/clickhouse/config.d/zookeeper.xml new file mode 100644 index 00000000000..96270e7b645 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/config.d/zookeeper.xml @@ -0,0 +1,10 @@ + + + + + zookeeper + 2181 + + 15000 + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse/config.xml b/tests/testflows/aes_encryption/configs/clickhouse/config.xml new file mode 100644 index 00000000000..d34d2c35253 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/config.xml @@ -0,0 +1,436 @@ + + + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + 8123 + 9000 + + + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 9009 + + + + + + + + + + + + + + + + + + + + 4096 + 3 + + + 100 + + + + + + 8589934592 + + + 5368709120 + + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + /var/lib/clickhouse/user_files/ + + + /var/lib/clickhouse/access/ + + + users.xml + + + default + + + + + + default + + + + + + + + + false + + + + + + + + localhost + 9000 + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + localhost + 9000 + + + + + localhost + 1 + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + 7500 + + + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 + + + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 + + + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse/ssl/dhparam.pem b/tests/testflows/aes_encryption/configs/clickhouse/ssl/dhparam.pem new file mode 100644 index 00000000000..2e6cee0798d --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/ssl/dhparam.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEAua92DDli13gJ+//ZXyGaggjIuidqB0crXfhUlsrBk9BV1hH3i7fR +XGP9rUdk2ubnB3k2ejBStL5oBrkHm9SzUFSQHqfDjLZjKoUpOEmuDc4cHvX1XTR5 +Pr1vf5cd0yEncJWG5W4zyUB8k++SUdL2qaeslSs+f491HBLDYn/h8zCgRbBvxhxb +9qeho1xcbnWeqkN6Kc9bgGozA16P9NLuuLttNnOblkH+lMBf42BSne/TWt3AlGZf +slKmmZcySUhF8aKfJnLKbkBCFqOtFRh8zBA9a7g+BT/lSANATCDPaAk1YVih2EKb +dpc3briTDbRsiqg2JKMI7+VdULY9bh3EawIBAg== +-----END DH PARAMETERS----- diff --git a/tests/testflows/aes_encryption/configs/clickhouse/ssl/server.crt b/tests/testflows/aes_encryption/configs/clickhouse/ssl/server.crt new file mode 100644 index 00000000000..7ade2d96273 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/ssl/server.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI +4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T +4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU +7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj +sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg +pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC +kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j +4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr +85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C +L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD ++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L +P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp +0g== +-----END CERTIFICATE----- diff --git a/tests/testflows/aes_encryption/configs/clickhouse/ssl/server.key b/tests/testflows/aes_encryption/configs/clickhouse/ssl/server.key new file mode 100644 index 00000000000..f0fb61ac443 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/ssl/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn +DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L +OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus +TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 +JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F +KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 +ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP +TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f +7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN +MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 +212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu +plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr +/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu +Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa +CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g +1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 +bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq +cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez +on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K +GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 +nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU +2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh +vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa +dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr +vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 +KCBtmIrQLqzMpnBpLNuSY+Q= +-----END PRIVATE KEY----- diff --git a/tests/testflows/aes_encryption/configs/clickhouse/users.xml b/tests/testflows/aes_encryption/configs/clickhouse/users.xml new file mode 100644 index 00000000000..86b2cd9e1e3 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse/users.xml @@ -0,0 +1,133 @@ + + + + + + + + 10000000000 + + + 0 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + 1 + + + + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse1/config.d/macros.xml b/tests/testflows/aes_encryption/configs/clickhouse1/config.d/macros.xml new file mode 100644 index 00000000000..6cdcc1b440c --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse1/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse1 + 01 + 01 + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse2/config.d/macros.xml b/tests/testflows/aes_encryption/configs/clickhouse2/config.d/macros.xml new file mode 100644 index 00000000000..a114a9ce4ab --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse2/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse2 + 01 + 02 + + diff --git a/tests/testflows/aes_encryption/configs/clickhouse3/config.d/macros.xml b/tests/testflows/aes_encryption/configs/clickhouse3/config.d/macros.xml new file mode 100644 index 00000000000..904a27b0172 --- /dev/null +++ b/tests/testflows/aes_encryption/configs/clickhouse3/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse3 + 01 + 03 + + diff --git a/tests/testflows/aes_encryption/docker-compose/clickhouse-service.yml b/tests/testflows/aes_encryption/docker-compose/clickhouse-service.yml new file mode 100644 index 00000000000..9787b37abbb --- /dev/null +++ b/tests/testflows/aes_encryption/docker-compose/clickhouse-service.yml @@ -0,0 +1,28 @@ +version: '2.3' + +services: + clickhouse: + image: yandex/clickhouse-integration-test + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/:/etc/clickhouse-server/users.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + healthcheck: + test: clickhouse client --query='select 1' + interval: 3s + timeout: 2s + retries: 40 + start_period: 2s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/aes_encryption/docker-compose/docker-compose.yml b/tests/testflows/aes_encryption/docker-compose/docker-compose.yml new file mode 100644 index 00000000000..04a51ad7ec0 --- /dev/null +++ b/tests/testflows/aes_encryption/docker-compose/docker-compose.yml @@ -0,0 +1,73 @@ +version: '2.3' + +services: + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + mysql1: + extends: + file: mysql-service.yml + service: mysql + hostname: mysql1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/mysql1/database:/var/lib/mysql" + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + mysql1: + condition: service_healthy + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy diff --git a/tests/testflows/aes_encryption/docker-compose/mysql-service.yml b/tests/testflows/aes_encryption/docker-compose/mysql-service.yml new file mode 100644 index 00000000000..6924bccfad5 --- /dev/null +++ b/tests/testflows/aes_encryption/docker-compose/mysql-service.yml @@ -0,0 +1,19 @@ +version: '2.3' + +services: + mysql: + image: mysql:5.7.30 + restart: always + environment: + MYSQL_DATABASE: 'db' + MYSQL_USER: 'user' + MYSQL_PASSWORD: 'password' + MYSQL_ROOT_PASSWORD: 'password' + expose: + - '3306' + healthcheck: + test: mysql -D db -u user --password=password -e "select 1;" + interval: 3s + timeout: 2s + retries: 40 + start_period: 2s diff --git a/tests/testflows/aes_encryption/docker-compose/zookeeper-service.yml b/tests/testflows/aes_encryption/docker-compose/zookeeper-service.yml new file mode 100644 index 00000000000..f3df33358be --- /dev/null +++ b/tests/testflows/aes_encryption/docker-compose/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 3s + timeout: 2s + retries: 5 + start_period: 2s + security_opt: + - label:disable diff --git a/tests/testflows/aes_encryption/regression.py b/tests/testflows/aes_encryption/regression.py new file mode 100755 index 00000000000..e50ac0a3f8b --- /dev/null +++ b/tests/testflows/aes_encryption/regression.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +import sys +from testflows.core import * + +append_path(sys.path, "..") + +from helpers.cluster import Cluster +from helpers.argparser import argparser +from aes_encryption.requirements import * + +xfails = { + # encrypt + "encrypt/invalid key or iv length for mode/mode=\"'aes-???-gcm'\", key_len=??, iv_len=12, aad=True/iv is too short": + [(Fail, "known issue")], + "encrypt/invalid key or iv length for mode/mode=\"'aes-???-gcm'\", key_len=??, iv_len=12, aad=True/iv is too long": + [(Fail, "known issue")], + # encrypt_mysql + "encrypt_mysql/key or iv length for mode/mode=\"'aes-???-ecb'\", key_len=??, iv_len=None": + [(Fail, "https://altinity.atlassian.net/browse/CH-190")], + "encrypt_mysql/invalid parameters/iv not valid for mode": + [(Fail, "https://altinity.atlassian.net/browse/CH-190")], + "encrypt_mysql/invalid parameters/no parameters": + [(Fail, "https://altinity.atlassian.net/browse/CH-191")], + # decrypt_mysql + "decrypt_mysql/key or iv length for mode/mode=\"'aes-???-ecb'\", key_len=??, iv_len=None:": + [(Fail, "https://altinity.atlassian.net/browse/CH-190")], + # compatibility + "compatibility/insert/encrypt using materialized view/:": + [(Fail, "https://altinity.atlassian.net/browse/CH-193")], + "compatibility/insert/decrypt using materialized view/:": + [(Error, "https://altinity.atlassian.net/browse/CH-193")], + "compatibility/insert/aes encrypt mysql using materialized view/:": + [(Fail, "https://altinity.atlassian.net/browse/CH-193")], + "compatibility/insert/aes decrypt mysql using materialized view/:": + [(Error, "https://altinity.atlassian.net/browse/CH-193")], + "compatibility/select/decrypt unique": + [(Fail, "https://altinity.atlassian.net/browse/CH-193")], + "compatibility/mysql/:engine/decrypt/mysql_datatype='TEXT'/:": + [(Fail, "https://altinity.atlassian.net/browse/CH-194")], + "compatibility/mysql/:engine/decrypt/mysql_datatype='VARCHAR(100)'/:": + [(Fail, "https://altinity.atlassian.net/browse/CH-194")], + "compatibility/mysql/:engine/encrypt/mysql_datatype='TEXT'/:": + [(Fail, "https://altinity.atlassian.net/browse/CH-194")], + "compatibility/mysql/:engine/encrypt/mysql_datatype='VARCHAR(100)'/:": + [(Fail, "https://altinity.atlassian.net/browse/CH-194")] +} + +@TestFeature +@Name("aes encryption") +@ArgumentParser(argparser) +@Requirements( + RQ_SRS008_AES_Functions("1.0"), + RQ_SRS008_AES_Functions_DifferentModes("1.0") +) +@XFails(xfails) +def regression(self, local, clickhouse_binary_path): + """ClickHouse AES encryption functions regression module. + """ + nodes = { + "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), + } + + with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: + self.context.cluster = cluster + + Feature(run=load("aes_encryption.tests.encrypt", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.decrypt", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.encrypt_mysql", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.decrypt_mysql", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.compatibility.feature", "feature"), flags=TE) + +if main(): + regression() diff --git a/tests/testflows/aes_encryption/requirements/__init__.py b/tests/testflows/aes_encryption/requirements/__init__.py new file mode 100644 index 00000000000..02f7d430154 --- /dev/null +++ b/tests/testflows/aes_encryption/requirements/__init__.py @@ -0,0 +1 @@ +from .requirements import * diff --git a/tests/testflows/aes_encryption/requirements/requirements.py b/tests/testflows/aes_encryption/requirements/requirements.py new file mode 100644 index 00000000000..bae8b5cc3c1 --- /dev/null +++ b/tests/testflows/aes_encryption/requirements/requirements.py @@ -0,0 +1,3663 @@ +# These requirements were auto generated +# from software requirements specification (SRS) +# document by TestFlows v1.6.200731.1222107. +# Do not edit by hand but re-generate instead +# using 'tfs requirements generate' command. +from testflows.core import Requirement + +RQ_SRS008_AES_Functions = Requirement( + name='RQ.SRS008.AES.Functions', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support [AES] encryption functions to encrypt and decrypt data.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Compatability_MySQL = Requirement( + name='RQ.SRS008.AES.Functions.Compatability.MySQL', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support [AES] encryption functions compatible with [MySQL 5.7].\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Compatability_Dictionaries = Requirement( + name='RQ.SRS008.AES.Functions.Compatability.Dictionaries', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support encryption and decryption of data accessed on remote\n' + '[MySQL] servers using [MySQL Dictionary].\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Compatability_Engine_Database_MySQL = Requirement( + name='RQ.SRS008.AES.Functions.Compatability.Engine.Database.MySQL', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Database Engine],\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Compatability_Engine_Table_MySQL = Requirement( + name='RQ.SRS008.AES.Functions.Compatability.Engine.Table.MySQL', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Engine].\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Compatability_TableFunction_MySQL = Requirement( + name='RQ.SRS008.AES.Functions.Compatability.TableFunction.MySQL', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support encryption and decryption of data accessed using [MySQL Table Function].\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_DifferentModes = Requirement( + name='RQ.SRS008.AES.Functions.DifferentModes', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL allow different modes to be supported in a single SQL statement\n' + 'using explicit function parameters.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_DataFromMultipleSources = Requirement( + name='RQ.SRS008.AES.Functions.DataFromMultipleSources', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support handling encryption and decryption of data from multiple sources\n' + 'in the `SELECT` statement, including [ClickHouse] [MergeTree] table as well as [MySQL Dictionary],\n' + '[MySQL Database Engine], [MySQL Table Engine], and [MySQL Table Function]\n' + 'with possibly different encryption schemes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_SuppressOutputOfSensitiveValues = Requirement( + name='RQ.SRS008.AES.Functions.SuppressOutputOfSensitiveValues', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL suppress output of [AES] `string` and `key` parameters to the system log,\n' + 'error log, and `query_log` table to prevent leakage of sensitive values.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_InvalidParameters = Requirement( + name='RQ.SRS008.AES.Functions.InvalidParameters', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when parameters are invalid.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Mismatched_Key = Requirement( + name='RQ.SRS008.AES.Functions.Mismatched.Key', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return garbage for mismatched keys.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Mismatched_IV = Requirement( + name='RQ.SRS008.AES.Functions.Mismatched.IV', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return garbage for mismatched initialization vector for the modes that use it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Mismatched_AAD = Requirement( + name='RQ.SRS008.AES.Functions.Mismatched.AAD', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return garbage for mismatched additional authentication data for the modes that use it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Mismatched_Mode = Requirement( + name='RQ.SRS008.AES.Functions.Mismatched.Mode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error or garbage for mismatched mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Check_Performance = Requirement( + name='RQ.SRS008.AES.Functions.Check.Performance', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + 'Performance of [AES] encryption functions SHALL be measured.\n' + ), + link=None + ) + +RQ_SRS008_AES_Function_Check_Performance_BestCase = Requirement( + name='RQ.SRS008.AES.Function.Check.Performance.BestCase', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + 'Performance of [AES] encryption functions SHALL be checked for the best case\n' + 'scenario where there is one key, one initialization vector, and one large stream of data.\n' + ), + link=None + ) + +RQ_SRS008_AES_Function_Check_Performance_WorstCase = Requirement( + name='RQ.SRS008.AES.Function.Check.Performance.WorstCase', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + 'Performance of [AES] encryption functions SHALL be checked for the worst case\n' + 'where there are `N` keys, `N` initialization vectors and `N` very small streams of data.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Check_Compression = Requirement( + name='RQ.SRS008.AES.Functions.Check.Compression', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + 'Effect of [AES] encryption on column compression SHALL be measured.\n' + ), + link=None + ) + +RQ_SRS008_AES_Functions_Check_Compression_LowCardinality = Requirement( + name='RQ.SRS008.AES.Functions.Check.Compression.LowCardinality', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + 'Effect of [AES] encryption on the compression of a column with [LowCardinality] data type\n' + 'SHALL be measured.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function = Requirement( + name='RQ.SRS008.AES.Encrypt.Function', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes_encrypt` function to encrypt data using [AES].\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Syntax = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Syntax', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support the following syntax for the `aes_encrypt` function\n' + '\n' + '```sql\n' + 'aes_encrypt(plaintext, key, mode, [iv, aad])\n' + '```\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_NIST_TestVectors = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.NIST.TestVectors', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] `aes_encrypt` function output SHALL produce output that matches [NIST test vectors].\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_PlainText = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.PlainText', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `plaintext` accepting any data type as\n' + 'the first parameter to the `aes_encrypt` function that SHALL specify the data to be encrypted.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Key = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Key', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' + 'as the second parameter to the `aes_encrypt` function that SHALL specify the encryption key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' + 'to the `aes_encrypt` function that SHALL specify encryption key length and block encryption mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.ValuesFormat', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n' + 'of the `aes_encrypt` function where\n' + 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' + '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' + 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as\n' + '[CTR] and [GCM] as the values. For example, `aes-256-ofb`.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt`\n' + 'function is not valid with the exception where such a mode is supported by the underlying\n' + '[OpenSSL] implementation.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-gcm` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key.\n' + 'An `AEAD` 16-byte tag is appended to the resulting ciphertext according to\n' + 'the [RFC5116].\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-gcm` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key.\n' + 'An `AEAD` 16-byte tag is appended to the resulting ciphertext according to\n' + 'the [RFC5116].\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-gcm` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key.\n' + 'An `AEAD` 16-byte tag is appended to the resulting ciphertext according to\n' + 'the [RFC5116].\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_encrypt` function\n' + 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_InitializationVector = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.InitializationVector', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' + 'parameter to the `aes_encrypt` function that SHALL specify the initialization vector for block modes that require\n' + 'it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_AdditionalAuthenticatedData = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.AdditionalAuthenticatedData', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth\n' + 'parameter to the `aes_encrypt` function that SHALL specify the additional authenticated data\n' + 'for block modes that require it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Parameters_ReturnValue = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Parameters.ReturnValue', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return the encrypted value of the data\n' + 'using `String` data type as the result of `aes_encrypt` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.Key.Length.InvalidLengthError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_encrypt` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.InitializationVector.Length.InvalidLengthError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` length is specified and not of the exact size for the `aes_encrypt` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.InitializationVector.NotValidForMode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt` function for a mode that does not need it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.NotValidForMode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_encrypt` function for a mode that does not need it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AdditionalAuthenticationData.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_encrypt` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes\n' + 'or `iv` or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes\n' + 'or `iv` or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes\n' + 'or `iv` or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes\n' + 'and if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes\n' + 'or `iv` is not specified or is less than 8 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes\n' + 'or `iv` is not specified or is less than 8 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes\n' + 'or `iv` is not specified or is less than 8 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Encrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Encrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function = Requirement( + name='RQ.SRS008.AES.Decrypt.Function', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes_decrypt` function to decrypt data using [AES].\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Syntax = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Syntax', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support the following syntax for the `aes_decrypt` function\n' + '\n' + '```sql\n' + 'aes_decrypt(ciphertext, key, mode, [iv, aad])\n' + '```\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.CipherText', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `ciphertext` accepting `FixedString` or `String` data types as\n' + 'the first parameter to the `aes_decrypt` function that SHALL specify the data to be decrypted.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Key = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Key', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' + 'as the second parameter to the `aes_decrypt` function that SHALL specify the encryption key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' + 'to the `aes_decrypt` function that SHALL specify encryption key length and block encryption mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.ValuesFormat', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n' + 'of the `aes_decrypt` function where\n' + 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' + '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' + 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB] as well as\n' + '[CTR] and [GCM] as the values. For example, `aes-256-ofb`.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt`\n' + 'function is not valid with the exception where such a mode is supported by the underlying\n' + '[OpenSSL] implementation.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-gcm` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 128 bit key.\n' + 'An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' + 'the [RFC5116].\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-gcm` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 192 bit key.\n' + 'An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' + 'the [RFC5116].\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-gcm` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [GCM] block mode encryption with a 256 bit key.\n' + 'An [AEAD] 16-byte tag is expected present at the end of the ciphertext according to\n' + 'the [RFC5116].\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ctr` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ctr` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ctr` as the value for the `mode` parameter of the `aes_decrypt` function\n' + 'and [AES] algorithm SHALL use the [CTR] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_InitializationVector = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.InitializationVector', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' + 'parameter to the `aes_decrypt` function that SHALL specify the initialization vector for block modes that require\n' + 'it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_AdditionalAuthenticatedData = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.AdditionalAuthenticatedData', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aad` with `String` or `FixedString` data types as the optional fifth\n' + 'parameter to the `aes_decrypt` function that SHALL specify the additional authenticated data\n' + 'for block modes that require it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Parameters_ReturnValue = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Parameters.ReturnValue', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return the decrypted value of the data\n' + 'using `String` data type as the result of `aes_decrypt` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.Key.Length.InvalidLengthError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `key` length is not exact for the `aes_decrypt` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.InitializationVector.Length.InvalidLengthError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` is speficified and the length is not exact for the `aes_decrypt` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.InitializationVector.NotValidForMode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt` function\n' + 'for a mode that does not need it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.NotValidForMode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `aad` is specified for the `aes_decrypt` function\n' + 'for a mode that does not need it.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AdditionalAuthenticationData.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not limit the size of the `aad` parameter passed to the `aes_decrypt` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ecb` and `key` is not 16 bytes\n' + 'or `iv` or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ecb` and `key` is not 24 bytes\n' + 'or `iv` or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ecb` and `key` is not 32 bytes\n' + 'or `iv` or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cbc` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cbc` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cbc` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb1` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb1` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb1` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb8` and `key` is not 16 bytes\n' + 'and if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb8` and `key` is not 24 bytes\n' + 'or `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb8` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-cfb128` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-cfb128` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-cfb128` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ofb` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ofb` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ofb` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes or `aad` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-GCM.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-gcm` and `key` is not 16 bytes\n' + 'or `iv` is not specified or is less than 8 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-GCM.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-gcm` and `key` is not 24 bytes\n' + 'or `iv` is not specified or is less than 8 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-GCM.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-gcm` and `key` is not 32 bytes\n' + 'or `iv` is not specified or is less than 8 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-128-CTR.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-128-ctr` and `key` is not 16 bytes\n' + 'or if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-192-CTR.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-192-ctr` and `key` is not 24 bytes\n' + 'or if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_Decrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.Decrypt.Function.AES-256-CTR.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt` function is set to `aes-256-ctr` and `key` is not 32 bytes\n' + 'or if specified `iv` is not 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes_encrypt_mysql` function to encrypt data using [AES].\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Syntax = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Syntax', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support the following syntax for the `aes_encrypt_mysql` function\n' + '\n' + '```sql\n' + 'aes_encrypt_mysql(plaintext, key, mode, [iv])\n' + '```\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_PlainText = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.PlainText', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `plaintext` accepting any data type as\n' + 'the first parameter to the `aes_encrypt_mysql` function that SHALL specify the data to be encrypted.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Key = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Key', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' + 'as the second parameter to the `aes_encrypt_mysql` function that SHALL specify the encryption key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' + 'to the `aes_encrypt_mysql` function that SHALL specify encryption key length and block encryption mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_ValuesFormat = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.ValuesFormat', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n' + 'of the `aes_encrypt_mysql` function where\n' + 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' + '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' + 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_Invalid = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_encrypt_mysql`\n' + 'function is not valid with the exception where such a mode is supported by the underlying\n' + '[OpenSSL] implementation.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_encrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the\n' + '`aes_encrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the\n' + '`aes_encrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the\n' + '`aes_encrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the\n' + '`aes_encrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the\n' + '`aes_encrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the\n' + '`aes_encrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_InitializationVector = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.InitializationVector', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' + 'parameter to the `aes_encrypt_mysql` function that SHALL specify the initialization vector for block modes that require\n' + 'it.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_ReturnValue = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Parameters.ReturnValue', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return the encrypted value of the data\n' + 'using `String` data type as the result of `aes_encrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooShortError = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooShortError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_encrypt_mysql`\n' + 'function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooLong = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.Key.Length.TooLong', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required\n' + 'for the `aes_encrypt_mysql` function for a given block mode.\n' + '\n' + '```python\n' + 'def fold_key(key, cipher_key_size):\n' + ' key = list(key) if not isinstance(key, (list, tuple)) else key\n' + '\t folded_key = key[:cipher_key_size]\n' + '\t for i in range(cipher_key_size, len(key)):\n' + '\t\t print(i % cipher_key_size, i)\n' + '\t\t folded_key[i % cipher_key_size] ^= key[i]\n' + '\t return folded_key\n' + '```\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooShortError = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooShortError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum\n' + 'that is required for the `aes_encrypt_mysql` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooLong = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.Length.TooLong', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and\n' + 'its length is longer than required for the `aes_encrypt_mysql` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.InitializationVector.NotValidForMode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_encrypt_mysql`\n' + 'function for a mode that does not need it.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes\n' + 'or `iv` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes\n' + 'or `iv` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes\n' + 'or `iv` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes\n' + 'and if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Encrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_encrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes_decrypt_mysql` function to decrypt data using [AES].\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Syntax = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Syntax', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support the following syntax for the `aes_decrypt_mysql` function\n' + '\n' + '```sql\n' + 'aes_decrypt_mysql(ciphertext, key, mode, [iv])\n' + '```\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.CipherText', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `ciphertext` accepting any data type as\n' + 'the first parameter to the `aes_decrypt_mysql` function that SHALL specify the data to be decrypted.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Key = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Key', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `key` with `String` or `FixedString` data types\n' + 'as the second parameter to the `aes_decrypt_mysql` function that SHALL specify the encryption key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `mode` with `String` or `FixedString` data types as the third parameter\n' + 'to the `aes_decrypt_mysql` function that SHALL specify encryption key length and block encryption mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_ValuesFormat = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.ValuesFormat', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support values of the form `aes-[key length]-[mode]` for the `mode` parameter\n' + 'of the `aes_decrypt_mysql` function where\n' + 'the `key_length` SHALL specifies the length of the key and SHALL accept\n' + '`128`, `192`, or `256` as the values and the `mode` SHALL specify the block encryption\n' + 'mode and SHALL accept [ECB], [CBC], [CFB1], [CFB8], [CFB128], or [OFB]. For example, `aes-256-ofb`.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_Invalid = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the specified value for the `mode` parameter of the `aes_decrypt_mysql`\n' + 'function is not valid with the exception where such a mode is supported by the underlying\n' + '[OpenSSL] implementation.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_ECB = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_ECB = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_ECB = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-ECB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ecb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [ECB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CBC = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CBC = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CBC = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CBC', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cbc` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CBC] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB1 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB1 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB1 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB1', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb1` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB1] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB8 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB8 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB8 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb8` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB8] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CFB128 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CFB128 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CFB128 = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CFB128', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-cfb128` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [CFB128] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_OFB = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-128-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 128 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_OFB = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-192-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 192 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_OFB = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-OFB', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `aes-256-ofb` as the value for the `mode` parameter of the `aes_decrypt_mysql` function\n' + 'and [AES] algorithm SHALL use the [OFB] block mode encryption with a 256 bit key.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-GCM.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-128-gcm` is specified as the value for the `mode` parameter of the\n' + '`aes_decrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-GCM.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-192-gcm` is specified as the value for the `mode` parameter of the\n' + '`aes_decrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-GCM.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-256-gcm` is specified as the value for the `mode` parameter of the\n' + '`aes_decrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-128-CTR.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-128-ctr` is specified as the value for the `mode` parameter of the\n' + '`aes_decrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-192-CTR.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-192-ctr` is specified as the value for the `mode` parameter of the\n' + '`aes_decrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.Mode.Value.AES-256-CTR.Error', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if `aes-256-ctr` is specified as the value for the `mode` parameter of the\n' + '`aes_decrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_InitializationVector = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.InitializationVector', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `iv` with `String` or `FixedString` data types as the optional fourth\n' + 'parameter to the `aes_decrypt_mysql` function that SHALL specify the initialization vector for block modes that require\n' + 'it.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_ReturnValue = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Parameters.ReturnValue', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return the decrypted value of the data\n' + 'using `String` data type as the result of `aes_decrypt_mysql` function.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooShortError = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooShortError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `key` length is less than the minimum for the `aes_decrypt_mysql`\n' + 'function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooLong = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.Key.Length.TooLong', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use folding algorithm specified below if the `key` length is longer than required\n' + 'for the `aes_decrypt_mysql` function for a given block mode.\n' + '\n' + '```python\n' + 'def fold_key(key, cipher_key_size):\n' + ' key = list(key) if not isinstance(key, (list, tuple)) else key\n' + '\t folded_key = key[:cipher_key_size]\n' + '\t for i in range(cipher_key_size, len(key)):\n' + '\t\t print(i % cipher_key_size, i)\n' + '\t\t folded_key[i % cipher_key_size] ^= key[i]\n' + '\t return folded_key\n' + '```\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooShortError = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooShortError', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` length is specified and is less than the minimum\n' + 'that is required for the `aes_decrypt_mysql` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooLong = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.Length.TooLong', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use the first `N` bytes that are required if the `iv` is specified and\n' + 'its length is longer than required for the `aes_decrypt_mysql` function for a given block mode.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.InitializationVector.NotValidForMode', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `iv` is specified for the `aes_decrypt_mysql`\n' + 'function for a mode that does not need it.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ecb` and `key` is less than 16 bytes\n' + 'or `iv` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ecb` and `key` is less than 24 bytes\n' + 'or `iv` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-ECB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ecb` and `key` is less than 32 bytes\n' + 'or `iv` is specified.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cbc` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cbc` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CBC.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cbc` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb1` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb1` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB1.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb1` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb8` and `key` is less than 16 bytes\n' + 'and if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb8` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB8.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb8` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-cfb128` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-cfb128` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-CFB128.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-cfb128` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-128-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-128-ofb` and `key` is less than 16 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-192-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-192-ofb` and `key` is less than 24 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) + +RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length = Requirement( + name='RQ.SRS008.AES.MySQL.Decrypt.Function.AES-256-OFB.KeyAndInitializationVector.Length', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error when `mode` for the `aes_decrypt_mysql` function is set to `aes-256-ofb` and `key` is less than 32 bytes\n' + 'or if specified `iv` is less than 16 bytes.\n' + ), + link=None + ) diff --git a/tests/testflows/aes_encryption/tests/common.py b/tests/testflows/aes_encryption/tests/common.py new file mode 100644 index 00000000000..5ed582563fb --- /dev/null +++ b/tests/testflows/aes_encryption/tests/common.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- +modes = [ + # mode, key_len, iv_len, aad + ("'aes-128-ecb'", 16, None, None), + ("'aes-192-ecb'", 24, None, None), + ("'aes-256-ecb'", 32, None, None), + # cbc + ("'aes-128-cbc'", 16, None, None), + ("'aes-192-cbc'", 24, None, None), + ("'aes-256-cbc'", 32, None, None), + ("'aes-128-cbc'", 16, 16, None), + ("'aes-192-cbc'", 24, 16, None), + ("'aes-256-cbc'", 32, 16, None), + # cfb1 + ("'aes-128-cfb1'", 16, None, None), + ("'aes-192-cfb1'", 24, None, None), + ("'aes-256-cfb1'", 32, None, None), + ("'aes-128-cfb1'", 16, 16, None), + ("'aes-192-cfb1'", 24, 16, None), + ("'aes-256-cfb1'", 32, 16, None), + # cfb8 + ("'aes-128-cfb8'", 16, None, None), + ("'aes-192-cfb8'", 24, None, None), + ("'aes-256-cfb8'", 32, None, None), + ("'aes-128-cfb8'", 16, 16, None), + ("'aes-192-cfb8'", 24, 16, None), + ("'aes-256-cfb8'", 32, 16, None), + # cfb128 + ("'aes-128-cfb128'", 16, None, None), + ("'aes-192-cfb128'", 24, None, None), + ("'aes-256-cfb128'", 32, None, None), + ("'aes-128-cfb128'", 16, 16, None), + ("'aes-192-cfb128'", 24, 16, None), + ("'aes-256-cfb128'", 32, 16, None), + # ofb + ("'aes-128-ofb'", 16, None, None), + ("'aes-192-ofb'", 24, None, None), + ("'aes-256-ofb'", 32, None, None), + ("'aes-128-ofb'", 16, 16, None), + ("'aes-192-ofb'", 24, 16, None), + ("'aes-256-ofb'", 32, 16, None), + # gcm + ("'aes-128-gcm'", 16, 12, None), + ("'aes-192-gcm'", 24, 12, None), + ("'aes-256-gcm'", 32, 12, None), + ("'aes-128-gcm'", 16, 12, True), + ("'aes-192-gcm'", 24, 12, True), + ("'aes-256-gcm'", 32, 12, True), + # ctr + ("'aes-128-ctr'", 16, None, None), + ("'aes-192-ctr'", 24, None, None), + ("'aes-256-ctr'", 32, None, None), + ("'aes-128-ctr'", 16, 16, None), + ("'aes-192-ctr'", 24, 16, None), + ("'aes-256-ctr'", 32, 16, None), +] + +mysql_modes = [ + # mode, key_len, iv_len + ("'aes-128-ecb'", 16, None), + ("'aes-128-ecb'", 24, None), + ("'aes-192-ecb'", 24, None), + ("'aes-192-ecb'", 32, None), + ("'aes-256-ecb'", 32, None), + ("'aes-256-ecb'", 64, None), + # cbc + ("'aes-128-cbc'", 16, None), + ("'aes-192-cbc'", 24, None), + ("'aes-256-cbc'", 32, None), + ("'aes-128-cbc'", 16, 16), + ("'aes-128-cbc'", 24, 24), + ("'aes-192-cbc'", 24, 16), + ("'aes-192-cbc'", 32, 32), + ("'aes-256-cbc'", 32, 16), + ("'aes-256-cbc'", 64, 64), + # cfb1 + ("'aes-128-cfb1'", 16, None), + ("'aes-192-cfb1'", 24, None), + ("'aes-256-cfb1'", 32, None), + ("'aes-128-cfb1'", 16, 16), + ("'aes-128-cfb1'", 24, 24), + ("'aes-192-cfb1'", 24, 16), + ("'aes-192-cfb1'", 32, 32), + ("'aes-256-cfb1'", 32, 16), + ("'aes-256-cfb1'", 64, 64), + # cfb8 + ("'aes-128-cfb8'", 16, None), + ("'aes-192-cfb8'", 24, None), + ("'aes-256-cfb8'", 32, None), + ("'aes-128-cfb8'", 16, 16), + ("'aes-128-cfb8'", 24, 24), + ("'aes-192-cfb8'", 24, 16), + ("'aes-192-cfb8'", 32, 32), + ("'aes-256-cfb8'", 32, 16), + ("'aes-256-cfb8'", 64, 64), + # cfb128 + ("'aes-128-cfb128'", 16, None), + ("'aes-192-cfb128'", 24, None), + ("'aes-256-cfb128'", 32, None), + ("'aes-128-cfb128'", 16, 16), + ("'aes-128-cfb128'", 24, 24), + ("'aes-192-cfb128'", 24, 16), + ("'aes-192-cfb128'", 32, 32), + ("'aes-256-cfb128'", 32, 16), + ("'aes-256-cfb128'", 64, 64), + # ofb + ("'aes-128-ofb'", 16, None), + ("'aes-192-ofb'", 24, None), + ("'aes-256-ofb'", 32, None), + ("'aes-128-ofb'", 16, 16), + ("'aes-128-ofb'", 24, 24), + ("'aes-192-ofb'", 24, 16), + ("'aes-192-ofb'", 32, 32), + ("'aes-256-ofb'", 32, 16), + ("'aes-256-ofb'", 64, 64), +] + +plaintexts = [ + ("bytes", "unhex('0')"), + ("emptystring", "''"), + ("utf8string", "'Gãńdåłf_Thê_Gręât'"), + ("utf8fixedstring", "toFixedString('Gãńdåłf_Thê_Gręât', 24)"), + ("String", "'1'"), + ("FixedString", "toFixedString('1', 1)"), + ("UInt8", "toUInt8('1')"), + ("UInt16", "toUInt16('1')"), + ("UInt32", "toUInt32('1')"), + ("UInt64", "toUInt64('1')"), + ("Int8", "toInt8('1')"), + ("Int16", "toInt16('1')"), + ("Int32", "toInt32('1')"), + ("Int64", "toInt64('1')"), + ("Float32", "toFloat32('1')"), + ("Float64", "toFloat64('1')"), + ("Decimal32", "toDecimal32(2, 4)"), + ("Decimal64", "toDecimal64(2, 4)"), + ("Decimal128", "toDecimal128(2, 4)"), + ("UUID", "toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')"), + ("Date", "toDate('2020-01-01')"), + ("DateTime", "toDateTime('2020-01-01 20:01:02')"), + ("DateTime64", "toDateTime64('2020-01-01 20:01:02.123', 3)"), + ("LowCardinality", "toLowCardinality('1')"), + ("Array", "[1,2]"), + #("Tuple", "(1,'a')") - not supported + #("Nullable, "Nullable(X)") - not supported + ("NULL", "toDateOrNull('foo')"), + ("IPv4", "toIPv4('171.225.130.45')"), + ("IPv6", "toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001')"), + ("Enum8", r"CAST('a', 'Enum8(\'a\' = 1, \'b\' = 2)')"), + ("Enum16", r"CAST('a', 'Enum16(\'a\' = 1, \'b\' = 2)')"), +] + +_hex = hex + +def hex(s): + """Convert string to hex. + """ + if isinstance(s, str): + return "".join(['%X' % ord(c) for c in s]) + if isinstance(s, bytes): + return "".join(['%X' % c for c in s]) + return _hex(s) diff --git a/tests/testflows/aes_encryption/tests/compatibility/__init__.py b/tests/testflows/aes_encryption/tests/compatibility/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/aes_encryption/tests/compatibility/feature.py b/tests/testflows/aes_encryption/tests/compatibility/feature.py new file mode 100644 index 00000000000..5ef547e43f4 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/feature.py @@ -0,0 +1,17 @@ +from testflows.core import * + +from aes_encryption.requirements import * + +@TestFeature +@Name("compatibility") +@Requirements( + RQ_SRS008_AES_Functions_DataFromMultipleSources("1.0") +) +def feature(self, node="clickhouse1"): + """Check encryption functions usage compatibility. + """ + self.context.node = self.context.cluster.node(node) + + Feature(run=load("aes_encryption.tests.compatibility.insert", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.compatibility.select", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.compatibility.mysql.feature", "feature"), flags=TE) \ No newline at end of file diff --git a/tests/testflows/aes_encryption/tests/compatibility/insert.py b/tests/testflows/aes_encryption/tests/compatibility/insert.py new file mode 100644 index 00000000000..6ddcc11b584 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/insert.py @@ -0,0 +1,414 @@ +import os +import textwrap +from contextlib import contextmanager +from importlib.machinery import SourceFileLoader + +from testflows.core import * +from testflows.core.name import basename +from testflows.asserts.helpers import varname +from testflows.asserts import values, error, snapshot + +from aes_encryption.tests.common import modes, mysql_modes + +@contextmanager +def table(name): + node = current().context.node + try: + with Given("table"): + sql = f""" + CREATE TABLE {name} + ( + date Nullable(Date), + name Nullable(String), + secret Nullable(String) + ) + ENGINE = Memory() + """ + with By("dropping table if exists"): + node.query(f"DROP TABLE IF EXISTS {name}") + with And("creating a table"): + node.query(textwrap.dedent(sql)) + yield + finally: + with Finally("I drop the table", flags=TE): + node.query(f"DROP TABLE IF EXISTS {name}") + +@contextmanager +def mv_transform(table, transform): + node = current().context.node + try: + with Given("tables for input transformation"): + with By("creating Null input table"): + sql = f""" + CREATE TABLE {table}_input + ( + date Nullable(Date), + name Nullable(String), + secret Nullable(String), + mode String, + key String, + iv String, + aad String + ) + ENGINE=Null() + """ + node.query(textwrap.dedent(sql)) + + with And("creating materialized view table"): + sql = f""" + CREATE MATERIALIZED VIEW {table}_input_mv TO {table} AS + SELECT date, name, {transform} + FROM {table}_input + """ + node.query(textwrap.dedent(sql)) + yield + finally: + with Finally("I drop tables for input transformation", flags=TE): + with By("dropping materialized view table", flags=TE): + node.query(f"DROP TABLE IF EXISTS {table}_input_mv") + + with And("dropping Null input table", flags=TE): + node.query(f"DROP TABLE IF EXISTS {table}_input") + +@TestScenario +def encrypt_using_materialized_view(self): + """Check that we can use `encrypt` function when inserting + data into a table using a materialized view for input + data transformation. + """ + node = self.context.node + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_aad = None if not aad_len else f"'{aad}'" + example_transform = f"encrypt(mode, secret, key{', iv' if example_iv else ''}{', aad' if example_aad else ''})" + + with table("user_data"): + with mv_transform("user_data", example_transform): + with When("I insert encrypted data"): + node.query(f""" + INSERT INTO user_data_input + (date, name, secret, mode, key) + VALUES + ('2020-01-01', 'user0', 'user0_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}), + ('2020-01-02', 'user1', 'user1_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}), + ('2020-01-03', 'user2', 'user2_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) + """) + + with And("I read inserted data back"): + node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date") + + with Then("output must match the snapshot"): + with values() as that: + assert that(snapshot(r.output.strip(), "insert", name=f"encrypt_mv_example_{varname(basename(self.name))}")), error() + +@TestScenario +def aes_encrypt_mysql_using_materialized_view(self): + """Check that we can use `aes_encrypt_mysql` function when inserting + data into a table using a materialized view for input + data transformation. + """ + node = self.context.node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + aad = "some random aad" + + for mode, key_len, iv_len in mysql_modes: + with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_transform = f"aes_encrypt_mysql(mode, secret, key{', iv' if example_iv else ''})" + + with table("user_data"): + with mv_transform("user_data", example_transform): + with When("I insert encrypted data"): + node.query(f""" + INSERT INTO user_data_input + (date, name, secret, mode, key) + VALUES + ('2020-01-01', 'user0', 'user0_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}), + ('2020-01-02', 'user1', 'user1_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}), + ('2020-01-03', 'user2', 'user2_secret', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}) + """) + + with And("I read inserted data back"): + node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date") + + with Then("output must match the snapshot"): + with values() as that: + assert that(snapshot(r.output.strip(), "insert", name=f"aes_encrypt_mysql_mv_example_{varname(basename(self.name))}")), error() + +@TestScenario +def encrypt_using_input_table_function(self): + """Check that we can use `encrypt` function when inserting + data into a table using insert select and `input()` table + function. + """ + node = self.context.node + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_aad = None if not aad_len else f"'{aad}'" + example_transform = f"encrypt({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''}{(', ' + example_aad) if example_aad else ''})" + + with table("user_data"): + with When("I insert encrypted data"): + node.query(f""" + INSERT INTO + user_data + SELECT + date, name, {example_transform} + FROM + input('date Date, name String, secret String') + FORMAT Values ('2020-01-01', 'user0', 'user0_secret'), ('2020-01-02', 'user1', 'user1_secret'), ('2020-01-03', 'user2', 'user2_secret') + """) + + with And("I read inserted data back"): + r = node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date") + + with Then("output must match the snapshot"): + with values() as that: + assert that(snapshot(r.output.strip(), "insert", name=f"encrypt_input_example_{varname(basename(example.name))}")), error() + +@TestScenario +def aes_encrypt_mysql_using_input_table_function(self): + """Check that we can use `aes_encrypt_mysql` function when inserting + data into a table using insert select and `input()` table + function. + """ + node = self.context.node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + aad = "some random aad" + + for mode, key_len, iv_len in mysql_modes: + with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_transform = f"aes_encrypt_mysql({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''})" + + with table("user_data"): + with When("I insert encrypted data"): + node.query(f""" + INSERT INTO + user_data + SELECT + date, name, {example_transform} + FROM + input('date Date, name String, secret String') + FORMAT Values ('2020-01-01', 'user0', 'user0_secret'), ('2020-01-02', 'user1', 'user1_secret'), ('2020-01-03', 'user2', 'user2_secret') + """) + + with And("I read inserted data back"): + r = node.query("SELECT date, name, hex(secret) FROM user_data ORDER BY date") + + with Then("output must match the snapshot"): + with values() as that: + assert that(snapshot(r.output.strip(), "insert", name=f"aes_encrypt_mysql_input_example_{varname(basename(example.name))}")), error() + +@TestScenario +def decrypt_using_materialized_view(self): + """Check that we can use `decrypt` function when inserting + data into a table using a materialized view for input + data transformation. + """ + node = self.context.node + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_aad = None if not aad_len else f"'{aad}'" + example_transform = f"decrypt(mode, secret, key{', iv' if example_iv else ''}{', aad' if example_aad else ''})" + + with Given("I have ciphertexts"): + example_name = basename(example.name) + ciphertexts = getattr(snapshot_module, varname(f"encrypt_mv_example_{example_name}")) + example_ciphertexts = ["'{}'".format(l.split("\t")[-1].strup("'")) for l in ciphertexts.split("\n")] + + with table("user_data"): + with mv_transform("user_data", example_transform): + with When("I insert encrypted data"): + node.query(f""" + INSERT INTO user_data_input + (date, name, secret, mode, key) + VALUES + ('2020-01-01', 'user0', 'unhex({example_ciphertexts[0]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}), + ('2020-01-02', 'user1', 'unhex({example_ciphertexts[1]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}), + ('2020-01-03', 'user2', 'unhex({example_ciphertexts[2]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) + """) + + with And("I read inserted data back"): + r = node.query("SELECT date, name, secret FROM user_data ORDER BY date") + + with Then("output must match the expected"): + expected = r"""'2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret'""" + assert r.output == expected, error() + +@TestScenario +def aes_decrypt_mysql_using_materialized_view(self): + """Check that we can use `aes_decrypt_mysql` function when inserting + data into a table using a materialized view for input + data transformation. + """ + node = self.context.node + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_aad = None if not aad_len else f"'{aad}'" + example_transform = f"aes_decrypt_mysql(mode, secret, key{', iv' if example_iv else ''})" + + with Given("I have ciphertexts"): + example_name = basename(example.name) + ciphertexts = getattr(snapshot_module, varname(f"aes_encrypt_mysql_mv_example_{example_name}")) + example_ciphertexts = ["'{}'".format(l.split("\t")[-1].strup("'")) for l in ciphertexts.split("\n")] + + with table("user_data"): + with mv_transform("user_data", example_transform): + with When("I insert encrypted data"): + node.query(f""" + INSERT INTO user_data_input + (date, name, secret, mode, key) + VALUES + ('2020-01-01', 'user0', 'unhex({example_ciphertexts[0]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}), + ('2020-01-02', 'user1', 'unhex({example_ciphertexts[1]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}), + ('2020-01-03', 'user2', 'unhex({example_ciphertexts[2]})', {example_mode}, {example_key}{(", " + example_iv) if example_iv else ""}) + """) + + with And("I read inserted data back"): + r = node.query("SELECT date, name, secret FROM user_data ORDER BY date") + + with Then("output must match the expected"): + expected = r"""'2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret'""" + assert r.output == expected, error() + +@TestScenario +def decrypt_using_input_table_function(self): + """Check that we can use `decrypt` function when inserting + data into a table using insert select and `input()` table + function. + """ + node = self.context.node + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_aad = None if not aad_len else f"'{aad}'" + example_transform = f"decrypt({mode}, unhex(secret), {example_key}{(', ' + example_iv) if example_iv else ''}{(', ' + example_aad) if example_aad else ''})" + + with Given("I have ciphertexts"): + example_name = basename(example.name) + ciphertexts = getattr(snapshot_module, varname(f"encrypt_input_example_{example_name}")) + example_ciphertexts = [l.split("\\t")[-1].strip("'") for l in ciphertexts.split("\\n")] + + with table("user_data"): + with When("I insert decrypted data"): + node.query(textwrap.dedent(f""" + INSERT INTO + user_data + SELECT + date, name, {example_transform} + FROM + input('date Date, name String, secret String') + FORMAT Values ('2020-01-01', 'user0', '{example_ciphertexts[0]}'), ('2020-01-02', 'user1', '{example_ciphertexts[1]}'), ('2020-01-03', 'user2', '{example_ciphertexts[2]}') + """)) + + with And("I read inserted data back"): + r = node.query("SELECT date, name, secret FROM user_data ORDER BY date") + + expected = """2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret""" + with Then("output must match the expected", description=expected): + assert r.output == expected, error() + +@TestScenario +def aes_decrypt_mysql_using_input_table_function(self): + """Check that we can use `aes_decrypt_mysql` function when inserting + data into a table using insert select and `input()` table + function. + """ + node = self.context.node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + aad = "some random aad" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "insert.py.insert.snapshot")).load_module() + + for mode, key_len, iv_len in mysql_modes: + with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}""") as example: + example_key = f"'{key[:key_len]}'" + example_mode = mode + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_transform = f"aes_decrypt_mysql({mode}, unhex(secret), {example_key}{(', ' + example_iv) if example_iv else ''})" + + with Given("I have ciphertexts"): + example_name = basename(example.name) + ciphertexts = getattr(snapshot_module, varname(f"aes_encrypt_mysql_input_example_{example_name}")) + example_ciphertexts = [l.split("\\t")[-1].strip("'") for l in ciphertexts.split("\\n")] + + with table("user_data"): + with When("I insert decrypted data"): + node.query(textwrap.dedent(f""" + INSERT INTO + user_data + SELECT + date, name, {example_transform} + FROM + input('date Date, name String, secret String') + FORMAT Values ('2020-01-01', 'user0', '{example_ciphertexts[0]}'), ('2020-01-02', 'user1', '{example_ciphertexts[1]}'), ('2020-01-03', 'user2', '{example_ciphertexts[2]}') + """)) + + with And("I read inserted data back"): + r = node.query("SELECT date, name, secret FROM user_data ORDER BY date") + + expected = """2020-01-01\tuser0\tuser0_secret\n2020-01-02\tuser1\tuser1_secret\n2020-01-03\tuser2\tuser2_secret""" + with Then("output must match the expected", description=expected): + assert r.output == expected, error() + +@TestFeature +@Name("insert") +def feature(self, node="clickhouse1"): + """Check encryption functions when used during data insertion into a table. + """ + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/__init__.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/database_engine.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/database_engine.py new file mode 100644 index 00000000000..3547dc95ab0 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/mysql/database_engine.py @@ -0,0 +1,196 @@ +import textwrap +from contextlib import contextmanager + +from testflows.core import * +from testflows.asserts import error + +from aes_encryption.requirements import * +from aes_encryption.tests.common import mysql_modes, hex + +@contextmanager +def table(name, node, mysql_node, secret_type): + """Create a table that can be accessed using MySQL database engine. + """ + try: + with Given("table in MySQL"): + sql = f""" + CREATE TABLE {name}( + id INT NOT NULL AUTO_INCREMENT, + date DATE, + name VARCHAR(100), + secret {secret_type}, + PRIMARY KEY ( id ) + ); + """ + with When("I drop the table if exists"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + with And("I create a table"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I create a database using MySQL database engine"): + sql = f""" + CREATE DATABASE mysql_db + ENGINE = MySQL('{mysql_node.name}:3306', 'db', 'user', 'password') + """ + with When("I drop database if exists"): + node.query(f"DROP DATABASE IF EXISTS mysql_db") + with And("I create database"): + node.query(textwrap.dedent(sql)) + yield + + finally: + with And("I drop the database that is using MySQL database engine", flags=TE): + node.query(f"DROP DATABASE IF EXISTS mysql_db") + + with And("I drop a table in MySQL", flags=TE): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + +@TestOutline(Scenario) +@Examples("mysql_datatype", [ + ("VARBINARY(100)",), + #("VARCHAR(100)",), + ("BLOB", ), + #("TEXT",) +]) +def decrypt(self, mysql_datatype): + """Check that when using a table provided by MySQL database engine that + contains a column encrypted in MySQL stored using specified data type + I can decrypt data in the column using the `decrypt` and `aes_decrypt_mysql` + functions in the select query. + """ + node = self.context.node + mysql_node = self.context.mysql_node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for func in ["decrypt", "aes_decrypt_mysql"]: + for mode, key_len, iv_len in mysql_modes: + exact_key_size = int(mode.split("-")[1])//8 + + if "ecb" not in mode and not iv_len: + continue + if func == "decrypt": + if iv_len and iv_len != 16: + continue + if key_len != exact_key_size: + continue + + with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with table("user_data", node, mysql_node, mysql_datatype): + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + + with When("I insert encrypted data in MySQL"): + sql = f""" + SET block_encryption_mode = {example_mode}; + INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"})); + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read encrypted data in MySQL to make sure it is valid"): + sql = f""" + SET block_encryption_mode = {example_mode}; + SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data; + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read raw encrypted data in MySQL"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0) + + with And("I read raw data using MySQL database engine"): + output = node.query("SELECT id, date, name, hex(secret) AS secret FROM mysql_db.user_data") + + with And("I read decrypted data using MySQL database engine"): + output = node.query(f"""SELECT hex({func}({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_db.user_data""").output.strip() + + with Then("output should match the original plain text"): + assert output == hex("secret"), error() + +@TestOutline(Scenario) +@Examples("mysql_datatype", [ + ("VARBINARY(100)",), + #("VARCHAR(100)",), + ("BLOB", ), + #("TEXT",) +]) +def encrypt(self, mysql_datatype): + """Check that when using a table provided by MySQL database engine that + we can encrypt data during insert using the `aes_encrypt_mysql` function + and decrypt it in MySQL. + """ + node = self.context.node + mysql_node = self.context.mysql_node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for func in ["encrypt", "aes_encrypt_mysql"]: + for mode, key_len, iv_len in mysql_modes: + exact_key_size = int(mode.split("-")[1])//8 + + if "ecb" not in mode and not iv_len: + continue + if func == "encrypt": + if iv_len and iv_len != 16: + continue + if key_len != exact_key_size: + continue + + with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with table("user_data", node, mysql_node, mysql_datatype): + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_transform = f"{func}({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''})" + + with When("I insert encrypted data into a table provided by MySQL database engine"): + node.query(textwrap.dedent(f""" + INSERT INTO + mysql_db.user_data + SELECT + id, date, name, {example_transform} + FROM + input('id Int32, date Date, name String, secret String') + FORMAT Values (1, '2020-01-01', 'user0', 'secret') + """)) + + with And("I read decrypted data using MySQL database engine"): + output = node.query(f"""SELECT hex(aes_decrypt_mysql({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_db.user_data""").output.strip() + + with Then("decrypted data from MySQL database engine should should match the original plain text"): + assert output == hex("secret"), error() + + with And("I read raw data using MySQL database engine to get expected raw data"): + expected_raw_data = node.query("SELECT hex(secret) AS secret FROM mysql_db.user_data").output.strip() + + with And("I read raw encrypted data in MySQL"): + output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT hex(secret) as secret FROM user_data;\"", exitcode=0).output.strip() + + with Then("check that raw encryted data in MySQL matches the expected"): + assert expected_raw_data in output, error() + + with And("I decrypt data in MySQL to make sure it is valid"): + sql = f""" + SET block_encryption_mode = {example_mode}; + SELECT id, date, name, hex(AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"})) AS secret FROM user_data; + """ + output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0).output.strip() + + with Then("decryted data in MySQL should match the original plain text"): + assert hex("secret") in output, error() + +@TestFeature +@Name("database engine") +@Requirements( + RQ_SRS008_AES_Functions_Compatability_Engine_Database_MySQL("1.0") +) +def feature(self, node="clickhouse1", mysql_node="mysql1"): + """Check usage of encryption functions with [MySQL database engine]. + + [MySQL database engine]: https://clickhouse.tech/docs/en/engines/database-engines/mysql/ + """ + self.context.node = self.context.cluster.node(node) + self.context.mysql_node = self.context.cluster.node(mysql_node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/dictionary.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/dictionary.py new file mode 100644 index 00000000000..66b9e3acbf8 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/mysql/dictionary.py @@ -0,0 +1,251 @@ +import textwrap +from contextlib import contextmanager + +from testflows.core import * +from testflows.asserts import error + +from aes_encryption.requirements import * +from aes_encryption.tests.common import mysql_modes, hex + +@contextmanager +def dictionary(name, node, mysql_node, secret_type): + """Create a table in MySQL and use it a source for a dictionary. + """ + try: + with Given("table in MySQL"): + sql = f""" + CREATE TABLE {name}( + id INT NOT NULL AUTO_INCREMENT, + date DATE, + name VARCHAR(100), + secret {secret_type}, + PRIMARY KEY ( id ) + ); + """ + with When("I drop the table if exists"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + with And("I create a table"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("dictionary that uses MySQL table as the external source"): + with When("I drop the dictionary if exists"): + node.query(f"DROP DICTIONARY IF EXISTS dict_{name}") + with And("I create the dictionary"): + sql = f""" + CREATE DICTIONARY dict_{name} + ( + id Int32, + date Date, + name String, + secret String + ) + PRIMARY KEY id + SOURCE(MYSQL( + USER 'user' + PASSWORD 'password' + DB 'db' + TABLE '{name}' + REPLICA(PRIORITY 1 HOST '{mysql_node.name}' PORT 3306) + )) + LAYOUT(HASHED()) + LIFETIME(0) + """ + node.query(textwrap.dedent(sql)) + + yield f"dict_{name}" + + finally: + with Finally("I drop the dictionary", flags=TE): + node.query(f"DROP DICTIONARY IF EXISTS dict_{name}") + + with And("I drop a table in MySQL", flags=TE): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + +@contextmanager +def parameters_dictionary(name, node, mysql_node): + """Create a table in MySQL and use it a source for a dictionary + that stores parameters for the encryption functions. + """ + try: + with Given("table in MySQL"): + sql = f""" + CREATE TABLE {name}( + `id` INT NOT NULL AUTO_INCREMENT, + `name` VARCHAR(100), + `mode` VARCHAR(100), + `key` BLOB, + `iv` BLOB, + `text` BLOB, + PRIMARY KEY ( id ) + ); + """ + with When("I drop the table if exists"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + with And("I create a table"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("dictionary that uses MySQL table as the external source"): + with When("I drop the dictionary if exists"): + node.query(f"DROP DICTIONARY IF EXISTS dict_{name}") + with And("I create the dictionary"): + sql = f""" + CREATE DICTIONARY dict_{name} + ( + id Int32, + name String, + mode String, + key String, + iv String, + text String + ) + PRIMARY KEY id + SOURCE(MYSQL( + USER 'user' + PASSWORD 'password' + DB 'db' + TABLE '{name}' + REPLICA(PRIORITY 1 HOST '{mysql_node.name}' PORT 3306) + )) + LAYOUT(HASHED()) + LIFETIME(0) + """ + node.query(textwrap.dedent(sql)) + + yield f"dict_{name}" + + finally: + with Finally("I drop the dictionary", flags=TE): + node.query(f"DROP DICTIONARY IF EXISTS dict_{name}") + + with And("I drop a table in MySQL", flags=TE): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + +@TestScenario +def parameter_values(self): + """Check that we can use a dictionary that uses MySQL table as a source + can be used as a parameters store for the `encrypt`, `decrypt`, and + `aes_encrypt_mysql`, `aes_decrypt_mysql` functions. + """ + node = self.context.node + mysql_node = self.context.mysql_node + mode = "'aes-128-cbc'" + key = f"'{'1' * 16}'" + iv = f"'{'2' * 16}'" + plaintext = "'secret'" + + for encrypt, decrypt in [ + ("encrypt", "decrypt"), + ("aes_encrypt_mysql", "aes_decrypt_mysql") + ]: + with Example(f"{encrypt} and {decrypt}", description=f"Check using dictionary for parameters of {encrypt} and {decrypt} functions."): + with parameters_dictionary("parameters_data", node, mysql_node) as dict_name: + with When("I insert parameters values in MySQL"): + sql = f""" + INSERT INTO parameters_data VALUES (1, 'user0', {mode}, {key}, {iv}, {plaintext}); + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I use dictionary values as parameters"): + sql = f""" + SELECT {decrypt}( + dictGet('default.{dict_name}', 'mode', toUInt64(1)), + {encrypt}( + dictGet('default.{dict_name}', 'mode', toUInt64(1)), + dictGet('default.{dict_name}', 'text', toUInt64(1)), + dictGet('default.{dict_name}', 'key', toUInt64(1)), + dictGet('default.{dict_name}', 'iv', toUInt64(1)) + ), + dictGet('default.{dict_name}', 'key', toUInt64(1)), + dictGet('default.{dict_name}', 'iv', toUInt64(1)) + ) + """ + output = node.query(textwrap.dedent(sql)).output.strip() + + with Then("output should match the plain text"): + assert f"'{output}'" == plaintext, error() + +@TestOutline(Scenario) +@Examples("mysql_datatype", [ + ("VARBINARY(100)",), + #("VARCHAR(100)",), + ("BLOB", ), + #("TEXT",) +]) +def decrypt(self, mysql_datatype): + """Check that when using a dictionary that uses MySQL table as a source and + contains a data encrypted in MySQL and stored using specified data type + that we can decrypt data from the dictionary using + the `aes_decrypt_mysql` or `decrypt` functions in the select query. + """ + node = self.context.node + mysql_node = self.context.mysql_node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for func in ["decrypt", "aes_decrypt_mysql"]: + for mode, key_len, iv_len in mysql_modes: + exact_key_size = int(mode.split("-")[1])//8 + + if "ecb" not in mode and not iv_len: + continue + if func == "decrypt": + if iv_len and iv_len != 16: + continue + if key_len != exact_key_size: + continue + + with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with dictionary("user_data", node, mysql_node, mysql_datatype) as dict_name: + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + + with When("I insert encrypted data in MySQL"): + sql = f""" + SET block_encryption_mode = {example_mode}; + INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"})); + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read encrypted data in MySQL to make sure it is valid"): + sql = f""" + SET block_encryption_mode = {example_mode}; + SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data; + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read raw encrypted data in MySQL"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0) + + with And("I read raw data using MySQL dictionary"): + output = node.query(f"SELECT hex(dictGet('default.{dict_name}', 'secret', toUInt64(1))) AS secret") + + with And("I read decrypted data using MySQL dictionary"): + output = node.query(textwrap.dedent(f""" + SELECT hex( + {func}( + {example_mode}, + dictGet('default.{dict_name}', 'secret', toUInt64(1)), + {example_key}{(", " + example_iv) if example_iv else ""} + ) + ) + """)).output.strip() + + with Then("output should match the original plain text"): + assert output == hex("secret"), error() + +@TestFeature +@Name("dictionary") +@Requirements( + RQ_SRS008_AES_Functions_Compatability_Dictionaries("1.0") +) +def feature(self, node="clickhouse1", mysql_node="mysql1"): + """Check usage of encryption functions with [MySQL dictionary]. + + [MySQL dictionary]: https://clickhouse.tech/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources/#dicts-external_dicts_dict_sources-mysql + """ + self.context.node = self.context.cluster.node(node) + self.context.mysql_node = self.context.cluster.node(mysql_node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/feature.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/feature.py new file mode 100644 index 00000000000..5c6338bfdff --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/mysql/feature.py @@ -0,0 +1,18 @@ +from testflows.core import * + +from aes_encryption.requirements import * + +@TestFeature +@Name("mysql") +@Requirements( + RQ_SRS008_AES_Functions_Compatability_MySQL("1.0") +) +def feature(self, node="clickhouse1"): + """Check encryption functions usage compatibility with MySQL. + """ + self.context.node = self.context.cluster.node(node) + + Feature(run=load("aes_encryption.tests.compatibility.mysql.table_engine", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.compatibility.mysql.database_engine", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.compatibility.mysql.table_function", "feature"), flags=TE) + Feature(run=load("aes_encryption.tests.compatibility.mysql.dictionary", "feature"), flags=TE) diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/table_engine.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/table_engine.py new file mode 100644 index 00000000000..5a5c7d9d58a --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/mysql/table_engine.py @@ -0,0 +1,202 @@ +import textwrap +from contextlib import contextmanager + +from testflows.core import * +from testflows.asserts import error + +from aes_encryption.requirements import * +from aes_encryption.tests.common import mysql_modes, hex + +@contextmanager +def table(name, node, mysql_node, secret_type): + """Create a table that can be accessed using MySQL table engine. + """ + try: + with Given("table in MySQL"): + sql = f""" + CREATE TABLE {name}( + id INT NOT NULL AUTO_INCREMENT, + date DATE, + name VARCHAR(100), + secret {secret_type}, + PRIMARY KEY ( id ) + ); + """ + with When("I drop the table if exists"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + with And("I create a table"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I create a table using MySQL table engine"): + sql = f""" + CREATE TABLE mysql_{name} + ( + id Nullable(Int32), + date Nullable(Date), + name Nullable(String), + secret Nullable(String) + ) + ENGINE = MySQL('{mysql_node.name}:3306', 'db', '{name}', 'user', 'password') + """ + with When("I drop table if exists"): + node.query(f"DROP TABLE IF EXISTS mysql_{name}") + with And("I create table"): + node.query(textwrap.dedent(sql)) + yield + + finally: + with And("I drop a table using MySQL table engine", flags=TE): + node.query(f"DROP TABLE IF EXISTS mysql_{name}") + + with And("I drop a table in MySQL", flags=TE): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + +@TestOutline(Scenario) +@Examples("mysql_datatype", [ + ("VARBINARY(100)",), + #("VARCHAR(100)",), + ("BLOB", ), + #("TEXT",) +]) +def decrypt(self, mysql_datatype): + """Check that when using a table with MySQL table engine that + contains a column encrypted in MySQL stored using specified data type + I can decrypt data in the column using the `decrypt` and `aes_decrypt_mysql` + functions in the select query. + """ + node = self.context.node + mysql_node = self.context.mysql_node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for func in ["decrypt", "aes_decrypt_mysql"]: + for mode, key_len, iv_len in mysql_modes: + exact_key_size = int(mode.split("-")[1])//8 + + if "ecb" not in mode and not iv_len: + continue + if func == "decrypt": + if iv_len and iv_len != 16: + continue + if key_len != exact_key_size: + continue + + with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with table("user_data", node, mysql_node, mysql_datatype): + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + + with When("I insert encrypted data in MySQL"): + sql = f""" + SET block_encryption_mode = {example_mode}; + INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"})); + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read encrypted data in MySQL to make sure it is valid"): + sql = f""" + SET block_encryption_mode = {example_mode}; + SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data; + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read raw encrypted data in MySQL"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0) + + with And("I read raw data using MySQL table engine"): + output = node.query("SELECT id, date, name, hex(secret) AS secret FROM mysql_user_data") + + with And("I read decrypted data via MySQL table engine"): + output = node.query(f"""SELECT hex({func}({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_user_data""").output.strip() + + with Then("the output should match the original plain text"): + assert output == hex("secret"), error() + +@TestOutline(Scenario) +@Examples("mysql_datatype", [ + ("VARBINARY(100)",), + #("VARCHAR(100)",), + ("BLOB", ), + #("TEXT",) +]) +def encrypt(self, mysql_datatype): + """Check that when using a table with MySQL table engine that + we can encrypt data during insert using the `encrypt` and `aes_encrypt_mysql` + functions and decrypt it in MySQL. + """ + node = self.context.node + mysql_node = self.context.mysql_node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for func in ["encrypt", "aes_encrypt_mysql"]: + for mode, key_len, iv_len in mysql_modes: + exact_key_size = int(mode.split("-")[1])//8 + + if "ecb" not in mode and not iv_len: + continue + if func == "encrypt": + if iv_len and iv_len != 16: + continue + if key_len != exact_key_size: + continue + + with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with table("user_data", node, mysql_node, mysql_datatype): + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_transform = f"{func}({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''})" + + with When("I insert encrypted data into MySQL table engine"): + node.query(textwrap.dedent(f""" + INSERT INTO + mysql_user_data + SELECT + id, date, name, {example_transform} + FROM + input('id Nullable(Int32), date Date, name String, secret String') + FORMAT Values (null, '2020-01-01', 'user0', 'secret') + """)) + + with And("I read decrypted data via MySQL table engine"): + output = node.query(f"""SELECT hex(aes_decrypt_mysql({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM mysql_user_data""").output.strip() + + with Then("decrypted data from MySQL table engine should should match the original plain text"): + assert output == hex("secret"), error() + + with And("I read raw data using MySQL table engine to get expected raw data"): + expected_raw_data = node.query("SELECT hex(secret) AS secret FROM mysql_user_data").output.strip() + + with And("I read raw encrypted data in MySQL"): + output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT hex(secret) as secret FROM user_data;\"", exitcode=0).output.strip() + + with Then("check that raw encryted data in MySQL matches the expected"): + assert expected_raw_data in output, error() + + with And("I decrypt data in MySQL to make sure it is valid"): + sql = f""" + SET block_encryption_mode = {example_mode}; + SELECT id, date, name, hex(AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"})) AS secret FROM user_data; + """ + output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0).output.strip() + + with Then("decryted data in MySQL should match the original plain text"): + assert hex("secret") in output, error() + +@TestFeature +@Name("table engine") +@Requirements( + RQ_SRS008_AES_Functions_Compatability_Engine_Table_MySQL("1.0") +) +def feature(self, node="clickhouse1", mysql_node="mysql1"): + """Check usage of encryption functions with [MySQL table engine]. + + [MySQL table engine]: https://clickhouse.tech/docs/en/engines/table-engines/integrations/mysql/ + """ + self.context.node = self.context.cluster.node(node) + self.context.mysql_node = self.context.cluster.node(mysql_node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/compatibility/mysql/table_function.py b/tests/testflows/aes_encryption/tests/compatibility/mysql/table_function.py new file mode 100644 index 00000000000..cd3487c5c74 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/mysql/table_function.py @@ -0,0 +1,183 @@ +import textwrap +from contextlib import contextmanager + +from testflows.core import * +from testflows.asserts import error + +from aes_encryption.requirements import * +from aes_encryption.tests.common import mysql_modes, hex + +@contextmanager +def table(name, node, mysql_node, secret_type): + """Create a table that can be accessed using MySQL table function. + """ + try: + with Given("table in MySQL"): + sql = f""" + CREATE TABLE {name}( + id INT NOT NULL AUTO_INCREMENT, + date DATE, + name VARCHAR(100), + secret {secret_type}, + PRIMARY KEY ( id ) + ); + """ + with When("I drop the table if exists"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + with And("I create a table"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + yield f"mysql('{mysql_node.name}:3306', 'db', 'user_data', 'user', 'password')" + + finally: + with And("I drop a table in MySQL", flags=TE): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"DROP TABLE IF EXISTS {name};\"", exitcode=0) + +@TestOutline(Scenario) +@Examples("mysql_datatype", [ + ("VARBINARY(100)",), + #("VARCHAR(100)",), + ("BLOB", ), + #("TEXT",) +]) +def decrypt(self, mysql_datatype): + """Check that when using a table accessed through MySQL table function that + contains a column encrypted in MySQL stored using specified data type + I can decrypt data in the column using the `decrypt` and `aes_decrypt_mysql` + functions in the select query. + """ + node = self.context.node + mysql_node = self.context.mysql_node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for func in ["decrypt", "aes_decrypt_mysql"]: + for mode, key_len, iv_len in mysql_modes: + exact_key_size = int(mode.split("-")[1])//8 + + if "ecb" not in mode and not iv_len: + continue + if func == "decrypt": + if iv_len and iv_len != 16: + continue + if key_len != exact_key_size: + continue + + with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with table("user_data", node, mysql_node, mysql_datatype) as table_function: + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + + with When("I insert encrypted data in MySQL"): + sql = f""" + SET block_encryption_mode = {example_mode}; + INSERT INTO user_data VALUES (NULL, '2020-01-01', 'user0', AES_ENCRYPT('secret', {example_key}{(", " + example_iv) if example_iv else ", ''"})); + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read encrypted data in MySQL to make sure it is valid"): + sql = f""" + SET block_encryption_mode = {example_mode}; + SELECT id, date, name, AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"}) AS secret FROM user_data; + """ + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0) + + with And("I read raw encrypted data in MySQL"): + mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT id, date, name, hex(secret) as secret FROM user_data;\"", exitcode=0) + + with And("I read raw data using MySQL table function"): + output = node.query(f"SELECT id, date, name, hex(secret) AS secret FROM {table_function}") + + with And("I read decrypted data using MySQL table function"): + output = node.query(f"""SELECT hex({func}({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM {table_function}""").output.strip() + + with Then("output should match the original plain text"): + assert output == hex("secret"), error() + +@TestOutline(Scenario) +@Examples("mysql_datatype", [ + ("VARBINARY(100)",), + #("VARCHAR(100)",), + ("BLOB", ), + #("TEXT",) +]) +def encrypt(self, mysql_datatype): + """Check that when using a table accessed through MySQL table function that + we can encrypt data during insert using the `aes_encrypt_mysql` function + and decrypt it in MySQL. + """ + node = self.context.node + mysql_node = self.context.mysql_node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for func in ["encrypt", "aes_encrypt_mysql"]: + for mode, key_len, iv_len in mysql_modes: + exact_key_size = int(mode.split("-")[1])//8 + + if "ecb" not in mode and not iv_len: + continue + if func == "encrypt": + if iv_len and iv_len != 16: + continue + if key_len != exact_key_size: + continue + + with Example(f"""{func} mode={mode.strip("'")} key={key_len} iv={iv_len}"""): + with table("user_data", node, mysql_node, mysql_datatype) as table_function: + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_transform = f"{func}({mode}, secret, {example_key}{(', ' + example_iv) if example_iv else ''})" + + with When("I insert encrypted data into a table provided by MySQL database engine"): + node.query(textwrap.dedent(f""" + INSERT INTO TABLE FUNCTION + {table_function} + SELECT + id, date, name, {example_transform} + FROM + input('id Int32, date Date, name String, secret String') + FORMAT Values (1, '2020-01-01', 'user0', 'secret') + """)) + + with And("I read decrypted data using MySQL database engine"): + output = node.query(f"""SELECT hex(aes_decrypt_mysql({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""})) FROM {table_function}""").output.strip() + + with Then("decrypted data from MySQL database engine should should match the original plain text"): + assert output == hex("secret"), error() + + with And("I read raw data using MySQL database engine to get expected raw data"): + expected_raw_data = node.query(f"SELECT hex(secret) AS secret FROM {table_function}").output.strip() + + with And("I read raw encrypted data in MySQL"): + output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user -e \"SELECT hex(secret) as secret FROM user_data;\"", exitcode=0).output.strip() + + with Then("check that raw encryted data in MySQL matches the expected"): + assert expected_raw_data in output, error() + + with And("I decrypt data in MySQL to make sure it is valid"): + sql = f""" + SET block_encryption_mode = {example_mode}; + SELECT id, date, name, hex(AES_DECRYPT(secret, {example_key}{(", " + example_iv) if example_iv else ", ''"})) AS secret FROM user_data; + """ + output = mysql_node.command(f"MYSQL_PWD=password mysql -D db -u user <<'EOF'{textwrap.dedent(sql)}\nEOF", exitcode=0).output.strip() + + with Then("decryted data in MySQL should match the original plain text"): + assert hex("secret") in output, error() + +@TestFeature +@Name("table function") +@Requirements( + RQ_SRS008_AES_Functions_Compatability_TableFunction_MySQL("1.0") +) +def feature(self, node="clickhouse1", mysql_node="mysql1"): + """Check usage of encryption functions with [MySQL table function]. + + [MySQL table function]: https://clickhouse.tech/docs/en/sql-reference/table-functions/mysql/ + """ + self.context.node = self.context.cluster.node(node) + self.context.mysql_node = self.context.cluster.node(mysql_node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) \ No newline at end of file diff --git a/tests/testflows/aes_encryption/tests/compatibility/select.py b/tests/testflows/aes_encryption/tests/compatibility/select.py new file mode 100644 index 00000000000..f81920c65d3 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/select.py @@ -0,0 +1,186 @@ +import textwrap +from contextlib import contextmanager + +from testflows.core import * +from testflows.asserts.helpers import varname +from testflows.asserts import values, error, snapshot + +from aes_encryption.tests.common import modes, mysql_modes + +@contextmanager +def table(name, sql): + node = current().context.node + try: + with Given("table"): + + with By("dropping table if exists"): + node.query(f"DROP TABLE IF EXISTS {name}") + with And("creating a table"): + node.query(textwrap.dedent(sql.format(name=name))) + yield + finally: + with Finally("I drop the table", flags=TE): + node.query(f"DROP TABLE IF EXISTS {name}") + +@TestScenario +def decrypt(self): + """Check decrypting column when reading data from a table. + """ + node = self.context.node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + aad = "some random aad" + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len} aad={aad_len}""") as example: + with table("user_table", """ + CREATE TABLE {name} + ( + date Nullable(Date), + name Nullable(String), + secret Nullable(String) + ) + ENGINE = Memory() + """): + + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_aad = None if not aad_len else f"'{aad}'" + + with When("I insert encrypted data"): + encrypted_secret = node.query(f"""SELECT hex(encrypt({example_mode}, 'secret', {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}))""").output.strip() + node.query(textwrap.dedent(f""" + INSERT INTO user_table + (date, name, secret) + VALUES + ('2020-01-01', 'user0', unhex('{encrypted_secret}')) + """)) + + with And("I decrypt data during query"): + output = node.query(f"""SELECT name, decrypt({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) AS secret FROM user_table FORMAT JSONEachRow""").output.strip() + + with Then("I should get back the original plain text"): + assert output == '{"name":"user0","secret":"secret"}', error() + +@TestScenario +def decrypt_multiple(self, count=1000): + """Check decrypting column when reading multiple entries + encrypted with the same parameters for the same user + from a table. + """ + node = self.context.node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + aad = "some random aad" + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} key={key_len} iv={iv_len} aad={aad_len}""") as example: + with table("user_table", """ + CREATE TABLE {name} + ( + date Nullable(Date), + name Nullable(String), + secret Nullable(String) + ) + ENGINE = Memory() + """): + + example_mode = mode + example_key = f"'{key[:key_len]}'" + example_iv = None if not iv_len else f"'{iv[:iv_len]}'" + example_aad = None if not aad_len else f"'{aad}'" + + with When("I insert encrypted data"): + encrypted_secret = node.query(f"""SELECT hex(encrypt({example_mode}, 'secret', {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}))""").output.strip() + values = [f"('2020-01-01', 'user0', unhex('{encrypted_secret}'))"] * count + node.query( + "INSERT INTO user_table\n" + " (date, name, secret)\n" + f"VALUES {', '.join(values)}") + + with And("I decrypt data", description="using a subquery and get the number of entries that match the plaintext"): + output = node.query(f"""SELECT count() AS count FROM (SELECT name, decrypt({example_mode}, secret, {example_key}{(", " + example_iv) if example_iv else ""}{(", " + example_aad) if example_aad else ""}) AS secret FROM user_table) WHERE secret = 'secret' FORMAT JSONEachRow""").output.strip() + + with Then("I should get back the expected result", description=f"{count}"): + assert output == f'{{"count":"{count}"}}', error() + +@TestScenario +def decrypt_unique(self): + """Check decrypting column when reading multiple entries + encrypted with the different parameters for each user + from a table. + """ + node = self.context.node + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + aad = "some random aad" + + with table("user_table", """ + CREATE TABLE {name} + ( + id UInt64, + date Nullable(Date), + name Nullable(String), + secret Nullable(String) + ) + ENGINE = Memory() + """): + + user_modes = [] + user_keys = [] + + with When("I get encrypted data"): + user_id = 0 + values = [] + + for mode, key_len, iv_len, aad_len in modes: + if "gcm" in mode: + continue + user_modes.append(mode) + user_keys.append(f"'{key[:key_len]}'") + + with When(f"I get encrypted data for user {user_id}"): + encrypted_secret = node.query( + f"""SELECT hex(encrypt({user_modes[-1]}, 'secret', {user_keys[-1]}))""" + ).output.strip() + values.append(f"({user_id}, '2020-01-01', 'user{user_id}', unhex('{encrypted_secret}'))") + + user_id += 1 + + with And("I insert encrypted data for all users"): + node.query( + "INSERT INTO user_table\n" + " (id, date, name, secret)\n" + f"VALUES {', '.join(values)}") + + with And("I read decrypted data for all users"): + output = node.query(textwrap.dedent(f""" + SELECT + count() AS count + FROM + ( + SELECT + [{",".join(user_modes)}] AS modes, + [{",".join(user_keys)}] AS keys, + name, + decrypt(modes[id], secret, keys[id]) AS secret + FROM user_table + ) + WHERE + secret = 'secret' + FORMAT JSONEachRow + """)).output.strip() + + with Then("I should get back the expected result", description=f"{count}"): + assert output == f'{{"count":"{count}"}}', error() + +@TestFeature +@Name("select") +def feature(self, node="clickhouse1"): + """Check encryption functions when used during table querying. + """ + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/compatibility/snapshots/insert.py.insert.snapshot b/tests/testflows/aes_encryption/tests/compatibility/snapshots/insert.py.insert.snapshot new file mode 100644 index 00000000000..93d915bb9d6 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/compatibility/snapshots/insert.py.insert.snapshot @@ -0,0 +1,192 @@ +aes_encrypt_mysql_input_example_mode_aes_128_ecb_key_16_iv_None = r"""'2020-01-01\tuser0\t8C9598C3C8D8AC241DDF0D1B22020104\n2020-01-02\tuser1\tC5ECE31A240069D8F169B9F8CF687779\n2020-01-03\tuser2\t9FCFA4B05DD49D2B24BA61091F963CE3'""" + +aes_encrypt_mysql_input_example_mode_aes_128_ecb_key_24_iv_None = r"""'2020-01-01\tuser0\tB418FF12BCBF9E42FA7C19D6EE26BF0B\n2020-01-02\tuser1\t3147A3FEE47DF418D1D75CBC1BC14DE6\n2020-01-03\tuser2\tAECEFD40C6632A0FC033D040E44CCBCC'""" + +aes_encrypt_mysql_input_example_mode_aes_192_ecb_key_24_iv_None = r"""'2020-01-01\tuser0\t897F14C4E497962D986A7E7EA57AA043\n2020-01-02\tuser1\tED84AF2B3447113DA451E4577F649E36\n2020-01-03\tuser2\t4976F9D5AE195E61694A9ADCDD8A076F'""" + +aes_encrypt_mysql_input_example_mode_aes_192_ecb_key_32_iv_None = r"""'2020-01-01\tuser0\t044E715357AF77234FD95359666CAFF3\n2020-01-02\tuser1\tB633EF852CE85B4C97827401FD9B606B\n2020-01-03\tuser2\t2AFF7052C748E4BC3BDA8460AFD5A21D'""" + +aes_encrypt_mysql_input_example_mode_aes_256_ecb_key_32_iv_None = r"""'2020-01-01\tuser0\tBABD6C071FDEE1C9A33877006FBB0BE6\n2020-01-02\tuser1\t7753E81D1DB9DC91FC8148E88B3E9526\n2020-01-03\tuser2\tD77D1A8DF82C2273BF0D19A14526531F'""" + +aes_encrypt_mysql_input_example_mode_aes_256_ecb_key_64_iv_None = r"""'2020-01-01\tuser0\tBFFEC9DF7285A3EC799C941E1450839C\n2020-01-02\tuser1\t3EA0ECBD06326D227A7B9519B1A2955D\n2020-01-03\tuser2\t1478C57DD49523ABDB83A0917F0EDA60'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cbc_key_16_iv_None = r"""'2020-01-01\tuser0\t8C9598C3C8D8AC241DDF0D1B22020104\n2020-01-02\tuser1\tC5ECE31A240069D8F169B9F8CF687779\n2020-01-03\tuser2\t9FCFA4B05DD49D2B24BA61091F963CE3'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cbc_key_24_iv_None = r"""'2020-01-01\tuser0\t897F14C4E497962D986A7E7EA57AA043\n2020-01-02\tuser1\tED84AF2B3447113DA451E4577F649E36\n2020-01-03\tuser2\t4976F9D5AE195E61694A9ADCDD8A076F'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cbc_key_32_iv_None = r"""'2020-01-01\tuser0\tBABD6C071FDEE1C9A33877006FBB0BE6\n2020-01-02\tuser1\t7753E81D1DB9DC91FC8148E88B3E9526\n2020-01-03\tuser2\tD77D1A8DF82C2273BF0D19A14526531F'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cbc_key_16_iv_16 = r"""'2020-01-01\tuser0\tFC93C1D5E5E3B054C1F3A5692AAC0A61\n2020-01-02\tuser1\tD6DBC76ABCB14B7C6D93F1A5FCA66B9C\n2020-01-03\tuser2\tD4F4158A650D01EB505CC72EFE455486'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cbc_key_24_iv_24 = r"""'2020-01-01\tuser0\t26CEE6B6EBDDE1BF887FDEB75F28FB52\n2020-01-02\tuser1\tF9EC1A75BEEFF70B4DEB39AAD075CEFF\n2020-01-03\tuser2\t3FF84AB3BD40FAEEF70F06BCF6AF9C42'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cbc_key_24_iv_16 = r"""'2020-01-01\tuser0\t0E3BAF7F4E0BFCFFAE2589B67F71E277\n2020-01-02\tuser1\t2581CCEE9ABE5770480901D65B3D9222\n2020-01-03\tuser2\tED9F3BD8DB12FDF9F2462FFA572361E7'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cbc_key_32_iv_32 = r"""'2020-01-01\tuser0\t07371B5DE2E378EE08A3A8B6B9FEAD13\n2020-01-02\tuser1\t3C0BF5D187421ECFFD3E00474A154452\n2020-01-03\tuser2\t05B253FA783D78D864AF7C4D5E6A492D'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cbc_key_32_iv_16 = r"""'2020-01-01\tuser0\t72AC7BA6F283EA94A3C33C4D3E51C7D3\n2020-01-02\tuser1\tDACBBE79062F1C721A01CEEE3E85524F\n2020-01-03\tuser2\tFF5A09D19E5EB2ADD94581308588E44A'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cbc_key_64_iv_64 = r"""'2020-01-01\tuser0\t573924F0BB4AA1780D45DB6451F123D6\n2020-01-02\tuser1\t007A54AA7ADE8EF844D28936486D75BC\n2020-01-03\tuser2\tAA7249B514398FE1EE827C44402BCE57'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb1_key_16_iv_None = r"""'2020-01-01\tuser0\t750BE8662F57A095EC0E610C\n2020-01-02\tuser1\t750BE8662E444A6284C0FC72\n2020-01-03\tuser2\t750BE8662C000B61CDCF1C94'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb1_key_24_iv_None = r"""'2020-01-01\tuser0\t5DCC67A043EB776D8B7F5B70\n2020-01-02\tuser1\t5DCC67A042B46DFCC10EFD66\n2020-01-03\tuser2\t5DCC67A040243A8C1346D2DD'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb1_key_32_iv_None = r"""'2020-01-01\tuser0\tFAAC1A7D2CE844F8DEB4C44E\n2020-01-02\tuser1\tFAAC1A7D2DF85A43828C0FF8\n2020-01-03\tuser2\tFAAC1A7D2FC7582CCEFCF330'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb1_key_16_iv_16 = r"""'2020-01-01\tuser0\t7670A865D13B1B65AD46F8ED\n2020-01-02\tuser1\t7670A865D046007A1E218286\n2020-01-03\tuser2\t7670A865D2E5B091492ECCFB'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb1_key_24_iv_24 = r"""'2020-01-01\tuser0\t51EADDE82195C31118D0C171\n2020-01-02\tuser1\t51EADDE82009A46518270271\n2020-01-03\tuser2\t51EADDE8235CB38F95766481'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb1_key_24_iv_16 = r"""'2020-01-01\tuser0\t7F38C051539074C0A635C937\n2020-01-02\tuser1\t7F38C051520A30DFACBE9564\n2020-01-03\tuser2\t7F38C051500DA29FF0E7B799'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb1_key_32_iv_32 = r"""'2020-01-01\tuser0\t2067186DB91666DE730D0708\n2020-01-02\tuser1\t2067186DB83E2E8B0019F839\n2020-01-03\tuser2\t2067186DBB540332BFC84955'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb1_key_32_iv_16 = r"""'2020-01-01\tuser0\t0A216A58A5C0A33215E8E722\n2020-01-02\tuser1\t0A216A58A4E94067ABF030B6\n2020-01-03\tuser2\t0A216A58A6822CAB0318C632'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb1_key_64_iv_64 = r"""'2020-01-01\tuser0\t81BD636E1BF4CA02399943E3\n2020-01-02\tuser1\t81BD636E1A93D5D6DD9DCD8D\n2020-01-03\tuser2\t81BD636E18F15168D19C8117'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb8_key_16_iv_None = r"""'2020-01-01\tuser0\t650D96B9698D20DB12E2E437\n2020-01-02\tuser1\t650D96B968F00D16ABF2852E\n2020-01-03\tuser2\t650D96B96B8141F425E60D6B'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb8_key_24_iv_None = r"""'2020-01-01\tuser0\t72C4724B2F528724A12041C0\n2020-01-02\tuser1\t72C4724B2EF3C6A6FF9E09A9\n2020-01-03\tuser2\t72C4724B2D6EAB1D47709E15'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb8_key_32_iv_None = r"""'2020-01-01\tuser0\tC5FD6C94961765ED204F2BCA\n2020-01-02\tuser1\tC5FD6C9497AB1C1AF1DE671C\n2020-01-03\tuser2\tC5FD6C949491F4A3EA5069B3'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb8_key_16_iv_16 = r"""'2020-01-01\tuser0\t471D217E9CA3593FFEC955C8\n2020-01-02\tuser1\t471D217E9D7F484D85F81F19\n2020-01-03\tuser2\t471D217E9EBBFD2EA9841008'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb8_key_24_iv_24 = r"""'2020-01-01\tuser0\t2EE6147B830481BE36CBE350\n2020-01-02\tuser1\t2EE6147B82DE8F3197AF17A6\n2020-01-03\tuser2\t2EE6147B81FF826E798A0355'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb8_key_24_iv_16 = r"""'2020-01-01\tuser0\t1D98EFFAEB9907457BD3FCB2\n2020-01-02\tuser1\t1D98EFFAEA2D930825C6AE22\n2020-01-03\tuser2\t1D98EFFAE92C1D018438B98B'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb8_key_32_iv_32 = r"""'2020-01-01\tuser0\t4410165F7DCFDDBB1B15573F\n2020-01-02\tuser1\t4410165F7CFE6A0D2FD5CA9C\n2020-01-03\tuser2\t4410165F7FE8E0C081B3FB7B'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb8_key_32_iv_16 = r"""'2020-01-01\tuser0\t1C07B443BB7D7D60E9999C1D\n2020-01-02\tuser1\t1C07B443BA9674A3F68FF3FE\n2020-01-03\tuser2\t1C07B443B95F4B68161A616F'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb8_key_64_iv_64 = r"""'2020-01-01\tuser0\tA6D2368A5F177157D73FBD9D\n2020-01-02\tuser1\tA6D2368A5E695ADF99475359\n2020-01-03\tuser2\tA6D2368A5DB96AFD43311124'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb128_key_16_iv_None = r"""'2020-01-01\tuser0\t65ACA4C7C6338E0F7EB60812\n2020-01-02\tuser1\t65ACA4C7C7338E0F7EB60812\n2020-01-03\tuser2\t65ACA4C7C4338E0F7EB60812'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb128_key_24_iv_None = r"""'2020-01-01\tuser0\t72C47CEF0D63D2FB4FBC3CE4\n2020-01-02\tuser1\t72C47CEF0C63D2FB4FBC3CE4\n2020-01-03\tuser2\t72C47CEF0F63D2FB4FBC3CE4'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb128_key_32_iv_None = r"""'2020-01-01\tuser0\tC5FDAAECF7B42C68180AA151\n2020-01-02\tuser1\tC5FDAAECF6B42C68180AA151\n2020-01-03\tuser2\tC5FDAAECF5B42C68180AA151'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb128_key_16_iv_16 = r"""'2020-01-01\tuser0\t47FBCCF6ED598C3D8A4B05C5\n2020-01-02\tuser1\t47FBCCF6EC598C3D8A4B05C5\n2020-01-03\tuser2\t47FBCCF6EF598C3D8A4B05C5'""" + +aes_encrypt_mysql_input_example_mode_aes_128_cfb128_key_24_iv_24 = r"""'2020-01-01\tuser0\t2E046787D9EFFED25D69C908\n2020-01-02\tuser1\t2E046787D8EFFED25D69C908\n2020-01-03\tuser2\t2E046787DBEFFED25D69C908'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb128_key_24_iv_16 = r"""'2020-01-01\tuser0\t1DB482E0874F04D4E734607A\n2020-01-02\tuser1\t1DB482E0864F04D4E734607A\n2020-01-03\tuser2\t1DB482E0854F04D4E734607A'""" + +aes_encrypt_mysql_input_example_mode_aes_192_cfb128_key_32_iv_32 = r"""'2020-01-01\tuser0\t44D3EB069FF443A121590842\n2020-01-02\tuser1\t44D3EB069EF443A121590842\n2020-01-03\tuser2\t44D3EB069DF443A121590842'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb128_key_32_iv_16 = r"""'2020-01-01\tuser0\t1C2BED650C8137ED139226D3\n2020-01-02\tuser1\t1C2BED650D8137ED139226D3\n2020-01-03\tuser2\t1C2BED650E8137ED139226D3'""" + +aes_encrypt_mysql_input_example_mode_aes_256_cfb128_key_64_iv_64 = r"""'2020-01-01\tuser0\tA69DAA2E8B265618D25D5FE4\n2020-01-02\tuser1\tA69DAA2E8A265618D25D5FE4\n2020-01-03\tuser2\tA69DAA2E89265618D25D5FE4'""" + +aes_encrypt_mysql_input_example_mode_aes_128_ofb_key_16_iv_None = r"""'2020-01-01\tuser0\t65ACA4C7C6338E0F7EB60812\n2020-01-02\tuser1\t65ACA4C7C7338E0F7EB60812\n2020-01-03\tuser2\t65ACA4C7C4338E0F7EB60812'""" + +aes_encrypt_mysql_input_example_mode_aes_192_ofb_key_24_iv_None = r"""'2020-01-01\tuser0\t72C47CEF0D63D2FB4FBC3CE4\n2020-01-02\tuser1\t72C47CEF0C63D2FB4FBC3CE4\n2020-01-03\tuser2\t72C47CEF0F63D2FB4FBC3CE4'""" + +aes_encrypt_mysql_input_example_mode_aes_256_ofb_key_32_iv_None = r"""'2020-01-01\tuser0\tC5FDAAECF7B42C68180AA151\n2020-01-02\tuser1\tC5FDAAECF6B42C68180AA151\n2020-01-03\tuser2\tC5FDAAECF5B42C68180AA151'""" + +aes_encrypt_mysql_input_example_mode_aes_128_ofb_key_16_iv_16 = r"""'2020-01-01\tuser0\t47FBCCF6ED598C3D8A4B05C5\n2020-01-02\tuser1\t47FBCCF6EC598C3D8A4B05C5\n2020-01-03\tuser2\t47FBCCF6EF598C3D8A4B05C5'""" + +aes_encrypt_mysql_input_example_mode_aes_128_ofb_key_24_iv_24 = r"""'2020-01-01\tuser0\t2E046787D9EFFED25D69C908\n2020-01-02\tuser1\t2E046787D8EFFED25D69C908\n2020-01-03\tuser2\t2E046787DBEFFED25D69C908'""" + +aes_encrypt_mysql_input_example_mode_aes_192_ofb_key_24_iv_16 = r"""'2020-01-01\tuser0\t1DB482E0874F04D4E734607A\n2020-01-02\tuser1\t1DB482E0864F04D4E734607A\n2020-01-03\tuser2\t1DB482E0854F04D4E734607A'""" + +aes_encrypt_mysql_input_example_mode_aes_192_ofb_key_32_iv_32 = r"""'2020-01-01\tuser0\t44D3EB069FF443A121590842\n2020-01-02\tuser1\t44D3EB069EF443A121590842\n2020-01-03\tuser2\t44D3EB069DF443A121590842'""" + +aes_encrypt_mysql_input_example_mode_aes_256_ofb_key_32_iv_16 = r"""'2020-01-01\tuser0\t1C2BED650C8137ED139226D3\n2020-01-02\tuser1\t1C2BED650D8137ED139226D3\n2020-01-03\tuser2\t1C2BED650E8137ED139226D3'""" + +aes_encrypt_mysql_input_example_mode_aes_256_ofb_key_64_iv_64 = r"""'2020-01-01\tuser0\tA69DAA2E8B265618D25D5FE4\n2020-01-02\tuser1\tA69DAA2E8A265618D25D5FE4\n2020-01-03\tuser2\tA69DAA2E89265618D25D5FE4'""" + +encrypt_input_example_mode_aes_128_ecb_iv_None_aad_None = r"""'2020-01-01\tuser0\t8C9598C3C8D8AC241DDF0D1B22020104\n2020-01-02\tuser1\tC5ECE31A240069D8F169B9F8CF687779\n2020-01-03\tuser2\t9FCFA4B05DD49D2B24BA61091F963CE3'""" + +encrypt_input_example_mode_aes_192_ecb_iv_None_aad_None = r"""'2020-01-01\tuser0\t897F14C4E497962D986A7E7EA57AA043\n2020-01-02\tuser1\tED84AF2B3447113DA451E4577F649E36\n2020-01-03\tuser2\t4976F9D5AE195E61694A9ADCDD8A076F'""" + +encrypt_input_example_mode_aes_256_ecb_iv_None_aad_None = r"""'2020-01-01\tuser0\tBABD6C071FDEE1C9A33877006FBB0BE6\n2020-01-02\tuser1\t7753E81D1DB9DC91FC8148E88B3E9526\n2020-01-03\tuser2\tD77D1A8DF82C2273BF0D19A14526531F'""" + +encrypt_input_example_mode_aes_128_cbc_iv_None_aad_None = r"""'2020-01-01\tuser0\t8C9598C3C8D8AC241DDF0D1B22020104\n2020-01-02\tuser1\tC5ECE31A240069D8F169B9F8CF687779\n2020-01-03\tuser2\t9FCFA4B05DD49D2B24BA61091F963CE3'""" + +encrypt_input_example_mode_aes_192_cbc_iv_None_aad_None = r"""'2020-01-01\tuser0\t897F14C4E497962D986A7E7EA57AA043\n2020-01-02\tuser1\tED84AF2B3447113DA451E4577F649E36\n2020-01-03\tuser2\t4976F9D5AE195E61694A9ADCDD8A076F'""" + +encrypt_input_example_mode_aes_256_cbc_iv_None_aad_None = r"""'2020-01-01\tuser0\tBABD6C071FDEE1C9A33877006FBB0BE6\n2020-01-02\tuser1\t7753E81D1DB9DC91FC8148E88B3E9526\n2020-01-03\tuser2\tD77D1A8DF82C2273BF0D19A14526531F'""" + +encrypt_input_example_mode_aes_128_cbc_iv_16_aad_None = r"""'2020-01-01\tuser0\tFC93C1D5E5E3B054C1F3A5692AAC0A61\n2020-01-02\tuser1\tD6DBC76ABCB14B7C6D93F1A5FCA66B9C\n2020-01-03\tuser2\tD4F4158A650D01EB505CC72EFE455486'""" + +encrypt_input_example_mode_aes_192_cbc_iv_16_aad_None = r"""'2020-01-01\tuser0\t0E3BAF7F4E0BFCFFAE2589B67F71E277\n2020-01-02\tuser1\t2581CCEE9ABE5770480901D65B3D9222\n2020-01-03\tuser2\tED9F3BD8DB12FDF9F2462FFA572361E7'""" + +encrypt_input_example_mode_aes_256_cbc_iv_16_aad_None = r"""'2020-01-01\tuser0\t72AC7BA6F283EA94A3C33C4D3E51C7D3\n2020-01-02\tuser1\tDACBBE79062F1C721A01CEEE3E85524F\n2020-01-03\tuser2\tFF5A09D19E5EB2ADD94581308588E44A'""" + +encrypt_input_example_mode_aes_128_cfb1_iv_None_aad_None = r"""'2020-01-01\tuser0\t750BE8662F57A095EC0E610C\n2020-01-02\tuser1\t750BE8662E444A6284C0FC72\n2020-01-03\tuser2\t750BE8662C000B61CDCF1C94'""" + +encrypt_input_example_mode_aes_192_cfb1_iv_None_aad_None = r"""'2020-01-01\tuser0\t5DCC67A043EB776D8B7F5B70\n2020-01-02\tuser1\t5DCC67A042B46DFCC10EFD66\n2020-01-03\tuser2\t5DCC67A040243A8C1346D2DD'""" + +encrypt_input_example_mode_aes_256_cfb1_iv_None_aad_None = r"""'2020-01-01\tuser0\tFAAC1A7D2CE844F8DEB4C44E\n2020-01-02\tuser1\tFAAC1A7D2DF85A43828C0FF8\n2020-01-03\tuser2\tFAAC1A7D2FC7582CCEFCF330'""" + +encrypt_input_example_mode_aes_128_cfb1_iv_16_aad_None = r"""'2020-01-01\tuser0\t7670A865D13B1B65AD46F8ED\n2020-01-02\tuser1\t7670A865D046007A1E218286\n2020-01-03\tuser2\t7670A865D2E5B091492ECCFB'""" + +encrypt_input_example_mode_aes_192_cfb1_iv_16_aad_None = r"""'2020-01-01\tuser0\t7F38C051539074C0A635C937\n2020-01-02\tuser1\t7F38C051520A30DFACBE9564\n2020-01-03\tuser2\t7F38C051500DA29FF0E7B799'""" + +encrypt_input_example_mode_aes_256_cfb1_iv_16_aad_None = r"""'2020-01-01\tuser0\t0A216A58A5C0A33215E8E722\n2020-01-02\tuser1\t0A216A58A4E94067ABF030B6\n2020-01-03\tuser2\t0A216A58A6822CAB0318C632'""" + +encrypt_input_example_mode_aes_128_cfb8_iv_None_aad_None = r"""'2020-01-01\tuser0\t650D96B9698D20DB12E2E437\n2020-01-02\tuser1\t650D96B968F00D16ABF2852E\n2020-01-03\tuser2\t650D96B96B8141F425E60D6B'""" + +encrypt_input_example_mode_aes_192_cfb8_iv_None_aad_None = r"""'2020-01-01\tuser0\t72C4724B2F528724A12041C0\n2020-01-02\tuser1\t72C4724B2EF3C6A6FF9E09A9\n2020-01-03\tuser2\t72C4724B2D6EAB1D47709E15'""" + +encrypt_input_example_mode_aes_256_cfb8_iv_None_aad_None = r"""'2020-01-01\tuser0\tC5FD6C94961765ED204F2BCA\n2020-01-02\tuser1\tC5FD6C9497AB1C1AF1DE671C\n2020-01-03\tuser2\tC5FD6C949491F4A3EA5069B3'""" + +encrypt_input_example_mode_aes_128_cfb8_iv_16_aad_None = r"""'2020-01-01\tuser0\t471D217E9CA3593FFEC955C8\n2020-01-02\tuser1\t471D217E9D7F484D85F81F19\n2020-01-03\tuser2\t471D217E9EBBFD2EA9841008'""" + +encrypt_input_example_mode_aes_192_cfb8_iv_16_aad_None = r"""'2020-01-01\tuser0\t1D98EFFAEB9907457BD3FCB2\n2020-01-02\tuser1\t1D98EFFAEA2D930825C6AE22\n2020-01-03\tuser2\t1D98EFFAE92C1D018438B98B'""" + +encrypt_input_example_mode_aes_256_cfb8_iv_16_aad_None = r"""'2020-01-01\tuser0\t1C07B443BB7D7D60E9999C1D\n2020-01-02\tuser1\t1C07B443BA9674A3F68FF3FE\n2020-01-03\tuser2\t1C07B443B95F4B68161A616F'""" + +encrypt_input_example_mode_aes_128_cfb128_iv_None_aad_None = r"""'2020-01-01\tuser0\t65ACA4C7C6338E0F7EB60812\n2020-01-02\tuser1\t65ACA4C7C7338E0F7EB60812\n2020-01-03\tuser2\t65ACA4C7C4338E0F7EB60812'""" + +encrypt_input_example_mode_aes_192_cfb128_iv_None_aad_None = r"""'2020-01-01\tuser0\t72C47CEF0D63D2FB4FBC3CE4\n2020-01-02\tuser1\t72C47CEF0C63D2FB4FBC3CE4\n2020-01-03\tuser2\t72C47CEF0F63D2FB4FBC3CE4'""" + +encrypt_input_example_mode_aes_256_cfb128_iv_None_aad_None = r"""'2020-01-01\tuser0\tC5FDAAECF7B42C68180AA151\n2020-01-02\tuser1\tC5FDAAECF6B42C68180AA151\n2020-01-03\tuser2\tC5FDAAECF5B42C68180AA151'""" + +encrypt_input_example_mode_aes_128_cfb128_iv_16_aad_None = r"""'2020-01-01\tuser0\t47FBCCF6ED598C3D8A4B05C5\n2020-01-02\tuser1\t47FBCCF6EC598C3D8A4B05C5\n2020-01-03\tuser2\t47FBCCF6EF598C3D8A4B05C5'""" + +encrypt_input_example_mode_aes_192_cfb128_iv_16_aad_None = r"""'2020-01-01\tuser0\t1DB482E0874F04D4E734607A\n2020-01-02\tuser1\t1DB482E0864F04D4E734607A\n2020-01-03\tuser2\t1DB482E0854F04D4E734607A'""" + +encrypt_input_example_mode_aes_256_cfb128_iv_16_aad_None = r"""'2020-01-01\tuser0\t1C2BED650C8137ED139226D3\n2020-01-02\tuser1\t1C2BED650D8137ED139226D3\n2020-01-03\tuser2\t1C2BED650E8137ED139226D3'""" + +encrypt_input_example_mode_aes_128_ofb_iv_None_aad_None = r"""'2020-01-01\tuser0\t65ACA4C7C6338E0F7EB60812\n2020-01-02\tuser1\t65ACA4C7C7338E0F7EB60812\n2020-01-03\tuser2\t65ACA4C7C4338E0F7EB60812'""" + +encrypt_input_example_mode_aes_192_ofb_iv_None_aad_None = r"""'2020-01-01\tuser0\t72C47CEF0D63D2FB4FBC3CE4\n2020-01-02\tuser1\t72C47CEF0C63D2FB4FBC3CE4\n2020-01-03\tuser2\t72C47CEF0F63D2FB4FBC3CE4'""" + +encrypt_input_example_mode_aes_256_ofb_iv_None_aad_None = r"""'2020-01-01\tuser0\tC5FDAAECF7B42C68180AA151\n2020-01-02\tuser1\tC5FDAAECF6B42C68180AA151\n2020-01-03\tuser2\tC5FDAAECF5B42C68180AA151'""" + +encrypt_input_example_mode_aes_128_ofb_iv_16_aad_None = r"""'2020-01-01\tuser0\t47FBCCF6ED598C3D8A4B05C5\n2020-01-02\tuser1\t47FBCCF6EC598C3D8A4B05C5\n2020-01-03\tuser2\t47FBCCF6EF598C3D8A4B05C5'""" + +encrypt_input_example_mode_aes_192_ofb_iv_16_aad_None = r"""'2020-01-01\tuser0\t1DB482E0874F04D4E734607A\n2020-01-02\tuser1\t1DB482E0864F04D4E734607A\n2020-01-03\tuser2\t1DB482E0854F04D4E734607A'""" + +encrypt_input_example_mode_aes_256_ofb_iv_16_aad_None = r"""'2020-01-01\tuser0\t1C2BED650C8137ED139226D3\n2020-01-02\tuser1\t1C2BED650D8137ED139226D3\n2020-01-03\tuser2\t1C2BED650E8137ED139226D3'""" + +encrypt_input_example_mode_aes_128_gcm_iv_12_aad_None = r"""'2020-01-01\tuser0\t98E5A430C4A01C4429B0F37A4B3CDBC2BDB491651A36D7F904E231E0\n2020-01-02\tuser1\t98E5A430C5A01C4429B0F37A6E108322C2863C1ABF9BC7098CD369DB\n2020-01-03\tuser2\t98E5A430C6A01C4429B0F37A01646A0243D1CB9A516CF61814808196'""" + +encrypt_input_example_mode_aes_192_gcm_iv_12_aad_None = r"""'2020-01-01\tuser0\t3F89C3B657596C86202B59F4350807B364DA1E94682EAB679617575D\n2020-01-02\tuser1\t3F89C3B656596C86202B59F4FA03602ED37788B312FDE2AFDBB7F097\n2020-01-03\tuser2\t3F89C3B655596C86202B59F4691EC8880B8132DA9D8838F70D5618C8'""" + +encrypt_input_example_mode_aes_256_gcm_iv_12_aad_None = r"""'2020-01-01\tuser0\t23B80948CCDB54DC6D0B62F215132A07B30BA6F15593B4F946726B11\n2020-01-02\tuser1\t23B80948CDDB54DC6D0B62F2A01C1BAE07B8D6B26F60116040CDDB55\n2020-01-03\tuser2\t23B80948CEDB54DC6D0B62F2BD0D4954DA6D46772074FFCB4B0D0B98'""" + +encrypt_input_example_mode_aes_128_gcm_iv_12_aad_True = r"""'2020-01-01\tuser0\t98E5A430C4A01C4429B0F37AF9758E0EA4B44A50A7F964C8E51A913C\n2020-01-02\tuser1\t98E5A430C5A01C4429B0F37ADC59D6EEDB86E72F025474386D2BC907\n2020-01-03\tuser2\t98E5A430C6A01C4429B0F37AB32D3FCE5AD110AFECA34529F578214A'""" + +encrypt_input_example_mode_aes_192_gcm_iv_12_aad_True = r"""'2020-01-01\tuser0\t3F89C3B657596C86202B59F4B6C662DFF6347EF3B46C170A2F80E946\n2020-01-02\tuser1\t3F89C3B656596C86202B59F479CD05424199E8D4CEBF5EC262204E8C\n2020-01-03\tuser2\t3F89C3B655596C86202B59F4EAD0ADE4996F52BD41CA849AB4C1A6D3'""" + +encrypt_input_example_mode_aes_256_gcm_iv_12_aad_True = r"""'2020-01-01\tuser0\t23B80948CCDB54DC6D0B62F28787710BBF3F9A594C387B9F7CA2372B\n2020-01-02\tuser1\t23B80948CDDB54DC6D0B62F2328840A20B8CEA1A76CBDE067A1D876F\n2020-01-03\tuser2\t23B80948CEDB54DC6D0B62F22F991258D6597ADF39DF30AD71DD57A2'""" + +encrypt_input_example_mode_aes_128_ctr_iv_None_aad_None = r"""'2020-01-01\tuser0\t65ACA4C7C6338E0F7EB60812\n2020-01-02\tuser1\t65ACA4C7C7338E0F7EB60812\n2020-01-03\tuser2\t65ACA4C7C4338E0F7EB60812'""" + +encrypt_input_example_mode_aes_192_ctr_iv_None_aad_None = r"""'2020-01-01\tuser0\t72C47CEF0D63D2FB4FBC3CE4\n2020-01-02\tuser1\t72C47CEF0C63D2FB4FBC3CE4\n2020-01-03\tuser2\t72C47CEF0F63D2FB4FBC3CE4'""" + +encrypt_input_example_mode_aes_256_ctr_iv_None_aad_None = r"""'2020-01-01\tuser0\tC5FDAAECF7B42C68180AA151\n2020-01-02\tuser1\tC5FDAAECF6B42C68180AA151\n2020-01-03\tuser2\tC5FDAAECF5B42C68180AA151'""" + +encrypt_input_example_mode_aes_128_ctr_iv_16_aad_None = r"""'2020-01-01\tuser0\t47FBCCF6ED598C3D8A4B05C5\n2020-01-02\tuser1\t47FBCCF6EC598C3D8A4B05C5\n2020-01-03\tuser2\t47FBCCF6EF598C3D8A4B05C5'""" + +encrypt_input_example_mode_aes_192_ctr_iv_16_aad_None = r"""'2020-01-01\tuser0\t1DB482E0874F04D4E734607A\n2020-01-02\tuser1\t1DB482E0864F04D4E734607A\n2020-01-03\tuser2\t1DB482E0854F04D4E734607A'""" + +encrypt_input_example_mode_aes_256_ctr_iv_16_aad_None = r"""'2020-01-01\tuser0\t1C2BED650C8137ED139226D3\n2020-01-02\tuser1\t1C2BED650D8137ED139226D3\n2020-01-03\tuser2\t1C2BED650E8137ED139226D3'""" + diff --git a/tests/testflows/aes_encryption/tests/decrypt.py b/tests/testflows/aes_encryption/tests/decrypt.py new file mode 100644 index 00000000000..faba5363f2f --- /dev/null +++ b/tests/testflows/aes_encryption/tests/decrypt.py @@ -0,0 +1,634 @@ +# -*- coding: utf-8 -*- +import os +from importlib.machinery import SourceFileLoader + +from testflows.core import * +from testflows.core.name import basename +from testflows.asserts.helpers import varname +from testflows.asserts import error + +from aes_encryption.requirements.requirements import * +from aes_encryption.tests.common import * + +@TestOutline +def decrypt(self, ciphertext=None, key=None, mode=None, iv=None, aad=None, exitcode=0, message=None, step=When, cast=None, endcast=None, compare=None, no_checks=False): + """Execute `decrypt` function with the specified parameters. + """ + params = [] + if mode is not None: + params.append(mode) + if ciphertext is not None: + params.append(ciphertext) + if key is not None: + params.append(key) + if iv is not None: + params.append(iv) + if aad is not None: + params.append(aad) + + sql = f"decrypt(" + ", ".join(params) + ")" + if cast: + sql = f"{cast}({sql}){endcast or ''}" + if compare: + sql = f"{compare} = {sql}" + sql = f"SELECT {sql}" + + return current().context.node.query(sql, step=step, exitcode=exitcode, message=message, no_checks=no_checks) + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText("1.0"), +) +def invalid_ciphertext(self): + """Check that `decrypt` function does not crash when invalid + `ciphertext` is passed as the first parameter. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + invalid_ciphertexts = plaintexts + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} iv={iv_len} aad={aad_len}"""): + d_iv = None if not iv_len else f"'{iv[:iv_len]}'" + d_aad = None if not aad_len else f"'{aad}'" + + for datatype, ciphertext in invalid_ciphertexts: + if datatype in ["NULL"]: + continue + with When(f"invalid ciphertext={ciphertext}"): + if "cfb" in mode or "ofb" in mode or "ctr" in mode: + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, aad=d_aad, cast="hex") + else: + with When("I execute decrypt function"): + r = decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, aad=d_aad, no_checks=True, step=By) + with Then("exitcode is not zero"): + assert r.exitcode in [198, 36] + with And("exception is present in the output"): + assert "DB::Exception:" in r.output + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_InvalidParameters("1.0") +) +def invalid_parameters(self): + """Check that `decrypt` function returns an error when + we call it with invalid parameters. + """ + ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')" + + with Example("no parameters"): + decrypt(exitcode=42, message="DB::Exception: Incorrect number of arguments for function decrypt provided 0, expected 3 to 5") + + with Example("missing key and mode"): + decrypt(ciphertext=ciphertext, exitcode=42, + message="DB::Exception: Incorrect number of arguments for function decrypt provided 1") + + with Example("missing mode"): + decrypt(ciphertext=ciphertext, key="'123'", exitcode=42, + message="DB::Exception: Incorrect number of arguments for function decrypt provided 2") + + with Example("bad key type - UInt8"): + decrypt(ciphertext=ciphertext, key="123", mode="'aes-128-ecb'", exitcode=43, + message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3") + + with Example("bad mode type - forgot quotes"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="aes-128-ecb", exitcode=47, + message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query") + + with Example("bad mode type - UInt8"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="128", exitcode=43, + message="DB::Exception: Illegal type of argument #1 'mode'") + + with Example("bad iv type - UInt8"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43, + message="DB::Exception: Illegal type of argument") + + with Example("bad aad type - UInt8"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-gcm'", iv="'012345678912'", aad="123", exitcode=43, + message="DB::Exception: Illegal type of argument") + + with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=36, + message="DB::Exception: aes-128-ecb does not support IV") + + with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]): + decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=36, + message="DB::Exception: aes-128-ecb does not support IV") + + with Example("aad not valid for mode", requirements=[RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0")]): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv="'0123456789123456'", aad="'aad'", exitcode=36, + message="DB::Exception: AAD can be only set for GCM-mode") + + with Example("invalid mode value", requirements=[RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_Invalid("1.0")]): + with When("typo in the block algorithm"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128-eeb") + + with When("typo in the key size"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-127-ecb") + + with When("typo in the aes prefix"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aee-128-ecb") + + with When("missing last dash"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128ecb") + + with When("missing first dash"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes128-ecb") + + with When("all capitals"): + decrypt(ciphertext=ciphertext, key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36, + message="DB::Exception: Invalid mode: AES-128-ECB") + +@TestOutline(Scenario) +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0") +) +@Examples("mode key_len iv_len aad", [ + # ECB + ("'aes-128-ecb'", 16, None, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ecb'", 24, None, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ecb'", 32, None, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length("1.0"))), + # CBC + ("'aes-128-cbc'", 16, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cbc'", 24, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cbc'", 32, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length("1.0"))), + # CFB1 + ("'aes-128-cfb1'", 16, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb1'", 24, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb1'", 32, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length("1.0"))), + # CFB8 + ("'aes-128-cfb8'", 16, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb8'", 24, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb8'", 32, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length("1.0"))), + # CFB128 + ("'aes-128-cfb128'", 16, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb128'", 24, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb128'", 32, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length("1.0"))), + # OFB + ("'aes-128-ofb'", 16, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ofb'", 24, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ofb'", 32, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length("1.0"))), + # CTR + ("'aes-128-ctr'", 16, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ctr'", 24, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ctr'", 32, 16, None, + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length("1.0"))), +], "%-16s %-10s %-10s %-10s") +def invalid_key_or_iv_length_for_mode_non_gcm(self, mode, key_len, iv_len, aad): + """Check that an error is returned when key or iv length does not match + the expected value for the mode. + """ + ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')" + key = "0123456789" * 4 + iv = "0123456789" * 4 + + with When("key is too short"): + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + with When("key is too long"): + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + if iv_len is not None: + with When("iv is too short"): + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + + with When("iv is too long"): + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + + if aad is None: + with When("aad is specified but not needed"): + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1] if iv_len is not None else ''}'", aad="'AAD'", mode=mode, exitcode=36, message="DB::Exception: AAD can be only set for GCM-mode") + + else: + with When("iv is specified but not needed"): + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=36, message="DB::Exception: {} does not support IV".format(mode.strip("'"))) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Key_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Decrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0") +) +@Examples("mode key_len iv_len aad", [ + # GCM + ("'aes-128-gcm'", 16, 8, "'hello there aad'", + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length("1.0"))), + ("'aes-128-gcm'", 16, None, "'hello there aad'", + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-gcm'", 24, 8, "''", + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-gcm'", 24, None, "''", + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-gcm'", 32, 8, "'a'", + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-gcm'", 32, None, "'a'", + Requirements(RQ_SRS008_AES_Decrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length("1.0"))), +], "%-16s %-10s %-10s %-10s") +def invalid_key_or_iv_length_for_gcm(self, mode, key_len, iv_len, aad): + """Check that an error is returned when key or iv length does not match + the expected value for the GCM mode. + """ + ciphertext = "'hello there'" + plaintext = "hello there" + key = "0123456789" * 4 + iv = "0123456789" * 4 + + with When("key is too short"): + ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')" + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len-1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + with When("key is too long"): + ciphertext = "unhex('24AEBFEA049D6F4CF85AAB8CADEDF39CCCAA1C3C2AFF99E194789D')" + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + if iv_len is not None: + with When(f"iv is too short"): + ciphertext = "unhex('24AEBFEA049D6F4CF85AAB8CADEDF39CCCAA1C3C2AFF99E194789D')" + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=198, message="DB::Exception:") + else: + with When("iv is not specified"): + ciphertext = "unhex('1CD4EC93A4B0C687926E8F8C2AA3B4CE1943D006DAE3A774CB1AE5')" + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, exitcode=198, message="DB::Exception: Failed to set custom IV length to 0") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Parameters_AdditionalAuthenticatedData("1.0"), + RQ_SRS008_AES_Decrypt_Function_AdditionalAuthenticationData_Length("1.0") +) +def aad_parameter_types_and_length(self): + """Check that `decrypt` function accepts `aad` parameter as the fifth argument + of either `String` or `FixedString` types and that the length is not limited. + """ + plaintext = "hello there" + iv = "'012345678912'" + mode = "'aes-128-gcm'" + key = "'0123456789123456'" + + with When("aad is specified using String type"): + ciphertext = "unhex('19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526')" + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="'aad'", message=plaintext) + + with When("aad is specified using String with UTF8 characters"): + ciphertext = "unhex('19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39')" + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="'Gãńdåłf_Thê_Gręât'", message=plaintext) + + with When("aad is specified using FixedString type"): + ciphertext = "unhex('19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526')" + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="toFixedString('aad', 3)", message=plaintext) + + with When("aad is specified using FixedString with UTF8 characters"): + ciphertext = "unhex('19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39')" + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="toFixedString('Gãńdåłf_Thê_Gręât', 24)", message=plaintext) + + with When("aad is 0 bytes"): + ciphertext = "unhex('19A1183335B374C626B242DF92BB3F57F5D82BEDF41FD5D49F8BC9')" + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="''", message=plaintext) + + with When("aad is 1 byte"): + ciphertext = "unhex('19A1183335B374C626B242D1BCFC63B09CFE9EAD20285044A01035')" + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad="'1'", message=plaintext) + + with When("aad is 256 bytes"): + ciphertext = "unhex('19A1183335B374C626B242355AD3DD2C5D7E36AEECBB847BF9E8A7')" + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, aad=f"'{'1' * 256}'", message=plaintext) + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Parameters_InitializationVector("1.0") +) +def iv_parameter_types(self): + """Check that `decrypt` function accepts `iv` parameter as the fourth argument + of either `String` or `FixedString` types. + """ + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("iv is specified using String type"): + decrypt(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=iv, message="hello there") + + with When("iv is specified using String with UTF8 characters"): + decrypt(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="hello there") + + with When("iv is specified using FixedString type"): + decrypt(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="hello there") + + with When("iv is specified using FixedString with UTF8 characters"): + decrypt(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv=f"toFixedString('Gãńdåłf_Thê', 16)", message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Parameters_Key("1.0") +) +def key_parameter_types(self): + """Check that `decrypt` function accepts `key` parameter as the second argument + of either `String` or `FixedString` types. + """ + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("key is specified using String type"): + decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there") + + with When("key is specified using String with UTF8 characters"): + decrypt(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key="'Gãńdåłf_Thê'", mode=mode, message="hello there") + + with When("key is specified using FixedString type"): + decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=f"toFixedString({key}, 16)", mode=mode, message="hello there") + + with When("key is specified using FixedString with UTF8 characters"): + decrypt(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key=f"toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Parameters_Mode("1.0"), +) +def mode_parameter_types(self): + """Check that `decrypt` function accepts `mode` parameter as the third argument + of either `String` or `FixedString` types. + """ + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("mode is specified using String type"): + decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there") + + with When("mode is specified using FixedString type"): + decrypt(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=f"toFixedString({mode}, 12)", message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Parameters_ReturnValue("1.0") +) +def return_value(self): + """Check that `decrypt` functions returns String data type. + """ + ciphertext = "unhex('F024F9372FA0D8B974894D29FFB8A7F7')" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("I get type of the return value"): + sql = "SELECT toTypeName(decrypt(" + mode + "," + ciphertext + "," + key + "," + iv + "))" + r = self.context.node.query(sql) + + with Then("type should be String"): + assert r.output.strip() == "String", error() + + with When("I get the return value"): + decrypt(ciphertext=ciphertext, key=key, mode=mode, iv=iv, message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Syntax("1.0"), +) +def syntax(self): + """Check that `decrypt` function supports syntax + + ```sql + decrypt(ciphertext, key, mode, [iv, aad]) + ``` + """ + ciphertext = "19A1183335B374C626B242A6F6E8712E2B64DCDC6A468B2F654614" + sql = f"SELECT decrypt('aes-128-gcm', unhex('{ciphertext}'), '0123456789123456', '012345678912', 'AAD')" + self.context.node.query(sql, step=When, message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Decrypt_Function_Parameters_CipherText("1.0"), + RQ_SRS008_AES_Decrypt_Function_Parameters_Mode("1.0"), + RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_ValuesFormat("1.0"), +) +def decryption(self): + """Check that `decrypt` functions accepts `ciphertext` as the second parameter + and `mode` as the first parameter and we can convert the decrypted value into the original + value with the original data type. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + for datatype, plaintext in plaintexts: + + requirement = globals().get(f"""RQ_SRS008_AES_Decrypt_Function_Parameters_Mode_Value_{mode.strip("'").replace("-","_").upper()}""")("1.0") + + with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""", + requirements=[requirement]) as example: + + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + cast = None + endcast = None + ciphertext = f"unhex({ciphertext})" + compare = plaintext + + if datatype == "IPv4": + cast = "toIPv4(IPv4NumToString(reinterpretAsUInt32" + endcast = "))" + elif datatype in ["DateTime64", "UUID", "IPv6", "LowCardinality", "Enum8", "Enum16", "Decimal32", "Decimal64", "Decimal128", "Array"]: + xfail(reason="no conversion") + elif datatype == "NULL": + ciphertext = "NULL" + cast = "isNull" + compare = None + elif datatype in ["Float32", "Float64", "Date", "DateTime"] or "Int" in datatype: + cast = f"reinterpretAs{datatype}" + + decrypt(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), + aad=(None if not aad_len else f"'{aad}'"), + cast=cast, endcast=endcast, compare=compare, message="1") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_Mismatched_Key("1.0") +) +def mismatched_key(self): + """Check that `decrypt` function returns garbage or an error when key parameter does not match. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + datatype = "String" + plaintext = "'1'" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example: + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + with When("I decrypt using a mismatched key"): + r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'a{key[:key_len-1]}'", mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), + aad=(None if not aad_len else f"'{aad}'"), no_checks=True, cast="hex") + + with Then("exitcode shoud be 0 or 198"): + assert r.exitcode in [0, 198], error() + + with And("output should be garbage or an error"): + output = r.output.strip() + assert "Exception: Failed to decrypt" in output or output != "31", error() + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_Mismatched_IV("1.0") +) +def mismatched_iv(self): + """Check that `decrypt` function returns garbage or an error when iv parameter does not match. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + datatype = "String" + plaintext = "'1'" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + if not iv_len: + continue + with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example: + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + with When("I decrypt using a mismatched iv"): + r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mode, + iv=f"'a{iv[:iv_len-1]}'", + aad=(None if not aad_len else f"'{aad}'"), no_checks=True, cast="hex") + + with Then("exitcode shoud be 0 or 198"): + assert r.exitcode in [0, 198], error() + + with And("output should be garbage or an error"): + output = r.output.strip() + assert "Exception: Failed to decrypt" in output or output != "31", error() + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_Mismatched_AAD("1.0") +) +def mismatched_aad(self): + """Check that `decrypt` function returns garbage or an error when aad parameter does not match. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + datatype = "String" + plaintext = "'1'" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + if not aad_len: + continue + with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""") as example: + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + with When("I decrypt using a mismatched aad"): + r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), + aad=(None if not aad_len else f"'a{aad}'"), no_checks=True, cast="hex") + + with Then("exitcode shoud be 0 or 198"): + assert r.exitcode in [0, 198], error() + + with And("output should be garbage or an error"): + output = r.output.strip() + assert "Exception: Failed to decrypt" in output or output != "31", error() + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_Mismatched_Mode("1.0") +) +def mismatched_mode(self): + """Check that `decrypt` function returns garbage or an error when mode parameter does not match. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + plaintext = hex('Gãńdåłf_Thê_Gręât'.encode("utf-8")) + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt.py.encrypt.snapshot")).load_module() + + for mode, key_len, iv_len, aad_len in modes: + with Example(f"""mode={mode.strip("'")} datatype=utf8string iv={iv_len} aad={aad_len}""") as example: + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + for mismatched_mode, _, _, _ in modes: + if mismatched_mode == mode: + continue + + with When(f"I decrypt using mismatched mode {mismatched_mode}"): + r = decrypt(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mismatched_mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), + aad=(None if not aad_len else f"'{aad}'"), no_checks=True, cast="hex") + + with Then("exitcode shoud be 0 or 36 or 198"): + assert r.exitcode in [0, 36, 198], error() + + with And("output should be garbage or an error"): + output = r.output.strip() + condition = "Exception: Failed to decrypt" in output \ + or 'Exception: Invalid key size' in output \ + or output != plaintext + assert condition, error() + +@TestFeature +@Name("decrypt") +@Requirements( + RQ_SRS008_AES_Decrypt_Function("1.0") +) +def feature(self, node="clickhouse1"): + """Check the behavior of the `decrypt` function. + """ + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/decrypt_mysql.py b/tests/testflows/aes_encryption/tests/decrypt_mysql.py new file mode 100644 index 00000000000..b3df2121cd4 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/decrypt_mysql.py @@ -0,0 +1,521 @@ +# -*- coding: utf-8 -*- +import os +from importlib.machinery import SourceFileLoader + +from testflows.core import * +from testflows.core.name import basename +from testflows.asserts.helpers import varname +from testflows.asserts import error + +from aes_encryption.requirements.requirements import * +from aes_encryption.tests.common import * + +@TestOutline +def aes_decrypt_mysql(self, ciphertext=None, key=None, mode=None, iv=None, aad=None, exitcode=0, message=None, + step=When, cast=None, endcast=None, compare=None, no_checks=False): + """Execute `aes_decrypt_mysql` function with the specified parameters. + """ + params = [] + if mode is not None: + params.append(mode) + if ciphertext is not None: + params.append(ciphertext) + if key is not None: + params.append(key) + if iv is not None: + params.append(iv) + if aad is not None: + params.append(aad) + + sql = f"aes_decrypt_mysql(" + ", ".join(params) + ")" + if cast: + sql = f"{cast}({sql}){endcast or ''}" + if compare: + sql = f"{compare} = {sql}" + sql = f"SELECT {sql}" + + return current().context.node.query(sql, step=step, exitcode=exitcode, message=message, no_checks=no_checks) + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText("1.0"), +) +def invalid_ciphertext(self): + """Check that `aes_decrypt_mysql` function does not crash when invalid + `ciphertext` is passed as the first parameter. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + invalid_ciphertexts = plaintexts + + for mode, key_len, iv_len in mysql_modes: + with Example(f"""mode={mode.strip("'")} iv={iv_len}"""): + d_iv = None if not iv_len else f"'{iv[:iv_len]}'" + + for datatype, ciphertext in invalid_ciphertexts: + if datatype in ["NULL"]: + continue + with When(f"invalid ciphertext={ciphertext}"): + if "cfb" in mode or "ofb" in mode or "ctr" in mode: + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, cast="hex") + else: + with When("I execute aes_decrypt_mysql function"): + r = aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, iv=d_iv, no_checks=True, step=By) + with Then("exitcode is not zero"): + assert r.exitcode in [198, 36] + with And("exception is present in the output"): + assert "DB::Exception:" in r.output + +@TestOutline(Scenario) +@Examples("mode", [ + ("'aes-128-gcm'", Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error("1.0"))), + ("'aes-192-gcm'", Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error("1.0"))), + ("'aes-256-gcm'", Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error("1.0"))), + ("'aes-128-ctr'", Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error("1.0"))), + ("'aes-192-ctr'", Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error("1.0"))), + ("'aes-256-ctr'", Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error("1.0"))), +]) +def unsupported_modes(self, mode): + """Check that `aes_decrypt_mysql` function returns an error when unsupported modes are specified. + """ + ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')" + + aes_decrypt_mysql(ciphertext=ciphertext, mode=mode, key=f"'{'1'* 32}'", exitcode=36, message="DB::Exception: Unsupported cipher mode") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_InvalidParameters("1.0") +) +def invalid_parameters(self): + """Check that `aes_decrypt_mysql` function returns an error when + we call it with invalid parameters. + """ + ciphertext = "unhex('AA1826B5F66A903C888D5DCDA9FB63D1D9CCA10EC55F59D6C00D37')" + + with Example("no parameters"): + aes_decrypt_mysql(exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 0, expected 3 to 4") + + with Example("missing key and mode"): + aes_decrypt_mysql(ciphertext=ciphertext, exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 1") + + with Example("missing mode"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'123'", exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 2") + + with Example("bad key type - UInt8"): + aes_decrypt_mysql(ciphertext=ciphertext, key="123", mode="'aes-128-ecb'", exitcode=43, + message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3") + + with Example("bad mode type - forgot quotes"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="aes-128-ecb", exitcode=47, + message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query") + + with Example("bad mode type - UInt8"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="128", exitcode=43, + message="DB::Exception: Illegal type of argument #1 'mode'") + + with Example("bad iv type - UInt8"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43, + message="DB::Exception: Illegal type of argument") + + with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]): + aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=0, + message=None) + + with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode("1.0")]): + aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=0, + message=None) + + with Example("aad passed by mistake"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-cbc'", iv="'0123456789123456'", aad="'aad'", exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 5") + + with Example("aad passed by mistake type - UInt8"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-gcm'", iv="'012345678912'", aad="123", exitcode=42, + message="DB::Exception: Incorrect number of arguments for function aes_decrypt_mysql provided 5") + + with Example("invalid mode value", requirements=[RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_Invalid("1.0")]): + with When("typo in the block algorithm"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128-eeb") + + with When("typo in the key size"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-127-ecb") + + with When("typo in the aes prefix"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aee-128-ecb") + + with When("missing last dash"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128ecb") + + with When("missing first dash"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes128-ecb") + + with When("all capitals"): + aes_decrypt_mysql(ciphertext=ciphertext, key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36, + message="DB::Exception: Invalid mode: AES-128-ECB") + +@TestOutline(Scenario) +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooShortError("1.0"), + RQ_SRS008_AES_MySQL_Decrypt_Function_Key_Length_TooLong("1.0"), + RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooShortError("1.0"), + RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_Length_TooLong("1.0"), + RQ_SRS008_AES_MySQL_Decrypt_Function_InitializationVector_NotValidForMode("1.0") +) +@Examples("mode key_len iv_len", [ + # ECB + ("'aes-128-ecb'", 16, None, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ecb'", 24, None, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ecb'", 32, None, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length("1.0"))), + # CBC + ("'aes-128-cbc'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cbc'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cbc'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length("1.0"))), + # CFB1 + ("'aes-128-cfb1'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb1'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb1'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length("1.0"))), + # CFB8 + ("'aes-128-cfb8'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb8'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb8'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length("1.0"))), + # CFB128 + ("'aes-128-cfb128'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb128'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb128'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length("1.0"))), + # OFB + ("'aes-128-ofb'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ofb'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ofb'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Decrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length("1.0"))), +], "%-16s %-10s %-10s") +def key_or_iv_length_for_mode(self, mode, key_len, iv_len): + """Check that key or iv length for mode. + """ + ciphertext = "unhex('31F4C847CAB873AB34584368E3E85E3A')" + if mode == "'aes-128-ecb'": + ciphertext = "unhex('31F4C847CAB873AB34584368E3E85E3B')" + elif mode == "'aes-192-ecb'": + ciphertext = "unhex('073868ECDECA94133A61A0FFA282E877')" + elif mode == "'aes-256-ecb'": + ciphertext = "unhex('1729E5354D6EC44D89900ABDB09DC297')" + key = "0123456789" * 4 + iv = "0123456789" * 4 + + with When("key is too short"): + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + with When("key is too long"): + if "ecb" in mode or "cbc" in mode: + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", mode=mode, exitcode=198, message="DB::Exception: Failed to decrypt") + else: + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len+1]}'", mode=mode, cast="hex") + + if iv_len is not None: + with When("iv is too short"): + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + + with When("iv is too long"): + if "ecb" in mode or "cbc" in mode: + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, exitcode=198, message="DB::Exception: Failed to decrypt") + else: + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, cast="hex") + else: + with When("iv is specified but not needed"): + if "ecb" in mode or "cbc" in mode: + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=198, message="DB::Exception: Failed to decrypt") + else: + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode) + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_InitializationVector("1.0") +) +def iv_parameter_types(self): + """Check that `aes_decrypt_mysql` function accepts `iv` parameter as the fourth argument + of either `String` or `FixedString` types. + """ + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("iv is specified using String type"): + aes_decrypt_mysql(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=iv, message="hello there") + + with When("iv is specified using String with UTF8 characters"): + aes_decrypt_mysql(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="hello there") + + with When("iv is specified using FixedString type"): + aes_decrypt_mysql(ciphertext="unhex('F024F9372FA0D8B974894D29FFB8A7F7')", key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="hello there") + + with When("iv is specified using FixedString with UTF8 characters"): + aes_decrypt_mysql(ciphertext="unhex('7A4EC0FF3796F46BED281F4778ACE1DC')", key=key, mode=mode, iv=f"toFixedString('Gãńdåłf_Thê', 16)", message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Key("1.0") +) +def key_parameter_types(self): + """Check that `aes_decrypt` function accepts `key` parameter as the second argument + of either `String` or `FixedString` types. + """ + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("key is specified using String type"): + aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there") + + with When("key is specified using String with UTF8 characters"): + aes_decrypt_mysql(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key="'Gãńdåłf_Thê'", mode=mode, message="hello there") + + with When("key is specified using FixedString type"): + aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=f"toFixedString({key}, 16)", mode=mode, message="hello there") + + with When("key is specified using FixedString with UTF8 characters"): + aes_decrypt_mysql(ciphertext="unhex('180086AA42AD57B71C706EEC372D0C3D')", key=f"toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="hello there") + + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode("1.0"), +) +def mode_parameter_types(self): + """Check that `aes_decrypt_mysql` function accepts `mode` parameter as the third argument + of either `String` or `FixedString` types. + """ + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("mode is specified using String type"): + aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=mode, message="hello there") + + with When("mode is specified using FixedString type"): + aes_decrypt_mysql(ciphertext="unhex('49C9ADB81BA9B58C485E7ADB90E70576')", key=key, mode=f"toFixedString({mode}, 12)", message="hello there") + + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_ReturnValue("1.0") +) +def return_value(self): + """Check that `aes_decrypt_mysql` functions returns String data type. + """ + ciphertext = "unhex('F024F9372FA0D8B974894D29FFB8A7F7')" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("I get type of the return value"): + sql = "SELECT toTypeName(aes_decrypt_mysql(" + mode + "," + ciphertext + "," + key + "," + iv + "))" + r = self.context.node.query(sql) + + with Then("type should be String"): + assert r.output.strip() == "String", error() + + with When("I get the return value"): + aes_decrypt_mysql(ciphertext=ciphertext, key=key, mode=mode, iv=iv, message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Syntax("1.0"), +) +def syntax(self): + """Check that `aes_decrypt_mysql` function supports syntax + + ```sql + aes_decrypt_mysql(ciphertext, key, mode, [iv]) + ``` + """ + ciphertext = "70FE78410D6EE237C2DE4A" + sql = f"SELECT aes_decrypt_mysql('aes-128-ofb', unhex('{ciphertext}'), '0123456789123456', '0123456789123456')" + self.context.node.query(sql, step=When, message="hello there") + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_CipherText("1.0"), + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode("1.0"), + RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_ValuesFormat("1.0"), +) +def decryption(self): + """Check that `aes_decrypt_mysql` functions accepts `mode` as the first parameter + and `ciphertext` as the second parameter and we can convert the decrypted value into the original + value with the original data type. + """ + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module() + + for mode, key_len, iv_len in mysql_modes: + for datatype, plaintext in plaintexts: + + requirement = globals().get(f"""RQ_SRS008_AES_MySQL_Decrypt_Function_Parameters_Mode_Value_{mode.strip("'").replace("-","_").upper()}""")("1.0") + + with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} key={key_len} iv={iv_len}""", + requirements=[requirement]) as example: + + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + cast = None + endcast = None + ciphertext = f"unhex({ciphertext})" + compare = plaintext + + if datatype == "IPv4": + cast = "toIPv4(IPv4NumToString(reinterpretAsUInt32" + endcast = "))" + elif datatype in ["DateTime64", "UUID", "IPv6", "LowCardinality", "Enum8", "Enum16", "Decimal32", "Decimal64", "Decimal128", "Array"]: + xfail(reason="no conversion") + elif datatype == "NULL": + ciphertext = "NULL" + cast = "isNull" + compare = None + elif datatype in ["Float32", "Float64", "Date", "DateTime"] or "Int" in datatype: + cast = f"reinterpretAs{datatype}" + + aes_decrypt_mysql(ciphertext=ciphertext, key=f"'{key[:key_len]}'", mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), + cast=cast, endcast=endcast, compare=compare, message="1") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_Mismatched_Key("1.0") +) +def mismatched_key(self): + """Check that `aes_decrypt_mysql` function returns garbage or an error when key parameter does not match. + """ + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module() + + for mode, key_len, iv_len in mysql_modes: + with Example(f"""mode={mode.strip("'")} datatype=String key={key_len} iv={iv_len}""") as example: + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + with When("I decrypt using a mismatched key"): + r = aes_decrypt_mysql(ciphertext=f"unhex({ciphertext})", key=f"'a{key[:key_len-1]}'", mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), + cast="hex", no_checks=True) + + with Then("exitcode shoud be 0 or 198"): + assert r.exitcode in [0, 198], error() + + with And("output should be garbage or an error"): + output = r.output.strip() + assert "Exception: Failed to decrypt" in output or output != "31", error() + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_Mismatched_IV("1.0") +) +def mismatched_iv(self): + """Check that `aes_decrypt_mysql` function returns garbage or an error when iv parameter does not match. + """ + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module() + + for mode, key_len, iv_len in mysql_modes: + if not iv_len: + continue + with Example(f"""mode={mode.strip("'")} datatype=String key={key_len} iv={iv_len}""") as example: + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + with When("I decrypt using a mismatched key"): + r = aes_decrypt_mysql(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mode, + iv=f"'a{iv[:iv_len-1]}'", + cast="hex", no_checks=True) + + with Then("exitcode shoud be 0 or 198"): + assert r.exitcode in [0, 198], error() + + with And("output should be garbage or an error"): + output = r.output.strip() + assert "Exception: Failed to decrypt" in output or output != "31", error() + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_Mismatched_Mode("1.0") +) +def mismatched_mode(self): + """Check that `aes_decrypt_mysql` function returns garbage or an error when mode parameter does not match. + """ + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + plaintext = hex('Gãńdåłf_Thê_Gręât'.encode("utf-8")) + + with Given("I load encrypt snapshots"): + snapshot_module = SourceFileLoader("snapshot", os.path.join(current_dir(), "snapshots", "encrypt_mysql.py.encrypt_mysql.snapshot")).load_module() + + for mode, key_len, iv_len in mysql_modes: + if not iv_len: + continue + + with Example(f"""mode={mode.strip("'")} datatype=utf8string key={key_len} iv={iv_len}""") as example: + with Given("I have ciphertext"): + example_name = basename(example.name) + ciphertext = getattr(snapshot_module, varname(f"example_{example_name}")) + + for mismatched_mode, _, _ in mysql_modes: + if mismatched_mode == mode: + continue + + with When(f"I decrypt using a mismatched mode {mismatched_mode}"): + r = aes_decrypt_mysql(ciphertext=f"unhex({ciphertext})", key=f"'{key[:key_len]}'", mode=mismatched_mode, + iv=f"'{iv[:iv_len]}'", + cast="hex", no_checks=True) + + with Then("exitcode shoud be 0 or 36 or 198"): + assert r.exitcode in [0, 36, 198], error() + + with And("output should be garbage or an error"): + output = r.output.strip() + assert "Exception: Failed to decrypt" in output or output != plaintext, error() + +@TestFeature +@Name("decrypt_mysql") +@Requirements( + RQ_SRS008_AES_MySQL_Decrypt_Function("1.0") +) +def feature(self, node="clickhouse1"): + """Check the behavior of the `aes_decrypt_mysql` function. + """ + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) \ No newline at end of file diff --git a/tests/testflows/aes_encryption/tests/encrypt.py b/tests/testflows/aes_encryption/tests/encrypt.py new file mode 100644 index 00000000000..2fdd75fa0c8 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/encrypt.py @@ -0,0 +1,408 @@ +# -*- coding: utf-8 -*- +from testflows.core import * +from testflows.core.name import basename +from testflows.asserts import values, error, snapshot + +from aes_encryption.requirements.requirements import * +from aes_encryption.tests.common import * + +@TestOutline +def encrypt(self, plaintext=None, key=None, mode=None, iv=None, aad=None, exitcode=0, message=None, step=When): + """Execute `encrypt` function with the specified parameters. + """ + params = [] + if mode is not None: + params.append(mode) + if plaintext is not None: + params.append(plaintext) + if key is not None: + params.append(key) + if iv is not None: + params.append(iv) + if aad is not None: + params.append(aad) + + sql = "SELECT hex(encrypt(" + ", ".join(params) + "))" + + return current().context.node.query(sql, step=step, exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_InvalidParameters("1.0") +) +def invalid_parameters(self): + """Check that `encrypt` function returns an error when + we call it with invalid parameters. + """ + with Example("no parameters"): + encrypt(exitcode=42, message="DB::Exception: Incorrect number of arguments for function encrypt provided 0, expected 3 to 5") + + with Example("missing key and mode"): + encrypt(plaintext="'hello there'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function encrypt provided 1") + + with Example("missing mode"): + encrypt(plaintext="'hello there'", key="'123'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function encrypt provided 2") + + with Example("bad key type - UInt8"): + encrypt(plaintext="'hello there'", key="123", mode="'aes-128-ecb'", exitcode=43, + message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3") + + with Example("bad mode type - forgot quotes"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="aes-128-ecb", exitcode=47, + message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query") + + with Example("bad mode type - UInt8"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="128", exitcode=43, + message="DB::Exception: Illegal type of argument #1 'mode'") + + with Example("bad iv type - UInt8"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43, + message="DB::Exception: Illegal type of argument") + + with Example("bad aad type - UInt8"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-gcm'", iv="'012345678912'", aad="123", exitcode=43, + message="DB::Exception: Illegal type of argument") + + with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=36, + message="DB::Exception: aes-128-ecb does not support IV") + + with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=36, + message="DB::Exception: aes-128-ecb does not support IV") + + with Example("aad not valid for mode", requirements=[RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0")]): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cbc'", iv="'0123456789123456'", aad="'aad'", exitcode=36, + message="DB::Exception: AAD can be only set for GCM-mode") + + with Example("invalid mode value", requirements=[RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_Invalid("1.0")]): + with When("typo in the block algorithm"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128-eeb") + + with When("typo in the key size"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-127-ecb") + + with When("typo in the aes prefix"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aee-128-ecb") + + with When("missing last dash"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128ecb") + + with When("missing first dash"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes128-ecb") + + with When("all capitals"): + encrypt(plaintext="'hello there'", key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36, + message="DB::Exception: Invalid mode: AES-128-ECB") + +@TestOutline(Scenario) +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0") +) +@Examples("mode key_len iv_len aad", [ + # ECB + ("'aes-128-ecb'", 16, None, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ecb'", 24, None, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ecb'", 32, None, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length("1.0"))), + # CBC + ("'aes-128-cbc'", 16, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cbc'", 24, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cbc'", 32, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length("1.0"))), + # CFB1 + ("'aes-128-cfb1'", 16, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb1'", 24, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb1'", 32, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length("1.0"))), + # CFB8 + ("'aes-128-cfb8'", 16, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb8'", 24, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb8'", 32, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length("1.0"))), + # CFB128 + ("'aes-128-cfb128'", 16, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb128'", 24, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb128'", 32, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length("1.0"))), + # OFB + ("'aes-128-ofb'", 16, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ofb'", 24, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ofb'", 32, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length("1.0"))), + # CTR + ("'aes-128-ctr'", 16, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_CTR_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ctr'", 24, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_CTR_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ctr'", 32, 16, None, + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_CTR_KeyAndInitializationVector_Length("1.0"))), +], "%-16s %-10s %-10s %-10s") +def invalid_key_or_iv_length_for_mode_non_gcm(self, mode, key_len, iv_len, aad): + """Check that an error is returned when key or iv length does not match + the expected value for the mode. + """ + plaintext = "'hello there'" + key = "0123456789" * 4 + iv = "0123456789" * 4 + + with When("key is too short"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + with When("key is too long"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + if iv_len is not None: + with When("iv is too short"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + + with When("iv is too long"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + + if aad is None: + with When("aad is specified but not needed"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1] if iv_len is not None else ''}'", aad="'AAD'", mode=mode, exitcode=36, message="DB::Exception: AAD can be only set for GCM-mode") + + else: + with When("iv is specified but not needed"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=36, message="DB::Exception: {} does not support IV".format(mode.strip("'"))) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Key_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Encrypt_Function_InitializationVector_Length_InvalidLengthError("1.0"), + RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_NotValidForMode("1.0") +) +@Examples("mode key_len iv_len aad", [ + # GCM + ("'aes-128-gcm'", 16, 8, "'hello there aad'", + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_128_GCM_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-gcm'", 24, 8, "''", + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_192_GCM_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-gcm'", 32, 8, "'a'", + Requirements(RQ_SRS008_AES_Encrypt_Function_AES_256_GCM_KeyAndInitializationVector_Length("1.0"))), +], "%-16s %-10s %-10s %-10s") +def invalid_key_or_iv_length_for_gcm(self, mode, key_len, iv_len, aad): + """Check that an error is returned when key or iv length does not match + the expected value for the GCM mode. + """ + plaintext = "'hello there'" + key = "0123456789" * 4 + iv = "0123456789" * 4 + + with When("key is too short"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len-1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + with When("key is too long"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len+1]}'", iv=f"'{iv[:iv_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + if iv_len is not None: + with When(f"iv is too short"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=198, message="DB::Exception:") + else: + with When("iv is not specified"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + + if aad is not None: + with When(f"aad is {aad}"): + encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len]}'", aad=f"{aad}", mode=mode) + +@TestScenario +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Parameters_AdditionalAuthenticatedData("1.0"), + RQ_SRS008_AES_Encrypt_Function_AdditionalAuthenticationData_Length("1.0") +) +def aad_parameter_types_and_length(self): + """Check that `encrypt` function accepts `aad` parameter as the fifth argument + of either `String` or `FixedString` types and that the length is not limited. + """ + plaintext = "'hello there'" + iv = "'012345678912'" + mode = "'aes-128-gcm'" + key = "'0123456789123456'" + + with When("aad is specified using String type"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="'aad'", message="19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526") + + with When("aad is specified using String with UTF8 characters"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="'Gãńdåłf_Thê_Gręât'", message="19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39") + + with When("aad is specified using FixedString type"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="toFixedString('aad', 3)", message="19A1183335B374C626B24208AAEC97F148732CE05621AC87B21526") + + with When("aad is specified using FixedString with UTF8 characters"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="toFixedString('Gãńdåłf_Thê_Gręât', 24)", message="19A1183335B374C626B242C68D9618A8C2664D7B6A3FE978104B39") + + with When("aad is 0 bytes"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="''", message="19A1183335B374C626B242DF92BB3F57F5D82BEDF41FD5D49F8BC9") + + with When("aad is 1 byte"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad="'1'", message="19A1183335B374C626B242D1BCFC63B09CFE9EAD20285044A01035") + + with When("aad is 256 bytes"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, aad=f"'{'1' * 256}'", message="19A1183335B374C626B242355AD3DD2C5D7E36AEECBB847BF9E8A7") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Parameters_InitializationVector("1.0") +) +def iv_parameter_types(self): + """Check that `encrypt` function accepts `iv` parameter as the fourth argument + of either `String` or `FixedString` types. + """ + plaintext = "'hello there'" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("iv is specified using String type"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7") + + with When("iv is specified using String with UTF8 characters"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="7A4EC0FF3796F46BED281F4778ACE1DC") + + with When("iv is specified using FixedString type"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="F024F9372FA0D8B974894D29FFB8A7F7") + + with When("iv is specified using FixedString with UTF8 characters"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv="toFixedString('Gãńdåłf_Thê', 16)", message="7A4EC0FF3796F46BED281F4778ACE1DC") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Parameters_Key("1.0") +) +def key_parameter_types(self): + """Check that `encrypt` function accepts `key` parameter as the second argument + of either `String` or `FixedString` types. + """ + plaintext = "'hello there'" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("key is specified using String type"): + encrypt(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + + with When("key is specified using String with UTF8 characters"): + encrypt(plaintext=plaintext, key="'Gãńdåłf_Thê'", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D") + + with When("key is specified using FixedString type"): + encrypt(plaintext=plaintext, key=f"toFixedString({key}, 16)", mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + + with When("key is specified using FixedString with UTF8 characters"): + encrypt(plaintext=plaintext, key="toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Parameters_Mode("1.0"), +) +def mode_parameter_types(self): + """Check that `encrypt` function accepts `mode` parameter as the third argument + of either `String` or `FixedString` types. + """ + plaintext = "'hello there'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("mode is specified using String type"): + encrypt(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + + with When("mode is specified using FixedString type"): + encrypt(plaintext=plaintext, key=key, mode=f"toFixedString({mode}, 12)", message="49C9ADB81BA9B58C485E7ADB90E70576") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Parameters_PlainText("1.0"), + RQ_SRS008_AES_Encrypt_Function_Parameters_Mode("1.0"), + RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_ValuesFormat("1.0"), +) +def encryption(self): + """Check that `encrypt` functions accepts `plaintext` as the second parameter + with any data type and `mode` as the first parameter. + """ + key = f"{'1' * 36}" + iv = f"{'2' * 16}" + aad = "some random aad" + + for mode, key_len, iv_len, aad_len in modes: + for datatype, plaintext in plaintexts: + + requirement = globals().get(f"""RQ_SRS008_AES_Encrypt_Function_Parameters_Mode_Value_{mode.strip("'").replace("-","_").upper()}""")("1.0") + + with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} iv={iv_len} aad={aad_len}""", + requirements=[requirement]) as example: + r = encrypt(plaintext=plaintext, key=f"'{key[:key_len]}'", mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'"), aad=(None if not aad_len else f"'{aad}'")) + + with Then("I check output against snapshot"): + with values() as that: + example_name = basename(example.name) + assert that(snapshot(r.output.strip(), "encrypt", name=f"example_{example_name.replace(' ', '_')}")), error() + +@TestScenario +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Parameters_ReturnValue("1.0") +) +def return_value(self): + """Check that `encrypt` functions returns String data type. + """ + plaintext = "'hello there'" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("I get type of the return value"): + sql = "SELECT toTypeName(encrypt(" + mode + "," + plaintext + "," + key + "," + iv + "))" + r = self.context.node.query(sql) + + with Then("type should be String"): + assert r.output.strip() == "String", error() + + with When("I get return ciphertext as hex"): + encrypt(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Encrypt_Function_Syntax("1.0"), +) +def syntax(self): + """Check that `encrypt` function supports syntax + + ```sql + encrypt(plaintext, key, mode, [iv, aad]) + ``` + """ + sql = "SELECT hex(encrypt('aes-128-gcm', 'hello there', '0123456789123456', '012345678912', 'AAD'))" + self.context.node.query(sql, step=When, message="19A1183335B374C626B242A6F6E8712E2B64DCDC6A468B2F654614") + +@TestFeature +@Name("encrypt") +@Requirements( + RQ_SRS008_AES_Encrypt_Function("1.0") +) +def feature(self, node="clickhouse1"): + """Check the behavior of the `encrypt` function. + """ + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/encrypt_mysql.py b/tests/testflows/aes_encryption/tests/encrypt_mysql.py new file mode 100644 index 00000000000..56183aff125 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/encrypt_mysql.py @@ -0,0 +1,326 @@ +from testflows.core import * +from testflows.core.name import basename +from testflows.asserts import values, error, snapshot + +from aes_encryption.requirements.requirements import * +from aes_encryption.tests.common import * + +@TestOutline +def aes_encrypt_mysql(self, plaintext=None, key=None, mode=None, iv=None, exitcode=0, message=None, step=When): + """Execute `aes_encrypt_mysql` function with the specified parameters. + """ + params = [] + if mode is not None: + params.append(mode) + if plaintext is not None: + params.append(plaintext) + if key is not None: + params.append(key) + if iv is not None: + params.append(iv) + + sql = "SELECT hex(aes_encrypt_mysql(" + ", ".join(params) + "))" + + return current().context.node.query(sql, step=step, exitcode=exitcode, message=message) + +@TestOutline(Scenario) +@Examples("mode", [ + ("'aes-128-gcm'", Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_GCM_Error("1.0"))), + ("'aes-192-gcm'", Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_GCM_Error("1.0"))), + ("'aes-256-gcm'", Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_GCM_Error("1.0"))), + ("'aes-128-ctr'", Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_128_CTR_Error("1.0"))), + ("'aes-192-ctr'", Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_192_CTR_Error("1.0"))), + ("'aes-256-ctr'", Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_AES_256_CTR_Error("1.0"))), +]) +def unsupported_modes(self, mode): + """Check that `aes_encrypt_mysql` function returns an error when unsupported modes are specified. + """ + aes_encrypt_mysql(plaintext="'hello there'", mode=mode, key=f"'{'1'* 32}'", exitcode=36, message="DB::Exception: Unsupported cipher mode") + +@TestScenario +@Requirements( + RQ_SRS008_AES_Functions_InvalidParameters("1.0") +) +def invalid_parameters(self): + """Check that `aes_encrypt_mysql` function returns an error when + we call it with invalid parameters. + """ + with Example("no parameters"): + aes_encrypt_mysql(exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_encrypt provided 0, expected 3 to 4") + + with Example("missing key and mode"): + aes_encrypt_mysql(plaintext="'hello there'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_encrypt_mysql provided 1") + + with Example("missing mode"): + aes_encrypt_mysql(plaintext="'hello there'", key="'123'", exitcode=42, message="DB::Exception: Incorrect number of arguments for function aes_encrypt_mysql provided 2") + + with Example("bad key type - UInt8"): + aes_encrypt_mysql(plaintext="'hello there'", key="123", mode="'aes-128-ecb'", exitcode=43, + message="DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #3") + + with Example("bad mode type - forgot quotes"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="aes-128-ecb", exitcode=47, + message="DB::Exception: Missing columns: 'ecb' 'aes' while processing query") + + with Example("bad mode type - UInt8"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="128", exitcode=43, + message="DB::Exception: Illegal type of argument #1 'mode'") + + with Example("bad iv type - UInt8"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-cbc'", iv='128', exitcode=43, + message="DB::Exception: Illegal type of argument") + + with Example("iv not valid for mode", requirements=[RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="'012345678912'", exitcode=36, + message="DB::Exception: aes-128-ecb does not support IV") + + with Example("iv not valid for mode - size 0", requirements=[RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode("1.0")]): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-ecb'", iv="''", exitcode=0, + message=None) + + with Example("invalid mode value", requirements=[RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_Invalid("1.0")]): + with When("typo in the block algorithm"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128-eeb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128-eeb") + + with When("typo in the key size"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-127-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-127-ecb") + + with When("typo in the aes prefix"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aee-128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aee-128-ecb") + + with When("missing last dash"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes-128ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes-128ecb") + + with When("missing first dash"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'aes128-ecb'", exitcode=36, + message="DB::Exception: Invalid mode: aes128-ecb") + + with When("all capitals"): + aes_encrypt_mysql(plaintext="'hello there'", key="'0123456789123456'", mode="'AES-128-ECB'", exitcode=36, + message="DB::Exception: Invalid mode: AES-128-ECB") + +@TestOutline(Scenario) +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooShortError("1.0"), + RQ_SRS008_AES_MySQL_Encrypt_Function_Key_Length_TooLong("1.0"), + RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooShortError("1.0"), + RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_Length_TooLong("1.0"), + RQ_SRS008_AES_MySQL_Encrypt_Function_InitializationVector_NotValidForMode("1.0") +) +@Examples("mode key_len iv_len", [ + # ECB + ("'aes-128-ecb'", 16, None, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ecb'", 24, None, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_ECB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ecb'", 32, None, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_ECB_KeyAndInitializationVector_Length("1.0"))), + # CBC + ("'aes-128-cbc'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cbc'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CBC_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cbc'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CBC_KeyAndInitializationVector_Length("1.0"))), + # CFB1 + ("'aes-128-cfb1'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb1'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB1_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb1'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB1_KeyAndInitializationVector_Length("1.0"))), + # CFB8 + ("'aes-128-cfb8'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb8'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB8_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb8'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB8_KeyAndInitializationVector_Length("1.0"))), + # CFB128 + ("'aes-128-cfb128'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-cfb128'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_CFB128_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-cfb128'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_CFB128_KeyAndInitializationVector_Length("1.0"))), + # OFB + ("'aes-128-ofb'", 16, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_128_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-192-ofb'", 24, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_192_OFB_KeyAndInitializationVector_Length("1.0"))), + ("'aes-256-ofb'", 32, 16, + Requirements(RQ_SRS008_AES_MySQL_Encrypt_Function_AES_256_OFB_KeyAndInitializationVector_Length("1.0"))), +], "%-16s %-10s %-10s") +def key_or_iv_length_for_mode(self, mode, key_len, iv_len): + """Check that key or iv length for mode. + """ + plaintext = "'hello there'" + key = "0123456789" * 4 + iv = "0123456789" * 4 + + with When("key is too short"): + aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid key size") + + with When("key is too long"): + aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len+1]}'", mode=mode) + + if iv_len is not None: + with When("iv is too short"): + aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len-1]}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + + with When("iv is too long"): + aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv[:iv_len+1]}'", mode=mode) + else: + with When("iv is specified but not needed"): + aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", iv=f"'{iv}'", mode=mode, exitcode=36, message="DB::Exception: Invalid IV size") + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_InitializationVector("1.0") +) +def iv_parameter_types(self): + """Check that `aes_encrypt_mysql` function accepts `iv` parameter as the fourth argument + of either `String` or `FixedString` types. + """ + plaintext = "'hello there'" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("iv is specified using String type"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7") + + with When("iv is specified using String with UTF8 characters"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv="'Gãńdåłf_Thê'", message="7A4EC0FF3796F46BED281F4778ACE1DC") + + with When("iv is specified using FixedString type"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv=f"toFixedString({iv}, 16)", message="F024F9372FA0D8B974894D29FFB8A7F7") + + with When("iv is specified using FixedString with UTF8 characters"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv="toFixedString('Gãńdåłf_Thê', 16)", message="7A4EC0FF3796F46BED281F4778ACE1DC") + + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Key("1.0") +) +def key_parameter_types(self): + """Check that `aes_encrypt_mysql` function accepts `key` parameter as the second argument + of either `String` or `FixedString` types. + """ + plaintext = "'hello there'" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("key is specified using String type"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + + with When("key is specified using String with UTF8 characters"): + aes_encrypt_mysql(plaintext=plaintext, key="'Gãńdåłf_Thê'", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D") + + with When("key is specified using FixedString type"): + aes_encrypt_mysql(plaintext=plaintext, key=f"toFixedString({key}, 16)", mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + + with When("key is specified using FixedString with UTF8 characters"): + aes_encrypt_mysql(plaintext=plaintext, key="toFixedString('Gãńdåłf_Thê', 16)", mode=mode, message="180086AA42AD57B71C706EEC372D0C3D") + + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode("1.0"), +) +def mode_parameter_types(self): + """Check that `aes_encrypt_mysql` function accepts `mode` parameter as the third argument + of either `String` or `FixedString` types. + """ + plaintext = "'hello there'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("mode is specified using String type"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, message="49C9ADB81BA9B58C485E7ADB90E70576") + + with When("mode is specified using FixedString type"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=f"toFixedString({mode}, 12)", message="49C9ADB81BA9B58C485E7ADB90E70576") + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_ReturnValue("1.0") +) +def return_value(self): + """Check that `aes_encrypt_mysql` functions returns String data type. + """ + plaintext = "'hello there'" + iv = "'0123456789123456'" + mode = "'aes-128-cbc'" + key = "'0123456789123456'" + + with When("I get type of the return value"): + sql = "SELECT toTypeName(aes_encrypt_mysql("+ mode + "," + plaintext + "," + key + "," + iv + "))" + r = self.context.node.query(sql) + + with Then("type should be String"): + assert r.output.strip() == "String", error() + + with When("I get return ciphertext as hex"): + aes_encrypt_mysql(plaintext=plaintext, key=key, mode=mode, iv=iv, message="F024F9372FA0D8B974894D29FFB8A7F7") + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function_Syntax("1.0"), +) +def syntax(self): + """Check that `aes_encrypt_mysql` function supports syntax + + ```sql + aes_encrypt_mysql(plaintext, key, mode, [iv]) + ``` + """ + sql = "SELECT hex(aes_encrypt_mysql('aes-128-ofb', 'hello there', '0123456789123456', '0123456789123456'))" + self.context.node.query(sql, step=When, message="70FE78410D6EE237C2DE4A") + +@TestScenario +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_PlainText("1.0"), + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode("1.0"), + RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_ValuesFormat("1.0"), +) +def encryption(self): + """Check that `aes_encrypt_mysql` functions accepts `plaintext` as the second parameter + with any data type and `mode` as the first parameter. + """ + key = f"{'1' * 64}" + iv = f"{'2' * 64}" + + for mode, key_len, iv_len in mysql_modes: + for datatype, plaintext in plaintexts: + requirement = globals().get(f"""RQ_SRS008_AES_MySQL_Encrypt_Function_Parameters_Mode_Value_{mode.strip("'").replace("-","_").upper()}""")("1.0") + + with Example(f"""mode={mode.strip("'")} datatype={datatype.strip("'")} key={key_len} iv={iv_len}""", + requirements=[requirement]) as example: + + r = aes_encrypt_mysql(plaintext=plaintext, key=f"'{key[:key_len]}'", mode=mode, + iv=(None if not iv_len else f"'{iv[:iv_len]}'")) + + with Then("I check output against snapshot"): + with values() as that: + example_name = basename(example.name) + assert that(snapshot(r.output.strip(), "encrypt_mysql", name=f"example_{example_name.replace(' ', '_')}")), error() + +@TestFeature +@Name("encrypt_mysql") +@Requirements( + RQ_SRS008_AES_MySQL_Encrypt_Function("1.0") +) +def feature(self, node="clickhouse1"): + """Check the behavior of the `aes_encrypt_mysql` function. + """ + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/aes_encryption/tests/snapshots/encrypt.py.encrypt.snapshot b/tests/testflows/aes_encryption/tests/snapshots/encrypt.py.encrypt.snapshot new file mode 100644 index 00000000000..566c1074efb --- /dev/null +++ b/tests/testflows/aes_encryption/tests/snapshots/encrypt.py.encrypt.snapshot @@ -0,0 +1,2700 @@ +example_mode_aes_128_ecb_datatype_String_iv_None_aad_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_ecb_datatype_FixedString_iv_None_aad_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_ecb_datatype_UInt8_iv_None_aad_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_ecb_datatype_UInt16_iv_None_aad_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_ecb_datatype_UInt32_iv_None_aad_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_ecb_datatype_UInt64_iv_None_aad_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_ecb_datatype_Int8_iv_None_aad_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_ecb_datatype_Int16_iv_None_aad_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_ecb_datatype_Int32_iv_None_aad_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_ecb_datatype_Int64_iv_None_aad_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_ecb_datatype_Float32_iv_None_aad_None = r"""'FF4D70D9A1050E6BBDD0325FC45CC22D'""" + +example_mode_aes_128_ecb_datatype_Float64_iv_None_aad_None = r"""'75FE6B4A722A31D7760680CC1B9F131D'""" + +example_mode_aes_128_ecb_datatype_Decimal32_iv_None_aad_None = r"""'83BBD7CCE7E5A38071653870475D48D2'""" + +example_mode_aes_128_ecb_datatype_Decimal64_iv_None_aad_None = r"""'BE0DD9302B2952CE9CC3721DD85C8E66'""" + +example_mode_aes_128_ecb_datatype_Decimal128_iv_None_aad_None = r"""'5F3DBFA74809E45E03980357B26787AFF30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_ecb_datatype_UUID_iv_None_aad_None = r"""'FF9161B222B4A67481271035745F06D9F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_ecb_datatype_Date_iv_None_aad_None = r"""'1E4FBE33752D96D147E890C29A409BFE'""" + +example_mode_aes_128_ecb_datatype_DateTime_iv_None_aad_None = r"""'384F3D97B78D52C73CD06C0E1B6DE399'""" + +example_mode_aes_128_ecb_datatype_DateTime64_iv_None_aad_None = r"""'C7F50A2D0175F3ED280AD42FF01FF5F2'""" + +example_mode_aes_128_ecb_datatype_LowCardinality_iv_None_aad_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_ecb_datatype_Array_iv_None_aad_None = r"""'D9152D05CFA9E162983A5A2E883109B4'""" + +example_mode_aes_128_ecb_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_128_ecb_datatype_IPv4_iv_None_aad_None = r"""'4F32782638C1F33C6A7202CA83F0C12C'""" + +example_mode_aes_128_ecb_datatype_IPv6_iv_None_aad_None = r"""'F54700FF04ADAD342BA6830DB12AD7E9F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_ecb_datatype_Enum8_iv_None_aad_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_ecb_datatype_Enum16_iv_None_aad_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_192_ecb_datatype_String_iv_None_aad_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_ecb_datatype_FixedString_iv_None_aad_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_ecb_datatype_UInt8_iv_None_aad_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_ecb_datatype_UInt16_iv_None_aad_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_ecb_datatype_UInt32_iv_None_aad_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_ecb_datatype_UInt64_iv_None_aad_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_ecb_datatype_Int8_iv_None_aad_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_ecb_datatype_Int16_iv_None_aad_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_ecb_datatype_Int32_iv_None_aad_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_ecb_datatype_Int64_iv_None_aad_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_ecb_datatype_Float32_iv_None_aad_None = r"""'4E0C122631ED64EAD726833291A81878'""" + +example_mode_aes_192_ecb_datatype_Float64_iv_None_aad_None = r"""'3F723599278E22E4692CE7D7D5F9A12F'""" + +example_mode_aes_192_ecb_datatype_Decimal32_iv_None_aad_None = r"""'2420D49DBAA5CEF7D853C98DA1BD33BF'""" + +example_mode_aes_192_ecb_datatype_Decimal64_iv_None_aad_None = r"""'FDF594113FCC2776653ED109A51FADF1'""" + +example_mode_aes_192_ecb_datatype_Decimal128_iv_None_aad_None = r"""'79207931793E374FB5A3A2AC1ECA857AD8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_ecb_datatype_UUID_iv_None_aad_None = r"""'9FDB738E78D0D2F774C484ED82A854E4D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_ecb_datatype_Date_iv_None_aad_None = r"""'2CDD4685168FA3E2A7FA2092E86F44D4'""" + +example_mode_aes_192_ecb_datatype_DateTime_iv_None_aad_None = r"""'A4BEE097872E44FAD94D6707D6643DF5'""" + +example_mode_aes_192_ecb_datatype_DateTime64_iv_None_aad_None = r"""'1798B23C09F783623943560DF142E0F3'""" + +example_mode_aes_192_ecb_datatype_LowCardinality_iv_None_aad_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_ecb_datatype_Array_iv_None_aad_None = r"""'7C0B9021CAF2CBBB06DBF589740DCC65'""" + +example_mode_aes_192_ecb_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_192_ecb_datatype_IPv4_iv_None_aad_None = r"""'B20465C932A0719BA04E2F76371510D8'""" + +example_mode_aes_192_ecb_datatype_IPv6_iv_None_aad_None = r"""'CCCDC9B9C3F182254591DFEDDCE9F232D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_ecb_datatype_Enum8_iv_None_aad_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_ecb_datatype_Enum16_iv_None_aad_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_256_ecb_datatype_String_iv_None_aad_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_ecb_datatype_FixedString_iv_None_aad_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_ecb_datatype_UInt8_iv_None_aad_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_ecb_datatype_UInt16_iv_None_aad_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_ecb_datatype_UInt32_iv_None_aad_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_ecb_datatype_UInt64_iv_None_aad_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_ecb_datatype_Int8_iv_None_aad_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_ecb_datatype_Int16_iv_None_aad_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_ecb_datatype_Int32_iv_None_aad_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_ecb_datatype_Int64_iv_None_aad_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_ecb_datatype_Float32_iv_None_aad_None = r"""'A11ED1B75CF1C04C6CA3A31E76627D4C'""" + +example_mode_aes_256_ecb_datatype_Float64_iv_None_aad_None = r"""'464C85EB7DB36D95CF48A3431CC7B2BC'""" + +example_mode_aes_256_ecb_datatype_Decimal32_iv_None_aad_None = r"""'988C793BD81036C1D05EC47F43851269'""" + +example_mode_aes_256_ecb_datatype_Decimal64_iv_None_aad_None = r"""'50FFB9C104DBFF3F415F12BA73D6FF1C'""" + +example_mode_aes_256_ecb_datatype_Decimal128_iv_None_aad_None = r"""'B04C40C085A262E3AA27F8E7F6831DCB217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_ecb_datatype_UUID_iv_None_aad_None = r"""'6A36D74ACB38B95FA77BC757A7AB2C34217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_ecb_datatype_Date_iv_None_aad_None = r"""'F1CFA361A9B08FC101F3A4707A3E04D2'""" + +example_mode_aes_256_ecb_datatype_DateTime_iv_None_aad_None = r"""'D58178485CD1AE1C30F68383307B8BC5'""" + +example_mode_aes_256_ecb_datatype_DateTime64_iv_None_aad_None = r"""'A19B65BCB740B2AF4D421CE1DEC43608'""" + +example_mode_aes_256_ecb_datatype_LowCardinality_iv_None_aad_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_ecb_datatype_Array_iv_None_aad_None = r"""'C4071E4FD44F004347EA9932326B7038'""" + +example_mode_aes_256_ecb_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_256_ecb_datatype_IPv4_iv_None_aad_None = r"""'6C7950041CB4041D4D8036FCD22E3B06'""" + +example_mode_aes_256_ecb_datatype_IPv6_iv_None_aad_None = r"""'8CBF2DC164F4086B8DD14B75E3065621217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_ecb_datatype_Enum8_iv_None_aad_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_ecb_datatype_Enum16_iv_None_aad_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_128_cbc_datatype_String_iv_None_aad_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_cbc_datatype_FixedString_iv_None_aad_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_cbc_datatype_UInt8_iv_None_aad_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_cbc_datatype_UInt16_iv_None_aad_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_cbc_datatype_UInt32_iv_None_aad_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_cbc_datatype_UInt64_iv_None_aad_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_cbc_datatype_Int8_iv_None_aad_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_cbc_datatype_Int16_iv_None_aad_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_cbc_datatype_Int32_iv_None_aad_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_cbc_datatype_Int64_iv_None_aad_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_cbc_datatype_Float32_iv_None_aad_None = r"""'FF4D70D9A1050E6BBDD0325FC45CC22D'""" + +example_mode_aes_128_cbc_datatype_Float64_iv_None_aad_None = r"""'75FE6B4A722A31D7760680CC1B9F131D'""" + +example_mode_aes_128_cbc_datatype_Decimal32_iv_None_aad_None = r"""'83BBD7CCE7E5A38071653870475D48D2'""" + +example_mode_aes_128_cbc_datatype_Decimal64_iv_None_aad_None = r"""'BE0DD9302B2952CE9CC3721DD85C8E66'""" + +example_mode_aes_128_cbc_datatype_Decimal128_iv_None_aad_None = r"""'5F3DBFA74809E45E03980357B26787AF0D55B905F5525D3F5916FF811D8A6E7E'""" + +example_mode_aes_128_cbc_datatype_UUID_iv_None_aad_None = r"""'FF9161B222B4A67481271035745F06D991B6833DF67CBA9BC6E1AAEADBE363BB'""" + +example_mode_aes_128_cbc_datatype_Date_iv_None_aad_None = r"""'1E4FBE33752D96D147E890C29A409BFE'""" + +example_mode_aes_128_cbc_datatype_DateTime_iv_None_aad_None = r"""'384F3D97B78D52C73CD06C0E1B6DE399'""" + +example_mode_aes_128_cbc_datatype_DateTime64_iv_None_aad_None = r"""'C7F50A2D0175F3ED280AD42FF01FF5F2'""" + +example_mode_aes_128_cbc_datatype_LowCardinality_iv_None_aad_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_cbc_datatype_Array_iv_None_aad_None = r"""'D9152D05CFA9E162983A5A2E883109B4'""" + +example_mode_aes_128_cbc_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_128_cbc_datatype_IPv4_iv_None_aad_None = r"""'4F32782638C1F33C6A7202CA83F0C12C'""" + +example_mode_aes_128_cbc_datatype_IPv6_iv_None_aad_None = r"""'F54700FF04ADAD342BA6830DB12AD7E9B1B4BE8B15BAE0B2C9196D69E3D53C6C'""" + +example_mode_aes_128_cbc_datatype_Enum8_iv_None_aad_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_cbc_datatype_Enum16_iv_None_aad_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_192_cbc_datatype_String_iv_None_aad_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_cbc_datatype_FixedString_iv_None_aad_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_cbc_datatype_UInt8_iv_None_aad_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_cbc_datatype_UInt16_iv_None_aad_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_cbc_datatype_UInt32_iv_None_aad_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_cbc_datatype_UInt64_iv_None_aad_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_cbc_datatype_Int8_iv_None_aad_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_cbc_datatype_Int16_iv_None_aad_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_cbc_datatype_Int32_iv_None_aad_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_cbc_datatype_Int64_iv_None_aad_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_cbc_datatype_Float32_iv_None_aad_None = r"""'4E0C122631ED64EAD726833291A81878'""" + +example_mode_aes_192_cbc_datatype_Float64_iv_None_aad_None = r"""'3F723599278E22E4692CE7D7D5F9A12F'""" + +example_mode_aes_192_cbc_datatype_Decimal32_iv_None_aad_None = r"""'2420D49DBAA5CEF7D853C98DA1BD33BF'""" + +example_mode_aes_192_cbc_datatype_Decimal64_iv_None_aad_None = r"""'FDF594113FCC2776653ED109A51FADF1'""" + +example_mode_aes_192_cbc_datatype_Decimal128_iv_None_aad_None = r"""'79207931793E374FB5A3A2AC1ECA857A583603B3047000A843425EECA4C35311'""" + +example_mode_aes_192_cbc_datatype_UUID_iv_None_aad_None = r"""'9FDB738E78D0D2F774C484ED82A854E46B580C61DBE08478DC523DA6AD605078'""" + +example_mode_aes_192_cbc_datatype_Date_iv_None_aad_None = r"""'2CDD4685168FA3E2A7FA2092E86F44D4'""" + +example_mode_aes_192_cbc_datatype_DateTime_iv_None_aad_None = r"""'A4BEE097872E44FAD94D6707D6643DF5'""" + +example_mode_aes_192_cbc_datatype_DateTime64_iv_None_aad_None = r"""'1798B23C09F783623943560DF142E0F3'""" + +example_mode_aes_192_cbc_datatype_LowCardinality_iv_None_aad_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_cbc_datatype_Array_iv_None_aad_None = r"""'7C0B9021CAF2CBBB06DBF589740DCC65'""" + +example_mode_aes_192_cbc_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_192_cbc_datatype_IPv4_iv_None_aad_None = r"""'B20465C932A0719BA04E2F76371510D8'""" + +example_mode_aes_192_cbc_datatype_IPv6_iv_None_aad_None = r"""'CCCDC9B9C3F182254591DFEDDCE9F2326879326F3973401A6293A92BCB8EDFC4'""" + +example_mode_aes_192_cbc_datatype_Enum8_iv_None_aad_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_cbc_datatype_Enum16_iv_None_aad_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_256_cbc_datatype_String_iv_None_aad_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_cbc_datatype_FixedString_iv_None_aad_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_cbc_datatype_UInt8_iv_None_aad_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_cbc_datatype_UInt16_iv_None_aad_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_cbc_datatype_UInt32_iv_None_aad_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_cbc_datatype_UInt64_iv_None_aad_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_cbc_datatype_Int8_iv_None_aad_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_cbc_datatype_Int16_iv_None_aad_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_cbc_datatype_Int32_iv_None_aad_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_cbc_datatype_Int64_iv_None_aad_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_cbc_datatype_Float32_iv_None_aad_None = r"""'A11ED1B75CF1C04C6CA3A31E76627D4C'""" + +example_mode_aes_256_cbc_datatype_Float64_iv_None_aad_None = r"""'464C85EB7DB36D95CF48A3431CC7B2BC'""" + +example_mode_aes_256_cbc_datatype_Decimal32_iv_None_aad_None = r"""'988C793BD81036C1D05EC47F43851269'""" + +example_mode_aes_256_cbc_datatype_Decimal64_iv_None_aad_None = r"""'50FFB9C104DBFF3F415F12BA73D6FF1C'""" + +example_mode_aes_256_cbc_datatype_Decimal128_iv_None_aad_None = r"""'B04C40C085A262E3AA27F8E7F6831DCB36585C228B0286E7A8D8DBAF754C4C38'""" + +example_mode_aes_256_cbc_datatype_UUID_iv_None_aad_None = r"""'6A36D74ACB38B95FA77BC757A7AB2C3428548E6132D69A22B320775A21ABA11F'""" + +example_mode_aes_256_cbc_datatype_Date_iv_None_aad_None = r"""'F1CFA361A9B08FC101F3A4707A3E04D2'""" + +example_mode_aes_256_cbc_datatype_DateTime_iv_None_aad_None = r"""'D58178485CD1AE1C30F68383307B8BC5'""" + +example_mode_aes_256_cbc_datatype_DateTime64_iv_None_aad_None = r"""'A19B65BCB740B2AF4D421CE1DEC43608'""" + +example_mode_aes_256_cbc_datatype_LowCardinality_iv_None_aad_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_cbc_datatype_Array_iv_None_aad_None = r"""'C4071E4FD44F004347EA9932326B7038'""" + +example_mode_aes_256_cbc_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_256_cbc_datatype_IPv4_iv_None_aad_None = r"""'6C7950041CB4041D4D8036FCD22E3B06'""" + +example_mode_aes_256_cbc_datatype_IPv6_iv_None_aad_None = r"""'8CBF2DC164F4086B8DD14B75E3065621393DE8421BAA5AE5E87096AEA7087507'""" + +example_mode_aes_256_cbc_datatype_Enum8_iv_None_aad_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_cbc_datatype_Enum16_iv_None_aad_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_128_cbc_datatype_String_iv_16_aad_None = r"""'D017D171B3865D6EA347E14167261F41'""" + +example_mode_aes_128_cbc_datatype_FixedString_iv_16_aad_None = r"""'D017D171B3865D6EA347E14167261F41'""" + +example_mode_aes_128_cbc_datatype_UInt8_iv_16_aad_None = r"""'A5BD67663C14A01DC9AB3B5F7B0F9383'""" + +example_mode_aes_128_cbc_datatype_UInt16_iv_16_aad_None = r"""'02D98283BEADCA1AC6EF925F9BF86960'""" + +example_mode_aes_128_cbc_datatype_UInt32_iv_16_aad_None = r"""'E72BD2245C3B2B7474300D09DBD85F3F'""" + +example_mode_aes_128_cbc_datatype_UInt64_iv_16_aad_None = r"""'C9032C59328DEA2EE03ACDBEDFAE7475'""" + +example_mode_aes_128_cbc_datatype_Int8_iv_16_aad_None = r"""'A5BD67663C14A01DC9AB3B5F7B0F9383'""" + +example_mode_aes_128_cbc_datatype_Int16_iv_16_aad_None = r"""'02D98283BEADCA1AC6EF925F9BF86960'""" + +example_mode_aes_128_cbc_datatype_Int32_iv_16_aad_None = r"""'E72BD2245C3B2B7474300D09DBD85F3F'""" + +example_mode_aes_128_cbc_datatype_Int64_iv_16_aad_None = r"""'C9032C59328DEA2EE03ACDBEDFAE7475'""" + +example_mode_aes_128_cbc_datatype_Float32_iv_16_aad_None = r"""'A5425BDEB6B83E311C45249DAF3153F5'""" + +example_mode_aes_128_cbc_datatype_Float64_iv_16_aad_None = r"""'EEDA98EC4045C7D351F3905313073B79'""" + +example_mode_aes_128_cbc_datatype_Decimal32_iv_16_aad_None = r"""'52EBB74292ECD37A29E9809166CC77DB'""" + +example_mode_aes_128_cbc_datatype_Decimal64_iv_16_aad_None = r"""'95EF455767EC8FBD32BAAEFFB44FEEB7'""" + +example_mode_aes_128_cbc_datatype_Decimal128_iv_16_aad_None = r"""'94C066884FA09B0D3C750F20A2823304A2FE20B6B69AB18373E3F58623E0D7FB'""" + +example_mode_aes_128_cbc_datatype_UUID_iv_16_aad_None = r"""'1D909C15BB882E89AD68B1EFEAC72148DCD05E2303B6BE19007A945AFB778B42'""" + +example_mode_aes_128_cbc_datatype_Date_iv_16_aad_None = r"""'24A4F8CE8A9FAE48A0AFEB8A6203EFEA'""" + +example_mode_aes_128_cbc_datatype_DateTime_iv_16_aad_None = r"""'0DD5554819E3995B1B6B00362AEE9424'""" + +example_mode_aes_128_cbc_datatype_DateTime64_iv_16_aad_None = r"""'0E55319903957C9D1FDA4FB65C3871CB'""" + +example_mode_aes_128_cbc_datatype_LowCardinality_iv_16_aad_None = r"""'D017D171B3865D6EA347E14167261F41'""" + +example_mode_aes_128_cbc_datatype_Array_iv_16_aad_None = r"""'D53C82A5D13256B88DF41C1C1D924E40'""" + +example_mode_aes_128_cbc_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_128_cbc_datatype_IPv4_iv_16_aad_None = r"""'C0D81AAB3134EAB5B1F190958C6A29F9'""" + +example_mode_aes_128_cbc_datatype_IPv6_iv_16_aad_None = r"""'AE1A36F75C9BB387121445069A9968CA247FA4459ED3C8809089FEE334EB1EC7'""" + +example_mode_aes_128_cbc_datatype_Enum8_iv_16_aad_None = r"""'A5BD67663C14A01DC9AB3B5F7B0F9383'""" + +example_mode_aes_128_cbc_datatype_Enum16_iv_16_aad_None = r"""'02D98283BEADCA1AC6EF925F9BF86960'""" + +example_mode_aes_192_cbc_datatype_String_iv_16_aad_None = r"""'A3DB45D129A5C9FDB5ED66E782B28BD2'""" + +example_mode_aes_192_cbc_datatype_FixedString_iv_16_aad_None = r"""'A3DB45D129A5C9FDB5ED66E782B28BD2'""" + +example_mode_aes_192_cbc_datatype_UInt8_iv_16_aad_None = r"""'F2A751470B32C58822F23B1417C11279'""" + +example_mode_aes_192_cbc_datatype_UInt16_iv_16_aad_None = r"""'CA1ECFEA89CF520D8FA14A38235E5FA5'""" + +example_mode_aes_192_cbc_datatype_UInt32_iv_16_aad_None = r"""'57F211370522621F23B59C8304878904'""" + +example_mode_aes_192_cbc_datatype_UInt64_iv_16_aad_None = r"""'DCF974CD88752B215284625F9164F5D4'""" + +example_mode_aes_192_cbc_datatype_Int8_iv_16_aad_None = r"""'F2A751470B32C58822F23B1417C11279'""" + +example_mode_aes_192_cbc_datatype_Int16_iv_16_aad_None = r"""'CA1ECFEA89CF520D8FA14A38235E5FA5'""" + +example_mode_aes_192_cbc_datatype_Int32_iv_16_aad_None = r"""'57F211370522621F23B59C8304878904'""" + +example_mode_aes_192_cbc_datatype_Int64_iv_16_aad_None = r"""'DCF974CD88752B215284625F9164F5D4'""" + +example_mode_aes_192_cbc_datatype_Float32_iv_16_aad_None = r"""'62EBE4FD1035D405BBD6C41436780E13'""" + +example_mode_aes_192_cbc_datatype_Float64_iv_16_aad_None = r"""'5706FC9892A4C1AB48FC93E13C9C72FE'""" + +example_mode_aes_192_cbc_datatype_Decimal32_iv_16_aad_None = r"""'BB056843D369A5E55982C92AD52EEC07'""" + +example_mode_aes_192_cbc_datatype_Decimal64_iv_16_aad_None = r"""'70ACD4156F9AC1444A75EFCB9202CA00'""" + +example_mode_aes_192_cbc_datatype_Decimal128_iv_16_aad_None = r"""'04748A45840A0CAAC83F139DB01C504B01FC56631A8B2FFBE68F2FC85B6FEEDE'""" + +example_mode_aes_192_cbc_datatype_UUID_iv_16_aad_None = r"""'D7B2ABC08F67823F61C3E8F680C12B3A8AA3E3711D412CB55ACFBC89C14949A8'""" + +example_mode_aes_192_cbc_datatype_Date_iv_16_aad_None = r"""'734BBE526E56B280E90E53DDEA7DB69B'""" + +example_mode_aes_192_cbc_datatype_DateTime_iv_16_aad_None = r"""'9B9BE7CC20F75DA3F39F688DE3A1ADAA'""" + +example_mode_aes_192_cbc_datatype_DateTime64_iv_16_aad_None = r"""'554FCAAF985378A561F7C6ED91E20C89'""" + +example_mode_aes_192_cbc_datatype_LowCardinality_iv_16_aad_None = r"""'A3DB45D129A5C9FDB5ED66E782B28BD2'""" + +example_mode_aes_192_cbc_datatype_Array_iv_16_aad_None = r"""'D85AF1078F110329896EFC462340171E'""" + +example_mode_aes_192_cbc_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_192_cbc_datatype_IPv4_iv_16_aad_None = r"""'6AF45078B1E924B6C107D4C0236EA937'""" + +example_mode_aes_192_cbc_datatype_IPv6_iv_16_aad_None = r"""'9E4F8E54B265A340090DC7FE4F53BB50048442F5632A7B1630AE80DFD938E9AA'""" + +example_mode_aes_192_cbc_datatype_Enum8_iv_16_aad_None = r"""'F2A751470B32C58822F23B1417C11279'""" + +example_mode_aes_192_cbc_datatype_Enum16_iv_16_aad_None = r"""'CA1ECFEA89CF520D8FA14A38235E5FA5'""" + +example_mode_aes_256_cbc_datatype_String_iv_16_aad_None = r"""'5E22454D9AC4F1A47B04E2FD98A76140'""" + +example_mode_aes_256_cbc_datatype_FixedString_iv_16_aad_None = r"""'5E22454D9AC4F1A47B04E2FD98A76140'""" + +example_mode_aes_256_cbc_datatype_UInt8_iv_16_aad_None = r"""'FE35EEF14D6AA67AA2EBA474253CA19A'""" + +example_mode_aes_256_cbc_datatype_UInt16_iv_16_aad_None = r"""'2D22C6B58140E591BEF7986C7770FF21'""" + +example_mode_aes_256_cbc_datatype_UInt32_iv_16_aad_None = r"""'4EB4923E19AA24206B135D5B25CB31AB'""" + +example_mode_aes_256_cbc_datatype_UInt64_iv_16_aad_None = r"""'173B7CAFFCBED9B814C0ECD50A9477F6'""" + +example_mode_aes_256_cbc_datatype_Int8_iv_16_aad_None = r"""'FE35EEF14D6AA67AA2EBA474253CA19A'""" + +example_mode_aes_256_cbc_datatype_Int16_iv_16_aad_None = r"""'2D22C6B58140E591BEF7986C7770FF21'""" + +example_mode_aes_256_cbc_datatype_Int32_iv_16_aad_None = r"""'4EB4923E19AA24206B135D5B25CB31AB'""" + +example_mode_aes_256_cbc_datatype_Int64_iv_16_aad_None = r"""'173B7CAFFCBED9B814C0ECD50A9477F6'""" + +example_mode_aes_256_cbc_datatype_Float32_iv_16_aad_None = r"""'E639AA3E45D8C2759181FD736CD58EDC'""" + +example_mode_aes_256_cbc_datatype_Float64_iv_16_aad_None = r"""'CFEF3FDC054997559DF5DCFB5F215B58'""" + +example_mode_aes_256_cbc_datatype_Decimal32_iv_16_aad_None = r"""'E2F57A092A1759D39F4AE67C9543FAB8'""" + +example_mode_aes_256_cbc_datatype_Decimal64_iv_16_aad_None = r"""'6259A2CFD3D83352A44C03DB050077B3'""" + +example_mode_aes_256_cbc_datatype_Decimal128_iv_16_aad_None = r"""'AEC71CA2D87098392689F9EB2ED93A84FA5787E643E28CB3C2013F8FCC24E387'""" + +example_mode_aes_256_cbc_datatype_UUID_iv_16_aad_None = r"""'88BA86B14A468DC92084B7152B172E142D88CBFB639A8FF2F480F1727972251C'""" + +example_mode_aes_256_cbc_datatype_Date_iv_16_aad_None = r"""'C67C84B1C6BF4527A7E730499FF39C86'""" + +example_mode_aes_256_cbc_datatype_DateTime_iv_16_aad_None = r"""'7FDC1B0797A5F3C04CDA82729A1EA4AA'""" + +example_mode_aes_256_cbc_datatype_DateTime64_iv_16_aad_None = r"""'B1B7401FB2B65BCB3448C1BE179F6AA6'""" + +example_mode_aes_256_cbc_datatype_LowCardinality_iv_16_aad_None = r"""'5E22454D9AC4F1A47B04E2FD98A76140'""" + +example_mode_aes_256_cbc_datatype_Array_iv_16_aad_None = r"""'6BB1E8429CC612B0AA74282B81D4FE8A'""" + +example_mode_aes_256_cbc_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_256_cbc_datatype_IPv4_iv_16_aad_None = r"""'51364C8DC6882CA1F03CF7FB45117EEF'""" + +example_mode_aes_256_cbc_datatype_IPv6_iv_16_aad_None = r"""'87A1C4D4672EFE64DC98E040EAD6B3126C899C263577B3D8EE8A3952BE5CDC1B'""" + +example_mode_aes_256_cbc_datatype_Enum8_iv_16_aad_None = r"""'FE35EEF14D6AA67AA2EBA474253CA19A'""" + +example_mode_aes_256_cbc_datatype_Enum16_iv_16_aad_None = r"""'2D22C6B58140E591BEF7986C7770FF21'""" + +example_mode_aes_128_cfb1_datatype_String_iv_None_aad_None = r"""'32'""" + +example_mode_aes_128_cfb1_datatype_FixedString_iv_None_aad_None = r"""'32'""" + +example_mode_aes_128_cfb1_datatype_UInt8_iv_None_aad_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_UInt16_iv_None_aad_None = r"""'0173'""" + +example_mode_aes_128_cfb1_datatype_UInt32_iv_None_aad_None = r"""'01732E6B'""" + +example_mode_aes_128_cfb1_datatype_UInt64_iv_None_aad_None = r"""'01732E6B82FCBDF6'""" + +example_mode_aes_128_cfb1_datatype_Int8_iv_None_aad_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Int16_iv_None_aad_None = r"""'0173'""" + +example_mode_aes_128_cfb1_datatype_Int32_iv_None_aad_None = r"""'01732E6B'""" + +example_mode_aes_128_cfb1_datatype_Int64_iv_None_aad_None = r"""'01732E6B82FCBDF6'""" + +example_mode_aes_128_cfb1_datatype_Float32_iv_None_aad_None = r"""'0000B9AB'""" + +example_mode_aes_128_cfb1_datatype_Float64_iv_None_aad_None = r"""'000000000000FFF6'""" + +example_mode_aes_128_cfb1_datatype_Decimal32_iv_None_aad_None = r"""'2E09CA6A'""" + +example_mode_aes_128_cfb1_datatype_Decimal64_iv_None_aad_None = r"""'2E09CA6A6DBEE799'""" + +example_mode_aes_128_cfb1_datatype_Decimal128_iv_None_aad_None = r"""'2E09CA6A6DBEE79923BA65C6B78FD199'""" + +example_mode_aes_128_cfb1_datatype_UUID_iv_None_aad_None = r"""'E590DFB515D3A518F85C66A6A5EC9C6E'""" + +example_mode_aes_128_cfb1_datatype_Date_iv_None_aad_None = r"""'42F0'""" + +example_mode_aes_128_cfb1_datatype_DateTime_iv_None_aad_None = r"""'5475EC3D'""" + +example_mode_aes_128_cfb1_datatype_DateTime64_iv_None_aad_None = r"""'21CDF1128AE44A37'""" + +example_mode_aes_128_cfb1_datatype_LowCardinality_iv_None_aad_None = r"""'32'""" + +example_mode_aes_128_cfb1_datatype_Array_iv_None_aad_None = r"""'0170'""" + +example_mode_aes_128_cfb1_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_128_cfb1_datatype_IPv4_iv_None_aad_None = r"""'240A9E43'""" + +example_mode_aes_128_cfb1_datatype_IPv6_iv_None_aad_None = r"""'2E642EF4B07D9B1251BE3B3CBDBCC6F6'""" + +example_mode_aes_128_cfb1_datatype_Enum8_iv_None_aad_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Enum16_iv_None_aad_None = r"""'0173'""" + +example_mode_aes_192_cfb1_datatype_String_iv_None_aad_None = r"""'23'""" + +example_mode_aes_192_cfb1_datatype_FixedString_iv_None_aad_None = r"""'23'""" + +example_mode_aes_192_cfb1_datatype_UInt8_iv_None_aad_None = r"""'01'""" + +example_mode_aes_192_cfb1_datatype_UInt16_iv_None_aad_None = r"""'01F9'""" + +example_mode_aes_192_cfb1_datatype_UInt32_iv_None_aad_None = r"""'01F92AD3'""" + +example_mode_aes_192_cfb1_datatype_UInt64_iv_None_aad_None = r"""'01F92AD38CB10028'""" + +example_mode_aes_192_cfb1_datatype_Int8_iv_None_aad_None = r"""'01'""" + +example_mode_aes_192_cfb1_datatype_Int16_iv_None_aad_None = r"""'01F9'""" + +example_mode_aes_192_cfb1_datatype_Int32_iv_None_aad_None = r"""'01F92AD3'""" + +example_mode_aes_192_cfb1_datatype_Int64_iv_None_aad_None = r"""'01F92AD38CB10028'""" + +example_mode_aes_192_cfb1_datatype_Float32_iv_None_aad_None = r"""'0000FCAE'""" + +example_mode_aes_192_cfb1_datatype_Float64_iv_None_aad_None = r"""'000000000000A79C'""" + +example_mode_aes_192_cfb1_datatype_Decimal32_iv_None_aad_None = r"""'3F406C3F'""" + +example_mode_aes_192_cfb1_datatype_Decimal64_iv_None_aad_None = r"""'3F406C3F3A41B134'""" + +example_mode_aes_192_cfb1_datatype_Decimal128_iv_None_aad_None = r"""'3F406C3F3A41B134310D6B68BEBC5708'""" + +example_mode_aes_192_cfb1_datatype_UUID_iv_None_aad_None = r"""'B7F80F1BDCA1C4193E5AB11078FEA213'""" + +example_mode_aes_192_cfb1_datatype_Date_iv_None_aad_None = r"""'6FF6'""" + +example_mode_aes_192_cfb1_datatype_DateTime_iv_None_aad_None = r"""'7013E555'""" + +example_mode_aes_192_cfb1_datatype_DateTime64_iv_None_aad_None = r"""'371AF0291536F5B7'""" + +example_mode_aes_192_cfb1_datatype_LowCardinality_iv_None_aad_None = r"""'23'""" + +example_mode_aes_192_cfb1_datatype_Array_iv_None_aad_None = r"""'01FA'""" + +example_mode_aes_192_cfb1_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_192_cfb1_datatype_IPv4_iv_None_aad_None = r"""'33895F70'""" + +example_mode_aes_192_cfb1_datatype_IPv6_iv_None_aad_None = r"""'3F24552946522B931290F904186B055A'""" + +example_mode_aes_192_cfb1_datatype_Enum8_iv_None_aad_None = r"""'01'""" + +example_mode_aes_192_cfb1_datatype_Enum16_iv_None_aad_None = r"""'01F9'""" + +example_mode_aes_256_cfb1_datatype_String_iv_None_aad_None = r"""'9E'""" + +example_mode_aes_256_cfb1_datatype_FixedString_iv_None_aad_None = r"""'9E'""" + +example_mode_aes_256_cfb1_datatype_UInt8_iv_None_aad_None = r"""'B9'""" + +example_mode_aes_256_cfb1_datatype_UInt16_iv_None_aad_None = r"""'B9ED'""" + +example_mode_aes_256_cfb1_datatype_UInt32_iv_None_aad_None = r"""'B9ED4764'""" + +example_mode_aes_256_cfb1_datatype_UInt64_iv_None_aad_None = r"""'B9ED4764E7BF3C1C'""" + +example_mode_aes_256_cfb1_datatype_Int8_iv_None_aad_None = r"""'B9'""" + +example_mode_aes_256_cfb1_datatype_Int16_iv_None_aad_None = r"""'B9ED'""" + +example_mode_aes_256_cfb1_datatype_Int32_iv_None_aad_None = r"""'B9ED4764'""" + +example_mode_aes_256_cfb1_datatype_Int64_iv_None_aad_None = r"""'B9ED4764E7BF3C1C'""" + +example_mode_aes_256_cfb1_datatype_Float32_iv_None_aad_None = r"""'B85F0E63'""" + +example_mode_aes_256_cfb1_datatype_Float64_iv_None_aad_None = r"""'B85FDB5A8FE0C0BB'""" + +example_mode_aes_256_cfb1_datatype_Decimal32_iv_None_aad_None = r"""'891B85B3'""" + +example_mode_aes_256_cfb1_datatype_Decimal64_iv_None_aad_None = r"""'891B85B3C1BA6EE1'""" + +example_mode_aes_256_cfb1_datatype_Decimal128_iv_None_aad_None = r"""'891B85B3C1BA6EE137EF658F618D1F3F'""" + +example_mode_aes_256_cfb1_datatype_UUID_iv_None_aad_None = r"""'121B5EE9929417BC1CDBDB390BC93B4A'""" + +example_mode_aes_256_cfb1_datatype_Date_iv_None_aad_None = r"""'D40F'""" + +example_mode_aes_256_cfb1_datatype_DateTime_iv_None_aad_None = r"""'CF27297C'""" + +example_mode_aes_256_cfb1_datatype_DateTime64_iv_None_aad_None = r"""'8773F350CD394D36'""" + +example_mode_aes_256_cfb1_datatype_LowCardinality_iv_None_aad_None = r"""'9E'""" + +example_mode_aes_256_cfb1_datatype_Array_iv_None_aad_None = r"""'B9EE'""" + +example_mode_aes_256_cfb1_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_256_cfb1_datatype_IPv4_iv_None_aad_None = r"""'8383FD3C'""" + +example_mode_aes_256_cfb1_datatype_IPv6_iv_None_aad_None = r"""'897A84A02FD451D3DDB92FF290BF9B7C'""" + +example_mode_aes_256_cfb1_datatype_Enum8_iv_None_aad_None = r"""'B9'""" + +example_mode_aes_256_cfb1_datatype_Enum16_iv_None_aad_None = r"""'B9ED'""" + +example_mode_aes_128_cfb1_datatype_String_iv_16_aad_None = r"""'37'""" + +example_mode_aes_128_cfb1_datatype_FixedString_iv_16_aad_None = r"""'37'""" + +example_mode_aes_128_cfb1_datatype_UInt8_iv_16_aad_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_UInt16_iv_16_aad_None = r"""'0188'""" + +example_mode_aes_128_cfb1_datatype_UInt32_iv_16_aad_None = r"""'01882D46'""" + +example_mode_aes_128_cfb1_datatype_UInt64_iv_16_aad_None = r"""'01882D46FCCCD695'""" + +example_mode_aes_128_cfb1_datatype_Int8_iv_16_aad_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Int16_iv_16_aad_None = r"""'0188'""" + +example_mode_aes_128_cfb1_datatype_Int32_iv_16_aad_None = r"""'01882D46'""" + +example_mode_aes_128_cfb1_datatype_Int64_iv_16_aad_None = r"""'01882D46FCCCD695'""" + +example_mode_aes_128_cfb1_datatype_Float32_iv_16_aad_None = r"""'00B931F2'""" + +example_mode_aes_128_cfb1_datatype_Float64_iv_16_aad_None = r"""'00B99AAE199C3C93'""" + +example_mode_aes_128_cfb1_datatype_Decimal32_iv_16_aad_None = r"""'2D557511'""" + +example_mode_aes_128_cfb1_datatype_Decimal64_iv_16_aad_None = r"""'2D557511511F90FB'""" + +example_mode_aes_128_cfb1_datatype_Decimal128_iv_16_aad_None = r"""'2D557511511F90FBC464352E8A02FC51'""" + +example_mode_aes_128_cfb1_datatype_UUID_iv_16_aad_None = r"""'8AE269086C72AD682EB92ABA6CA58E49'""" + +example_mode_aes_128_cfb1_datatype_Date_iv_16_aad_None = r"""'5FC9'""" + +example_mode_aes_128_cfb1_datatype_DateTime_iv_16_aad_None = r"""'42970865'""" + +example_mode_aes_128_cfb1_datatype_DateTime64_iv_16_aad_None = r"""'20B310A2F7EF8460'""" + +example_mode_aes_128_cfb1_datatype_LowCardinality_iv_16_aad_None = r"""'37'""" + +example_mode_aes_128_cfb1_datatype_Array_iv_16_aad_None = r"""'018A'""" + +example_mode_aes_128_cfb1_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_128_cfb1_datatype_IPv4_iv_16_aad_None = r"""'27476DAF'""" + +example_mode_aes_128_cfb1_datatype_IPv6_iv_16_aad_None = r"""'2D311FBDC0A5C652AAD863398F94C5C3'""" + +example_mode_aes_128_cfb1_datatype_Enum8_iv_16_aad_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Enum16_iv_16_aad_None = r"""'0188'""" + +example_mode_aes_192_cfb1_datatype_String_iv_16_aad_None = r"""'38'""" + +example_mode_aes_192_cfb1_datatype_FixedString_iv_16_aad_None = r"""'38'""" + +example_mode_aes_192_cfb1_datatype_UInt8_iv_16_aad_None = r"""'06'""" + +example_mode_aes_192_cfb1_datatype_UInt16_iv_16_aad_None = r"""'069E'""" + +example_mode_aes_192_cfb1_datatype_UInt32_iv_16_aad_None = r"""'069E2E37'""" + +example_mode_aes_192_cfb1_datatype_UInt64_iv_16_aad_None = r"""'069E2E370A6D9872'""" + +example_mode_aes_192_cfb1_datatype_Int8_iv_16_aad_None = r"""'06'""" + +example_mode_aes_192_cfb1_datatype_Int16_iv_16_aad_None = r"""'069E'""" + +example_mode_aes_192_cfb1_datatype_Int32_iv_16_aad_None = r"""'069E2E37'""" + +example_mode_aes_192_cfb1_datatype_Int64_iv_16_aad_None = r"""'069E2E370A6D9872'""" + +example_mode_aes_192_cfb1_datatype_Float32_iv_16_aad_None = r"""'07955BCF'""" + +example_mode_aes_192_cfb1_datatype_Float64_iv_16_aad_None = r"""'0795A57CA222A36E'""" + +example_mode_aes_192_cfb1_datatype_Decimal32_iv_16_aad_None = r"""'2A15BB86'""" + +example_mode_aes_192_cfb1_datatype_Decimal64_iv_16_aad_None = r"""'2A15BB86FB961E7D'""" + +example_mode_aes_192_cfb1_datatype_Decimal128_iv_16_aad_None = r"""'2A15BB86FB961E7D0DD5055987176AF4'""" + +example_mode_aes_192_cfb1_datatype_UUID_iv_16_aad_None = r"""'DA2338793C7B9E0F6722E272062F5EA1'""" + +example_mode_aes_192_cfb1_datatype_Date_iv_16_aad_None = r"""'4AAB'""" + +example_mode_aes_192_cfb1_datatype_DateTime_iv_16_aad_None = r"""'5B6A8EE6'""" + +example_mode_aes_192_cfb1_datatype_DateTime64_iv_16_aad_None = r"""'23C4E2A707F73EF4'""" + +example_mode_aes_192_cfb1_datatype_LowCardinality_iv_16_aad_None = r"""'38'""" + +example_mode_aes_192_cfb1_datatype_Array_iv_16_aad_None = r"""'069C'""" + +example_mode_aes_192_cfb1_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_192_cfb1_datatype_IPv4_iv_16_aad_None = r"""'2470A839'""" + +example_mode_aes_192_cfb1_datatype_IPv6_iv_16_aad_None = r"""'2A712A746781131B2DC4EB92E31C72FA'""" + +example_mode_aes_192_cfb1_datatype_Enum8_iv_16_aad_None = r"""'06'""" + +example_mode_aes_192_cfb1_datatype_Enum16_iv_16_aad_None = r"""'069E'""" + +example_mode_aes_256_cfb1_datatype_String_iv_16_aad_None = r"""'5A'""" + +example_mode_aes_256_cfb1_datatype_FixedString_iv_16_aad_None = r"""'5A'""" + +example_mode_aes_256_cfb1_datatype_UInt8_iv_16_aad_None = r"""'7E'""" + +example_mode_aes_256_cfb1_datatype_UInt16_iv_16_aad_None = r"""'7EA1'""" + +example_mode_aes_256_cfb1_datatype_UInt32_iv_16_aad_None = r"""'7EA17214'""" + +example_mode_aes_256_cfb1_datatype_UInt64_iv_16_aad_None = r"""'7EA172144C6F5578'""" + +example_mode_aes_256_cfb1_datatype_Int8_iv_16_aad_None = r"""'7E'""" + +example_mode_aes_256_cfb1_datatype_Int16_iv_16_aad_None = r"""'7EA1'""" + +example_mode_aes_256_cfb1_datatype_Int32_iv_16_aad_None = r"""'7EA17214'""" + +example_mode_aes_256_cfb1_datatype_Int64_iv_16_aad_None = r"""'7EA172144C6F5578'""" + +example_mode_aes_256_cfb1_datatype_Float32_iv_16_aad_None = r"""'7F630BBA'""" + +example_mode_aes_256_cfb1_datatype_Float64_iv_16_aad_None = r"""'7F638DFAAA434E6B'""" + +example_mode_aes_256_cfb1_datatype_Decimal32_iv_16_aad_None = r"""'4F430FBA'""" + +example_mode_aes_256_cfb1_datatype_Decimal64_iv_16_aad_None = r"""'4F430FBAA3AAF884'""" + +example_mode_aes_256_cfb1_datatype_Decimal128_iv_16_aad_None = r"""'4F430FBAA3AAF8845DB7BBA7F98F49C4'""" + +example_mode_aes_256_cfb1_datatype_UUID_iv_16_aad_None = r"""'B06F4A8C3BF3A8D32D113D0D40397C8F'""" + +example_mode_aes_256_cfb1_datatype_Date_iv_16_aad_None = r"""'30CE'""" + +example_mode_aes_256_cfb1_datatype_DateTime_iv_16_aad_None = r"""'206545FA'""" + +example_mode_aes_256_cfb1_datatype_DateTime64_iv_16_aad_None = r"""'43756F28C68E3D55'""" + +example_mode_aes_256_cfb1_datatype_LowCardinality_iv_16_aad_None = r"""'5A'""" + +example_mode_aes_256_cfb1_datatype_Array_iv_16_aad_None = r"""'7EA3'""" + +example_mode_aes_256_cfb1_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_256_cfb1_datatype_IPv4_iv_16_aad_None = r"""'4526FCCF'""" + +example_mode_aes_256_cfb1_datatype_IPv6_iv_16_aad_None = r"""'4F23BDAC741DB8767CE6AE24888545A2'""" + +example_mode_aes_256_cfb1_datatype_Enum8_iv_16_aad_None = r"""'7E'""" + +example_mode_aes_256_cfb1_datatype_Enum16_iv_16_aad_None = r"""'7EA1'""" + +example_mode_aes_128_cfb8_datatype_String_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_cfb8_datatype_FixedString_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_cfb8_datatype_UInt8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_cfb8_datatype_UInt16_iv_None_aad_None = r"""'11FF'""" + +example_mode_aes_128_cfb8_datatype_UInt32_iv_None_aad_None = r"""'11FF20C0'""" + +example_mode_aes_128_cfb8_datatype_UInt64_iv_None_aad_None = r"""'11FF20C07A65C524'""" + +example_mode_aes_128_cfb8_datatype_Int8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_cfb8_datatype_Int16_iv_None_aad_None = r"""'11FF'""" + +example_mode_aes_128_cfb8_datatype_Int32_iv_None_aad_None = r"""'11FF20C0'""" + +example_mode_aes_128_cfb8_datatype_Int64_iv_None_aad_None = r"""'11FF20C07A65C524'""" + +example_mode_aes_128_cfb8_datatype_Float32_iv_None_aad_None = r"""'10671940'""" + +example_mode_aes_128_cfb8_datatype_Float64_iv_None_aad_None = r"""'106799607DBF56DA'""" + +example_mode_aes_128_cfb8_datatype_Decimal32_iv_None_aad_None = r"""'30756C94'""" + +example_mode_aes_128_cfb8_datatype_Decimal64_iv_None_aad_None = r"""'30756C9417D3C023'""" + +example_mode_aes_128_cfb8_datatype_Decimal128_iv_None_aad_None = r"""'30756C9417D3C023705550B7BEF872FF'""" + +example_mode_aes_128_cfb8_datatype_UUID_iv_None_aad_None = r"""'F7FE50CF0647659CB0D401B5C0E259D3'""" + +example_mode_aes_128_cfb8_datatype_Date_iv_None_aad_None = r"""'46EA'""" + +example_mode_aes_128_cfb8_datatype_DateTime_iv_None_aad_None = r"""'5EB4905E'""" + +example_mode_aes_128_cfb8_datatype_DateTime64_iv_None_aad_None = r"""'3BB70F8E64D7C6A7'""" + +example_mode_aes_128_cfb8_datatype_LowCardinality_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_cfb8_datatype_Array_iv_None_aad_None = r"""'11FD'""" + +example_mode_aes_128_cfb8_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_128_cfb8_datatype_IPv4_iv_None_aad_None = r"""'3DC2BE9E'""" + +example_mode_aes_128_cfb8_datatype_IPv6_iv_None_aad_None = r"""'303ABAC6F4F380D9F077DFC79C82D1A1'""" + +example_mode_aes_128_cfb8_datatype_Enum8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_cfb8_datatype_Enum16_iv_None_aad_None = r"""'11FF'""" + +example_mode_aes_192_cfb8_datatype_String_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_cfb8_datatype_FixedString_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_cfb8_datatype_UInt8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_cfb8_datatype_UInt16_iv_None_aad_None = r"""'0683'""" + +example_mode_aes_192_cfb8_datatype_UInt32_iv_None_aad_None = r"""'0683139D'""" + +example_mode_aes_192_cfb8_datatype_UInt64_iv_None_aad_None = r"""'0683139D83E2EFAC'""" + +example_mode_aes_192_cfb8_datatype_Int8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_cfb8_datatype_Int16_iv_None_aad_None = r"""'0683'""" + +example_mode_aes_192_cfb8_datatype_Int32_iv_None_aad_None = r"""'0683139D'""" + +example_mode_aes_192_cfb8_datatype_Int64_iv_None_aad_None = r"""'0683139D83E2EFAC'""" + +example_mode_aes_192_cfb8_datatype_Float32_iv_None_aad_None = r"""'07EDB300'""" + +example_mode_aes_192_cfb8_datatype_Float64_iv_None_aad_None = r"""'07ED3359B91DEF3B'""" + +example_mode_aes_192_cfb8_datatype_Decimal32_iv_None_aad_None = r"""'275947FE'""" + +example_mode_aes_192_cfb8_datatype_Decimal64_iv_None_aad_None = r"""'275947FE4B3390EE'""" + +example_mode_aes_192_cfb8_datatype_Decimal128_iv_None_aad_None = r"""'275947FE4B3390EE7A2541BC8E2F58D7'""" + +example_mode_aes_192_cfb8_datatype_UUID_iv_None_aad_None = r"""'E0C082C032FB8ED756F9345E270A283B'""" + +example_mode_aes_192_cfb8_datatype_Date_iv_None_aad_None = r"""'5109'""" + +example_mode_aes_192_cfb8_datatype_DateTime_iv_None_aad_None = r"""'49713150'""" + +example_mode_aes_192_cfb8_datatype_DateTime64_iv_None_aad_None = r"""'2C10FB4FEC471EF7'""" + +example_mode_aes_192_cfb8_datatype_LowCardinality_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_cfb8_datatype_Array_iv_None_aad_None = r"""'0681'""" + +example_mode_aes_192_cfb8_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_192_cfb8_datatype_IPv4_iv_None_aad_None = r"""'2A41C8F2'""" + +example_mode_aes_192_cfb8_datatype_IPv6_iv_None_aad_None = r"""'271682C9379C5A46C68488DC33D0C278'""" + +example_mode_aes_192_cfb8_datatype_Enum8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_cfb8_datatype_Enum16_iv_None_aad_None = r"""'0683'""" + +example_mode_aes_256_cfb8_datatype_String_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_cfb8_datatype_FixedString_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_cfb8_datatype_UInt8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_cfb8_datatype_UInt16_iv_None_aad_None = r"""'B15F'""" + +example_mode_aes_256_cfb8_datatype_UInt32_iv_None_aad_None = r"""'B15FD91F'""" + +example_mode_aes_256_cfb8_datatype_UInt64_iv_None_aad_None = r"""'B15FD91F702960CB'""" + +example_mode_aes_256_cfb8_datatype_Int8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_cfb8_datatype_Int16_iv_None_aad_None = r"""'B15F'""" + +example_mode_aes_256_cfb8_datatype_Int32_iv_None_aad_None = r"""'B15FD91F'""" + +example_mode_aes_256_cfb8_datatype_Int64_iv_None_aad_None = r"""'B15FD91F702960CB'""" + +example_mode_aes_256_cfb8_datatype_Float32_iv_None_aad_None = r"""'B05A05BE'""" + +example_mode_aes_256_cfb8_datatype_Float64_iv_None_aad_None = r"""'B05A8510DB2F16A0'""" + +example_mode_aes_256_cfb8_datatype_Decimal32_iv_None_aad_None = r"""'906B5777'""" + +example_mode_aes_256_cfb8_datatype_Decimal64_iv_None_aad_None = r"""'906B57771CB81F37'""" + +example_mode_aes_256_cfb8_datatype_Decimal128_iv_None_aad_None = r"""'906B57771CB81F378D932AE788527DE2'""" + +example_mode_aes_256_cfb8_datatype_UUID_iv_None_aad_None = r"""'57FB06BA6F4BA51D7A61D65A7827A18D'""" + +example_mode_aes_256_cfb8_datatype_Date_iv_None_aad_None = r"""'E652'""" + +example_mode_aes_256_cfb8_datatype_DateTime_iv_None_aad_None = r"""'FEEEADA4'""" + +example_mode_aes_256_cfb8_datatype_DateTime64_iv_None_aad_None = r"""'9BB36DEF05FF5975'""" + +example_mode_aes_256_cfb8_datatype_LowCardinality_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_cfb8_datatype_Array_iv_None_aad_None = r"""'B15D'""" + +example_mode_aes_256_cfb8_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_256_cfb8_datatype_IPv4_iv_None_aad_None = r"""'9DC836F3'""" + +example_mode_aes_256_cfb8_datatype_IPv6_iv_None_aad_None = r"""'90242F0083C8B0221DF3B5755EC8D99C'""" + +example_mode_aes_256_cfb8_datatype_Enum8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_cfb8_datatype_Enum16_iv_None_aad_None = r"""'B15F'""" + +example_mode_aes_128_cfb8_datatype_String_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_cfb8_datatype_FixedString_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_cfb8_datatype_UInt8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_cfb8_datatype_UInt16_iv_16_aad_None = r"""'3368'""" + +example_mode_aes_128_cfb8_datatype_UInt32_iv_16_aad_None = r"""'3368AB64'""" + +example_mode_aes_128_cfb8_datatype_UInt64_iv_16_aad_None = r"""'3368AB6421744B7E'""" + +example_mode_aes_128_cfb8_datatype_Int8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_cfb8_datatype_Int16_iv_16_aad_None = r"""'3368'""" + +example_mode_aes_128_cfb8_datatype_Int32_iv_16_aad_None = r"""'3368AB64'""" + +example_mode_aes_128_cfb8_datatype_Int64_iv_16_aad_None = r"""'3368AB6421744B7E'""" + +example_mode_aes_128_cfb8_datatype_Float32_iv_16_aad_None = r"""'3232B23D'""" + +example_mode_aes_128_cfb8_datatype_Float64_iv_16_aad_None = r"""'323232323232C2A6'""" + +example_mode_aes_128_cfb8_datatype_Decimal32_iv_16_aad_None = r"""'12ABA873'""" + +example_mode_aes_128_cfb8_datatype_Decimal64_iv_16_aad_None = r"""'12ABA873E2E24473'""" + +example_mode_aes_128_cfb8_datatype_Decimal128_iv_16_aad_None = r"""'12ABA873E2E24473166434D82270A19C'""" + +example_mode_aes_128_cfb8_datatype_UUID_iv_16_aad_None = r"""'D529D970A38CCB794F856E4458D0E2D4'""" + +example_mode_aes_128_cfb8_datatype_Date_iv_16_aad_None = r"""'6445'""" + +example_mode_aes_128_cfb8_datatype_DateTime_iv_16_aad_None = r"""'7CBF2FDA'""" + +example_mode_aes_128_cfb8_datatype_DateTime64_iv_16_aad_None = r"""'191C7B5A063F562D'""" + +example_mode_aes_128_cfb8_datatype_LowCardinality_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_cfb8_datatype_Array_iv_16_aad_None = r"""'336A'""" + +example_mode_aes_128_cfb8_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_128_cfb8_datatype_IPv4_iv_16_aad_None = r"""'1F0A367A'""" + +example_mode_aes_128_cfb8_datatype_IPv6_iv_16_aad_None = r"""'12E4B19D97DC9F2C61A671C51B1201D2'""" + +example_mode_aes_128_cfb8_datatype_Enum8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_cfb8_datatype_Enum16_iv_16_aad_None = r"""'3368'""" + +example_mode_aes_192_cfb8_datatype_String_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_cfb8_datatype_FixedString_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_cfb8_datatype_UInt8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_cfb8_datatype_UInt16_iv_16_aad_None = r"""'6924'""" + +example_mode_aes_192_cfb8_datatype_UInt32_iv_16_aad_None = r"""'6924A086'""" + +example_mode_aes_192_cfb8_datatype_UInt64_iv_16_aad_None = r"""'6924A086F8F61C3C'""" + +example_mode_aes_192_cfb8_datatype_Int8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_cfb8_datatype_Int16_iv_16_aad_None = r"""'6924'""" + +example_mode_aes_192_cfb8_datatype_Int32_iv_16_aad_None = r"""'6924A086'""" + +example_mode_aes_192_cfb8_datatype_Int64_iv_16_aad_None = r"""'6924A086F8F61C3C'""" + +example_mode_aes_192_cfb8_datatype_Float32_iv_16_aad_None = r"""'6861DF7A'""" + +example_mode_aes_192_cfb8_datatype_Float64_iv_16_aad_None = r"""'68615FBC184B8D50'""" + +example_mode_aes_192_cfb8_datatype_Decimal32_iv_16_aad_None = r"""'48041B5C'""" + +example_mode_aes_192_cfb8_datatype_Decimal64_iv_16_aad_None = r"""'48041B5C6BEF70DD'""" + +example_mode_aes_192_cfb8_datatype_Decimal128_iv_16_aad_None = r"""'48041B5C6BEF70DD4CDABC1FC8C2C684'""" + +example_mode_aes_192_cfb8_datatype_UUID_iv_16_aad_None = r"""'8FF1142976A9808C0F475A3D2A34D06D'""" + +example_mode_aes_192_cfb8_datatype_Date_iv_16_aad_None = r"""'3E6D'""" + +example_mode_aes_192_cfb8_datatype_DateTime_iv_16_aad_None = r"""'269AFDC7'""" + +example_mode_aes_192_cfb8_datatype_DateTime64_iv_16_aad_None = r"""'4350703E05F43A50'""" + +example_mode_aes_192_cfb8_datatype_LowCardinality_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_cfb8_datatype_Array_iv_16_aad_None = r"""'6926'""" + +example_mode_aes_192_cfb8_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_192_cfb8_datatype_IPv4_iv_16_aad_None = r"""'45979A4C'""" + +example_mode_aes_192_cfb8_datatype_IPv6_iv_16_aad_None = r"""'484BFA49756D837181B7EE03EBCF2B79'""" + +example_mode_aes_192_cfb8_datatype_Enum8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_cfb8_datatype_Enum16_iv_16_aad_None = r"""'6924'""" + +example_mode_aes_256_cfb8_datatype_String_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_cfb8_datatype_FixedString_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_cfb8_datatype_UInt8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_cfb8_datatype_UInt16_iv_16_aad_None = r"""'682C'""" + +example_mode_aes_256_cfb8_datatype_UInt32_iv_16_aad_None = r"""'682CE0A9'""" + +example_mode_aes_256_cfb8_datatype_UInt64_iv_16_aad_None = r"""'682CE0A9FFAF55AE'""" + +example_mode_aes_256_cfb8_datatype_Int8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_cfb8_datatype_Int16_iv_16_aad_None = r"""'682C'""" + +example_mode_aes_256_cfb8_datatype_Int32_iv_16_aad_None = r"""'682CE0A9'""" + +example_mode_aes_256_cfb8_datatype_Int64_iv_16_aad_None = r"""'682CE0A9FFAF55AE'""" + +example_mode_aes_256_cfb8_datatype_Float32_iv_16_aad_None = r"""'69B127F9'""" + +example_mode_aes_256_cfb8_datatype_Float64_iv_16_aad_None = r"""'69B1A72CB81A0FFF'""" + +example_mode_aes_256_cfb8_datatype_Decimal32_iv_16_aad_None = r"""'49378750'""" + +example_mode_aes_256_cfb8_datatype_Decimal64_iv_16_aad_None = r"""'493787505DFF5606'""" + +example_mode_aes_256_cfb8_datatype_Decimal128_iv_16_aad_None = r"""'493787505DFF5606774649C631E6E0E7'""" + +example_mode_aes_256_cfb8_datatype_UUID_iv_16_aad_None = r"""'8E09A60AA21565C888B2D92942896930'""" + +example_mode_aes_256_cfb8_datatype_Date_iv_16_aad_None = r"""'3FF1'""" + +example_mode_aes_256_cfb8_datatype_DateTime_iv_16_aad_None = r"""'274E13D8'""" + +example_mode_aes_256_cfb8_datatype_DateTime64_iv_16_aad_None = r"""'4211DFF611769F37'""" + +example_mode_aes_256_cfb8_datatype_LowCardinality_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_cfb8_datatype_Array_iv_16_aad_None = r"""'682E'""" + +example_mode_aes_256_cfb8_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_256_cfb8_datatype_IPv4_iv_16_aad_None = r"""'442DB771'""" + +example_mode_aes_256_cfb8_datatype_IPv6_iv_16_aad_None = r"""'4978AF3EED91F4AD14F7C326CCD96804'""" + +example_mode_aes_256_cfb8_datatype_Enum8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_cfb8_datatype_Enum16_iv_16_aad_None = r"""'682C'""" + +example_mode_aes_128_cfb128_datatype_String_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_cfb128_datatype_FixedString_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_cfb128_datatype_UInt8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_cfb128_datatype_UInt16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_128_cfb128_datatype_UInt32_iv_None_aad_None = r"""'11DFC1B5'""" + +example_mode_aes_128_cfb128_datatype_UInt64_iv_None_aad_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_cfb128_datatype_Int8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_cfb128_datatype_Int16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_128_cfb128_datatype_Int32_iv_None_aad_None = r"""'11DFC1B5'""" + +example_mode_aes_128_cfb128_datatype_Int64_iv_None_aad_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_cfb128_datatype_Float32_iv_None_aad_None = r"""'10DF418A'""" + +example_mode_aes_128_cfb128_datatype_Float64_iv_None_aad_None = r"""'10DFC1B5F66C0D55'""" + +example_mode_aes_128_cfb128_datatype_Decimal32_iv_None_aad_None = r"""'3091C1B5'""" + +example_mode_aes_128_cfb128_datatype_Decimal64_iv_None_aad_None = r"""'3091C1B5F66CFD6A'""" + +example_mode_aes_128_cfb128_datatype_Decimal128_iv_None_aad_None = r"""'3091C1B5F66CFD6A1DC46D66907BEEB1'""" + +example_mode_aes_128_cfb128_datatype_UUID_iv_None_aad_None = r"""'F7CE72E9F2A80D0BBD1FBE0C90DD9521'""" + +example_mode_aes_128_cfb128_datatype_Date_iv_None_aad_None = r"""'4698'""" + +example_mode_aes_128_cfb128_datatype_DateTime_iv_None_aad_None = r"""'5E0FCDEB'""" + +example_mode_aes_128_cfb128_datatype_DateTime64_iv_None_aad_None = r"""'3B6ECCD7996DFD6A'""" + +example_mode_aes_128_cfb128_datatype_LowCardinality_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_cfb128_datatype_Array_iv_None_aad_None = r"""'11DD'""" + +example_mode_aes_128_cfb128_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_128_cfb128_datatype_IPv4_iv_None_aad_None = r"""'3D5D201E'""" + +example_mode_aes_128_cfb128_datatype_IPv6_iv_None_aad_None = r"""'30DECC0DF66C78C91DC46D663C646EB0'""" + +example_mode_aes_128_cfb128_datatype_Enum8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_cfb128_datatype_Enum16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_192_cfb128_datatype_String_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_cfb128_datatype_FixedString_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_cfb128_datatype_UInt8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_cfb128_datatype_UInt16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_192_cfb128_datatype_UInt32_iv_None_aad_None = r"""'06B7199D'""" + +example_mode_aes_192_cfb128_datatype_UInt64_iv_None_aad_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_cfb128_datatype_Int8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_cfb128_datatype_Int16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_192_cfb128_datatype_Int32_iv_None_aad_None = r"""'06B7199D'""" + +example_mode_aes_192_cfb128_datatype_Int64_iv_None_aad_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_cfb128_datatype_Float32_iv_None_aad_None = r"""'07B799A2'""" + +example_mode_aes_192_cfb128_datatype_Float64_iv_None_aad_None = r"""'07B7199D3D3C51A1'""" + +example_mode_aes_192_cfb128_datatype_Decimal32_iv_None_aad_None = r"""'27F9199D'""" + +example_mode_aes_192_cfb128_datatype_Decimal64_iv_None_aad_None = r"""'27F9199D3D3CA19E'""" + +example_mode_aes_192_cfb128_datatype_Decimal128_iv_None_aad_None = r"""'27F9199D3D3CA19E2CCE5990D7551E73'""" + +example_mode_aes_192_cfb128_datatype_UUID_iv_None_aad_None = r"""'E0A6AAC139F851FF8C158AFAD7F365E3'""" + +example_mode_aes_192_cfb128_datatype_Date_iv_None_aad_None = r"""'51F0'""" + +example_mode_aes_192_cfb128_datatype_DateTime_iv_None_aad_None = r"""'496715C3'""" + +example_mode_aes_192_cfb128_datatype_DateTime64_iv_None_aad_None = r"""'2C0614FF523DA19E'""" + +example_mode_aes_192_cfb128_datatype_LowCardinality_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_cfb128_datatype_Array_iv_None_aad_None = r"""'06B5'""" + +example_mode_aes_192_cfb128_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_192_cfb128_datatype_IPv4_iv_None_aad_None = r"""'2A35F836'""" + +example_mode_aes_192_cfb128_datatype_IPv6_iv_None_aad_None = r"""'27B614253D3C243D2CCE59907B4A9E72'""" + +example_mode_aes_192_cfb128_datatype_Enum8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_cfb128_datatype_Enum16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_256_cfb128_datatype_String_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_cfb128_datatype_FixedString_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_cfb128_datatype_UInt8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_cfb128_datatype_UInt16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_256_cfb128_datatype_UInt32_iv_None_aad_None = r"""'B18ECF9E'""" + +example_mode_aes_256_cfb128_datatype_UInt64_iv_None_aad_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_cfb128_datatype_Int8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_cfb128_datatype_Int16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_256_cfb128_datatype_Int32_iv_None_aad_None = r"""'B18ECF9E'""" + +example_mode_aes_256_cfb128_datatype_Int64_iv_None_aad_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_cfb128_datatype_Float32_iv_None_aad_None = r"""'B08E4FA1'""" + +example_mode_aes_256_cfb128_datatype_Float64_iv_None_aad_None = r"""'B08ECF9EC7EBAF32'""" + +example_mode_aes_256_cfb128_datatype_Decimal32_iv_None_aad_None = r"""'90C0CF9E'""" + +example_mode_aes_256_cfb128_datatype_Decimal64_iv_None_aad_None = r"""'90C0CF9EC7EB5F0D'""" + +example_mode_aes_256_cfb128_datatype_Decimal128_iv_None_aad_None = r"""'90C0CF9EC7EB5F0D7B78C42556D668AC'""" + +example_mode_aes_256_cfb128_datatype_UUID_iv_None_aad_None = r"""'579F7CC2C32FAF6CDBA3174F5670133C'""" + +example_mode_aes_256_cfb128_datatype_Date_iv_None_aad_None = r"""'E6C9'""" + +example_mode_aes_256_cfb128_datatype_DateTime_iv_None_aad_None = r"""'FE5EC3C0'""" + +example_mode_aes_256_cfb128_datatype_DateTime64_iv_None_aad_None = r"""'9B3FC2FCA8EA5F0D'""" + +example_mode_aes_256_cfb128_datatype_LowCardinality_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_cfb128_datatype_Array_iv_None_aad_None = r"""'B18C'""" + +example_mode_aes_256_cfb128_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_256_cfb128_datatype_IPv4_iv_None_aad_None = r"""'9D0C2E35'""" + +example_mode_aes_256_cfb128_datatype_IPv6_iv_None_aad_None = r"""'908FC226C7EBDAAE7B78C425FAC9E8AD'""" + +example_mode_aes_256_cfb128_datatype_Enum8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_cfb128_datatype_Enum16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_128_cfb128_datatype_String_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_cfb128_datatype_FixedString_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_cfb128_datatype_UInt8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_cfb128_datatype_UInt16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_128_cfb128_datatype_UInt32_iv_16_aad_None = r"""'3388A984'""" + +example_mode_aes_128_cfb128_datatype_UInt64_iv_16_aad_None = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_cfb128_datatype_Int8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_cfb128_datatype_Int16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_128_cfb128_datatype_Int32_iv_16_aad_None = r"""'3388A984'""" + +example_mode_aes_128_cfb128_datatype_Int64_iv_16_aad_None = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_cfb128_datatype_Float32_iv_16_aad_None = r"""'328829BB'""" + +example_mode_aes_128_cfb128_datatype_Float64_iv_16_aad_None = r"""'3288A984DD060F67'""" + +example_mode_aes_128_cfb128_datatype_Decimal32_iv_16_aad_None = r"""'12C6A984'""" + +example_mode_aes_128_cfb128_datatype_Decimal64_iv_16_aad_None = r"""'12C6A984DD06FF58'""" + +example_mode_aes_128_cfb128_datatype_Decimal128_iv_16_aad_None = r"""'12C6A984DD06FF58E93960B1DEC50F1E'""" + +example_mode_aes_128_cfb128_datatype_UUID_iv_16_aad_None = r"""'D5991AD8D9C20F3949E2B3DBDE63748E'""" + +example_mode_aes_128_cfb128_datatype_Date_iv_16_aad_None = r"""'64CF'""" + +example_mode_aes_128_cfb128_datatype_DateTime_iv_16_aad_None = r"""'7C58A5DA'""" + +example_mode_aes_128_cfb128_datatype_DateTime64_iv_16_aad_None = r"""'1939A4E6B207FF58'""" + +example_mode_aes_128_cfb128_datatype_LowCardinality_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_cfb128_datatype_Array_iv_16_aad_None = r"""'338A'""" + +example_mode_aes_128_cfb128_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_128_cfb128_datatype_IPv4_iv_16_aad_None = r"""'1F0A482F'""" + +example_mode_aes_128_cfb128_datatype_IPv6_iv_16_aad_None = r"""'1289A43CDD067AFBE93960B172DA8F1F'""" + +example_mode_aes_128_cfb128_datatype_Enum8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_cfb128_datatype_Enum16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_192_cfb128_datatype_String_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_cfb128_datatype_FixedString_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_cfb128_datatype_UInt8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_cfb128_datatype_UInt16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_192_cfb128_datatype_UInt32_iv_16_aad_None = r"""'69C7E792'""" + +example_mode_aes_192_cfb128_datatype_UInt64_iv_16_aad_None = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_cfb128_datatype_Int8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_cfb128_datatype_Int16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_192_cfb128_datatype_Int32_iv_16_aad_None = r"""'69C7E792'""" + +example_mode_aes_192_cfb128_datatype_Int64_iv_16_aad_None = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_cfb128_datatype_Float32_iv_16_aad_None = r"""'68C767AD'""" + +example_mode_aes_192_cfb128_datatype_Float64_iv_16_aad_None = r"""'68C7E792B710878E'""" + +example_mode_aes_192_cfb128_datatype_Decimal32_iv_16_aad_None = r"""'4889E792'""" + +example_mode_aes_192_cfb128_datatype_Decimal64_iv_16_aad_None = r"""'4889E792B71077B1'""" + +example_mode_aes_192_cfb128_datatype_Decimal128_iv_16_aad_None = r"""'4889E792B71077B18446050EBFD861B5'""" + +example_mode_aes_192_cfb128_datatype_UUID_iv_16_aad_None = r"""'8FD654CEB3D487D0249DD664BF7E1A25'""" + +example_mode_aes_192_cfb128_datatype_Date_iv_16_aad_None = r"""'3E80'""" + +example_mode_aes_192_cfb128_datatype_DateTime_iv_16_aad_None = r"""'2617EBCC'""" + +example_mode_aes_192_cfb128_datatype_DateTime64_iv_16_aad_None = r"""'4376EAF0D81177B1'""" + +example_mode_aes_192_cfb128_datatype_LowCardinality_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_cfb128_datatype_Array_iv_16_aad_None = r"""'69C5'""" + +example_mode_aes_192_cfb128_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_192_cfb128_datatype_IPv4_iv_16_aad_None = r"""'45450639'""" + +example_mode_aes_192_cfb128_datatype_IPv6_iv_16_aad_None = r"""'48C6EA2AB710F2128446050E13C7E1B4'""" + +example_mode_aes_192_cfb128_datatype_Enum8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_cfb128_datatype_Enum16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_256_cfb128_datatype_String_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_cfb128_datatype_FixedString_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_cfb128_datatype_UInt8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_cfb128_datatype_UInt16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_256_cfb128_datatype_UInt32_iv_16_aad_None = r"""'68588817'""" + +example_mode_aes_256_cfb128_datatype_UInt64_iv_16_aad_None = r"""'685888173CDE4488'""" + +example_mode_aes_256_cfb128_datatype_Int8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_cfb128_datatype_Int16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_256_cfb128_datatype_Int32_iv_16_aad_None = r"""'68588817'""" + +example_mode_aes_256_cfb128_datatype_Int64_iv_16_aad_None = r"""'685888173CDE4488'""" + +example_mode_aes_256_cfb128_datatype_Float32_iv_16_aad_None = r"""'69580828'""" + +example_mode_aes_256_cfb128_datatype_Float64_iv_16_aad_None = r"""'695888173CDEB4B7'""" + +example_mode_aes_256_cfb128_datatype_Decimal32_iv_16_aad_None = r"""'49168817'""" + +example_mode_aes_256_cfb128_datatype_Decimal64_iv_16_aad_None = r"""'491688173CDE4488'""" + +example_mode_aes_256_cfb128_datatype_Decimal128_iv_16_aad_None = r"""'491688173CDE448870E043A7733CC726'""" + +example_mode_aes_256_cfb128_datatype_UUID_iv_16_aad_None = r"""'8E493B4B381AB4E9D03B90CD739ABCB6'""" + +example_mode_aes_256_cfb128_datatype_Date_iv_16_aad_None = r"""'3F1F'""" + +example_mode_aes_256_cfb128_datatype_DateTime_iv_16_aad_None = r"""'27888449'""" + +example_mode_aes_256_cfb128_datatype_DateTime64_iv_16_aad_None = r"""'42E9857553DF4488'""" + +example_mode_aes_256_cfb128_datatype_LowCardinality_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_cfb128_datatype_Array_iv_16_aad_None = r"""'685A'""" + +example_mode_aes_256_cfb128_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_256_cfb128_datatype_IPv4_iv_16_aad_None = r"""'44DA69BC'""" + +example_mode_aes_256_cfb128_datatype_IPv6_iv_16_aad_None = r"""'495985AF3CDEC12B70E043A7DF234727'""" + +example_mode_aes_256_cfb128_datatype_Enum8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_cfb128_datatype_Enum16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_128_ofb_datatype_String_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_ofb_datatype_FixedString_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_ofb_datatype_UInt8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_ofb_datatype_UInt16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_128_ofb_datatype_UInt32_iv_None_aad_None = r"""'11DFC1B5'""" + +example_mode_aes_128_ofb_datatype_UInt64_iv_None_aad_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_ofb_datatype_Int8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_ofb_datatype_Int16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_128_ofb_datatype_Int32_iv_None_aad_None = r"""'11DFC1B5'""" + +example_mode_aes_128_ofb_datatype_Int64_iv_None_aad_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_ofb_datatype_Float32_iv_None_aad_None = r"""'10DF418A'""" + +example_mode_aes_128_ofb_datatype_Float64_iv_None_aad_None = r"""'10DFC1B5F66C0D55'""" + +example_mode_aes_128_ofb_datatype_Decimal32_iv_None_aad_None = r"""'3091C1B5'""" + +example_mode_aes_128_ofb_datatype_Decimal64_iv_None_aad_None = r"""'3091C1B5F66CFD6A'""" + +example_mode_aes_128_ofb_datatype_Decimal128_iv_None_aad_None = r"""'3091C1B5F66CFD6A1DC46D66907BEEB1'""" + +example_mode_aes_128_ofb_datatype_UUID_iv_None_aad_None = r"""'F7CE72E9F2A80D0BBD1FBE0C90DD9521'""" + +example_mode_aes_128_ofb_datatype_Date_iv_None_aad_None = r"""'4698'""" + +example_mode_aes_128_ofb_datatype_DateTime_iv_None_aad_None = r"""'5E0FCDEB'""" + +example_mode_aes_128_ofb_datatype_DateTime64_iv_None_aad_None = r"""'3B6ECCD7996DFD6A'""" + +example_mode_aes_128_ofb_datatype_LowCardinality_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_ofb_datatype_Array_iv_None_aad_None = r"""'11DD'""" + +example_mode_aes_128_ofb_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_128_ofb_datatype_IPv4_iv_None_aad_None = r"""'3D5D201E'""" + +example_mode_aes_128_ofb_datatype_IPv6_iv_None_aad_None = r"""'30DECC0DF66C78C91DC46D663C646EB0'""" + +example_mode_aes_128_ofb_datatype_Enum8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_ofb_datatype_Enum16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_192_ofb_datatype_String_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_ofb_datatype_FixedString_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_ofb_datatype_UInt8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_ofb_datatype_UInt16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_192_ofb_datatype_UInt32_iv_None_aad_None = r"""'06B7199D'""" + +example_mode_aes_192_ofb_datatype_UInt64_iv_None_aad_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_ofb_datatype_Int8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_ofb_datatype_Int16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_192_ofb_datatype_Int32_iv_None_aad_None = r"""'06B7199D'""" + +example_mode_aes_192_ofb_datatype_Int64_iv_None_aad_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_ofb_datatype_Float32_iv_None_aad_None = r"""'07B799A2'""" + +example_mode_aes_192_ofb_datatype_Float64_iv_None_aad_None = r"""'07B7199D3D3C51A1'""" + +example_mode_aes_192_ofb_datatype_Decimal32_iv_None_aad_None = r"""'27F9199D'""" + +example_mode_aes_192_ofb_datatype_Decimal64_iv_None_aad_None = r"""'27F9199D3D3CA19E'""" + +example_mode_aes_192_ofb_datatype_Decimal128_iv_None_aad_None = r"""'27F9199D3D3CA19E2CCE5990D7551E73'""" + +example_mode_aes_192_ofb_datatype_UUID_iv_None_aad_None = r"""'E0A6AAC139F851FF8C158AFAD7F365E3'""" + +example_mode_aes_192_ofb_datatype_Date_iv_None_aad_None = r"""'51F0'""" + +example_mode_aes_192_ofb_datatype_DateTime_iv_None_aad_None = r"""'496715C3'""" + +example_mode_aes_192_ofb_datatype_DateTime64_iv_None_aad_None = r"""'2C0614FF523DA19E'""" + +example_mode_aes_192_ofb_datatype_LowCardinality_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_ofb_datatype_Array_iv_None_aad_None = r"""'06B5'""" + +example_mode_aes_192_ofb_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_192_ofb_datatype_IPv4_iv_None_aad_None = r"""'2A35F836'""" + +example_mode_aes_192_ofb_datatype_IPv6_iv_None_aad_None = r"""'27B614253D3C243D2CCE59907B4A9E72'""" + +example_mode_aes_192_ofb_datatype_Enum8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_ofb_datatype_Enum16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_256_ofb_datatype_String_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_ofb_datatype_FixedString_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_ofb_datatype_UInt8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_ofb_datatype_UInt16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_256_ofb_datatype_UInt32_iv_None_aad_None = r"""'B18ECF9E'""" + +example_mode_aes_256_ofb_datatype_UInt64_iv_None_aad_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_ofb_datatype_Int8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_ofb_datatype_Int16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_256_ofb_datatype_Int32_iv_None_aad_None = r"""'B18ECF9E'""" + +example_mode_aes_256_ofb_datatype_Int64_iv_None_aad_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_ofb_datatype_Float32_iv_None_aad_None = r"""'B08E4FA1'""" + +example_mode_aes_256_ofb_datatype_Float64_iv_None_aad_None = r"""'B08ECF9EC7EBAF32'""" + +example_mode_aes_256_ofb_datatype_Decimal32_iv_None_aad_None = r"""'90C0CF9E'""" + +example_mode_aes_256_ofb_datatype_Decimal64_iv_None_aad_None = r"""'90C0CF9EC7EB5F0D'""" + +example_mode_aes_256_ofb_datatype_Decimal128_iv_None_aad_None = r"""'90C0CF9EC7EB5F0D7B78C42556D668AC'""" + +example_mode_aes_256_ofb_datatype_UUID_iv_None_aad_None = r"""'579F7CC2C32FAF6CDBA3174F5670133C'""" + +example_mode_aes_256_ofb_datatype_Date_iv_None_aad_None = r"""'E6C9'""" + +example_mode_aes_256_ofb_datatype_DateTime_iv_None_aad_None = r"""'FE5EC3C0'""" + +example_mode_aes_256_ofb_datatype_DateTime64_iv_None_aad_None = r"""'9B3FC2FCA8EA5F0D'""" + +example_mode_aes_256_ofb_datatype_LowCardinality_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_ofb_datatype_Array_iv_None_aad_None = r"""'B18C'""" + +example_mode_aes_256_ofb_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_256_ofb_datatype_IPv4_iv_None_aad_None = r"""'9D0C2E35'""" + +example_mode_aes_256_ofb_datatype_IPv6_iv_None_aad_None = r"""'908FC226C7EBDAAE7B78C425FAC9E8AD'""" + +example_mode_aes_256_ofb_datatype_Enum8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_ofb_datatype_Enum16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_128_ofb_datatype_String_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_ofb_datatype_FixedString_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_ofb_datatype_UInt8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_ofb_datatype_UInt16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_128_ofb_datatype_UInt32_iv_16_aad_None = r"""'3388A984'""" + +example_mode_aes_128_ofb_datatype_UInt64_iv_16_aad_None = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_ofb_datatype_Int8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_ofb_datatype_Int16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_128_ofb_datatype_Int32_iv_16_aad_None = r"""'3388A984'""" + +example_mode_aes_128_ofb_datatype_Int64_iv_16_aad_None = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_ofb_datatype_Float32_iv_16_aad_None = r"""'328829BB'""" + +example_mode_aes_128_ofb_datatype_Float64_iv_16_aad_None = r"""'3288A984DD060F67'""" + +example_mode_aes_128_ofb_datatype_Decimal32_iv_16_aad_None = r"""'12C6A984'""" + +example_mode_aes_128_ofb_datatype_Decimal64_iv_16_aad_None = r"""'12C6A984DD06FF58'""" + +example_mode_aes_128_ofb_datatype_Decimal128_iv_16_aad_None = r"""'12C6A984DD06FF58E93960B1DEC50F1E'""" + +example_mode_aes_128_ofb_datatype_UUID_iv_16_aad_None = r"""'D5991AD8D9C20F3949E2B3DBDE63748E'""" + +example_mode_aes_128_ofb_datatype_Date_iv_16_aad_None = r"""'64CF'""" + +example_mode_aes_128_ofb_datatype_DateTime_iv_16_aad_None = r"""'7C58A5DA'""" + +example_mode_aes_128_ofb_datatype_DateTime64_iv_16_aad_None = r"""'1939A4E6B207FF58'""" + +example_mode_aes_128_ofb_datatype_LowCardinality_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_ofb_datatype_Array_iv_16_aad_None = r"""'338A'""" + +example_mode_aes_128_ofb_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_128_ofb_datatype_IPv4_iv_16_aad_None = r"""'1F0A482F'""" + +example_mode_aes_128_ofb_datatype_IPv6_iv_16_aad_None = r"""'1289A43CDD067AFBE93960B172DA8F1F'""" + +example_mode_aes_128_ofb_datatype_Enum8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_ofb_datatype_Enum16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_192_ofb_datatype_String_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_ofb_datatype_FixedString_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_ofb_datatype_UInt8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_ofb_datatype_UInt16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_192_ofb_datatype_UInt32_iv_16_aad_None = r"""'69C7E792'""" + +example_mode_aes_192_ofb_datatype_UInt64_iv_16_aad_None = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_ofb_datatype_Int8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_ofb_datatype_Int16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_192_ofb_datatype_Int32_iv_16_aad_None = r"""'69C7E792'""" + +example_mode_aes_192_ofb_datatype_Int64_iv_16_aad_None = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_ofb_datatype_Float32_iv_16_aad_None = r"""'68C767AD'""" + +example_mode_aes_192_ofb_datatype_Float64_iv_16_aad_None = r"""'68C7E792B710878E'""" + +example_mode_aes_192_ofb_datatype_Decimal32_iv_16_aad_None = r"""'4889E792'""" + +example_mode_aes_192_ofb_datatype_Decimal64_iv_16_aad_None = r"""'4889E792B71077B1'""" + +example_mode_aes_192_ofb_datatype_Decimal128_iv_16_aad_None = r"""'4889E792B71077B18446050EBFD861B5'""" + +example_mode_aes_192_ofb_datatype_UUID_iv_16_aad_None = r"""'8FD654CEB3D487D0249DD664BF7E1A25'""" + +example_mode_aes_192_ofb_datatype_Date_iv_16_aad_None = r"""'3E80'""" + +example_mode_aes_192_ofb_datatype_DateTime_iv_16_aad_None = r"""'2617EBCC'""" + +example_mode_aes_192_ofb_datatype_DateTime64_iv_16_aad_None = r"""'4376EAF0D81177B1'""" + +example_mode_aes_192_ofb_datatype_LowCardinality_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_ofb_datatype_Array_iv_16_aad_None = r"""'69C5'""" + +example_mode_aes_192_ofb_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_192_ofb_datatype_IPv4_iv_16_aad_None = r"""'45450639'""" + +example_mode_aes_192_ofb_datatype_IPv6_iv_16_aad_None = r"""'48C6EA2AB710F2128446050E13C7E1B4'""" + +example_mode_aes_192_ofb_datatype_Enum8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_ofb_datatype_Enum16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_256_ofb_datatype_String_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_ofb_datatype_FixedString_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_ofb_datatype_UInt8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_ofb_datatype_UInt16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_256_ofb_datatype_UInt32_iv_16_aad_None = r"""'68588817'""" + +example_mode_aes_256_ofb_datatype_UInt64_iv_16_aad_None = r"""'685888173CDE4488'""" + +example_mode_aes_256_ofb_datatype_Int8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_ofb_datatype_Int16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_256_ofb_datatype_Int32_iv_16_aad_None = r"""'68588817'""" + +example_mode_aes_256_ofb_datatype_Int64_iv_16_aad_None = r"""'685888173CDE4488'""" + +example_mode_aes_256_ofb_datatype_Float32_iv_16_aad_None = r"""'69580828'""" + +example_mode_aes_256_ofb_datatype_Float64_iv_16_aad_None = r"""'695888173CDEB4B7'""" + +example_mode_aes_256_ofb_datatype_Decimal32_iv_16_aad_None = r"""'49168817'""" + +example_mode_aes_256_ofb_datatype_Decimal64_iv_16_aad_None = r"""'491688173CDE4488'""" + +example_mode_aes_256_ofb_datatype_Decimal128_iv_16_aad_None = r"""'491688173CDE448870E043A7733CC726'""" + +example_mode_aes_256_ofb_datatype_UUID_iv_16_aad_None = r"""'8E493B4B381AB4E9D03B90CD739ABCB6'""" + +example_mode_aes_256_ofb_datatype_Date_iv_16_aad_None = r"""'3F1F'""" + +example_mode_aes_256_ofb_datatype_DateTime_iv_16_aad_None = r"""'27888449'""" + +example_mode_aes_256_ofb_datatype_DateTime64_iv_16_aad_None = r"""'42E9857553DF4488'""" + +example_mode_aes_256_ofb_datatype_LowCardinality_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_ofb_datatype_Array_iv_16_aad_None = r"""'685A'""" + +example_mode_aes_256_ofb_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_256_ofb_datatype_IPv4_iv_16_aad_None = r"""'44DA69BC'""" + +example_mode_aes_256_ofb_datatype_IPv6_iv_16_aad_None = r"""'495985AF3CDEC12B70E043A7DF234727'""" + +example_mode_aes_256_ofb_datatype_Enum8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_ofb_datatype_Enum16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_128_gcm_datatype_String_iv_12_aad_None = r"""'DC48B85D412AEF42C46DA18E25139D5D9D'""" + +example_mode_aes_128_gcm_datatype_FixedString_iv_12_aad_None = r"""'DC48B85D412AEF42C46DA18E25139D5D9D'""" + +example_mode_aes_128_gcm_datatype_UInt8_iv_12_aad_None = r"""'EC55C7254FC59C53DDE89F0AF034F08A19'""" + +example_mode_aes_128_gcm_datatype_UInt16_iv_12_aad_None = r"""'EC9644C4CEFA3BB71CD48C619B03BD238C18'""" + +example_mode_aes_128_gcm_datatype_UInt32_iv_12_aad_None = r"""'EC96C142DE5B4B621603E2ADD041DBAEB3F54F50'""" + +example_mode_aes_128_gcm_datatype_UInt64_iv_12_aad_None = r"""'EC96C142F4FF6F21DA0515DBB392E4C83187679A10F9D879'""" + +example_mode_aes_128_gcm_datatype_Int8_iv_12_aad_None = r"""'EC55C7254FC59C53DDE89F0AF034F08A19'""" + +example_mode_aes_128_gcm_datatype_Int16_iv_12_aad_None = r"""'EC9644C4CEFA3BB71CD48C619B03BD238C18'""" + +example_mode_aes_128_gcm_datatype_Int32_iv_12_aad_None = r"""'EC96C142DE5B4B621603E2ADD041DBAEB3F54F50'""" + +example_mode_aes_128_gcm_datatype_Int64_iv_12_aad_None = r"""'EC96C142F4FF6F21DA0515DBB392E4C83187679A10F9D879'""" + +example_mode_aes_128_gcm_datatype_Float32_iv_12_aad_None = r"""'ED96417D9A89D5DA7B2DA96F78D0D545BD724061'""" + +example_mode_aes_128_gcm_datatype_Float64_iv_12_aad_None = r"""'ED96C142F4FF9F1E36E4396725C81E2B56D2E0A42CB04CE9'""" + +example_mode_aes_128_gcm_datatype_Decimal32_iv_12_aad_None = r"""'CDD8C142960BD83213972E95DB26FFEC37308C56'""" + +example_mode_aes_128_gcm_datatype_Decimal64_iv_12_aad_None = r"""'CDD8C142F4FF6F219255868BB60628F03AE043D8943C1B7F'""" + +example_mode_aes_128_gcm_datatype_Decimal128_iv_12_aad_None = r"""'CDD8C142F4FF6F214AC2960E7EA36A6B1B1FC500659A9E81A843F13DCFC70751'""" + +example_mode_aes_128_gcm_datatype_UUID_iv_12_aad_None = r"""'0A87721EF03B9F40EA1945647E0511FB0552FE1E5264499B02AE50562A94AFF7'""" + +example_mode_aes_128_gcm_datatype_Date_iv_12_aad_None = r"""'BBD12669328AE5725A634A005E5B2C4BA877'""" + +example_mode_aes_128_gcm_datatype_DateTime_iv_12_aad_None = r"""'A346CD1C097FB5A48C84628E9C0C9F3BF4E3CBB2'""" + +example_mode_aes_128_gcm_datatype_DateTime64_iv_12_aad_None = r"""'C627CC209BFE6F21638D7F1465C11D12578CF38254273DC9'""" + +example_mode_aes_128_gcm_datatype_LowCardinality_iv_12_aad_None = r"""'DC48B85D412AEF42C46DA18E25139D5D9D'""" + +example_mode_aes_128_gcm_datatype_Array_iv_12_aad_None = r"""'EC94283AABA0C4FC46F56D71F9B3CB40288E'""" + +example_mode_aes_128_gcm_datatype_NULL_iv_12_aad_None = r"""'\\N'""" + +example_mode_aes_128_gcm_datatype_IPv4_iv_12_aad_None = r"""'C01420E9B1BBD6ED16DC1CC0ED7AE9773CD72297'""" + +example_mode_aes_128_gcm_datatype_IPv6_iv_12_aad_None = r"""'CD97CCFAF4FFEA824AC2960ED2BCEA6AC9427AAD6328E977BF356C8AEF93B556'""" + +example_mode_aes_128_gcm_datatype_Enum8_iv_12_aad_None = r"""'EC55C7254FC59C53DDE89F0AF034F08A19'""" + +example_mode_aes_128_gcm_datatype_Enum16_iv_12_aad_None = r"""'EC9644C4CEFA3BB71CD48C619B03BD238C18'""" + +example_mode_aes_192_gcm_datatype_String_iv_12_aad_None = r"""'7B34E3F4BAFCD2F3D493F843FFEBF9A415'""" + +example_mode_aes_192_gcm_datatype_FixedString_iv_12_aad_None = r"""'7B34E3F4BAFCD2F3D493F843FFEBF9A415'""" + +example_mode_aes_192_gcm_datatype_UInt8_iv_12_aad_None = r"""'4B5F8852220B8F5659FDE6C603D1C7E127'""" + +example_mode_aes_192_gcm_datatype_UInt16_iv_12_aad_None = r"""'4BFA728CD0043E08485EF39EEFEF8554BDD1'""" + +example_mode_aes_192_gcm_datatype_UInt32_iv_12_aad_None = r"""'4BFAA6C499D3637CF257889C690120FA80FBDBFD'""" + +example_mode_aes_192_gcm_datatype_UInt64_iv_12_aad_None = r"""'4BFAA6C467061FE32600E6AE21DD8A6D7AA0A38D2BE318EE'""" + +example_mode_aes_192_gcm_datatype_Int8_iv_12_aad_None = r"""'4B5F8852220B8F5659FDE6C603D1C7E127'""" + +example_mode_aes_192_gcm_datatype_Int16_iv_12_aad_None = r"""'4BFA728CD0043E08485EF39EEFEF8554BDD1'""" + +example_mode_aes_192_gcm_datatype_Int32_iv_12_aad_None = r"""'4BFAA6C499D3637CF257889C690120FA80FBDBFD'""" + +example_mode_aes_192_gcm_datatype_Int64_iv_12_aad_None = r"""'4BFAA6C467061FE32600E6AE21DD8A6D7AA0A38D2BE318EE'""" + +example_mode_aes_192_gcm_datatype_Float32_iv_12_aad_None = r"""'4AFA26FB812573A4DCCCFFB53F91E180A3F6EB95'""" + +example_mode_aes_192_gcm_datatype_Float64_iv_12_aad_None = r"""'4AFAA6C46706EFDC2968FA31495521544C0E792959D3DBA7'""" + +example_mode_aes_192_gcm_datatype_Decimal32_iv_12_aad_None = r"""'6AB4A6C4382BD5F8E4915BD5701D917BA8651BCA'""" + +example_mode_aes_192_gcm_datatype_Decimal64_iv_12_aad_None = r"""'6AB4A6C467061FE387F8502A371B592463BC120C037DD8D9'""" + +example_mode_aes_192_gcm_datatype_Decimal128_iv_12_aad_None = r"""'6AB4A6C467061FE343593C80B73515B94130370E40D09BDBE9EDB5BEAF29C541'""" + +example_mode_aes_192_gcm_datatype_UUID_iv_12_aad_None = r"""'ADEB159863C2EF82E382EFEAB7936E296A91D29B6A49269870CC866604C82451'""" + +example_mode_aes_192_gcm_datatype_Date_iv_12_aad_None = r"""'1CBD949DFC326906B13FCB565C1B28516877'""" + +example_mode_aes_192_gcm_datatype_DateTime_iv_12_aad_None = r"""'042AAA9AB3B5BBD234DF77B8C35BD4EBF561AEC0'""" + +example_mode_aes_192_gcm_datatype_DateTime64_iv_12_aad_None = r"""'614BABA608071FE304B3939091C0CFC91D1371D5FBD3F6BE'""" + +example_mode_aes_192_gcm_datatype_LowCardinality_iv_12_aad_None = r"""'7B34E3F4BAFCD2F3D493F843FFEBF9A415'""" + +example_mode_aes_192_gcm_datatype_Array_iv_12_aad_None = r"""'4BF887E38B2870FDEECD6305AEA010298096'""" + +example_mode_aes_192_gcm_datatype_NULL_iv_12_aad_None = r"""'\\N'""" + +example_mode_aes_192_gcm_datatype_IPv4_iv_12_aad_None = r"""'6778476F9B3687FD4D022727F4641E697D35D8D5'""" + +example_mode_aes_192_gcm_datatype_IPv6_iv_12_aad_None = r"""'6AFBAB7C67069A4043593C801B2A95B81CF6DF4E6D63A84A0DF7939A3FBE5B2D'""" + +example_mode_aes_192_gcm_datatype_Enum8_iv_12_aad_None = r"""'4B5F8852220B8F5659FDE6C603D1C7E127'""" + +example_mode_aes_192_gcm_datatype_Enum16_iv_12_aad_None = r"""'4BFA728CD0043E08485EF39EEFEF8554BDD1'""" + +example_mode_aes_256_gcm_datatype_String_iv_12_aad_None = r"""'67B83EFC31C169D7613D6881E954F624C2'""" + +example_mode_aes_256_gcm_datatype_FixedString_iv_12_aad_None = r"""'67B83EFC31C169D7613D6881E954F624C2'""" + +example_mode_aes_256_gcm_datatype_UInt8_iv_12_aad_None = r"""'574B67F06530273DD181658D384D73A2F5'""" + +example_mode_aes_256_gcm_datatype_UInt16_iv_12_aad_None = r"""'57CBC5BA7DF134BBAE0153E139EB9D7B8D18'""" + +example_mode_aes_256_gcm_datatype_UInt32_iv_12_aad_None = r"""'57CB6C3A286A4C6BDCEBF8ECEE345F12B5F44729'""" + +example_mode_aes_256_gcm_datatype_UInt64_iv_12_aad_None = r"""'57CB6C3AFC8427B9E3A61ABE0A227F1C2ADB2694F324393D'""" + +example_mode_aes_256_gcm_datatype_Int8_iv_12_aad_None = r"""'574B67F06530273DD181658D384D73A2F5'""" + +example_mode_aes_256_gcm_datatype_Int16_iv_12_aad_None = r"""'57CBC5BA7DF134BBAE0153E139EB9D7B8D18'""" + +example_mode_aes_256_gcm_datatype_Int32_iv_12_aad_None = r"""'57CB6C3A286A4C6BDCEBF8ECEE345F12B5F44729'""" + +example_mode_aes_256_gcm_datatype_Int64_iv_12_aad_None = r"""'57CB6C3AFC8427B9E3A61ABE0A227F1C2ADB2694F324393D'""" + +example_mode_aes_256_gcm_datatype_Float32_iv_12_aad_None = r"""'56CBEC055F35D9E71D5AA0252DB60716A99B1CA2'""" + +example_mode_aes_256_gcm_datatype_Float64_iv_12_aad_None = r"""'56CB6C3AFC84D786DF3A9796E0FDCECFCBBC26274A60F542'""" + +example_mode_aes_256_gcm_datatype_Decimal32_iv_12_aad_None = r"""'76856C3AE585D8FCE599B3C2E0F077FAD5FFF7D4'""" + +example_mode_aes_256_gcm_datatype_Decimal64_iv_12_aad_None = r"""'76856C3AFC8427B92E498E2933503432241F0E7C932F89C0'""" + +example_mode_aes_256_gcm_datatype_Decimal128_iv_12_aad_None = r"""'76856C3AFC8427B90E79078689172F14CAA5CA7BDC5F7FCD915CAE4EC302FF1C'""" + +example_mode_aes_256_gcm_datatype_UUID_iv_12_aad_None = r"""'B1DADF66F840D7D8AEA2D4EC89B1548437C22BB5B8D92A3A139ACED61DB3F2B3'""" + +example_mode_aes_256_gcm_datatype_Date_iv_12_aad_None = r"""'008CFC6AFC8484AADA010876EF4686BC3528'""" + +example_mode_aes_256_gcm_datatype_DateTime_iv_12_aad_None = r"""'181B606489B8E56FEB8D9950EAE27205E36C94FB'""" + +example_mode_aes_256_gcm_datatype_DateTime64_iv_12_aad_None = r"""'7D7A6158938527B9F938A05622B5957BD801AE303908AD22'""" + +example_mode_aes_256_gcm_datatype_LowCardinality_iv_12_aad_None = r"""'67B83EFC31C169D7613D6881E954F624C2'""" + +example_mode_aes_256_gcm_datatype_Array_iv_12_aad_None = r"""'57C908D31B11B2CE494A61EC468B149A9D77'""" + +example_mode_aes_256_gcm_datatype_NULL_iv_12_aad_None = r"""'\\N'""" + +example_mode_aes_256_gcm_datatype_IPv4_iv_12_aad_None = r"""'7B498D91B274D5D6748C43E4345552318BD05DA6'""" + +example_mode_aes_256_gcm_datatype_IPv6_iv_12_aad_None = r"""'76CA6182FC84A21A0E7907862508AF15CCE4089698265E4B0656598079CE56CC'""" + +example_mode_aes_256_gcm_datatype_Enum8_iv_12_aad_None = r"""'574B67F06530273DD181658D384D73A2F5'""" + +example_mode_aes_256_gcm_datatype_Enum16_iv_12_aad_None = r"""'57CBC5BA7DF134BBAE0153E139EB9D7B8D18'""" + +example_mode_aes_128_gcm_datatype_String_iv_12_aad_True = r"""'DCFAF1088D33EF99F1D06E3D14F265FD41'""" + +example_mode_aes_128_gcm_datatype_FixedString_iv_12_aad_True = r"""'DCFAF1088D33EF99F1D06E3D14F265FD41'""" + +example_mode_aes_128_gcm_datatype_UInt8_iv_12_aad_True = r"""'ECE78E7083DC9C88E85550B9C1D5082AC5'""" + +example_mode_aes_128_gcm_datatype_UInt16_iv_12_aad_True = r"""'EC96F68D9B3622B7C7E131AE28325CDB2CC4'""" + +example_mode_aes_128_gcm_datatype_UInt32_iv_12_aad_True = r"""'EC96C1426C121EAE0F0339986D8E689F520DEF8C'""" + +example_mode_aes_128_gcm_datatype_UInt64_iv_12_aad_True = r"""'EC96C142F4FF6F21684C4017AA923FFD8C48D4ABF10178A5'""" + +example_mode_aes_128_gcm_datatype_Int8_iv_12_aad_True = r"""'ECE78E7083DC9C88E85550B9C1D5082AC5'""" + +example_mode_aes_128_gcm_datatype_Int16_iv_12_aad_True = r"""'EC96F68D9B3622B7C7E131AE28325CDB2CC4'""" + +example_mode_aes_128_gcm_datatype_Int32_iv_12_aad_True = r"""'EC96C1426C121EAE0F0339986D8E689F520DEF8C'""" + +example_mode_aes_128_gcm_datatype_Int64_iv_12_aad_True = r"""'EC96C142F4FF6F21684C4017AA923FFD8C48D4ABF10178A5'""" + +example_mode_aes_128_gcm_datatype_Float32_iv_12_aad_True = r"""'ED96417D28C08016622D725AC51F66745C8AE0BD'""" + +example_mode_aes_128_gcm_datatype_Float64_iv_12_aad_True = r"""'ED96C142F4FF9F1E84AD6CAB3CC8C51EEB1D5395CD48EC35'""" + +example_mode_aes_128_gcm_datatype_Decimal32_iv_12_aad_True = r"""'CDD8C14224428DFE0A97F5A066E94CDDD6C82C8A'""" + +example_mode_aes_128_gcm_datatype_Decimal64_iv_12_aad_True = r"""'CDD8C142F4FF6F21201CD347AF06F3C5872FF0E975C4BBA3'""" + +example_mode_aes_128_gcm_datatype_Decimal128_iv_12_aad_True = r"""'CDD8C142F4FF6F214AC2960E7EA36A6BA95690CC7C9A45B4158C420C2E3FA78D'""" + +example_mode_aes_128_gcm_datatype_UUID_iv_12_aad_True = r"""'0A87721EF03B9F40EA1945647E0511FBB71BABD24B6492AEBF61E367CB6C0F2B'""" + +example_mode_aes_128_gcm_datatype_Date_iv_12_aad_True = r"""'BBD194206746FC728156F7CFED6ACDB308AB'""" + +example_mode_aes_128_gcm_datatype_DateTime_iv_12_aad_True = r"""'A346CD1CBB36E0689584B9BB21C32C0A151B6B6E'""" + +example_mode_aes_128_gcm_datatype_DateTime64_iv_12_aad_True = r"""'C627CC209BFE6F21D1C42AD87CC1C627EA4340B3B5DF9D15'""" + +example_mode_aes_128_gcm_datatype_LowCardinality_iv_12_aad_True = r"""'DCFAF1088D33EF99F1D06E3D14F265FD41'""" + +example_mode_aes_128_gcm_datatype_Array_iv_12_aad_True = r"""'EC949A73FE6CDDFC9DC0D0BE4A822AB88852'""" + +example_mode_aes_128_gcm_datatype_NULL_iv_12_aad_True = r"""'\\N'""" + +example_mode_aes_128_gcm_datatype_IPv4_iv_12_aad_True = r"""'C01420E903F283210FDCC7F550B55A46DD2F824B'""" + +example_mode_aes_128_gcm_datatype_IPv6_iv_12_aad_True = r"""'CD97CCFAF4FFEA824AC2960ED2BCEA6A7B0B2F617A28324202FADFBB0E6B158A'""" + +example_mode_aes_128_gcm_datatype_Enum8_iv_12_aad_True = r"""'ECE78E7083DC9C88E85550B9C1D5082AC5'""" + +example_mode_aes_128_gcm_datatype_Enum16_iv_12_aad_True = r"""'EC96F68D9B3622B7C7E131AE28325CDB2CC4'""" + +example_mode_aes_192_gcm_datatype_String_iv_12_aad_True = r"""'7BB72D91D66E3C93B34FBAFF92526E1A0E'""" + +example_mode_aes_192_gcm_datatype_FixedString_iv_12_aad_True = r"""'7BB72D91D66E3C93B34FBAFF92526E1A0E'""" + +example_mode_aes_192_gcm_datatype_UInt8_iv_12_aad_True = r"""'4BDC46374E9961363E21A47A6E68505F3C'""" + +example_mode_aes_192_gcm_datatype_UInt16_iv_12_aad_True = r"""'4BFAF142B568ACE628392FDC53823CC303CA'""" + +example_mode_aes_192_gcm_datatype_UInt32_iv_12_aad_True = r"""'4BFAA6C41A1D061060B9E8FBB5439C97396C65E6'""" + +example_mode_aes_192_gcm_datatype_UInt64_iv_12_aad_True = r"""'4BFAA6C467061FE3A5CE83C2B333EA0AA6E21FE09274A6F5'""" + +example_mode_aes_192_gcm_datatype_Int8_iv_12_aad_True = r"""'4BDC46374E9961363E21A47A6E68505F3C'""" + +example_mode_aes_192_gcm_datatype_Int16_iv_12_aad_True = r"""'4BFAF142B568ACE628392FDC53823CC303CA'""" + +example_mode_aes_192_gcm_datatype_Int32_iv_12_aad_True = r"""'4BFAA6C41A1D061060B9E8FBB5439C97396C65E6'""" + +example_mode_aes_192_gcm_datatype_Int64_iv_12_aad_True = r"""'4BFAA6C467061FE3A5CE83C2B333EA0AA6E21FE09274A6F5'""" + +example_mode_aes_192_gcm_datatype_Float32_iv_12_aad_True = r"""'4AFA26FB02EB16C84E229FD2E3D35DED1A61558E'""" + +example_mode_aes_192_gcm_datatype_Float64_iv_12_aad_True = r"""'4AFAA6C46706EFDCAAA69F5DDBBB4133904CC544E04465BC'""" + +example_mode_aes_192_gcm_datatype_Decimal32_iv_12_aad_True = r"""'6AB4A6C4BBE5B094767F3BB2AC5F2D1611F2A5D1'""" + +example_mode_aes_192_gcm_datatype_Decimal64_iv_12_aad_True = r"""'6AB4A6C467061FE304363546A5F53943BFFEAE61BAEA66C2'""" + +example_mode_aes_192_gcm_datatype_Decimal128_iv_12_aad_True = r"""'6AB4A6C467061FE343593C80B73515B9C2FE5262D23EFBBC35AF09D316BE7B5A'""" + +example_mode_aes_192_gcm_datatype_UUID_iv_12_aad_True = r"""'ADEB159863C2EF82E382EFEAB7936E29E95FB7F7F8A746FFAC8E3A0BBD5F9A4A'""" + +example_mode_aes_192_gcm_datatype_Date_iv_12_aad_True = r"""'1CBD1753995EFBE8D1581714E07691C6D66C'""" + +example_mode_aes_192_gcm_datatype_DateTime_iv_12_aad_True = r"""'042AAA9A307BDEBEA63117DF1F1968864CF610DB'""" + +example_mode_aes_192_gcm_datatype_DateTime64_iv_12_aad_True = r"""'614BABA608071FE3877DF6FC032EAFAEC151CDB8424448A5'""" + +example_mode_aes_192_gcm_datatype_LowCardinality_iv_12_aad_True = r"""'7BB72D91D66E3C93B34FBAFF92526E1A0E'""" + +example_mode_aes_192_gcm_datatype_Array_iv_12_aad_True = r"""'4BF8042DEE44E2138EAABF4712CDA9BE3E8D'""" + +example_mode_aes_192_gcm_datatype_NULL_iv_12_aad_True = r"""'\\N'""" + +example_mode_aes_192_gcm_datatype_IPv4_iv_12_aad_True = r"""'6778476F18F8E291DFEC47402826A204C4A266CE'""" + +example_mode_aes_192_gcm_datatype_IPv6_iv_12_aad_True = r"""'6AFBAB7C67069A4043593C801B2A95B89F38BA22FF8DC82DD1B52FF78629E536'""" + +example_mode_aes_192_gcm_datatype_Enum8_iv_12_aad_True = r"""'4BDC46374E9961363E21A47A6E68505F3C'""" + +example_mode_aes_192_gcm_datatype_Enum16_iv_12_aad_True = r"""'4BFAF142B568ACE628392FDC53823CC303CA'""" + +example_mode_aes_256_gcm_datatype_String_iv_12_aad_True = r"""'672AAAA73DCD5DEBC924C34E8F6E2678F8'""" + +example_mode_aes_256_gcm_datatype_FixedString_iv_12_aad_True = r"""'672AAAA73DCD5DEBC924C34E8F6E2678F8'""" + +example_mode_aes_256_gcm_datatype_UInt8_iv_12_aad_True = r"""'57D9F3AB693C13017998CE425E77A3FECF'""" + +example_mode_aes_256_gcm_datatype_UInt16_iv_12_aad_True = r"""'57CB572E26FD388F92A94A4AF68DA7ABD122'""" + +example_mode_aes_256_gcm_datatype_UInt32_iv_12_aad_True = r"""'57CB6C3ABAFE1767D0DFC444F79F90748F241B13'""" + +example_mode_aes_256_gcm_datatype_UInt64_iv_12_aad_True = r"""'57CB6C3AFC8427B9713241B2061643B43370E9F2C9F46507'""" + +example_mode_aes_256_gcm_datatype_Int8_iv_12_aad_True = r"""'57D9F3AB693C13017998CE425E77A3FECF'""" + +example_mode_aes_256_gcm_datatype_Int16_iv_12_aad_True = r"""'57CB572E26FD388F92A94A4AF68DA7ABD122'""" + +example_mode_aes_256_gcm_datatype_Int32_iv_12_aad_True = r"""'57CB6C3ABAFE1767D0DFC444F79F90748F241B13'""" + +example_mode_aes_256_gcm_datatype_Int64_iv_12_aad_True = r"""'57CB6C3AFC8427B9713241B2061643B43370E9F2C9F46507'""" + +example_mode_aes_256_gcm_datatype_Float32_iv_12_aad_True = r"""'56CBEC05CDA182EB116E9C8D341DC870934B4098'""" + +example_mode_aes_256_gcm_datatype_Float64_iv_12_aad_True = r"""'56CB6C3AFC84D7864DAECC9AECC9F267D217E94170B0A978'""" + +example_mode_aes_256_gcm_datatype_Decimal32_iv_12_aad_True = r"""'76856C3A771183F0E9AD8F6AF95BB89CEF2FABEE'""" + +example_mode_aes_256_gcm_datatype_Decimal64_iv_12_aad_True = r"""'76856C3AFC8427B9BCDDD5253F64089A3DB4C11AA9FFD5FA'""" + +example_mode_aes_256_gcm_datatype_Decimal128_iv_12_aad_True = r"""'76856C3AFC8427B90E79078689172F1458319177D06B436588F76128F9D2A326'""" + +example_mode_aes_256_gcm_datatype_UUID_iv_12_aad_True = r"""'B1DADF66F840D7D8AEA2D4EC89B15484A55670B9B4ED16920A3101B02763AE89'""" + +example_mode_aes_256_gcm_datatype_Date_iv_12_aad_True = r"""'008C6EFEA788889EE6A911DD2020BC6C6912'""" + +example_mode_aes_256_gcm_datatype_DateTime_iv_12_aad_True = r"""'181B60641B2CBE63E7B9A5F8F349BD63D9BCC8C1'""" + +example_mode_aes_256_gcm_datatype_DateTime64_iv_12_aad_True = r"""'7D7A6158938527B96BACFB5A2E81A9D3C1AA615603D8F118'""" + +example_mode_aes_256_gcm_datatype_LowCardinality_iv_12_aad_True = r"""'672AAAA73DCD5DEBC924C34E8F6E2678F8'""" + +example_mode_aes_256_gcm_datatype_Array_iv_12_aad_True = r"""'57C99A47401DBEFA75E2784789ED2E4AC14D'""" + +example_mode_aes_256_gcm_datatype_NULL_iv_12_aad_True = r"""'\\N'""" + +example_mode_aes_256_gcm_datatype_IPv4_iv_12_aad_True = r"""'7B498D9120E08EDA78B87F4C2DFE9D57B100019C'""" + +example_mode_aes_256_gcm_datatype_IPv6_iv_12_aad_True = r"""'76CA6182FC84A21A0E7907862508AF155E70539A941262E31FFD96E6431E0AF6'""" + +example_mode_aes_256_gcm_datatype_Enum8_iv_12_aad_True = r"""'57D9F3AB693C13017998CE425E77A3FECF'""" + +example_mode_aes_256_gcm_datatype_Enum16_iv_12_aad_True = r"""'57CB572E26FD388F92A94A4AF68DA7ABD122'""" + +example_mode_aes_128_ctr_datatype_String_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_ctr_datatype_FixedString_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_ctr_datatype_UInt8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_ctr_datatype_UInt16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_128_ctr_datatype_UInt32_iv_None_aad_None = r"""'11DFC1B5'""" + +example_mode_aes_128_ctr_datatype_UInt64_iv_None_aad_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_ctr_datatype_Int8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_ctr_datatype_Int16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_128_ctr_datatype_Int32_iv_None_aad_None = r"""'11DFC1B5'""" + +example_mode_aes_128_ctr_datatype_Int64_iv_None_aad_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_ctr_datatype_Float32_iv_None_aad_None = r"""'10DF418A'""" + +example_mode_aes_128_ctr_datatype_Float64_iv_None_aad_None = r"""'10DFC1B5F66C0D55'""" + +example_mode_aes_128_ctr_datatype_Decimal32_iv_None_aad_None = r"""'3091C1B5'""" + +example_mode_aes_128_ctr_datatype_Decimal64_iv_None_aad_None = r"""'3091C1B5F66CFD6A'""" + +example_mode_aes_128_ctr_datatype_Decimal128_iv_None_aad_None = r"""'3091C1B5F66CFD6A1DC46D66907BEEB1'""" + +example_mode_aes_128_ctr_datatype_UUID_iv_None_aad_None = r"""'F7CE72E9F2A80D0BBD1FBE0C90DD9521'""" + +example_mode_aes_128_ctr_datatype_Date_iv_None_aad_None = r"""'4698'""" + +example_mode_aes_128_ctr_datatype_DateTime_iv_None_aad_None = r"""'5E0FCDEB'""" + +example_mode_aes_128_ctr_datatype_DateTime64_iv_None_aad_None = r"""'3B6ECCD7996DFD6A'""" + +example_mode_aes_128_ctr_datatype_LowCardinality_iv_None_aad_None = r"""'21'""" + +example_mode_aes_128_ctr_datatype_Array_iv_None_aad_None = r"""'11DD'""" + +example_mode_aes_128_ctr_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_128_ctr_datatype_IPv4_iv_None_aad_None = r"""'3D5D201E'""" + +example_mode_aes_128_ctr_datatype_IPv6_iv_None_aad_None = r"""'30DECC0DF66C78C91DC46D663C646EB0'""" + +example_mode_aes_128_ctr_datatype_Enum8_iv_None_aad_None = r"""'11'""" + +example_mode_aes_128_ctr_datatype_Enum16_iv_None_aad_None = r"""'11DF'""" + +example_mode_aes_192_ctr_datatype_String_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_ctr_datatype_FixedString_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_ctr_datatype_UInt8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_ctr_datatype_UInt16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_192_ctr_datatype_UInt32_iv_None_aad_None = r"""'06B7199D'""" + +example_mode_aes_192_ctr_datatype_UInt64_iv_None_aad_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_ctr_datatype_Int8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_ctr_datatype_Int16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_192_ctr_datatype_Int32_iv_None_aad_None = r"""'06B7199D'""" + +example_mode_aes_192_ctr_datatype_Int64_iv_None_aad_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_ctr_datatype_Float32_iv_None_aad_None = r"""'07B799A2'""" + +example_mode_aes_192_ctr_datatype_Float64_iv_None_aad_None = r"""'07B7199D3D3C51A1'""" + +example_mode_aes_192_ctr_datatype_Decimal32_iv_None_aad_None = r"""'27F9199D'""" + +example_mode_aes_192_ctr_datatype_Decimal64_iv_None_aad_None = r"""'27F9199D3D3CA19E'""" + +example_mode_aes_192_ctr_datatype_Decimal128_iv_None_aad_None = r"""'27F9199D3D3CA19E2CCE5990D7551E73'""" + +example_mode_aes_192_ctr_datatype_UUID_iv_None_aad_None = r"""'E0A6AAC139F851FF8C158AFAD7F365E3'""" + +example_mode_aes_192_ctr_datatype_Date_iv_None_aad_None = r"""'51F0'""" + +example_mode_aes_192_ctr_datatype_DateTime_iv_None_aad_None = r"""'496715C3'""" + +example_mode_aes_192_ctr_datatype_DateTime64_iv_None_aad_None = r"""'2C0614FF523DA19E'""" + +example_mode_aes_192_ctr_datatype_LowCardinality_iv_None_aad_None = r"""'36'""" + +example_mode_aes_192_ctr_datatype_Array_iv_None_aad_None = r"""'06B5'""" + +example_mode_aes_192_ctr_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_192_ctr_datatype_IPv4_iv_None_aad_None = r"""'2A35F836'""" + +example_mode_aes_192_ctr_datatype_IPv6_iv_None_aad_None = r"""'27B614253D3C243D2CCE59907B4A9E72'""" + +example_mode_aes_192_ctr_datatype_Enum8_iv_None_aad_None = r"""'06'""" + +example_mode_aes_192_ctr_datatype_Enum16_iv_None_aad_None = r"""'06B7'""" + +example_mode_aes_256_ctr_datatype_String_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_ctr_datatype_FixedString_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_ctr_datatype_UInt8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_ctr_datatype_UInt16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_256_ctr_datatype_UInt32_iv_None_aad_None = r"""'B18ECF9E'""" + +example_mode_aes_256_ctr_datatype_UInt64_iv_None_aad_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_ctr_datatype_Int8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_ctr_datatype_Int16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_256_ctr_datatype_Int32_iv_None_aad_None = r"""'B18ECF9E'""" + +example_mode_aes_256_ctr_datatype_Int64_iv_None_aad_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_ctr_datatype_Float32_iv_None_aad_None = r"""'B08E4FA1'""" + +example_mode_aes_256_ctr_datatype_Float64_iv_None_aad_None = r"""'B08ECF9EC7EBAF32'""" + +example_mode_aes_256_ctr_datatype_Decimal32_iv_None_aad_None = r"""'90C0CF9E'""" + +example_mode_aes_256_ctr_datatype_Decimal64_iv_None_aad_None = r"""'90C0CF9EC7EB5F0D'""" + +example_mode_aes_256_ctr_datatype_Decimal128_iv_None_aad_None = r"""'90C0CF9EC7EB5F0D7B78C42556D668AC'""" + +example_mode_aes_256_ctr_datatype_UUID_iv_None_aad_None = r"""'579F7CC2C32FAF6CDBA3174F5670133C'""" + +example_mode_aes_256_ctr_datatype_Date_iv_None_aad_None = r"""'E6C9'""" + +example_mode_aes_256_ctr_datatype_DateTime_iv_None_aad_None = r"""'FE5EC3C0'""" + +example_mode_aes_256_ctr_datatype_DateTime64_iv_None_aad_None = r"""'9B3FC2FCA8EA5F0D'""" + +example_mode_aes_256_ctr_datatype_LowCardinality_iv_None_aad_None = r"""'81'""" + +example_mode_aes_256_ctr_datatype_Array_iv_None_aad_None = r"""'B18C'""" + +example_mode_aes_256_ctr_datatype_NULL_iv_None_aad_None = r"""'\\N'""" + +example_mode_aes_256_ctr_datatype_IPv4_iv_None_aad_None = r"""'9D0C2E35'""" + +example_mode_aes_256_ctr_datatype_IPv6_iv_None_aad_None = r"""'908FC226C7EBDAAE7B78C425FAC9E8AD'""" + +example_mode_aes_256_ctr_datatype_Enum8_iv_None_aad_None = r"""'B1'""" + +example_mode_aes_256_ctr_datatype_Enum16_iv_None_aad_None = r"""'B18E'""" + +example_mode_aes_128_ctr_datatype_String_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_ctr_datatype_FixedString_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_ctr_datatype_UInt8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_ctr_datatype_UInt16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_128_ctr_datatype_UInt32_iv_16_aad_None = r"""'3388A984'""" + +example_mode_aes_128_ctr_datatype_UInt64_iv_16_aad_None = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_ctr_datatype_Int8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_ctr_datatype_Int16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_128_ctr_datatype_Int32_iv_16_aad_None = r"""'3388A984'""" + +example_mode_aes_128_ctr_datatype_Int64_iv_16_aad_None = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_ctr_datatype_Float32_iv_16_aad_None = r"""'328829BB'""" + +example_mode_aes_128_ctr_datatype_Float64_iv_16_aad_None = r"""'3288A984DD060F67'""" + +example_mode_aes_128_ctr_datatype_Decimal32_iv_16_aad_None = r"""'12C6A984'""" + +example_mode_aes_128_ctr_datatype_Decimal64_iv_16_aad_None = r"""'12C6A984DD06FF58'""" + +example_mode_aes_128_ctr_datatype_Decimal128_iv_16_aad_None = r"""'12C6A984DD06FF58E93960B1DEC50F1E'""" + +example_mode_aes_128_ctr_datatype_UUID_iv_16_aad_None = r"""'D5991AD8D9C20F3949E2B3DBDE63748E'""" + +example_mode_aes_128_ctr_datatype_Date_iv_16_aad_None = r"""'64CF'""" + +example_mode_aes_128_ctr_datatype_DateTime_iv_16_aad_None = r"""'7C58A5DA'""" + +example_mode_aes_128_ctr_datatype_DateTime64_iv_16_aad_None = r"""'1939A4E6B207FF58'""" + +example_mode_aes_128_ctr_datatype_LowCardinality_iv_16_aad_None = r"""'03'""" + +example_mode_aes_128_ctr_datatype_Array_iv_16_aad_None = r"""'338A'""" + +example_mode_aes_128_ctr_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_128_ctr_datatype_IPv4_iv_16_aad_None = r"""'1F0A482F'""" + +example_mode_aes_128_ctr_datatype_IPv6_iv_16_aad_None = r"""'1289A43CDD067AFBE93960B172DA8F1F'""" + +example_mode_aes_128_ctr_datatype_Enum8_iv_16_aad_None = r"""'33'""" + +example_mode_aes_128_ctr_datatype_Enum16_iv_16_aad_None = r"""'3388'""" + +example_mode_aes_192_ctr_datatype_String_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_ctr_datatype_FixedString_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_ctr_datatype_UInt8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_ctr_datatype_UInt16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_192_ctr_datatype_UInt32_iv_16_aad_None = r"""'69C7E792'""" + +example_mode_aes_192_ctr_datatype_UInt64_iv_16_aad_None = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_ctr_datatype_Int8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_ctr_datatype_Int16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_192_ctr_datatype_Int32_iv_16_aad_None = r"""'69C7E792'""" + +example_mode_aes_192_ctr_datatype_Int64_iv_16_aad_None = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_ctr_datatype_Float32_iv_16_aad_None = r"""'68C767AD'""" + +example_mode_aes_192_ctr_datatype_Float64_iv_16_aad_None = r"""'68C7E792B710878E'""" + +example_mode_aes_192_ctr_datatype_Decimal32_iv_16_aad_None = r"""'4889E792'""" + +example_mode_aes_192_ctr_datatype_Decimal64_iv_16_aad_None = r"""'4889E792B71077B1'""" + +example_mode_aes_192_ctr_datatype_Decimal128_iv_16_aad_None = r"""'4889E792B71077B18446050EBFD861B5'""" + +example_mode_aes_192_ctr_datatype_UUID_iv_16_aad_None = r"""'8FD654CEB3D487D0249DD664BF7E1A25'""" + +example_mode_aes_192_ctr_datatype_Date_iv_16_aad_None = r"""'3E80'""" + +example_mode_aes_192_ctr_datatype_DateTime_iv_16_aad_None = r"""'2617EBCC'""" + +example_mode_aes_192_ctr_datatype_DateTime64_iv_16_aad_None = r"""'4376EAF0D81177B1'""" + +example_mode_aes_192_ctr_datatype_LowCardinality_iv_16_aad_None = r"""'59'""" + +example_mode_aes_192_ctr_datatype_Array_iv_16_aad_None = r"""'69C5'""" + +example_mode_aes_192_ctr_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_192_ctr_datatype_IPv4_iv_16_aad_None = r"""'45450639'""" + +example_mode_aes_192_ctr_datatype_IPv6_iv_16_aad_None = r"""'48C6EA2AB710F2128446050E13C7E1B4'""" + +example_mode_aes_192_ctr_datatype_Enum8_iv_16_aad_None = r"""'69'""" + +example_mode_aes_192_ctr_datatype_Enum16_iv_16_aad_None = r"""'69C7'""" + +example_mode_aes_256_ctr_datatype_String_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_ctr_datatype_FixedString_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_ctr_datatype_UInt8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_ctr_datatype_UInt16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_256_ctr_datatype_UInt32_iv_16_aad_None = r"""'68588817'""" + +example_mode_aes_256_ctr_datatype_UInt64_iv_16_aad_None = r"""'685888173CDE4488'""" + +example_mode_aes_256_ctr_datatype_Int8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_ctr_datatype_Int16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_256_ctr_datatype_Int32_iv_16_aad_None = r"""'68588817'""" + +example_mode_aes_256_ctr_datatype_Int64_iv_16_aad_None = r"""'685888173CDE4488'""" + +example_mode_aes_256_ctr_datatype_Float32_iv_16_aad_None = r"""'69580828'""" + +example_mode_aes_256_ctr_datatype_Float64_iv_16_aad_None = r"""'695888173CDEB4B7'""" + +example_mode_aes_256_ctr_datatype_Decimal32_iv_16_aad_None = r"""'49168817'""" + +example_mode_aes_256_ctr_datatype_Decimal64_iv_16_aad_None = r"""'491688173CDE4488'""" + +example_mode_aes_256_ctr_datatype_Decimal128_iv_16_aad_None = r"""'491688173CDE448870E043A7733CC726'""" + +example_mode_aes_256_ctr_datatype_UUID_iv_16_aad_None = r"""'8E493B4B381AB4E9D03B90CD739ABCB6'""" + +example_mode_aes_256_ctr_datatype_Date_iv_16_aad_None = r"""'3F1F'""" + +example_mode_aes_256_ctr_datatype_DateTime_iv_16_aad_None = r"""'27888449'""" + +example_mode_aes_256_ctr_datatype_DateTime64_iv_16_aad_None = r"""'42E9857553DF4488'""" + +example_mode_aes_256_ctr_datatype_LowCardinality_iv_16_aad_None = r"""'58'""" + +example_mode_aes_256_ctr_datatype_Array_iv_16_aad_None = r"""'685A'""" + +example_mode_aes_256_ctr_datatype_NULL_iv_16_aad_None = r"""'\\N'""" + +example_mode_aes_256_ctr_datatype_IPv4_iv_16_aad_None = r"""'44DA69BC'""" + +example_mode_aes_256_ctr_datatype_IPv6_iv_16_aad_None = r"""'495985AF3CDEC12B70E043A7DF234727'""" + +example_mode_aes_256_ctr_datatype_Enum8_iv_16_aad_None = r"""'68'""" + +example_mode_aes_256_ctr_datatype_Enum16_iv_16_aad_None = r"""'6858'""" + +example_mode_aes_128_ecb_datatype_bytes_iv_None_aad_None = r"""'7B62A15720E13327948BF706B89CF2BE'""" + +example_mode_aes_128_ecb_datatype_emptystring_iv_None_aad_None = r"""'F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_192_ecb_datatype_bytes_iv_None_aad_None = r"""'C60D7A90C41260E3CD03422E9163144A'""" + +example_mode_aes_192_ecb_datatype_emptystring_iv_None_aad_None = r"""'D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_256_ecb_datatype_bytes_iv_None_aad_None = r"""'B73CDD4E7705F0C516612F860715EBE3'""" + +example_mode_aes_256_ecb_datatype_emptystring_iv_None_aad_None = r"""'217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_128_cbc_datatype_bytes_iv_None_aad_None = r"""'7B62A15720E13327948BF706B89CF2BE'""" + +example_mode_aes_128_cbc_datatype_emptystring_iv_None_aad_None = r"""'F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_192_cbc_datatype_bytes_iv_None_aad_None = r"""'C60D7A90C41260E3CD03422E9163144A'""" + +example_mode_aes_192_cbc_datatype_emptystring_iv_None_aad_None = r"""'D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_256_cbc_datatype_bytes_iv_None_aad_None = r"""'B73CDD4E7705F0C516612F860715EBE3'""" + +example_mode_aes_256_cbc_datatype_emptystring_iv_None_aad_None = r"""'217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_128_cbc_datatype_bytes_iv_16_aad_None = r"""'CDA4B7027137998B9A33C2096C9A50DD'""" + +example_mode_aes_128_cbc_datatype_emptystring_iv_16_aad_None = r"""'56A77308430BA344FFBF016999795ED5'""" + +example_mode_aes_192_cbc_datatype_bytes_iv_16_aad_None = r"""'67771349942D4F812553F2E1E3FFB276'""" + +example_mode_aes_192_cbc_datatype_emptystring_iv_16_aad_None = r"""'62E9214DB5E239F0CAD31ADF26AB313F'""" + +example_mode_aes_256_cbc_datatype_bytes_iv_16_aad_None = r"""'6046ECF8094941C6DEC9278FF6F137E9'""" + +example_mode_aes_256_cbc_datatype_emptystring_iv_16_aad_None = r"""'4EC7785DA650D55B71B52816B1DB5AD3'""" + +example_mode_aes_128_cfb1_datatype_bytes_iv_None_aad_None = r"""'00'""" + +example_mode_aes_128_cfb1_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_192_cfb1_datatype_bytes_iv_None_aad_None = r"""'00'""" + +example_mode_aes_192_cfb1_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_256_cfb1_datatype_bytes_iv_None_aad_None = r"""'B8'""" + +example_mode_aes_256_cfb1_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_128_cfb1_datatype_bytes_iv_16_aad_None = r"""'00'""" + +example_mode_aes_128_cfb1_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_192_cfb1_datatype_bytes_iv_16_aad_None = r"""'07'""" + +example_mode_aes_192_cfb1_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_256_cfb1_datatype_bytes_iv_16_aad_None = r"""'7F'""" + +example_mode_aes_256_cfb1_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_128_cfb8_datatype_bytes_iv_None_aad_None = r"""'10'""" + +example_mode_aes_128_cfb8_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_192_cfb8_datatype_bytes_iv_None_aad_None = r"""'07'""" + +example_mode_aes_192_cfb8_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_256_cfb8_datatype_bytes_iv_None_aad_None = r"""'B0'""" + +example_mode_aes_256_cfb8_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_128_cfb8_datatype_bytes_iv_16_aad_None = r"""'32'""" + +example_mode_aes_128_cfb8_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_192_cfb8_datatype_bytes_iv_16_aad_None = r"""'68'""" + +example_mode_aes_192_cfb8_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_256_cfb8_datatype_bytes_iv_16_aad_None = r"""'69'""" + +example_mode_aes_256_cfb8_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_128_cfb128_datatype_bytes_iv_None_aad_None = r"""'10'""" + +example_mode_aes_128_cfb128_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_192_cfb128_datatype_bytes_iv_None_aad_None = r"""'07'""" + +example_mode_aes_192_cfb128_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_256_cfb128_datatype_bytes_iv_None_aad_None = r"""'B0'""" + +example_mode_aes_256_cfb128_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_128_cfb128_datatype_bytes_iv_16_aad_None = r"""'32'""" + +example_mode_aes_128_cfb128_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_192_cfb128_datatype_bytes_iv_16_aad_None = r"""'68'""" + +example_mode_aes_192_cfb128_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_256_cfb128_datatype_bytes_iv_16_aad_None = r"""'69'""" + +example_mode_aes_256_cfb128_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_128_ofb_datatype_bytes_iv_None_aad_None = r"""'10'""" + +example_mode_aes_128_ofb_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_192_ofb_datatype_bytes_iv_None_aad_None = r"""'07'""" + +example_mode_aes_192_ofb_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_256_ofb_datatype_bytes_iv_None_aad_None = r"""'B0'""" + +example_mode_aes_256_ofb_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_128_ofb_datatype_bytes_iv_16_aad_None = r"""'32'""" + +example_mode_aes_128_ofb_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_192_ofb_datatype_bytes_iv_16_aad_None = r"""'68'""" + +example_mode_aes_192_ofb_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_256_ofb_datatype_bytes_iv_16_aad_None = r"""'69'""" + +example_mode_aes_256_ofb_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_128_gcm_datatype_bytes_iv_12_aad_None = r"""'ED7AF588306031432D60AE52CB0522C131'""" + +example_mode_aes_128_gcm_datatype_emptystring_iv_12_aad_None = r"""'BF165C92C4F89A189DCD3A9CCD549D64'""" + +example_mode_aes_192_gcm_datatype_bytes_iv_12_aad_None = r"""'4AF425C405715C1F91B04661C96F5942E9'""" + +example_mode_aes_192_gcm_datatype_emptystring_iv_12_aad_None = r"""'F95400D38FC8B708F21C1A5CC97E2611'""" + +example_mode_aes_256_gcm_datatype_bytes_iv_12_aad_None = r"""'56C3D480260AD4984887DA3D7CBDFB952B'""" + +example_mode_aes_256_gcm_datatype_emptystring_iv_12_aad_None = r"""'4C24D7C3019165A77A8AE2FB9E130FBB'""" + +example_mode_aes_128_gcm_datatype_bytes_iv_12_aad_True = r"""'EDC8BCDDFC79319818DD61E1FAE4DA61ED'""" + +example_mode_aes_128_gcm_datatype_emptystring_iv_12_aad_True = r"""'9E4606A8AD25466858006BD90DA530F5'""" + +example_mode_aes_192_gcm_datatype_bytes_iv_12_aad_True = r"""'4A77EBA169E3B27FF66C04DDA4D6CEFCF2'""" + +example_mode_aes_192_gcm_datatype_emptystring_iv_12_aad_True = r"""'43B2949BA8DFB2A1511C294394E0DFB4'""" + +example_mode_aes_256_gcm_datatype_bytes_iv_12_aad_True = r"""'565140DB2A06E0A4E09E71F21A872BC911'""" + +example_mode_aes_256_gcm_datatype_emptystring_iv_12_aad_True = r"""'555A1764B65804E27174F272CF88FC11'""" + +example_mode_aes_128_ctr_datatype_bytes_iv_None_aad_None = r"""'10'""" + +example_mode_aes_128_ctr_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_192_ctr_datatype_bytes_iv_None_aad_None = r"""'07'""" + +example_mode_aes_192_ctr_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_256_ctr_datatype_bytes_iv_None_aad_None = r"""'B0'""" + +example_mode_aes_256_ctr_datatype_emptystring_iv_None_aad_None = r"""''""" + +example_mode_aes_128_ctr_datatype_bytes_iv_16_aad_None = r"""'32'""" + +example_mode_aes_128_ctr_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_192_ctr_datatype_bytes_iv_16_aad_None = r"""'68'""" + +example_mode_aes_192_ctr_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_256_ctr_datatype_bytes_iv_16_aad_None = r"""'69'""" + +example_mode_aes_256_ctr_datatype_emptystring_iv_16_aad_None = r"""''""" + +example_mode_aes_128_ecb_datatype_utf8string_iv_None_aad_None = r"""'4F5025E938F215F36B4EFEBD8A328B5CB8D530E00C83AD075E2D845A2425D8B5'""" + +example_mode_aes_128_ecb_datatype_utf8fixedstring_iv_None_aad_None = r"""'4F5025E938F215F36B4EFEBD8A328B5CB8D530E00C83AD075E2D845A2425D8B5'""" + +example_mode_aes_192_ecb_datatype_utf8string_iv_None_aad_None = r"""'7C1CE735A57407291267928DE0E2E479822D3586BD475686D9DAB17103D7B162'""" + +example_mode_aes_192_ecb_datatype_utf8fixedstring_iv_None_aad_None = r"""'7C1CE735A57407291267928DE0E2E479822D3586BD475686D9DAB17103D7B162'""" + +example_mode_aes_256_ecb_datatype_utf8string_iv_None_aad_None = r"""'3303F819796DDB5046AAFAB8A39FC3AB8A858B18916A30D2E2C3C9C9BBC961FD'""" + +example_mode_aes_256_ecb_datatype_utf8fixedstring_iv_None_aad_None = r"""'3303F819796DDB5046AAFAB8A39FC3AB8A858B18916A30D2E2C3C9C9BBC961FD'""" + +example_mode_aes_128_cbc_datatype_utf8string_iv_None_aad_None = r"""'4F5025E938F215F36B4EFEBD8A328B5C00AF476CA1EE03B7C0D297C2BF287339'""" + +example_mode_aes_128_cbc_datatype_utf8fixedstring_iv_None_aad_None = r"""'4F5025E938F215F36B4EFEBD8A328B5C00AF476CA1EE03B7C0D297C2BF287339'""" + +example_mode_aes_192_cbc_datatype_utf8string_iv_None_aad_None = r"""'7C1CE735A57407291267928DE0E2E47918C12F093BCD530F69669FC25B23195A'""" + +example_mode_aes_192_cbc_datatype_utf8fixedstring_iv_None_aad_None = r"""'7C1CE735A57407291267928DE0E2E47918C12F093BCD530F69669FC25B23195A'""" + +example_mode_aes_256_cbc_datatype_utf8string_iv_None_aad_None = r"""'3303F819796DDB5046AAFAB8A39FC3ABC19300E0966158A167939EDD20D39907'""" + +example_mode_aes_256_cbc_datatype_utf8fixedstring_iv_None_aad_None = r"""'3303F819796DDB5046AAFAB8A39FC3ABC19300E0966158A167939EDD20D39907'""" + +example_mode_aes_128_cbc_datatype_utf8string_iv_16_aad_None = r"""'0BD95BFF6DE2DC43D936DAF23937B06D602786A6770B627EB56BC7F681B1C9DB'""" + +example_mode_aes_128_cbc_datatype_utf8fixedstring_iv_16_aad_None = r"""'0BD95BFF6DE2DC43D936DAF23937B06D602786A6770B627EB56BC7F681B1C9DB'""" + +example_mode_aes_192_cbc_datatype_utf8string_iv_16_aad_None = r"""'D619039D0956015C34336196DB3EB5A4710B2B8860344AB2625E9269C5E4A6CC'""" + +example_mode_aes_192_cbc_datatype_utf8fixedstring_iv_16_aad_None = r"""'D619039D0956015C34336196DB3EB5A4710B2B8860344AB2625E9269C5E4A6CC'""" + +example_mode_aes_256_cbc_datatype_utf8string_iv_16_aad_None = r"""'A7663A9F621A26398B51DFBC099A6FA09032C25FE48CB9D2DE29A8DFD581714D'""" + +example_mode_aes_256_cbc_datatype_utf8fixedstring_iv_16_aad_None = r"""'A7663A9F621A26398B51DFBC099A6FA09032C25FE48CB9D2DE29A8DFD581714D'""" + +example_mode_aes_128_cfb1_datatype_utf8string_iv_None_aad_None = r"""'5BA033EA7B9901874F4F863E229069EBA414B7C317D37DF0'""" + +example_mode_aes_128_cfb1_datatype_utf8fixedstring_iv_None_aad_None = r"""'5BA033EA7B9901874F4F863E229069EBA414B7C317D37DF0'""" + +example_mode_aes_192_cfb1_datatype_utf8string_iv_None_aad_None = r"""'7B69A097857549357D008DB662730D8735DE1673D9E8CFF9'""" + +example_mode_aes_192_cfb1_datatype_utf8fixedstring_iv_None_aad_None = r"""'7B69A097857549357D008DB662730D8735DE1673D9E8CFF9'""" + +example_mode_aes_256_cfb1_datatype_utf8string_iv_None_aad_None = r"""'C797191D9D99944E674425A4275A8A3263E1E1357DF8E11E'""" + +example_mode_aes_256_cfb1_datatype_utf8fixedstring_iv_None_aad_None = r"""'C797191D9D99944E674425A4275A8A3263E1E1357DF8E11E'""" + +example_mode_aes_128_cfb1_datatype_utf8string_iv_16_aad_None = r"""'49CFD4F9B884A17F67C8CDD639EB4D367BE2B1656CA442A4'""" + +example_mode_aes_128_cfb1_datatype_utf8fixedstring_iv_16_aad_None = r"""'49CFD4F9B884A17F67C8CDD639EB4D367BE2B1656CA442A4'""" + +example_mode_aes_192_cfb1_datatype_utf8string_iv_16_aad_None = r"""'54C01F88A909EC7B2AEE59F81C138B8EE2DF205E2ED74210'""" + +example_mode_aes_192_cfb1_datatype_utf8fixedstring_iv_16_aad_None = r"""'54C01F88A909EC7B2AEE59F81C138B8EE2DF205E2ED74210'""" + +example_mode_aes_256_cfb1_datatype_utf8string_iv_16_aad_None = r"""'2E1863FFF5FEC47DBB3F50BCC912976E2777442C693EB1B5'""" + +example_mode_aes_256_cfb1_datatype_utf8fixedstring_iv_16_aad_None = r"""'2E1863FFF5FEC47DBB3F50BCC912976E2777442C693EB1B5'""" + +example_mode_aes_128_cfb8_datatype_utf8string_iv_None_aad_None = r"""'5716349E99199E0EF18EB43C06B54AB2F3E0C4CEA0BC11F9'""" + +example_mode_aes_128_cfb8_datatype_utf8fixedstring_iv_None_aad_None = r"""'5716349E99199E0EF18EB43C06B54AB2F3E0C4CEA0BC11F9'""" + +example_mode_aes_192_cfb8_datatype_utf8string_iv_None_aad_None = r"""'40541136D8EEE5DCFC55722D8FB56ED9D4CCDF0CB104B14D'""" + +example_mode_aes_192_cfb8_datatype_utf8fixedstring_iv_None_aad_None = r"""'40541136D8EEE5DCFC55722D8FB56ED9D4CCDF0CB104B14D'""" + +example_mode_aes_256_cfb8_datatype_utf8string_iv_None_aad_None = r"""'F796638AA3A076EA9816324AE8A93F420280C33AA2DE4AF8'""" + +example_mode_aes_256_cfb8_datatype_utf8fixedstring_iv_None_aad_None = r"""'F796638AA3A076EA9816324AE8A93F420280C33AA2DE4AF8'""" + +example_mode_aes_128_cfb8_datatype_utf8string_iv_16_aad_None = r"""'75BA4CC259943AF0B6E20E8FCC78C0427601F60930A5F980'""" + +example_mode_aes_128_cfb8_datatype_utf8fixedstring_iv_16_aad_None = r"""'75BA4CC259943AF0B6E20E8FCC78C0427601F60930A5F980'""" + +example_mode_aes_192_cfb8_datatype_utf8string_iv_16_aad_None = r"""'2F48574D4D12E6B2C1EF1B43346E437333FFD386067A9398'""" + +example_mode_aes_192_cfb8_datatype_utf8fixedstring_iv_16_aad_None = r"""'2F48574D4D12E6B2C1EF1B43346E437333FFD386067A9398'""" + +example_mode_aes_256_cfb8_datatype_utf8string_iv_16_aad_None = r"""'2E79EE96B485FC5BF3BE56AD461AAC2B1CCB425F51679553'""" + +example_mode_aes_256_cfb8_datatype_utf8fixedstring_iv_16_aad_None = r"""'2E79EE96B485FC5BF3BE56AD461AAC2B1CCB425F51679553'""" + +example_mode_aes_128_cfb128_datatype_utf8string_iv_None_aad_None = r"""'571C627072083ECFD8460B39C4132D1B2802275B5B24EF73'""" + +example_mode_aes_128_cfb128_datatype_utf8fixedstring_iv_None_aad_None = r"""'571C627072083ECFD8460B39C4132D1B2802275B5B24EF73'""" + +example_mode_aes_192_cfb128_datatype_utf8string_iv_None_aad_None = r"""'4074BA58B958623BE94C3FCF833DDDD9AC9F875CC2784719'""" + +example_mode_aes_192_cfb128_datatype_utf8fixedstring_iv_None_aad_None = r"""'4074BA58B958623BE94C3FCF833DDDD9AC9F875CC2784719'""" + +example_mode_aes_256_cfb128_datatype_utf8string_iv_None_aad_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06F5E4BB666EC25FE2'""" + +example_mode_aes_256_cfb128_datatype_utf8fixedstring_iv_None_aad_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06F5E4BB666EC25FE2'""" + +example_mode_aes_128_cfb128_datatype_utf8string_iv_16_aad_None = r"""'754B0A4159623CFD2CBB06EE8AADCCB4581E4F5FB9F091DD'""" + +example_mode_aes_128_cfb128_datatype_utf8fixedstring_iv_16_aad_None = r"""'754B0A4159623CFD2CBB06EE8AADCCB4581E4F5FB9F091DD'""" + +example_mode_aes_192_cfb128_datatype_utf8string_iv_16_aad_None = r"""'2F0444573374B41441C46351EBB0A21FD2D5B29B19D817D8'""" + +example_mode_aes_192_cfb128_datatype_utf8fixedstring_iv_16_aad_None = r"""'2F0444573374B41441C46351EBB0A21FD2D5B29B19D817D8'""" + +example_mode_aes_256_cfb128_datatype_utf8string_iv_16_aad_None = r"""'2E9B2BD2B8BA872DB56225F82754048C22AA31B7F22AD276'""" + +example_mode_aes_256_cfb128_datatype_utf8fixedstring_iv_16_aad_None = r"""'2E9B2BD2B8BA872DB56225F82754048C22AA31B7F22AD276'""" + +example_mode_aes_128_ofb_datatype_utf8string_iv_None_aad_None = r"""'571C627072083ECFD8460B39C4132D1B1EFEEBE7197398AE'""" + +example_mode_aes_128_ofb_datatype_utf8fixedstring_iv_None_aad_None = r"""'571C627072083ECFD8460B39C4132D1B1EFEEBE7197398AE'""" + +example_mode_aes_192_ofb_datatype_utf8string_iv_None_aad_None = r"""'4074BA58B958623BE94C3FCF833DDDD95F6EFF17F7823E17'""" + +example_mode_aes_192_ofb_datatype_utf8fixedstring_iv_None_aad_None = r"""'4074BA58B958623BE94C3FCF833DDDD95F6EFF17F7823E17'""" + +example_mode_aes_256_ofb_datatype_utf8string_iv_None_aad_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06B24181EFC9F2663B'""" + +example_mode_aes_256_ofb_datatype_utf8fixedstring_iv_None_aad_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06B24181EFC9F2663B'""" + +example_mode_aes_128_ofb_datatype_utf8string_iv_16_aad_None = r"""'754B0A4159623CFD2CBB06EE8AADCCB46A1C2A356E7D91D8'""" + +example_mode_aes_128_ofb_datatype_utf8fixedstring_iv_16_aad_None = r"""'754B0A4159623CFD2CBB06EE8AADCCB46A1C2A356E7D91D8'""" + +example_mode_aes_192_ofb_datatype_utf8string_iv_16_aad_None = r"""'2F0444573374B41441C46351EBB0A21F81C68E6CF92A6AF3'""" + +example_mode_aes_192_ofb_datatype_utf8fixedstring_iv_16_aad_None = r"""'2F0444573374B41441C46351EBB0A21F81C68E6CF92A6AF3'""" + +example_mode_aes_256_ofb_datatype_utf8string_iv_16_aad_None = r"""'2E9B2BD2B8BA872DB56225F82754048CE38E2C23393CF6FD'""" + +example_mode_aes_256_ofb_datatype_utf8fixedstring_iv_16_aad_None = r"""'2E9B2BD2B8BA872DB56225F82754048CE38E2C23393CF6FD'""" + +example_mode_aes_128_gcm_datatype_utf8string_iv_12_aad_None = r"""'AA556287709BAC848F40F0512ACBA9C1D3324C9C90260B0CDDC3AD7EBB18C53625907FE8745D8FFA'""" + +example_mode_aes_128_gcm_datatype_utf8fixedstring_iv_12_aad_None = r"""'AA556287709BAC848F40F0512ACBA9C1D3324C9C90260B0CDDC3AD7EBB18C53625907FE8745D8FFA'""" + +example_mode_aes_192_gcm_datatype_utf8string_iv_12_aad_None = r"""'0D390501E362DC4686DB5ADFE35DD613EA762CEE8E06DB20D0A9639BEF09294270D1352D22DB2CAF'""" + +example_mode_aes_192_gcm_datatype_utf8fixedstring_iv_12_aad_None = r"""'0D390501E362DC4686DB5ADFE35DD613EA762CEE8E06DB20D0A9639BEF09294270D1352D22DB2CAF'""" + +example_mode_aes_256_gcm_datatype_utf8string_iv_12_aad_None = r"""'1108CFFF78E0E41CCBFB61D9DD7FECBEF8AE22FF80D345FCAC905BDB791BC316A9A7D21FB34951F0'""" + +example_mode_aes_256_gcm_datatype_utf8fixedstring_iv_12_aad_None = r"""'1108CFFF78E0E41CCBFB61D9DD7FECBEF8AE22FF80D345FCAC905BDB791BC316A9A7D21FB34951F0'""" + +example_mode_aes_128_gcm_datatype_utf8string_iv_12_aad_True = r"""'AA556287709BAC848F40F0512ACBA9C1D3324C9C90260B0CD181E1471DC596491F22269EF75F4E5F'""" + +example_mode_aes_128_gcm_datatype_utf8fixedstring_iv_12_aad_True = r"""'AA556287709BAC848F40F0512ACBA9C1D3324C9C90260B0CD181E1471DC596491F22269EF75F4E5F'""" + +example_mode_aes_192_gcm_datatype_utf8string_iv_12_aad_True = r"""'0D390501E362DC4686DB5ADFE35DD613EA762CEE8E06DB20936BD43E66BAD9322C30B6E5D64110BB'""" + +example_mode_aes_192_gcm_datatype_utf8fixedstring_iv_12_aad_True = r"""'0D390501E362DC4686DB5ADFE35DD613EA762CEE8E06DB20936BD43E66BAD9322C30B6E5D64110BB'""" + +example_mode_aes_256_gcm_datatype_utf8string_iv_12_aad_True = r"""'1108CFFF78E0E41CCBFB61D9DD7FECBEF8AE22FF80D345FC92F054FDB351668A3790BE74D20869DD'""" + +example_mode_aes_256_gcm_datatype_utf8fixedstring_iv_12_aad_True = r"""'1108CFFF78E0E41CCBFB61D9DD7FECBEF8AE22FF80D345FC92F054FDB351668A3790BE74D20869DD'""" + +example_mode_aes_128_ctr_datatype_utf8string_iv_None_aad_None = r"""'571C627072083ECFD8460B39C4132D1B60215F9423235540'""" + +example_mode_aes_128_ctr_datatype_utf8fixedstring_iv_None_aad_None = r"""'571C627072083ECFD8460B39C4132D1B60215F9423235540'""" + +example_mode_aes_192_ctr_datatype_utf8string_iv_None_aad_None = r"""'4074BA58B958623BE94C3FCF833DDDD9EE89664A09F10327'""" + +example_mode_aes_192_ctr_datatype_utf8fixedstring_iv_None_aad_None = r"""'4074BA58B958623BE94C3FCF833DDDD9EE89664A09F10327'""" + +example_mode_aes_256_ctr_datatype_utf8string_iv_None_aad_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06519900DEC1DB5CF5'""" + +example_mode_aes_256_ctr_datatype_utf8fixedstring_iv_None_aad_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06519900DEC1DB5CF5'""" + +example_mode_aes_128_ctr_datatype_utf8string_iv_16_aad_None = r"""'754B0A4159623CFD2CBB06EE8AADCCB437EE2D5260EB4958'""" + +example_mode_aes_128_ctr_datatype_utf8fixedstring_iv_16_aad_None = r"""'754B0A4159623CFD2CBB06EE8AADCCB437EE2D5260EB4958'""" + +example_mode_aes_192_ctr_datatype_utf8string_iv_16_aad_None = r"""'2F0444573374B41441C46351EBB0A21F039E1E85A0813F1D'""" + +example_mode_aes_192_ctr_datatype_utf8fixedstring_iv_16_aad_None = r"""'2F0444573374B41441C46351EBB0A21F039E1E85A0813F1D'""" + +example_mode_aes_256_ctr_datatype_utf8string_iv_16_aad_None = r"""'2E9B2BD2B8BA872DB56225F82754048C944F1E670DF94BAA'""" + +example_mode_aes_256_ctr_datatype_utf8fixedstring_iv_16_aad_None = r"""'2E9B2BD2B8BA872DB56225F82754048C944F1E670DF94BAA'""" + diff --git a/tests/testflows/aes_encryption/tests/snapshots/encrypt_mysql.py.encrypt_mysql.snapshot b/tests/testflows/aes_encryption/tests/snapshots/encrypt_mysql.py.encrypt_mysql.snapshot new file mode 100644 index 00000000000..280dbf35be2 --- /dev/null +++ b/tests/testflows/aes_encryption/tests/snapshots/encrypt_mysql.py.encrypt_mysql.snapshot @@ -0,0 +1,3060 @@ +example_mode_aes_128_ecb_datatype_bytes_key_16_iv_None = r"""'7B62A15720E13327948BF706B89CF2BE'""" + +example_mode_aes_128_ecb_datatype_emptystring_key_16_iv_None = r"""'F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_ecb_datatype_utf8string_key_16_iv_None = r"""'4F5025E938F215F36B4EFEBD8A328B5CB8D530E00C83AD075E2D845A2425D8B5'""" + +example_mode_aes_128_ecb_datatype_utf8fixedstring_key_16_iv_None = r"""'4F5025E938F215F36B4EFEBD8A328B5CB8D530E00C83AD075E2D845A2425D8B5'""" + +example_mode_aes_128_ecb_datatype_String_key_16_iv_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_ecb_datatype_FixedString_key_16_iv_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_ecb_datatype_UInt8_key_16_iv_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_ecb_datatype_UInt16_key_16_iv_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_ecb_datatype_UInt32_key_16_iv_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_ecb_datatype_UInt64_key_16_iv_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_ecb_datatype_Int8_key_16_iv_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_ecb_datatype_Int16_key_16_iv_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_ecb_datatype_Int32_key_16_iv_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_ecb_datatype_Int64_key_16_iv_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_ecb_datatype_Float32_key_16_iv_None = r"""'FF4D70D9A1050E6BBDD0325FC45CC22D'""" + +example_mode_aes_128_ecb_datatype_Float64_key_16_iv_None = r"""'75FE6B4A722A31D7760680CC1B9F131D'""" + +example_mode_aes_128_ecb_datatype_Decimal32_key_16_iv_None = r"""'83BBD7CCE7E5A38071653870475D48D2'""" + +example_mode_aes_128_ecb_datatype_Decimal64_key_16_iv_None = r"""'BE0DD9302B2952CE9CC3721DD85C8E66'""" + +example_mode_aes_128_ecb_datatype_Decimal128_key_16_iv_None = r"""'5F3DBFA74809E45E03980357B26787AFF30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_ecb_datatype_UUID_key_16_iv_None = r"""'FF9161B222B4A67481271035745F06D9F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_ecb_datatype_Date_key_16_iv_None = r"""'1E4FBE33752D96D147E890C29A409BFE'""" + +example_mode_aes_128_ecb_datatype_DateTime_key_16_iv_None = r"""'384F3D97B78D52C73CD06C0E1B6DE399'""" + +example_mode_aes_128_ecb_datatype_DateTime64_key_16_iv_None = r"""'C7F50A2D0175F3ED280AD42FF01FF5F2'""" + +example_mode_aes_128_ecb_datatype_LowCardinality_key_16_iv_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_ecb_datatype_Array_key_16_iv_None = r"""'D9152D05CFA9E162983A5A2E883109B4'""" + +example_mode_aes_128_ecb_datatype_NULL_key_16_iv_None = r"""'\\N'""" + +example_mode_aes_128_ecb_datatype_IPv4_key_16_iv_None = r"""'4F32782638C1F33C6A7202CA83F0C12C'""" + +example_mode_aes_128_ecb_datatype_IPv6_key_16_iv_None = r"""'F54700FF04ADAD342BA6830DB12AD7E9F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_ecb_datatype_Enum8_key_16_iv_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_ecb_datatype_Enum16_key_16_iv_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_ecb_datatype_bytes_key_24_iv_None = r"""'0E7A1A2ED5C8BCC0B811B44D7FEA9E51'""" + +example_mode_aes_128_ecb_datatype_emptystring_key_24_iv_None = r"""'644DD62B737880C0203A16C9844616A6'""" + +example_mode_aes_128_ecb_datatype_utf8string_key_24_iv_None = r"""'4A970096F9F40F5507A678BF07F22B6B0644E044B974D161C223EA94DCB43B5D'""" + +example_mode_aes_128_ecb_datatype_utf8fixedstring_key_24_iv_None = r"""'4A970096F9F40F5507A678BF07F22B6B0644E044B974D161C223EA94DCB43B5D'""" + +example_mode_aes_128_ecb_datatype_String_key_24_iv_None = r"""'697175286BC73A26C572DBD9480738F3'""" + +example_mode_aes_128_ecb_datatype_FixedString_key_24_iv_None = r"""'697175286BC73A26C572DBD9480738F3'""" + +example_mode_aes_128_ecb_datatype_UInt8_key_24_iv_None = r"""'113A22E44AC1C4B397CC8204C069F5F4'""" + +example_mode_aes_128_ecb_datatype_UInt16_key_24_iv_None = r"""'94DD57978311932F2E9FED922796A023'""" + +example_mode_aes_128_ecb_datatype_UInt32_key_24_iv_None = r"""'CB84F00C70A72890AF6F7106AE8109CB'""" + +example_mode_aes_128_ecb_datatype_UInt64_key_24_iv_None = r"""'973944561BDA0D954449BEBD64C9ED7A'""" + +example_mode_aes_128_ecb_datatype_Int8_key_24_iv_None = r"""'113A22E44AC1C4B397CC8204C069F5F4'""" + +example_mode_aes_128_ecb_datatype_Int16_key_24_iv_None = r"""'94DD57978311932F2E9FED922796A023'""" + +example_mode_aes_128_ecb_datatype_Int32_key_24_iv_None = r"""'CB84F00C70A72890AF6F7106AE8109CB'""" + +example_mode_aes_128_ecb_datatype_Int64_key_24_iv_None = r"""'973944561BDA0D954449BEBD64C9ED7A'""" + +example_mode_aes_128_ecb_datatype_Float32_key_24_iv_None = r"""'7BCD3C1EB87CDA7BBA4B19929367243E'""" + +example_mode_aes_128_ecb_datatype_Float64_key_24_iv_None = r"""'3712B5B24D8F17CA7BE784AB7D57514E'""" + +example_mode_aes_128_ecb_datatype_Decimal32_key_24_iv_None = r"""'37BE2C0520C1A4C6F88CCBA2EEC020E5'""" + +example_mode_aes_128_ecb_datatype_Decimal64_key_24_iv_None = r"""'9ACC742A0FD36A80FEA2DBA2D73C4BFA'""" + +example_mode_aes_128_ecb_datatype_Decimal128_key_24_iv_None = r"""'EE8E9F15CAE8C5B6F2E0547636DEAF14644DD62B737880C0203A16C9844616A6'""" + +example_mode_aes_128_ecb_datatype_UUID_key_24_iv_None = r"""'6FD5580CC0B329922B575B79D4AD6E70644DD62B737880C0203A16C9844616A6'""" + +example_mode_aes_128_ecb_datatype_Date_key_24_iv_None = r"""'B0ADFA31B83C5B33B277097C33B06CB4'""" + +example_mode_aes_128_ecb_datatype_DateTime_key_24_iv_None = r"""'0632CF6416213B4153D247F4A85EAB19'""" + +example_mode_aes_128_ecb_datatype_DateTime64_key_24_iv_None = r"""'3D1049375E6EA3599E7AC2C753B1AA7F'""" + +example_mode_aes_128_ecb_datatype_LowCardinality_key_24_iv_None = r"""'697175286BC73A26C572DBD9480738F3'""" + +example_mode_aes_128_ecb_datatype_Array_key_24_iv_None = r"""'129253539B09D45C2147884BB2B866BD'""" + +example_mode_aes_128_ecb_datatype_NULL_key_24_iv_None = r"""'\\N'""" + +example_mode_aes_128_ecb_datatype_IPv4_key_24_iv_None = r"""'B0D8302DB6F6B772E883A9353CDC28F0'""" + +example_mode_aes_128_ecb_datatype_IPv6_key_24_iv_None = r"""'781ABF0C605281F93A00B9BADD2CD1E3644DD62B737880C0203A16C9844616A6'""" + +example_mode_aes_128_ecb_datatype_Enum8_key_24_iv_None = r"""'113A22E44AC1C4B397CC8204C069F5F4'""" + +example_mode_aes_128_ecb_datatype_Enum16_key_24_iv_None = r"""'94DD57978311932F2E9FED922796A023'""" + +example_mode_aes_192_ecb_datatype_bytes_key_24_iv_None = r"""'C60D7A90C41260E3CD03422E9163144A'""" + +example_mode_aes_192_ecb_datatype_emptystring_key_24_iv_None = r"""'D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_ecb_datatype_utf8string_key_24_iv_None = r"""'7C1CE735A57407291267928DE0E2E479822D3586BD475686D9DAB17103D7B162'""" + +example_mode_aes_192_ecb_datatype_utf8fixedstring_key_24_iv_None = r"""'7C1CE735A57407291267928DE0E2E479822D3586BD475686D9DAB17103D7B162'""" + +example_mode_aes_192_ecb_datatype_String_key_24_iv_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_ecb_datatype_FixedString_key_24_iv_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_ecb_datatype_UInt8_key_24_iv_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_ecb_datatype_UInt16_key_24_iv_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_ecb_datatype_UInt32_key_24_iv_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_ecb_datatype_UInt64_key_24_iv_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_ecb_datatype_Int8_key_24_iv_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_ecb_datatype_Int16_key_24_iv_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_ecb_datatype_Int32_key_24_iv_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_ecb_datatype_Int64_key_24_iv_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_ecb_datatype_Float32_key_24_iv_None = r"""'4E0C122631ED64EAD726833291A81878'""" + +example_mode_aes_192_ecb_datatype_Float64_key_24_iv_None = r"""'3F723599278E22E4692CE7D7D5F9A12F'""" + +example_mode_aes_192_ecb_datatype_Decimal32_key_24_iv_None = r"""'2420D49DBAA5CEF7D853C98DA1BD33BF'""" + +example_mode_aes_192_ecb_datatype_Decimal64_key_24_iv_None = r"""'FDF594113FCC2776653ED109A51FADF1'""" + +example_mode_aes_192_ecb_datatype_Decimal128_key_24_iv_None = r"""'79207931793E374FB5A3A2AC1ECA857AD8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_ecb_datatype_UUID_key_24_iv_None = r"""'9FDB738E78D0D2F774C484ED82A854E4D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_ecb_datatype_Date_key_24_iv_None = r"""'2CDD4685168FA3E2A7FA2092E86F44D4'""" + +example_mode_aes_192_ecb_datatype_DateTime_key_24_iv_None = r"""'A4BEE097872E44FAD94D6707D6643DF5'""" + +example_mode_aes_192_ecb_datatype_DateTime64_key_24_iv_None = r"""'1798B23C09F783623943560DF142E0F3'""" + +example_mode_aes_192_ecb_datatype_LowCardinality_key_24_iv_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_ecb_datatype_Array_key_24_iv_None = r"""'7C0B9021CAF2CBBB06DBF589740DCC65'""" + +example_mode_aes_192_ecb_datatype_NULL_key_24_iv_None = r"""'\\N'""" + +example_mode_aes_192_ecb_datatype_IPv4_key_24_iv_None = r"""'B20465C932A0719BA04E2F76371510D8'""" + +example_mode_aes_192_ecb_datatype_IPv6_key_24_iv_None = r"""'CCCDC9B9C3F182254591DFEDDCE9F232D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_ecb_datatype_Enum8_key_24_iv_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_ecb_datatype_Enum16_key_24_iv_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_ecb_datatype_bytes_key_32_iv_None = r"""'4977677DCA6485E59B6D2AEC781DB50E'""" + +example_mode_aes_192_ecb_datatype_emptystring_key_32_iv_None = r"""'4F9776A389026F399064946440DF432F'""" + +example_mode_aes_192_ecb_datatype_utf8string_key_32_iv_None = r"""'718D9D0FCE92DF4C73C6ADF970082475F5B91A80060E2F74BDB70F4D61D51128'""" + +example_mode_aes_192_ecb_datatype_utf8fixedstring_key_32_iv_None = r"""'718D9D0FCE92DF4C73C6ADF970082475F5B91A80060E2F74BDB70F4D61D51128'""" + +example_mode_aes_192_ecb_datatype_String_key_32_iv_None = r"""'851106E40808E28682DAC1AD840A7E92'""" + +example_mode_aes_192_ecb_datatype_FixedString_key_32_iv_None = r"""'851106E40808E28682DAC1AD840A7E92'""" + +example_mode_aes_192_ecb_datatype_UInt8_key_32_iv_None = r"""'077626FAA6FD46322732E0A107849CBE'""" + +example_mode_aes_192_ecb_datatype_UInt16_key_32_iv_None = r"""'C55FFF6925F48B2DDDD3F8696A6EE21A'""" + +example_mode_aes_192_ecb_datatype_UInt32_key_32_iv_None = r"""'7F3A3604968AC5BB1578D256A221442A'""" + +example_mode_aes_192_ecb_datatype_UInt64_key_32_iv_None = r"""'FCC1BC19F3A2F8F0484BD2BF3A069BB9'""" + +example_mode_aes_192_ecb_datatype_Int8_key_32_iv_None = r"""'077626FAA6FD46322732E0A107849CBE'""" + +example_mode_aes_192_ecb_datatype_Int16_key_32_iv_None = r"""'C55FFF6925F48B2DDDD3F8696A6EE21A'""" + +example_mode_aes_192_ecb_datatype_Int32_key_32_iv_None = r"""'7F3A3604968AC5BB1578D256A221442A'""" + +example_mode_aes_192_ecb_datatype_Int64_key_32_iv_None = r"""'FCC1BC19F3A2F8F0484BD2BF3A069BB9'""" + +example_mode_aes_192_ecb_datatype_Float32_key_32_iv_None = r"""'57AC570C827D0B10A340A635080E4BED'""" + +example_mode_aes_192_ecb_datatype_Float64_key_32_iv_None = r"""'EB761A3EB9A06676C875E70C6323B6D3'""" + +example_mode_aes_192_ecb_datatype_Decimal32_key_32_iv_None = r"""'07757D905A8DF38EEC7EAC2436BC883D'""" + +example_mode_aes_192_ecb_datatype_Decimal64_key_32_iv_None = r"""'B3DD7C625F5C9CAEB24B014AAF1660CF'""" + +example_mode_aes_192_ecb_datatype_Decimal128_key_32_iv_None = r"""'AF4EE89B714CBB6ED41802268A6C291F4F9776A389026F399064946440DF432F'""" + +example_mode_aes_192_ecb_datatype_UUID_key_32_iv_None = r"""'6DDADFA4AD0FEA6DD4AE756F5E13E0EE4F9776A389026F399064946440DF432F'""" + +example_mode_aes_192_ecb_datatype_Date_key_32_iv_None = r"""'D2212BB509C49D9DCBE970F86D34C4BE'""" + +example_mode_aes_192_ecb_datatype_DateTime_key_32_iv_None = r"""'C3AB19DCEFE1F61019484A2589D69037'""" + +example_mode_aes_192_ecb_datatype_DateTime64_key_32_iv_None = r"""'FB427597072B72E5C3D1D65247DB6A8C'""" + +example_mode_aes_192_ecb_datatype_LowCardinality_key_32_iv_None = r"""'851106E40808E28682DAC1AD840A7E92'""" + +example_mode_aes_192_ecb_datatype_Array_key_32_iv_None = r"""'D0FE4D4B34CFA03960FA609F1AA18D79'""" + +example_mode_aes_192_ecb_datatype_NULL_key_32_iv_None = r"""'\\N'""" + +example_mode_aes_192_ecb_datatype_IPv4_key_32_iv_None = r"""'F07B737C70748F1ACDF5DBB874B4D78B'""" + +example_mode_aes_192_ecb_datatype_IPv6_key_32_iv_None = r"""'C95A36120328FDE78278655287FF91F44F9776A389026F399064946440DF432F'""" + +example_mode_aes_192_ecb_datatype_Enum8_key_32_iv_None = r"""'077626FAA6FD46322732E0A107849CBE'""" + +example_mode_aes_192_ecb_datatype_Enum16_key_32_iv_None = r"""'C55FFF6925F48B2DDDD3F8696A6EE21A'""" + +example_mode_aes_256_ecb_datatype_bytes_key_32_iv_None = r"""'B73CDD4E7705F0C516612F860715EBE3'""" + +example_mode_aes_256_ecb_datatype_emptystring_key_32_iv_None = r"""'217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_ecb_datatype_utf8string_key_32_iv_None = r"""'3303F819796DDB5046AAFAB8A39FC3AB8A858B18916A30D2E2C3C9C9BBC961FD'""" + +example_mode_aes_256_ecb_datatype_utf8fixedstring_key_32_iv_None = r"""'3303F819796DDB5046AAFAB8A39FC3AB8A858B18916A30D2E2C3C9C9BBC961FD'""" + +example_mode_aes_256_ecb_datatype_String_key_32_iv_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_ecb_datatype_FixedString_key_32_iv_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_ecb_datatype_UInt8_key_32_iv_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_ecb_datatype_UInt16_key_32_iv_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_ecb_datatype_UInt32_key_32_iv_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_ecb_datatype_UInt64_key_32_iv_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_ecb_datatype_Int8_key_32_iv_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_ecb_datatype_Int16_key_32_iv_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_ecb_datatype_Int32_key_32_iv_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_ecb_datatype_Int64_key_32_iv_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_ecb_datatype_Float32_key_32_iv_None = r"""'A11ED1B75CF1C04C6CA3A31E76627D4C'""" + +example_mode_aes_256_ecb_datatype_Float64_key_32_iv_None = r"""'464C85EB7DB36D95CF48A3431CC7B2BC'""" + +example_mode_aes_256_ecb_datatype_Decimal32_key_32_iv_None = r"""'988C793BD81036C1D05EC47F43851269'""" + +example_mode_aes_256_ecb_datatype_Decimal64_key_32_iv_None = r"""'50FFB9C104DBFF3F415F12BA73D6FF1C'""" + +example_mode_aes_256_ecb_datatype_Decimal128_key_32_iv_None = r"""'B04C40C085A262E3AA27F8E7F6831DCB217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_ecb_datatype_UUID_key_32_iv_None = r"""'6A36D74ACB38B95FA77BC757A7AB2C34217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_ecb_datatype_Date_key_32_iv_None = r"""'F1CFA361A9B08FC101F3A4707A3E04D2'""" + +example_mode_aes_256_ecb_datatype_DateTime_key_32_iv_None = r"""'D58178485CD1AE1C30F68383307B8BC5'""" + +example_mode_aes_256_ecb_datatype_DateTime64_key_32_iv_None = r"""'A19B65BCB740B2AF4D421CE1DEC43608'""" + +example_mode_aes_256_ecb_datatype_LowCardinality_key_32_iv_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_ecb_datatype_Array_key_32_iv_None = r"""'C4071E4FD44F004347EA9932326B7038'""" + +example_mode_aes_256_ecb_datatype_NULL_key_32_iv_None = r"""'\\N'""" + +example_mode_aes_256_ecb_datatype_IPv4_key_32_iv_None = r"""'6C7950041CB4041D4D8036FCD22E3B06'""" + +example_mode_aes_256_ecb_datatype_IPv6_key_32_iv_None = r"""'8CBF2DC164F4086B8DD14B75E3065621217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_ecb_datatype_Enum8_key_32_iv_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_ecb_datatype_Enum16_key_32_iv_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_ecb_datatype_bytes_key_64_iv_None = r"""'2600697679EF0B3989C4EA3C0323CB8B'""" + +example_mode_aes_256_ecb_datatype_emptystring_key_64_iv_None = r"""'1F788FE6D86C317549697FBF0C07FA43'""" + +example_mode_aes_256_ecb_datatype_utf8string_key_64_iv_None = r"""'AB49D11AECD4A57A8BB4155C9F6733FBC9E3E51C40CDF6FA420050B461F48FA7'""" + +example_mode_aes_256_ecb_datatype_utf8fixedstring_key_64_iv_None = r"""'AB49D11AECD4A57A8BB4155C9F6733FBC9E3E51C40CDF6FA420050B461F48FA7'""" + +example_mode_aes_256_ecb_datatype_String_key_64_iv_None = r"""'7492B9A2D0E86DAF1DBCAEDBAD9E3D7E'""" + +example_mode_aes_256_ecb_datatype_FixedString_key_64_iv_None = r"""'7492B9A2D0E86DAF1DBCAEDBAD9E3D7E'""" + +example_mode_aes_256_ecb_datatype_UInt8_key_64_iv_None = r"""'FE2DE0EEF32A0510DC312ED77D1293EB'""" + +example_mode_aes_256_ecb_datatype_UInt16_key_64_iv_None = r"""'6805EDF8559E85ECBC4CA0AC3E241CB5'""" + +example_mode_aes_256_ecb_datatype_UInt32_key_64_iv_None = r"""'B2E6C9CE7EB187B7F56E754587C6BDBE'""" + +example_mode_aes_256_ecb_datatype_UInt64_key_64_iv_None = r"""'4F805F6D67E44124754951AEC9FDCEF3'""" + +example_mode_aes_256_ecb_datatype_Int8_key_64_iv_None = r"""'FE2DE0EEF32A0510DC312ED77D1293EB'""" + +example_mode_aes_256_ecb_datatype_Int16_key_64_iv_None = r"""'6805EDF8559E85ECBC4CA0AC3E241CB5'""" + +example_mode_aes_256_ecb_datatype_Int32_key_64_iv_None = r"""'B2E6C9CE7EB187B7F56E754587C6BDBE'""" + +example_mode_aes_256_ecb_datatype_Int64_key_64_iv_None = r"""'4F805F6D67E44124754951AEC9FDCEF3'""" + +example_mode_aes_256_ecb_datatype_Float32_key_64_iv_None = r"""'8014D3F1CFF0B7A66AE06FBDFA006FA6'""" + +example_mode_aes_256_ecb_datatype_Float64_key_64_iv_None = r"""'2C734E3A5A82C65E8918FA329B936114'""" + +example_mode_aes_256_ecb_datatype_Decimal32_key_64_iv_None = r"""'95245BC3292B4749E8CC7B5FDD26CAD9'""" + +example_mode_aes_256_ecb_datatype_Decimal64_key_64_iv_None = r"""'443682D0541F666078718D4790C3CE4E'""" + +example_mode_aes_256_ecb_datatype_Decimal128_key_64_iv_None = r"""'BE7F01084B171062CB4CCCCF9BB77D671F788FE6D86C317549697FBF0C07FA43'""" + +example_mode_aes_256_ecb_datatype_UUID_key_64_iv_None = r"""'65F2B1003C30A2E7148652BA06EF09FC1F788FE6D86C317549697FBF0C07FA43'""" + +example_mode_aes_256_ecb_datatype_Date_key_64_iv_None = r"""'D72724C85F90712153FC49FB33432644'""" + +example_mode_aes_256_ecb_datatype_DateTime_key_64_iv_None = r"""'B734143D37365E5C3325E0396BABC2AB'""" + +example_mode_aes_256_ecb_datatype_DateTime64_key_64_iv_None = r"""'78DDC273BE606E8F546538FC02508360'""" + +example_mode_aes_256_ecb_datatype_LowCardinality_key_64_iv_None = r"""'7492B9A2D0E86DAF1DBCAEDBAD9E3D7E'""" + +example_mode_aes_256_ecb_datatype_Array_key_64_iv_None = r"""'592C364BE5AEBE911096DEB1F6C75AB9'""" + +example_mode_aes_256_ecb_datatype_NULL_key_64_iv_None = r"""'\\N'""" + +example_mode_aes_256_ecb_datatype_IPv4_key_64_iv_None = r"""'F5401B3B979784C3FF4C86DD726872F9'""" + +example_mode_aes_256_ecb_datatype_IPv6_key_64_iv_None = r"""'3F810185F7D07B5E5A897E96BCA930ED1F788FE6D86C317549697FBF0C07FA43'""" + +example_mode_aes_256_ecb_datatype_Enum8_key_64_iv_None = r"""'FE2DE0EEF32A0510DC312ED77D1293EB'""" + +example_mode_aes_256_ecb_datatype_Enum16_key_64_iv_None = r"""'6805EDF8559E85ECBC4CA0AC3E241CB5'""" + +example_mode_aes_128_cbc_datatype_bytes_key_16_iv_None = r"""'7B62A15720E13327948BF706B89CF2BE'""" + +example_mode_aes_128_cbc_datatype_emptystring_key_16_iv_None = r"""'F30C69C4F945E654EBD4B388B1C8F790'""" + +example_mode_aes_128_cbc_datatype_utf8string_key_16_iv_None = r"""'4F5025E938F215F36B4EFEBD8A328B5C00AF476CA1EE03B7C0D297C2BF287339'""" + +example_mode_aes_128_cbc_datatype_utf8fixedstring_key_16_iv_None = r"""'4F5025E938F215F36B4EFEBD8A328B5C00AF476CA1EE03B7C0D297C2BF287339'""" + +example_mode_aes_128_cbc_datatype_String_key_16_iv_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_cbc_datatype_FixedString_key_16_iv_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_cbc_datatype_UInt8_key_16_iv_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_cbc_datatype_UInt16_key_16_iv_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_cbc_datatype_UInt32_key_16_iv_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_cbc_datatype_UInt64_key_16_iv_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_cbc_datatype_Int8_key_16_iv_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_cbc_datatype_Int16_key_16_iv_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_128_cbc_datatype_Int32_key_16_iv_None = r"""'E86C0858C6D9CCD970BA6DC320038306'""" + +example_mode_aes_128_cbc_datatype_Int64_key_16_iv_None = r"""'2D43D83E0250AE8AC4403551B639F694'""" + +example_mode_aes_128_cbc_datatype_Float32_key_16_iv_None = r"""'FF4D70D9A1050E6BBDD0325FC45CC22D'""" + +example_mode_aes_128_cbc_datatype_Float64_key_16_iv_None = r"""'75FE6B4A722A31D7760680CC1B9F131D'""" + +example_mode_aes_128_cbc_datatype_Decimal32_key_16_iv_None = r"""'83BBD7CCE7E5A38071653870475D48D2'""" + +example_mode_aes_128_cbc_datatype_Decimal64_key_16_iv_None = r"""'BE0DD9302B2952CE9CC3721DD85C8E66'""" + +example_mode_aes_128_cbc_datatype_Decimal128_key_16_iv_None = r"""'5F3DBFA74809E45E03980357B26787AF0D55B905F5525D3F5916FF811D8A6E7E'""" + +example_mode_aes_128_cbc_datatype_UUID_key_16_iv_None = r"""'FF9161B222B4A67481271035745F06D991B6833DF67CBA9BC6E1AAEADBE363BB'""" + +example_mode_aes_128_cbc_datatype_Date_key_16_iv_None = r"""'1E4FBE33752D96D147E890C29A409BFE'""" + +example_mode_aes_128_cbc_datatype_DateTime_key_16_iv_None = r"""'384F3D97B78D52C73CD06C0E1B6DE399'""" + +example_mode_aes_128_cbc_datatype_DateTime64_key_16_iv_None = r"""'C7F50A2D0175F3ED280AD42FF01FF5F2'""" + +example_mode_aes_128_cbc_datatype_LowCardinality_key_16_iv_None = r"""'7C51909F95C1E9B886A3487CD3EBED69'""" + +example_mode_aes_128_cbc_datatype_Array_key_16_iv_None = r"""'D9152D05CFA9E162983A5A2E883109B4'""" + +example_mode_aes_128_cbc_datatype_NULL_key_16_iv_None = r"""'\\N'""" + +example_mode_aes_128_cbc_datatype_IPv4_key_16_iv_None = r"""'4F32782638C1F33C6A7202CA83F0C12C'""" + +example_mode_aes_128_cbc_datatype_IPv6_key_16_iv_None = r"""'F54700FF04ADAD342BA6830DB12AD7E9B1B4BE8B15BAE0B2C9196D69E3D53C6C'""" + +example_mode_aes_128_cbc_datatype_Enum8_key_16_iv_None = r"""'4CDF8A192A06AC6EDBDCE2BFB53B7D73'""" + +example_mode_aes_128_cbc_datatype_Enum16_key_16_iv_None = r"""'12FB5B75B1CB5DF0DC70D8039758691D'""" + +example_mode_aes_192_cbc_datatype_bytes_key_24_iv_None = r"""'C60D7A90C41260E3CD03422E9163144A'""" + +example_mode_aes_192_cbc_datatype_emptystring_key_24_iv_None = r"""'D8ED6FC305C161EFCF57A383DAF31A83'""" + +example_mode_aes_192_cbc_datatype_utf8string_key_24_iv_None = r"""'7C1CE735A57407291267928DE0E2E47918C12F093BCD530F69669FC25B23195A'""" + +example_mode_aes_192_cbc_datatype_utf8fixedstring_key_24_iv_None = r"""'7C1CE735A57407291267928DE0E2E47918C12F093BCD530F69669FC25B23195A'""" + +example_mode_aes_192_cbc_datatype_String_key_24_iv_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_cbc_datatype_FixedString_key_24_iv_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_cbc_datatype_UInt8_key_24_iv_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_cbc_datatype_UInt16_key_24_iv_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_cbc_datatype_UInt32_key_24_iv_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_cbc_datatype_UInt64_key_24_iv_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_cbc_datatype_Int8_key_24_iv_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_cbc_datatype_Int16_key_24_iv_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_192_cbc_datatype_Int32_key_24_iv_None = r"""'9F32F3F6B3C3B1830F56B5B94C93875D'""" + +example_mode_aes_192_cbc_datatype_Int64_key_24_iv_None = r"""'8DE807D54B7717BFC773567D9FFE292D'""" + +example_mode_aes_192_cbc_datatype_Float32_key_24_iv_None = r"""'4E0C122631ED64EAD726833291A81878'""" + +example_mode_aes_192_cbc_datatype_Float64_key_24_iv_None = r"""'3F723599278E22E4692CE7D7D5F9A12F'""" + +example_mode_aes_192_cbc_datatype_Decimal32_key_24_iv_None = r"""'2420D49DBAA5CEF7D853C98DA1BD33BF'""" + +example_mode_aes_192_cbc_datatype_Decimal64_key_24_iv_None = r"""'FDF594113FCC2776653ED109A51FADF1'""" + +example_mode_aes_192_cbc_datatype_Decimal128_key_24_iv_None = r"""'79207931793E374FB5A3A2AC1ECA857A583603B3047000A843425EECA4C35311'""" + +example_mode_aes_192_cbc_datatype_UUID_key_24_iv_None = r"""'9FDB738E78D0D2F774C484ED82A854E46B580C61DBE08478DC523DA6AD605078'""" + +example_mode_aes_192_cbc_datatype_Date_key_24_iv_None = r"""'2CDD4685168FA3E2A7FA2092E86F44D4'""" + +example_mode_aes_192_cbc_datatype_DateTime_key_24_iv_None = r"""'A4BEE097872E44FAD94D6707D6643DF5'""" + +example_mode_aes_192_cbc_datatype_DateTime64_key_24_iv_None = r"""'1798B23C09F783623943560DF142E0F3'""" + +example_mode_aes_192_cbc_datatype_LowCardinality_key_24_iv_None = r"""'1AE38A541D466EDFED572EE839B0907F'""" + +example_mode_aes_192_cbc_datatype_Array_key_24_iv_None = r"""'7C0B9021CAF2CBBB06DBF589740DCC65'""" + +example_mode_aes_192_cbc_datatype_NULL_key_24_iv_None = r"""'\\N'""" + +example_mode_aes_192_cbc_datatype_IPv4_key_24_iv_None = r"""'B20465C932A0719BA04E2F76371510D8'""" + +example_mode_aes_192_cbc_datatype_IPv6_key_24_iv_None = r"""'CCCDC9B9C3F182254591DFEDDCE9F2326879326F3973401A6293A92BCB8EDFC4'""" + +example_mode_aes_192_cbc_datatype_Enum8_key_24_iv_None = r"""'01CC3C67F07C3FA6E5EFB7AE5F19130B'""" + +example_mode_aes_192_cbc_datatype_Enum16_key_24_iv_None = r"""'B50A3019F16B9C643FB40259E4B09308'""" + +example_mode_aes_256_cbc_datatype_bytes_key_32_iv_None = r"""'B73CDD4E7705F0C516612F860715EBE3'""" + +example_mode_aes_256_cbc_datatype_emptystring_key_32_iv_None = r"""'217E121CBD32CEC1F6FD3EBDF414BC34'""" + +example_mode_aes_256_cbc_datatype_utf8string_key_32_iv_None = r"""'3303F819796DDB5046AAFAB8A39FC3ABC19300E0966158A167939EDD20D39907'""" + +example_mode_aes_256_cbc_datatype_utf8fixedstring_key_32_iv_None = r"""'3303F819796DDB5046AAFAB8A39FC3ABC19300E0966158A167939EDD20D39907'""" + +example_mode_aes_256_cbc_datatype_String_key_32_iv_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_cbc_datatype_FixedString_key_32_iv_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_cbc_datatype_UInt8_key_32_iv_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_cbc_datatype_UInt16_key_32_iv_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_cbc_datatype_UInt32_key_32_iv_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_cbc_datatype_UInt64_key_32_iv_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_cbc_datatype_Int8_key_32_iv_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_cbc_datatype_Int16_key_32_iv_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_256_cbc_datatype_Int32_key_32_iv_None = r"""'D8876CDF9B97DD110E780F958C1EA2AA'""" + +example_mode_aes_256_cbc_datatype_Int64_key_32_iv_None = r"""'F6E11A48B6D830F7B8D0817885C05D3C'""" + +example_mode_aes_256_cbc_datatype_Float32_key_32_iv_None = r"""'A11ED1B75CF1C04C6CA3A31E76627D4C'""" + +example_mode_aes_256_cbc_datatype_Float64_key_32_iv_None = r"""'464C85EB7DB36D95CF48A3431CC7B2BC'""" + +example_mode_aes_256_cbc_datatype_Decimal32_key_32_iv_None = r"""'988C793BD81036C1D05EC47F43851269'""" + +example_mode_aes_256_cbc_datatype_Decimal64_key_32_iv_None = r"""'50FFB9C104DBFF3F415F12BA73D6FF1C'""" + +example_mode_aes_256_cbc_datatype_Decimal128_key_32_iv_None = r"""'B04C40C085A262E3AA27F8E7F6831DCB36585C228B0286E7A8D8DBAF754C4C38'""" + +example_mode_aes_256_cbc_datatype_UUID_key_32_iv_None = r"""'6A36D74ACB38B95FA77BC757A7AB2C3428548E6132D69A22B320775A21ABA11F'""" + +example_mode_aes_256_cbc_datatype_Date_key_32_iv_None = r"""'F1CFA361A9B08FC101F3A4707A3E04D2'""" + +example_mode_aes_256_cbc_datatype_DateTime_key_32_iv_None = r"""'D58178485CD1AE1C30F68383307B8BC5'""" + +example_mode_aes_256_cbc_datatype_DateTime64_key_32_iv_None = r"""'A19B65BCB740B2AF4D421CE1DEC43608'""" + +example_mode_aes_256_cbc_datatype_LowCardinality_key_32_iv_None = r"""'C91184ED1E67F0CDED89B097D5D3B130'""" + +example_mode_aes_256_cbc_datatype_Array_key_32_iv_None = r"""'C4071E4FD44F004347EA9932326B7038'""" + +example_mode_aes_256_cbc_datatype_NULL_key_32_iv_None = r"""'\\N'""" + +example_mode_aes_256_cbc_datatype_IPv4_key_32_iv_None = r"""'6C7950041CB4041D4D8036FCD22E3B06'""" + +example_mode_aes_256_cbc_datatype_IPv6_key_32_iv_None = r"""'8CBF2DC164F4086B8DD14B75E3065621393DE8421BAA5AE5E87096AEA7087507'""" + +example_mode_aes_256_cbc_datatype_Enum8_key_32_iv_None = r"""'3605C5E38A448F5FEFABADF3B9983FDF'""" + +example_mode_aes_256_cbc_datatype_Enum16_key_32_iv_None = r"""'2E5299C7A5672D8779BA9DDDE1DBCE00'""" + +example_mode_aes_128_cbc_datatype_bytes_key_16_iv_16 = r"""'CDA4B7027137998B9A33C2096C9A50DD'""" + +example_mode_aes_128_cbc_datatype_emptystring_key_16_iv_16 = r"""'56A77308430BA344FFBF016999795ED5'""" + +example_mode_aes_128_cbc_datatype_utf8string_key_16_iv_16 = r"""'0BD95BFF6DE2DC43D936DAF23937B06D602786A6770B627EB56BC7F681B1C9DB'""" + +example_mode_aes_128_cbc_datatype_utf8fixedstring_key_16_iv_16 = r"""'0BD95BFF6DE2DC43D936DAF23937B06D602786A6770B627EB56BC7F681B1C9DB'""" + +example_mode_aes_128_cbc_datatype_String_key_16_iv_16 = r"""'D017D171B3865D6EA347E14167261F41'""" + +example_mode_aes_128_cbc_datatype_FixedString_key_16_iv_16 = r"""'D017D171B3865D6EA347E14167261F41'""" + +example_mode_aes_128_cbc_datatype_UInt8_key_16_iv_16 = r"""'A5BD67663C14A01DC9AB3B5F7B0F9383'""" + +example_mode_aes_128_cbc_datatype_UInt16_key_16_iv_16 = r"""'02D98283BEADCA1AC6EF925F9BF86960'""" + +example_mode_aes_128_cbc_datatype_UInt32_key_16_iv_16 = r"""'E72BD2245C3B2B7474300D09DBD85F3F'""" + +example_mode_aes_128_cbc_datatype_UInt64_key_16_iv_16 = r"""'C9032C59328DEA2EE03ACDBEDFAE7475'""" + +example_mode_aes_128_cbc_datatype_Int8_key_16_iv_16 = r"""'A5BD67663C14A01DC9AB3B5F7B0F9383'""" + +example_mode_aes_128_cbc_datatype_Int16_key_16_iv_16 = r"""'02D98283BEADCA1AC6EF925F9BF86960'""" + +example_mode_aes_128_cbc_datatype_Int32_key_16_iv_16 = r"""'E72BD2245C3B2B7474300D09DBD85F3F'""" + +example_mode_aes_128_cbc_datatype_Int64_key_16_iv_16 = r"""'C9032C59328DEA2EE03ACDBEDFAE7475'""" + +example_mode_aes_128_cbc_datatype_Float32_key_16_iv_16 = r"""'A5425BDEB6B83E311C45249DAF3153F5'""" + +example_mode_aes_128_cbc_datatype_Float64_key_16_iv_16 = r"""'EEDA98EC4045C7D351F3905313073B79'""" + +example_mode_aes_128_cbc_datatype_Decimal32_key_16_iv_16 = r"""'52EBB74292ECD37A29E9809166CC77DB'""" + +example_mode_aes_128_cbc_datatype_Decimal64_key_16_iv_16 = r"""'95EF455767EC8FBD32BAAEFFB44FEEB7'""" + +example_mode_aes_128_cbc_datatype_Decimal128_key_16_iv_16 = r"""'94C066884FA09B0D3C750F20A2823304A2FE20B6B69AB18373E3F58623E0D7FB'""" + +example_mode_aes_128_cbc_datatype_UUID_key_16_iv_16 = r"""'1D909C15BB882E89AD68B1EFEAC72148DCD05E2303B6BE19007A945AFB778B42'""" + +example_mode_aes_128_cbc_datatype_Date_key_16_iv_16 = r"""'24A4F8CE8A9FAE48A0AFEB8A6203EFEA'""" + +example_mode_aes_128_cbc_datatype_DateTime_key_16_iv_16 = r"""'0DD5554819E3995B1B6B00362AEE9424'""" + +example_mode_aes_128_cbc_datatype_DateTime64_key_16_iv_16 = r"""'0E55319903957C9D1FDA4FB65C3871CB'""" + +example_mode_aes_128_cbc_datatype_LowCardinality_key_16_iv_16 = r"""'D017D171B3865D6EA347E14167261F41'""" + +example_mode_aes_128_cbc_datatype_Array_key_16_iv_16 = r"""'D53C82A5D13256B88DF41C1C1D924E40'""" + +example_mode_aes_128_cbc_datatype_NULL_key_16_iv_16 = r"""'\\N'""" + +example_mode_aes_128_cbc_datatype_IPv4_key_16_iv_16 = r"""'C0D81AAB3134EAB5B1F190958C6A29F9'""" + +example_mode_aes_128_cbc_datatype_IPv6_key_16_iv_16 = r"""'AE1A36F75C9BB387121445069A9968CA247FA4459ED3C8809089FEE334EB1EC7'""" + +example_mode_aes_128_cbc_datatype_Enum8_key_16_iv_16 = r"""'A5BD67663C14A01DC9AB3B5F7B0F9383'""" + +example_mode_aes_128_cbc_datatype_Enum16_key_16_iv_16 = r"""'02D98283BEADCA1AC6EF925F9BF86960'""" + +example_mode_aes_128_cbc_datatype_bytes_key_24_iv_24 = r"""'FD4D81969EDCB22A5B5DE4E21BDFE267'""" + +example_mode_aes_128_cbc_datatype_emptystring_key_24_iv_24 = r"""'BEFE724909AC17B32920D0400312227E'""" + +example_mode_aes_128_cbc_datatype_utf8string_key_24_iv_24 = r"""'A4FBBF549A9D453212CB69882DF12D34DAA659D7D176B78DE14F6AF36E0BE2C9'""" + +example_mode_aes_128_cbc_datatype_utf8fixedstring_key_24_iv_24 = r"""'A4FBBF549A9D453212CB69882DF12D34DAA659D7D176B78DE14F6AF36E0BE2C9'""" + +example_mode_aes_128_cbc_datatype_String_key_24_iv_24 = r"""'73C9874744984892250CCCEC8541D690'""" + +example_mode_aes_128_cbc_datatype_FixedString_key_24_iv_24 = r"""'73C9874744984892250CCCEC8541D690'""" + +example_mode_aes_128_cbc_datatype_UInt8_key_24_iv_24 = r"""'0E97B8F125240D96125B8AC23A798981'""" + +example_mode_aes_128_cbc_datatype_UInt16_key_24_iv_24 = r"""'9118F2C6B4E17730C62F85CB22E9A446'""" + +example_mode_aes_128_cbc_datatype_UInt32_key_24_iv_24 = r"""'2A189EB114A9487142573673C0F30929'""" + +example_mode_aes_128_cbc_datatype_UInt64_key_24_iv_24 = r"""'B6CA3E3C1C7830203FABD429664E81B3'""" + +example_mode_aes_128_cbc_datatype_Int8_key_24_iv_24 = r"""'0E97B8F125240D96125B8AC23A798981'""" + +example_mode_aes_128_cbc_datatype_Int16_key_24_iv_24 = r"""'9118F2C6B4E17730C62F85CB22E9A446'""" + +example_mode_aes_128_cbc_datatype_Int32_key_24_iv_24 = r"""'2A189EB114A9487142573673C0F30929'""" + +example_mode_aes_128_cbc_datatype_Int64_key_24_iv_24 = r"""'B6CA3E3C1C7830203FABD429664E81B3'""" + +example_mode_aes_128_cbc_datatype_Float32_key_24_iv_24 = r"""'79B44B6E26BC4BBACBBE312329F8C86D'""" + +example_mode_aes_128_cbc_datatype_Float64_key_24_iv_24 = r"""'C23864E616278AD006ED806C1EF89F71'""" + +example_mode_aes_128_cbc_datatype_Decimal32_key_24_iv_24 = r"""'4E14763E17CD954A2754768FF664CECE'""" + +example_mode_aes_128_cbc_datatype_Decimal64_key_24_iv_24 = r"""'A5D655F79EA638ECD6879A515853DD50'""" + +example_mode_aes_128_cbc_datatype_Decimal128_key_24_iv_24 = r"""'5C1C88BE7B68B91167BD317D3E6291A0D7920AEDA123237D0223EFB15F9C8ADC'""" + +example_mode_aes_128_cbc_datatype_UUID_key_24_iv_24 = r"""'ADF1CF6C8AF83CBD4BB3313E7E88FE8C377730890CC89DD91E95436ABC5E4F3A'""" + +example_mode_aes_128_cbc_datatype_Date_key_24_iv_24 = r"""'3127F80211A13ED02CED473BF2BFC28B'""" + +example_mode_aes_128_cbc_datatype_DateTime_key_24_iv_24 = r"""'A6B567CCFCE4F693BD6575D2D4DF498B'""" + +example_mode_aes_128_cbc_datatype_DateTime64_key_24_iv_24 = r"""'410F1A2A9E39722A1D0B549C3ADF9526'""" + +example_mode_aes_128_cbc_datatype_LowCardinality_key_24_iv_24 = r"""'73C9874744984892250CCCEC8541D690'""" + +example_mode_aes_128_cbc_datatype_Array_key_24_iv_24 = r"""'14A8202D2CACBE584BE9313C72F0478D'""" + +example_mode_aes_128_cbc_datatype_NULL_key_24_iv_24 = r"""'\\N'""" + +example_mode_aes_128_cbc_datatype_IPv4_key_24_iv_24 = r"""'9C554B2C04C5C9EAC0E7E36F1BD5CB80'""" + +example_mode_aes_128_cbc_datatype_IPv6_key_24_iv_24 = r"""'D548D293E5F41FE72A940A6A29EDBCDA8AAEBAC8BA27FB0B4DB65DAD63895B9F'""" + +example_mode_aes_128_cbc_datatype_Enum8_key_24_iv_24 = r"""'0E97B8F125240D96125B8AC23A798981'""" + +example_mode_aes_128_cbc_datatype_Enum16_key_24_iv_24 = r"""'9118F2C6B4E17730C62F85CB22E9A446'""" + +example_mode_aes_192_cbc_datatype_bytes_key_24_iv_16 = r"""'67771349942D4F812553F2E1E3FFB276'""" + +example_mode_aes_192_cbc_datatype_emptystring_key_24_iv_16 = r"""'62E9214DB5E239F0CAD31ADF26AB313F'""" + +example_mode_aes_192_cbc_datatype_utf8string_key_24_iv_16 = r"""'D619039D0956015C34336196DB3EB5A4710B2B8860344AB2625E9269C5E4A6CC'""" + +example_mode_aes_192_cbc_datatype_utf8fixedstring_key_24_iv_16 = r"""'D619039D0956015C34336196DB3EB5A4710B2B8860344AB2625E9269C5E4A6CC'""" + +example_mode_aes_192_cbc_datatype_String_key_24_iv_16 = r"""'A3DB45D129A5C9FDB5ED66E782B28BD2'""" + +example_mode_aes_192_cbc_datatype_FixedString_key_24_iv_16 = r"""'A3DB45D129A5C9FDB5ED66E782B28BD2'""" + +example_mode_aes_192_cbc_datatype_UInt8_key_24_iv_16 = r"""'F2A751470B32C58822F23B1417C11279'""" + +example_mode_aes_192_cbc_datatype_UInt16_key_24_iv_16 = r"""'CA1ECFEA89CF520D8FA14A38235E5FA5'""" + +example_mode_aes_192_cbc_datatype_UInt32_key_24_iv_16 = r"""'57F211370522621F23B59C8304878904'""" + +example_mode_aes_192_cbc_datatype_UInt64_key_24_iv_16 = r"""'DCF974CD88752B215284625F9164F5D4'""" + +example_mode_aes_192_cbc_datatype_Int8_key_24_iv_16 = r"""'F2A751470B32C58822F23B1417C11279'""" + +example_mode_aes_192_cbc_datatype_Int16_key_24_iv_16 = r"""'CA1ECFEA89CF520D8FA14A38235E5FA5'""" + +example_mode_aes_192_cbc_datatype_Int32_key_24_iv_16 = r"""'57F211370522621F23B59C8304878904'""" + +example_mode_aes_192_cbc_datatype_Int64_key_24_iv_16 = r"""'DCF974CD88752B215284625F9164F5D4'""" + +example_mode_aes_192_cbc_datatype_Float32_key_24_iv_16 = r"""'62EBE4FD1035D405BBD6C41436780E13'""" + +example_mode_aes_192_cbc_datatype_Float64_key_24_iv_16 = r"""'5706FC9892A4C1AB48FC93E13C9C72FE'""" + +example_mode_aes_192_cbc_datatype_Decimal32_key_24_iv_16 = r"""'BB056843D369A5E55982C92AD52EEC07'""" + +example_mode_aes_192_cbc_datatype_Decimal64_key_24_iv_16 = r"""'70ACD4156F9AC1444A75EFCB9202CA00'""" + +example_mode_aes_192_cbc_datatype_Decimal128_key_24_iv_16 = r"""'04748A45840A0CAAC83F139DB01C504B01FC56631A8B2FFBE68F2FC85B6FEEDE'""" + +example_mode_aes_192_cbc_datatype_UUID_key_24_iv_16 = r"""'D7B2ABC08F67823F61C3E8F680C12B3A8AA3E3711D412CB55ACFBC89C14949A8'""" + +example_mode_aes_192_cbc_datatype_Date_key_24_iv_16 = r"""'734BBE526E56B280E90E53DDEA7DB69B'""" + +example_mode_aes_192_cbc_datatype_DateTime_key_24_iv_16 = r"""'9B9BE7CC20F75DA3F39F688DE3A1ADAA'""" + +example_mode_aes_192_cbc_datatype_DateTime64_key_24_iv_16 = r"""'554FCAAF985378A561F7C6ED91E20C89'""" + +example_mode_aes_192_cbc_datatype_LowCardinality_key_24_iv_16 = r"""'A3DB45D129A5C9FDB5ED66E782B28BD2'""" + +example_mode_aes_192_cbc_datatype_Array_key_24_iv_16 = r"""'D85AF1078F110329896EFC462340171E'""" + +example_mode_aes_192_cbc_datatype_NULL_key_24_iv_16 = r"""'\\N'""" + +example_mode_aes_192_cbc_datatype_IPv4_key_24_iv_16 = r"""'6AF45078B1E924B6C107D4C0236EA937'""" + +example_mode_aes_192_cbc_datatype_IPv6_key_24_iv_16 = r"""'9E4F8E54B265A340090DC7FE4F53BB50048442F5632A7B1630AE80DFD938E9AA'""" + +example_mode_aes_192_cbc_datatype_Enum8_key_24_iv_16 = r"""'F2A751470B32C58822F23B1417C11279'""" + +example_mode_aes_192_cbc_datatype_Enum16_key_24_iv_16 = r"""'CA1ECFEA89CF520D8FA14A38235E5FA5'""" + +example_mode_aes_192_cbc_datatype_bytes_key_32_iv_32 = r"""'34FFC84A0D7CAD43F6BDB7F0C57CE511'""" + +example_mode_aes_192_cbc_datatype_emptystring_key_32_iv_32 = r"""'B9937ECA1C6EADCCA37CD7E8BA89D939'""" + +example_mode_aes_192_cbc_datatype_utf8string_key_32_iv_32 = r"""'00A1C691A68F5BF50BA973D72A5DBFBD579279FD953AE222F41FD0DAF48A3C90'""" + +example_mode_aes_192_cbc_datatype_utf8fixedstring_key_32_iv_32 = r"""'00A1C691A68F5BF50BA973D72A5DBFBD579279FD953AE222F41FD0DAF48A3C90'""" + +example_mode_aes_192_cbc_datatype_String_key_32_iv_32 = r"""'508551DA505F6538F90DC607423CFAD4'""" + +example_mode_aes_192_cbc_datatype_FixedString_key_32_iv_32 = r"""'508551DA505F6538F90DC607423CFAD4'""" + +example_mode_aes_192_cbc_datatype_UInt8_key_32_iv_32 = r"""'C2D3A1603DBAC305D8180F85A3830300'""" + +example_mode_aes_192_cbc_datatype_UInt16_key_32_iv_32 = r"""'A7EEA5561C4BB75015632824CAE9AC1E'""" + +example_mode_aes_192_cbc_datatype_UInt32_key_32_iv_32 = r"""'97EE21986E9A9F3A4F67851D05C93830'""" + +example_mode_aes_192_cbc_datatype_UInt64_key_32_iv_32 = r"""'474F106EAC813E4B432B2CA58D11F0E1'""" + +example_mode_aes_192_cbc_datatype_Int8_key_32_iv_32 = r"""'C2D3A1603DBAC305D8180F85A3830300'""" + +example_mode_aes_192_cbc_datatype_Int16_key_32_iv_32 = r"""'A7EEA5561C4BB75015632824CAE9AC1E'""" + +example_mode_aes_192_cbc_datatype_Int32_key_32_iv_32 = r"""'97EE21986E9A9F3A4F67851D05C93830'""" + +example_mode_aes_192_cbc_datatype_Int64_key_32_iv_32 = r"""'474F106EAC813E4B432B2CA58D11F0E1'""" + +example_mode_aes_192_cbc_datatype_Float32_key_32_iv_32 = r"""'3C76B134B1FF458CBE4430DCDF3EE8F9'""" + +example_mode_aes_192_cbc_datatype_Float64_key_32_iv_32 = r"""'0C2C54B82F3B7A889696EACBBBC5AA96'""" + +example_mode_aes_192_cbc_datatype_Decimal32_key_32_iv_32 = r"""'A038F21DD2F3C6FE3382C849E6BCC6B8'""" + +example_mode_aes_192_cbc_datatype_Decimal64_key_32_iv_32 = r"""'CE5E44EF4BAC3455C1FE57C82498FC32'""" + +example_mode_aes_192_cbc_datatype_Decimal128_key_32_iv_32 = r"""'DF51ED3C3BE2A229F07C4C9BB64902DD9383D2999BDD4E3FEFE023671B90CA88'""" + +example_mode_aes_192_cbc_datatype_UUID_key_32_iv_32 = r"""'B4A04107B9945309227C08EEE5516367F2C64F1587FEB4720620AA04485CA18C'""" + +example_mode_aes_192_cbc_datatype_Date_key_32_iv_32 = r"""'60253CED16ECAB1D34A5CC131E55B769'""" + +example_mode_aes_192_cbc_datatype_DateTime_key_32_iv_32 = r"""'99C6EE6DF7A06D02FE4D60018784F129'""" + +example_mode_aes_192_cbc_datatype_DateTime64_key_32_iv_32 = r"""'C0329F9CB5CFDDDEA74275618A099FAE'""" + +example_mode_aes_192_cbc_datatype_LowCardinality_key_32_iv_32 = r"""'508551DA505F6538F90DC607423CFAD4'""" + +example_mode_aes_192_cbc_datatype_Array_key_32_iv_32 = r"""'53161975B2A33AAFB8379B39B255D45A'""" + +example_mode_aes_192_cbc_datatype_NULL_key_32_iv_32 = r"""'\\N'""" + +example_mode_aes_192_cbc_datatype_IPv4_key_32_iv_32 = r"""'2B2ACE9C2914F3F2F9E206D25CD29429'""" + +example_mode_aes_192_cbc_datatype_IPv6_key_32_iv_32 = r"""'16AF2466967A2705EC5D588C0C155457A3EC9F82C1C5FB98CE86AC5697EF7223'""" + +example_mode_aes_192_cbc_datatype_Enum8_key_32_iv_32 = r"""'C2D3A1603DBAC305D8180F85A3830300'""" + +example_mode_aes_192_cbc_datatype_Enum16_key_32_iv_32 = r"""'A7EEA5561C4BB75015632824CAE9AC1E'""" + +example_mode_aes_256_cbc_datatype_bytes_key_32_iv_16 = r"""'6046ECF8094941C6DEC9278FF6F137E9'""" + +example_mode_aes_256_cbc_datatype_emptystring_key_32_iv_16 = r"""'4EC7785DA650D55B71B52816B1DB5AD3'""" + +example_mode_aes_256_cbc_datatype_utf8string_key_32_iv_16 = r"""'A7663A9F621A26398B51DFBC099A6FA09032C25FE48CB9D2DE29A8DFD581714D'""" + +example_mode_aes_256_cbc_datatype_utf8fixedstring_key_32_iv_16 = r"""'A7663A9F621A26398B51DFBC099A6FA09032C25FE48CB9D2DE29A8DFD581714D'""" + +example_mode_aes_256_cbc_datatype_String_key_32_iv_16 = r"""'5E22454D9AC4F1A47B04E2FD98A76140'""" + +example_mode_aes_256_cbc_datatype_FixedString_key_32_iv_16 = r"""'5E22454D9AC4F1A47B04E2FD98A76140'""" + +example_mode_aes_256_cbc_datatype_UInt8_key_32_iv_16 = r"""'FE35EEF14D6AA67AA2EBA474253CA19A'""" + +example_mode_aes_256_cbc_datatype_UInt16_key_32_iv_16 = r"""'2D22C6B58140E591BEF7986C7770FF21'""" + +example_mode_aes_256_cbc_datatype_UInt32_key_32_iv_16 = r"""'4EB4923E19AA24206B135D5B25CB31AB'""" + +example_mode_aes_256_cbc_datatype_UInt64_key_32_iv_16 = r"""'173B7CAFFCBED9B814C0ECD50A9477F6'""" + +example_mode_aes_256_cbc_datatype_Int8_key_32_iv_16 = r"""'FE35EEF14D6AA67AA2EBA474253CA19A'""" + +example_mode_aes_256_cbc_datatype_Int16_key_32_iv_16 = r"""'2D22C6B58140E591BEF7986C7770FF21'""" + +example_mode_aes_256_cbc_datatype_Int32_key_32_iv_16 = r"""'4EB4923E19AA24206B135D5B25CB31AB'""" + +example_mode_aes_256_cbc_datatype_Int64_key_32_iv_16 = r"""'173B7CAFFCBED9B814C0ECD50A9477F6'""" + +example_mode_aes_256_cbc_datatype_Float32_key_32_iv_16 = r"""'E639AA3E45D8C2759181FD736CD58EDC'""" + +example_mode_aes_256_cbc_datatype_Float64_key_32_iv_16 = r"""'CFEF3FDC054997559DF5DCFB5F215B58'""" + +example_mode_aes_256_cbc_datatype_Decimal32_key_32_iv_16 = r"""'E2F57A092A1759D39F4AE67C9543FAB8'""" + +example_mode_aes_256_cbc_datatype_Decimal64_key_32_iv_16 = r"""'6259A2CFD3D83352A44C03DB050077B3'""" + +example_mode_aes_256_cbc_datatype_Decimal128_key_32_iv_16 = r"""'AEC71CA2D87098392689F9EB2ED93A84FA5787E643E28CB3C2013F8FCC24E387'""" + +example_mode_aes_256_cbc_datatype_UUID_key_32_iv_16 = r"""'88BA86B14A468DC92084B7152B172E142D88CBFB639A8FF2F480F1727972251C'""" + +example_mode_aes_256_cbc_datatype_Date_key_32_iv_16 = r"""'C67C84B1C6BF4527A7E730499FF39C86'""" + +example_mode_aes_256_cbc_datatype_DateTime_key_32_iv_16 = r"""'7FDC1B0797A5F3C04CDA82729A1EA4AA'""" + +example_mode_aes_256_cbc_datatype_DateTime64_key_32_iv_16 = r"""'B1B7401FB2B65BCB3448C1BE179F6AA6'""" + +example_mode_aes_256_cbc_datatype_LowCardinality_key_32_iv_16 = r"""'5E22454D9AC4F1A47B04E2FD98A76140'""" + +example_mode_aes_256_cbc_datatype_Array_key_32_iv_16 = r"""'6BB1E8429CC612B0AA74282B81D4FE8A'""" + +example_mode_aes_256_cbc_datatype_NULL_key_32_iv_16 = r"""'\\N'""" + +example_mode_aes_256_cbc_datatype_IPv4_key_32_iv_16 = r"""'51364C8DC6882CA1F03CF7FB45117EEF'""" + +example_mode_aes_256_cbc_datatype_IPv6_key_32_iv_16 = r"""'87A1C4D4672EFE64DC98E040EAD6B3126C899C263577B3D8EE8A3952BE5CDC1B'""" + +example_mode_aes_256_cbc_datatype_Enum8_key_32_iv_16 = r"""'FE35EEF14D6AA67AA2EBA474253CA19A'""" + +example_mode_aes_256_cbc_datatype_Enum16_key_32_iv_16 = r"""'2D22C6B58140E591BEF7986C7770FF21'""" + +example_mode_aes_256_cbc_datatype_bytes_key_64_iv_64 = r"""'BC91CDADF0BD3A8E9FF014B1E494FE7E'""" + +example_mode_aes_256_cbc_datatype_emptystring_key_64_iv_64 = r"""'A574B24357860C4B9C580D4CBE6C22C7'""" + +example_mode_aes_256_cbc_datatype_utf8string_key_64_iv_64 = r"""'3DA0F0C80E8067C1A0B5C82EF7A2D12A1EAD3C258827367A46D13D522B4D85DC'""" + +example_mode_aes_256_cbc_datatype_utf8fixedstring_key_64_iv_64 = r"""'3DA0F0C80E8067C1A0B5C82EF7A2D12A1EAD3C258827367A46D13D522B4D85DC'""" + +example_mode_aes_256_cbc_datatype_String_key_64_iv_64 = r"""'4CE9C9AFDC1E1E1EF2D1F4C141CE1874'""" + +example_mode_aes_256_cbc_datatype_FixedString_key_64_iv_64 = r"""'4CE9C9AFDC1E1E1EF2D1F4C141CE1874'""" + +example_mode_aes_256_cbc_datatype_UInt8_key_64_iv_64 = r"""'89E81101BF6A6E1DBEB6C0EAA67BEF14'""" + +example_mode_aes_256_cbc_datatype_UInt16_key_64_iv_64 = r"""'3124E8A62599D6C8A0C16A7ED0F01C58'""" + +example_mode_aes_256_cbc_datatype_UInt32_key_64_iv_64 = r"""'DD8866C2B2E443073D2F7B5A6FEE92E9'""" + +example_mode_aes_256_cbc_datatype_UInt64_key_64_iv_64 = r"""'C45A6D56D06F8A3A6A753D9BADBE88A4'""" + +example_mode_aes_256_cbc_datatype_Int8_key_64_iv_64 = r"""'89E81101BF6A6E1DBEB6C0EAA67BEF14'""" + +example_mode_aes_256_cbc_datatype_Int16_key_64_iv_64 = r"""'3124E8A62599D6C8A0C16A7ED0F01C58'""" + +example_mode_aes_256_cbc_datatype_Int32_key_64_iv_64 = r"""'DD8866C2B2E443073D2F7B5A6FEE92E9'""" + +example_mode_aes_256_cbc_datatype_Int64_key_64_iv_64 = r"""'C45A6D56D06F8A3A6A753D9BADBE88A4'""" + +example_mode_aes_256_cbc_datatype_Float32_key_64_iv_64 = r"""'66778B8F543C2C8402865CFB41187A09'""" + +example_mode_aes_256_cbc_datatype_Float64_key_64_iv_64 = r"""'F05D98EC189434B1F9177D0915FE939F'""" + +example_mode_aes_256_cbc_datatype_Decimal32_key_64_iv_64 = r"""'81190FEC7ADDBD8B919DD85AD71FF0B3'""" + +example_mode_aes_256_cbc_datatype_Decimal64_key_64_iv_64 = r"""'AB8B69F837890ED5684B071FF2359BED'""" + +example_mode_aes_256_cbc_datatype_Decimal128_key_64_iv_64 = r"""'765BDA3E9FA6177EB1839155F9C55A856315AB458B1DC334DF821A6FD768BFC8'""" + +example_mode_aes_256_cbc_datatype_UUID_key_64_iv_64 = r"""'99588BF38D9D6A2ABBEBD22E02F4D0F1CD5787AF78FE18679B8CF02A53D2600E'""" + +example_mode_aes_256_cbc_datatype_Date_key_64_iv_64 = r"""'ABEEA4FD2E771A3936EB591F674E4CDA'""" + +example_mode_aes_256_cbc_datatype_DateTime_key_64_iv_64 = r"""'C7EC851BCD2FBA03E58AC9F027E06444'""" + +example_mode_aes_256_cbc_datatype_DateTime64_key_64_iv_64 = r"""'53D42045D02E68F0D565EB8E093C83B8'""" + +example_mode_aes_256_cbc_datatype_LowCardinality_key_64_iv_64 = r"""'4CE9C9AFDC1E1E1EF2D1F4C141CE1874'""" + +example_mode_aes_256_cbc_datatype_Array_key_64_iv_64 = r"""'5A3063DCD98C7C59B41068F26DA0CC52'""" + +example_mode_aes_256_cbc_datatype_NULL_key_64_iv_64 = r"""'\\N'""" + +example_mode_aes_256_cbc_datatype_IPv4_key_64_iv_64 = r"""'1F12AE92E01133B2BB3F4F5AC6041FC9'""" + +example_mode_aes_256_cbc_datatype_IPv6_key_64_iv_64 = r"""'32406EB5546AECA4EE235A586577B14A5F6070FE879D6E07E6431748E3C2E80F'""" + +example_mode_aes_256_cbc_datatype_Enum8_key_64_iv_64 = r"""'89E81101BF6A6E1DBEB6C0EAA67BEF14'""" + +example_mode_aes_256_cbc_datatype_Enum16_key_64_iv_64 = r"""'3124E8A62599D6C8A0C16A7ED0F01C58'""" + +example_mode_aes_128_cfb1_datatype_bytes_key_16_iv_None = r"""'00'""" + +example_mode_aes_128_cfb1_datatype_emptystring_key_16_iv_None = r"""''""" + +example_mode_aes_128_cfb1_datatype_utf8string_key_16_iv_None = r"""'5BA033EA7B9901874F4F863E229069EBA414B7C317D37DF0'""" + +example_mode_aes_128_cfb1_datatype_utf8fixedstring_key_16_iv_None = r"""'5BA033EA7B9901874F4F863E229069EBA414B7C317D37DF0'""" + +example_mode_aes_128_cfb1_datatype_String_key_16_iv_None = r"""'32'""" + +example_mode_aes_128_cfb1_datatype_FixedString_key_16_iv_None = r"""'32'""" + +example_mode_aes_128_cfb1_datatype_UInt8_key_16_iv_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_UInt16_key_16_iv_None = r"""'0173'""" + +example_mode_aes_128_cfb1_datatype_UInt32_key_16_iv_None = r"""'01732E6B'""" + +example_mode_aes_128_cfb1_datatype_UInt64_key_16_iv_None = r"""'01732E6B82FCBDF6'""" + +example_mode_aes_128_cfb1_datatype_Int8_key_16_iv_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Int16_key_16_iv_None = r"""'0173'""" + +example_mode_aes_128_cfb1_datatype_Int32_key_16_iv_None = r"""'01732E6B'""" + +example_mode_aes_128_cfb1_datatype_Int64_key_16_iv_None = r"""'01732E6B82FCBDF6'""" + +example_mode_aes_128_cfb1_datatype_Float32_key_16_iv_None = r"""'0000B9AB'""" + +example_mode_aes_128_cfb1_datatype_Float64_key_16_iv_None = r"""'000000000000FFF6'""" + +example_mode_aes_128_cfb1_datatype_Decimal32_key_16_iv_None = r"""'2E09CA6A'""" + +example_mode_aes_128_cfb1_datatype_Decimal64_key_16_iv_None = r"""'2E09CA6A6DBEE799'""" + +example_mode_aes_128_cfb1_datatype_Decimal128_key_16_iv_None = r"""'2E09CA6A6DBEE79923BA65C6B78FD199'""" + +example_mode_aes_128_cfb1_datatype_UUID_key_16_iv_None = r"""'E590DFB515D3A518F85C66A6A5EC9C6E'""" + +example_mode_aes_128_cfb1_datatype_Date_key_16_iv_None = r"""'42F0'""" + +example_mode_aes_128_cfb1_datatype_DateTime_key_16_iv_None = r"""'5475EC3D'""" + +example_mode_aes_128_cfb1_datatype_DateTime64_key_16_iv_None = r"""'21CDF1128AE44A37'""" + +example_mode_aes_128_cfb1_datatype_LowCardinality_key_16_iv_None = r"""'32'""" + +example_mode_aes_128_cfb1_datatype_Array_key_16_iv_None = r"""'0170'""" + +example_mode_aes_128_cfb1_datatype_NULL_key_16_iv_None = r"""'\\N'""" + +example_mode_aes_128_cfb1_datatype_IPv4_key_16_iv_None = r"""'240A9E43'""" + +example_mode_aes_128_cfb1_datatype_IPv6_key_16_iv_None = r"""'2E642EF4B07D9B1251BE3B3CBDBCC6F6'""" + +example_mode_aes_128_cfb1_datatype_Enum8_key_16_iv_None = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Enum16_key_16_iv_None = r"""'0173'""" + +example_mode_aes_192_cfb1_datatype_bytes_key_24_iv_None = r"""'00'""" + +example_mode_aes_192_cfb1_datatype_emptystring_key_24_iv_None = r"""''""" + +example_mode_aes_192_cfb1_datatype_utf8string_key_24_iv_None = r"""'7B69A097857549357D008DB662730D8735DE1673D9E8CFF9'""" + +example_mode_aes_192_cfb1_datatype_utf8fixedstring_key_24_iv_None = r"""'7B69A097857549357D008DB662730D8735DE1673D9E8CFF9'""" + +example_mode_aes_192_cfb1_datatype_String_key_24_iv_None = r"""'23'""" + +example_mode_aes_192_cfb1_datatype_FixedString_key_24_iv_None = r"""'23'""" + +example_mode_aes_192_cfb1_datatype_UInt8_key_24_iv_None = r"""'01'""" + +example_mode_aes_192_cfb1_datatype_UInt16_key_24_iv_None = r"""'01F9'""" + +example_mode_aes_192_cfb1_datatype_UInt32_key_24_iv_None = r"""'01F92AD3'""" + +example_mode_aes_192_cfb1_datatype_UInt64_key_24_iv_None = r"""'01F92AD38CB10028'""" + +example_mode_aes_192_cfb1_datatype_Int8_key_24_iv_None = r"""'01'""" + +example_mode_aes_192_cfb1_datatype_Int16_key_24_iv_None = r"""'01F9'""" + +example_mode_aes_192_cfb1_datatype_Int32_key_24_iv_None = r"""'01F92AD3'""" + +example_mode_aes_192_cfb1_datatype_Int64_key_24_iv_None = r"""'01F92AD38CB10028'""" + +example_mode_aes_192_cfb1_datatype_Float32_key_24_iv_None = r"""'0000FCAE'""" + +example_mode_aes_192_cfb1_datatype_Float64_key_24_iv_None = r"""'000000000000A79C'""" + +example_mode_aes_192_cfb1_datatype_Decimal32_key_24_iv_None = r"""'3F406C3F'""" + +example_mode_aes_192_cfb1_datatype_Decimal64_key_24_iv_None = r"""'3F406C3F3A41B134'""" + +example_mode_aes_192_cfb1_datatype_Decimal128_key_24_iv_None = r"""'3F406C3F3A41B134310D6B68BEBC5708'""" + +example_mode_aes_192_cfb1_datatype_UUID_key_24_iv_None = r"""'B7F80F1BDCA1C4193E5AB11078FEA213'""" + +example_mode_aes_192_cfb1_datatype_Date_key_24_iv_None = r"""'6FF6'""" + +example_mode_aes_192_cfb1_datatype_DateTime_key_24_iv_None = r"""'7013E555'""" + +example_mode_aes_192_cfb1_datatype_DateTime64_key_24_iv_None = r"""'371AF0291536F5B7'""" + +example_mode_aes_192_cfb1_datatype_LowCardinality_key_24_iv_None = r"""'23'""" + +example_mode_aes_192_cfb1_datatype_Array_key_24_iv_None = r"""'01FA'""" + +example_mode_aes_192_cfb1_datatype_NULL_key_24_iv_None = r"""'\\N'""" + +example_mode_aes_192_cfb1_datatype_IPv4_key_24_iv_None = r"""'33895F70'""" + +example_mode_aes_192_cfb1_datatype_IPv6_key_24_iv_None = r"""'3F24552946522B931290F904186B055A'""" + +example_mode_aes_192_cfb1_datatype_Enum8_key_24_iv_None = r"""'01'""" + +example_mode_aes_192_cfb1_datatype_Enum16_key_24_iv_None = r"""'01F9'""" + +example_mode_aes_256_cfb1_datatype_bytes_key_32_iv_None = r"""'B8'""" + +example_mode_aes_256_cfb1_datatype_emptystring_key_32_iv_None = r"""''""" + +example_mode_aes_256_cfb1_datatype_utf8string_key_32_iv_None = r"""'C797191D9D99944E674425A4275A8A3263E1E1357DF8E11E'""" + +example_mode_aes_256_cfb1_datatype_utf8fixedstring_key_32_iv_None = r"""'C797191D9D99944E674425A4275A8A3263E1E1357DF8E11E'""" + +example_mode_aes_256_cfb1_datatype_String_key_32_iv_None = r"""'9E'""" + +example_mode_aes_256_cfb1_datatype_FixedString_key_32_iv_None = r"""'9E'""" + +example_mode_aes_256_cfb1_datatype_UInt8_key_32_iv_None = r"""'B9'""" + +example_mode_aes_256_cfb1_datatype_UInt16_key_32_iv_None = r"""'B9ED'""" + +example_mode_aes_256_cfb1_datatype_UInt32_key_32_iv_None = r"""'B9ED4764'""" + +example_mode_aes_256_cfb1_datatype_UInt64_key_32_iv_None = r"""'B9ED4764E7BF3C1C'""" + +example_mode_aes_256_cfb1_datatype_Int8_key_32_iv_None = r"""'B9'""" + +example_mode_aes_256_cfb1_datatype_Int16_key_32_iv_None = r"""'B9ED'""" + +example_mode_aes_256_cfb1_datatype_Int32_key_32_iv_None = r"""'B9ED4764'""" + +example_mode_aes_256_cfb1_datatype_Int64_key_32_iv_None = r"""'B9ED4764E7BF3C1C'""" + +example_mode_aes_256_cfb1_datatype_Float32_key_32_iv_None = r"""'B85F0E63'""" + +example_mode_aes_256_cfb1_datatype_Float64_key_32_iv_None = r"""'B85FDB5A8FE0C0BB'""" + +example_mode_aes_256_cfb1_datatype_Decimal32_key_32_iv_None = r"""'891B85B3'""" + +example_mode_aes_256_cfb1_datatype_Decimal64_key_32_iv_None = r"""'891B85B3C1BA6EE1'""" + +example_mode_aes_256_cfb1_datatype_Decimal128_key_32_iv_None = r"""'891B85B3C1BA6EE137EF658F618D1F3F'""" + +example_mode_aes_256_cfb1_datatype_UUID_key_32_iv_None = r"""'121B5EE9929417BC1CDBDB390BC93B4A'""" + +example_mode_aes_256_cfb1_datatype_Date_key_32_iv_None = r"""'D40F'""" + +example_mode_aes_256_cfb1_datatype_DateTime_key_32_iv_None = r"""'CF27297C'""" + +example_mode_aes_256_cfb1_datatype_DateTime64_key_32_iv_None = r"""'8773F350CD394D36'""" + +example_mode_aes_256_cfb1_datatype_LowCardinality_key_32_iv_None = r"""'9E'""" + +example_mode_aes_256_cfb1_datatype_Array_key_32_iv_None = r"""'B9EE'""" + +example_mode_aes_256_cfb1_datatype_NULL_key_32_iv_None = r"""'\\N'""" + +example_mode_aes_256_cfb1_datatype_IPv4_key_32_iv_None = r"""'8383FD3C'""" + +example_mode_aes_256_cfb1_datatype_IPv6_key_32_iv_None = r"""'897A84A02FD451D3DDB92FF290BF9B7C'""" + +example_mode_aes_256_cfb1_datatype_Enum8_key_32_iv_None = r"""'B9'""" + +example_mode_aes_256_cfb1_datatype_Enum16_key_32_iv_None = r"""'B9ED'""" + +example_mode_aes_128_cfb1_datatype_bytes_key_16_iv_16 = r"""'00'""" + +example_mode_aes_128_cfb1_datatype_emptystring_key_16_iv_16 = r"""''""" + +example_mode_aes_128_cfb1_datatype_utf8string_key_16_iv_16 = r"""'49CFD4F9B884A17F67C8CDD639EB4D367BE2B1656CA442A4'""" + +example_mode_aes_128_cfb1_datatype_utf8fixedstring_key_16_iv_16 = r"""'49CFD4F9B884A17F67C8CDD639EB4D367BE2B1656CA442A4'""" + +example_mode_aes_128_cfb1_datatype_String_key_16_iv_16 = r"""'37'""" + +example_mode_aes_128_cfb1_datatype_FixedString_key_16_iv_16 = r"""'37'""" + +example_mode_aes_128_cfb1_datatype_UInt8_key_16_iv_16 = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_UInt16_key_16_iv_16 = r"""'0188'""" + +example_mode_aes_128_cfb1_datatype_UInt32_key_16_iv_16 = r"""'01882D46'""" + +example_mode_aes_128_cfb1_datatype_UInt64_key_16_iv_16 = r"""'01882D46FCCCD695'""" + +example_mode_aes_128_cfb1_datatype_Int8_key_16_iv_16 = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Int16_key_16_iv_16 = r"""'0188'""" + +example_mode_aes_128_cfb1_datatype_Int32_key_16_iv_16 = r"""'01882D46'""" + +example_mode_aes_128_cfb1_datatype_Int64_key_16_iv_16 = r"""'01882D46FCCCD695'""" + +example_mode_aes_128_cfb1_datatype_Float32_key_16_iv_16 = r"""'00B931F2'""" + +example_mode_aes_128_cfb1_datatype_Float64_key_16_iv_16 = r"""'00B99AAE199C3C93'""" + +example_mode_aes_128_cfb1_datatype_Decimal32_key_16_iv_16 = r"""'2D557511'""" + +example_mode_aes_128_cfb1_datatype_Decimal64_key_16_iv_16 = r"""'2D557511511F90FB'""" + +example_mode_aes_128_cfb1_datatype_Decimal128_key_16_iv_16 = r"""'2D557511511F90FBC464352E8A02FC51'""" + +example_mode_aes_128_cfb1_datatype_UUID_key_16_iv_16 = r"""'8AE269086C72AD682EB92ABA6CA58E49'""" + +example_mode_aes_128_cfb1_datatype_Date_key_16_iv_16 = r"""'5FC9'""" + +example_mode_aes_128_cfb1_datatype_DateTime_key_16_iv_16 = r"""'42970865'""" + +example_mode_aes_128_cfb1_datatype_DateTime64_key_16_iv_16 = r"""'20B310A2F7EF8460'""" + +example_mode_aes_128_cfb1_datatype_LowCardinality_key_16_iv_16 = r"""'37'""" + +example_mode_aes_128_cfb1_datatype_Array_key_16_iv_16 = r"""'018A'""" + +example_mode_aes_128_cfb1_datatype_NULL_key_16_iv_16 = r"""'\\N'""" + +example_mode_aes_128_cfb1_datatype_IPv4_key_16_iv_16 = r"""'27476DAF'""" + +example_mode_aes_128_cfb1_datatype_IPv6_key_16_iv_16 = r"""'2D311FBDC0A5C652AAD863398F94C5C3'""" + +example_mode_aes_128_cfb1_datatype_Enum8_key_16_iv_16 = r"""'01'""" + +example_mode_aes_128_cfb1_datatype_Enum16_key_16_iv_16 = r"""'0188'""" + +example_mode_aes_128_cfb1_datatype_bytes_key_24_iv_24 = r"""'1C'""" + +example_mode_aes_128_cfb1_datatype_emptystring_key_24_iv_24 = r"""''""" + +example_mode_aes_128_cfb1_datatype_utf8string_key_24_iv_24 = r"""'612A502D86BAEA94E98C1381AE057BD8606FB3776DC79E52'""" + +example_mode_aes_128_cfb1_datatype_utf8fixedstring_key_24_iv_24 = r"""'612A502D86BAEA94E98C1381AE057BD8606FB3776DC79E52'""" + +example_mode_aes_128_cfb1_datatype_String_key_24_iv_24 = r"""'30'""" + +example_mode_aes_128_cfb1_datatype_FixedString_key_24_iv_24 = r"""'30'""" + +example_mode_aes_128_cfb1_datatype_UInt8_key_24_iv_24 = r"""'1D'""" + +example_mode_aes_128_cfb1_datatype_UInt16_key_24_iv_24 = r"""'1D52'""" + +example_mode_aes_128_cfb1_datatype_UInt32_key_24_iv_24 = r"""'1D52BAD5'""" + +example_mode_aes_128_cfb1_datatype_UInt64_key_24_iv_24 = r"""'1D52BAD5E209AC18'""" + +example_mode_aes_128_cfb1_datatype_Int8_key_24_iv_24 = r"""'1D'""" + +example_mode_aes_128_cfb1_datatype_Int16_key_24_iv_24 = r"""'1D52'""" + +example_mode_aes_128_cfb1_datatype_Int32_key_24_iv_24 = r"""'1D52BAD5'""" + +example_mode_aes_128_cfb1_datatype_Int64_key_24_iv_24 = r"""'1D52BAD5E209AC18'""" + +example_mode_aes_128_cfb1_datatype_Float32_key_24_iv_24 = r"""'1C3165A0'""" + +example_mode_aes_128_cfb1_datatype_Float64_key_24_iv_24 = r"""'1C31E017FDE1A7AC'""" + +example_mode_aes_128_cfb1_datatype_Decimal32_key_24_iv_24 = r"""'2CCB18B8'""" + +example_mode_aes_128_cfb1_datatype_Decimal64_key_24_iv_24 = r"""'2CCB18B874E51783'""" + +example_mode_aes_128_cfb1_datatype_Decimal128_key_24_iv_24 = r"""'2CCB18B874E51783A6ADE353B5D95026'""" + +example_mode_aes_128_cfb1_datatype_UUID_key_24_iv_24 = r"""'EF014A2E68CFB6C3BE44AE59F4A58888'""" + +example_mode_aes_128_cfb1_datatype_Date_key_24_iv_24 = r"""'70F3'""" + +example_mode_aes_128_cfb1_datatype_DateTime_key_24_iv_24 = r"""'6FF3F637'""" + +example_mode_aes_128_cfb1_datatype_DateTime64_key_24_iv_24 = r"""'2432F8327D29D7FF'""" + +example_mode_aes_128_cfb1_datatype_LowCardinality_key_24_iv_24 = r"""'30'""" + +example_mode_aes_128_cfb1_datatype_Array_key_24_iv_24 = r"""'1D50'""" + +example_mode_aes_128_cfb1_datatype_NULL_key_24_iv_24 = r"""'\\N'""" + +example_mode_aes_128_cfb1_datatype_IPv4_key_24_iv_24 = r"""'20BA7FDE'""" + +example_mode_aes_128_cfb1_datatype_IPv6_key_24_iv_24 = r"""'2C8F73DFF566693C8E482419FAC24837'""" + +example_mode_aes_128_cfb1_datatype_Enum8_key_24_iv_24 = r"""'1D'""" + +example_mode_aes_128_cfb1_datatype_Enum16_key_24_iv_24 = r"""'1D52'""" + +example_mode_aes_192_cfb1_datatype_bytes_key_24_iv_16 = r"""'07'""" + +example_mode_aes_192_cfb1_datatype_emptystring_key_24_iv_16 = r"""''""" + +example_mode_aes_192_cfb1_datatype_utf8string_key_24_iv_16 = r"""'54C01F88A909EC7B2AEE59F81C138B8EE2DF205E2ED74210'""" + +example_mode_aes_192_cfb1_datatype_utf8fixedstring_key_24_iv_16 = r"""'54C01F88A909EC7B2AEE59F81C138B8EE2DF205E2ED74210'""" + +example_mode_aes_192_cfb1_datatype_String_key_24_iv_16 = r"""'38'""" + +example_mode_aes_192_cfb1_datatype_FixedString_key_24_iv_16 = r"""'38'""" + +example_mode_aes_192_cfb1_datatype_UInt8_key_24_iv_16 = r"""'06'""" + +example_mode_aes_192_cfb1_datatype_UInt16_key_24_iv_16 = r"""'069E'""" + +example_mode_aes_192_cfb1_datatype_UInt32_key_24_iv_16 = r"""'069E2E37'""" + +example_mode_aes_192_cfb1_datatype_UInt64_key_24_iv_16 = r"""'069E2E370A6D9872'""" + +example_mode_aes_192_cfb1_datatype_Int8_key_24_iv_16 = r"""'06'""" + +example_mode_aes_192_cfb1_datatype_Int16_key_24_iv_16 = r"""'069E'""" + +example_mode_aes_192_cfb1_datatype_Int32_key_24_iv_16 = r"""'069E2E37'""" + +example_mode_aes_192_cfb1_datatype_Int64_key_24_iv_16 = r"""'069E2E370A6D9872'""" + +example_mode_aes_192_cfb1_datatype_Float32_key_24_iv_16 = r"""'07955BCF'""" + +example_mode_aes_192_cfb1_datatype_Float64_key_24_iv_16 = r"""'0795A57CA222A36E'""" + +example_mode_aes_192_cfb1_datatype_Decimal32_key_24_iv_16 = r"""'2A15BB86'""" + +example_mode_aes_192_cfb1_datatype_Decimal64_key_24_iv_16 = r"""'2A15BB86FB961E7D'""" + +example_mode_aes_192_cfb1_datatype_Decimal128_key_24_iv_16 = r"""'2A15BB86FB961E7D0DD5055987176AF4'""" + +example_mode_aes_192_cfb1_datatype_UUID_key_24_iv_16 = r"""'DA2338793C7B9E0F6722E272062F5EA1'""" + +example_mode_aes_192_cfb1_datatype_Date_key_24_iv_16 = r"""'4AAB'""" + +example_mode_aes_192_cfb1_datatype_DateTime_key_24_iv_16 = r"""'5B6A8EE6'""" + +example_mode_aes_192_cfb1_datatype_DateTime64_key_24_iv_16 = r"""'23C4E2A707F73EF4'""" + +example_mode_aes_192_cfb1_datatype_LowCardinality_key_24_iv_16 = r"""'38'""" + +example_mode_aes_192_cfb1_datatype_Array_key_24_iv_16 = r"""'069C'""" + +example_mode_aes_192_cfb1_datatype_NULL_key_24_iv_16 = r"""'\\N'""" + +example_mode_aes_192_cfb1_datatype_IPv4_key_24_iv_16 = r"""'2470A839'""" + +example_mode_aes_192_cfb1_datatype_IPv6_key_24_iv_16 = r"""'2A712A746781131B2DC4EB92E31C72FA'""" + +example_mode_aes_192_cfb1_datatype_Enum8_key_24_iv_16 = r"""'06'""" + +example_mode_aes_192_cfb1_datatype_Enum16_key_24_iv_16 = r"""'069E'""" + +example_mode_aes_192_cfb1_datatype_bytes_key_32_iv_32 = r"""'6A'""" + +example_mode_aes_192_cfb1_datatype_emptystring_key_32_iv_32 = r"""''""" + +example_mode_aes_192_cfb1_datatype_utf8string_key_32_iv_32 = r"""'1317B03E3411FE499C234D87DD44014E46B15B10C9F76A3C'""" + +example_mode_aes_192_cfb1_datatype_utf8fixedstring_key_32_iv_32 = r"""'1317B03E3411FE499C234D87DD44014E46B15B10C9F76A3C'""" + +example_mode_aes_192_cfb1_datatype_String_key_32_iv_32 = r"""'45'""" + +example_mode_aes_192_cfb1_datatype_FixedString_key_32_iv_32 = r"""'45'""" + +example_mode_aes_192_cfb1_datatype_UInt8_key_32_iv_32 = r"""'6B'""" + +example_mode_aes_192_cfb1_datatype_UInt16_key_32_iv_32 = r"""'6B43'""" + +example_mode_aes_192_cfb1_datatype_UInt32_key_32_iv_32 = r"""'6B43D96E'""" + +example_mode_aes_192_cfb1_datatype_UInt64_key_32_iv_32 = r"""'6B43D96E7D63C177'""" + +example_mode_aes_192_cfb1_datatype_Int8_key_32_iv_32 = r"""'6B'""" + +example_mode_aes_192_cfb1_datatype_Int16_key_32_iv_32 = r"""'6B43'""" + +example_mode_aes_192_cfb1_datatype_Int32_key_32_iv_32 = r"""'6B43D96E'""" + +example_mode_aes_192_cfb1_datatype_Int64_key_32_iv_32 = r"""'6B43D96E7D63C177'""" + +example_mode_aes_192_cfb1_datatype_Float32_key_32_iv_32 = r"""'6AF301CD'""" + +example_mode_aes_192_cfb1_datatype_Float64_key_32_iv_32 = r"""'6AF3B787907E47A9'""" + +example_mode_aes_192_cfb1_datatype_Decimal32_key_32_iv_32 = r"""'515FA59C'""" + +example_mode_aes_192_cfb1_datatype_Decimal64_key_32_iv_32 = r"""'515FA59C52E93CA7'""" + +example_mode_aes_192_cfb1_datatype_Decimal128_key_32_iv_32 = r"""'515FA59C52E93CA74BFFFC3D5F3D3163'""" + +example_mode_aes_192_cfb1_datatype_UUID_key_32_iv_32 = r"""'E5F370149365AFB35B9245146396A914'""" + +example_mode_aes_192_cfb1_datatype_Date_key_32_iv_32 = r"""'0685'""" + +example_mode_aes_192_cfb1_datatype_DateTime_key_32_iv_32 = r"""'1D0054E1'""" + +example_mode_aes_192_cfb1_datatype_DateTime64_key_32_iv_32 = r"""'5BD19E5F525FA135'""" + +example_mode_aes_192_cfb1_datatype_LowCardinality_key_32_iv_32 = r"""'45'""" + +example_mode_aes_192_cfb1_datatype_Array_key_32_iv_32 = r"""'6B40'""" + +example_mode_aes_192_cfb1_datatype_NULL_key_32_iv_32 = r"""'\\N'""" + +example_mode_aes_192_cfb1_datatype_IPv4_key_32_iv_32 = r"""'5D77DE04'""" + +example_mode_aes_192_cfb1_datatype_IPv6_key_32_iv_32 = r"""'513F29EF9D609D6DBD12084B5E11D796'""" + +example_mode_aes_192_cfb1_datatype_Enum8_key_32_iv_32 = r"""'6B'""" + +example_mode_aes_192_cfb1_datatype_Enum16_key_32_iv_32 = r"""'6B43'""" + +example_mode_aes_256_cfb1_datatype_bytes_key_32_iv_16 = r"""'7F'""" + +example_mode_aes_256_cfb1_datatype_emptystring_key_32_iv_16 = r"""''""" + +example_mode_aes_256_cfb1_datatype_utf8string_key_32_iv_16 = r"""'2E1863FFF5FEC47DBB3F50BCC912976E2777442C693EB1B5'""" + +example_mode_aes_256_cfb1_datatype_utf8fixedstring_key_32_iv_16 = r"""'2E1863FFF5FEC47DBB3F50BCC912976E2777442C693EB1B5'""" + +example_mode_aes_256_cfb1_datatype_String_key_32_iv_16 = r"""'5A'""" + +example_mode_aes_256_cfb1_datatype_FixedString_key_32_iv_16 = r"""'5A'""" + +example_mode_aes_256_cfb1_datatype_UInt8_key_32_iv_16 = r"""'7E'""" + +example_mode_aes_256_cfb1_datatype_UInt16_key_32_iv_16 = r"""'7EA1'""" + +example_mode_aes_256_cfb1_datatype_UInt32_key_32_iv_16 = r"""'7EA17214'""" + +example_mode_aes_256_cfb1_datatype_UInt64_key_32_iv_16 = r"""'7EA172144C6F5578'""" + +example_mode_aes_256_cfb1_datatype_Int8_key_32_iv_16 = r"""'7E'""" + +example_mode_aes_256_cfb1_datatype_Int16_key_32_iv_16 = r"""'7EA1'""" + +example_mode_aes_256_cfb1_datatype_Int32_key_32_iv_16 = r"""'7EA17214'""" + +example_mode_aes_256_cfb1_datatype_Int64_key_32_iv_16 = r"""'7EA172144C6F5578'""" + +example_mode_aes_256_cfb1_datatype_Float32_key_32_iv_16 = r"""'7F630BBA'""" + +example_mode_aes_256_cfb1_datatype_Float64_key_32_iv_16 = r"""'7F638DFAAA434E6B'""" + +example_mode_aes_256_cfb1_datatype_Decimal32_key_32_iv_16 = r"""'4F430FBA'""" + +example_mode_aes_256_cfb1_datatype_Decimal64_key_32_iv_16 = r"""'4F430FBAA3AAF884'""" + +example_mode_aes_256_cfb1_datatype_Decimal128_key_32_iv_16 = r"""'4F430FBAA3AAF8845DB7BBA7F98F49C4'""" + +example_mode_aes_256_cfb1_datatype_UUID_key_32_iv_16 = r"""'B06F4A8C3BF3A8D32D113D0D40397C8F'""" + +example_mode_aes_256_cfb1_datatype_Date_key_32_iv_16 = r"""'30CE'""" + +example_mode_aes_256_cfb1_datatype_DateTime_key_32_iv_16 = r"""'206545FA'""" + +example_mode_aes_256_cfb1_datatype_DateTime64_key_32_iv_16 = r"""'43756F28C68E3D55'""" + +example_mode_aes_256_cfb1_datatype_LowCardinality_key_32_iv_16 = r"""'5A'""" + +example_mode_aes_256_cfb1_datatype_Array_key_32_iv_16 = r"""'7EA3'""" + +example_mode_aes_256_cfb1_datatype_NULL_key_32_iv_16 = r"""'\\N'""" + +example_mode_aes_256_cfb1_datatype_IPv4_key_32_iv_16 = r"""'4526FCCF'""" + +example_mode_aes_256_cfb1_datatype_IPv6_key_32_iv_16 = r"""'4F23BDAC741DB8767CE6AE24888545A2'""" + +example_mode_aes_256_cfb1_datatype_Enum8_key_32_iv_16 = r"""'7E'""" + +example_mode_aes_256_cfb1_datatype_Enum16_key_32_iv_16 = r"""'7EA1'""" + +example_mode_aes_256_cfb1_datatype_bytes_key_64_iv_64 = r"""'DF'""" + +example_mode_aes_256_cfb1_datatype_emptystring_key_64_iv_64 = r"""''""" + +example_mode_aes_256_cfb1_datatype_utf8string_key_64_iv_64 = r"""'A72DF5A969944B55E9D33B4E5207FB80DAB5BA6787ACB19F'""" + +example_mode_aes_256_cfb1_datatype_utf8fixedstring_key_64_iv_64 = r"""'A72DF5A969944B55E9D33B4E5207FB80DAB5BA6787ACB19F'""" + +example_mode_aes_256_cfb1_datatype_String_key_64_iv_64 = r"""'E9'""" + +example_mode_aes_256_cfb1_datatype_FixedString_key_64_iv_64 = r"""'E9'""" + +example_mode_aes_256_cfb1_datatype_UInt8_key_64_iv_64 = r"""'DE'""" + +example_mode_aes_256_cfb1_datatype_UInt16_key_64_iv_64 = r"""'DEBD'""" + +example_mode_aes_256_cfb1_datatype_UInt32_key_64_iv_64 = r"""'DEBD9C91'""" + +example_mode_aes_256_cfb1_datatype_UInt64_key_64_iv_64 = r"""'DEBD9C915CAA4813'""" + +example_mode_aes_256_cfb1_datatype_Int8_key_64_iv_64 = r"""'DE'""" + +example_mode_aes_256_cfb1_datatype_Int16_key_64_iv_64 = r"""'DEBD'""" + +example_mode_aes_256_cfb1_datatype_Int32_key_64_iv_64 = r"""'DEBD9C91'""" + +example_mode_aes_256_cfb1_datatype_Int64_key_64_iv_64 = r"""'DEBD9C915CAA4813'""" + +example_mode_aes_256_cfb1_datatype_Float32_key_64_iv_64 = r"""'DF845158'""" + +example_mode_aes_256_cfb1_datatype_Float64_key_64_iv_64 = r"""'DF84903EDD435363'""" + +example_mode_aes_256_cfb1_datatype_Decimal32_key_64_iv_64 = r"""'F2FE11DD'""" + +example_mode_aes_256_cfb1_datatype_Decimal64_key_64_iv_64 = r"""'F2FE11DDDF1B0FB8'""" + +example_mode_aes_256_cfb1_datatype_Decimal128_key_64_iv_64 = r"""'F2FE11DDDF1B0FB8F5BA32A48936DF25'""" + +example_mode_aes_256_cfb1_datatype_UUID_key_64_iv_64 = r"""'580073A89DF8ED751F3E63D07B5AAE93'""" + +example_mode_aes_256_cfb1_datatype_Date_key_64_iv_64 = r"""'BCC3'""" + +example_mode_aes_256_cfb1_datatype_DateTime_key_64_iv_64 = r"""'AF6825D1'""" + +example_mode_aes_256_cfb1_datatype_DateTime64_key_64_iv_64 = r"""'FD4A027DF5DD3405'""" + +example_mode_aes_256_cfb1_datatype_LowCardinality_key_64_iv_64 = r"""'E9'""" + +example_mode_aes_256_cfb1_datatype_Array_key_64_iv_64 = r"""'DEBE'""" + +example_mode_aes_256_cfb1_datatype_NULL_key_64_iv_64 = r"""'\\N'""" + +example_mode_aes_256_cfb1_datatype_IPv4_key_64_iv_64 = r"""'FB2ABD13'""" + +example_mode_aes_256_cfb1_datatype_IPv6_key_64_iv_64 = r"""'F2B6BE1E1DC716AD0490FF91D509F72B'""" + +example_mode_aes_256_cfb1_datatype_Enum8_key_64_iv_64 = r"""'DE'""" + +example_mode_aes_256_cfb1_datatype_Enum16_key_64_iv_64 = r"""'DEBD'""" + +example_mode_aes_128_cfb8_datatype_bytes_key_16_iv_None = r"""'10'""" + +example_mode_aes_128_cfb8_datatype_emptystring_key_16_iv_None = r"""''""" + +example_mode_aes_128_cfb8_datatype_utf8string_key_16_iv_None = r"""'5716349E99199E0EF18EB43C06B54AB2F3E0C4CEA0BC11F9'""" + +example_mode_aes_128_cfb8_datatype_utf8fixedstring_key_16_iv_None = r"""'5716349E99199E0EF18EB43C06B54AB2F3E0C4CEA0BC11F9'""" + +example_mode_aes_128_cfb8_datatype_String_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_cfb8_datatype_FixedString_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_cfb8_datatype_UInt8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_cfb8_datatype_UInt16_key_16_iv_None = r"""'11FF'""" + +example_mode_aes_128_cfb8_datatype_UInt32_key_16_iv_None = r"""'11FF20C0'""" + +example_mode_aes_128_cfb8_datatype_UInt64_key_16_iv_None = r"""'11FF20C07A65C524'""" + +example_mode_aes_128_cfb8_datatype_Int8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_cfb8_datatype_Int16_key_16_iv_None = r"""'11FF'""" + +example_mode_aes_128_cfb8_datatype_Int32_key_16_iv_None = r"""'11FF20C0'""" + +example_mode_aes_128_cfb8_datatype_Int64_key_16_iv_None = r"""'11FF20C07A65C524'""" + +example_mode_aes_128_cfb8_datatype_Float32_key_16_iv_None = r"""'10671940'""" + +example_mode_aes_128_cfb8_datatype_Float64_key_16_iv_None = r"""'106799607DBF56DA'""" + +example_mode_aes_128_cfb8_datatype_Decimal32_key_16_iv_None = r"""'30756C94'""" + +example_mode_aes_128_cfb8_datatype_Decimal64_key_16_iv_None = r"""'30756C9417D3C023'""" + +example_mode_aes_128_cfb8_datatype_Decimal128_key_16_iv_None = r"""'30756C9417D3C023705550B7BEF872FF'""" + +example_mode_aes_128_cfb8_datatype_UUID_key_16_iv_None = r"""'F7FE50CF0647659CB0D401B5C0E259D3'""" + +example_mode_aes_128_cfb8_datatype_Date_key_16_iv_None = r"""'46EA'""" + +example_mode_aes_128_cfb8_datatype_DateTime_key_16_iv_None = r"""'5EB4905E'""" + +example_mode_aes_128_cfb8_datatype_DateTime64_key_16_iv_None = r"""'3BB70F8E64D7C6A7'""" + +example_mode_aes_128_cfb8_datatype_LowCardinality_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_cfb8_datatype_Array_key_16_iv_None = r"""'11FD'""" + +example_mode_aes_128_cfb8_datatype_NULL_key_16_iv_None = r"""'\\N'""" + +example_mode_aes_128_cfb8_datatype_IPv4_key_16_iv_None = r"""'3DC2BE9E'""" + +example_mode_aes_128_cfb8_datatype_IPv6_key_16_iv_None = r"""'303ABAC6F4F380D9F077DFC79C82D1A1'""" + +example_mode_aes_128_cfb8_datatype_Enum8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_cfb8_datatype_Enum16_key_16_iv_None = r"""'11FF'""" + +example_mode_aes_192_cfb8_datatype_bytes_key_24_iv_None = r"""'07'""" + +example_mode_aes_192_cfb8_datatype_emptystring_key_24_iv_None = r"""''""" + +example_mode_aes_192_cfb8_datatype_utf8string_key_24_iv_None = r"""'40541136D8EEE5DCFC55722D8FB56ED9D4CCDF0CB104B14D'""" + +example_mode_aes_192_cfb8_datatype_utf8fixedstring_key_24_iv_None = r"""'40541136D8EEE5DCFC55722D8FB56ED9D4CCDF0CB104B14D'""" + +example_mode_aes_192_cfb8_datatype_String_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_cfb8_datatype_FixedString_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_cfb8_datatype_UInt8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_cfb8_datatype_UInt16_key_24_iv_None = r"""'0683'""" + +example_mode_aes_192_cfb8_datatype_UInt32_key_24_iv_None = r"""'0683139D'""" + +example_mode_aes_192_cfb8_datatype_UInt64_key_24_iv_None = r"""'0683139D83E2EFAC'""" + +example_mode_aes_192_cfb8_datatype_Int8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_cfb8_datatype_Int16_key_24_iv_None = r"""'0683'""" + +example_mode_aes_192_cfb8_datatype_Int32_key_24_iv_None = r"""'0683139D'""" + +example_mode_aes_192_cfb8_datatype_Int64_key_24_iv_None = r"""'0683139D83E2EFAC'""" + +example_mode_aes_192_cfb8_datatype_Float32_key_24_iv_None = r"""'07EDB300'""" + +example_mode_aes_192_cfb8_datatype_Float64_key_24_iv_None = r"""'07ED3359B91DEF3B'""" + +example_mode_aes_192_cfb8_datatype_Decimal32_key_24_iv_None = r"""'275947FE'""" + +example_mode_aes_192_cfb8_datatype_Decimal64_key_24_iv_None = r"""'275947FE4B3390EE'""" + +example_mode_aes_192_cfb8_datatype_Decimal128_key_24_iv_None = r"""'275947FE4B3390EE7A2541BC8E2F58D7'""" + +example_mode_aes_192_cfb8_datatype_UUID_key_24_iv_None = r"""'E0C082C032FB8ED756F9345E270A283B'""" + +example_mode_aes_192_cfb8_datatype_Date_key_24_iv_None = r"""'5109'""" + +example_mode_aes_192_cfb8_datatype_DateTime_key_24_iv_None = r"""'49713150'""" + +example_mode_aes_192_cfb8_datatype_DateTime64_key_24_iv_None = r"""'2C10FB4FEC471EF7'""" + +example_mode_aes_192_cfb8_datatype_LowCardinality_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_cfb8_datatype_Array_key_24_iv_None = r"""'0681'""" + +example_mode_aes_192_cfb8_datatype_NULL_key_24_iv_None = r"""'\\N'""" + +example_mode_aes_192_cfb8_datatype_IPv4_key_24_iv_None = r"""'2A41C8F2'""" + +example_mode_aes_192_cfb8_datatype_IPv6_key_24_iv_None = r"""'271682C9379C5A46C68488DC33D0C278'""" + +example_mode_aes_192_cfb8_datatype_Enum8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_cfb8_datatype_Enum16_key_24_iv_None = r"""'0683'""" + +example_mode_aes_256_cfb8_datatype_bytes_key_32_iv_None = r"""'B0'""" + +example_mode_aes_256_cfb8_datatype_emptystring_key_32_iv_None = r"""''""" + +example_mode_aes_256_cfb8_datatype_utf8string_key_32_iv_None = r"""'F796638AA3A076EA9816324AE8A93F420280C33AA2DE4AF8'""" + +example_mode_aes_256_cfb8_datatype_utf8fixedstring_key_32_iv_None = r"""'F796638AA3A076EA9816324AE8A93F420280C33AA2DE4AF8'""" + +example_mode_aes_256_cfb8_datatype_String_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_cfb8_datatype_FixedString_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_cfb8_datatype_UInt8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_cfb8_datatype_UInt16_key_32_iv_None = r"""'B15F'""" + +example_mode_aes_256_cfb8_datatype_UInt32_key_32_iv_None = r"""'B15FD91F'""" + +example_mode_aes_256_cfb8_datatype_UInt64_key_32_iv_None = r"""'B15FD91F702960CB'""" + +example_mode_aes_256_cfb8_datatype_Int8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_cfb8_datatype_Int16_key_32_iv_None = r"""'B15F'""" + +example_mode_aes_256_cfb8_datatype_Int32_key_32_iv_None = r"""'B15FD91F'""" + +example_mode_aes_256_cfb8_datatype_Int64_key_32_iv_None = r"""'B15FD91F702960CB'""" + +example_mode_aes_256_cfb8_datatype_Float32_key_32_iv_None = r"""'B05A05BE'""" + +example_mode_aes_256_cfb8_datatype_Float64_key_32_iv_None = r"""'B05A8510DB2F16A0'""" + +example_mode_aes_256_cfb8_datatype_Decimal32_key_32_iv_None = r"""'906B5777'""" + +example_mode_aes_256_cfb8_datatype_Decimal64_key_32_iv_None = r"""'906B57771CB81F37'""" + +example_mode_aes_256_cfb8_datatype_Decimal128_key_32_iv_None = r"""'906B57771CB81F378D932AE788527DE2'""" + +example_mode_aes_256_cfb8_datatype_UUID_key_32_iv_None = r"""'57FB06BA6F4BA51D7A61D65A7827A18D'""" + +example_mode_aes_256_cfb8_datatype_Date_key_32_iv_None = r"""'E652'""" + +example_mode_aes_256_cfb8_datatype_DateTime_key_32_iv_None = r"""'FEEEADA4'""" + +example_mode_aes_256_cfb8_datatype_DateTime64_key_32_iv_None = r"""'9BB36DEF05FF5975'""" + +example_mode_aes_256_cfb8_datatype_LowCardinality_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_cfb8_datatype_Array_key_32_iv_None = r"""'B15D'""" + +example_mode_aes_256_cfb8_datatype_NULL_key_32_iv_None = r"""'\\N'""" + +example_mode_aes_256_cfb8_datatype_IPv4_key_32_iv_None = r"""'9DC836F3'""" + +example_mode_aes_256_cfb8_datatype_IPv6_key_32_iv_None = r"""'90242F0083C8B0221DF3B5755EC8D99C'""" + +example_mode_aes_256_cfb8_datatype_Enum8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_cfb8_datatype_Enum16_key_32_iv_None = r"""'B15F'""" + +example_mode_aes_128_cfb8_datatype_bytes_key_16_iv_16 = r"""'32'""" + +example_mode_aes_128_cfb8_datatype_emptystring_key_16_iv_16 = r"""''""" + +example_mode_aes_128_cfb8_datatype_utf8string_key_16_iv_16 = r"""'75BA4CC259943AF0B6E20E8FCC78C0427601F60930A5F980'""" + +example_mode_aes_128_cfb8_datatype_utf8fixedstring_key_16_iv_16 = r"""'75BA4CC259943AF0B6E20E8FCC78C0427601F60930A5F980'""" + +example_mode_aes_128_cfb8_datatype_String_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_cfb8_datatype_FixedString_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_cfb8_datatype_UInt8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_cfb8_datatype_UInt16_key_16_iv_16 = r"""'3368'""" + +example_mode_aes_128_cfb8_datatype_UInt32_key_16_iv_16 = r"""'3368AB64'""" + +example_mode_aes_128_cfb8_datatype_UInt64_key_16_iv_16 = r"""'3368AB6421744B7E'""" + +example_mode_aes_128_cfb8_datatype_Int8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_cfb8_datatype_Int16_key_16_iv_16 = r"""'3368'""" + +example_mode_aes_128_cfb8_datatype_Int32_key_16_iv_16 = r"""'3368AB64'""" + +example_mode_aes_128_cfb8_datatype_Int64_key_16_iv_16 = r"""'3368AB6421744B7E'""" + +example_mode_aes_128_cfb8_datatype_Float32_key_16_iv_16 = r"""'3232B23D'""" + +example_mode_aes_128_cfb8_datatype_Float64_key_16_iv_16 = r"""'323232323232C2A6'""" + +example_mode_aes_128_cfb8_datatype_Decimal32_key_16_iv_16 = r"""'12ABA873'""" + +example_mode_aes_128_cfb8_datatype_Decimal64_key_16_iv_16 = r"""'12ABA873E2E24473'""" + +example_mode_aes_128_cfb8_datatype_Decimal128_key_16_iv_16 = r"""'12ABA873E2E24473166434D82270A19C'""" + +example_mode_aes_128_cfb8_datatype_UUID_key_16_iv_16 = r"""'D529D970A38CCB794F856E4458D0E2D4'""" + +example_mode_aes_128_cfb8_datatype_Date_key_16_iv_16 = r"""'6445'""" + +example_mode_aes_128_cfb8_datatype_DateTime_key_16_iv_16 = r"""'7CBF2FDA'""" + +example_mode_aes_128_cfb8_datatype_DateTime64_key_16_iv_16 = r"""'191C7B5A063F562D'""" + +example_mode_aes_128_cfb8_datatype_LowCardinality_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_cfb8_datatype_Array_key_16_iv_16 = r"""'336A'""" + +example_mode_aes_128_cfb8_datatype_NULL_key_16_iv_16 = r"""'\\N'""" + +example_mode_aes_128_cfb8_datatype_IPv4_key_16_iv_16 = r"""'1F0A367A'""" + +example_mode_aes_128_cfb8_datatype_IPv6_key_16_iv_16 = r"""'12E4B19D97DC9F2C61A671C51B1201D2'""" + +example_mode_aes_128_cfb8_datatype_Enum8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_cfb8_datatype_Enum16_key_16_iv_16 = r"""'3368'""" + +example_mode_aes_128_cfb8_datatype_bytes_key_24_iv_24 = r"""'5B'""" + +example_mode_aes_128_cfb8_datatype_emptystring_key_24_iv_24 = r"""''""" + +example_mode_aes_128_cfb8_datatype_utf8string_key_24_iv_24 = r"""'1CE086607CFD8523D699E2588307E8F47EB1C6DB33141C6A'""" + +example_mode_aes_128_cfb8_datatype_utf8fixedstring_key_24_iv_24 = r"""'1CE086607CFD8523D699E2588307E8F47EB1C6DB33141C6A'""" + +example_mode_aes_128_cfb8_datatype_String_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_cfb8_datatype_FixedString_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_cfb8_datatype_UInt8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_cfb8_datatype_UInt16_key_24_iv_24 = r"""'5A37'""" + +example_mode_aes_128_cfb8_datatype_UInt32_key_24_iv_24 = r"""'5A37BD02'""" + +example_mode_aes_128_cfb8_datatype_UInt64_key_24_iv_24 = r"""'5A37BD02ECFA56DB'""" + +example_mode_aes_128_cfb8_datatype_Int8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_cfb8_datatype_Int16_key_24_iv_24 = r"""'5A37'""" + +example_mode_aes_128_cfb8_datatype_Int32_key_24_iv_24 = r"""'5A37BD02'""" + +example_mode_aes_128_cfb8_datatype_Int64_key_24_iv_24 = r"""'5A37BD02ECFA56DB'""" + +example_mode_aes_128_cfb8_datatype_Float32_key_24_iv_24 = r"""'5BC333D3'""" + +example_mode_aes_128_cfb8_datatype_Float64_key_24_iv_24 = r"""'5BC3B3495464BBE8'""" + +example_mode_aes_128_cfb8_datatype_Decimal32_key_24_iv_24 = r"""'7BE3C6D3'""" + +example_mode_aes_128_cfb8_datatype_Decimal64_key_24_iv_24 = r"""'7BE3C6D356A31F6E'""" + +example_mode_aes_128_cfb8_datatype_Decimal128_key_24_iv_24 = r"""'7BE3C6D356A31F6E1A17102C88C40C85'""" + +example_mode_aes_128_cfb8_datatype_UUID_key_24_iv_24 = r"""'BC30ED9E33095BED0825C18860C622AC'""" + +example_mode_aes_128_cfb8_datatype_Date_key_24_iv_24 = r"""'0D03'""" + +example_mode_aes_128_cfb8_datatype_DateTime_key_24_iv_24 = r"""'15F76197'""" + +example_mode_aes_128_cfb8_datatype_DateTime64_key_24_iv_24 = r"""'7035C3E2D0F5F702'""" + +example_mode_aes_128_cfb8_datatype_LowCardinality_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_cfb8_datatype_Array_key_24_iv_24 = r"""'5A35'""" + +example_mode_aes_128_cfb8_datatype_NULL_key_24_iv_24 = r"""'\\N'""" + +example_mode_aes_128_cfb8_datatype_IPv4_key_24_iv_24 = r"""'769533D5'""" + +example_mode_aes_128_cfb8_datatype_IPv6_key_24_iv_24 = r"""'7BAC6FE32B92AB3A07AA8D90F0E175BA'""" + +example_mode_aes_128_cfb8_datatype_Enum8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_cfb8_datatype_Enum16_key_24_iv_24 = r"""'5A37'""" + +example_mode_aes_192_cfb8_datatype_bytes_key_24_iv_16 = r"""'68'""" + +example_mode_aes_192_cfb8_datatype_emptystring_key_24_iv_16 = r"""''""" + +example_mode_aes_192_cfb8_datatype_utf8string_key_24_iv_16 = r"""'2F48574D4D12E6B2C1EF1B43346E437333FFD386067A9398'""" + +example_mode_aes_192_cfb8_datatype_utf8fixedstring_key_24_iv_16 = r"""'2F48574D4D12E6B2C1EF1B43346E437333FFD386067A9398'""" + +example_mode_aes_192_cfb8_datatype_String_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_cfb8_datatype_FixedString_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_cfb8_datatype_UInt8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_cfb8_datatype_UInt16_key_24_iv_16 = r"""'6924'""" + +example_mode_aes_192_cfb8_datatype_UInt32_key_24_iv_16 = r"""'6924A086'""" + +example_mode_aes_192_cfb8_datatype_UInt64_key_24_iv_16 = r"""'6924A086F8F61C3C'""" + +example_mode_aes_192_cfb8_datatype_Int8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_cfb8_datatype_Int16_key_24_iv_16 = r"""'6924'""" + +example_mode_aes_192_cfb8_datatype_Int32_key_24_iv_16 = r"""'6924A086'""" + +example_mode_aes_192_cfb8_datatype_Int64_key_24_iv_16 = r"""'6924A086F8F61C3C'""" + +example_mode_aes_192_cfb8_datatype_Float32_key_24_iv_16 = r"""'6861DF7A'""" + +example_mode_aes_192_cfb8_datatype_Float64_key_24_iv_16 = r"""'68615FBC184B8D50'""" + +example_mode_aes_192_cfb8_datatype_Decimal32_key_24_iv_16 = r"""'48041B5C'""" + +example_mode_aes_192_cfb8_datatype_Decimal64_key_24_iv_16 = r"""'48041B5C6BEF70DD'""" + +example_mode_aes_192_cfb8_datatype_Decimal128_key_24_iv_16 = r"""'48041B5C6BEF70DD4CDABC1FC8C2C684'""" + +example_mode_aes_192_cfb8_datatype_UUID_key_24_iv_16 = r"""'8FF1142976A9808C0F475A3D2A34D06D'""" + +example_mode_aes_192_cfb8_datatype_Date_key_24_iv_16 = r"""'3E6D'""" + +example_mode_aes_192_cfb8_datatype_DateTime_key_24_iv_16 = r"""'269AFDC7'""" + +example_mode_aes_192_cfb8_datatype_DateTime64_key_24_iv_16 = r"""'4350703E05F43A50'""" + +example_mode_aes_192_cfb8_datatype_LowCardinality_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_cfb8_datatype_Array_key_24_iv_16 = r"""'6926'""" + +example_mode_aes_192_cfb8_datatype_NULL_key_24_iv_16 = r"""'\\N'""" + +example_mode_aes_192_cfb8_datatype_IPv4_key_24_iv_16 = r"""'45979A4C'""" + +example_mode_aes_192_cfb8_datatype_IPv6_key_24_iv_16 = r"""'484BFA49756D837181B7EE03EBCF2B79'""" + +example_mode_aes_192_cfb8_datatype_Enum8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_cfb8_datatype_Enum16_key_24_iv_16 = r"""'6924'""" + +example_mode_aes_192_cfb8_datatype_bytes_key_32_iv_32 = r"""'31'""" + +example_mode_aes_192_cfb8_datatype_emptystring_key_32_iv_32 = r"""''""" + +example_mode_aes_192_cfb8_datatype_utf8string_key_32_iv_32 = r"""'7625D50FBB8F39E353302002F643A769A00CA77A81D316AB'""" + +example_mode_aes_192_cfb8_datatype_utf8fixedstring_key_32_iv_32 = r"""'7625D50FBB8F39E353302002F643A769A00CA77A81D316AB'""" + +example_mode_aes_192_cfb8_datatype_String_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_cfb8_datatype_FixedString_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_cfb8_datatype_UInt8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_cfb8_datatype_UInt16_key_32_iv_32 = r"""'3032'""" + +example_mode_aes_192_cfb8_datatype_UInt32_key_32_iv_32 = r"""'30325F8E'""" + +example_mode_aes_192_cfb8_datatype_UInt64_key_32_iv_32 = r"""'30325F8E9BE4C241'""" + +example_mode_aes_192_cfb8_datatype_Int8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_cfb8_datatype_Int16_key_32_iv_32 = r"""'3032'""" + +example_mode_aes_192_cfb8_datatype_Int32_key_32_iv_32 = r"""'30325F8E'""" + +example_mode_aes_192_cfb8_datatype_Int64_key_32_iv_32 = r"""'30325F8E9BE4C241'""" + +example_mode_aes_192_cfb8_datatype_Float32_key_32_iv_32 = r"""'3132C141'""" + +example_mode_aes_192_cfb8_datatype_Float64_key_32_iv_32 = r"""'3132410E0E8B4763'""" + +example_mode_aes_192_cfb8_datatype_Decimal32_key_32_iv_32 = r"""'1143551C'""" + +example_mode_aes_192_cfb8_datatype_Decimal64_key_32_iv_32 = r"""'1143551CF390615B'""" + +example_mode_aes_192_cfb8_datatype_Decimal128_key_32_iv_32 = r"""'1143551CF390615BB25617FAD1010676'""" + +example_mode_aes_192_cfb8_datatype_UUID_key_32_iv_32 = r"""'D63F190E24A3EF3805EF4E394CD5C620'""" + +example_mode_aes_192_cfb8_datatype_Date_key_32_iv_32 = r"""'67FC'""" + +example_mode_aes_192_cfb8_datatype_DateTime_key_32_iv_32 = r"""'7FDB8ABC'""" + +example_mode_aes_192_cfb8_datatype_DateTime64_key_32_iv_32 = r"""'1A304FE168BFC721'""" + +example_mode_aes_192_cfb8_datatype_LowCardinality_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_cfb8_datatype_Array_key_32_iv_32 = r"""'3030'""" + +example_mode_aes_192_cfb8_datatype_NULL_key_32_iv_32 = r"""'\\N'""" + +example_mode_aes_192_cfb8_datatype_IPv4_key_32_iv_32 = r"""'1CE26C68'""" + +example_mode_aes_192_cfb8_datatype_IPv6_key_32_iv_32 = r"""'110CAE7AAE3AF3436C8D155FD4DFACB9'""" + +example_mode_aes_192_cfb8_datatype_Enum8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_cfb8_datatype_Enum16_key_32_iv_32 = r"""'3032'""" + +example_mode_aes_256_cfb8_datatype_bytes_key_32_iv_16 = r"""'69'""" + +example_mode_aes_256_cfb8_datatype_emptystring_key_32_iv_16 = r"""''""" + +example_mode_aes_256_cfb8_datatype_utf8string_key_32_iv_16 = r"""'2E79EE96B485FC5BF3BE56AD461AAC2B1CCB425F51679553'""" + +example_mode_aes_256_cfb8_datatype_utf8fixedstring_key_32_iv_16 = r"""'2E79EE96B485FC5BF3BE56AD461AAC2B1CCB425F51679553'""" + +example_mode_aes_256_cfb8_datatype_String_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_cfb8_datatype_FixedString_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_cfb8_datatype_UInt8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_cfb8_datatype_UInt16_key_32_iv_16 = r"""'682C'""" + +example_mode_aes_256_cfb8_datatype_UInt32_key_32_iv_16 = r"""'682CE0A9'""" + +example_mode_aes_256_cfb8_datatype_UInt64_key_32_iv_16 = r"""'682CE0A9FFAF55AE'""" + +example_mode_aes_256_cfb8_datatype_Int8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_cfb8_datatype_Int16_key_32_iv_16 = r"""'682C'""" + +example_mode_aes_256_cfb8_datatype_Int32_key_32_iv_16 = r"""'682CE0A9'""" + +example_mode_aes_256_cfb8_datatype_Int64_key_32_iv_16 = r"""'682CE0A9FFAF55AE'""" + +example_mode_aes_256_cfb8_datatype_Float32_key_32_iv_16 = r"""'69B127F9'""" + +example_mode_aes_256_cfb8_datatype_Float64_key_32_iv_16 = r"""'69B1A72CB81A0FFF'""" + +example_mode_aes_256_cfb8_datatype_Decimal32_key_32_iv_16 = r"""'49378750'""" + +example_mode_aes_256_cfb8_datatype_Decimal64_key_32_iv_16 = r"""'493787505DFF5606'""" + +example_mode_aes_256_cfb8_datatype_Decimal128_key_32_iv_16 = r"""'493787505DFF5606774649C631E6E0E7'""" + +example_mode_aes_256_cfb8_datatype_UUID_key_32_iv_16 = r"""'8E09A60AA21565C888B2D92942896930'""" + +example_mode_aes_256_cfb8_datatype_Date_key_32_iv_16 = r"""'3FF1'""" + +example_mode_aes_256_cfb8_datatype_DateTime_key_32_iv_16 = r"""'274E13D8'""" + +example_mode_aes_256_cfb8_datatype_DateTime64_key_32_iv_16 = r"""'4211DFF611769F37'""" + +example_mode_aes_256_cfb8_datatype_LowCardinality_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_cfb8_datatype_Array_key_32_iv_16 = r"""'682E'""" + +example_mode_aes_256_cfb8_datatype_NULL_key_32_iv_16 = r"""'\\N'""" + +example_mode_aes_256_cfb8_datatype_IPv4_key_32_iv_16 = r"""'442DB771'""" + +example_mode_aes_256_cfb8_datatype_IPv6_key_32_iv_16 = r"""'4978AF3EED91F4AD14F7C326CCD96804'""" + +example_mode_aes_256_cfb8_datatype_Enum8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_cfb8_datatype_Enum16_key_32_iv_16 = r"""'682C'""" + +example_mode_aes_256_cfb8_datatype_bytes_key_64_iv_64 = r"""'D3'""" + +example_mode_aes_256_cfb8_datatype_emptystring_key_64_iv_64 = r"""''""" + +example_mode_aes_256_cfb8_datatype_utf8string_key_64_iv_64 = r"""'94336E21D7AF28FAA2179962917236281AB6CA2EB17A5F52'""" + +example_mode_aes_256_cfb8_datatype_utf8fixedstring_key_64_iv_64 = r"""'94336E21D7AF28FAA2179962917236281AB6CA2EB17A5F52'""" + +example_mode_aes_256_cfb8_datatype_String_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_cfb8_datatype_FixedString_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_cfb8_datatype_UInt8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_cfb8_datatype_UInt16_key_64_iv_64 = r"""'D2F0'""" + +example_mode_aes_256_cfb8_datatype_UInt32_key_64_iv_64 = r"""'D2F009AE'""" + +example_mode_aes_256_cfb8_datatype_UInt64_key_64_iv_64 = r"""'D2F009AE234CE3B2'""" + +example_mode_aes_256_cfb8_datatype_Int8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_cfb8_datatype_Int16_key_64_iv_64 = r"""'D2F0'""" + +example_mode_aes_256_cfb8_datatype_Int32_key_64_iv_64 = r"""'D2F009AE'""" + +example_mode_aes_256_cfb8_datatype_Int64_key_64_iv_64 = r"""'D2F009AE234CE3B2'""" + +example_mode_aes_256_cfb8_datatype_Float32_key_64_iv_64 = r"""'D323A6EE'""" + +example_mode_aes_256_cfb8_datatype_Float64_key_64_iv_64 = r"""'D32326952E7D73B6'""" + +example_mode_aes_256_cfb8_datatype_Decimal32_key_64_iv_64 = r"""'F3600DB1'""" + +example_mode_aes_256_cfb8_datatype_Decimal64_key_64_iv_64 = r"""'F3600DB150D7A5EF'""" + +example_mode_aes_256_cfb8_datatype_Decimal128_key_64_iv_64 = r"""'F3600DB150D7A5EFA1934028A5BDD856'""" + +example_mode_aes_256_cfb8_datatype_UUID_key_64_iv_64 = r"""'34B0729B2D7A5D08043DBF031FE464BC'""" + +example_mode_aes_256_cfb8_datatype_Date_key_64_iv_64 = r"""'85A9'""" + +example_mode_aes_256_cfb8_datatype_DateTime_key_64_iv_64 = r"""'9D298352'""" + +example_mode_aes_256_cfb8_datatype_DateTime64_key_64_iv_64 = r"""'F8BC2B190E2139CC'""" + +example_mode_aes_256_cfb8_datatype_LowCardinality_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_cfb8_datatype_Array_key_64_iv_64 = r"""'D2F2'""" + +example_mode_aes_256_cfb8_datatype_NULL_key_64_iv_64 = r"""'\\N'""" + +example_mode_aes_256_cfb8_datatype_IPv4_key_64_iv_64 = r"""'FEEA2143'""" + +example_mode_aes_256_cfb8_datatype_IPv6_key_64_iv_64 = r"""'F32F6C4967EF6EFF9975263D9074D4E1'""" + +example_mode_aes_256_cfb8_datatype_Enum8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_cfb8_datatype_Enum16_key_64_iv_64 = r"""'D2F0'""" + +example_mode_aes_128_cfb128_datatype_bytes_key_16_iv_None = r"""'10'""" + +example_mode_aes_128_cfb128_datatype_emptystring_key_16_iv_None = r"""''""" + +example_mode_aes_128_cfb128_datatype_utf8string_key_16_iv_None = r"""'571C627072083ECFD8460B39C4132D1B2802275B5B24EF73'""" + +example_mode_aes_128_cfb128_datatype_utf8fixedstring_key_16_iv_None = r"""'571C627072083ECFD8460B39C4132D1B2802275B5B24EF73'""" + +example_mode_aes_128_cfb128_datatype_String_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_cfb128_datatype_FixedString_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_cfb128_datatype_UInt8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_cfb128_datatype_UInt16_key_16_iv_None = r"""'11DF'""" + +example_mode_aes_128_cfb128_datatype_UInt32_key_16_iv_None = r"""'11DFC1B5'""" + +example_mode_aes_128_cfb128_datatype_UInt64_key_16_iv_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_cfb128_datatype_Int8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_cfb128_datatype_Int16_key_16_iv_None = r"""'11DF'""" + +example_mode_aes_128_cfb128_datatype_Int32_key_16_iv_None = r"""'11DFC1B5'""" + +example_mode_aes_128_cfb128_datatype_Int64_key_16_iv_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_cfb128_datatype_Float32_key_16_iv_None = r"""'10DF418A'""" + +example_mode_aes_128_cfb128_datatype_Float64_key_16_iv_None = r"""'10DFC1B5F66C0D55'""" + +example_mode_aes_128_cfb128_datatype_Decimal32_key_16_iv_None = r"""'3091C1B5'""" + +example_mode_aes_128_cfb128_datatype_Decimal64_key_16_iv_None = r"""'3091C1B5F66CFD6A'""" + +example_mode_aes_128_cfb128_datatype_Decimal128_key_16_iv_None = r"""'3091C1B5F66CFD6A1DC46D66907BEEB1'""" + +example_mode_aes_128_cfb128_datatype_UUID_key_16_iv_None = r"""'F7CE72E9F2A80D0BBD1FBE0C90DD9521'""" + +example_mode_aes_128_cfb128_datatype_Date_key_16_iv_None = r"""'4698'""" + +example_mode_aes_128_cfb128_datatype_DateTime_key_16_iv_None = r"""'5E0FCDEB'""" + +example_mode_aes_128_cfb128_datatype_DateTime64_key_16_iv_None = r"""'3B6ECCD7996DFD6A'""" + +example_mode_aes_128_cfb128_datatype_LowCardinality_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_cfb128_datatype_Array_key_16_iv_None = r"""'11DD'""" + +example_mode_aes_128_cfb128_datatype_NULL_key_16_iv_None = r"""'\\N'""" + +example_mode_aes_128_cfb128_datatype_IPv4_key_16_iv_None = r"""'3D5D201E'""" + +example_mode_aes_128_cfb128_datatype_IPv6_key_16_iv_None = r"""'30DECC0DF66C78C91DC46D663C646EB0'""" + +example_mode_aes_128_cfb128_datatype_Enum8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_cfb128_datatype_Enum16_key_16_iv_None = r"""'11DF'""" + +example_mode_aes_192_cfb128_datatype_bytes_key_24_iv_None = r"""'07'""" + +example_mode_aes_192_cfb128_datatype_emptystring_key_24_iv_None = r"""''""" + +example_mode_aes_192_cfb128_datatype_utf8string_key_24_iv_None = r"""'4074BA58B958623BE94C3FCF833DDDD9AC9F875CC2784719'""" + +example_mode_aes_192_cfb128_datatype_utf8fixedstring_key_24_iv_None = r"""'4074BA58B958623BE94C3FCF833DDDD9AC9F875CC2784719'""" + +example_mode_aes_192_cfb128_datatype_String_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_cfb128_datatype_FixedString_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_cfb128_datatype_UInt8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_cfb128_datatype_UInt16_key_24_iv_None = r"""'06B7'""" + +example_mode_aes_192_cfb128_datatype_UInt32_key_24_iv_None = r"""'06B7199D'""" + +example_mode_aes_192_cfb128_datatype_UInt64_key_24_iv_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_cfb128_datatype_Int8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_cfb128_datatype_Int16_key_24_iv_None = r"""'06B7'""" + +example_mode_aes_192_cfb128_datatype_Int32_key_24_iv_None = r"""'06B7199D'""" + +example_mode_aes_192_cfb128_datatype_Int64_key_24_iv_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_cfb128_datatype_Float32_key_24_iv_None = r"""'07B799A2'""" + +example_mode_aes_192_cfb128_datatype_Float64_key_24_iv_None = r"""'07B7199D3D3C51A1'""" + +example_mode_aes_192_cfb128_datatype_Decimal32_key_24_iv_None = r"""'27F9199D'""" + +example_mode_aes_192_cfb128_datatype_Decimal64_key_24_iv_None = r"""'27F9199D3D3CA19E'""" + +example_mode_aes_192_cfb128_datatype_Decimal128_key_24_iv_None = r"""'27F9199D3D3CA19E2CCE5990D7551E73'""" + +example_mode_aes_192_cfb128_datatype_UUID_key_24_iv_None = r"""'E0A6AAC139F851FF8C158AFAD7F365E3'""" + +example_mode_aes_192_cfb128_datatype_Date_key_24_iv_None = r"""'51F0'""" + +example_mode_aes_192_cfb128_datatype_DateTime_key_24_iv_None = r"""'496715C3'""" + +example_mode_aes_192_cfb128_datatype_DateTime64_key_24_iv_None = r"""'2C0614FF523DA19E'""" + +example_mode_aes_192_cfb128_datatype_LowCardinality_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_cfb128_datatype_Array_key_24_iv_None = r"""'06B5'""" + +example_mode_aes_192_cfb128_datatype_NULL_key_24_iv_None = r"""'\\N'""" + +example_mode_aes_192_cfb128_datatype_IPv4_key_24_iv_None = r"""'2A35F836'""" + +example_mode_aes_192_cfb128_datatype_IPv6_key_24_iv_None = r"""'27B614253D3C243D2CCE59907B4A9E72'""" + +example_mode_aes_192_cfb128_datatype_Enum8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_cfb128_datatype_Enum16_key_24_iv_None = r"""'06B7'""" + +example_mode_aes_256_cfb128_datatype_bytes_key_32_iv_None = r"""'B0'""" + +example_mode_aes_256_cfb128_datatype_emptystring_key_32_iv_None = r"""''""" + +example_mode_aes_256_cfb128_datatype_utf8string_key_32_iv_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06F5E4BB666EC25FE2'""" + +example_mode_aes_256_cfb128_datatype_utf8fixedstring_key_32_iv_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06F5E4BB666EC25FE2'""" + +example_mode_aes_256_cfb128_datatype_String_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_cfb128_datatype_FixedString_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_cfb128_datatype_UInt8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_cfb128_datatype_UInt16_key_32_iv_None = r"""'B18E'""" + +example_mode_aes_256_cfb128_datatype_UInt32_key_32_iv_None = r"""'B18ECF9E'""" + +example_mode_aes_256_cfb128_datatype_UInt64_key_32_iv_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_cfb128_datatype_Int8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_cfb128_datatype_Int16_key_32_iv_None = r"""'B18E'""" + +example_mode_aes_256_cfb128_datatype_Int32_key_32_iv_None = r"""'B18ECF9E'""" + +example_mode_aes_256_cfb128_datatype_Int64_key_32_iv_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_cfb128_datatype_Float32_key_32_iv_None = r"""'B08E4FA1'""" + +example_mode_aes_256_cfb128_datatype_Float64_key_32_iv_None = r"""'B08ECF9EC7EBAF32'""" + +example_mode_aes_256_cfb128_datatype_Decimal32_key_32_iv_None = r"""'90C0CF9E'""" + +example_mode_aes_256_cfb128_datatype_Decimal64_key_32_iv_None = r"""'90C0CF9EC7EB5F0D'""" + +example_mode_aes_256_cfb128_datatype_Decimal128_key_32_iv_None = r"""'90C0CF9EC7EB5F0D7B78C42556D668AC'""" + +example_mode_aes_256_cfb128_datatype_UUID_key_32_iv_None = r"""'579F7CC2C32FAF6CDBA3174F5670133C'""" + +example_mode_aes_256_cfb128_datatype_Date_key_32_iv_None = r"""'E6C9'""" + +example_mode_aes_256_cfb128_datatype_DateTime_key_32_iv_None = r"""'FE5EC3C0'""" + +example_mode_aes_256_cfb128_datatype_DateTime64_key_32_iv_None = r"""'9B3FC2FCA8EA5F0D'""" + +example_mode_aes_256_cfb128_datatype_LowCardinality_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_cfb128_datatype_Array_key_32_iv_None = r"""'B18C'""" + +example_mode_aes_256_cfb128_datatype_NULL_key_32_iv_None = r"""'\\N'""" + +example_mode_aes_256_cfb128_datatype_IPv4_key_32_iv_None = r"""'9D0C2E35'""" + +example_mode_aes_256_cfb128_datatype_IPv6_key_32_iv_None = r"""'908FC226C7EBDAAE7B78C425FAC9E8AD'""" + +example_mode_aes_256_cfb128_datatype_Enum8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_cfb128_datatype_Enum16_key_32_iv_None = r"""'B18E'""" + +example_mode_aes_128_cfb128_datatype_bytes_key_16_iv_16 = r"""'32'""" + +example_mode_aes_128_cfb128_datatype_emptystring_key_16_iv_16 = r"""''""" + +example_mode_aes_128_cfb128_datatype_utf8string_key_16_iv_16 = r"""'754B0A4159623CFD2CBB06EE8AADCCB4581E4F5FB9F091DD'""" + +example_mode_aes_128_cfb128_datatype_utf8fixedstring_key_16_iv_16 = r"""'754B0A4159623CFD2CBB06EE8AADCCB4581E4F5FB9F091DD'""" + +example_mode_aes_128_cfb128_datatype_String_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_cfb128_datatype_FixedString_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_cfb128_datatype_UInt8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_cfb128_datatype_UInt16_key_16_iv_16 = r"""'3388'""" + +example_mode_aes_128_cfb128_datatype_UInt32_key_16_iv_16 = r"""'3388A984'""" + +example_mode_aes_128_cfb128_datatype_UInt64_key_16_iv_16 = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_cfb128_datatype_Int8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_cfb128_datatype_Int16_key_16_iv_16 = r"""'3388'""" + +example_mode_aes_128_cfb128_datatype_Int32_key_16_iv_16 = r"""'3388A984'""" + +example_mode_aes_128_cfb128_datatype_Int64_key_16_iv_16 = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_cfb128_datatype_Float32_key_16_iv_16 = r"""'328829BB'""" + +example_mode_aes_128_cfb128_datatype_Float64_key_16_iv_16 = r"""'3288A984DD060F67'""" + +example_mode_aes_128_cfb128_datatype_Decimal32_key_16_iv_16 = r"""'12C6A984'""" + +example_mode_aes_128_cfb128_datatype_Decimal64_key_16_iv_16 = r"""'12C6A984DD06FF58'""" + +example_mode_aes_128_cfb128_datatype_Decimal128_key_16_iv_16 = r"""'12C6A984DD06FF58E93960B1DEC50F1E'""" + +example_mode_aes_128_cfb128_datatype_UUID_key_16_iv_16 = r"""'D5991AD8D9C20F3949E2B3DBDE63748E'""" + +example_mode_aes_128_cfb128_datatype_Date_key_16_iv_16 = r"""'64CF'""" + +example_mode_aes_128_cfb128_datatype_DateTime_key_16_iv_16 = r"""'7C58A5DA'""" + +example_mode_aes_128_cfb128_datatype_DateTime64_key_16_iv_16 = r"""'1939A4E6B207FF58'""" + +example_mode_aes_128_cfb128_datatype_LowCardinality_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_cfb128_datatype_Array_key_16_iv_16 = r"""'338A'""" + +example_mode_aes_128_cfb128_datatype_NULL_key_16_iv_16 = r"""'\\N'""" + +example_mode_aes_128_cfb128_datatype_IPv4_key_16_iv_16 = r"""'1F0A482F'""" + +example_mode_aes_128_cfb128_datatype_IPv6_key_16_iv_16 = r"""'1289A43CDD067AFBE93960B172DA8F1F'""" + +example_mode_aes_128_cfb128_datatype_Enum8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_cfb128_datatype_Enum16_key_16_iv_16 = r"""'3388'""" + +example_mode_aes_128_cfb128_datatype_bytes_key_24_iv_24 = r"""'5B'""" + +example_mode_aes_128_cfb128_datatype_emptystring_key_24_iv_24 = r"""''""" + +example_mode_aes_128_cfb128_datatype_utf8string_key_24_iv_24 = r"""'1CB4A1306DD44E12FB99CA2398A56D5C6FFFA5BE700B5B9B'""" + +example_mode_aes_128_cfb128_datatype_utf8fixedstring_key_24_iv_24 = r"""'1CB4A1306DD44E12FB99CA2398A56D5C6FFFA5BE700B5B9B'""" + +example_mode_aes_128_cfb128_datatype_String_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_cfb128_datatype_FixedString_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_cfb128_datatype_UInt8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_cfb128_datatype_UInt16_key_24_iv_24 = r"""'5A77'""" + +example_mode_aes_128_cfb128_datatype_UInt32_key_24_iv_24 = r"""'5A7702F5'""" + +example_mode_aes_128_cfb128_datatype_UInt64_key_24_iv_24 = r"""'5A7702F5E9B08DB7'""" + +example_mode_aes_128_cfb128_datatype_Int8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_cfb128_datatype_Int16_key_24_iv_24 = r"""'5A77'""" + +example_mode_aes_128_cfb128_datatype_Int32_key_24_iv_24 = r"""'5A7702F5'""" + +example_mode_aes_128_cfb128_datatype_Int64_key_24_iv_24 = r"""'5A7702F5E9B08DB7'""" + +example_mode_aes_128_cfb128_datatype_Float32_key_24_iv_24 = r"""'5B7782CA'""" + +example_mode_aes_128_cfb128_datatype_Float64_key_24_iv_24 = r"""'5B7702F5E9B07D88'""" + +example_mode_aes_128_cfb128_datatype_Decimal32_key_24_iv_24 = r"""'7B3902F5'""" + +example_mode_aes_128_cfb128_datatype_Decimal64_key_24_iv_24 = r"""'7B3902F5E9B08DB7'""" + +example_mode_aes_128_cfb128_datatype_Decimal128_key_24_iv_24 = r"""'7B3902F5E9B08DB73E1BAC7CCCCDAEF6'""" + +example_mode_aes_128_cfb128_datatype_UUID_key_24_iv_24 = r"""'BC66B1A9ED747DD69EC07F16CC6BD566'""" + +example_mode_aes_128_cfb128_datatype_Date_key_24_iv_24 = r"""'0D30'""" + +example_mode_aes_128_cfb128_datatype_DateTime_key_24_iv_24 = r"""'15A70EAB'""" + +example_mode_aes_128_cfb128_datatype_DateTime64_key_24_iv_24 = r"""'70C60F9786B18DB7'""" + +example_mode_aes_128_cfb128_datatype_LowCardinality_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_cfb128_datatype_Array_key_24_iv_24 = r"""'5A75'""" + +example_mode_aes_128_cfb128_datatype_NULL_key_24_iv_24 = r"""'\\N'""" + +example_mode_aes_128_cfb128_datatype_IPv4_key_24_iv_24 = r"""'76F5E35E'""" + +example_mode_aes_128_cfb128_datatype_IPv6_key_24_iv_24 = r"""'7B760F4DE9B008143E1BAC7C60D22EF7'""" + +example_mode_aes_128_cfb128_datatype_Enum8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_cfb128_datatype_Enum16_key_24_iv_24 = r"""'5A77'""" + +example_mode_aes_192_cfb128_datatype_bytes_key_24_iv_16 = r"""'68'""" + +example_mode_aes_192_cfb128_datatype_emptystring_key_24_iv_16 = r"""''""" + +example_mode_aes_192_cfb128_datatype_utf8string_key_24_iv_16 = r"""'2F0444573374B41441C46351EBB0A21FD2D5B29B19D817D8'""" + +example_mode_aes_192_cfb128_datatype_utf8fixedstring_key_24_iv_16 = r"""'2F0444573374B41441C46351EBB0A21FD2D5B29B19D817D8'""" + +example_mode_aes_192_cfb128_datatype_String_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_cfb128_datatype_FixedString_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_cfb128_datatype_UInt8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_cfb128_datatype_UInt16_key_24_iv_16 = r"""'69C7'""" + +example_mode_aes_192_cfb128_datatype_UInt32_key_24_iv_16 = r"""'69C7E792'""" + +example_mode_aes_192_cfb128_datatype_UInt64_key_24_iv_16 = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_cfb128_datatype_Int8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_cfb128_datatype_Int16_key_24_iv_16 = r"""'69C7'""" + +example_mode_aes_192_cfb128_datatype_Int32_key_24_iv_16 = r"""'69C7E792'""" + +example_mode_aes_192_cfb128_datatype_Int64_key_24_iv_16 = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_cfb128_datatype_Float32_key_24_iv_16 = r"""'68C767AD'""" + +example_mode_aes_192_cfb128_datatype_Float64_key_24_iv_16 = r"""'68C7E792B710878E'""" + +example_mode_aes_192_cfb128_datatype_Decimal32_key_24_iv_16 = r"""'4889E792'""" + +example_mode_aes_192_cfb128_datatype_Decimal64_key_24_iv_16 = r"""'4889E792B71077B1'""" + +example_mode_aes_192_cfb128_datatype_Decimal128_key_24_iv_16 = r"""'4889E792B71077B18446050EBFD861B5'""" + +example_mode_aes_192_cfb128_datatype_UUID_key_24_iv_16 = r"""'8FD654CEB3D487D0249DD664BF7E1A25'""" + +example_mode_aes_192_cfb128_datatype_Date_key_24_iv_16 = r"""'3E80'""" + +example_mode_aes_192_cfb128_datatype_DateTime_key_24_iv_16 = r"""'2617EBCC'""" + +example_mode_aes_192_cfb128_datatype_DateTime64_key_24_iv_16 = r"""'4376EAF0D81177B1'""" + +example_mode_aes_192_cfb128_datatype_LowCardinality_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_cfb128_datatype_Array_key_24_iv_16 = r"""'69C5'""" + +example_mode_aes_192_cfb128_datatype_NULL_key_24_iv_16 = r"""'\\N'""" + +example_mode_aes_192_cfb128_datatype_IPv4_key_24_iv_16 = r"""'45450639'""" + +example_mode_aes_192_cfb128_datatype_IPv6_key_24_iv_16 = r"""'48C6EA2AB710F2128446050E13C7E1B4'""" + +example_mode_aes_192_cfb128_datatype_Enum8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_cfb128_datatype_Enum16_key_24_iv_16 = r"""'69C7'""" + +example_mode_aes_192_cfb128_datatype_bytes_key_32_iv_32 = r"""'31'""" + +example_mode_aes_192_cfb128_datatype_emptystring_key_32_iv_32 = r"""''""" + +example_mode_aes_192_cfb128_datatype_utf8string_key_32_iv_32 = r"""'76632DB12BCFF36187A90B6990CFA6D8D9CFB425308D13E0'""" + +example_mode_aes_192_cfb128_datatype_utf8fixedstring_key_32_iv_32 = r"""'76632DB12BCFF36187A90B6990CFA6D8D9CFB425308D13E0'""" + +example_mode_aes_192_cfb128_datatype_String_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_cfb128_datatype_FixedString_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_cfb128_datatype_UInt8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_cfb128_datatype_UInt16_key_32_iv_32 = r"""'30A0'""" + +example_mode_aes_192_cfb128_datatype_UInt32_key_32_iv_32 = r"""'30A08E74'""" + +example_mode_aes_192_cfb128_datatype_UInt64_key_32_iv_32 = r"""'30A08E74AFAB30C4'""" + +example_mode_aes_192_cfb128_datatype_Int8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_cfb128_datatype_Int16_key_32_iv_32 = r"""'30A0'""" + +example_mode_aes_192_cfb128_datatype_Int32_key_32_iv_32 = r"""'30A08E74'""" + +example_mode_aes_192_cfb128_datatype_Int64_key_32_iv_32 = r"""'30A08E74AFAB30C4'""" + +example_mode_aes_192_cfb128_datatype_Float32_key_32_iv_32 = r"""'31A00E4B'""" + +example_mode_aes_192_cfb128_datatype_Float64_key_32_iv_32 = r"""'31A08E74AFABC0FB'""" + +example_mode_aes_192_cfb128_datatype_Decimal32_key_32_iv_32 = r"""'11EE8E74'""" + +example_mode_aes_192_cfb128_datatype_Decimal64_key_32_iv_32 = r"""'11EE8E74AFAB30C4'""" + +example_mode_aes_192_cfb128_datatype_Decimal128_key_32_iv_32 = r"""'11EE8E74AFAB30C4422B6D36C4A76572'""" + +example_mode_aes_192_cfb128_datatype_UUID_key_32_iv_32 = r"""'D6B13D28AB6FC0A5E2F0BE5CC4011EE2'""" + +example_mode_aes_192_cfb128_datatype_Date_key_32_iv_32 = r"""'67E7'""" + +example_mode_aes_192_cfb128_datatype_DateTime_key_32_iv_32 = r"""'7F70822A'""" + +example_mode_aes_192_cfb128_datatype_DateTime64_key_32_iv_32 = r"""'1A118316C0AA30C4'""" + +example_mode_aes_192_cfb128_datatype_LowCardinality_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_cfb128_datatype_Array_key_32_iv_32 = r"""'30A2'""" + +example_mode_aes_192_cfb128_datatype_NULL_key_32_iv_32 = r"""'\\N'""" + +example_mode_aes_192_cfb128_datatype_IPv4_key_32_iv_32 = r"""'1C226FDF'""" + +example_mode_aes_192_cfb128_datatype_IPv6_key_32_iv_32 = r"""'11A183CCAFABB567422B6D3668B8E573'""" + +example_mode_aes_192_cfb128_datatype_Enum8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_cfb128_datatype_Enum16_key_32_iv_32 = r"""'30A0'""" + +example_mode_aes_256_cfb128_datatype_bytes_key_32_iv_16 = r"""'69'""" + +example_mode_aes_256_cfb128_datatype_emptystring_key_32_iv_16 = r"""''""" + +example_mode_aes_256_cfb128_datatype_utf8string_key_32_iv_16 = r"""'2E9B2BD2B8BA872DB56225F82754048C22AA31B7F22AD276'""" + +example_mode_aes_256_cfb128_datatype_utf8fixedstring_key_32_iv_16 = r"""'2E9B2BD2B8BA872DB56225F82754048C22AA31B7F22AD276'""" + +example_mode_aes_256_cfb128_datatype_String_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_cfb128_datatype_FixedString_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_cfb128_datatype_UInt8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_cfb128_datatype_UInt16_key_32_iv_16 = r"""'6858'""" + +example_mode_aes_256_cfb128_datatype_UInt32_key_32_iv_16 = r"""'68588817'""" + +example_mode_aes_256_cfb128_datatype_UInt64_key_32_iv_16 = r"""'685888173CDE4488'""" + +example_mode_aes_256_cfb128_datatype_Int8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_cfb128_datatype_Int16_key_32_iv_16 = r"""'6858'""" + +example_mode_aes_256_cfb128_datatype_Int32_key_32_iv_16 = r"""'68588817'""" + +example_mode_aes_256_cfb128_datatype_Int64_key_32_iv_16 = r"""'685888173CDE4488'""" + +example_mode_aes_256_cfb128_datatype_Float32_key_32_iv_16 = r"""'69580828'""" + +example_mode_aes_256_cfb128_datatype_Float64_key_32_iv_16 = r"""'695888173CDEB4B7'""" + +example_mode_aes_256_cfb128_datatype_Decimal32_key_32_iv_16 = r"""'49168817'""" + +example_mode_aes_256_cfb128_datatype_Decimal64_key_32_iv_16 = r"""'491688173CDE4488'""" + +example_mode_aes_256_cfb128_datatype_Decimal128_key_32_iv_16 = r"""'491688173CDE448870E043A7733CC726'""" + +example_mode_aes_256_cfb128_datatype_UUID_key_32_iv_16 = r"""'8E493B4B381AB4E9D03B90CD739ABCB6'""" + +example_mode_aes_256_cfb128_datatype_Date_key_32_iv_16 = r"""'3F1F'""" + +example_mode_aes_256_cfb128_datatype_DateTime_key_32_iv_16 = r"""'27888449'""" + +example_mode_aes_256_cfb128_datatype_DateTime64_key_32_iv_16 = r"""'42E9857553DF4488'""" + +example_mode_aes_256_cfb128_datatype_LowCardinality_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_cfb128_datatype_Array_key_32_iv_16 = r"""'685A'""" + +example_mode_aes_256_cfb128_datatype_NULL_key_32_iv_16 = r"""'\\N'""" + +example_mode_aes_256_cfb128_datatype_IPv4_key_32_iv_16 = r"""'44DA69BC'""" + +example_mode_aes_256_cfb128_datatype_IPv6_key_32_iv_16 = r"""'495985AF3CDEC12B70E043A7DF234727'""" + +example_mode_aes_256_cfb128_datatype_Enum8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_cfb128_datatype_Enum16_key_32_iv_16 = r"""'6858'""" + +example_mode_aes_256_cfb128_datatype_bytes_key_64_iv_64 = r"""'D3'""" + +example_mode_aes_256_cfb128_datatype_emptystring_key_64_iv_64 = r"""''""" + +example_mode_aes_256_cfb128_datatype_utf8string_key_64_iv_64 = r"""'942D6C993F1DE6D874AD5CCF2109CE7D9EC333A5AE718F82'""" + +example_mode_aes_256_cfb128_datatype_utf8fixedstring_key_64_iv_64 = r"""'942D6C993F1DE6D874AD5CCF2109CE7D9EC333A5AE718F82'""" + +example_mode_aes_256_cfb128_datatype_String_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_cfb128_datatype_FixedString_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_cfb128_datatype_UInt8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_cfb128_datatype_UInt16_key_64_iv_64 = r"""'D2EE'""" + +example_mode_aes_256_cfb128_datatype_UInt32_key_64_iv_64 = r"""'D2EECF5C'""" + +example_mode_aes_256_cfb128_datatype_UInt64_key_64_iv_64 = r"""'D2EECF5CBB79257D'""" + +example_mode_aes_256_cfb128_datatype_Int8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_cfb128_datatype_Int16_key_64_iv_64 = r"""'D2EE'""" + +example_mode_aes_256_cfb128_datatype_Int32_key_64_iv_64 = r"""'D2EECF5C'""" + +example_mode_aes_256_cfb128_datatype_Int64_key_64_iv_64 = r"""'D2EECF5CBB79257D'""" + +example_mode_aes_256_cfb128_datatype_Float32_key_64_iv_64 = r"""'D3EE4F63'""" + +example_mode_aes_256_cfb128_datatype_Float64_key_64_iv_64 = r"""'D3EECF5CBB79D542'""" + +example_mode_aes_256_cfb128_datatype_Decimal32_key_64_iv_64 = r"""'F3A0CF5C'""" + +example_mode_aes_256_cfb128_datatype_Decimal64_key_64_iv_64 = r"""'F3A0CF5CBB79257D'""" + +example_mode_aes_256_cfb128_datatype_Decimal128_key_64_iv_64 = r"""'F3A0CF5CBB79257DB12F3A9075610DD7'""" + +example_mode_aes_256_cfb128_datatype_UUID_key_64_iv_64 = r"""'34FF7C00BFBDD51C11F4E9FA75C77647'""" + +example_mode_aes_256_cfb128_datatype_Date_key_64_iv_64 = r"""'85A9'""" + +example_mode_aes_256_cfb128_datatype_DateTime_key_64_iv_64 = r"""'9D3EC302'""" + +example_mode_aes_256_cfb128_datatype_DateTime64_key_64_iv_64 = r"""'F85FC23ED478257D'""" + +example_mode_aes_256_cfb128_datatype_LowCardinality_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_cfb128_datatype_Array_key_64_iv_64 = r"""'D2EC'""" + +example_mode_aes_256_cfb128_datatype_NULL_key_64_iv_64 = r"""'\\N'""" + +example_mode_aes_256_cfb128_datatype_IPv4_key_64_iv_64 = r"""'FE6C2EF7'""" + +example_mode_aes_256_cfb128_datatype_IPv6_key_64_iv_64 = r"""'F3EFC2E4BB79A0DEB12F3A90D97E8DD6'""" + +example_mode_aes_256_cfb128_datatype_Enum8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_cfb128_datatype_Enum16_key_64_iv_64 = r"""'D2EE'""" + +example_mode_aes_128_ofb_datatype_bytes_key_16_iv_None = r"""'10'""" + +example_mode_aes_128_ofb_datatype_emptystring_key_16_iv_None = r"""''""" + +example_mode_aes_128_ofb_datatype_utf8string_key_16_iv_None = r"""'571C627072083ECFD8460B39C4132D1B1EFEEBE7197398AE'""" + +example_mode_aes_128_ofb_datatype_utf8fixedstring_key_16_iv_None = r"""'571C627072083ECFD8460B39C4132D1B1EFEEBE7197398AE'""" + +example_mode_aes_128_ofb_datatype_String_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_ofb_datatype_FixedString_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_ofb_datatype_UInt8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_ofb_datatype_UInt16_key_16_iv_None = r"""'11DF'""" + +example_mode_aes_128_ofb_datatype_UInt32_key_16_iv_None = r"""'11DFC1B5'""" + +example_mode_aes_128_ofb_datatype_UInt64_key_16_iv_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_ofb_datatype_Int8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_ofb_datatype_Int16_key_16_iv_None = r"""'11DF'""" + +example_mode_aes_128_ofb_datatype_Int32_key_16_iv_None = r"""'11DFC1B5'""" + +example_mode_aes_128_ofb_datatype_Int64_key_16_iv_None = r"""'11DFC1B5F66CFD6A'""" + +example_mode_aes_128_ofb_datatype_Float32_key_16_iv_None = r"""'10DF418A'""" + +example_mode_aes_128_ofb_datatype_Float64_key_16_iv_None = r"""'10DFC1B5F66C0D55'""" + +example_mode_aes_128_ofb_datatype_Decimal32_key_16_iv_None = r"""'3091C1B5'""" + +example_mode_aes_128_ofb_datatype_Decimal64_key_16_iv_None = r"""'3091C1B5F66CFD6A'""" + +example_mode_aes_128_ofb_datatype_Decimal128_key_16_iv_None = r"""'3091C1B5F66CFD6A1DC46D66907BEEB1'""" + +example_mode_aes_128_ofb_datatype_UUID_key_16_iv_None = r"""'F7CE72E9F2A80D0BBD1FBE0C90DD9521'""" + +example_mode_aes_128_ofb_datatype_Date_key_16_iv_None = r"""'4698'""" + +example_mode_aes_128_ofb_datatype_DateTime_key_16_iv_None = r"""'5E0FCDEB'""" + +example_mode_aes_128_ofb_datatype_DateTime64_key_16_iv_None = r"""'3B6ECCD7996DFD6A'""" + +example_mode_aes_128_ofb_datatype_LowCardinality_key_16_iv_None = r"""'21'""" + +example_mode_aes_128_ofb_datatype_Array_key_16_iv_None = r"""'11DD'""" + +example_mode_aes_128_ofb_datatype_NULL_key_16_iv_None = r"""'\\N'""" + +example_mode_aes_128_ofb_datatype_IPv4_key_16_iv_None = r"""'3D5D201E'""" + +example_mode_aes_128_ofb_datatype_IPv6_key_16_iv_None = r"""'30DECC0DF66C78C91DC46D663C646EB0'""" + +example_mode_aes_128_ofb_datatype_Enum8_key_16_iv_None = r"""'11'""" + +example_mode_aes_128_ofb_datatype_Enum16_key_16_iv_None = r"""'11DF'""" + +example_mode_aes_192_ofb_datatype_bytes_key_24_iv_None = r"""'07'""" + +example_mode_aes_192_ofb_datatype_emptystring_key_24_iv_None = r"""''""" + +example_mode_aes_192_ofb_datatype_utf8string_key_24_iv_None = r"""'4074BA58B958623BE94C3FCF833DDDD95F6EFF17F7823E17'""" + +example_mode_aes_192_ofb_datatype_utf8fixedstring_key_24_iv_None = r"""'4074BA58B958623BE94C3FCF833DDDD95F6EFF17F7823E17'""" + +example_mode_aes_192_ofb_datatype_String_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_ofb_datatype_FixedString_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_ofb_datatype_UInt8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_ofb_datatype_UInt16_key_24_iv_None = r"""'06B7'""" + +example_mode_aes_192_ofb_datatype_UInt32_key_24_iv_None = r"""'06B7199D'""" + +example_mode_aes_192_ofb_datatype_UInt64_key_24_iv_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_ofb_datatype_Int8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_ofb_datatype_Int16_key_24_iv_None = r"""'06B7'""" + +example_mode_aes_192_ofb_datatype_Int32_key_24_iv_None = r"""'06B7199D'""" + +example_mode_aes_192_ofb_datatype_Int64_key_24_iv_None = r"""'06B7199D3D3CA19E'""" + +example_mode_aes_192_ofb_datatype_Float32_key_24_iv_None = r"""'07B799A2'""" + +example_mode_aes_192_ofb_datatype_Float64_key_24_iv_None = r"""'07B7199D3D3C51A1'""" + +example_mode_aes_192_ofb_datatype_Decimal32_key_24_iv_None = r"""'27F9199D'""" + +example_mode_aes_192_ofb_datatype_Decimal64_key_24_iv_None = r"""'27F9199D3D3CA19E'""" + +example_mode_aes_192_ofb_datatype_Decimal128_key_24_iv_None = r"""'27F9199D3D3CA19E2CCE5990D7551E73'""" + +example_mode_aes_192_ofb_datatype_UUID_key_24_iv_None = r"""'E0A6AAC139F851FF8C158AFAD7F365E3'""" + +example_mode_aes_192_ofb_datatype_Date_key_24_iv_None = r"""'51F0'""" + +example_mode_aes_192_ofb_datatype_DateTime_key_24_iv_None = r"""'496715C3'""" + +example_mode_aes_192_ofb_datatype_DateTime64_key_24_iv_None = r"""'2C0614FF523DA19E'""" + +example_mode_aes_192_ofb_datatype_LowCardinality_key_24_iv_None = r"""'36'""" + +example_mode_aes_192_ofb_datatype_Array_key_24_iv_None = r"""'06B5'""" + +example_mode_aes_192_ofb_datatype_NULL_key_24_iv_None = r"""'\\N'""" + +example_mode_aes_192_ofb_datatype_IPv4_key_24_iv_None = r"""'2A35F836'""" + +example_mode_aes_192_ofb_datatype_IPv6_key_24_iv_None = r"""'27B614253D3C243D2CCE59907B4A9E72'""" + +example_mode_aes_192_ofb_datatype_Enum8_key_24_iv_None = r"""'06'""" + +example_mode_aes_192_ofb_datatype_Enum16_key_24_iv_None = r"""'06B7'""" + +example_mode_aes_256_ofb_datatype_bytes_key_32_iv_None = r"""'B0'""" + +example_mode_aes_256_ofb_datatype_emptystring_key_32_iv_None = r"""''""" + +example_mode_aes_256_ofb_datatype_utf8string_key_32_iv_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06B24181EFC9F2663B'""" + +example_mode_aes_256_ofb_datatype_utf8fixedstring_key_32_iv_None = r"""'F74D6C5B438F9CA8BEFAA27A02BEAB06B24181EFC9F2663B'""" + +example_mode_aes_256_ofb_datatype_String_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_ofb_datatype_FixedString_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_ofb_datatype_UInt8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_ofb_datatype_UInt16_key_32_iv_None = r"""'B18E'""" + +example_mode_aes_256_ofb_datatype_UInt32_key_32_iv_None = r"""'B18ECF9E'""" + +example_mode_aes_256_ofb_datatype_UInt64_key_32_iv_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_ofb_datatype_Int8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_ofb_datatype_Int16_key_32_iv_None = r"""'B18E'""" + +example_mode_aes_256_ofb_datatype_Int32_key_32_iv_None = r"""'B18ECF9E'""" + +example_mode_aes_256_ofb_datatype_Int64_key_32_iv_None = r"""'B18ECF9EC7EB5F0D'""" + +example_mode_aes_256_ofb_datatype_Float32_key_32_iv_None = r"""'B08E4FA1'""" + +example_mode_aes_256_ofb_datatype_Float64_key_32_iv_None = r"""'B08ECF9EC7EBAF32'""" + +example_mode_aes_256_ofb_datatype_Decimal32_key_32_iv_None = r"""'90C0CF9E'""" + +example_mode_aes_256_ofb_datatype_Decimal64_key_32_iv_None = r"""'90C0CF9EC7EB5F0D'""" + +example_mode_aes_256_ofb_datatype_Decimal128_key_32_iv_None = r"""'90C0CF9EC7EB5F0D7B78C42556D668AC'""" + +example_mode_aes_256_ofb_datatype_UUID_key_32_iv_None = r"""'579F7CC2C32FAF6CDBA3174F5670133C'""" + +example_mode_aes_256_ofb_datatype_Date_key_32_iv_None = r"""'E6C9'""" + +example_mode_aes_256_ofb_datatype_DateTime_key_32_iv_None = r"""'FE5EC3C0'""" + +example_mode_aes_256_ofb_datatype_DateTime64_key_32_iv_None = r"""'9B3FC2FCA8EA5F0D'""" + +example_mode_aes_256_ofb_datatype_LowCardinality_key_32_iv_None = r"""'81'""" + +example_mode_aes_256_ofb_datatype_Array_key_32_iv_None = r"""'B18C'""" + +example_mode_aes_256_ofb_datatype_NULL_key_32_iv_None = r"""'\\N'""" + +example_mode_aes_256_ofb_datatype_IPv4_key_32_iv_None = r"""'9D0C2E35'""" + +example_mode_aes_256_ofb_datatype_IPv6_key_32_iv_None = r"""'908FC226C7EBDAAE7B78C425FAC9E8AD'""" + +example_mode_aes_256_ofb_datatype_Enum8_key_32_iv_None = r"""'B1'""" + +example_mode_aes_256_ofb_datatype_Enum16_key_32_iv_None = r"""'B18E'""" + +example_mode_aes_128_ofb_datatype_bytes_key_16_iv_16 = r"""'32'""" + +example_mode_aes_128_ofb_datatype_emptystring_key_16_iv_16 = r"""''""" + +example_mode_aes_128_ofb_datatype_utf8string_key_16_iv_16 = r"""'754B0A4159623CFD2CBB06EE8AADCCB46A1C2A356E7D91D8'""" + +example_mode_aes_128_ofb_datatype_utf8fixedstring_key_16_iv_16 = r"""'754B0A4159623CFD2CBB06EE8AADCCB46A1C2A356E7D91D8'""" + +example_mode_aes_128_ofb_datatype_String_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_ofb_datatype_FixedString_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_ofb_datatype_UInt8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_ofb_datatype_UInt16_key_16_iv_16 = r"""'3388'""" + +example_mode_aes_128_ofb_datatype_UInt32_key_16_iv_16 = r"""'3388A984'""" + +example_mode_aes_128_ofb_datatype_UInt64_key_16_iv_16 = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_ofb_datatype_Int8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_ofb_datatype_Int16_key_16_iv_16 = r"""'3388'""" + +example_mode_aes_128_ofb_datatype_Int32_key_16_iv_16 = r"""'3388A984'""" + +example_mode_aes_128_ofb_datatype_Int64_key_16_iv_16 = r"""'3388A984DD06FF58'""" + +example_mode_aes_128_ofb_datatype_Float32_key_16_iv_16 = r"""'328829BB'""" + +example_mode_aes_128_ofb_datatype_Float64_key_16_iv_16 = r"""'3288A984DD060F67'""" + +example_mode_aes_128_ofb_datatype_Decimal32_key_16_iv_16 = r"""'12C6A984'""" + +example_mode_aes_128_ofb_datatype_Decimal64_key_16_iv_16 = r"""'12C6A984DD06FF58'""" + +example_mode_aes_128_ofb_datatype_Decimal128_key_16_iv_16 = r"""'12C6A984DD06FF58E93960B1DEC50F1E'""" + +example_mode_aes_128_ofb_datatype_UUID_key_16_iv_16 = r"""'D5991AD8D9C20F3949E2B3DBDE63748E'""" + +example_mode_aes_128_ofb_datatype_Date_key_16_iv_16 = r"""'64CF'""" + +example_mode_aes_128_ofb_datatype_DateTime_key_16_iv_16 = r"""'7C58A5DA'""" + +example_mode_aes_128_ofb_datatype_DateTime64_key_16_iv_16 = r"""'1939A4E6B207FF58'""" + +example_mode_aes_128_ofb_datatype_LowCardinality_key_16_iv_16 = r"""'03'""" + +example_mode_aes_128_ofb_datatype_Array_key_16_iv_16 = r"""'338A'""" + +example_mode_aes_128_ofb_datatype_NULL_key_16_iv_16 = r"""'\\N'""" + +example_mode_aes_128_ofb_datatype_IPv4_key_16_iv_16 = r"""'1F0A482F'""" + +example_mode_aes_128_ofb_datatype_IPv6_key_16_iv_16 = r"""'1289A43CDD067AFBE93960B172DA8F1F'""" + +example_mode_aes_128_ofb_datatype_Enum8_key_16_iv_16 = r"""'33'""" + +example_mode_aes_128_ofb_datatype_Enum16_key_16_iv_16 = r"""'3388'""" + +example_mode_aes_128_ofb_datatype_bytes_key_24_iv_24 = r"""'5B'""" + +example_mode_aes_128_ofb_datatype_emptystring_key_24_iv_24 = r"""''""" + +example_mode_aes_128_ofb_datatype_utf8string_key_24_iv_24 = r"""'1CB4A1306DD44E12FB99CA2398A56D5CE691EFBA49066039'""" + +example_mode_aes_128_ofb_datatype_utf8fixedstring_key_24_iv_24 = r"""'1CB4A1306DD44E12FB99CA2398A56D5CE691EFBA49066039'""" + +example_mode_aes_128_ofb_datatype_String_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_ofb_datatype_FixedString_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_ofb_datatype_UInt8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_ofb_datatype_UInt16_key_24_iv_24 = r"""'5A77'""" + +example_mode_aes_128_ofb_datatype_UInt32_key_24_iv_24 = r"""'5A7702F5'""" + +example_mode_aes_128_ofb_datatype_UInt64_key_24_iv_24 = r"""'5A7702F5E9B08DB7'""" + +example_mode_aes_128_ofb_datatype_Int8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_ofb_datatype_Int16_key_24_iv_24 = r"""'5A77'""" + +example_mode_aes_128_ofb_datatype_Int32_key_24_iv_24 = r"""'5A7702F5'""" + +example_mode_aes_128_ofb_datatype_Int64_key_24_iv_24 = r"""'5A7702F5E9B08DB7'""" + +example_mode_aes_128_ofb_datatype_Float32_key_24_iv_24 = r"""'5B7782CA'""" + +example_mode_aes_128_ofb_datatype_Float64_key_24_iv_24 = r"""'5B7702F5E9B07D88'""" + +example_mode_aes_128_ofb_datatype_Decimal32_key_24_iv_24 = r"""'7B3902F5'""" + +example_mode_aes_128_ofb_datatype_Decimal64_key_24_iv_24 = r"""'7B3902F5E9B08DB7'""" + +example_mode_aes_128_ofb_datatype_Decimal128_key_24_iv_24 = r"""'7B3902F5E9B08DB73E1BAC7CCCCDAEF6'""" + +example_mode_aes_128_ofb_datatype_UUID_key_24_iv_24 = r"""'BC66B1A9ED747DD69EC07F16CC6BD566'""" + +example_mode_aes_128_ofb_datatype_Date_key_24_iv_24 = r"""'0D30'""" + +example_mode_aes_128_ofb_datatype_DateTime_key_24_iv_24 = r"""'15A70EAB'""" + +example_mode_aes_128_ofb_datatype_DateTime64_key_24_iv_24 = r"""'70C60F9786B18DB7'""" + +example_mode_aes_128_ofb_datatype_LowCardinality_key_24_iv_24 = r"""'6A'""" + +example_mode_aes_128_ofb_datatype_Array_key_24_iv_24 = r"""'5A75'""" + +example_mode_aes_128_ofb_datatype_NULL_key_24_iv_24 = r"""'\\N'""" + +example_mode_aes_128_ofb_datatype_IPv4_key_24_iv_24 = r"""'76F5E35E'""" + +example_mode_aes_128_ofb_datatype_IPv6_key_24_iv_24 = r"""'7B760F4DE9B008143E1BAC7C60D22EF7'""" + +example_mode_aes_128_ofb_datatype_Enum8_key_24_iv_24 = r"""'5A'""" + +example_mode_aes_128_ofb_datatype_Enum16_key_24_iv_24 = r"""'5A77'""" + +example_mode_aes_192_ofb_datatype_bytes_key_24_iv_16 = r"""'68'""" + +example_mode_aes_192_ofb_datatype_emptystring_key_24_iv_16 = r"""''""" + +example_mode_aes_192_ofb_datatype_utf8string_key_24_iv_16 = r"""'2F0444573374B41441C46351EBB0A21F81C68E6CF92A6AF3'""" + +example_mode_aes_192_ofb_datatype_utf8fixedstring_key_24_iv_16 = r"""'2F0444573374B41441C46351EBB0A21F81C68E6CF92A6AF3'""" + +example_mode_aes_192_ofb_datatype_String_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_ofb_datatype_FixedString_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_ofb_datatype_UInt8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_ofb_datatype_UInt16_key_24_iv_16 = r"""'69C7'""" + +example_mode_aes_192_ofb_datatype_UInt32_key_24_iv_16 = r"""'69C7E792'""" + +example_mode_aes_192_ofb_datatype_UInt64_key_24_iv_16 = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_ofb_datatype_Int8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_ofb_datatype_Int16_key_24_iv_16 = r"""'69C7'""" + +example_mode_aes_192_ofb_datatype_Int32_key_24_iv_16 = r"""'69C7E792'""" + +example_mode_aes_192_ofb_datatype_Int64_key_24_iv_16 = r"""'69C7E792B71077B1'""" + +example_mode_aes_192_ofb_datatype_Float32_key_24_iv_16 = r"""'68C767AD'""" + +example_mode_aes_192_ofb_datatype_Float64_key_24_iv_16 = r"""'68C7E792B710878E'""" + +example_mode_aes_192_ofb_datatype_Decimal32_key_24_iv_16 = r"""'4889E792'""" + +example_mode_aes_192_ofb_datatype_Decimal64_key_24_iv_16 = r"""'4889E792B71077B1'""" + +example_mode_aes_192_ofb_datatype_Decimal128_key_24_iv_16 = r"""'4889E792B71077B18446050EBFD861B5'""" + +example_mode_aes_192_ofb_datatype_UUID_key_24_iv_16 = r"""'8FD654CEB3D487D0249DD664BF7E1A25'""" + +example_mode_aes_192_ofb_datatype_Date_key_24_iv_16 = r"""'3E80'""" + +example_mode_aes_192_ofb_datatype_DateTime_key_24_iv_16 = r"""'2617EBCC'""" + +example_mode_aes_192_ofb_datatype_DateTime64_key_24_iv_16 = r"""'4376EAF0D81177B1'""" + +example_mode_aes_192_ofb_datatype_LowCardinality_key_24_iv_16 = r"""'59'""" + +example_mode_aes_192_ofb_datatype_Array_key_24_iv_16 = r"""'69C5'""" + +example_mode_aes_192_ofb_datatype_NULL_key_24_iv_16 = r"""'\\N'""" + +example_mode_aes_192_ofb_datatype_IPv4_key_24_iv_16 = r"""'45450639'""" + +example_mode_aes_192_ofb_datatype_IPv6_key_24_iv_16 = r"""'48C6EA2AB710F2128446050E13C7E1B4'""" + +example_mode_aes_192_ofb_datatype_Enum8_key_24_iv_16 = r"""'69'""" + +example_mode_aes_192_ofb_datatype_Enum16_key_24_iv_16 = r"""'69C7'""" + +example_mode_aes_192_ofb_datatype_bytes_key_32_iv_32 = r"""'31'""" + +example_mode_aes_192_ofb_datatype_emptystring_key_32_iv_32 = r"""''""" + +example_mode_aes_192_ofb_datatype_utf8string_key_32_iv_32 = r"""'76632DB12BCFF36187A90B6990CFA6D86DA0963C4A14697B'""" + +example_mode_aes_192_ofb_datatype_utf8fixedstring_key_32_iv_32 = r"""'76632DB12BCFF36187A90B6990CFA6D86DA0963C4A14697B'""" + +example_mode_aes_192_ofb_datatype_String_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_ofb_datatype_FixedString_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_ofb_datatype_UInt8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_ofb_datatype_UInt16_key_32_iv_32 = r"""'30A0'""" + +example_mode_aes_192_ofb_datatype_UInt32_key_32_iv_32 = r"""'30A08E74'""" + +example_mode_aes_192_ofb_datatype_UInt64_key_32_iv_32 = r"""'30A08E74AFAB30C4'""" + +example_mode_aes_192_ofb_datatype_Int8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_ofb_datatype_Int16_key_32_iv_32 = r"""'30A0'""" + +example_mode_aes_192_ofb_datatype_Int32_key_32_iv_32 = r"""'30A08E74'""" + +example_mode_aes_192_ofb_datatype_Int64_key_32_iv_32 = r"""'30A08E74AFAB30C4'""" + +example_mode_aes_192_ofb_datatype_Float32_key_32_iv_32 = r"""'31A00E4B'""" + +example_mode_aes_192_ofb_datatype_Float64_key_32_iv_32 = r"""'31A08E74AFABC0FB'""" + +example_mode_aes_192_ofb_datatype_Decimal32_key_32_iv_32 = r"""'11EE8E74'""" + +example_mode_aes_192_ofb_datatype_Decimal64_key_32_iv_32 = r"""'11EE8E74AFAB30C4'""" + +example_mode_aes_192_ofb_datatype_Decimal128_key_32_iv_32 = r"""'11EE8E74AFAB30C4422B6D36C4A76572'""" + +example_mode_aes_192_ofb_datatype_UUID_key_32_iv_32 = r"""'D6B13D28AB6FC0A5E2F0BE5CC4011EE2'""" + +example_mode_aes_192_ofb_datatype_Date_key_32_iv_32 = r"""'67E7'""" + +example_mode_aes_192_ofb_datatype_DateTime_key_32_iv_32 = r"""'7F70822A'""" + +example_mode_aes_192_ofb_datatype_DateTime64_key_32_iv_32 = r"""'1A118316C0AA30C4'""" + +example_mode_aes_192_ofb_datatype_LowCardinality_key_32_iv_32 = r"""'00'""" + +example_mode_aes_192_ofb_datatype_Array_key_32_iv_32 = r"""'30A2'""" + +example_mode_aes_192_ofb_datatype_NULL_key_32_iv_32 = r"""'\\N'""" + +example_mode_aes_192_ofb_datatype_IPv4_key_32_iv_32 = r"""'1C226FDF'""" + +example_mode_aes_192_ofb_datatype_IPv6_key_32_iv_32 = r"""'11A183CCAFABB567422B6D3668B8E573'""" + +example_mode_aes_192_ofb_datatype_Enum8_key_32_iv_32 = r"""'30'""" + +example_mode_aes_192_ofb_datatype_Enum16_key_32_iv_32 = r"""'30A0'""" + +example_mode_aes_256_ofb_datatype_bytes_key_32_iv_16 = r"""'69'""" + +example_mode_aes_256_ofb_datatype_emptystring_key_32_iv_16 = r"""''""" + +example_mode_aes_256_ofb_datatype_utf8string_key_32_iv_16 = r"""'2E9B2BD2B8BA872DB56225F82754048CE38E2C23393CF6FD'""" + +example_mode_aes_256_ofb_datatype_utf8fixedstring_key_32_iv_16 = r"""'2E9B2BD2B8BA872DB56225F82754048CE38E2C23393CF6FD'""" + +example_mode_aes_256_ofb_datatype_String_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_ofb_datatype_FixedString_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_ofb_datatype_UInt8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_ofb_datatype_UInt16_key_32_iv_16 = r"""'6858'""" + +example_mode_aes_256_ofb_datatype_UInt32_key_32_iv_16 = r"""'68588817'""" + +example_mode_aes_256_ofb_datatype_UInt64_key_32_iv_16 = r"""'685888173CDE4488'""" + +example_mode_aes_256_ofb_datatype_Int8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_ofb_datatype_Int16_key_32_iv_16 = r"""'6858'""" + +example_mode_aes_256_ofb_datatype_Int32_key_32_iv_16 = r"""'68588817'""" + +example_mode_aes_256_ofb_datatype_Int64_key_32_iv_16 = r"""'685888173CDE4488'""" + +example_mode_aes_256_ofb_datatype_Float32_key_32_iv_16 = r"""'69580828'""" + +example_mode_aes_256_ofb_datatype_Float64_key_32_iv_16 = r"""'695888173CDEB4B7'""" + +example_mode_aes_256_ofb_datatype_Decimal32_key_32_iv_16 = r"""'49168817'""" + +example_mode_aes_256_ofb_datatype_Decimal64_key_32_iv_16 = r"""'491688173CDE4488'""" + +example_mode_aes_256_ofb_datatype_Decimal128_key_32_iv_16 = r"""'491688173CDE448870E043A7733CC726'""" + +example_mode_aes_256_ofb_datatype_UUID_key_32_iv_16 = r"""'8E493B4B381AB4E9D03B90CD739ABCB6'""" + +example_mode_aes_256_ofb_datatype_Date_key_32_iv_16 = r"""'3F1F'""" + +example_mode_aes_256_ofb_datatype_DateTime_key_32_iv_16 = r"""'27888449'""" + +example_mode_aes_256_ofb_datatype_DateTime64_key_32_iv_16 = r"""'42E9857553DF4488'""" + +example_mode_aes_256_ofb_datatype_LowCardinality_key_32_iv_16 = r"""'58'""" + +example_mode_aes_256_ofb_datatype_Array_key_32_iv_16 = r"""'685A'""" + +example_mode_aes_256_ofb_datatype_NULL_key_32_iv_16 = r"""'\\N'""" + +example_mode_aes_256_ofb_datatype_IPv4_key_32_iv_16 = r"""'44DA69BC'""" + +example_mode_aes_256_ofb_datatype_IPv6_key_32_iv_16 = r"""'495985AF3CDEC12B70E043A7DF234727'""" + +example_mode_aes_256_ofb_datatype_Enum8_key_32_iv_16 = r"""'68'""" + +example_mode_aes_256_ofb_datatype_Enum16_key_32_iv_16 = r"""'6858'""" + +example_mode_aes_256_ofb_datatype_bytes_key_64_iv_64 = r"""'D3'""" + +example_mode_aes_256_ofb_datatype_emptystring_key_64_iv_64 = r"""''""" + +example_mode_aes_256_ofb_datatype_utf8string_key_64_iv_64 = r"""'942D6C993F1DE6D874AD5CCF2109CE7D063DC690F1843081'""" + +example_mode_aes_256_ofb_datatype_utf8fixedstring_key_64_iv_64 = r"""'942D6C993F1DE6D874AD5CCF2109CE7D063DC690F1843081'""" + +example_mode_aes_256_ofb_datatype_String_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_ofb_datatype_FixedString_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_ofb_datatype_UInt8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_ofb_datatype_UInt16_key_64_iv_64 = r"""'D2EE'""" + +example_mode_aes_256_ofb_datatype_UInt32_key_64_iv_64 = r"""'D2EECF5C'""" + +example_mode_aes_256_ofb_datatype_UInt64_key_64_iv_64 = r"""'D2EECF5CBB79257D'""" + +example_mode_aes_256_ofb_datatype_Int8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_ofb_datatype_Int16_key_64_iv_64 = r"""'D2EE'""" + +example_mode_aes_256_ofb_datatype_Int32_key_64_iv_64 = r"""'D2EECF5C'""" + +example_mode_aes_256_ofb_datatype_Int64_key_64_iv_64 = r"""'D2EECF5CBB79257D'""" + +example_mode_aes_256_ofb_datatype_Float32_key_64_iv_64 = r"""'D3EE4F63'""" + +example_mode_aes_256_ofb_datatype_Float64_key_64_iv_64 = r"""'D3EECF5CBB79D542'""" + +example_mode_aes_256_ofb_datatype_Decimal32_key_64_iv_64 = r"""'F3A0CF5C'""" + +example_mode_aes_256_ofb_datatype_Decimal64_key_64_iv_64 = r"""'F3A0CF5CBB79257D'""" + +example_mode_aes_256_ofb_datatype_Decimal128_key_64_iv_64 = r"""'F3A0CF5CBB79257DB12F3A9075610DD7'""" + +example_mode_aes_256_ofb_datatype_UUID_key_64_iv_64 = r"""'34FF7C00BFBDD51C11F4E9FA75C77647'""" + +example_mode_aes_256_ofb_datatype_Date_key_64_iv_64 = r"""'85A9'""" + +example_mode_aes_256_ofb_datatype_DateTime_key_64_iv_64 = r"""'9D3EC302'""" + +example_mode_aes_256_ofb_datatype_DateTime64_key_64_iv_64 = r"""'F85FC23ED478257D'""" + +example_mode_aes_256_ofb_datatype_LowCardinality_key_64_iv_64 = r"""'E2'""" + +example_mode_aes_256_ofb_datatype_Array_key_64_iv_64 = r"""'D2EC'""" + +example_mode_aes_256_ofb_datatype_NULL_key_64_iv_64 = r"""'\\N'""" + +example_mode_aes_256_ofb_datatype_IPv4_key_64_iv_64 = r"""'FE6C2EF7'""" + +example_mode_aes_256_ofb_datatype_IPv6_key_64_iv_64 = r"""'F3EFC2E4BB79A0DEB12F3A90D97E8DD6'""" + +example_mode_aes_256_ofb_datatype_Enum8_key_64_iv_64 = r"""'D2'""" + +example_mode_aes_256_ofb_datatype_Enum16_key_64_iv_64 = r"""'D2EE'""" + diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py index a80ae7fc07b..6f1a5acab14 100755 --- a/tests/testflows/regression.py +++ b/tests/testflows/regression.py @@ -17,6 +17,7 @@ def regression(self, local, clickhouse_binary_path): Feature(test=load("example.regression", "regression"))(**args) Feature(test=load("ldap.regression", "regression"))(**args) Feature(test=load("rbac.regression", "regression"))(**args) + Feature(test=load("aes_encryption.regression", "regression"))(**args) if main(): regression() From 7cc4118dabb68214c0654bc66e4e202ab19adf16 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 17 Aug 2020 17:49:21 +0300 Subject: [PATCH 069/411] Fixed AES encrypt/decrypt performance Improved execution time of EVP_DecryptInit_ex/EVP_EncryptInit_ex with some hacks Fixed all-const case --- src/Functions/FunctionsAES.cpp | 8 ++++-- src/Functions/FunctionsAES.h | 40 ++++++++++++--------------- src/Functions/tests/gtest_openssl.cpp | 0 3 files changed, 23 insertions(+), 25 deletions(-) create mode 100644 src/Functions/tests/gtest_openssl.cpp diff --git a/src/Functions/FunctionsAES.cpp b/src/Functions/FunctionsAES.cpp index 48533be054a..029b0727e0d 100644 --- a/src/Functions/FunctionsAES.cpp +++ b/src/Functions/FunctionsAES.cpp @@ -2,6 +2,7 @@ #if USE_SSL +#include #include #include @@ -34,7 +35,7 @@ StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const return StringRef(folded_key.data(), cipher_key_size); } -const EVP_CIPHER * getCipherByName(const StringRef & cipher_name) +CipherPtr getCipherByName(const StringRef & cipher_name) { const auto *evp_cipher = EVP_get_cipherbyname(cipher_name.data); if (evp_cipher == nullptr) @@ -48,7 +49,10 @@ const EVP_CIPHER * getCipherByName(const StringRef & cipher_name) evp_cipher = EVP_aes_256_cfb128(); } - return evp_cipher; + // HACK: To speed up context initialization with EVP_EncryptInit_ex (which is called at least once per row) + // Apparently cipher from EVP_get_cipherbyname may require additional initialization of context, + // while cipher from EVP_CIPHER_fetch causes less operations => faster context initialization. + return CipherPtr{EVP_CIPHER_fetch(nullptr, EVP_CIPHER_name(evp_cipher), nullptr), &EVP_CIPHER_free}; } } diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index 8d062e9b12d..d126bcea6d6 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -34,7 +35,9 @@ namespace OpenSSLDetails { [[noreturn]] void onError(std::string error_message); StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array & folded_key); -const EVP_CIPHER * getCipherByName(const StringRef & name); + +using CipherPtr = std::unique_ptr; +CipherPtr getCipherByName(const StringRef & name); enum class CompatibilityMode { @@ -148,6 +151,8 @@ private: String getName() const override { return name; } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } + bool useDefaultImplementationForConstants() const override { return true; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { @@ -174,11 +179,6 @@ private: return std::make_shared(); } - void executeImplDryRun(Block & block, const ColumnNumbers & /*arguments*/, size_t result, size_t /*input_rows_count*/) const override - { - block.getByPosition(result).column = block.getByPosition(result).type->createColumn(); - } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { using namespace OpenSSLDetails; @@ -188,9 +188,10 @@ private: if (mode.size == 0 || !std::string_view(mode).starts_with("aes-")) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); - const auto * evp_cipher = getCipherByName(mode); - if (evp_cipher == nullptr) + auto cipher = getCipherByName(mode); + if (cipher == nullptr) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); + const EVP_CIPHER * evp_cipher = cipher.get(); const auto cipher_mode = EVP_CIPHER_mode(evp_cipher); @@ -371,9 +372,6 @@ private: ++encrypted; encrypted_result_column_offsets.push_back(encrypted - encrypted_result_column_data.data()); - - if (EVP_CIPHER_CTX_reset(evp_ctx) != 1) - onError("Failed to reset context"); } // in case of block size of 1, we overestimate buffer required for encrypted data, fix it up. @@ -403,6 +401,8 @@ private: String getName() const override { return name; } bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } + bool useDefaultImplementationForConstants() const override { return true; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { @@ -429,23 +429,18 @@ private: return std::make_shared(); } - void executeImplDryRun(Block & block, const ColumnNumbers & /*arguments*/, size_t result, size_t /*input_rows_count*/) const override - { - block.getByPosition(result).column = block.getByPosition(result).type->createColumn(); - } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { using namespace OpenSSLDetails; const auto mode = block.getByPosition(arguments[0]).column->getDataAt(0); - if (mode.size == 0 || !std::string_view(mode).starts_with("aes-")) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); - const auto * evp_cipher = getCipherByName(mode); - if (evp_cipher == nullptr) + auto cipher = getCipherByName(mode); + if (cipher == nullptr) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); + const EVP_CIPHER * evp_cipher = cipher.get(); OpenSSLDetails::validateCipherMode(evp_cipher); @@ -575,13 +570,14 @@ private: // 1: Init CTX if constexpr (mode == CipherMode::RFC5116_AEAD_AES_GCM) { - // 1.a.1 : Init CTX with custom IV length and optionally with AAD if (EVP_DecryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr) != 1) - onError("Failed to initialize cipher context"); + onError("Failed to initialize cipher context 1"); + // 1.a.1 : Set custom IV length if (EVP_CIPHER_CTX_ctrl(evp_ctx, EVP_CTRL_AEAD_SET_IVLEN, iv_value.size, nullptr) != 1) onError("Failed to set custom IV length to " + std::to_string(iv_value.size)); + // 1.a.1 : Init CTX with key and IV if (EVP_DecryptInit_ex(evp_ctx, nullptr, nullptr, reinterpret_cast(key_value.data), reinterpret_cast(iv_value.data)) != 1) @@ -635,8 +631,6 @@ private: decrypted_result_column_offsets.push_back(decrypted - decrypted_result_column_data.data()); - if (EVP_CIPHER_CTX_reset(evp_ctx) != 1) - onError("Failed to reset context"); } // in case we overestimate buffer required for decrypted data, fix it up. diff --git a/src/Functions/tests/gtest_openssl.cpp b/src/Functions/tests/gtest_openssl.cpp new file mode 100644 index 00000000000..e69de29bb2d From bdaa012239e73177e162c4c65ad0085c885e223c Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 31 Aug 2020 18:06:33 +0300 Subject: [PATCH 070/411] Fixed compilation for older (pre-3.0.0) OpenSSL versions --- src/Functions/FunctionsAES.cpp | 18 +++++++++++++++++- src/Functions/FunctionsAES.h | 3 ++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionsAES.cpp b/src/Functions/FunctionsAES.cpp index 029b0727e0d..009226c035a 100644 --- a/src/Functions/FunctionsAES.cpp +++ b/src/Functions/FunctionsAES.cpp @@ -7,6 +7,18 @@ #include +namespace +{ +void CipherDeleter(const EVP_CIPHER * cipher [[maybe_unused]]) +{ +#if OPENSSL_VERSION_NUMBER >= 0x03 << 28 +// Used to free EVP_CIPHER poniter obtained with EVP_CIPHER_fetch, +// available only since OpenSSL ver 3.0.0. + EVP_CIPHER_free(const_cast(cipher)); +#endif +} +} + namespace DB { namespace ErrorCodes @@ -49,10 +61,14 @@ CipherPtr getCipherByName(const StringRef & cipher_name) evp_cipher = EVP_aes_256_cfb128(); } +#if OPENSSL_VERSION_NUMBER < 0x03 << 28 + return CipherPtr{evp_cipher, CipherDeleter}; +#else // HACK: To speed up context initialization with EVP_EncryptInit_ex (which is called at least once per row) // Apparently cipher from EVP_get_cipherbyname may require additional initialization of context, // while cipher from EVP_CIPHER_fetch causes less operations => faster context initialization. - return CipherPtr{EVP_CIPHER_fetch(nullptr, EVP_CIPHER_name(evp_cipher), nullptr), &EVP_CIPHER_free}; + return CipherPtr{EVP_CIPHER_fetch(nullptr, EVP_CIPHER_name(evp_cipher), nullptr), &CipherDeleter}; +#endif } } diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index d126bcea6d6..25b41a067a7 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -36,7 +36,8 @@ namespace OpenSSLDetails [[noreturn]] void onError(std::string error_message); StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array & folded_key); -using CipherPtr = std::unique_ptr; +using CipherDeleterType = void (*) (const EVP_CIPHER *cipher); +using CipherPtr = std::unique_ptr; CipherPtr getCipherByName(const StringRef & name); enum class CompatibilityMode From 9ba4613381942d920f26401e2c0c743771962698 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 1 Sep 2020 12:27:11 +0300 Subject: [PATCH 071/411] Excluding tests from fasttest runs since they depend on external libraries: OpenSSL --- docker/test/fasttest/run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 1f8d612a125..abd2fd62c13 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -206,6 +206,8 @@ TESTS_TO_SKIP=( 01411_bayesian_ab_testing 01238_http_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently 01281_group_by_limit_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently + 01318_encrypt # Depends on OpenSSL + 01318_decrypt # Depends on OpenSSL # Not sure why these two fail even in sequential mode. Disabled for now # to make some progress. From af23a27f9f38accbf85eebdef79253fb81e512a3 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 3 Sep 2020 15:53:34 +0200 Subject: [PATCH 072/411] Reorganizing ldap tests. --- .../testflows/ldap/authentication/__init__.py | 0 .../{ => authentication}/configs/CA/ca.crt | 0 .../{ => authentication}/configs/CA/ca.key | 0 .../{ => authentication}/configs/CA/ca.srl | 0 .../configs/CA/dhparam.pem | 0 .../configs/CA/passphrase.txt | 0 .../configs/clickhouse/common.xml | 0 .../configs/clickhouse/config.d/logs.xml | 0 .../configs/clickhouse/config.d/ports.xml | 0 .../configs/clickhouse/config.d/remote.xml | 0 .../configs/clickhouse/config.d/ssl.xml | 0 .../configs/clickhouse/config.d/storage.xml | 0 .../configs/clickhouse/config.d/zookeeper.xml | 0 .../configs/clickhouse/config.xml | 12 +++++ .../configs/clickhouse/ssl/dhparam.pem | 0 .../configs/clickhouse/ssl/server.crt | 0 .../configs/clickhouse/ssl/server.key | 0 .../configs/clickhouse/users.xml | 0 .../configs/clickhouse1/config.d/macros.xml | 0 .../configs/clickhouse2/config.d/macros.xml | 0 .../configs/clickhouse3/config.d/macros.xml | 0 .../configs/ldap1/config/export.ldif | 0 .../configs/ldap2/certs/ca.crt | 0 .../configs/ldap2/certs/dhparam.pem | 0 .../configs/ldap2/certs/ldap.crt | 0 .../configs/ldap2/certs/ldap.csr | 0 .../configs/ldap2/certs/ldap.key | 0 .../configs/ldap2/config/export.ldif | 0 .../configs/ldap3/certs/ca.crt | 0 .../configs/ldap3/certs/dhparam.pem | 0 .../configs/ldap3/certs/ldap.crt | 0 .../configs/ldap3/certs/ldap.csr | 0 .../configs/ldap3/certs/ldap.key | 0 .../configs/ldap3/config/export.ldif | 0 .../configs/ldap4/certs/ca.crt | 0 .../configs/ldap4/certs/dhparam.pem | 0 .../configs/ldap4/certs/ldap.crt | 0 .../configs/ldap4/certs/ldap.csr | 0 .../configs/ldap4/certs/ldap.key | 0 .../configs/ldap4/config/export.ldif | 0 .../configs/ldap5/config/export.ldif | 0 .../configs/ldap5/ldap2/certs/ca.crt | 0 .../configs/ldap5/ldap2/certs/dhparam.pem | 0 .../configs/ldap5/ldap2/certs/ldap.crt | 0 .../configs/ldap5/ldap2/certs/ldap.csr | 0 .../configs/ldap5/ldap2/certs/ldap.key | 0 .../configs/ldap5/ldap2/config/export.ldif | 0 .../docker-compose/clickhouse-service.yml | 0 .../docker-compose/docker-compose.yml | 0 .../docker-compose/openldap-service.yml | 0 .../docker-compose/zookeeper-service.yml | 0 .../ldap/authentication/regression.py | 54 +++++++++++++++++++ .../requirements/__init__.py | 0 .../requirements/requirements.md | 0 .../requirements/requirements.py | 0 .../tests/authentications.py | 4 +- .../ldap/{ => authentication}/tests/common.py | 2 +- .../{ => authentication}/tests/connections.py | 4 +- .../tests/multiple_servers.py | 4 +- .../ldap/{ => authentication}/tests/sanity.py | 2 +- .../tests/server_config.py | 4 +- .../{ => authentication}/tests/user_config.py | 4 +- tests/testflows/ldap/regression.py | 46 +++------------- 63 files changed, 84 insertions(+), 52 deletions(-) create mode 100644 tests/testflows/ldap/authentication/__init__.py rename tests/testflows/ldap/{ => authentication}/configs/CA/ca.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/CA/ca.key (100%) rename tests/testflows/ldap/{ => authentication}/configs/CA/ca.srl (100%) rename tests/testflows/ldap/{ => authentication}/configs/CA/dhparam.pem (100%) rename tests/testflows/ldap/{ => authentication}/configs/CA/passphrase.txt (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/common.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/config.d/logs.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/config.d/ports.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/config.d/remote.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/config.d/ssl.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/config.d/storage.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/config.d/zookeeper.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/config.xml (97%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/ssl/dhparam.pem (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/ssl/server.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/ssl/server.key (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse/users.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse1/config.d/macros.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse2/config.d/macros.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/clickhouse3/config.d/macros.xml (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap1/config/export.ldif (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap2/certs/ca.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap2/certs/dhparam.pem (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap2/certs/ldap.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap2/certs/ldap.csr (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap2/certs/ldap.key (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap2/config/export.ldif (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap3/certs/ca.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap3/certs/dhparam.pem (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap3/certs/ldap.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap3/certs/ldap.csr (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap3/certs/ldap.key (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap3/config/export.ldif (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap4/certs/ca.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap4/certs/dhparam.pem (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap4/certs/ldap.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap4/certs/ldap.csr (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap4/certs/ldap.key (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap4/config/export.ldif (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap5/config/export.ldif (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap5/ldap2/certs/ca.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap5/ldap2/certs/dhparam.pem (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap5/ldap2/certs/ldap.crt (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap5/ldap2/certs/ldap.csr (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap5/ldap2/certs/ldap.key (100%) rename tests/testflows/ldap/{ => authentication}/configs/ldap5/ldap2/config/export.ldif (100%) rename tests/testflows/ldap/{ => authentication}/docker-compose/clickhouse-service.yml (100%) rename tests/testflows/ldap/{ => authentication}/docker-compose/docker-compose.yml (100%) rename tests/testflows/ldap/{ => authentication}/docker-compose/openldap-service.yml (100%) rename tests/testflows/ldap/{ => authentication}/docker-compose/zookeeper-service.yml (100%) create mode 100755 tests/testflows/ldap/authentication/regression.py rename tests/testflows/ldap/{ => authentication}/requirements/__init__.py (100%) rename tests/testflows/ldap/{ => authentication}/requirements/requirements.md (100%) rename tests/testflows/ldap/{ => authentication}/requirements/requirements.py (100%) rename tests/testflows/ldap/{ => authentication}/tests/authentications.py (99%) rename tests/testflows/ldap/{ => authentication}/tests/common.py (99%) rename tests/testflows/ldap/{ => authentication}/tests/connections.py (98%) rename tests/testflows/ldap/{ => authentication}/tests/multiple_servers.py (88%) rename tests/testflows/ldap/{ => authentication}/tests/sanity.py (95%) rename tests/testflows/ldap/{ => authentication}/tests/server_config.py (99%) rename tests/testflows/ldap/{ => authentication}/tests/user_config.py (98%) diff --git a/tests/testflows/ldap/authentication/__init__.py b/tests/testflows/ldap/authentication/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/ldap/configs/CA/ca.crt b/tests/testflows/ldap/authentication/configs/CA/ca.crt similarity index 100% rename from tests/testflows/ldap/configs/CA/ca.crt rename to tests/testflows/ldap/authentication/configs/CA/ca.crt diff --git a/tests/testflows/ldap/configs/CA/ca.key b/tests/testflows/ldap/authentication/configs/CA/ca.key similarity index 100% rename from tests/testflows/ldap/configs/CA/ca.key rename to tests/testflows/ldap/authentication/configs/CA/ca.key diff --git a/tests/testflows/ldap/configs/CA/ca.srl b/tests/testflows/ldap/authentication/configs/CA/ca.srl similarity index 100% rename from tests/testflows/ldap/configs/CA/ca.srl rename to tests/testflows/ldap/authentication/configs/CA/ca.srl diff --git a/tests/testflows/ldap/configs/CA/dhparam.pem b/tests/testflows/ldap/authentication/configs/CA/dhparam.pem similarity index 100% rename from tests/testflows/ldap/configs/CA/dhparam.pem rename to tests/testflows/ldap/authentication/configs/CA/dhparam.pem diff --git a/tests/testflows/ldap/configs/CA/passphrase.txt b/tests/testflows/ldap/authentication/configs/CA/passphrase.txt similarity index 100% rename from tests/testflows/ldap/configs/CA/passphrase.txt rename to tests/testflows/ldap/authentication/configs/CA/passphrase.txt diff --git a/tests/testflows/ldap/configs/clickhouse/common.xml b/tests/testflows/ldap/authentication/configs/clickhouse/common.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/common.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/common.xml diff --git a/tests/testflows/ldap/configs/clickhouse/config.d/logs.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.d/logs.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/config.d/logs.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/config.d/logs.xml diff --git a/tests/testflows/ldap/configs/clickhouse/config.d/ports.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.d/ports.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/config.d/ports.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/config.d/ports.xml diff --git a/tests/testflows/ldap/configs/clickhouse/config.d/remote.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.d/remote.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/config.d/remote.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/config.d/remote.xml diff --git a/tests/testflows/ldap/configs/clickhouse/config.d/ssl.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.d/ssl.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/config.d/ssl.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/config.d/ssl.xml diff --git a/tests/testflows/ldap/configs/clickhouse/config.d/storage.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.d/storage.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/config.d/storage.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/config.d/storage.xml diff --git a/tests/testflows/ldap/configs/clickhouse/config.d/zookeeper.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.d/zookeeper.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/config.d/zookeeper.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/config.d/zookeeper.xml diff --git a/tests/testflows/ldap/configs/clickhouse/config.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.xml similarity index 97% rename from tests/testflows/ldap/configs/clickhouse/config.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/config.xml index d34d2c35253..80c3150d326 100644 --- a/tests/testflows/ldap/configs/clickhouse/config.xml +++ b/tests/testflows/ldap/authentication/configs/clickhouse/config.xml @@ -120,6 +120,18 @@ /var/lib/clickhouse/access/ + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + users.xml diff --git a/tests/testflows/ldap/configs/clickhouse/ssl/dhparam.pem b/tests/testflows/ldap/authentication/configs/clickhouse/ssl/dhparam.pem similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/ssl/dhparam.pem rename to tests/testflows/ldap/authentication/configs/clickhouse/ssl/dhparam.pem diff --git a/tests/testflows/ldap/configs/clickhouse/ssl/server.crt b/tests/testflows/ldap/authentication/configs/clickhouse/ssl/server.crt similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/ssl/server.crt rename to tests/testflows/ldap/authentication/configs/clickhouse/ssl/server.crt diff --git a/tests/testflows/ldap/configs/clickhouse/ssl/server.key b/tests/testflows/ldap/authentication/configs/clickhouse/ssl/server.key similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/ssl/server.key rename to tests/testflows/ldap/authentication/configs/clickhouse/ssl/server.key diff --git a/tests/testflows/ldap/configs/clickhouse/users.xml b/tests/testflows/ldap/authentication/configs/clickhouse/users.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse/users.xml rename to tests/testflows/ldap/authentication/configs/clickhouse/users.xml diff --git a/tests/testflows/ldap/configs/clickhouse1/config.d/macros.xml b/tests/testflows/ldap/authentication/configs/clickhouse1/config.d/macros.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse1/config.d/macros.xml rename to tests/testflows/ldap/authentication/configs/clickhouse1/config.d/macros.xml diff --git a/tests/testflows/ldap/configs/clickhouse2/config.d/macros.xml b/tests/testflows/ldap/authentication/configs/clickhouse2/config.d/macros.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse2/config.d/macros.xml rename to tests/testflows/ldap/authentication/configs/clickhouse2/config.d/macros.xml diff --git a/tests/testflows/ldap/configs/clickhouse3/config.d/macros.xml b/tests/testflows/ldap/authentication/configs/clickhouse3/config.d/macros.xml similarity index 100% rename from tests/testflows/ldap/configs/clickhouse3/config.d/macros.xml rename to tests/testflows/ldap/authentication/configs/clickhouse3/config.d/macros.xml diff --git a/tests/testflows/ldap/configs/ldap1/config/export.ldif b/tests/testflows/ldap/authentication/configs/ldap1/config/export.ldif similarity index 100% rename from tests/testflows/ldap/configs/ldap1/config/export.ldif rename to tests/testflows/ldap/authentication/configs/ldap1/config/export.ldif diff --git a/tests/testflows/ldap/configs/ldap2/certs/ca.crt b/tests/testflows/ldap/authentication/configs/ldap2/certs/ca.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap2/certs/ca.crt rename to tests/testflows/ldap/authentication/configs/ldap2/certs/ca.crt diff --git a/tests/testflows/ldap/configs/ldap2/certs/dhparam.pem b/tests/testflows/ldap/authentication/configs/ldap2/certs/dhparam.pem similarity index 100% rename from tests/testflows/ldap/configs/ldap2/certs/dhparam.pem rename to tests/testflows/ldap/authentication/configs/ldap2/certs/dhparam.pem diff --git a/tests/testflows/ldap/configs/ldap2/certs/ldap.crt b/tests/testflows/ldap/authentication/configs/ldap2/certs/ldap.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap2/certs/ldap.crt rename to tests/testflows/ldap/authentication/configs/ldap2/certs/ldap.crt diff --git a/tests/testflows/ldap/configs/ldap2/certs/ldap.csr b/tests/testflows/ldap/authentication/configs/ldap2/certs/ldap.csr similarity index 100% rename from tests/testflows/ldap/configs/ldap2/certs/ldap.csr rename to tests/testflows/ldap/authentication/configs/ldap2/certs/ldap.csr diff --git a/tests/testflows/ldap/configs/ldap2/certs/ldap.key b/tests/testflows/ldap/authentication/configs/ldap2/certs/ldap.key similarity index 100% rename from tests/testflows/ldap/configs/ldap2/certs/ldap.key rename to tests/testflows/ldap/authentication/configs/ldap2/certs/ldap.key diff --git a/tests/testflows/ldap/configs/ldap2/config/export.ldif b/tests/testflows/ldap/authentication/configs/ldap2/config/export.ldif similarity index 100% rename from tests/testflows/ldap/configs/ldap2/config/export.ldif rename to tests/testflows/ldap/authentication/configs/ldap2/config/export.ldif diff --git a/tests/testflows/ldap/configs/ldap3/certs/ca.crt b/tests/testflows/ldap/authentication/configs/ldap3/certs/ca.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap3/certs/ca.crt rename to tests/testflows/ldap/authentication/configs/ldap3/certs/ca.crt diff --git a/tests/testflows/ldap/configs/ldap3/certs/dhparam.pem b/tests/testflows/ldap/authentication/configs/ldap3/certs/dhparam.pem similarity index 100% rename from tests/testflows/ldap/configs/ldap3/certs/dhparam.pem rename to tests/testflows/ldap/authentication/configs/ldap3/certs/dhparam.pem diff --git a/tests/testflows/ldap/configs/ldap3/certs/ldap.crt b/tests/testflows/ldap/authentication/configs/ldap3/certs/ldap.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap3/certs/ldap.crt rename to tests/testflows/ldap/authentication/configs/ldap3/certs/ldap.crt diff --git a/tests/testflows/ldap/configs/ldap3/certs/ldap.csr b/tests/testflows/ldap/authentication/configs/ldap3/certs/ldap.csr similarity index 100% rename from tests/testflows/ldap/configs/ldap3/certs/ldap.csr rename to tests/testflows/ldap/authentication/configs/ldap3/certs/ldap.csr diff --git a/tests/testflows/ldap/configs/ldap3/certs/ldap.key b/tests/testflows/ldap/authentication/configs/ldap3/certs/ldap.key similarity index 100% rename from tests/testflows/ldap/configs/ldap3/certs/ldap.key rename to tests/testflows/ldap/authentication/configs/ldap3/certs/ldap.key diff --git a/tests/testflows/ldap/configs/ldap3/config/export.ldif b/tests/testflows/ldap/authentication/configs/ldap3/config/export.ldif similarity index 100% rename from tests/testflows/ldap/configs/ldap3/config/export.ldif rename to tests/testflows/ldap/authentication/configs/ldap3/config/export.ldif diff --git a/tests/testflows/ldap/configs/ldap4/certs/ca.crt b/tests/testflows/ldap/authentication/configs/ldap4/certs/ca.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap4/certs/ca.crt rename to tests/testflows/ldap/authentication/configs/ldap4/certs/ca.crt diff --git a/tests/testflows/ldap/configs/ldap4/certs/dhparam.pem b/tests/testflows/ldap/authentication/configs/ldap4/certs/dhparam.pem similarity index 100% rename from tests/testflows/ldap/configs/ldap4/certs/dhparam.pem rename to tests/testflows/ldap/authentication/configs/ldap4/certs/dhparam.pem diff --git a/tests/testflows/ldap/configs/ldap4/certs/ldap.crt b/tests/testflows/ldap/authentication/configs/ldap4/certs/ldap.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap4/certs/ldap.crt rename to tests/testflows/ldap/authentication/configs/ldap4/certs/ldap.crt diff --git a/tests/testflows/ldap/configs/ldap4/certs/ldap.csr b/tests/testflows/ldap/authentication/configs/ldap4/certs/ldap.csr similarity index 100% rename from tests/testflows/ldap/configs/ldap4/certs/ldap.csr rename to tests/testflows/ldap/authentication/configs/ldap4/certs/ldap.csr diff --git a/tests/testflows/ldap/configs/ldap4/certs/ldap.key b/tests/testflows/ldap/authentication/configs/ldap4/certs/ldap.key similarity index 100% rename from tests/testflows/ldap/configs/ldap4/certs/ldap.key rename to tests/testflows/ldap/authentication/configs/ldap4/certs/ldap.key diff --git a/tests/testflows/ldap/configs/ldap4/config/export.ldif b/tests/testflows/ldap/authentication/configs/ldap4/config/export.ldif similarity index 100% rename from tests/testflows/ldap/configs/ldap4/config/export.ldif rename to tests/testflows/ldap/authentication/configs/ldap4/config/export.ldif diff --git a/tests/testflows/ldap/configs/ldap5/config/export.ldif b/tests/testflows/ldap/authentication/configs/ldap5/config/export.ldif similarity index 100% rename from tests/testflows/ldap/configs/ldap5/config/export.ldif rename to tests/testflows/ldap/authentication/configs/ldap5/config/export.ldif diff --git a/tests/testflows/ldap/configs/ldap5/ldap2/certs/ca.crt b/tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ca.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap5/ldap2/certs/ca.crt rename to tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ca.crt diff --git a/tests/testflows/ldap/configs/ldap5/ldap2/certs/dhparam.pem b/tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/dhparam.pem similarity index 100% rename from tests/testflows/ldap/configs/ldap5/ldap2/certs/dhparam.pem rename to tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/dhparam.pem diff --git a/tests/testflows/ldap/configs/ldap5/ldap2/certs/ldap.crt b/tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ldap.crt similarity index 100% rename from tests/testflows/ldap/configs/ldap5/ldap2/certs/ldap.crt rename to tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ldap.crt diff --git a/tests/testflows/ldap/configs/ldap5/ldap2/certs/ldap.csr b/tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ldap.csr similarity index 100% rename from tests/testflows/ldap/configs/ldap5/ldap2/certs/ldap.csr rename to tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ldap.csr diff --git a/tests/testflows/ldap/configs/ldap5/ldap2/certs/ldap.key b/tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ldap.key similarity index 100% rename from tests/testflows/ldap/configs/ldap5/ldap2/certs/ldap.key rename to tests/testflows/ldap/authentication/configs/ldap5/ldap2/certs/ldap.key diff --git a/tests/testflows/ldap/configs/ldap5/ldap2/config/export.ldif b/tests/testflows/ldap/authentication/configs/ldap5/ldap2/config/export.ldif similarity index 100% rename from tests/testflows/ldap/configs/ldap5/ldap2/config/export.ldif rename to tests/testflows/ldap/authentication/configs/ldap5/ldap2/config/export.ldif diff --git a/tests/testflows/ldap/docker-compose/clickhouse-service.yml b/tests/testflows/ldap/authentication/docker-compose/clickhouse-service.yml similarity index 100% rename from tests/testflows/ldap/docker-compose/clickhouse-service.yml rename to tests/testflows/ldap/authentication/docker-compose/clickhouse-service.yml diff --git a/tests/testflows/ldap/docker-compose/docker-compose.yml b/tests/testflows/ldap/authentication/docker-compose/docker-compose.yml similarity index 100% rename from tests/testflows/ldap/docker-compose/docker-compose.yml rename to tests/testflows/ldap/authentication/docker-compose/docker-compose.yml diff --git a/tests/testflows/ldap/docker-compose/openldap-service.yml b/tests/testflows/ldap/authentication/docker-compose/openldap-service.yml similarity index 100% rename from tests/testflows/ldap/docker-compose/openldap-service.yml rename to tests/testflows/ldap/authentication/docker-compose/openldap-service.yml diff --git a/tests/testflows/ldap/docker-compose/zookeeper-service.yml b/tests/testflows/ldap/authentication/docker-compose/zookeeper-service.yml similarity index 100% rename from tests/testflows/ldap/docker-compose/zookeeper-service.yml rename to tests/testflows/ldap/authentication/docker-compose/zookeeper-service.yml diff --git a/tests/testflows/ldap/authentication/regression.py b/tests/testflows/ldap/authentication/regression.py new file mode 100755 index 00000000000..9d0a5ca743f --- /dev/null +++ b/tests/testflows/ldap/authentication/regression.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +import sys +from testflows.core import * + +append_path(sys.path, "..", "..") + +from helpers.cluster import Cluster +from helpers.argparser import argparser +from ldap.authentication.requirements import * + +# Cross-outs of known fails +xfails = { + "connection protocols/tls/tls_require_cert='try'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/tls/tls_require_cert='demand'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/starttls/tls_require_cert='try'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/starttls/tls_require_cert='demand'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/tls require cert default demand": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/starttls with custom port": + [(Fail, "it seems that starttls is not enabled by default on custom plain-text ports in LDAP server")], + "connection protocols/tls cipher suite": + [(Fail, "can't get it to work")] +} + +@TestFeature +@Name("authentication") +@ArgumentParser(argparser) +@Requirements( + RQ_SRS_007_LDAP_Authentication("1.0") +) +@XFails(xfails) +def regression(self, local, clickhouse_binary_path): + """ClickHouse integration with LDAP regression module. + """ + nodes = { + "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), + } + + with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: + self.context.cluster = cluster + + Scenario(run=load("ldap.authentication.tests.sanity", "scenario")) + Scenario(run=load("ldap.authentication.tests.multiple_servers", "scenario")) + Feature(run=load("ldap.authentication.tests.connections", "feature")) + Feature(run=load("ldap.authentication.tests.server_config", "feature")) + Feature(run=load("ldap.authentication.tests.user_config", "feature")) + Feature(run=load("ldap.authentication.tests.authentications", "feature")) + +if main(): + regression() diff --git a/tests/testflows/ldap/requirements/__init__.py b/tests/testflows/ldap/authentication/requirements/__init__.py similarity index 100% rename from tests/testflows/ldap/requirements/__init__.py rename to tests/testflows/ldap/authentication/requirements/__init__.py diff --git a/tests/testflows/ldap/requirements/requirements.md b/tests/testflows/ldap/authentication/requirements/requirements.md similarity index 100% rename from tests/testflows/ldap/requirements/requirements.md rename to tests/testflows/ldap/authentication/requirements/requirements.md diff --git a/tests/testflows/ldap/requirements/requirements.py b/tests/testflows/ldap/authentication/requirements/requirements.py similarity index 100% rename from tests/testflows/ldap/requirements/requirements.py rename to tests/testflows/ldap/authentication/requirements/requirements.py diff --git a/tests/testflows/ldap/tests/authentications.py b/tests/testflows/ldap/authentication/tests/authentications.py similarity index 99% rename from tests/testflows/ldap/tests/authentications.py rename to tests/testflows/ldap/authentication/tests/authentications.py index a1fb27bd51a..1b21dce7cc1 100644 --- a/tests/testflows/ldap/tests/authentications.py +++ b/tests/testflows/ldap/authentication/tests/authentications.py @@ -4,8 +4,8 @@ import random from multiprocessing.dummy import Pool from testflows.core import * from testflows.asserts import error -from ldap.tests.common import * -from ldap.requirements import * +from ldap.authentication.tests.common import * +from ldap.authentication.requirements import * servers = { "openldap1": { diff --git a/tests/testflows/ldap/tests/common.py b/tests/testflows/ldap/authentication/tests/common.py similarity index 99% rename from tests/testflows/ldap/tests/common.py rename to tests/testflows/ldap/authentication/tests/common.py index eb21923e930..1e382f8942c 100644 --- a/tests/testflows/ldap/tests/common.py +++ b/tests/testflows/ldap/authentication/tests/common.py @@ -172,7 +172,7 @@ def create_ldap_users_config_content(*users, config_d_dir="/etc/clickhouse-serve return Config(content, path, name, uid, "users.xml") def add_users_identified_with_ldap(*users): - """Add one or more users that are identified via + """Add one or more users that are identified via an ldap server using RBAC. """ node = current().context.node diff --git a/tests/testflows/ldap/tests/connections.py b/tests/testflows/ldap/authentication/tests/connections.py similarity index 98% rename from tests/testflows/ldap/tests/connections.py rename to tests/testflows/ldap/authentication/tests/connections.py index 8de4b3f4d01..f16f6c29b0e 100644 --- a/tests/testflows/ldap/tests/connections.py +++ b/tests/testflows/ldap/authentication/tests/connections.py @@ -1,8 +1,8 @@ from testflows.core import * from testflows.asserts import error -from ldap.tests.common import login -from ldap.requirements import * +from ldap.authentication.tests.common import login +from ldap.authentication.requirements import * @TestScenario @Requirements( diff --git a/tests/testflows/ldap/tests/multiple_servers.py b/tests/testflows/ldap/authentication/tests/multiple_servers.py similarity index 88% rename from tests/testflows/ldap/tests/multiple_servers.py rename to tests/testflows/ldap/authentication/tests/multiple_servers.py index aefc0116fa2..6e906023b0a 100644 --- a/tests/testflows/ldap/tests/multiple_servers.py +++ b/tests/testflows/ldap/authentication/tests/multiple_servers.py @@ -1,8 +1,8 @@ from testflows.core import * from testflows.asserts import error -from ldap.tests.common import login -from ldap.requirements import RQ_SRS_007_LDAP_Authentication_MultipleServers +from ldap.authentication.tests.common import login +from ldap.authentication.requirements import RQ_SRS_007_LDAP_Authentication_MultipleServers @TestScenario @Name("multiple servers") diff --git a/tests/testflows/ldap/tests/sanity.py b/tests/testflows/ldap/authentication/tests/sanity.py similarity index 95% rename from tests/testflows/ldap/tests/sanity.py rename to tests/testflows/ldap/authentication/tests/sanity.py index 9e5d8a2ddd7..542fa2a48b1 100644 --- a/tests/testflows/ldap/tests/sanity.py +++ b/tests/testflows/ldap/authentication/tests/sanity.py @@ -1,7 +1,7 @@ from testflows.core import * from testflows.asserts import error -from ldap.tests.common import add_user_to_ldap, delete_user_from_ldap +from ldap.authentication.tests.common import add_user_to_ldap, delete_user_from_ldap @TestScenario @Name("sanity") diff --git a/tests/testflows/ldap/tests/server_config.py b/tests/testflows/ldap/authentication/tests/server_config.py similarity index 99% rename from tests/testflows/ldap/tests/server_config.py rename to tests/testflows/ldap/authentication/tests/server_config.py index f3d03434afe..5658f7a9399 100644 --- a/tests/testflows/ldap/tests/server_config.py +++ b/tests/testflows/ldap/authentication/tests/server_config.py @@ -1,7 +1,7 @@ from testflows.core import * -from ldap.tests.common import * -from ldap.requirements import * +from ldap.authentication.tests.common import * +from ldap.authentication.requirements import * @TestScenario @Requirements( diff --git a/tests/testflows/ldap/tests/user_config.py b/tests/testflows/ldap/authentication/tests/user_config.py similarity index 98% rename from tests/testflows/ldap/tests/user_config.py rename to tests/testflows/ldap/authentication/tests/user_config.py index edc85a5877e..7927d72b510 100644 --- a/tests/testflows/ldap/tests/user_config.py +++ b/tests/testflows/ldap/authentication/tests/user_config.py @@ -2,8 +2,8 @@ import xml.etree.ElementTree as xmltree from testflows.core import * -from ldap.tests.common import * -from ldap.requirements import * +from ldap.authentication.tests.common import * +from ldap.authentication.requirements import * @TestScenario @Requirements( diff --git a/tests/testflows/ldap/regression.py b/tests/testflows/ldap/regression.py index 567807fc0a8..8520941ed0b 100755 --- a/tests/testflows/ldap/regression.py +++ b/tests/testflows/ldap/regression.py @@ -2,53 +2,19 @@ import sys from testflows.core import * -append_path(sys.path, "..") +append_path(sys.path, "..") -from helpers.cluster import Cluster from helpers.argparser import argparser -from ldap.requirements import * -# Cross-outs of known fails -xfails = { - "connection protocols/tls/tls_require_cert='try'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/tls/tls_require_cert='demand'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/starttls/tls_require_cert='try'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/starttls/tls_require_cert='demand'": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/tls require cert default demand": - [(Fail, "can't be tested with self-signed certificates")], - "connection protocols/starttls with custom port": - [(Fail, "it seems that starttls is not enabled by default on custom plain-text ports in LDAP server")], - "connection protocols/tls cipher suite": - [(Fail, "can't get it to work")] -} - -@TestFeature -@Name("ldap authentication") +@TestModule +@Name("ldap") @ArgumentParser(argparser) -@Requirements( - RQ_SRS_007_LDAP_Authentication("1.0") -) -@XFails(xfails) def regression(self, local, clickhouse_binary_path): - """ClickHouse integration with LDAP regression module. + """ClickHouse LDAP integration regression module. """ - nodes = { - "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), - } - - with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: - self.context.cluster = cluster + args = {"local": local, "clickhouse_binary_path": clickhouse_binary_path} - Scenario(run=load("ldap.tests.sanity", "scenario")) - Scenario(run=load("ldap.tests.multiple_servers", "scenario")) - Feature(run=load("ldap.tests.connections", "feature")) - Feature(run=load("ldap.tests.server_config", "feature")) - Feature(run=load("ldap.tests.user_config", "feature")) - Feature(run=load("ldap.tests.authentications", "feature")) + Feature(test=load("ldap.authentication.regression", "regression"))(**args) if main(): regression() From 85b913deaac7ac1bb36b5c06a118da5974a626f7 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 3 Sep 2020 16:53:09 +0200 Subject: [PATCH 073/411] Adding regression tests for LDAP external user directory. --- tests/testflows/helpers/cluster.py | 5 +- .../ldap/external_user_directory/__init__.py | 0 .../external_user_directory/configs/CA/ca.crt | 22 + .../external_user_directory/configs/CA/ca.key | 30 + .../external_user_directory/configs/CA/ca.srl | 1 + .../configs/CA/dhparam.pem | 8 + .../configs/CA/passphrase.txt | 1 + .../configs/clickhouse/common.xml | 6 + .../configs/clickhouse/config.d/logs.xml | 17 + .../configs/clickhouse/config.d/ports.xml | 5 + .../configs/clickhouse/config.d/remote.xml | 107 ++ .../configs/clickhouse/config.d/ssl.xml | 17 + .../configs/clickhouse/config.d/storage.xml | 20 + .../configs/clickhouse/config.d/zookeeper.xml | 10 + .../configs/clickhouse/config.xml | 448 +++++++ .../configs/clickhouse/ssl/dhparam.pem | 8 + .../configs/clickhouse/ssl/server.crt | 19 + .../configs/clickhouse/ssl/server.key | 28 + .../configs/clickhouse/users.xml | 133 ++ .../configs/clickhouse1/config.d/macros.xml | 8 + .../configs/clickhouse2/config.d/macros.xml | 8 + .../configs/clickhouse3/config.d/macros.xml | 8 + .../configs/ldap1/config/export.ldif | 64 + .../configs/ldap2/certs/ca.crt | 22 + .../configs/ldap2/certs/dhparam.pem | 5 + .../configs/ldap2/certs/ldap.crt | 20 + .../configs/ldap2/certs/ldap.csr | 17 + .../configs/ldap2/certs/ldap.key | 27 + .../configs/ldap2/config/export.ldif | 64 + .../configs/ldap3/certs/ca.crt | 22 + .../configs/ldap3/certs/dhparam.pem | 5 + .../configs/ldap3/certs/ldap.crt | 20 + .../configs/ldap3/certs/ldap.csr | 17 + .../configs/ldap3/certs/ldap.key | 27 + .../configs/ldap3/config/export.ldif | 64 + .../configs/ldap4/certs/ca.crt | 22 + .../configs/ldap4/certs/dhparam.pem | 5 + .../configs/ldap4/certs/ldap.crt | 20 + .../configs/ldap4/certs/ldap.csr | 17 + .../configs/ldap4/certs/ldap.key | 27 + .../configs/ldap4/config/export.ldif | 64 + .../configs/ldap5/config/export.ldif | 64 + .../configs/ldap5/ldap2/certs/ca.crt | 22 + .../configs/ldap5/ldap2/certs/dhparam.pem | 5 + .../configs/ldap5/ldap2/certs/ldap.crt | 20 + .../configs/ldap5/ldap2/certs/ldap.csr | 17 + .../configs/ldap5/ldap2/certs/ldap.key | 27 + .../configs/ldap5/ldap2/config/export.ldif | 64 + .../docker-compose/clickhouse-service.yml | 28 + .../docker-compose/docker-compose.yml | 162 +++ .../docker-compose/openldap-service.yml | 40 + .../docker-compose/zookeeper-service.yml | 18 + .../external_user_directory/regression.py | 55 + .../requirements/__init__.py | 1 + .../requirements/requirements.py | 1133 +++++++++++++++++ .../tests/authentications.py | 509 ++++++++ .../external_user_directory/tests/common.py | 181 +++ .../tests/connections.py | 270 ++++ .../tests/external_user_directory_config.py | 283 ++++ .../external_user_directory/tests/roles.py | 314 +++++ .../tests/server_config.py | 305 +++++ .../external_user_directory/tests/simple.py | 24 + tests/testflows/ldap/regression.py | 1 + 63 files changed, 4979 insertions(+), 2 deletions(-) create mode 100644 tests/testflows/ldap/external_user_directory/__init__.py create mode 100644 tests/testflows/ldap/external_user_directory/configs/CA/ca.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/CA/ca.key create mode 100644 tests/testflows/ldap/external_user_directory/configs/CA/ca.srl create mode 100644 tests/testflows/ldap/external_user_directory/configs/CA/dhparam.pem create mode 100644 tests/testflows/ldap/external_user_directory/configs/CA/passphrase.txt create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/common.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/logs.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ports.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/remote.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ssl.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/storage.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/zookeeper.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/dhparam.pem create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.key create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse/users.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse1/config.d/macros.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse2/config.d/macros.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/clickhouse3/config.d/macros.xml create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap1/config/export.ldif create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ca.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap2/certs/dhparam.pem create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.csr create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.key create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap2/config/export.ldif create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ca.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap3/certs/dhparam.pem create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.csr create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.key create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap3/config/export.ldif create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ca.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap4/certs/dhparam.pem create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.csr create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.key create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap4/config/export.ldif create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap5/config/export.ldif create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ca.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/dhparam.pem create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.crt create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.csr create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.key create mode 100644 tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/config/export.ldif create mode 100644 tests/testflows/ldap/external_user_directory/docker-compose/clickhouse-service.yml create mode 100644 tests/testflows/ldap/external_user_directory/docker-compose/docker-compose.yml create mode 100644 tests/testflows/ldap/external_user_directory/docker-compose/openldap-service.yml create mode 100644 tests/testflows/ldap/external_user_directory/docker-compose/zookeeper-service.yml create mode 100755 tests/testflows/ldap/external_user_directory/regression.py create mode 100644 tests/testflows/ldap/external_user_directory/requirements/__init__.py create mode 100644 tests/testflows/ldap/external_user_directory/requirements/requirements.py create mode 100644 tests/testflows/ldap/external_user_directory/tests/authentications.py create mode 100644 tests/testflows/ldap/external_user_directory/tests/common.py create mode 100644 tests/testflows/ldap/external_user_directory/tests/connections.py create mode 100644 tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py create mode 100644 tests/testflows/ldap/external_user_directory/tests/roles.py create mode 100644 tests/testflows/ldap/external_user_directory/tests/server_config.py create mode 100644 tests/testflows/ldap/external_user_directory/tests/simple.py diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py index 8288e700e3b..761acce40d3 100644 --- a/tests/testflows/helpers/cluster.py +++ b/tests/testflows/helpers/cluster.py @@ -53,7 +53,7 @@ class ClickHouseNode(Node): continue assert False, "container is not healthy" - def restart(self, timeout=120, safe=True): + def restart(self, timeout=120, safe=True, wait_healthy=True): """Restart node. """ if safe: @@ -73,7 +73,8 @@ class ClickHouseNode(Node): self.cluster.command(None, f'{self.cluster.docker_compose} restart {self.name}', timeout=timeout) - self.wait_healthy(timeout) + if wait_healthy: + self.wait_healthy(timeout) def query(self, sql, message=None, exitcode=None, steps=True, no_checks=False, raise_on_exception=False, step=By, settings=None, *args, **kwargs): diff --git a/tests/testflows/ldap/external_user_directory/__init__.py b/tests/testflows/ldap/external_user_directory/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/testflows/ldap/external_user_directory/configs/CA/ca.crt b/tests/testflows/ldap/external_user_directory/configs/CA/ca.crt new file mode 100644 index 00000000000..8c71e3afc91 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/CA/ca.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlTCCAn2gAwIBAgIUJBqw2dHM2DDCZjYSkPOESlvDH6swDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQHDAZPdHRhd2Ex +ETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTENMAsGA1UEAwwEcm9vdDAe +Fw0yMDA2MTExOTAzNDhaFw0zMDA2MDkxOTAzNDhaMFoxCzAJBgNVBAYTAkNBMQsw +CQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYDVQQKDAhBbHRpbml0eTEL +MAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC9Irr0zGV+HCI2fZ0ht4hR5It4Sbjz4RwZV8ENRP/+TEz8l9eK +J6ygxhKX7SMYzIs/jS9Gsq4plX1r2ujW1qRf8yLpR4+dGLP+jBRi1drj0XjZXosT +SERjWzgPauWxL9LN8+l26eBAqz6fw5e0W8WRSTgf5iGiCcKOTmaATIUjP0CdfWKK +qpktI4vhe++CXZFJ3usR+8KZ/FwwbCLJM/3J2HnbcXfcaYPYvr1tfqLudKSTbG9H +M3+AVwjctdesc/0sbd51Zsm0ClQptMbuKnDCYauGg61kNkgbgPgRmH9Pzo67DtxF +/WW+PtOzq8xLOifciQ9Piboy9QBSQZGwf4wzAgMBAAGjUzBRMB0GA1UdDgQWBBSi +njya0RDozx3OZTLYFpwqYnlpIDAfBgNVHSMEGDAWgBSinjya0RDozx3OZTLYFpwq +YnlpIDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBAD7VyFg7F +U1C25KFvtauchAOjCW6w7U/b3z1dVZvcQ88/kH1VsLUcfGixlSilUEfPTJsi7OA0 +R5BQdh2GGcjUJv4iqEFGU05KvMVmRRKn08P62+ZhJxKMxG26VzcliRZzCMkI6d0W +lFwI6nM45yeqdHVh5k4xbuJzqpbD9BtXXLI+/Ra9Fx8S9ETA3GdidpZLU5P1VLxq +UuedfqyAVWZXpr6TAURGxouRmRzul9yFzbSUex+MLEIPrstjtEwV3+tBQZJz9xAS +TVPj+Nv3LO7GCq54bdwkq1ioWbSL2hEmABkj6kdW/JwmfhGHf/2rirDVMzrTYw07 +dFJfAZC+FEsv +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/CA/ca.key b/tests/testflows/ldap/external_user_directory/configs/CA/ca.key new file mode 100644 index 00000000000..e7a7f664dcf --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/CA/ca.key @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,D06B9754A2069EBB4E77065DC9B605A1 + +FJT794Z6AUuUB5Vp5W2iR6zzCvQUg2dtKoE+xhFdbgC7lmSfA2W/O9fx15Il67Yj +Bbpm9Y6yteUSDQpJrvBdkhXeBkYEa5K1CA+0Jdx98nzwP3KBhHNxVVrTWRc5kniB +LMV3iBQEbAafxgL7gN+EWr3eV7w7ZSqT7D5br/mlBALU62gv2UzwTXLu1CgyNWMC +HIPjIX50Zga+BnhZhtQvM4Yj1gOsn+X6AaEZ3KjTfCDqthYQf2ldswW4gAlPAq83 ++INq9Spx+QG97Z+1XO2DmmGTZL0z+OFLT+3y26/UcftM26ODY09Dcf3gt0n6RIUV +0KsD1qQL0ppu4CHVnbIkOKMBe86qBl+kG8FVmyhgZ8D9ULlF1tpyTVKvHR82V2B5 +ztbc5EY1Fhb+r7OVVJlbCeo/bWmWybODZrpN49x5gGZpM3+8ApaHupGZ+cRFkQKG +rDpqC5gflT3WwFNxtP5noWcV+Gzb3riXNM3c8G5aIpLZwmmaTLK9ahKqMcq4Ljf+ +hir8kuCMqIKt3m7Ceoj4wAHSP8xO0y/cc1WYNb3CI0Emk795aR6IPUw4vDEXHG27 +OLoCJTvl/JKRWJGkdQx8wKAs/uw/qwtbhYoQJccTjfvy4NXH3tpSgxCE8OTWuEch +TAN8ra1PDGAUu+1MeT5gZ9uI1BEU6hXMME4mVRpJdcmw9MVy3V+B6rkUqX3kFAfR +e2ueF5qgIp+A4/UlVe5cKdWAQxu4BnUESLooA7cbgcLypdao9bRx9bXH8S3aNgxW +IdgICpc/v8wAX2yqMe191KgR9Vh1p0RCw/kEGVgWfY/IaQpsaYuq5quZbvr/fN5T +d++ySAMaPysaCadLUdZJLw56uk4Y+PYzR+ygjTX9dCCHedrAU8RYM55FJ/fyD3bQ +Hn9/n7PZyWy6u/TYt6dhlcYxaS3Opzw4eAQB8tGZJRYQ3AKpHpTEC57lXoMnUPKo ++nBmb0+YulylMZdns0WIBJlcv6qzIaNhDMrjyi18n1ezzPIGH7ivUjoXy2FL23q5 +f3aqJK4UUDEDkC8IeZkS+ykYxnohjFDhUyBe5gjryLqdMdy9EerehCWPf425AztX +c/EWPzDl46qmxWhugOlz3Fiw95VlYu0MUDRayHuZiYPplgJypChuU4EHJ+q8V2z3 +BwjSo1bD4nfc8f68qEOtdZ1u/ClcolMwlZQYDJz/DiE4JOcd2Gx4QSF5vaInm0/4 +mMj/ZWna4DAYFbH8IGh7xUPDqeIWhBYlgrD69ajKyay5Vu3La/d2QW20BhX35Ro2 +ZJVR+lfioMmxn4y481H2pv+5gOlGwh02Oa8qLhZBb8W+DvFShNk6mk87eCForFFT +CDgmvfsC/cS2wZkcFTecq6vbjFlt+OF13NCKlcO3wCm44D+bwVPeMrU6HycCVQw7 +SASrnP/th5sJbv11byb2lKgVdVHWk090bqnDwB9H2hGIb9JnPC9ZpaL/mocYyzTi +H9fcBrMYkL09FJGr3Uff7qEY4XQTMlLadXue3iKd19PRgV8cRyKp37MYI9/3iLwv +eYHLtMfrifZahf1ksOPeBphnlfzWo9qqfooUCaGxfSlNPUHhrHZ4aMiRyTE8Xeh2 +-----END RSA PRIVATE KEY----- diff --git a/tests/testflows/ldap/external_user_directory/configs/CA/ca.srl b/tests/testflows/ldap/external_user_directory/configs/CA/ca.srl new file mode 100644 index 00000000000..66feb9c8a35 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/CA/ca.srl @@ -0,0 +1 @@ +227B125D27B6B1A4B5955361365DF8EC2D7098C1 diff --git a/tests/testflows/ldap/external_user_directory/configs/CA/dhparam.pem b/tests/testflows/ldap/external_user_directory/configs/CA/dhparam.pem new file mode 100644 index 00000000000..554d75696ee --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/CA/dhparam.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEA1iatTn4jdw1WIu09qeLj8OEeLhzG/w2lI4RUeJT9nU+WTwegpvLN +/MvrIMIKHRmItyxgraYFau2moC7RKm7OKLmFt6e34QeMvM1vXpuwQav6mfp8GsYL +mEIw5riFcB73E32NN3g7qmfmurkTF28BohmqhuQp2et7FNoGBKQ6ePZzGHWil3yG +nEnCwyK0o3eP2IEytx2N50uUWVdfg3MN34L3wqpUivArrjBkoMpqm3/V3wdfoYG9 +ZQkH0gIxT/2FIixCLGlfBsJ1qA/Apz1BJZbGqVu5M5iiQmq+LWN5JLS3xYai4wJL +rIY8DhjbciSNVWkwTJHzaLwIQa9a6p6mUwIBAg== +-----END DH PARAMETERS----- diff --git a/tests/testflows/ldap/external_user_directory/configs/CA/passphrase.txt b/tests/testflows/ldap/external_user_directory/configs/CA/passphrase.txt new file mode 100644 index 00000000000..2cf58b2364c --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/CA/passphrase.txt @@ -0,0 +1 @@ +altinity diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/common.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/common.xml new file mode 100644 index 00000000000..df952b28c82 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/common.xml @@ -0,0 +1,6 @@ + + Europe/Moscow + 0.0.0.0 + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/logs.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/logs.xml new file mode 100644 index 00000000000..bdf1bbc11c1 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/logs.xml @@ -0,0 +1,17 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + system + part_log
+ 500 + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ports.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ports.xml new file mode 100644 index 00000000000..fbc6cea74c0 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ports.xml @@ -0,0 +1,5 @@ + + + 8443 + 9440 + \ No newline at end of file diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/remote.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/remote.xml new file mode 100644 index 00000000000..51be2a6e8e3 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/remote.xml @@ -0,0 +1,107 @@ + + + + + + true + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + + + + true + + clickhouse1 + 9440 + 1 + + + clickhouse2 + 9440 + 1 + + + clickhouse3 + 9440 + 1 + + + + + + + clickhouse1 + 9000 + + + + + clickhouse2 + 9000 + + + + + clickhouse3 + 9000 + + + + + + + clickhouse1 + 9440 + 1 + + + + + clickhouse2 + 9440 + 1 + + + + + clickhouse3 + 9440 + 1 + + + + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ssl.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ssl.xml new file mode 100644 index 00000000000..ca65ffd5e04 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/ssl.xml @@ -0,0 +1,17 @@ + + + + /etc/clickhouse-server/ssl/server.crt + /etc/clickhouse-server/ssl/server.key + none + true + + + true + none + + AcceptCertificateHandler + + + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/storage.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/storage.xml new file mode 100644 index 00000000000..618fd6b6d24 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/storage.xml @@ -0,0 +1,20 @@ + + + + + + 1024 + + + + + + + default + + + + + + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/zookeeper.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/zookeeper.xml new file mode 100644 index 00000000000..96270e7b645 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.d/zookeeper.xml @@ -0,0 +1,10 @@ + + + + + zookeeper + 2181 + + 15000 + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml new file mode 100644 index 00000000000..80c3150d326 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml @@ -0,0 +1,448 @@ + + + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + 8123 + 9000 + + + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 9009 + + + + + + + + + + + + + + + + + + + + 4096 + 3 + + + 100 + + + + + + 8589934592 + + + 5368709120 + + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + /var/lib/clickhouse/user_files/ + + + /var/lib/clickhouse/access/ + + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + + + users.xml + + + default + + + + + + default + + + + + + + + + false + + + + + + + + localhost + 9000 + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + localhost + 9000 + + + + + localhost + 1 + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + 7500 + + + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 + + + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 + + + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/dhparam.pem b/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/dhparam.pem new file mode 100644 index 00000000000..2e6cee0798d --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/dhparam.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEAua92DDli13gJ+//ZXyGaggjIuidqB0crXfhUlsrBk9BV1hH3i7fR +XGP9rUdk2ubnB3k2ejBStL5oBrkHm9SzUFSQHqfDjLZjKoUpOEmuDc4cHvX1XTR5 +Pr1vf5cd0yEncJWG5W4zyUB8k++SUdL2qaeslSs+f491HBLDYn/h8zCgRbBvxhxb +9qeho1xcbnWeqkN6Kc9bgGozA16P9NLuuLttNnOblkH+lMBf42BSne/TWt3AlGZf +slKmmZcySUhF8aKfJnLKbkBCFqOtFRh8zBA9a7g+BT/lSANATCDPaAk1YVih2EKb +dpc3briTDbRsiqg2JKMI7+VdULY9bh3EawIBAg== +-----END DH PARAMETERS----- diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.crt b/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.crt new file mode 100644 index 00000000000..7ade2d96273 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI +4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T +4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU +7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj +sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg +pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC +kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j +4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr +85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C +L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD ++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L +P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp +0g== +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.key b/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.key new file mode 100644 index 00000000000..f0fb61ac443 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/ssl/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn +DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L +OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus +TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 +JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F +KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 +ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP +TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f +7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN +MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 +212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu +plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr +/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu +Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa +CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g +1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 +bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq +cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez +on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K +GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 +nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU +2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh +vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa +dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr +vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 +KCBtmIrQLqzMpnBpLNuSY+Q= +-----END PRIVATE KEY----- diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/users.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/users.xml new file mode 100644 index 00000000000..86b2cd9e1e3 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/users.xml @@ -0,0 +1,133 @@ + + + + + + + + 10000000000 + + + 0 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + 1 + + + + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse1/config.d/macros.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse1/config.d/macros.xml new file mode 100644 index 00000000000..6cdcc1b440c --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse1/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse1 + 01 + 01 + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse2/config.d/macros.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse2/config.d/macros.xml new file mode 100644 index 00000000000..a114a9ce4ab --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse2/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse2 + 01 + 02 + + diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse3/config.d/macros.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse3/config.d/macros.xml new file mode 100644 index 00000000000..904a27b0172 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse3/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse3 + 01 + 03 + + diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap1/config/export.ldif b/tests/testflows/ldap/external_user_directory/configs/ldap1/config/export.ldif new file mode 100644 index 00000000000..621dd32ca0c --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap1/config/export.ldif @@ -0,0 +1,64 @@ +# LDIF Export for dc=company,dc=com +# Server: openldap (openldap) +# Search Scope: sub +# Search Filter: (objectClass=*) +# Total Entries: 7 +# +# Generated by phpLDAPadmin (http://phpldapadmin.sourceforge.net) on May 22, 2020 5:51 pm +# Version: 1.2.5 + +# Entry 1: dc=company,dc=com +#dn: dc=company,dc=com +#dc: company +#o: company +#objectclass: top +#objectclass: dcObject +#objectclass: organization + +# Entry 2: cn=admin,dc=company,dc=com +#dn: cn=admin,dc=company,dc=com +#cn: admin +#description: LDAP administrator +#objectclass: simpleSecurityObject +#objectclass: organizationalRole +#userpassword: {SSHA}eUEupkQCTvq9SkrxfWGSe5rX+orrjVbF + +# Entry 3: ou=groups,dc=company,dc=com +dn: ou=groups,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: groups + +# Entry 4: cn=admin,ou=groups,dc=company,dc=com +dn: cn=admin,ou=groups,dc=company,dc=com +cn: admin +gidnumber: 500 +objectclass: posixGroup +objectclass: top + +# Entry 5: cn=users,ou=groups,dc=company,dc=com +dn: cn=users,ou=groups,dc=company,dc=com +cn: users +gidnumber: 501 +objectclass: posixGroup +objectclass: top + +# Entry 6: ou=users,dc=company,dc=com +dn: ou=users,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: users + +# Entry 7: cn=user1,ou=users,dc=company,dc=com +dn: cn=user1,ou=users,dc=company,dc=com +cn: user1 +gidnumber: 501 +givenname: John +homedirectory: /home/users/user1 +objectclass: inetOrgPerson +objectclass: posixAccount +objectclass: top +sn: User +uid: user1 +uidnumber: 1101 +userpassword: user1 diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ca.crt b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ca.crt new file mode 100644 index 00000000000..8c71e3afc91 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ca.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlTCCAn2gAwIBAgIUJBqw2dHM2DDCZjYSkPOESlvDH6swDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQHDAZPdHRhd2Ex +ETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTENMAsGA1UEAwwEcm9vdDAe +Fw0yMDA2MTExOTAzNDhaFw0zMDA2MDkxOTAzNDhaMFoxCzAJBgNVBAYTAkNBMQsw +CQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYDVQQKDAhBbHRpbml0eTEL +MAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC9Irr0zGV+HCI2fZ0ht4hR5It4Sbjz4RwZV8ENRP/+TEz8l9eK +J6ygxhKX7SMYzIs/jS9Gsq4plX1r2ujW1qRf8yLpR4+dGLP+jBRi1drj0XjZXosT +SERjWzgPauWxL9LN8+l26eBAqz6fw5e0W8WRSTgf5iGiCcKOTmaATIUjP0CdfWKK +qpktI4vhe++CXZFJ3usR+8KZ/FwwbCLJM/3J2HnbcXfcaYPYvr1tfqLudKSTbG9H +M3+AVwjctdesc/0sbd51Zsm0ClQptMbuKnDCYauGg61kNkgbgPgRmH9Pzo67DtxF +/WW+PtOzq8xLOifciQ9Piboy9QBSQZGwf4wzAgMBAAGjUzBRMB0GA1UdDgQWBBSi +njya0RDozx3OZTLYFpwqYnlpIDAfBgNVHSMEGDAWgBSinjya0RDozx3OZTLYFpwq +YnlpIDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBAD7VyFg7F +U1C25KFvtauchAOjCW6w7U/b3z1dVZvcQ88/kH1VsLUcfGixlSilUEfPTJsi7OA0 +R5BQdh2GGcjUJv4iqEFGU05KvMVmRRKn08P62+ZhJxKMxG26VzcliRZzCMkI6d0W +lFwI6nM45yeqdHVh5k4xbuJzqpbD9BtXXLI+/Ra9Fx8S9ETA3GdidpZLU5P1VLxq +UuedfqyAVWZXpr6TAURGxouRmRzul9yFzbSUex+MLEIPrstjtEwV3+tBQZJz9xAS +TVPj+Nv3LO7GCq54bdwkq1ioWbSL2hEmABkj6kdW/JwmfhGHf/2rirDVMzrTYw07 +dFJfAZC+FEsv +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/dhparam.pem b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/dhparam.pem new file mode 100644 index 00000000000..0a96faffd62 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/dhparam.pem @@ -0,0 +1,5 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAJitt2hhnpDViQ5ko2ipBMdjy+bZ6FR/WdZ987R7lQvBkKehPXmxtEyV +AO6ofv5CZSDJokc5bUeBOAtg0EhMTCH82uPdwQvt58jRXcxXBg4JTjkx+oW9LBv2 +FdZsbaX8+SYivmiZ0Jp8T/HBm/4DA9VBS0O5GFRS4C7dHhmSTPfDAgEC +-----END DH PARAMETERS----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.crt b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.crt new file mode 100644 index 00000000000..9167cbf861d --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQDCCAigCFCJ7El0ntrGktZVTYTZd+OwtcJjBMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAkNBMQswCQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYD +VQQKDAhBbHRpbml0eTELMAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwHhcNMjAw +NjExMTkxMTQzWhcNMzAwNjA5MTkxMTQzWjBfMQswCQYDVQQGEwJDQTELMAkGA1UE +CAwCT04xDzANBgNVBAcMBk90dGF3YTERMA8GA1UECgwIQWx0aW5pdHkxCzAJBgNV +BAsMAlFBMRIwEAYDVQQDDAlvcGVubGRhcDIwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC0Mbn//U56URavMgXm82FWP6vBdKuRydFX/L0M5XLlnAtk/IXG +/T+4t7nOBJxWmTp/xpsPtSMALE4eFJpEUEqlpVbG5DfBzVWcYOWoMeRAcHWCDkzr +PkB6I0dfF0Mm5hoaDhn+ZXjBWvoh/IlJdAnPg5mlejflJBQ7xtFC9eN6WjldXuRO +vyntGNuMfVLgITHwXuH2yZ98G0mFO6TU/9dRY/Z3D6RTSzKdb17Yk/VnG+ry92u2 +0sgXIBvhuJuC3ksWLArwwFoMl8DVa05D4O2H76goGdCcQ0KzqBV8RPXAh3UcgP2e +Zu90p2EGIhIk+sZTCkPd4dorxjL9nkRR86HdAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggEBAJWiCxJaTksv/BTsh/etxlDY5eHwqStqIuiovEQ8bhGAcKJ3bfWd/YTb8DUS +hrLvXrXdOVC+U8PqPFXBpdOqcm5Dc233z52VgUCb+0EKv3lAzgKXRIo32h52skdK +NnRrCHDeDzgfEIXR4MEJ99cLEaxWyXQhremmTYWHYznry9/4NYz40gCDxHn9dJAi +KxFyDNxhtuKs58zp4PrBoo+542JurAoLPtRGOhdXpU2RkQVU/ho38HsAXDStAB5D +vAoSxPuMHKgo17ffrb0oqU3didwaA9fIsz7Mr6RxmI7X03s7hLzNBq9FCqu0U3RR +CX4zWGFNJu/ieSGVWLYKQzbYxp8= +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.csr b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.csr new file mode 100644 index 00000000000..bf569f727d6 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.csr @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICpDCCAYwCAQAwXzELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQH +DAZPdHRhd2ExETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTESMBAGA1UE +AwwJb3BlbmxkYXAyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtDG5 +//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyFxv0/uLe5zgScVpk6f8ab +D7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M6z5AeiNHXxdDJuYaGg4Z +/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7kTr8p7RjbjH1S4CEx8F7h +9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdrttLIFyAb4bibgt5LFiwK +8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9nmbvdKdhBiISJPrGUwpD +3eHaK8Yy/Z5EUfOh3QIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAEzIjZQOT5R7 +mEJg+RFpCSIoPn3xJ4/VMMyWqA3bTGZKpb4S6GxgsierY/87kPL7jZrMdGYB4Dc3 +2M3VWZGXlYo8vctH1zLE9VW6CzosUpl20lhdgydoCMz3RQqdJyK8aGeFTeLtk7G/ +TRCCUFUE6jaA+VtaCPCnOJSff3jUf76xguEu7dgTZgCKV7dtBqald8gIzF3D+AJJ +7pEN2UrC3UR0xpe2cj2GhndQJ+WsIyft3zpNFzAO13j8ZPibuVP7oDWcW3ixNCWC +213aeRVplJGof8Eo6llDxP+6Fwp1YmOoQmwB1Xm3t4ADn7FLJ14LONLB7q40KviG +RyLyqu3IVOI= +-----END CERTIFICATE REQUEST----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.key b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.key new file mode 100644 index 00000000000..5ab3a3f8b59 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap2/certs/ldap.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAtDG5//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyF +xv0/uLe5zgScVpk6f8abD7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M +6z5AeiNHXxdDJuYaGg4Z/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7k +Tr8p7RjbjH1S4CEx8F7h9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdr +ttLIFyAb4bibgt5LFiwK8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9 +nmbvdKdhBiISJPrGUwpD3eHaK8Yy/Z5EUfOh3QIDAQABAoIBADugMMIKWcuTxYPX +c6iGZHEbxIPRTWyCcalB0nTQAAMGbabPAJ1l8432DZ+kWu806OybFXhPIfPOtVKy +0pFEWE8TtPE/V0vj3C5Qye2sBLFmBRwyCzXUdZV00wseMXRPs9dnTyalAR5KMnbI +j80kfpKSI2dkV9aU57UYBuq3Xrx/TCGItwL769D4ZZW9BvbpiTZApQQFZ0gwUFFn +btPXGU9Ti8H4mfBuZWL+5CaZdqOo76+CXvMPaUK0F9MJp4yX3XxQLRNH3qz/Tyn7 +h7QOOo0XTqoUmzRw0N9QRVH5LRdSE5yq3aF9aFKjNW59exz+62pufOFadngzkpkn +OKCzgWkCgYEA4mOWWMzdYwMn3GtfG7whqlqy7wOmMkNb81zTDQejHBV98dnj0AHr +deurfKWzHrAh3DXo6tFeqUIgXabhBPS/0dEx/S5sgLFmuUZP05EUYahfWBgzzmM9 +C6Oe5xIMLzxsZCJczolsfkEsoFe4o0vkvuLYoQrQL7InzewcDy8cUxsCgYEAy8Na +YCnanSNDY03Bulcni+5sF+opaHseeki1pv3nlw8TwsWuZF9ApS+yL7ck9jJjxBRR +RC3KGmpoqIr0vTmUYS946ngQWXPE90zfuhJfM+NRv/q0oCjH0qAcxRbTkls5On9v +oxJ8rO7gD6K85eHqasWdbCVzdZrobOXzay37tmcCgYBfyUUmw190cjReZauzH3Gb +E48b5A5gu/Fe0cqWe8G+szU7rDZgnz9SAGnpbm6QMHPTKZgoKngD42+wUFhq8Wdr +zjh5aDgOZ4EQKTjDSmI2Q7g7nNnmnESK9SrZl+BB6C3wXD2qQaj+7nKEUTlVFlpt +jaucz+dwFtASp7Djl8pDOwKBgEtr2c3ycArt/ImLRIP2spqm+7e2YvFbcSKOOz6+ +iLRvTj8v8KcSYtlB2FC1F6dRa4AujQ4RbNduP6LzHDfWUkfOzJDtNBAIPAXVnJJB +LqAEKkRHRghqT9x0i3GgS1vHDF3MwcO4mhFgserXr9ffUWeIEgbvrdcAKbv1Oa6Y +bK1NAoGAGPm8ISmboDJynjBl9wMrkcy23Pwg9kmyocdWUHh0zMLDKriZNKYB6u/U +C+/RTfkohPoHPzkeqWiHp7z3JhMItYUfTkNW6vMCxEGc0NEN6ZyMIjtiDPGN1n6O +E7jmODFmj1AQICQGdV5SHp+yKvKyb0YHKyDwETbs4SZBXxVvjEw= +-----END RSA PRIVATE KEY----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap2/config/export.ldif b/tests/testflows/ldap/external_user_directory/configs/ldap2/config/export.ldif new file mode 100644 index 00000000000..6766aaae6f1 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap2/config/export.ldif @@ -0,0 +1,64 @@ +# LDIF Export for dc=company,dc=com +# Server: openldap (openldap) +# Search Scope: sub +# Search Filter: (objectClass=*) +# Total Entries: 7 +# +# Generated by phpLDAPadmin (http://phpldapadmin.sourceforge.net) on May 22, 2020 5:51 pm +# Version: 1.2.5 + +# Entry 1: dc=company,dc=com +#dn: dc=company,dc=com +#dc: company +#o: company +#objectclass: top +#objectclass: dcObject +#objectclass: organization + +# Entry 2: cn=admin,dc=company,dc=com +#dn: cn=admin,dc=company,dc=com +#cn: admin +#description: LDAP administrator +#objectclass: simpleSecurityObject +#objectclass: organizationalRole +#userpassword: {SSHA}eUEupkQCTvq9SkrxfWGSe5rX+orrjVbF + +# Entry 3: ou=groups,dc=company,dc=com +dn: ou=groups,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: groups + +# Entry 4: cn=admin,ou=groups,dc=company,dc=com +dn: cn=admin,ou=groups,dc=company,dc=com +cn: admin +gidnumber: 500 +objectclass: posixGroup +objectclass: top + +# Entry 5: cn=users,ou=groups,dc=company,dc=com +dn: cn=users,ou=groups,dc=company,dc=com +cn: users +gidnumber: 501 +objectclass: posixGroup +objectclass: top + +# Entry 6: ou=users,dc=company,dc=com +dn: ou=users,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: users + +# Entry 7: cn=user2,ou=users,dc=company,dc=com +dn: cn=user2,ou=users,dc=company,dc=com +cn: user2 +gidnumber: 501 +givenname: John +homedirectory: /home/users/user2 +objectclass: inetOrgPerson +objectclass: posixAccount +objectclass: top +sn: User +uid: user2 +uidnumber: 1002 +userpassword: user2 diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ca.crt b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ca.crt new file mode 100644 index 00000000000..8c71e3afc91 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ca.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlTCCAn2gAwIBAgIUJBqw2dHM2DDCZjYSkPOESlvDH6swDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQHDAZPdHRhd2Ex +ETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTENMAsGA1UEAwwEcm9vdDAe +Fw0yMDA2MTExOTAzNDhaFw0zMDA2MDkxOTAzNDhaMFoxCzAJBgNVBAYTAkNBMQsw +CQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYDVQQKDAhBbHRpbml0eTEL +MAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC9Irr0zGV+HCI2fZ0ht4hR5It4Sbjz4RwZV8ENRP/+TEz8l9eK +J6ygxhKX7SMYzIs/jS9Gsq4plX1r2ujW1qRf8yLpR4+dGLP+jBRi1drj0XjZXosT +SERjWzgPauWxL9LN8+l26eBAqz6fw5e0W8WRSTgf5iGiCcKOTmaATIUjP0CdfWKK +qpktI4vhe++CXZFJ3usR+8KZ/FwwbCLJM/3J2HnbcXfcaYPYvr1tfqLudKSTbG9H +M3+AVwjctdesc/0sbd51Zsm0ClQptMbuKnDCYauGg61kNkgbgPgRmH9Pzo67DtxF +/WW+PtOzq8xLOifciQ9Piboy9QBSQZGwf4wzAgMBAAGjUzBRMB0GA1UdDgQWBBSi +njya0RDozx3OZTLYFpwqYnlpIDAfBgNVHSMEGDAWgBSinjya0RDozx3OZTLYFpwq +YnlpIDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBAD7VyFg7F +U1C25KFvtauchAOjCW6w7U/b3z1dVZvcQ88/kH1VsLUcfGixlSilUEfPTJsi7OA0 +R5BQdh2GGcjUJv4iqEFGU05KvMVmRRKn08P62+ZhJxKMxG26VzcliRZzCMkI6d0W +lFwI6nM45yeqdHVh5k4xbuJzqpbD9BtXXLI+/Ra9Fx8S9ETA3GdidpZLU5P1VLxq +UuedfqyAVWZXpr6TAURGxouRmRzul9yFzbSUex+MLEIPrstjtEwV3+tBQZJz9xAS +TVPj+Nv3LO7GCq54bdwkq1ioWbSL2hEmABkj6kdW/JwmfhGHf/2rirDVMzrTYw07 +dFJfAZC+FEsv +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/dhparam.pem b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/dhparam.pem new file mode 100644 index 00000000000..0a96faffd62 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/dhparam.pem @@ -0,0 +1,5 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAJitt2hhnpDViQ5ko2ipBMdjy+bZ6FR/WdZ987R7lQvBkKehPXmxtEyV +AO6ofv5CZSDJokc5bUeBOAtg0EhMTCH82uPdwQvt58jRXcxXBg4JTjkx+oW9LBv2 +FdZsbaX8+SYivmiZ0Jp8T/HBm/4DA9VBS0O5GFRS4C7dHhmSTPfDAgEC +-----END DH PARAMETERS----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.crt b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.crt new file mode 100644 index 00000000000..9167cbf861d --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQDCCAigCFCJ7El0ntrGktZVTYTZd+OwtcJjBMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAkNBMQswCQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYD +VQQKDAhBbHRpbml0eTELMAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwHhcNMjAw +NjExMTkxMTQzWhcNMzAwNjA5MTkxMTQzWjBfMQswCQYDVQQGEwJDQTELMAkGA1UE +CAwCT04xDzANBgNVBAcMBk90dGF3YTERMA8GA1UECgwIQWx0aW5pdHkxCzAJBgNV +BAsMAlFBMRIwEAYDVQQDDAlvcGVubGRhcDIwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC0Mbn//U56URavMgXm82FWP6vBdKuRydFX/L0M5XLlnAtk/IXG +/T+4t7nOBJxWmTp/xpsPtSMALE4eFJpEUEqlpVbG5DfBzVWcYOWoMeRAcHWCDkzr +PkB6I0dfF0Mm5hoaDhn+ZXjBWvoh/IlJdAnPg5mlejflJBQ7xtFC9eN6WjldXuRO +vyntGNuMfVLgITHwXuH2yZ98G0mFO6TU/9dRY/Z3D6RTSzKdb17Yk/VnG+ry92u2 +0sgXIBvhuJuC3ksWLArwwFoMl8DVa05D4O2H76goGdCcQ0KzqBV8RPXAh3UcgP2e +Zu90p2EGIhIk+sZTCkPd4dorxjL9nkRR86HdAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggEBAJWiCxJaTksv/BTsh/etxlDY5eHwqStqIuiovEQ8bhGAcKJ3bfWd/YTb8DUS +hrLvXrXdOVC+U8PqPFXBpdOqcm5Dc233z52VgUCb+0EKv3lAzgKXRIo32h52skdK +NnRrCHDeDzgfEIXR4MEJ99cLEaxWyXQhremmTYWHYznry9/4NYz40gCDxHn9dJAi +KxFyDNxhtuKs58zp4PrBoo+542JurAoLPtRGOhdXpU2RkQVU/ho38HsAXDStAB5D +vAoSxPuMHKgo17ffrb0oqU3didwaA9fIsz7Mr6RxmI7X03s7hLzNBq9FCqu0U3RR +CX4zWGFNJu/ieSGVWLYKQzbYxp8= +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.csr b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.csr new file mode 100644 index 00000000000..bf569f727d6 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.csr @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICpDCCAYwCAQAwXzELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQH +DAZPdHRhd2ExETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTESMBAGA1UE +AwwJb3BlbmxkYXAyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtDG5 +//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyFxv0/uLe5zgScVpk6f8ab +D7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M6z5AeiNHXxdDJuYaGg4Z +/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7kTr8p7RjbjH1S4CEx8F7h +9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdrttLIFyAb4bibgt5LFiwK +8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9nmbvdKdhBiISJPrGUwpD +3eHaK8Yy/Z5EUfOh3QIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAEzIjZQOT5R7 +mEJg+RFpCSIoPn3xJ4/VMMyWqA3bTGZKpb4S6GxgsierY/87kPL7jZrMdGYB4Dc3 +2M3VWZGXlYo8vctH1zLE9VW6CzosUpl20lhdgydoCMz3RQqdJyK8aGeFTeLtk7G/ +TRCCUFUE6jaA+VtaCPCnOJSff3jUf76xguEu7dgTZgCKV7dtBqald8gIzF3D+AJJ +7pEN2UrC3UR0xpe2cj2GhndQJ+WsIyft3zpNFzAO13j8ZPibuVP7oDWcW3ixNCWC +213aeRVplJGof8Eo6llDxP+6Fwp1YmOoQmwB1Xm3t4ADn7FLJ14LONLB7q40KviG +RyLyqu3IVOI= +-----END CERTIFICATE REQUEST----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.key b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.key new file mode 100644 index 00000000000..5ab3a3f8b59 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap3/certs/ldap.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAtDG5//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyF +xv0/uLe5zgScVpk6f8abD7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M +6z5AeiNHXxdDJuYaGg4Z/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7k +Tr8p7RjbjH1S4CEx8F7h9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdr +ttLIFyAb4bibgt5LFiwK8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9 +nmbvdKdhBiISJPrGUwpD3eHaK8Yy/Z5EUfOh3QIDAQABAoIBADugMMIKWcuTxYPX +c6iGZHEbxIPRTWyCcalB0nTQAAMGbabPAJ1l8432DZ+kWu806OybFXhPIfPOtVKy +0pFEWE8TtPE/V0vj3C5Qye2sBLFmBRwyCzXUdZV00wseMXRPs9dnTyalAR5KMnbI +j80kfpKSI2dkV9aU57UYBuq3Xrx/TCGItwL769D4ZZW9BvbpiTZApQQFZ0gwUFFn +btPXGU9Ti8H4mfBuZWL+5CaZdqOo76+CXvMPaUK0F9MJp4yX3XxQLRNH3qz/Tyn7 +h7QOOo0XTqoUmzRw0N9QRVH5LRdSE5yq3aF9aFKjNW59exz+62pufOFadngzkpkn +OKCzgWkCgYEA4mOWWMzdYwMn3GtfG7whqlqy7wOmMkNb81zTDQejHBV98dnj0AHr +deurfKWzHrAh3DXo6tFeqUIgXabhBPS/0dEx/S5sgLFmuUZP05EUYahfWBgzzmM9 +C6Oe5xIMLzxsZCJczolsfkEsoFe4o0vkvuLYoQrQL7InzewcDy8cUxsCgYEAy8Na +YCnanSNDY03Bulcni+5sF+opaHseeki1pv3nlw8TwsWuZF9ApS+yL7ck9jJjxBRR +RC3KGmpoqIr0vTmUYS946ngQWXPE90zfuhJfM+NRv/q0oCjH0qAcxRbTkls5On9v +oxJ8rO7gD6K85eHqasWdbCVzdZrobOXzay37tmcCgYBfyUUmw190cjReZauzH3Gb +E48b5A5gu/Fe0cqWe8G+szU7rDZgnz9SAGnpbm6QMHPTKZgoKngD42+wUFhq8Wdr +zjh5aDgOZ4EQKTjDSmI2Q7g7nNnmnESK9SrZl+BB6C3wXD2qQaj+7nKEUTlVFlpt +jaucz+dwFtASp7Djl8pDOwKBgEtr2c3ycArt/ImLRIP2spqm+7e2YvFbcSKOOz6+ +iLRvTj8v8KcSYtlB2FC1F6dRa4AujQ4RbNduP6LzHDfWUkfOzJDtNBAIPAXVnJJB +LqAEKkRHRghqT9x0i3GgS1vHDF3MwcO4mhFgserXr9ffUWeIEgbvrdcAKbv1Oa6Y +bK1NAoGAGPm8ISmboDJynjBl9wMrkcy23Pwg9kmyocdWUHh0zMLDKriZNKYB6u/U +C+/RTfkohPoHPzkeqWiHp7z3JhMItYUfTkNW6vMCxEGc0NEN6ZyMIjtiDPGN1n6O +E7jmODFmj1AQICQGdV5SHp+yKvKyb0YHKyDwETbs4SZBXxVvjEw= +-----END RSA PRIVATE KEY----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap3/config/export.ldif b/tests/testflows/ldap/external_user_directory/configs/ldap3/config/export.ldif new file mode 100644 index 00000000000..6ac9a995efd --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap3/config/export.ldif @@ -0,0 +1,64 @@ +# LDIF Export for dc=company,dc=com +# Server: openldap (openldap) +# Search Scope: sub +# Search Filter: (objectClass=*) +# Total Entries: 7 +# +# Generated by phpLDAPadmin (http://phpldapadmin.sourceforge.net) on May 22, 2020 5:51 pm +# Version: 1.2.5 + +# Entry 1: dc=company,dc=com +#dn: dc=company,dc=com +#dc: company +#o: company +#objectclass: top +#objectclass: dcObject +#objectclass: organization + +# Entry 2: cn=admin,dc=company,dc=com +#dn: cn=admin,dc=company,dc=com +#cn: admin +#description: LDAP administrator +#objectclass: simpleSecurityObject +#objectclass: organizationalRole +#userpassword: {SSHA}eUEupkQCTvq9SkrxfWGSe5rX+orrjVbF + +# Entry 3: ou=groups,dc=company,dc=com +dn: ou=groups,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: groups + +# Entry 4: cn=admin,ou=groups,dc=company,dc=com +dn: cn=admin,ou=groups,dc=company,dc=com +cn: admin +gidnumber: 500 +objectclass: posixGroup +objectclass: top + +# Entry 5: cn=users,ou=groups,dc=company,dc=com +dn: cn=users,ou=groups,dc=company,dc=com +cn: users +gidnumber: 501 +objectclass: posixGroup +objectclass: top + +# Entry 6: ou=users,dc=company,dc=com +dn: ou=users,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: users + +# Entry 7: cn=user3,ou=users,dc=company,dc=com +dn: cn=user3,ou=users,dc=company,dc=com +cn: user3 +gidnumber: 501 +givenname: John +homedirectory: /home/users/user3 +objectclass: inetOrgPerson +objectclass: posixAccount +objectclass: top +sn: User +uid: user3 +uidnumber: 1003 +userpassword: user3 diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ca.crt b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ca.crt new file mode 100644 index 00000000000..8c71e3afc91 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ca.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlTCCAn2gAwIBAgIUJBqw2dHM2DDCZjYSkPOESlvDH6swDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQHDAZPdHRhd2Ex +ETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTENMAsGA1UEAwwEcm9vdDAe +Fw0yMDA2MTExOTAzNDhaFw0zMDA2MDkxOTAzNDhaMFoxCzAJBgNVBAYTAkNBMQsw +CQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYDVQQKDAhBbHRpbml0eTEL +MAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC9Irr0zGV+HCI2fZ0ht4hR5It4Sbjz4RwZV8ENRP/+TEz8l9eK +J6ygxhKX7SMYzIs/jS9Gsq4plX1r2ujW1qRf8yLpR4+dGLP+jBRi1drj0XjZXosT +SERjWzgPauWxL9LN8+l26eBAqz6fw5e0W8WRSTgf5iGiCcKOTmaATIUjP0CdfWKK +qpktI4vhe++CXZFJ3usR+8KZ/FwwbCLJM/3J2HnbcXfcaYPYvr1tfqLudKSTbG9H +M3+AVwjctdesc/0sbd51Zsm0ClQptMbuKnDCYauGg61kNkgbgPgRmH9Pzo67DtxF +/WW+PtOzq8xLOifciQ9Piboy9QBSQZGwf4wzAgMBAAGjUzBRMB0GA1UdDgQWBBSi +njya0RDozx3OZTLYFpwqYnlpIDAfBgNVHSMEGDAWgBSinjya0RDozx3OZTLYFpwq +YnlpIDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBAD7VyFg7F +U1C25KFvtauchAOjCW6w7U/b3z1dVZvcQ88/kH1VsLUcfGixlSilUEfPTJsi7OA0 +R5BQdh2GGcjUJv4iqEFGU05KvMVmRRKn08P62+ZhJxKMxG26VzcliRZzCMkI6d0W +lFwI6nM45yeqdHVh5k4xbuJzqpbD9BtXXLI+/Ra9Fx8S9ETA3GdidpZLU5P1VLxq +UuedfqyAVWZXpr6TAURGxouRmRzul9yFzbSUex+MLEIPrstjtEwV3+tBQZJz9xAS +TVPj+Nv3LO7GCq54bdwkq1ioWbSL2hEmABkj6kdW/JwmfhGHf/2rirDVMzrTYw07 +dFJfAZC+FEsv +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/dhparam.pem b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/dhparam.pem new file mode 100644 index 00000000000..0a96faffd62 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/dhparam.pem @@ -0,0 +1,5 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAJitt2hhnpDViQ5ko2ipBMdjy+bZ6FR/WdZ987R7lQvBkKehPXmxtEyV +AO6ofv5CZSDJokc5bUeBOAtg0EhMTCH82uPdwQvt58jRXcxXBg4JTjkx+oW9LBv2 +FdZsbaX8+SYivmiZ0Jp8T/HBm/4DA9VBS0O5GFRS4C7dHhmSTPfDAgEC +-----END DH PARAMETERS----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.crt b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.crt new file mode 100644 index 00000000000..9167cbf861d --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQDCCAigCFCJ7El0ntrGktZVTYTZd+OwtcJjBMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAkNBMQswCQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYD +VQQKDAhBbHRpbml0eTELMAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwHhcNMjAw +NjExMTkxMTQzWhcNMzAwNjA5MTkxMTQzWjBfMQswCQYDVQQGEwJDQTELMAkGA1UE +CAwCT04xDzANBgNVBAcMBk90dGF3YTERMA8GA1UECgwIQWx0aW5pdHkxCzAJBgNV +BAsMAlFBMRIwEAYDVQQDDAlvcGVubGRhcDIwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC0Mbn//U56URavMgXm82FWP6vBdKuRydFX/L0M5XLlnAtk/IXG +/T+4t7nOBJxWmTp/xpsPtSMALE4eFJpEUEqlpVbG5DfBzVWcYOWoMeRAcHWCDkzr +PkB6I0dfF0Mm5hoaDhn+ZXjBWvoh/IlJdAnPg5mlejflJBQ7xtFC9eN6WjldXuRO +vyntGNuMfVLgITHwXuH2yZ98G0mFO6TU/9dRY/Z3D6RTSzKdb17Yk/VnG+ry92u2 +0sgXIBvhuJuC3ksWLArwwFoMl8DVa05D4O2H76goGdCcQ0KzqBV8RPXAh3UcgP2e +Zu90p2EGIhIk+sZTCkPd4dorxjL9nkRR86HdAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggEBAJWiCxJaTksv/BTsh/etxlDY5eHwqStqIuiovEQ8bhGAcKJ3bfWd/YTb8DUS +hrLvXrXdOVC+U8PqPFXBpdOqcm5Dc233z52VgUCb+0EKv3lAzgKXRIo32h52skdK +NnRrCHDeDzgfEIXR4MEJ99cLEaxWyXQhremmTYWHYznry9/4NYz40gCDxHn9dJAi +KxFyDNxhtuKs58zp4PrBoo+542JurAoLPtRGOhdXpU2RkQVU/ho38HsAXDStAB5D +vAoSxPuMHKgo17ffrb0oqU3didwaA9fIsz7Mr6RxmI7X03s7hLzNBq9FCqu0U3RR +CX4zWGFNJu/ieSGVWLYKQzbYxp8= +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.csr b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.csr new file mode 100644 index 00000000000..bf569f727d6 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.csr @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICpDCCAYwCAQAwXzELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQH +DAZPdHRhd2ExETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTESMBAGA1UE +AwwJb3BlbmxkYXAyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtDG5 +//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyFxv0/uLe5zgScVpk6f8ab +D7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M6z5AeiNHXxdDJuYaGg4Z +/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7kTr8p7RjbjH1S4CEx8F7h +9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdrttLIFyAb4bibgt5LFiwK +8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9nmbvdKdhBiISJPrGUwpD +3eHaK8Yy/Z5EUfOh3QIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAEzIjZQOT5R7 +mEJg+RFpCSIoPn3xJ4/VMMyWqA3bTGZKpb4S6GxgsierY/87kPL7jZrMdGYB4Dc3 +2M3VWZGXlYo8vctH1zLE9VW6CzosUpl20lhdgydoCMz3RQqdJyK8aGeFTeLtk7G/ +TRCCUFUE6jaA+VtaCPCnOJSff3jUf76xguEu7dgTZgCKV7dtBqald8gIzF3D+AJJ +7pEN2UrC3UR0xpe2cj2GhndQJ+WsIyft3zpNFzAO13j8ZPibuVP7oDWcW3ixNCWC +213aeRVplJGof8Eo6llDxP+6Fwp1YmOoQmwB1Xm3t4ADn7FLJ14LONLB7q40KviG +RyLyqu3IVOI= +-----END CERTIFICATE REQUEST----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.key b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.key new file mode 100644 index 00000000000..5ab3a3f8b59 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap4/certs/ldap.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAtDG5//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyF +xv0/uLe5zgScVpk6f8abD7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M +6z5AeiNHXxdDJuYaGg4Z/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7k +Tr8p7RjbjH1S4CEx8F7h9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdr +ttLIFyAb4bibgt5LFiwK8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9 +nmbvdKdhBiISJPrGUwpD3eHaK8Yy/Z5EUfOh3QIDAQABAoIBADugMMIKWcuTxYPX +c6iGZHEbxIPRTWyCcalB0nTQAAMGbabPAJ1l8432DZ+kWu806OybFXhPIfPOtVKy +0pFEWE8TtPE/V0vj3C5Qye2sBLFmBRwyCzXUdZV00wseMXRPs9dnTyalAR5KMnbI +j80kfpKSI2dkV9aU57UYBuq3Xrx/TCGItwL769D4ZZW9BvbpiTZApQQFZ0gwUFFn +btPXGU9Ti8H4mfBuZWL+5CaZdqOo76+CXvMPaUK0F9MJp4yX3XxQLRNH3qz/Tyn7 +h7QOOo0XTqoUmzRw0N9QRVH5LRdSE5yq3aF9aFKjNW59exz+62pufOFadngzkpkn +OKCzgWkCgYEA4mOWWMzdYwMn3GtfG7whqlqy7wOmMkNb81zTDQejHBV98dnj0AHr +deurfKWzHrAh3DXo6tFeqUIgXabhBPS/0dEx/S5sgLFmuUZP05EUYahfWBgzzmM9 +C6Oe5xIMLzxsZCJczolsfkEsoFe4o0vkvuLYoQrQL7InzewcDy8cUxsCgYEAy8Na +YCnanSNDY03Bulcni+5sF+opaHseeki1pv3nlw8TwsWuZF9ApS+yL7ck9jJjxBRR +RC3KGmpoqIr0vTmUYS946ngQWXPE90zfuhJfM+NRv/q0oCjH0qAcxRbTkls5On9v +oxJ8rO7gD6K85eHqasWdbCVzdZrobOXzay37tmcCgYBfyUUmw190cjReZauzH3Gb +E48b5A5gu/Fe0cqWe8G+szU7rDZgnz9SAGnpbm6QMHPTKZgoKngD42+wUFhq8Wdr +zjh5aDgOZ4EQKTjDSmI2Q7g7nNnmnESK9SrZl+BB6C3wXD2qQaj+7nKEUTlVFlpt +jaucz+dwFtASp7Djl8pDOwKBgEtr2c3ycArt/ImLRIP2spqm+7e2YvFbcSKOOz6+ +iLRvTj8v8KcSYtlB2FC1F6dRa4AujQ4RbNduP6LzHDfWUkfOzJDtNBAIPAXVnJJB +LqAEKkRHRghqT9x0i3GgS1vHDF3MwcO4mhFgserXr9ffUWeIEgbvrdcAKbv1Oa6Y +bK1NAoGAGPm8ISmboDJynjBl9wMrkcy23Pwg9kmyocdWUHh0zMLDKriZNKYB6u/U +C+/RTfkohPoHPzkeqWiHp7z3JhMItYUfTkNW6vMCxEGc0NEN6ZyMIjtiDPGN1n6O +E7jmODFmj1AQICQGdV5SHp+yKvKyb0YHKyDwETbs4SZBXxVvjEw= +-----END RSA PRIVATE KEY----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap4/config/export.ldif b/tests/testflows/ldap/external_user_directory/configs/ldap4/config/export.ldif new file mode 100644 index 00000000000..36afdb4e350 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap4/config/export.ldif @@ -0,0 +1,64 @@ +# LDIF Export for dc=company,dc=com +# Server: openldap (openldap) +# Search Scope: sub +# Search Filter: (objectClass=*) +# Total Entries: 7 +# +# Generated by phpLDAPadmin (http://phpldapadmin.sourceforge.net) on May 22, 2020 5:51 pm +# Version: 1.2.5 + +# Entry 1: dc=company,dc=com +#dn: dc=company,dc=com +#dc: company +#o: company +#objectclass: top +#objectclass: dcObject +#objectclass: organization + +# Entry 2: cn=admin,dc=company,dc=com +#dn: cn=admin,dc=company,dc=com +#cn: admin +#description: LDAP administrator +#objectclass: simpleSecurityObject +#objectclass: organizationalRole +#userpassword: {SSHA}eUEupkQCTvq9SkrxfWGSe5rX+orrjVbF + +# Entry 3: ou=groups,dc=company,dc=com +dn: ou=groups,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: groups + +# Entry 4: cn=admin,ou=groups,dc=company,dc=com +dn: cn=admin,ou=groups,dc=company,dc=com +cn: admin +gidnumber: 500 +objectclass: posixGroup +objectclass: top + +# Entry 5: cn=users,ou=groups,dc=company,dc=com +dn: cn=users,ou=groups,dc=company,dc=com +cn: users +gidnumber: 501 +objectclass: posixGroup +objectclass: top + +# Entry 6: ou=users,dc=company,dc=com +dn: ou=users,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: users + +# Entry 7: cn=user4,ou=users,dc=company,dc=com +dn: cn=user4,ou=users,dc=company,dc=com +cn: user4 +gidnumber: 501 +givenname: John +homedirectory: /home/users/user4 +objectclass: inetOrgPerson +objectclass: posixAccount +objectclass: top +sn: User +uid: user4 +uidnumber: 1004 +userpassword: user4 diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap5/config/export.ldif b/tests/testflows/ldap/external_user_directory/configs/ldap5/config/export.ldif new file mode 100644 index 00000000000..bc3d2ff75fc --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap5/config/export.ldif @@ -0,0 +1,64 @@ +# LDIF Export for dc=company,dc=com +# Server: openldap (openldap) +# Search Scope: sub +# Search Filter: (objectClass=*) +# Total Entries: 7 +# +# Generated by phpLDAPadmin (http://phpldapadmin.sourceforge.net) on May 22, 2020 5:51 pm +# Version: 1.2.5 + +# Entry 1: dc=company,dc=com +#dn: dc=company,dc=com +#dc: company +#o: company +#objectclass: top +#objectclass: dcObject +#objectclass: organization + +# Entry 2: cn=admin,dc=company,dc=com +#dn: cn=admin,dc=company,dc=com +#cn: admin +#description: LDAP administrator +#objectclass: simpleSecurityObject +#objectclass: organizationalRole +#userpassword: {SSHA}eUEupkQCTvq9SkrxfWGSe5rX+orrjVbF + +# Entry 3: ou=groups,dc=company,dc=com +dn: ou=groups,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: groups + +# Entry 4: cn=admin,ou=groups,dc=company,dc=com +dn: cn=admin,ou=groups,dc=company,dc=com +cn: admin +gidnumber: 500 +objectclass: posixGroup +objectclass: top + +# Entry 5: cn=users,ou=groups,dc=company,dc=com +dn: cn=users,ou=groups,dc=company,dc=com +cn: users +gidnumber: 501 +objectclass: posixGroup +objectclass: top + +# Entry 6: ou=users,dc=company,dc=com +dn: ou=users,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: users + +# Entry 7: cn=user5,ou=users,dc=company,dc=com +dn: cn=user5,ou=users,dc=company,dc=com +cn: user5 +gidnumber: 501 +givenname: John +homedirectory: /home/users/user5 +objectclass: inetOrgPerson +objectclass: posixAccount +objectclass: top +sn: User +uid: user5 +uidnumber: 1005 +userpassword: user5 diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ca.crt b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ca.crt new file mode 100644 index 00000000000..8c71e3afc91 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ca.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlTCCAn2gAwIBAgIUJBqw2dHM2DDCZjYSkPOESlvDH6swDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQHDAZPdHRhd2Ex +ETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTENMAsGA1UEAwwEcm9vdDAe +Fw0yMDA2MTExOTAzNDhaFw0zMDA2MDkxOTAzNDhaMFoxCzAJBgNVBAYTAkNBMQsw +CQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYDVQQKDAhBbHRpbml0eTEL +MAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC9Irr0zGV+HCI2fZ0ht4hR5It4Sbjz4RwZV8ENRP/+TEz8l9eK +J6ygxhKX7SMYzIs/jS9Gsq4plX1r2ujW1qRf8yLpR4+dGLP+jBRi1drj0XjZXosT +SERjWzgPauWxL9LN8+l26eBAqz6fw5e0W8WRSTgf5iGiCcKOTmaATIUjP0CdfWKK +qpktI4vhe++CXZFJ3usR+8KZ/FwwbCLJM/3J2HnbcXfcaYPYvr1tfqLudKSTbG9H +M3+AVwjctdesc/0sbd51Zsm0ClQptMbuKnDCYauGg61kNkgbgPgRmH9Pzo67DtxF +/WW+PtOzq8xLOifciQ9Piboy9QBSQZGwf4wzAgMBAAGjUzBRMB0GA1UdDgQWBBSi +njya0RDozx3OZTLYFpwqYnlpIDAfBgNVHSMEGDAWgBSinjya0RDozx3OZTLYFpwq +YnlpIDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBAD7VyFg7F +U1C25KFvtauchAOjCW6w7U/b3z1dVZvcQ88/kH1VsLUcfGixlSilUEfPTJsi7OA0 +R5BQdh2GGcjUJv4iqEFGU05KvMVmRRKn08P62+ZhJxKMxG26VzcliRZzCMkI6d0W +lFwI6nM45yeqdHVh5k4xbuJzqpbD9BtXXLI+/Ra9Fx8S9ETA3GdidpZLU5P1VLxq +UuedfqyAVWZXpr6TAURGxouRmRzul9yFzbSUex+MLEIPrstjtEwV3+tBQZJz9xAS +TVPj+Nv3LO7GCq54bdwkq1ioWbSL2hEmABkj6kdW/JwmfhGHf/2rirDVMzrTYw07 +dFJfAZC+FEsv +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/dhparam.pem b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/dhparam.pem new file mode 100644 index 00000000000..0a96faffd62 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/dhparam.pem @@ -0,0 +1,5 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAJitt2hhnpDViQ5ko2ipBMdjy+bZ6FR/WdZ987R7lQvBkKehPXmxtEyV +AO6ofv5CZSDJokc5bUeBOAtg0EhMTCH82uPdwQvt58jRXcxXBg4JTjkx+oW9LBv2 +FdZsbaX8+SYivmiZ0Jp8T/HBm/4DA9VBS0O5GFRS4C7dHhmSTPfDAgEC +-----END DH PARAMETERS----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.crt b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.crt new file mode 100644 index 00000000000..9167cbf861d --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDQDCCAigCFCJ7El0ntrGktZVTYTZd+OwtcJjBMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAkNBMQswCQYDVQQIDAJPTjEPMA0GA1UEBwwGT3R0YXdhMREwDwYD +VQQKDAhBbHRpbml0eTELMAkGA1UECwwCUUExDTALBgNVBAMMBHJvb3QwHhcNMjAw +NjExMTkxMTQzWhcNMzAwNjA5MTkxMTQzWjBfMQswCQYDVQQGEwJDQTELMAkGA1UE +CAwCT04xDzANBgNVBAcMBk90dGF3YTERMA8GA1UECgwIQWx0aW5pdHkxCzAJBgNV +BAsMAlFBMRIwEAYDVQQDDAlvcGVubGRhcDIwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQC0Mbn//U56URavMgXm82FWP6vBdKuRydFX/L0M5XLlnAtk/IXG +/T+4t7nOBJxWmTp/xpsPtSMALE4eFJpEUEqlpVbG5DfBzVWcYOWoMeRAcHWCDkzr +PkB6I0dfF0Mm5hoaDhn+ZXjBWvoh/IlJdAnPg5mlejflJBQ7xtFC9eN6WjldXuRO +vyntGNuMfVLgITHwXuH2yZ98G0mFO6TU/9dRY/Z3D6RTSzKdb17Yk/VnG+ry92u2 +0sgXIBvhuJuC3ksWLArwwFoMl8DVa05D4O2H76goGdCcQ0KzqBV8RPXAh3UcgP2e +Zu90p2EGIhIk+sZTCkPd4dorxjL9nkRR86HdAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggEBAJWiCxJaTksv/BTsh/etxlDY5eHwqStqIuiovEQ8bhGAcKJ3bfWd/YTb8DUS +hrLvXrXdOVC+U8PqPFXBpdOqcm5Dc233z52VgUCb+0EKv3lAzgKXRIo32h52skdK +NnRrCHDeDzgfEIXR4MEJ99cLEaxWyXQhremmTYWHYznry9/4NYz40gCDxHn9dJAi +KxFyDNxhtuKs58zp4PrBoo+542JurAoLPtRGOhdXpU2RkQVU/ho38HsAXDStAB5D +vAoSxPuMHKgo17ffrb0oqU3didwaA9fIsz7Mr6RxmI7X03s7hLzNBq9FCqu0U3RR +CX4zWGFNJu/ieSGVWLYKQzbYxp8= +-----END CERTIFICATE----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.csr b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.csr new file mode 100644 index 00000000000..bf569f727d6 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.csr @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICpDCCAYwCAQAwXzELMAkGA1UEBhMCQ0ExCzAJBgNVBAgMAk9OMQ8wDQYDVQQH +DAZPdHRhd2ExETAPBgNVBAoMCEFsdGluaXR5MQswCQYDVQQLDAJRQTESMBAGA1UE +AwwJb3BlbmxkYXAyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtDG5 +//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyFxv0/uLe5zgScVpk6f8ab +D7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M6z5AeiNHXxdDJuYaGg4Z +/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7kTr8p7RjbjH1S4CEx8F7h +9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdrttLIFyAb4bibgt5LFiwK +8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9nmbvdKdhBiISJPrGUwpD +3eHaK8Yy/Z5EUfOh3QIDAQABoAAwDQYJKoZIhvcNAQELBQADggEBAEzIjZQOT5R7 +mEJg+RFpCSIoPn3xJ4/VMMyWqA3bTGZKpb4S6GxgsierY/87kPL7jZrMdGYB4Dc3 +2M3VWZGXlYo8vctH1zLE9VW6CzosUpl20lhdgydoCMz3RQqdJyK8aGeFTeLtk7G/ +TRCCUFUE6jaA+VtaCPCnOJSff3jUf76xguEu7dgTZgCKV7dtBqald8gIzF3D+AJJ +7pEN2UrC3UR0xpe2cj2GhndQJ+WsIyft3zpNFzAO13j8ZPibuVP7oDWcW3ixNCWC +213aeRVplJGof8Eo6llDxP+6Fwp1YmOoQmwB1Xm3t4ADn7FLJ14LONLB7q40KviG +RyLyqu3IVOI= +-----END CERTIFICATE REQUEST----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.key b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.key new file mode 100644 index 00000000000..5ab3a3f8b59 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/certs/ldap.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAtDG5//1OelEWrzIF5vNhVj+rwXSrkcnRV/y9DOVy5ZwLZPyF +xv0/uLe5zgScVpk6f8abD7UjACxOHhSaRFBKpaVWxuQ3wc1VnGDlqDHkQHB1gg5M +6z5AeiNHXxdDJuYaGg4Z/mV4wVr6IfyJSXQJz4OZpXo35SQUO8bRQvXjelo5XV7k +Tr8p7RjbjH1S4CEx8F7h9smffBtJhTuk1P/XUWP2dw+kU0synW9e2JP1Zxvq8vdr +ttLIFyAb4bibgt5LFiwK8MBaDJfA1WtOQ+Dth++oKBnQnENCs6gVfET1wId1HID9 +nmbvdKdhBiISJPrGUwpD3eHaK8Yy/Z5EUfOh3QIDAQABAoIBADugMMIKWcuTxYPX +c6iGZHEbxIPRTWyCcalB0nTQAAMGbabPAJ1l8432DZ+kWu806OybFXhPIfPOtVKy +0pFEWE8TtPE/V0vj3C5Qye2sBLFmBRwyCzXUdZV00wseMXRPs9dnTyalAR5KMnbI +j80kfpKSI2dkV9aU57UYBuq3Xrx/TCGItwL769D4ZZW9BvbpiTZApQQFZ0gwUFFn +btPXGU9Ti8H4mfBuZWL+5CaZdqOo76+CXvMPaUK0F9MJp4yX3XxQLRNH3qz/Tyn7 +h7QOOo0XTqoUmzRw0N9QRVH5LRdSE5yq3aF9aFKjNW59exz+62pufOFadngzkpkn +OKCzgWkCgYEA4mOWWMzdYwMn3GtfG7whqlqy7wOmMkNb81zTDQejHBV98dnj0AHr +deurfKWzHrAh3DXo6tFeqUIgXabhBPS/0dEx/S5sgLFmuUZP05EUYahfWBgzzmM9 +C6Oe5xIMLzxsZCJczolsfkEsoFe4o0vkvuLYoQrQL7InzewcDy8cUxsCgYEAy8Na +YCnanSNDY03Bulcni+5sF+opaHseeki1pv3nlw8TwsWuZF9ApS+yL7ck9jJjxBRR +RC3KGmpoqIr0vTmUYS946ngQWXPE90zfuhJfM+NRv/q0oCjH0qAcxRbTkls5On9v +oxJ8rO7gD6K85eHqasWdbCVzdZrobOXzay37tmcCgYBfyUUmw190cjReZauzH3Gb +E48b5A5gu/Fe0cqWe8G+szU7rDZgnz9SAGnpbm6QMHPTKZgoKngD42+wUFhq8Wdr +zjh5aDgOZ4EQKTjDSmI2Q7g7nNnmnESK9SrZl+BB6C3wXD2qQaj+7nKEUTlVFlpt +jaucz+dwFtASp7Djl8pDOwKBgEtr2c3ycArt/ImLRIP2spqm+7e2YvFbcSKOOz6+ +iLRvTj8v8KcSYtlB2FC1F6dRa4AujQ4RbNduP6LzHDfWUkfOzJDtNBAIPAXVnJJB +LqAEKkRHRghqT9x0i3GgS1vHDF3MwcO4mhFgserXr9ffUWeIEgbvrdcAKbv1Oa6Y +bK1NAoGAGPm8ISmboDJynjBl9wMrkcy23Pwg9kmyocdWUHh0zMLDKriZNKYB6u/U +C+/RTfkohPoHPzkeqWiHp7z3JhMItYUfTkNW6vMCxEGc0NEN6ZyMIjtiDPGN1n6O +E7jmODFmj1AQICQGdV5SHp+yKvKyb0YHKyDwETbs4SZBXxVvjEw= +-----END RSA PRIVATE KEY----- diff --git a/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/config/export.ldif b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/config/export.ldif new file mode 100644 index 00000000000..c6470176a5e --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/configs/ldap5/ldap2/config/export.ldif @@ -0,0 +1,64 @@ +# LDIF Export for dc=company,dc=com +# Server: openldap (openldap) +# Search Scope: sub +# Search Filter: (objectClass=*) +# Total Entries: 7 +# +# Generated by phpLDAPadmin (http://phpldapadmin.sourceforge.net) on May 22, 2020 5:51 pm +# Version: 1.2.5 + +# Entry 1: dc=company,dc=com +#dn: dc=company,dc=com +#dc: company +#o: company +#objectclass: top +#objectclass: dcObject +#objectclass: organization + +# Entry 2: cn=admin,dc=company,dc=com +#dn: cn=admin,dc=company,dc=com +#cn: admin +#description: LDAP administrator +#objectclass: simpleSecurityObject +#objectclass: organizationalRole +#userpassword: {SSHA}eUEupkQCTvq9SkrxfWGSe5rX+orrjVbF + +# Entry 3: ou=groups,dc=company,dc=com +dn: ou=groups,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: groups + +# Entry 4: cn=admin,ou=groups,dc=company,dc=com +dn: cn=admin,ou=groups,dc=company,dc=com +cn: admin +gidnumber: 500 +objectclass: posixGroup +objectclass: top + +# Entry 5: cn=users,ou=groups,dc=company,dc=com +dn: cn=users,ou=groups,dc=company,dc=com +cn: users +gidnumber: 501 +objectclass: posixGroup +objectclass: top + +# Entry 6: ou=users,dc=company,dc=com +dn: ou=users,dc=company,dc=com +objectclass: organizationalUnit +objectclass: top +ou: users + +# Entry 7: cn=user1,ou=users,dc=company,dc=com +dn: cn=user1,ou=users,dc=company,dc=com +cn: user1 +gidnumber: 501 +givenname: John1 +homedirectory: /home/users/user1 +objectclass: inetOrgPerson +objectclass: posixAccount +objectclass: top +sn: User1 +uid: user1 +uidnumber: 1001 +userpassword: user1 diff --git a/tests/testflows/ldap/external_user_directory/docker-compose/clickhouse-service.yml b/tests/testflows/ldap/external_user_directory/docker-compose/clickhouse-service.yml new file mode 100644 index 00000000000..2a56876c72e --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/docker-compose/clickhouse-service.yml @@ -0,0 +1,28 @@ +version: '2.3' + +services: + clickhouse: + image: yandex/clickhouse-integration-test + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.d/:/etc/clickhouse-server/users.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + healthcheck: + test: clickhouse client --query='select 1' + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/ldap/external_user_directory/docker-compose/docker-compose.yml b/tests/testflows/ldap/external_user_directory/docker-compose/docker-compose.yml new file mode 100644 index 00000000000..c8ff683df58 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/docker-compose/docker-compose.yml @@ -0,0 +1,162 @@ +version: '2.3' + +services: + openldap1: + # plain text + extends: + file: openldap-service.yml + service: openldap + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap1/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap2: + # TLS - never + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap2/certs:/container/service/slapd/assets/certs/" + + openldap3: + # plain text - custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + environment: + LDAP_PORT: "3089" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap3/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + + openldap4: + # TLS - never custom port + extends: + file: openldap-service.yml + service: openldap + expose: + - "3089" + - "6036" + environment: + LDAP_PORT: "3089" + LDAPS_PORT: "6036" + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "never" + LDAP_TLS_CIPHER_SUITE: "SECURE256:+SECURE128:-VERS-TLS-ALL:+VERS-TLS1.2:-RSA:-DHE-DSS:-CAMELLIA-128-CBC:-CAMELLIA-256-CBC" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap4/certs:/container/service/slapd/assets/certs/" + + openldap5: + # TLS - try + extends: + file: openldap-service.yml + service: openldap + environment: + LDAP_TLS: "true" + LDAP_TLS_CRT_FILENAME: "ldap.crt" + LDAP_TLS_KEY_FILENAME: "ldap.key" + LDAP_TLS_DH_PARAM_FILENAME: "dhparam.pem" + LDAP_TLS_CA_CRT_FILENAME: "ca.crt" + LDAP_TLS_ENFORCE: "false" + LDAP_TLS_VERIFY_CLIENT: "try" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/config:/container/service/slapd/assets/config/bootstrap/ldif/custom" + - "${CLICKHOUSE_TESTS_DIR}/configs/ldap5/certs:/container/service/slapd/assets/certs/" + + phpldapadmin: + extends: + file: openldap-service.yml + service: phpldapadmin + environment: + PHPLDAPADMIN_LDAP_HOSTS: "openldap1" + depends_on: + openldap1: + condition: service_healthy + + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy + openldap1: + condition: service_healthy + openldap2: + condition: service_healthy + openldap3: + condition: service_healthy + openldap4: + condition: service_healthy + openldap5: + condition: service_healthy + phpldapadmin: + condition: service_healthy diff --git a/tests/testflows/ldap/external_user_directory/docker-compose/openldap-service.yml b/tests/testflows/ldap/external_user_directory/docker-compose/openldap-service.yml new file mode 100644 index 00000000000..139907c513c --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/docker-compose/openldap-service.yml @@ -0,0 +1,40 @@ +version: '2.3' + +services: + openldap: + image: osixia/openldap:1.4.0 + command: "--copy-service --loglevel debug" + environment: + LDAP_ORGANIZATION: "company" + LDAP_DOMAIN: "company.com" + LDAP_ADMIN_PASSWORD: "admin" + LDAP_TLS: "false" + expose: + - "389" + - "636" + healthcheck: + test: ldapsearch -x -H ldap://localhost:$${LDAP_PORT:-389} -b "dc=company,dc=com" -D "cn=admin,dc=company,dc=com" -w admin + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable + + + phpldapadmin: + image: osixia/phpldapadmin:0.9.0 + container_name: phpldapadmin + environment: + PHPLDAPADMIN_HTTPS=false: + ports: + - "8080:80" + healthcheck: + test: echo 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable + diff --git a/tests/testflows/ldap/external_user_directory/docker-compose/zookeeper-service.yml b/tests/testflows/ldap/external_user_directory/docker-compose/zookeeper-service.yml new file mode 100644 index 00000000000..6691a2df31c --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/docker-compose/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/ldap/external_user_directory/regression.py b/tests/testflows/ldap/external_user_directory/regression.py new file mode 100755 index 00000000000..0082e37d98e --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/regression.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +import sys +from testflows.core import * + +append_path(sys.path, "..", "..") + +from helpers.cluster import Cluster +from helpers.argparser import argparser +from ldap.external_user_directory.requirements import * + +# Cross-outs of known fails +xfails = { + "connection protocols/tls/tls_require_cert='try'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/tls/tls_require_cert='demand'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/starttls/tls_require_cert='try'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/starttls/tls_require_cert='demand'": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/tls require cert default demand": + [(Fail, "can't be tested with self-signed certificates")], + "connection protocols/starttls with custom port": + [(Fail, "it seems that starttls is not enabled by default on custom plain-text ports in LDAP server")], + "connection protocols/tls cipher suite": + [(Fail, "can't get it to work")] +} + +@TestFeature +@Name("external user directory") +@ArgumentParser(argparser) +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication("1.0") +) +@XFails(xfails) +def regression(self, local, clickhouse_binary_path): + """ClickHouse LDAP external user directory regression module. + """ + nodes = { + "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), + } + + with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: + self.context.cluster = cluster + + Scenario(run=load("ldap.authentication.tests.sanity", "scenario")) + Scenario(run=load("ldap.external_user_directory.tests.simple", "scenario")) + Feature(run=load("ldap.external_user_directory.tests.server_config", "feature")) + Feature(run=load("ldap.external_user_directory.tests.external_user_directory_config", "feature")) + Feature(run=load("ldap.external_user_directory.tests.connections", "feature")) + Feature(run=load("ldap.external_user_directory.tests.authentications", "feature")) + Feature(run=load("ldap.external_user_directory.tests.roles", "feature")) + +if main(): + regression() diff --git a/tests/testflows/ldap/external_user_directory/requirements/__init__.py b/tests/testflows/ldap/external_user_directory/requirements/__init__.py new file mode 100644 index 00000000000..02f7d430154 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/requirements/__init__.py @@ -0,0 +1 @@ +from .requirements import * diff --git a/tests/testflows/ldap/external_user_directory/requirements/requirements.py b/tests/testflows/ldap/external_user_directory/requirements/requirements.py new file mode 100644 index 00000000000..7e3ced037fa --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/requirements/requirements.py @@ -0,0 +1,1133 @@ +# These requirements were auto generated +# from software requirements specification (SRS) +# document by TestFlows v1.6.200827.1211600. +# Do not edit by hand but re-generate instead +# using 'tfs requirements generate' command. +from testflows.core import Requirement + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authenticating users that are defined only on the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Authentication_NewUsers = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Authentication.NewUsers', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authenticating users that are defined only on the [LDAP] server\n' + 'as soon as they are added to the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_DeletedUsers = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.DeletedUsers', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not allow authentication of users that\n' + 'were previously defined only on the [LDAP] server but were removed\n' + 'from the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Valid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if\n' + 'user name and password match [LDAP] server records for the user\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication if either user name or password\n' + 'do not match [LDAP] server records for the user\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_UsernameChanged = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.UsernameChanged', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication if the username is changed\n' + 'on the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_PasswordChanged = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.PasswordChanged', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication if the password\n' + 'for the user is changed on the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_LDAPServerRestart = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.LDAPServerRestart', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_ClickHouseServerRestart = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.ClickHouseServerRestart', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authenticating users after server is restarted\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support parallel authentication of users using [LDAP] server\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.ValidAndInvalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authentication of valid users and\n' + 'prohibit authentication of invalid users using [LDAP] server\n' + 'in parallel without having invalid attempts affecting valid authentications\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_PlainText = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.PlainText', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol\n' + 'while connecting to the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol\n' + 'while connecting to the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_StartTLS = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.StartTLS', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a\n' + 'plain text `ldap://` protocol that is upgraded to [TLS] when connecting to the [LDAP] server\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS_Certificate_Validation = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.Validation', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support certificate validation used for [TLS] connections\n' + 'to the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS_Certificate_SelfSigned = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SelfSigned', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support self-signed certificates for [TLS] connections\n' + 'to the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS_Certificate_SpecificCertificationAuthority = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SpecificCertificationAuthority', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections\n' + 'to the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_Mechanism_Anonymous = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Anonymous', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind]\n' + 'authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_Mechanism_Unauthenticated = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Unauthenticated', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind]\n' + 'authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_Mechanism_NamePassword = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.NamePassword', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind]\n' + 'authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_UnreachableServer = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.UnreachableServer', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Lookup_Priority = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Lookup.Priority', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL lookup user presence in the following priority:\n' + '\n' + '1. access control\n' + '2. `users.xml`\n' + '3. LDAP\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Removed = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reject authentication attempt if any of the roles that are specified in the configuration\n' + 'of the external user directory are not defined at the time of the authentication attempt\n' + 'with an exception that if a user was able to authenticate in past and its internal user object was created and cached\n' + 'then the user SHALL be able to authenticate again, even if one of the roles is missing.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Removed_Privileges = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed.Privileges', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL remove the privileges provided by the role from all the LDAP\n' + 'users authenticated using external user directory if it is removed\n' + 'including currently cached users that are still able to authenticated where the removed\n' + 'role is specified in the configuration of the external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Readded_Privileges = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Readded.Privileges', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reassign the role and add the privileges provided by the role\n' + 'when it is re-added after removal for all LDAP users authenticated using external user directory\n' + 'including any cached users where the re-added role was specified in the configuration of the external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Role_New = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.New', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not allow any new roles to be assigned to any LDAP\n' + 'users authenticated using external user directory unless the role is specified\n' + 'in the configuration of the external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Role_NewPrivilege = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NewPrivilege', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL add new privilege to all the LDAP users authenticated using external user directory\n' + 'including cached users when new privilege is added to one of the roles specified\n' + 'in the configuration of the external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Role_RemovedPrivilege = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Role.RemovedPrivilege', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL remove privilege from all the LDAP users authenticated using external user directory\n' + 'including cached users when privilege is removed from all the roles specified\n' + 'in the configuration of the external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Definition = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Definition', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support using the [LDAP] servers defined in the\n' + '`ldap_servers` section of the `config.xml` as the server to be used\n' + 'for a external user directory that uses an [LDAP] server as a source of user definitions.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Name = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Name', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support empty string as a server name.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Host = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Host', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify [LDAP]\n' + 'server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify [LDAP] server port.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port_Default = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Prefix = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Prefix', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify the prefix\n' + 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Suffix = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Suffix', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify the suffix\n' + 'of value used to construct the DN to bound to during authentication via [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Value = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Value', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string.\n' + '\n' + "> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character.\n" + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_Default = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use `yes` value as the default for `` parameter\n' + 'to enable SSL/TLS `ldaps://` protocol.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_No = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.No', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable\n' + 'plain text `ldap://` protocol.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_Yes = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Yes', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable\n' + 'SSL/TLS `ldaps://` protocol.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_StartTLS = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.StartTLS', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable\n' + 'legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS].\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify\n' + 'the minimum protocol version of SSL/TLS.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion_Values = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Values', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2`\n' + 'as a value of the `` parameter.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion_Default = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify [TLS] peer\n' + 'certificate verification behavior.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Default = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Default', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL use `demand` value as the default for the `` parameter.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Demand = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Demand', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to\n' + 'enable requesting of client certificate. If no certificate is provided, or a bad certificate is\n' + 'provided, the session SHALL be immediately terminated.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Allow = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Allow', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to\n' + 'enable requesting of client certificate. If no\n' + 'certificate is provided, the session SHALL proceed normally.\n' + 'If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Try = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Try', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `try` as the value of `` parameter to\n' + 'enable requesting of client certificate. If no certificate is provided, the session\n' + 'SHALL proceed normally. If a bad certificate is provided, the session SHALL be\n' + 'immediately terminated.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Never = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Never', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support specifying `never` as the value of `` parameter to\n' + 'disable requesting of client certificate.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCertFile = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCertFile', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` to specify the path to certificate file used by\n' + '[ClickHouse] to establish connection with the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSKeyFile = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSKeyFile', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` to specify the path to key file for the certificate\n' + 'specified by the `` parameter.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCACertDir = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertDir', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify to a path to\n' + 'the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCACertFile = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertFile', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` parameter to specify a path to a specific\n' + '[CA] certificate file used to verify certificates provided by the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCipherSuite = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCipherSuite', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites.\n' + 'The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers].\n' + '\n' + 'For example,\n' + '\n' + '```xml\n' + 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n' + '```\n' + '\n' + 'The available suites SHALL depend on the [OpenSSL] library version and variant used to build\n' + '[ClickHouse] and therefore might change.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Syntax = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml`\n' + 'configuration file or of any configuration file inside the `config.d` directory.\n' + '\n' + '```xml\n' + '\n' + ' \n' + ' localhost\n' + ' 636\n' + ' cn=\n' + ' , ou=users, dc=example, dc=com\n' + ' yes\n' + ' tls1.2\n' + ' demand\n' + ' /path/to/tls_cert_file\n' + ' /path/to/tls_key_file\n' + ' /path/to/tls_ca_cert_file\n' + ' /path/to/tls_ca_cert_dir\n' + ' ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384\n' + ' \n' + '\n' + '```\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` sub-section in the `` section of the `config.xml`\n' + 'that SHALL define a external user directory that uses an [LDAP] server as a source of user definitions.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory_MoreThanOne = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL only use the first `` sub-section in the `` section of the `config.xml`\n' + 'if more than one `` sub-sections are present.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Syntax = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `` section with the following syntax\n' + '\n' + '```xml\n' + '\n' + ' \n' + ' \n' + ' my_ldap_server\n' + ' \n' + ' \n' + ' \n' + ' \n' + ' \n' + ' \n' + '\n' + '```\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `server` parameter in the `` sub-section in the ``\n' + 'section of the `config.xml` that SHALL specify one of LDAP server names\n' + 'defined in `` section.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Empty = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `server` parameter in the `` sub-section in the ``\n' + 'is empty.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Missing = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the `server` parameter in the `` sub-section in the ``\n' + 'is missing.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_MoreThanOne = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL only use the first definitition of the `server` parameter in the `` sub-section in the ``\n' + 'if more than one `server` parameter is defined in the configuration.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Invalid = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the server specified as the value of the ``\n' + 'parameter is not defined.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support `roles` parameter in the `` sub-section in the ``\n' + 'section of the `config.xml` that SHALL specify the names of a locally defined roles that SHALL\n' + 'be assigned to all users retrieved from the [LDAP] server.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_MoreThanOne = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL only use the first definitition of the `roles` parameter\n' + 'in the `` sub-section in the ``\n' + 'if more than one `roles` parameter is defined in the configuration.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Invalid = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL return an error if the role specified in the ``\n' + 'parameter does not exist locally.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Empty = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not allow users authenticated using LDAP external user directory\n' + 'to perform any action if the `roles` parameter in the `` sub-section in the ``\n' + 'section is empty.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Missing = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not allow users authenticated using LDAP external user directory\n' + 'to perform any action if the `roles` parameter in the `` sub-section in the ``\n' + 'section is missing.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_Empty = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Empty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support authenticating users with empty username\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_Long = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Long', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_UTF8 = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.UTF8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters\n' + 'when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Empty = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Empty', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not support authenticating users with empty passwords\n' + 'even if an empty password is valid for the user and\n' + 'is allowed by the [LDAP] server when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Long = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Long', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support long password of at least 256 bytes\n' + 'that can be used to authenticate users when using [LDAP] external user directory.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_UTF8 = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.UTF8', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support [UTF-8] characters in passwords\n' + 'used to authenticate users when using [LDAP] external user directory.\n' + ), + link=None + ) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py new file mode 100644 index 00000000000..76e55996f21 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -0,0 +1,509 @@ +# -*- coding: utf-8 -*- +import random + +from multiprocessing.dummy import Pool +from testflows.core import * +from testflows.asserts import error + +from ldap.external_user_directory.tests.common import * +from ldap.external_user_directory.requirements import * + +servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + "openldap2": { + "host": "openldap2", + "port": "636", + "enable_tls": "yes", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "never", + } +} + +@TestStep(When) +@Name("I login as {username} and execute query") +def login_and_execute_query(self, username, password, exitcode=None, message=None, steps=True): + self.context.node.query("SELECT 1", + settings=[("user", username), ("password", password)], + exitcode=exitcode or 0, + message=message, steps=steps) + +@TestOutline +def add_user_to_ldap_and_login(self, server, user=None, ch_user=None, login=None, exitcode=None, message=None): + """Add user to LDAP and ClickHouse and then try to login.""" + self.context.ldap_node = self.context.cluster.node(server) + + if ch_user is None: + ch_user = {} + if login is None: + login = {} + if user is None: + user = {"cn": "myuser", "userpassword": "myuser"} + + with ldap_user(**user) as user: + username = login.get("username", user["cn"]) + password = login.get("password", user["userpassword"]) + + login_and_execute_query(username=username, password=password, exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") +) +def parallel_login(self, server, user_count=10, timeout=200): + """Check that login of valid and invalid LDAP authenticated users works in parallel.""" + self.context.ldap_node = self.context.cluster.node(server) + user = None + + users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] + + with ldap_users(*users): + def login_with_valid_username_and_password(users, i, iterations=10): + with When(f"valid users try to login #{i}"): + for i in range(iterations): + random_user = users[random.randint(0, len(users)-1)] + login_and_execute_query(username=random_user["cn"], password=random_user["userpassword"], steps=False) + + def login_with_valid_username_and_invalid_password(users, i, iterations=10): + with When(f"users try to login with valid username and invalid password #{i}"): + for i in range(iterations): + random_user = users[random.randint(0, len(users)-1)] + login_and_execute_query(username=random_user["cn"], + password=(random_user["userpassword"] + randomword(1)), + exitcode=4, + message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", + steps=False) + + def login_with_invalid_username_and_valid_password(users, i, iterations=10): + with When(f"users try to login with invalid username and valid password #{i}"): + for i in range(iterations): + random_user = dict(users[random.randint(0, len(users)-1)]) + random_user["cn"] += randomword(1) + login_and_execute_query(username=random_user["cn"], + password=random_user["userpassword"], + exitcode=4, + message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", + steps=False) + + with When("I login in parallel"): + p = Pool(15) + tasks = [] + for i in range(5): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + + with Then("it should work"): + for task in tasks: + task.get(timeout=timeout) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Authentication_NewUsers("1.0") +) +def login_after_user_is_added_to_ldap(self, server): + """Check that user can login as soon as it is added to LDAP.""" + user = {"cn": "myuser", "userpassword": "myuser"} + + with When(f"I add user to LDAP and try to login"): + add_user_to_ldap_and_login(user=user, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_DeletedUsers("1.0") +) +def login_after_user_is_deleted_from_ldap(self, server): + """Check that login fails after user is deleted from LDAP.""" + self.context.ldap_node = self.context.cluster.node(server) + user = None + + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) + + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with When("I delete this user from LDAP"): + delete_user_from_ldap(user) + + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_PasswordChanged("1.0") +) +def login_after_user_password_changed_in_ldap(self, server): + """Check that login fails after user password is changed in LDAP.""" + self.context.ldap_node = self.context.cluster.node(server) + user = None + + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) + + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with When("I change user password in LDAP"): + change_user_password_in_ldap(user, "newpassword") + + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) + + with And("when I try to login with the new password it should work"): + login_and_execute_query(username=user["cn"], password="newpassword") + + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_UsernameChanged("1.0") +) +def login_after_user_cn_changed_in_ldap(self, server): + """Check that login fails after user cn is changed in LDAP.""" + self.context.ldap_node = self.context.cluster.node(server) + user = None + new_user = None + + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": "myuser"} + user = add_user_to_ldap(**user) + + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with When("I change user password in LDAP"): + new_user = change_user_cn_in_ldap(user, "myuser2") + + with Then("when I try to login again it should fail"): + login_and_execute_query(username=user["cn"], password=user["userpassword"], + exitcode=4, + message=f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + ) + finally: + with Finally("I make sure LDAP user is deleted"): + if new_user is not None: + delete_user_from_ldap(new_user, exitcode=None) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_LDAPServerRestart("1.0") +) +def login_after_ldap_server_is_restarted(self, server, timeout=60): + """Check that login succeeds after LDAP server is restarted.""" + self.context.ldap_node = self.context.cluster.node(server) + user = None + + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": getuid()} + user = add_user_to_ldap(**user) + + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with When("I restart LDAP server"): + self.context.ldap_node.restart() + + with Then("I try to login until it works", description=f"timeout {timeout} sec"): + started = time.time() + while True: + r = self.context.node.query("SELECT 1", + settings=[("user", user["cn"]), ("password", user["userpassword"])], + no_checks=True) + if r.exitcode == 0: + break + assert time.time() - started < timeout, error(r.output) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_ClickHouseServerRestart("1.0") +) +def login_after_clickhouse_server_is_restarted(self, server, timeout=60): + """Check that login succeeds after ClickHouse server is restarted.""" + self.context.ldap_node = self.context.cluster.node(server) + user = None + + try: + with Given(f"I add user to LDAP"): + user = {"cn": "myuser", "userpassword": getuid()} + user = add_user_to_ldap(**user) + + login_and_execute_query(username=user["cn"], password=user["userpassword"]) + + with When("I restart ClickHouse server"): + self.context.node.restart() + + with Then("I try to login until it works", description=f"timeout {timeout} sec"): + started = time.time() + while True: + r = self.context.node.query("SELECT 1", + settings=[("user", user["cn"]), ("password", user["userpassword"])], + no_checks=True) + if r.exitcode == 0: + break + assert time.time() - started < timeout, error(r.output) + finally: + with Finally("I make sure LDAP user is deleted"): + if user is not None: + delete_user_from_ldap(user, exitcode=None) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Empty("1.0") +) +def valid_username_with_valid_empty_password(self, server): + """Check that we can't login using valid username that has empty password.""" + user = {"cn": "empty_password", "userpassword": ""} + exitcode = 4 + message = f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, exitcode=exitcode, message=message, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Empty("1.0") +) +def valid_username_and_invalid_empty_password(self, server): + """Check that we can't login using valid username but invalid empty password.""" + username = "user_non_empty_password" + user = {"cn": username, "userpassword": username} + login = {"password": ""} + + exitcode = 4 + message = f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0") +) +def valid_username_and_password(self, server): + """Check that we can login using valid username and password.""" + username = "valid_username_and_password" + user = {"cn": username, "userpassword": username} + + with When(f"I add user {username} to LDAP and try to login"): + add_user_to_ldap_and_login(user=user, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0") +) +def valid_username_and_password_invalid_server(self, server=None): + """Check that we can't login using valid username and valid + password but for a different server.""" + self.context.ldap_node = self.context.cluster.node("openldap1") + + user = {"username": "user2", "userpassword": "user2", "server": "openldap1"} + + exitcode = 4 + message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" + + login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_Long("1.0"), +) +def valid_long_username_and_short_password(self, server): + """Check that we can login using valid very long username and short password.""" + username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890" + user = {"cn": username, "userpassword": "long_username"} + + add_user_to_ldap_and_login(user=user, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0") +) +def invalid_long_username_and_valid_short_password(self, server): + """Check that we can't login using slightly invalid long username but valid password.""" + username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890" + user = {"cn": username, "userpassword": "long_username"} + login = {"username": f"{username}?"} + + exitcode = 4 + message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Long("1.0") +) +def valid_short_username_and_long_password(self, server): + """Check that we can login using valid short username with very long password.""" + username = "long_password" + user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"} + add_user_to_ldap_and_login(user=user, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0") +) +def valid_short_username_and_invalid_long_password(self, server): + """Check that we can't login using valid short username and invalid long password.""" + username = "long_password" + user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"} + login = {"password": user["userpassword"] + "1"} + + exitcode = 4 + message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0") +) +def valid_username_and_invalid_password(self, server): + """Check that we can't login using valid username and invalid password.""" + username = "valid_username_and_invalid_password" + user = {"cn": username, "userpassword": username} + login = {"password": user["userpassword"] + "1"} + + exitcode = 4 + message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0") +) +def invalid_username_and_valid_password(self, server): + """Check that we can't login using slightly invalid username but valid password.""" + username = "invalid_username_and_valid_password" + user = {"cn": username, "userpassword": username} + login = {"username": user["cn"] + "1"} + + exitcode = 4 + message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name" + + add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_UTF8("1.0") +) +def valid_utf8_username_and_ascii_password(self, server): + """Check that we can login using valid utf-8 username with ascii password.""" + username = "utf8_username_Gãńdåłf_Thê_Gręât" + user = {"cn": username, "userpassword": "utf8_username"} + + add_user_to_ldap_and_login(user=user, server=server) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_UTF8("1.0") +) +def valid_ascii_username_and_utf8_password(self, server): + """Check that we can login using valid ascii username with utf-8 password.""" + username = "utf8_password" + user = {"cn": username, "userpassword": "utf8_password_Gãńdåłf_Thê_Gręât"} + + add_user_to_ldap_and_login(user=user, server=server) + +@TestScenario +def empty_username_and_empty_password(self, server=None): + """Check that we can login using empty username and empty password as + it will use the default user and that has an empty password.""" + login_and_execute_query(username="", password="") + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Lookup_Priority("1.0") +) +def user_lookup_priority(self, server): + """Check that users are looked up in the same priority + as they are defined in the `` section + of the `config.xml`. For this test we have the following priority list + as defined by the configuration files: + + * users.xml + * local directory + * LDAP external user directory + """ + self.context.ldap_node = self.context.cluster.node(server) + + message="DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name" + exitcode = 4 + + users = { + "default": {"username": "default", "password": "userdefault"}, + "local": {"username": "local", "password": "userlocal"}, + "ldap": {"username": "ldap", "password": "userldap"} + } + + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users.values()]): + with rbac_users("local"): + with When("I try to login as 'default' user which is also defined in users.xml it should fail"): + login_and_execute_query(**users["default"], exitcode=exitcode, message=message.format(username="default")) + + with When("I try to login as 'local' user which is also defined in local storage it should fail"): + login_and_execute_query(**users["local"], exitcode=exitcode, message=message.format(username="local")) + + with When("I try to login as 'ldap' user defined only in LDAP it should work"): + login_and_execute_query(**users["ldap"]) + + +@TestOutline(Feature) +@Name("user authentications") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_Mechanism_NamePassword("1.0") +) +def feature(self, servers=None, server=None, node="clickhouse1"): + """Check that users can be authenticated using an LDAP external user directory. + """ + self.context.node = self.context.cluster.node(node) + + if servers is None: + servers = globals()["servers"] + + if server is None: + server = "openldap1" + + with ldap_servers(servers): + with rbac_roles("ldap_role") as roles: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + for scenario in loads(current_module(), Scenario): + scenario(server=server) diff --git a/tests/testflows/ldap/external_user_directory/tests/common.py b/tests/testflows/ldap/external_user_directory/tests/common.py new file mode 100644 index 00000000000..4c7877035c8 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/tests/common.py @@ -0,0 +1,181 @@ +import os +import time +from contextlib import contextmanager + +import testflows.settings as settings +from testflows.core import * +from testflows.asserts import error +from ldap.authentication.tests.common import getuid, Config, ldap_servers, add_config +from ldap.authentication.tests.common import xmltree, xml_indent, xml_append, xml_with_utf8 +from ldap.authentication.tests.common import ldap_user, ldap_users, add_user_to_ldap, delete_user_from_ldap +from ldap.authentication.tests.common import change_user_password_in_ldap, change_user_cn_in_ldap +from ldap.authentication.tests.common import randomword + +@contextmanager +def table(name, create_statement, on_cluster=False): + node = current().context.node + try: + with Given(f"I have a {name} table"): + node.query(create_statement.format(name=name)) + yield name + finally: + with Finally("I drop the table"): + if on_cluster: + node.query(f"DROP TABLE IF EXISTS {name} ON CLUSTER {on_cluster}") + else: + node.query(f"DROP TABLE IF EXISTS {name}") + +@contextmanager +def rbac_users(*users): + node = current().context.node + try: + with Given("I have local users"): + for user in users: + with By(f"creating user {user}"): + node.query(f"CREATE USER OR REPLACE {user} IDENTIFIED WITH PLAINTEXT_PASSWORD BY '{user}'") + yield users + finally: + with Finally("I drop local users"): + for user in users: + with By(f"dropping user {user}", flags=TE): + node.query(f"DROP USER IF EXISTS {user}") + +@contextmanager +def rbac_roles(*roles): + node = current().context.node + try: + with Given("I have roles"): + for role in roles: + with By(f"creating role {role}"): + node.query(f"CREATE ROLE OR REPLACE {role}") + yield roles + finally: + with Finally("I drop the roles"): + for role in roles: + with By(f"dropping role {role}", flags=TE): + node.query(f"DROP ROLE IF EXISTS {role}") + +def create_ldap_external_user_directory_config_content(server=None, roles=None, **kwargs): + """Create LDAP external user directory configuration file content. + """ + return create_entries_ldap_external_user_directory_config_content(entries=[([server], [roles])], **kwargs) + +def create_entries_ldap_external_user_directory_config_content(entries, config_d_dir="/etc/clickhouse-server/config.d", + config_file="ldap_external_user_directories.xml"): + """Create configurattion file content that contains + one or more entries for the LDAP external user directory. + + For example, + + ```xml + + + my_ldap_server + my_user + + + ``` + """ + uid = getuid() + path = os.path.join(config_d_dir, config_file) + name = config_file + + root = xmltree.fromstring("") + xml_user_directories = root.find("user_directories") + xml_user_directories.append(xmltree.Comment(text=f"LDAP external user directories {uid}")) + + for entry in entries: + servers, roles_entries = entry + xml_directory = xmltree.Element("ldap") + for server in servers: + if server is not None: + xml_append(xml_directory, "server", server) + if roles_entries: + for roles_entry in roles_entries: + xml_roles = xmltree.Element("roles") + if roles_entry: + for role in roles_entry: + if role is not None: + xml_append(xml_roles, role, "") + xml_directory.append(xml_roles) + xml_user_directories.append(xml_directory) + + xml_indent(root) + content = xml_with_utf8 + str(xmltree.tostring(root, short_empty_elements=False, encoding="utf-8"), "utf-8") + + return Config(content, path, name, uid, "config.xml") + +def invalid_ldap_external_user_directory_config(server, roles, message, tail=20, timeout=20, config=None): + """Check that ClickHouse errors when trying to load invalid LDAP external user directory + configuration file. + """ + cluster = current().context.cluster + node = current().context.node + + if config is None: + config = create_ldap_external_user_directory_config_content(server=server, roles=roles) + + try: + with Given("I prepare the error log by writting empty lines into it"): + node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("-\\n" * tail)) + + with When("I add the config", description=config.path): + command = f"cat < {config.path}\n{config.content}\nHEREDOC" + node.command(command, steps=False, exitcode=0) + + with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): + started = time.time() + command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + + with When("I restart ClickHouse to apply the config changes"): + node.restart(safe=False, wait_healthy=False) + + finally: + with Finally(f"I remove {config.name}"): + with By("removing invalid configuration file"): + system_config_path = os.path.join(current_dir(), "..", "configs", node.name, "config.d", config.path.split("config.d/")[-1]) + cluster.command(None, f'rm -rf {system_config_path}', timeout=timeout, exitcode=0) + + with And("restarting the node"): + node.restart(safe=False) + + with Then("error log should contain the expected error message"): + started = time.time() + command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + +@contextmanager +def ldap_external_user_directory(server, roles, config_d_dir="/etc/clickhouse-server/config.d", + config_file=None, timeout=20, restart=True, config=None): + """Add LDAP external user directory. + """ + if config_file is None: + config_file = f"ldap_external_user_directory_{getuid()}.xml" + if config is None: + config = create_ldap_external_user_directory_config_content(server=server, roles=roles, config_d_dir=config_d_dir, config_file=config_file) + return add_config(config, restart=restart) + +def login(servers, directory_server, *users, config=None): + """Configure LDAP server and LDAP external user directory and + try to login and execute a query""" + with ldap_servers(servers): + with rbac_roles(f"role_{getuid()}") as roles: + with ldap_external_user_directory(server=servers[directory_server]["host"], roles=roles, restart=True, config=config): + for user in users: + if user.get("login", False): + with When(f"I login as {user['username']} and execute query"): + current().context.node.query("SELECT 1", + settings=[("user", user["username"]), ("password", user["password"])], + exitcode=user.get("exitcode", None), + message=user.get("message", None)) diff --git a/tests/testflows/ldap/external_user_directory/tests/connections.py b/tests/testflows/ldap/external_user_directory/tests/connections.py new file mode 100644 index 00000000000..ba734bb6c71 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/tests/connections.py @@ -0,0 +1,270 @@ +from testflows.core import * +from testflows.asserts import error + +from ldap.external_user_directory.tests.common import login +from ldap.external_user_directory.requirements import * + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_PlainText("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_No("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port_Default("1.0") +) +def plain_text(self): + """Check that we can perform LDAP user authentication using `plain text` connection protocol. + """ + servers = { + "openldap1": { + "host": "openldap1", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + ] + login(servers, "openldap1", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_PlainText("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port("1.0") +) +def plain_text_with_custom_port(self): + """Check that we can perform LDAP user authentication using `plain text` connection protocol + with the server that uses custom port. + """ + servers = { + "openldap3": { + "host": "openldap3", + "port": "3089", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap3", "username": "user3", "password": "user3", "login": True} + ] + login(servers, "openldap3", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port("1.0") +) +def tls_with_custom_port(self): + """Check that we can perform LDAP user authentication using `TLS` connection protocol + with the server that uses custom port. + """ + servers = { + "openldap4": { + "host": "openldap4", + "port": "6036", + "tls_require_cert": "never", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap4", "username": "user4", "password": "user4", "login": True} + ] + login(servers, "openldap4", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_StartTLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port("1.0") +) +def starttls_with_custom_port(self): + """Check that we can perform LDAP user authentication using `StartTLS` connection protocol + with the server that uses custom port. + """ + servers = { + "openldap4": { + "host": "openldap4", + "port": "3089", + "enable_tls": "starttls", + "tls_require_cert": "never", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap4", "username": "user4", "password": "user4", "login": True} + ] + login(servers, "openldap4", *users) + +def tls_connection(enable_tls, tls_require_cert): + """Try to login using LDAP user authentication over a TLS connection.""" + servers = { + "openldap2": { + "host": "openldap2", + "enable_tls": enable_tls, + "tls_require_cert": tls_require_cert, + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap2", "username": "user2", "password": "user2", "login": True} + ] + + requirements = [] + + if tls_require_cert == "never": + requirements = [RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Never("1.0")] + elif tls_require_cert == "allow": + requirements = [RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Allow("1.0")] + elif tls_require_cert == "try": + requirements = [RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Try("1.0")] + elif tls_require_cert == "demand": + requirements = [RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Demand("1.0")] + + with Example(name=f"tls_require_cert='{tls_require_cert}'", requirements=requirements): + login(servers, "openldap2", *users) + +@TestScenario +@Examples("enable_tls tls_require_cert", [ + ("yes", "never"), + ("yes", "allow"), + ("yes", "try"), + ("yes", "demand") +]) +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_TLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_Yes("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port_Default("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion_Default("1.0") +) +def tls(self): + """Check that we can perform LDAP user authentication using `TLS` connection protocol. + """ + for example in self.examples: + tls_connection(*example) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_Default("1.0") +) +def tls_enable_tls_default_yes(self): + """Check that the default value for the `enable_tls` is set to `yes`.""" + servers = { + "openldap2": { + "host": "openldap2", + "tls_require_cert": "never", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap2", "username": "user2", "password": "user2", "login": True} + ] + login(servers, "openldap2", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert_Options_Default("1.0") +) +def tls_require_cert_default_demand(self): + """Check that the default value for the `tls_require_cert` is set to `demand`.""" + servers = { + "openldap2": { + "host": "openldap2", + "enable_tls": "yes", + "port": "636", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap2", "username": "user2", "password": "user2", "login": True} + ] + login(servers, "openldap2", *users) + +@TestScenario +@Examples("enable_tls tls_require_cert", [ + ("starttls", "never"), + ("starttls", "allow"), + ("starttls", "try"), + ("starttls", "demand") +]) +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_StartTLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS_Options_StartTLS("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Port_Default("1.0") +) +def starttls(self): + """Check that we can perform LDAP user authentication using legacy `StartTLS` connection protocol. + """ + for example in self.examples: + tls_connection(*example) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCipherSuite("1.0") +) +def tls_cipher_suite(self): + """Check that `tls_cipher_suite` parameter can be used specify allowed cipher suites.""" + servers = { + "openldap4": { + "host": "openldap4", + "port": "6036", + "tls_require_cert": "never", + "tls_cipher_suite": "SECURE256:+SECURE128:-VERS-TLS-ALL:+VERS-TLS1.2:-RSA:-DHE-DSS:-CAMELLIA-128-CBC:-CAMELLIA-256-CBC", + "tls_minimum_protocol_version": "tls1.2", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + users = [ + {"server": "openldap4", "username": "user4", "password": "user4", "login": True} + ] + login(servers, "openldap4", *users) + +@TestOutline(Scenario) +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSMinimumProtocolVersion_Values("1.0") +) +@Examples("version exitcode message", [ + ("ssl2", None, None), + ("ssl3", None, None), + ("tls1.0", None, None), + ("tls1.1", None, None), + ("tls1.2", None, None) +]) +def tls_minimum_protocol_version(self, version, exitcode, message): + """Check that `tls_minimum_protocol_version` parameter can be used specify + to specify the minimum protocol version of SSL/TLS.""" + + servers = { + "openldap4": { + "host": "openldap4", + "port": "6036", + "tls_require_cert": "never", + "tls_minimum_protocol_version": version, + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + } + } + + users = [{ + "server": "openldap4", "username": "user4", "password": "user4", + "login": True, "exitcode": int(exitcode) if exitcode is not None else None, "message": message + }] + + login(servers,"openldap4", *users) + +@TestFeature +@Name("connection protocols") +def feature(self, node="clickhouse1"): + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + scenario() diff --git a/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py b/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py new file mode 100644 index 00000000000..d29d0124dc2 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py @@ -0,0 +1,283 @@ +from testflows.core import * + +from ldap.external_user_directory.tests.common import * +from ldap.external_user_directory.requirements import * + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory_MoreThanOne("1.0") +) +def more_than_one_user_directory(self, timeout=20): + """Check when more than one LDAP user directory is + defined inside a configuration file. + """ + message = "DB::Exception: Duplicate storage type 'ldap' at user_directories" + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + "openldap2": { + "host": "openldap2", "port": "636", "enable_tls": "yes", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "never" + } + } + users = [ + {"server": "openldap1", "username": "user1", "password": "user1", "login": True}, + {"server": "openldap2", "username": "user2", "password": "user2", "login": True} + ] + role = f"role_{getuid()}" + entries = [ + (["openldap1"], [(role,)]), + (["openldap2"], [(role,)]) + ] + + with ldap_servers(servers): + with rbac_roles(role) as roles: + config = create_entries_ldap_external_user_directory_config_content(entries) + invalid_ldap_external_user_directory_config(server=None, roles=None, message=message, timeout=timeout, config=config) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Empty("1.0") +) +def empty_server(self, timeout=20): + """Check that empty string in a `server` field is not allowed. + """ + message = "DB::Exception: Empty 'server' field for LDAP user directory" + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + + with ldap_servers(servers): + with rbac_roles(f"role_{getuid()}") as roles: + invalid_ldap_external_user_directory_config(server="", roles=roles, message=message, timeout=timeout) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Missing("1.0") +) +def missing_server(self, timeout=20): + """Check that missing `server` field is not allowed. + """ + message = "DB::Exception: Missing 'server' field for LDAP user directory" + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + + with ldap_servers(servers): + with rbac_roles(f"role_{getuid()}") as roles: + invalid_ldap_external_user_directory_config(server=None, roles=roles, message=message, timeout=timeout) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_MoreThanOne("1.0") +) +def defined_twice_server(self, timeout=20): + """Check that when `server` field is defined twice that only the first + entry is used. + """ + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + + role = f"role_{getuid()}" + entries = [ + (["openldap1", "openldap2"], [(role,)]) + ] + + with ldap_servers(servers): + with rbac_roles(role) as roles: + config = create_entries_ldap_external_user_directory_config_content(entries) + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with When(f"I login as {user['username']} and execute query"): + current().context.node.query("SELECT 1", + settings=[("user", user["username"]), ("password", user["password"])]) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server_Invalid("1.0") +) +def invalid_server(self, timeout=20): + """Check when `server` field value is invalid. + """ + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + role = f"role_{getuid()}" + + entries = [ + (["openldap2"], [(role,)]) + ] + + with ldap_servers(servers): + with rbac_roles(role) as roles: + config = create_entries_ldap_external_user_directory_config_content(entries) + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with When(f"I login as {user['username']} and execute query"): + current().context.node.query("SELECT 1", + settings=[("user", user["username"]), ("password", user["password"])], + exitcode=36, message="DB::Exception: LDAP server 'openldap2' is not configured") + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Empty("1.0") +) +def empty_roles(self, timeout=20): + """Check when `roles` parameter is empty then user can't read any tables. + """ + message = "DB::Exception: user1: Not enough privileges." + exitcode = 241 + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + user = {"server": "openldap1", "username": "user1", "password": "user1"} + + entries = [ + (["openldap1"], [[]]) + ] + + with ldap_servers(servers): + with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + config = create_entries_ldap_external_user_directory_config_content(entries) + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with When(f"I login as {user['username']} and execute query"): + current().context.node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", user["username"]), ("password", user["password"])], + exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_MoreThanOne("1.0") +) +def defined_twice_roles(self, timeout=20): + """Check that when `roles` is defined twice then only the first entry is used. + """ + node = self.context.node + + create_statement = "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()" + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + roles = [f"role0_{getuid()}", f"role1_{getuid()}"] + entries = [ + (["openldap1"], [[roles[0]],[roles[1]]]) + ] + + with ldap_servers(servers): + with rbac_roles(*roles): + with table(f"table0_{getuid()}", create_statement) as table0_name, \ + table(f"table1_{getuid()}", create_statement) as table1_name: + + with Given("I grant select privilege for the first table to the first role"): + node.query(f"GRANT SELECT ON {table0_name} TO {roles[0]}") + + with And("I grant select privilege for the second table to the second role"): + node.query(f"GRANT SELECT ON {table1_name} TO {roles[1]}") + + config = create_entries_ldap_external_user_directory_config_content(entries) + + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with When(f"I login as {user['username']} and try to read from the first table"): + current().context.node.query(f"SELECT * FROM {table0_name} LIMIT 1", + settings=[("user", user["username"]), ("password", user["password"])]) + + with And(f"I login as {user['username']} again and try to read from the second table"): + current().context.node.query(f"SELECT * FROM {table0_name} LIMIT 1", + settings=[("user", user["username"]), ("password", user["password"])]) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Invalid("1.0") +) +def invalid_role_in_roles(self, timeout=20): + """Check that an error is returned when LDAP users try to authenticate + if an invalid role is specified inside the `roles` section. + """ + exitcode = 4 + message = "DB::Exception: user1: Authentication failed" + + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + user = {"server": "openldap1", "username": "user1", "password": "user1"} + + with ldap_servers(servers): + with ldap_external_user_directory("openldap1", roles=["foo"], restart=True): + with When(f"I login as {user['username']} and execute query"): + current().context.node.query("SELECT 1", + settings=[("user", user["username"]), ("password", user["password"])], + exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles_Missing("1.0") +) +def missing_roles(self, timeout=20): + """Check that when the `roles` are missing then + LDAP users can still login but can't read from any table. + """ + message = "DB::Exception: user1: Not enough privileges." + exitcode = 241 + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + user = {"server": "openldap1", "username": "user1", "password": "user1"} + entries = [ + (["openldap1"], None) + ] + + with ldap_servers(servers): + with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + + config = create_entries_ldap_external_user_directory_config_content(entries) + + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with When(f"I login as {user['username']} and execute query"): + current().context.node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", user["username"]), ("password", user["password"])], + exitcode=exitcode, message=message) + +@TestFeature +@Name("external user directory config") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Syntax("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Server("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Definition("1.0") +) +def feature(self, node="clickhouse1"): + """Check LDAP external user directory configuration. + """ + self.context.node = self.context.cluster.node(node) + + for scenario in loads(current_module(), Scenario): + scenario() diff --git a/tests/testflows/ldap/external_user_directory/tests/roles.py b/tests/testflows/ldap/external_user_directory/tests/roles.py new file mode 100644 index 00000000000..d621993078b --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/tests/roles.py @@ -0,0 +1,314 @@ +from testflows.core import * + +from ldap.external_user_directory.tests.common import * +from ldap.external_user_directory.requirements import * + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Role_New("1.0") +) +def new_role(self, server, timeout=20): + """Check that new roles can't be assigned to any LDAP user + authenticated using external user directory. + """ + node = self.context.node + uid = getuid() + + self.context.ldap_node = self.context.cluster.node(server) + + users = [ + {"username": f"user0_{uid}", "password": "user0_password"}, + {"username": f"user1_{uid}", "password": "user1_password"} + ] + + with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: + with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + + with When(f"I login and execute query simple query to cache the LDAP user"): + node.query(f"SELECT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with rbac_roles(f"new_role0_{uid}") as new_roles: + + message = "DB::Exception: Cannot update user `{user}` in ldap because this storage is readonly" + exitcode = 239 + + with And("I try to grant new role to the cached LDAP user"): + node.query(f"GRANT {new_roles[0]} TO {users[0]['username']}", + exitcode=exitcode, message=message.format(user=users[0]["username"])) + + message = "DB::Exception: There is no role `{user}` in user directories" + exitcode = 255 + + with And("I try to grant new role to the non-cached LDAP user"): + node.query(f"GRANT {new_roles[0]} TO {users[1]['username']}", + exitcode=exitcode, message=message.format(user=users[1]["username"])) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Role_NewPrivilege("1.0") +) +def add_privilege(self, server, timeout=20): + """Check that we can add privilege to a role used + in the external user directory configuration. + """ + node = self.context.node + uid = getuid() + message = "DB::Exception: {user}: Not enough privileges." + exitcode = 241 + + self.context.ldap_node = self.context.cluster.node(server) + + users = [ + {"username": f"user0_{uid}", "password": "user0_password"}, + {"username": f"user1_{uid}", "password": "user1_password"} + ] + + with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: + with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + + with When(f"I login and execute query that requires no privileges"): + node.query(f"SELECT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And(f"I login and try to read from the table without having select privilege"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])], + exitcode=exitcode, message=message.format(user=users[0]["username"])) + + with When(f"I grant select privilege to one of the two roles assigned to LDAP users"): + node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") + + with And(f"I login again and expect that cached LDAP user can successfully read from the table"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And(f"I login again and expect that non-cached LDAP user can successfully read from the table"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[1]["username"]), ("password", users[1]["password"])]) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Role_RemovedPrivilege("1.0") +) +def remove_privilege(self, server, timeout=20): + """Check that we can remove privilege from a role used + in the external user directory configuration. + """ + node = self.context.node + uid = getuid() + message = "DB::Exception: {user}: Not enough privileges." + exitcode = 241 + + self.context.ldap_node = self.context.cluster.node(server) + + users = [ + {"username": f"user0_{uid}", "password": "user0_password"}, + {"username": f"user1_{uid}", "password": "user1_password"} + ] + + with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: + with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + + with When(f"I grant select privilege to one of the two roles assigned to LDAP users"): + node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") + + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + + with When(f"I login then LDAP user should be able to read from the table"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with When(f"I revoke select privilege from all the roles assigned to LDAP users"): + node.query(f"REVOKE SELECT ON {table_name} FROM {roles[0]}") + + with When(f"I login again then cached LDAP user should not be able to read from the table"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])], + exitcode=exitcode, message=message.format(user=users[0]["username"])) + + with When(f"I login with non-cached LDAP user then the user should also not be able to read from the table"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[1]["username"]), ("password", users[1]["password"])], + exitcode=exitcode, message=message.format(user=users[1]["username"])) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Removed("1.0") +) +def remove_role(self, server, timeout=20): + """Check that when a role used in the external user directory configuration + is dynamically removed then any non-cached LDAP users should not be authenticated using + LDAP external user directory. + """ + node = self.context.node + uid = getuid() + exitcode = 4 + message = "DB::Exception: {user}: Authentication failed: password is incorrect or there is no user with such name" + + self.context.ldap_node = self.context.cluster.node(server) + + users = [ + {"username": f"user0_{uid}", "password": "user0_password"}, + {"username": f"user1_{uid}", "password": "user1_password"} + ] + + with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + with When(f"I login and execute query that requires no privileges"): + node.query(f"SELECT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And("I remove one of the roles"): + node.query(f"DROP ROLE IF EXISTS {roles[1]}") + + with And(f"I try to login using cached LDAP user"): + node.query(f"SELECT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And(f"I try to login again using non-cached LDAP user"): + node.query(f"SELECT 1", + settings=[("user", users[1]["username"]), ("password", users[1]["password"])], + exitcode=exitcode, message=message.format(user=users[1]["username"])) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Removed_Privileges("1.0") +) +def remove_privilege_by_removing_role(self, server, timeout=20): + """Check that when the role used in the external user directory configuration + is dynamically removed then privileges are removed from all + LDAP users that are authenticated using external user directory. + """ + node = self.context.node + message = "DB::Exception: {user}: Not enough privileges." + exitcode = 241 + uid = getuid() + + self.context.ldap_node = self.context.cluster.node(server) + + users = [ + {"username": f"user0_{uid}", "password": "user0_password"}, + {"username": f"user1_{uid}", "password": "user1_password"} + ] + + with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: + with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + + with When(f"I grant select privilege to one of the two roles assigned to LDAP users"): + node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") + + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + + with When(f"I login and expect that LDAP user can read from the table"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And("I remove the role that grants the privilege"): + node.query(f"DROP ROLE IF EXISTS {roles[0]}") + + with And(f"I try to relogin and expect that cached LDAP user can login " + "but does not have privilege that was provided by the removed role"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])], + exitcode=exitcode, message=message.format(user=users[0]["username"])) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Role_Readded_Privileges("1.0") +) +def readd_privilege_by_readding_role(self, server, timeout=20): + """Check that when the role used in the external user directory configuration + is dynamically removed then all the privileges are removed from any + LDAP users authenticated using external user directory but when the role is re-added + then privileges are restored and non-cached users can login again. + """ + node = self.context.node + uid = getuid() + + self.context.ldap_node = self.context.cluster.node(server) + + users = [ + {"username": f"user0_{uid}", "password": "user0_password"}, + {"username": f"user1_{uid}", "password": "user1_password"} + ] + + with rbac_roles(f"role0_{uid}", f"role1_{uid}") as roles: + with table(f"table_{getuid()}", "CREATE TABLE {name} (d DATE, s String, i UInt8) ENGINE = Memory()") as table_name: + + with When(f"I grant select privilege to one of the two roles assigned to LDAP users"): + node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") + + with ldap_external_user_directory(server=server, roles=roles, restart=True): + with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users]): + + with When(f"I login and expect that LDAP user can read from the table"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And("I remove the role that grants the privilege"): + node.query(f"DROP ROLE IF EXISTS {roles[0]}") + + message = "DB::Exception: {user}: Not enough privileges." + exitcode = 241 + + with And(f"I try to relogin and expect that cached LDAP user can login " + "but does not have privilege that was provided by the removed role"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])], + exitcode=exitcode, message=message.format(user=users[0]["username"])) + + message = "DB::Exception: {user}: Authentication failed: password is incorrect or there is no user with such name" + exitcode = 4 + + with And(f"I try to login using non-cached LDAP user and expect it to fail"): + node.query(f"SELECT 1", + settings=[("user", users[1]["username"]), ("password", users[1]["password"])], + exitcode=exitcode, message=message.format(user=users[1]["username"])) + + with When("I re-add the role"): + node.query(f"CREATE ROLE IF NOT EXISTS {roles[0]}") + + with And(f"I grant select privilege to the re-added role"): + node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") + + with And(f"I try to relogin and expect that cached LDAP user can login " + "and again has the privilege that is provided by the role"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And("I try to login using non-cached LDAP expect it to work " + "with user also having privilege provided by the role"): + node.query(f"SELECT * FROM {table_name} LIMIT 1", + settings=[("user", users[1]["username"]), ("password", users[1]["password"])]) + +@TestFeature +@Name("roles") +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_Parameters_Roles("1.0") +) +def feature(self, node="clickhouse1"): + """Check that all the users that are authenticated using + LDAP external user directory are assigned the roles specified + in the configuration of the LDAP external user directory. + """ + self.context.node = self.context.cluster.node(node) + + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + user = {"server": "openldap1", "username": "user1", "password": "user1"} + + with ldap_servers(servers): + for scenario in loads(current_module(), Scenario): + scenario(server="openldap1") diff --git a/tests/testflows/ldap/external_user_directory/tests/server_config.py b/tests/testflows/ldap/external_user_directory/tests/server_config.py new file mode 100644 index 00000000000..b02b70eb6a7 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/tests/server_config.py @@ -0,0 +1,305 @@ +import time + +from testflows.core import * + +from ldap.external_user_directory.tests.common import * +from ldap.external_user_directory.requirements import * + +from ldap.authentication.tests.common import invalid_server_config + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Name("1.0") +) +def empty_server_name(self, timeout=20): + """Check that empty string as a server name is not allowed. + """ + servers = {"": {"host": "foo", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }} + invalid_server_config(servers, timeout=timeout) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_UnreachableServer("1.0") +) +def invalid_host(self): + """Check that server returns an error when LDAP server + host name is invalid. + """ + servers = {"foo": {"host": "foo", "port": "389", "enable_tls": "no"}} + users = [{ + "server": "foo", "username": "user1", "password": "user1", "login": True, + "exitcode": 20, "message": "DB::Exception: Can't contact LDAP server" + }] + login(servers, "foo", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Host("1.0") +) +def empty_host(self, tail=20, timeout=20): + """Check that server returns an error when LDAP server + host value is empty. + """ + node = current().context.node + message = "DB::Exception: Empty 'host' entry" + + servers = {"foo": {"host": "", "port": "389", "enable_tls": "no"}} + users = [{ + "server": "foo", "username": "user1", "password": "user1", "login": True, + "exitcode": 36, "message": "DB::Exception: LDAP server 'foo' is not configured." + }] + + with Given("I prepare the error log by writting empty lines into it"): + node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("-\\n" * tail)) + + with ldap_servers(servers): + with Then("server shall fail to merge the new config"): + started = time.time() + command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Host("1.0") +) +def missing_host(self, tail=20, timeout=20): + """Check that server returns an error when LDAP server + host is missing. + """ + node = current().context.node + message = "DB::Exception: Missing 'host' entry" + + servers = {"foo": {"port": "389", "enable_tls": "no"}} + users = [{ + "server": "foo", "username": "user1", "password": "user1", "login": True, + "exitcode": 36, "message": "DB::Exception: LDAP server 'foo' is not configured." + }] + + with Given("I prepare the error log by writting empty lines into it"): + node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("-\\n" * tail)) + + with ldap_servers(servers): + with Then("server shall fail to merge the new config"): + started = time.time() + command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), +) +def invalid_port(self): + """Check that server returns an error when LDAP server + port is not valid. + """ + servers = {"openldap1": {"host": "openldap1", "port": "3890", "enable_tls": "no"}} + users = [{ + "server": "openldap1", "username": "user1", "password": "user1", "login": True, + "exitcode": 20, "message": "DB::Exception: Can't contact LDAP server." + }] + login(servers, "openldap1", *users) + + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Prefix("1.0") +) +def invalid_auth_dn_prefix(self): + """Check that server returns an error when LDAP server definition + has invalid auth_dn_prefix. + """ + servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "foo=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }} + users = [{ + "server": "openldap1", "username": "user1", "password": "user1", "login": True, + "exitcode": 20, "message": "DB::Exception: Invalid DN syntax: invalid DN" + }] + login(servers, "openldap1", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Suffix("1.0") +) +def invalid_auth_dn_suffix(self): + """Check that server returns an error when LDAP server definition + has invalid auth_dn_suffix. + """ + servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",foo=users,dc=company,dc=com" + }} + users = [{ + "server": "openldap1", "username": "user1", "password": "user1", "login": True, + "exitcode": 20, "message": "DB::Exception: Invalid DN syntax: invalid DN" + }] + login(servers, "openldap1", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS("1.0") +) +def invalid_enable_tls_value(self): + """Check that server returns an error when enable_tls + option has invalid value. + """ + servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "foo", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }} + users = [{ + "server": "openldap1", "username": "user1", "password": "user1", "login": True, + "exitcode": 36, "message": "DB::Exception: LDAP server 'openldap1' is not configured" + }] + login(servers, "openldap1", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSRequireCert("1.0") +) +def invalid_tls_require_cert_value(self): + """Check that server returns an error when tls_require_cert + option has invalid value. + """ + servers = {"openldap2": { + "host": "openldap2", "port": "636", "enable_tls": "yes", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "foo", + "ca_cert_dir": "/container/service/slapd/assets/certs/", + "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt" + }} + users = [{ + "server": "openldap2", "username": "user2", "password": "user2", "login": True, + "exitcode": 36, "message": "DB::Exception: LDAP server 'openldap2' is not configured" + }] + login(servers, "openldap2", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCACertDir("1.0") +) +def empty_ca_cert_dir(self): + """Check that server returns an error when ca_cert_dir is empty. + """ + servers = {"openldap2": {"host": "openldap2", "port": "636", "enable_tls": "yes", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "demand", + "ca_cert_dir": "", + "ca_cert_file": "/container/service/slapd/assets/certs/ca.crt" + }} + users = [{ + "server": "openldap2", "username": "user2", "password": "user2", "login": True, + "exitcode": 20, + "message": "DB::Exception: Can't contact LDAP server: error:14000086:SSL routines::certificate verify failed (self signed certificate in certificate chain" + }] + login(servers, "openldap2", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_TLSCertFile("1.0") +) +def empty_ca_cert_file(self): + """Check that server returns an error when ca_cert_file is empty. + """ + servers = {"openldap2": {"host": "openldap2", "port": "636", "enable_tls": "yes", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "tls_require_cert": "demand", + "ca_cert_dir": "/container/service/slapd/assets/certs/", + "ca_cert_file": "" + }} + users = [{ + "server": "openldap2", "username": "user2", "password": "user2", "login": True, + "exitcode": 20, + "message": "Received from localhost:9000. DB::Exception: Can't contact LDAP server: error:14000086:SSL routines::certificate verify failed (self signed certificate in certificate chain)" + }] + login(servers, "openldap2", *users) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Value("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Prefix("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_AuthDN_Suffix("1.0") +) +def auth_dn_value(self): + """Check that server configuration can properly define the `dn` value of the user.""" + servers = { + "openldap1": { + "host": "openldap1", "port": "389", "enable_tls": "no", + "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }} + user = {"server": "openldap1", "username": "user1", "password": "user1", "login": True} + + login(servers, "openldap1", user) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Syntax("1.0") +) +def syntax(self): + """Check that server configuration with valid syntax can be loaded. + ```xml + + + localhost + 636 + cn= + , ou=users, dc=example, dc=com + yes + tls1.2 + demand + /path/to/tls_cert_file + /path/to/tls_key_file + /path/to/tls_ca_cert_file + /path/to/tls_ca_cert_dir + ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + + + ``` + """ + servers = { + "openldap2": { + "host": "openldap2", + "port": "389", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com", + "enable_tls": "yes", + "tls_minimum_protocol_version": "tls1.2" , + "tls_require_cert": "demand", + "tls_cert_file": "/container/service/slapd/assets/certs/ldap.crt", + "tls_key_file": "/container/service/slapd/assets/certs/ldap.key", + "tls_ca_cert_file": "/container/service/slapd/assets/certs/ca.crt", + "tls_ca_cert_dir": "/container/service/slapd/assets/certs/", + "tls_cipher_suite": "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384" + } + } + with ldap_servers(servers): + pass + +@TestFeature +@Name("server config") +def feature(self, node="clickhouse1"): + """Check LDAP server configuration. + """ + self.context.node = self.context.cluster.node(node) + for scenario in loads(current_module(), Scenario): + scenario() \ No newline at end of file diff --git a/tests/testflows/ldap/external_user_directory/tests/simple.py b/tests/testflows/ldap/external_user_directory/tests/simple.py new file mode 100644 index 00000000000..c48048833c7 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/tests/simple.py @@ -0,0 +1,24 @@ +from testflows.core import * +from testflows.asserts import error + +from ldap.external_user_directory.tests.common import login + +@TestScenario +@Name("simple") +def scenario(self, node="clickhouse1"): + """Check that an LDAP external user directory can be used to authenticate a user. + """ + self.context.node = self.context.cluster.node(node) + servers = { + "openldap1": { + "host": "openldap1", + "port": "389", + "enable_tls": "no", + "auth_dn_prefix": "cn=", + "auth_dn_suffix": ",ou=users,dc=company,dc=com" + }, + } + users = [ + {"server": "openldap1", "username": "user1", "password": "user1", "login": True}, + ] + login(servers, "openldap1", *users) diff --git a/tests/testflows/ldap/regression.py b/tests/testflows/ldap/regression.py index 8520941ed0b..0e9d06cf84a 100755 --- a/tests/testflows/ldap/regression.py +++ b/tests/testflows/ldap/regression.py @@ -15,6 +15,7 @@ def regression(self, local, clickhouse_binary_path): args = {"local": local, "clickhouse_binary_path": clickhouse_binary_path} Feature(test=load("ldap.authentication.regression", "regression"))(**args) + Feature(test=load("ldap.external_user_directory.regression", "regression"))(**args) if main(): regression() From d6c84291c95ffc0f5fd96c206747033ab6266522 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 8 Sep 2020 16:28:52 +0300 Subject: [PATCH 074/411] Fixed data race caused by EVP_CIPHER_fetch() --- src/Functions/FunctionsAES.cpp | 26 +++++--------------------- src/Functions/FunctionsAES.h | 14 +++++--------- 2 files changed, 10 insertions(+), 30 deletions(-) diff --git a/src/Functions/FunctionsAES.cpp b/src/Functions/FunctionsAES.cpp index 009226c035a..0d459b6095f 100644 --- a/src/Functions/FunctionsAES.cpp +++ b/src/Functions/FunctionsAES.cpp @@ -7,18 +7,6 @@ #include -namespace -{ -void CipherDeleter(const EVP_CIPHER * cipher [[maybe_unused]]) -{ -#if OPENSSL_VERSION_NUMBER >= 0x03 << 28 -// Used to free EVP_CIPHER poniter obtained with EVP_CIPHER_fetch, -// available only since OpenSSL ver 3.0.0. - EVP_CIPHER_free(const_cast(cipher)); -#endif -} -} - namespace DB { namespace ErrorCodes @@ -47,7 +35,7 @@ StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const return StringRef(folded_key.data(), cipher_key_size); } -CipherPtr getCipherByName(const StringRef & cipher_name) +const EVP_CIPHER * getCipherByName(const StringRef & cipher_name) { const auto *evp_cipher = EVP_get_cipherbyname(cipher_name.data); if (evp_cipher == nullptr) @@ -61,14 +49,10 @@ CipherPtr getCipherByName(const StringRef & cipher_name) evp_cipher = EVP_aes_256_cfb128(); } -#if OPENSSL_VERSION_NUMBER < 0x03 << 28 - return CipherPtr{evp_cipher, CipherDeleter}; -#else - // HACK: To speed up context initialization with EVP_EncryptInit_ex (which is called at least once per row) - // Apparently cipher from EVP_get_cipherbyname may require additional initialization of context, - // while cipher from EVP_CIPHER_fetch causes less operations => faster context initialization. - return CipherPtr{EVP_CIPHER_fetch(nullptr, EVP_CIPHER_name(evp_cipher), nullptr), &CipherDeleter}; -#endif + // NOTE: cipher obtained not via EVP_CIPHER_fetch() would cause extra work on each context reset + // with EVP_CIPHER_CTX_reset() or EVP_EncryptInit_ex(), but using EVP_CIPHER_fetch() + // causes data race, so we stick to the slower but safer alternative here. + return evp_cipher; } } diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index 25b41a067a7..8465c35a517 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -36,9 +36,7 @@ namespace OpenSSLDetails [[noreturn]] void onError(std::string error_message); StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array & folded_key); -using CipherDeleterType = void (*) (const EVP_CIPHER *cipher); -using CipherPtr = std::unique_ptr; -CipherPtr getCipherByName(const StringRef & name); +const EVP_CIPHER * getCipherByName(const StringRef & name); enum class CompatibilityMode { @@ -189,10 +187,9 @@ private: if (mode.size == 0 || !std::string_view(mode).starts_with("aes-")) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); - auto cipher = getCipherByName(mode); - if (cipher == nullptr) + auto evp_cipher = getCipherByName(mode); + if (evp_cipher == nullptr) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); - const EVP_CIPHER * evp_cipher = cipher.get(); const auto cipher_mode = EVP_CIPHER_mode(evp_cipher); @@ -438,10 +435,9 @@ private: if (mode.size == 0 || !std::string_view(mode).starts_with("aes-")) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); - auto cipher = getCipherByName(mode); - if (cipher == nullptr) + auto evp_cipher = getCipherByName(mode); + if (evp_cipher == nullptr) throw Exception("Invalid mode: " + mode.toString(), ErrorCodes::BAD_ARGUMENTS); - const EVP_CIPHER * evp_cipher = cipher.get(); OpenSSLDetails::validateCipherMode(evp_cipher); From f727f7bc99fb1ae606fe5a5dc7bc6010e1802ab0 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 9 Sep 2020 16:14:02 +0200 Subject: [PATCH 075/411] * Updating config.xml files to remove users_config and access_control_path parameter definition as these are now defined inside the user_directories section and cause a test failure due to https://github.com/ClickHouse/ClickHouse/issues/14511. * Removing the usage of `EXISTS` clauses used in tests outside of clean ups. --- .../ldap/authentication/configs/clickhouse/config.xml | 6 ------ .../external_user_directory/configs/clickhouse/config.xml | 6 ------ .../testflows/ldap/external_user_directory/tests/roles.py | 8 ++++---- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/tests/testflows/ldap/authentication/configs/clickhouse/config.xml b/tests/testflows/ldap/authentication/configs/clickhouse/config.xml index 80c3150d326..e28a0c8e255 100644 --- a/tests/testflows/ldap/authentication/configs/clickhouse/config.xml +++ b/tests/testflows/ldap/authentication/configs/clickhouse/config.xml @@ -117,9 +117,6 @@ /var/lib/clickhouse/user_files/ - - /var/lib/clickhouse/access/ - @@ -132,9 +129,6 @@ - - users.xml - default diff --git a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml index 80c3150d326..e28a0c8e255 100644 --- a/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml +++ b/tests/testflows/ldap/external_user_directory/configs/clickhouse/config.xml @@ -117,9 +117,6 @@ /var/lib/clickhouse/user_files/ - - /var/lib/clickhouse/access/ - @@ -132,9 +129,6 @@ - - users.xml - default diff --git a/tests/testflows/ldap/external_user_directory/tests/roles.py b/tests/testflows/ldap/external_user_directory/tests/roles.py index d621993078b..8a6c6f465d1 100644 --- a/tests/testflows/ldap/external_user_directory/tests/roles.py +++ b/tests/testflows/ldap/external_user_directory/tests/roles.py @@ -166,7 +166,7 @@ def remove_role(self, server, timeout=20): settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) with And("I remove one of the roles"): - node.query(f"DROP ROLE IF EXISTS {roles[1]}") + node.query(f"DROP ROLE {roles[1]}") with And(f"I try to login using cached LDAP user"): node.query(f"SELECT 1", @@ -212,7 +212,7 @@ def remove_privilege_by_removing_role(self, server, timeout=20): settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) with And("I remove the role that grants the privilege"): - node.query(f"DROP ROLE IF EXISTS {roles[0]}") + node.query(f"DROP ROLE {roles[0]}") with And(f"I try to relogin and expect that cached LDAP user can login " "but does not have privilege that was provided by the removed role"): @@ -254,7 +254,7 @@ def readd_privilege_by_readding_role(self, server, timeout=20): settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) with And("I remove the role that grants the privilege"): - node.query(f"DROP ROLE IF EXISTS {roles[0]}") + node.query(f"DROP ROLE {roles[0]}") message = "DB::Exception: {user}: Not enough privileges." exitcode = 241 @@ -274,7 +274,7 @@ def readd_privilege_by_readding_role(self, server, timeout=20): exitcode=exitcode, message=message.format(user=users[1]["username"])) with When("I re-add the role"): - node.query(f"CREATE ROLE IF NOT EXISTS {roles[0]}") + node.query(f"CREATE ROLE {roles[0]}") with And(f"I grant select privilege to the re-added role"): node.query(f"GRANT SELECT ON {table_name} TO {roles[0]}") From 8129169ead0e2ac3f7f7eb836732b3212dc317dc Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 10 Sep 2020 22:52:42 +0300 Subject: [PATCH 076/411] Fixed Yandex-specific builds :( --- src/Functions/FunctionsAES.h | 4 +++- src/Functions/aes_decrypt_mysql.cpp | 4 +++- src/Functions/aes_encrypt_mysql.cpp | 4 +++- src/Functions/decrypt.cpp | 4 +++- src/Functions/encrypt.cpp | 4 +++- src/Functions/registerFunctions.cpp | 4 +++- 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index 8465c35a517..fc9b9ef6f40 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -1,6 +1,8 @@ #pragma once -#include +#if !defined(ARCADIA_BUILD) +# include +#endif #if USE_SSL #include diff --git a/src/Functions/aes_decrypt_mysql.cpp b/src/Functions/aes_decrypt_mysql.cpp index 764fcf06c1a..d5d7a6660e4 100644 --- a/src/Functions/aes_decrypt_mysql.cpp +++ b/src/Functions/aes_decrypt_mysql.cpp @@ -1,4 +1,6 @@ -#include +#if !defined(ARCADIA_BUILD) +# include +#endif #if USE_SSL diff --git a/src/Functions/aes_encrypt_mysql.cpp b/src/Functions/aes_encrypt_mysql.cpp index 1d84824d9d6..bdf7d8d2cce 100644 --- a/src/Functions/aes_encrypt_mysql.cpp +++ b/src/Functions/aes_encrypt_mysql.cpp @@ -1,4 +1,6 @@ -#include +#if !defined(ARCADIA_BUILD) +# include +#endif #if USE_SSL diff --git a/src/Functions/decrypt.cpp b/src/Functions/decrypt.cpp index 1cbda0dba99..a9090a31f0e 100644 --- a/src/Functions/decrypt.cpp +++ b/src/Functions/decrypt.cpp @@ -1,4 +1,6 @@ -#include +#if !defined(ARCADIA_BUILD) +# include +#endif #if USE_SSL diff --git a/src/Functions/encrypt.cpp b/src/Functions/encrypt.cpp index 807443a2fb8..ba8ffa1bc59 100644 --- a/src/Functions/encrypt.cpp +++ b/src/Functions/encrypt.cpp @@ -1,4 +1,6 @@ -#include +#if !defined(ARCADIA_BUILD) +# include +#endif #if USE_SSL diff --git a/src/Functions/registerFunctions.cpp b/src/Functions/registerFunctions.cpp index b7fb3cecd66..e88a6704b56 100644 --- a/src/Functions/registerFunctions.cpp +++ b/src/Functions/registerFunctions.cpp @@ -1,4 +1,6 @@ -#include +#if !defined(ARCADIA_BUILD) +# include +#endif #include From bb1d126ce53794d814f5d7d9dfce2c62b51dd13e Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 9 Sep 2020 23:45:07 +0200 Subject: [PATCH 077/411] Updating TestFlows to 1.6.45 to fix stability issues with multi line commands. --- docker/test/testflows/runner/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index 6b4ec12b80c..219873e7be6 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN pip3 install urllib3 testflows==1.6.42 docker-compose docker dicttoxml kazoo tzlocal +RUN pip3 install urllib3 testflows==1.6.45 docker-compose docker dicttoxml kazoo tzlocal ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 17.09.1-ce From fe35de5fec942cd3aff88a0d819d8aec58a1d6af Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Fri, 11 Sep 2020 22:41:22 +0200 Subject: [PATCH 078/411] Updating TestFlows to 1.6.46 --- docker/test/testflows/runner/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index 898552ade56..ecbb43a88ec 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN pip3 install urllib3 testflows==1.6.42 docker-compose docker dicttoxml kazoo tzlocal +RUN pip3 install urllib3 testflows==1.6.46 docker-compose docker dicttoxml kazoo tzlocal ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 17.09.1-ce From 30b1831752209d1aabcf4ff80dc8d1c2bd919b27 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 14 Sep 2020 18:15:07 +0300 Subject: [PATCH 079/411] Moved default values for query_masking rules for encrypt/decrypt to config.xml --- debian/clickhouse-server.install | 1 - programs/server/CMakeLists.txt | 2 -- programs/server/config.xml | 14 +++++++++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/debian/clickhouse-server.install b/debian/clickhouse-server.install index 57f858407af..b1475fdf162 100644 --- a/debian/clickhouse-server.install +++ b/debian/clickhouse-server.install @@ -2,6 +2,5 @@ usr/bin/clickhouse-server usr/bin/clickhouse-copier usr/bin/clickhouse-report etc/clickhouse-server/config.xml -etc/clickhouse-server/config.d/*.xml etc/clickhouse-server/users.xml etc/systemd/system/clickhouse-server.service diff --git a/programs/server/CMakeLists.txt b/programs/server/CMakeLists.txt index adaddcf1762..5500a4680b7 100644 --- a/programs/server/CMakeLists.txt +++ b/programs/server/CMakeLists.txt @@ -29,8 +29,6 @@ set (CLICKHOUSE_SERVER_LINK clickhouse_program_add(server) install(FILES config.xml users.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server COMPONENT clickhouse) -install(FILES config.xml users.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server COMPONENT clickhouse) -install(FILES config.d/query_masking_rules.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server/config.d COMPONENT clickhouse) # TODO We actually need this on Mac, FreeBSD. if (OS_LINUX) diff --git a/programs/server/config.xml b/programs/server/config.xml index af01e880dc2..fc41e323e43 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -670,18 +670,22 @@ --> /var/lib/clickhouse/format_schemas/ - - hide SSN - \b\d{3}-\d{2}-\d{4}\b - 000-00-0000 + hide encrypt/decrypt arguments + ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) + + \1(???) - --> - \1(???) - From a93c579798738d0375bb2c0bb9e393288722c5fe Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 17 Sep 2020 00:36:57 +0300 Subject: [PATCH 085/411] Update ColumnString.cpp --- src/Columns/ColumnString.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index d38008c44c7..16f65c2eefa 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -615,7 +615,7 @@ void ColumnString::protect() void ColumnString::validate() const { if (!offsets.empty() && offsets.back() != chars.size()) - throw Exception(fmt::format("ColumnString validation failed: size mismatch (internal logical error) {} != {}", offsets.back(), chars.size()), ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "ColumnString validation failed: size mismatch (internal logical error) {} != {}", offsets.back(), chars.size()); } } From d97426fe9ad784a8fabbe83651bd5312050ac242 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 17 Sep 2020 00:47:43 +0300 Subject: [PATCH 086/411] Update FunctionsAES.cpp --- src/Functions/FunctionsAES.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Functions/FunctionsAES.cpp b/src/Functions/FunctionsAES.cpp index 0d459b6095f..120be3749f1 100644 --- a/src/Functions/FunctionsAES.cpp +++ b/src/Functions/FunctionsAES.cpp @@ -6,6 +6,8 @@ #include #include +#include + namespace DB { @@ -25,6 +27,7 @@ void onError(std::string error_message) StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, const StringRef & key, std::array & folded_key) { + assert(cipher_key_size <= EVP_MAX_KEY_LENGTH); memcpy(folded_key.data(), key.data, cipher_key_size); for (size_t i = cipher_key_size; i < key.size; ++i) From 60f91332b3732f46d6fbcc48f050e2baa2bd8901 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 17 Sep 2020 00:52:01 +0300 Subject: [PATCH 087/411] Update FunctionsAES.h --- src/Functions/FunctionsAES.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index fc9b9ef6f40..24e797f54cf 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -19,12 +19,12 @@ #include #include -#include #include #include #include + namespace DB { namespace ErrorCodes From 200bc9b9b2a390db7ecf4cc363cd047dde7d73df Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 17 Sep 2020 00:57:26 +0300 Subject: [PATCH 088/411] Update FunctionsAES.h --- src/Functions/FunctionsAES.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Functions/FunctionsAES.h b/src/Functions/FunctionsAES.h index 24e797f54cf..2bdcb23611c 100644 --- a/src/Functions/FunctionsAES.h +++ b/src/Functions/FunctionsAES.h @@ -7,10 +7,8 @@ #if USE_SSL #include #include -#include #include #include - #include #include From 03481f7a3a190a8a10d6d6d1fb412da6b20b651a Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 21 Sep 2020 02:51:38 +0400 Subject: [PATCH 089/411] Synchronize the code with respect to IAccessStorage::login() functionality --- src/Access/AccessControlManager.cpp | 9 +- src/Access/IAccessStorage.cpp | 12 --- src/Access/IAccessStorage.h | 7 -- src/Access/LDAPAccessStorage.cpp | 134 ++++++++++++++------------- src/Access/LDAPAccessStorage.h | 10 +- src/Access/MultipleAccessStorage.cpp | 17 ---- src/Access/MultipleAccessStorage.h | 1 - 7 files changed, 75 insertions(+), 115 deletions(-) diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index 9a9bca7734a..f5f1e4b3de8 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -257,9 +257,7 @@ void AccessControlManager::addMemoryStorage(const String & storage_name_) void AccessControlManager::addLDAPStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_) { - auto storage = std::make_shared(storage_name_); - storage->setConfiguration(this, config_, prefix_); - addStorage(storage); + addStorage(std::make_shared(storage_name_, this, config_, prefix_)); } @@ -279,8 +277,7 @@ void AccessControlManager::addStoragesFromUserDirectoriesConfig( String prefix = key + "." + key_in_user_directories; String type = key_in_user_directories; - const size_t bracket_pos = type.find('['); - if (bracket_pos != String::npos) + if (size_t bracket_pos = type.find('['); bracket_pos != String::npos) type.resize(bracket_pos); if ((type == "users_xml") || (type == "users_config")) type = UsersConfigAccessStorage::STORAGE_TYPE; @@ -310,8 +307,6 @@ void AccessControlManager::addStoragesFromUserDirectoriesConfig( } else if (type == LDAPAccessStorage::STORAGE_TYPE) { - if (bracket_pos != String::npos) - throw Exception("Duplicate storage type '" + type + "' at " + prefix + " in config", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); addLDAPStorage(name, config, prefix); } else diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index fbc95342972..e5170221e18 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -153,12 +153,6 @@ std::vector IAccessStorage::find(EntityType type, const Strings & names) c } -std::optional IAccessStorage::findOrGenerate(EntityType type, const String & name) const -{ - return findOrGenerateImpl(type, name); -} - - UUID IAccessStorage::getID(EntityType type, const String & name) const { auto id = findImpl(type, name); @@ -489,12 +483,6 @@ Poco::Logger * IAccessStorage::getLogger() const } -std::optional IAccessStorage::findOrGenerateImpl(EntityType type, const String & name) const -{ - return findImpl(type, name); -} - - void IAccessStorage::throwNotFound(const UUID & id) const { throw Exception(outputID(id) + " not found in " + getStorageName(), ErrorCodes::ACCESS_ENTITY_NOT_FOUND); diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index c47ab608c20..5a86e817fb2 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -53,12 +53,6 @@ public: template std::vector find(const Strings & names) const { return find(EntityClassT::TYPE, names); } - /// Searches for an entity with specified type and name. Returns std::nullopt if not found and cannot be generated. - std::optional findOrGenerate(EntityType type, const String & name) const; - - template - std::optional findOrGenerate(const String & name) const { return findOrGenerate(EntityClassT::TYPE, name); } - /// Searches for an entity with specified name and type. Throws an exception if not found. UUID getID(EntityType type, const String & name) const; @@ -158,7 +152,6 @@ public: protected: virtual std::optional findImpl(EntityType type, const String & name) const = 0; - virtual std::optional findOrGenerateImpl(EntityType type, const String & name) const; virtual std::vector findAllImpl(EntityType type) const = 0; virtual bool existsImpl(const UUID & id) const = 0; virtual AccessEntityPtr readImpl(const UUID & id) const = 0; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 59f061c51a5..ae61d912ebe 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -6,6 +6,10 @@ #include #include #include +#include +#include +#include +#include namespace DB @@ -16,9 +20,10 @@ namespace ErrorCodes } -LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_) +LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_, AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix) : IAccessStorage(storage_name_) { + setConfiguration(access_control_manager_, config, prefix); } @@ -62,12 +67,6 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m } -bool LDAPAccessStorage::isConfiguredNoLock() const -{ - return !ldap_server.empty() &&/* !roles.empty() &&*/ access_control_manager; -} - - void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr & entity) { auto role_ptr = typeid_cast>(entity); @@ -122,9 +121,17 @@ const char * LDAPAccessStorage::getStorageType() const } -bool LDAPAccessStorage::isStorageReadOnly() const +String LDAPAccessStorage::getStorageParamsJSON() const { - return true; + Poco::JSON::Object params_json; + + params_json.set("server", ldap_server); + params_json.set("roles", default_role_names); + + std::ostringstream oss; + Poco::JSON::Stringifier::stringify(params_json, oss); + + return oss.str(); } @@ -134,62 +141,6 @@ std::optional LDAPAccessStorage::findImpl(EntityType type, const String & } -std::optional LDAPAccessStorage::findOrGenerateImpl(EntityType type, const String & name) const -{ - if (type == EntityType::USER) - { - std::scoped_lock lock(mutex); - - // Return the id immediately if we already have it. - const auto id = memory_storage.find(type, name); - if (id.has_value()) - return id; - - if (!isConfiguredNoLock()) - return {}; - - // Stop if entity exists anywhere else, to avoid generating duplicates. - const auto * this_base = dynamic_cast(this); - const auto storages = access_control_manager->getStoragesPtr(); - for (const auto & storage : *storages) - { - if (storage.get() != this_base && storage->find(type, name)) - return {}; - } - - // Entity doesn't exist. We are going to create one. - const auto user = std::make_shared(); - user->setName(name); - user->authentication = Authentication(Authentication::Type::LDAP_SERVER); - user->authentication.setServerName(ldap_server); - - for (const auto& role_name : default_role_names) - { - std::optional role_id; - - try - { - role_id = access_control_manager->find(role_name); - if (!role_id) - throw Exception("Retrieved role info is empty", IAccessEntity::TypeInfo::get(IAccessEntity::Type::ROLE).not_found_error_code); - } - catch (...) - { - tryLogCurrentException(getLogger(), "Unable to retrieve role '" + role_name + "' info from access storage '" + access_control_manager->getStorageName() + "'"); - return {}; - } - - roles_of_interest.insert(role_id.value()); - user->granted_roles.grant(role_id.value()); - } - - return memory_storage.insert(user); - } - - return memory_storage.find(type, name); -} - - std::vector LDAPAccessStorage::findAllImpl(EntityType type) const { return memory_storage.findAll(type); @@ -262,4 +213,57 @@ bool LDAPAccessStorage::hasSubscriptionImpl(EntityType type) const { return memory_storage.hasSubscription(type); } + +UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const +{ + std::scoped_lock lock(mutex); + try + { + auto id = memory_storage.find(user_name); + if (id) + { + // We try to re-authenticate the existing user, and if not successful, we will remove it, since that would mean + // something changed and the user we authenticated previously cannot be authenticated anymore. + auto user = memory_storage.tryRead(*id); + try + { + if (user && isAddressAllowedImpl(*user, address) && isPasswordCorrectImpl(*user, password, external_authenticators)) + return *id; + } + catch (...) + { + memory_storage.remove(*id); + throw; + } + memory_storage.remove(*id); + } + else + { + // User does not exist, so we create one, and will add it if authentication is successful. + auto user = std::make_shared(); + user->setName(user_name); + user->authentication = Authentication(Authentication::Type::LDAP_SERVER); + user->authentication.setServerName(ldap_server); + + if (isAddressAllowedImpl(*user, address) && isPasswordCorrectImpl(*user, password, external_authenticators)) + { + for (const auto& role_name : default_role_names) + { + std::optional role_id = access_control_manager->find(role_name); + if (!role_id) + throw Exception("One of the default roles, the role '" + role_name + "', is not found", IAccessEntity::TypeInfo::get(IAccessEntity::Type::ROLE).not_found_error_code); + roles_of_interest.insert(role_id.value()); + user->granted_roles.grant(role_id.value()); + } + return memory_storage.insert(user); + } + } + } + catch (...) + { + tryLogCurrentException(getLogger(), "Authentication failed for user '" + user_name + "' from access storage '" + access_control_manager->getStorageName() + "'"); + } + throwCannotAuthenticate(user_name); +} + } diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index d52056fe947..35444944ec6 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -28,18 +28,15 @@ class LDAPAccessStorage : public IAccessStorage public: static constexpr char STORAGE_TYPE[] = "ldap"; - explicit LDAPAccessStorage(const String & storage_name_ = STORAGE_TYPE); + explicit LDAPAccessStorage(const String & storage_name_, AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix); virtual ~LDAPAccessStorage() override = default; - void setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix = ""); - public: // IAccessStorage implementations. virtual const char * getStorageType() const override; - virtual bool isStorageReadOnly() const override; + virtual String getStorageParamsJSON() const override; private: // IAccessStorage implementations. virtual std::optional findImpl(EntityType type, const String & name) const override; - virtual std::optional findOrGenerateImpl(EntityType type, const String & name) const override; virtual std::vector findAllImpl(EntityType type) const override; virtual bool existsImpl(const UUID & id) const override; virtual AccessEntityPtr readImpl(const UUID & id) const override; @@ -52,9 +49,10 @@ private: // IAccessStorage implementations. virtual ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; virtual bool hasSubscriptionImpl(const UUID & id) const override; virtual bool hasSubscriptionImpl(EntityType type) const override; + virtual UUID loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const override; private: - bool isConfiguredNoLock() const; + void setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix); void processRoleChange(const UUID & id, const AccessEntityPtr & entity); mutable std::recursive_mutex mutex; diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index e613279a466..8ddc7410d8d 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -104,23 +104,6 @@ std::optional MultipleAccessStorage::findImpl(EntityType type, const Strin } -std::optional MultipleAccessStorage::findOrGenerateImpl(EntityType type, const String & name) const -{ - auto storages = getStoragesInternal(); - for (const auto & storage : *storages) - { - auto id = storage->findOrGenerate(type, name); - if (id) - { - std::lock_guard lock{mutex}; - ids_cache.set(*id, storage); - return id; - } - } - return {}; -} - - std::vector MultipleAccessStorage::findAllImpl(EntityType type) const { std::vector all_ids; diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index 088e1e82e05..36551f1cbc8 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -35,7 +35,6 @@ public: protected: std::optional findImpl(EntityType type, const String & name) const override; - std::optional findOrGenerateImpl(EntityType type, const String & name) const override; std::vector findAllImpl(EntityType type) const override; bool existsImpl(const UUID & id) const override; AccessEntityPtr readImpl(const UUID & id) const override; From 2c6b6673f2d7350b59a55838a45d429e20af7090 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 21 Sep 2020 02:57:34 +0400 Subject: [PATCH 090/411] Remove extra declaration --- src/Access/AccessControlManager.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index f5f1e4b3de8..56d225f64f4 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -24,7 +24,6 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_ELEMENT_IN_CONFIG; - extern const int EXCESSIVE_ELEMENT_IN_CONFIG; extern const int UNKNOWN_SETTING; } From ab19bb25fd8c286713580649ad1c183d493ea5dc Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Tue, 22 Sep 2020 14:31:33 +0300 Subject: [PATCH 091/411] disable percpu arenas --- contrib/jemalloc-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index dd7f9f3e2bb..563d41301b1 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -81,7 +81,7 @@ if (NOT EXTERNAL_JEMALLOC_LIBRARY_FOUND OR NOT EXTERNAL_JEMALLOC_LIBRARY_WORKS) # avoid spurious latencies and additional work associated with # MADV_DONTNEED. See # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. - set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000") + set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:10000") else() set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:10000") endif() From 833c07f1f76b71527262362545fef3973fad686c Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 23 Sep 2020 11:31:18 +0300 Subject: [PATCH 092/411] Update compare.sh --- docker/test/performance-comparison/compare.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 2fd5641b9fd..7851cf8e81d 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -7,6 +7,10 @@ trap 'kill $(jobs -pr) ||:' EXIT stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +# https://github.com/jemalloc/jemalloc/wiki/Getting-Started +export MALLOC_CONF="percpu_arena:disabled" +echo "$MALLOC_CONF" > /etc/malloc.conf ||: + function wait_for_server # port, pid { for _ in {1..60} From bb51aade56d478588a8db60315c30870208aeec3 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 23 Sep 2020 12:02:22 +0300 Subject: [PATCH 093/411] Update docker/test/performance-comparison/compare.sh Co-authored-by: Azat Khuzhin --- docker/test/performance-comparison/compare.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 7851cf8e81d..35bb5890488 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -8,8 +8,7 @@ stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # https://github.com/jemalloc/jemalloc/wiki/Getting-Started -export MALLOC_CONF="percpu_arena:disabled" -echo "$MALLOC_CONF" > /etc/malloc.conf ||: +ln -s "percpu_arena:disabled" > /etc/malloc.conf function wait_for_server # port, pid { From d96c89972cc9e3ff7e8baca7a3899c1aeffed891 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 23 Sep 2020 12:04:13 +0300 Subject: [PATCH 094/411] Update compare.sh --- docker/test/performance-comparison/compare.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 35bb5890488..ed89e6f875c 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -8,7 +8,8 @@ stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # https://github.com/jemalloc/jemalloc/wiki/Getting-Started -ln -s "percpu_arena:disabled" > /etc/malloc.conf +export MALLOC_CONF="percpu_arena:disabled" +ln -s "percpu_arena:disabled" /etc/malloc.conf function wait_for_server # port, pid { From 26abe8cb30819eea1c1f3383cc99620ab2e9da9a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 24 Sep 2020 12:24:30 +0300 Subject: [PATCH 095/411] Update compare.sh --- docker/test/performance-comparison/compare.sh | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index ed89e6f875c..7708d11d81f 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -8,8 +8,7 @@ stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # https://github.com/jemalloc/jemalloc/wiki/Getting-Started -export MALLOC_CONF="percpu_arena:disabled" -ln -s "percpu_arena:disabled" /etc/malloc.conf +export MALLOC_CONF="confirm_conf:true" function wait_for_server # port, pid { @@ -83,18 +82,16 @@ function restart set -m # Spawn servers in their own process groups - numactl --cpunodebind=0 --localalloc \ - left/clickhouse-server --config-file=left/config/config.xml \ - -- --path left/db --user_files_path left/db/user_files \ - &>> left-server-log.log & + left/clickhouse-server --config-file=left/config/config.xml \ + -- --path left/db --user_files_path left/db/user_files \ + &>> left-server-log.log & left_pid=$! kill -0 $left_pid disown $left_pid - numactl --cpunodebind=0 --localalloc \ - right/clickhouse-server --config-file=right/config/config.xml \ - -- --path right/db --user_files_path right/db/user_files \ - &>> right-server-log.log & + right/clickhouse-server --config-file=right/config/config.xml \ + -- --path right/db --user_files_path right/db/user_files \ + &>> right-server-log.log & right_pid=$! kill -0 $right_pid disown $right_pid From 425150e78308941c5e7a7c097ac66c84721008df Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 25 Sep 2020 13:19:37 +0300 Subject: [PATCH 096/411] bind to different nodes --- docker/test/performance-comparison/compare.sh | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 7708d11d81f..98040b037f6 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -7,9 +7,6 @@ trap 'kill $(jobs -pr) ||:' EXIT stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -# https://github.com/jemalloc/jemalloc/wiki/Getting-Started -export MALLOC_CONF="confirm_conf:true" - function wait_for_server # port, pid { for _ in {1..60} @@ -80,24 +77,31 @@ function restart while killall clickhouse-server; do echo . ; sleep 1 ; done echo all killed - set -m # Spawn servers in their own process groups + # https://github.com/jemalloc/jemalloc/wiki/Getting-Started + export MALLOC_CONF="percpu_arena:disabled,confirm_conf:true" - left/clickhouse-server --config-file=left/config/config.xml \ - -- --path left/db --user_files_path left/db/user_files \ - &>> left-server-log.log & + set -m # Spawn servers in their own process groups + + numactl --cpunodebind=1 --localalloc \ + left/clickhouse-server --config-file=left/config/config.xml \ + -- --path left/db --user_files_path left/db/user_files \ + &>> left-server-log.log & left_pid=$! kill -0 $left_pid disown $left_pid - right/clickhouse-server --config-file=right/config/config.xml \ - -- --path right/db --user_files_path right/db/user_files \ - &>> right-server-log.log & + numactl --cpunodebind=0 --localalloc \ + right/clickhouse-server --config-file=right/config/config.xml \ + -- --path right/db --user_files_path right/db/user_files \ + &>> right-server-log.log & right_pid=$! kill -0 $right_pid disown $right_pid set +m + unset MALLOC_CONF + wait_for_server 9001 $left_pid echo left ok From bde19bf240571ca6d05450412a838ae4bea2f782 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 28 Sep 2020 15:26:51 +0300 Subject: [PATCH 097/411] restart the build From a1f4d38019a408c2a840df1dc877ea04983cb089 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 29 Sep 2020 12:44:28 +0300 Subject: [PATCH 098/411] Add ability to run tests multiple time in stateless image --- docker/test/stateless/Dockerfile | 3 +++ docker/test/stateless/run.sh | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 516d8d5842b..c7529686cbe 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -33,5 +33,8 @@ RUN mkdir -p /tmp/clickhouse-odbc-tmp \ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +ENV NUM_TRIES=1 +ENV MAX_RUN_TIME=0 #unlimited + COPY run.sh / CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 9f2bb9bf62d..e736ef835c7 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -17,4 +17,13 @@ if cat /usr/bin/clickhouse-test | grep -q -- "--use-skip-list"; then SKIP_LIST_OPT="--use-skip-list" fi -clickhouse-test --testname --shard --zookeeper "$SKIP_LIST_OPT" $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt +function run_tests() +{ + for i in $(seq 1 $NUM_TRIES); do + clickhouse-test --testname --shard --zookeeper "$SKIP_LIST_OPT" $ADDITIONAL_OPTIONS $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a test_output/test_result.txt + done +} + +export -f run_tests + +timeout $MAX_RUN_TIME bash -c run_tests ||: From 162541217fb165ca0f273f6019e91e417b2755b6 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Tue, 29 Sep 2020 22:58:09 +0200 Subject: [PATCH 099/411] Updating tests to match changes in the behavior. --- .../tests/authentications.py | 2 +- .../tests/external_user_directory_config.py | 14 ++++- .../tests/server_config.py | 53 ++++++------------- 3 files changed, 30 insertions(+), 39 deletions(-) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index 76e55996f21..e4df466b1ea 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -506,4 +506,4 @@ def feature(self, servers=None, server=None, node="clickhouse1"): with rbac_roles("ldap_role") as roles: with ldap_external_user_directory(server=server, roles=roles, restart=True): for scenario in loads(current_module(), Scenario): - scenario(server=server) + Scenario(test=scenario, flags=TE)(server=server) diff --git a/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py b/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py index d29d0124dc2..d95a4a674a1 100644 --- a/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py +++ b/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py @@ -36,7 +36,17 @@ def more_than_one_user_directory(self, timeout=20): with ldap_servers(servers): with rbac_roles(role) as roles: config = create_entries_ldap_external_user_directory_config_content(entries) - invalid_ldap_external_user_directory_config(server=None, roles=None, message=message, timeout=timeout, config=config) + + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with When(f"I login as {users[0]['username']} authenticated using openldap1"): + current().context.node.query(f"SELECT 1", + settings=[("user", users[0]["username"]), ("password", users[0]["password"])]) + + with And(f"I login as {users[1]['username']} authenticated using openldap2"): + current().context.node.query(f"SELECT 1", + settings=[("user", users[1]["username"]), ("password", users[1]["password"])]) + + @TestScenario @Requirements( @@ -132,7 +142,7 @@ def invalid_server(self, timeout=20): with When(f"I login as {user['username']} and execute query"): current().context.node.query("SELECT 1", settings=[("user", user["username"]), ("password", user["password"])], - exitcode=36, message="DB::Exception: LDAP server 'openldap2' is not configured") + exitcode=4, message="DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name.") @TestScenario @Requirements( diff --git a/tests/testflows/ldap/external_user_directory/tests/server_config.py b/tests/testflows/ldap/external_user_directory/tests/server_config.py index b02b70eb6a7..5df343b53df 100644 --- a/tests/testflows/ldap/external_user_directory/tests/server_config.py +++ b/tests/testflows/ldap/external_user_directory/tests/server_config.py @@ -12,7 +12,7 @@ from ldap.authentication.tests.common import invalid_server_config RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Name("1.0") ) -def empty_server_name(self, timeout=20): +def empty_server_name(self, timeout=60): """Check that empty string as a server name is not allowed. """ servers = {"": {"host": "foo", "port": "389", "enable_tls": "no", @@ -32,7 +32,7 @@ def invalid_host(self): servers = {"foo": {"host": "foo", "port": "389", "enable_tls": "no"}} users = [{ "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Can't contact LDAP server" + "exitcode": 4, "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name." }] login(servers, "foo", *users) @@ -41,7 +41,7 @@ def invalid_host(self): RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Host("1.0") ) -def empty_host(self, tail=20, timeout=20): +def empty_host(self, tail=20, timeout=60): """Check that server returns an error when LDAP server host value is empty. """ @@ -49,31 +49,15 @@ def empty_host(self, tail=20, timeout=20): message = "DB::Exception: Empty 'host' entry" servers = {"foo": {"host": "", "port": "389", "enable_tls": "no"}} - users = [{ - "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 36, "message": "DB::Exception: LDAP server 'foo' is not configured." - }] - with Given("I prepare the error log by writting empty lines into it"): - node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("-\\n" * tail)) - - with ldap_servers(servers): - with Then("server shall fail to merge the new config"): - started = time.time() - command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{message}\"" - while time.time() - started < timeout: - exitcode = node.command(command, steps=False).exitcode - if exitcode == 0: - break - time.sleep(1) - assert exitcode == 0, error() + invalid_server_config(servers, message=message, tail=16, timeout=timeout) @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Host("1.0") ) -def missing_host(self, tail=20, timeout=20): +def missing_host(self, tail=20, timeout=60): """Check that server returns an error when LDAP server host is missing. """ @@ -111,7 +95,7 @@ def invalid_port(self): servers = {"openldap1": {"host": "openldap1", "port": "3890", "enable_tls": "no"}} users = [{ "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Can't contact LDAP server." + "exitcode": 4, "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name." }] login(servers, "openldap1", *users) @@ -130,7 +114,7 @@ def invalid_auth_dn_prefix(self): }} users = [{ "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Invalid DN syntax: invalid DN" + "exitcode": 4, "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name." }] login(servers, "openldap1", *users) @@ -148,7 +132,7 @@ def invalid_auth_dn_suffix(self): }} users = [{ "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Invalid DN syntax: invalid DN" + "exitcode": 4, "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name." }] login(servers, "openldap1", *users) @@ -157,18 +141,15 @@ def invalid_auth_dn_suffix(self): RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_Invalid("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Server_EnableTLS("1.0") ) -def invalid_enable_tls_value(self): +def invalid_enable_tls_value(self, timeout=60): """Check that server returns an error when enable_tls option has invalid value. """ + message = "Syntax error: Cannot convert to boolean: foo" servers = {"openldap1": {"host": "openldap1", "port": "389", "enable_tls": "foo", "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" }} - users = [{ - "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 36, "message": "DB::Exception: LDAP server 'openldap1' is not configured" - }] - login(servers, "openldap1", *users) + invalid_server_config(servers, message=message, tail=17, timeout=timeout) @TestScenario @Requirements( @@ -188,7 +169,7 @@ def invalid_tls_require_cert_value(self): }} users = [{ "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 36, "message": "DB::Exception: LDAP server 'openldap2' is not configured" + "exitcode": 4, "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name." }] login(servers, "openldap2", *users) @@ -208,8 +189,8 @@ def empty_ca_cert_dir(self): }} users = [{ "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 20, - "message": "DB::Exception: Can't contact LDAP server: error:14000086:SSL routines::certificate verify failed (self signed certificate in certificate chain" + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, "openldap2", *users) @@ -229,8 +210,8 @@ def empty_ca_cert_file(self): }} users = [{ "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 20, - "message": "Received from localhost:9000. DB::Exception: Can't contact LDAP server: error:14000086:SSL routines::certificate verify failed (self signed certificate in certificate chain)" + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name." }] login(servers, "openldap2", *users) @@ -302,4 +283,4 @@ def feature(self, node="clickhouse1"): """ self.context.node = self.context.cluster.node(node) for scenario in loads(current_module(), Scenario): - scenario() \ No newline at end of file + scenario() From 824d5b093c453506cb64f2abc67ce3034a184da6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 30 Sep 2020 14:32:49 +0300 Subject: [PATCH 100/411] bind everything to node 0 --- contrib/jemalloc-cmake/CMakeLists.txt | 2 +- docker/test/performance-comparison/Dockerfile | 9 ++++++++- docker/test/performance-comparison/compare.sh | 18 ++++++++++-------- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 563d41301b1..dd7f9f3e2bb 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -81,7 +81,7 @@ if (NOT EXTERNAL_JEMALLOC_LIBRARY_FOUND OR NOT EXTERNAL_JEMALLOC_LIBRARY_WORKS) # avoid spurious latencies and additional work associated with # MADV_DONTNEED. See # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. - set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:10000") + set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000") else() set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:10000") endif() diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index a4f8af2f388..99f2f9b2b4b 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -37,7 +37,14 @@ RUN apt-get update \ COPY * / -CMD /entrypoint.sh +# Bind everything to node 0 early. We have to bind both servers and the tmpfs +# on which the database is stored. How to do it through Yandex Sandbox API is +# unclear, but by default tmpfs uses 'process allocation policy', not sure +# which process but hopefully the one that writes to it, so just bind the +# downloader script as well. +# We could also try to remount it with proper options in Sandbox task. +# https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt +CMD ['numactl', '--cpunodebind=0', '--localalloc', '/entrypoint.sh'] # docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-performance-comparison diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 0134d03aea1..2f03ecc9ad7 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -77,23 +77,25 @@ function restart while killall clickhouse-server; do echo . ; sleep 1 ; done echo all killed + # Disable percpu arenas because they segfault when the process is bound to + # a particular NUMA node: https://github.com/jemalloc/jemalloc/pull/1939 + # + # About the jemalloc settings: # https://github.com/jemalloc/jemalloc/wiki/Getting-Started export MALLOC_CONF="percpu_arena:disabled,confirm_conf:true" set -m # Spawn servers in their own process groups - numactl --cpunodebind=1 --localalloc \ - left/clickhouse-server --config-file=left/config/config.xml \ - -- --path left/db --user_files_path left/db/user_files \ - &>> left-server-log.log & + left/clickhouse-server --config-file=left/config/config.xml \ + -- --path left/db --user_files_path left/db/user_files \ + &>> left-server-log.log & left_pid=$! kill -0 $left_pid disown $left_pid - numactl --cpunodebind=0 --localalloc \ - right/clickhouse-server --config-file=right/config/config.xml \ - -- --path right/db --user_files_path right/db/user_files \ - &>> right-server-log.log & + right/clickhouse-server --config-file=right/config/config.xml \ + -- --path right/db --user_files_path right/db/user_files \ + &>> right-server-log.log & right_pid=$! kill -0 $right_pid disown $right_pid From aa543a2d3d51f16c6d043a36bb9f4249ba35cd05 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 30 Sep 2020 17:40:24 +0300 Subject: [PATCH 101/411] quotes --- docker/test/performance-comparison/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index 99f2f9b2b4b..535f7de9e29 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -44,7 +44,7 @@ COPY * / # downloader script as well. # We could also try to remount it with proper options in Sandbox task. # https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt -CMD ['numactl', '--cpunodebind=0', '--localalloc', '/entrypoint.sh'] +CMD ["numactl", "--cpunodebind=0", "--localalloc", "/entrypoint.sh"] # docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-performance-comparison From 22a0ec0892c6058acb9b0e09f3f4b57f5b99647e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 30 Sep 2020 17:55:40 +0300 Subject: [PATCH 102/411] try split debug -Og build in fasttest --- docker/test/fasttest/run.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 4a47fcfe4dc..e16a70fc3b2 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -124,7 +124,20 @@ git submodule foreach git clean -xfd function run_cmake { -CMAKE_LIBS_CONFIG=("-DENABLE_LIBRARIES=0" "-DENABLE_TESTS=0" "-DENABLE_UTILS=0" "-DENABLE_EMBEDDED_COMPILER=0" "-DENABLE_THINLTO=0" "-DUSE_UNWIND=1") +CMAKE_LIBS_CONFIG=( + "-DENABLE_LIBRARIES=0" + "-DENABLE_TESTS=0" + "-DENABLE_UTILS=0" + "-DENABLE_EMBEDDED_COMPILER=0" + "-DENABLE_THINLTO=0" + "-DUSE_UNWIND=1" + "-DUSE_STATIC_LIBRARIES=0" + "-DSPLIT_SHARED_LIBRARIES=1" + "-DCLICKHOUSE_SPLIT_BINARY=1" + "-DCMAKE_BUILD_TYPE=Debug" + "-DCMAKE_C_FLAGS=-Og" + "-DCMAKE_CXX_FLAGS=-Og" +) # TODO remove this? we don't use ccache anyway. An option would be to download it # from S3 simultaneously with cloning. From c5d1f51f5836da49a33c531dad6cd46dccc271d5 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 30 Sep 2020 19:14:20 +0300 Subject: [PATCH 103/411] just split --- docker/test/fasttest/run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index e16a70fc3b2..f769b342846 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -134,9 +134,9 @@ CMAKE_LIBS_CONFIG=( "-DUSE_STATIC_LIBRARIES=0" "-DSPLIT_SHARED_LIBRARIES=1" "-DCLICKHOUSE_SPLIT_BINARY=1" - "-DCMAKE_BUILD_TYPE=Debug" - "-DCMAKE_C_FLAGS=-Og" - "-DCMAKE_CXX_FLAGS=-Og" +# "-DCMAKE_BUILD_TYPE=Debug" +# "-DCMAKE_C_FLAGS=-Og" +# "-DCMAKE_CXX_FLAGS=-Og" ) # TODO remove this? we don't use ccache anyway. An option would be to download it @@ -255,7 +255,7 @@ TESTS_TO_SKIP=( 00974_query_profiler # Look at DistributedFilesToInsert, so cannot run in parallel. - 01460_DistributedFilesToInsert + 01457_DistributedFilesToInsert ) time clickhouse-test -j 8 --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt" From 581a14be50eba7c57bdf724e75ad366ad4498c5e Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 30 Sep 2020 22:48:32 +0200 Subject: [PATCH 104/411] Adding SRS source for LDAP external user directory. Small updates to helpers/cluster.py. Updating link in the ldap/authentication/requirements/requirements.md. --- tests/testflows/helpers/cluster.py | 6 +- .../requirements/requirements.md | 7 +- .../requirements/requirements.md | 712 ++++++++++++++++++ .../requirements/requirements.py | 104 ++- .../tests/authentications.py | 2 +- .../tests/external_user_directory_config.py | 2 +- 6 files changed, 813 insertions(+), 20 deletions(-) create mode 100644 tests/testflows/ldap/external_user_directory/requirements/requirements.md diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py index 761acce40d3..8f386691638 100644 --- a/tests/testflows/helpers/cluster.py +++ b/tests/testflows/helpers/cluster.py @@ -296,12 +296,12 @@ class Cluster(object): :param steps: don't break command into steps, default: True """ debug(f"command() {node}, {command}") - with By("executing command", description=command) if steps else NullStep(): + with By("executing command", description=command, format_description=False) if steps else NullStep(): r = self.bash(node)(command, *args, **kwargs) if exitcode is not None: - with Then(f"exitcode should be {exitcode}") if steps else NullStep(): + with Then(f"exitcode should be {exitcode}", format_name=False) if steps else NullStep(): assert r.exitcode == exitcode, error(r.output) if message is not None: - with Then(f"output should contain message", description=message) if steps else NullStep(): + with Then(f"output should contain message", description=message, format_description=False) if steps else NullStep(): assert message in r.output, error(r.output) return r diff --git a/tests/testflows/ldap/authentication/requirements/requirements.md b/tests/testflows/ldap/authentication/requirements/requirements.md index 6d787670138..d322db70330 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.md +++ b/tests/testflows/ldap/authentication/requirements/requirements.md @@ -524,9 +524,6 @@ used to authenticate users using an [LDAP] server. ## References * **ClickHouse:** https://clickhouse.tech -* **GitHub repository:** https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/requirements/requirements.md -* **Revision history:** https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/requirements/requirements.md -* **Git:** https://git-scm.com/ [Anonymous Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-AnonymousAuthenticationMechanismOfSimpleBind [Unauthenticated Authentication Mechanism of Simple Bind]: https://ldapwiki.com/wiki/Simple%20Authentication#section-Simple+Authentication-UnauthenticatedAuthenticationMechanismOfSimpleBind @@ -539,6 +536,6 @@ used to authenticate users using an [LDAP] server. [LDAP]: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol [ClickHouse]: https://clickhouse.tech [GitHub]: https://github.com -[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/requirements/requirements.md -[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/requirements/requirements.md +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/authentication/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/authentication/requirements/requirements.md [Git]: https://git-scm.com/ diff --git a/tests/testflows/ldap/external_user_directory/requirements/requirements.md b/tests/testflows/ldap/external_user_directory/requirements/requirements.md new file mode 100644 index 00000000000..37a8be04b66 --- /dev/null +++ b/tests/testflows/ldap/external_user_directory/requirements/requirements.md @@ -0,0 +1,712 @@ +# SRS-009 ClickHouse LDAP External User Directory +# Software Requirements Specification + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Terminology](#terminology) + * 3.1 [LDAP](#ldap) +* 4 [Requirements](#requirements) + * 4.1 [Generic](#generic) + * 4.1.1 [User Authentication](#user-authentication) + * 4.1.1.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication](#rqsrs-009ldapexternaluserdirectoryauthentication) + * 4.1.1.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories](#rqsrs-009ldapexternaluserdirectorymultipleuserdirectories) + * 4.1.1.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories.Lookup](#rqsrs-009ldapexternaluserdirectorymultipleuserdirectorieslookup) + * 4.1.1.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Authentication.NewUsers](#rqsrs-009ldapexternaluserdirectoryusersauthenticationnewusers) + * 4.1.1.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.DeletedUsers](#rqsrs-009ldapexternaluserdirectoryauthenticationdeletedusers) + * 4.1.1.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Valid](#rqsrs-009ldapexternaluserdirectoryauthenticationvalid) + * 4.1.1.7 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Invalid](#rqsrs-009ldapexternaluserdirectoryauthenticationinvalid) + * 4.1.1.8 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.UsernameChanged](#rqsrs-009ldapexternaluserdirectoryauthenticationusernamechanged) + * 4.1.1.9 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.PasswordChanged](#rqsrs-009ldapexternaluserdirectoryauthenticationpasswordchanged) + * 4.1.1.10 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.LDAPServerRestart](#rqsrs-009ldapexternaluserdirectoryauthenticationldapserverrestart) + * 4.1.1.11 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.ClickHouseServerRestart](#rqsrs-009ldapexternaluserdirectoryauthenticationclickhouseserverrestart) + * 4.1.1.12 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel](#rqsrs-009ldapexternaluserdirectoryauthenticationparallel) + * 4.1.1.13 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.ValidAndInvalid](#rqsrs-009ldapexternaluserdirectoryauthenticationparallelvalidandinvalid) + * 4.1.1.14 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.MultipleServers](#rqsrs-009ldapexternaluserdirectoryauthenticationparallelmultipleservers) + * 4.1.1.15 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalOnly](#rqsrs-009ldapexternaluserdirectoryauthenticationparallellocalonly) + * 4.1.1.16 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalAndMultipleLDAP](#rqsrs-009ldapexternaluserdirectoryauthenticationparallellocalandmultipleldap) + * 4.1.1.17 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.SameUser](#rqsrs-009ldapexternaluserdirectoryauthenticationparallelsameuser) + * 4.1.2 [Connection](#connection) + * 4.1.2.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.PlainText](#rqsrs-009ldapexternaluserdirectoryconnectionprotocolplaintext) + * 4.1.2.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS](#rqsrs-009ldapexternaluserdirectoryconnectionprotocoltls) + * 4.1.2.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.StartTLS](#rqsrs-009ldapexternaluserdirectoryconnectionprotocolstarttls) + * 4.1.2.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.Validation](#rqsrs-009ldapexternaluserdirectoryconnectionprotocoltlscertificatevalidation) + * 4.1.2.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SelfSigned](#rqsrs-009ldapexternaluserdirectoryconnectionprotocoltlscertificateselfsigned) + * 4.1.2.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SpecificCertificationAuthority](#rqsrs-009ldapexternaluserdirectoryconnectionprotocoltlscertificatespecificcertificationauthority) + * 4.1.2.7 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Anonymous](#rqsrs-009ldapexternaluserdirectoryconnectionauthenticationmechanismanonymous) + * 4.1.2.8 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Unauthenticated](#rqsrs-009ldapexternaluserdirectoryconnectionauthenticationmechanismunauthenticated) + * 4.1.2.9 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.NamePassword](#rqsrs-009ldapexternaluserdirectoryconnectionauthenticationmechanismnamepassword) + * 4.1.2.10 [RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.UnreachableServer](#rqsrs-009ldapexternaluserdirectoryconnectionauthenticationunreachableserver) + * 4.2 [Specific](#specific) + * 4.2.1 [User Discovery](#user-discovery) + * 4.2.1.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Lookup.Priority](#rqsrs-009ldapexternaluserdirectoryuserslookuppriority) + * 4.2.2 [Roles](#roles) + * 4.2.2.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed](#rqsrs-009ldapexternaluserdirectoryroleremoved) + * 4.2.2.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed.Privileges](#rqsrs-009ldapexternaluserdirectoryroleremovedprivileges) + * 4.2.2.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Readded.Privileges](#rqsrs-009ldapexternaluserdirectoryrolereaddedprivileges) + * 4.2.2.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.New](#rqsrs-009ldapexternaluserdirectoryrolenew) + * 4.2.2.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NewPrivilege](#rqsrs-009ldapexternaluserdirectoryrolenewprivilege) + * 4.2.2.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Role.RemovedPrivilege](#rqsrs-009ldapexternaluserdirectoryroleremovedprivilege) + * 4.2.3 [Configuration](#configuration) + * 4.2.3.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationserverinvalid) + * 4.2.3.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Definition](#rqsrs-009ldapexternaluserdirectoryconfigurationserverdefinition) + * 4.2.3.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Name](#rqsrs-009ldapexternaluserdirectoryconfigurationservername) + * 4.2.3.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Host](#rqsrs-009ldapexternaluserdirectoryconfigurationserverhost) + * 4.2.3.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port](#rqsrs-009ldapexternaluserdirectoryconfigurationserverport) + * 4.2.3.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationserverportdefault) + * 4.2.3.7 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Prefix](#rqsrs-009ldapexternaluserdirectoryconfigurationserverauthdnprefix) + * 4.2.3.8 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Suffix](#rqsrs-009ldapexternaluserdirectoryconfigurationserverauthdnsuffix) + * 4.2.3.9 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Value](#rqsrs-009ldapexternaluserdirectoryconfigurationserverauthdnvalue) + * 4.2.3.10 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS](#rqsrs-009ldapexternaluserdirectoryconfigurationserverenabletls) + * 4.2.3.11 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationserverenabletlsoptionsdefault) + * 4.2.3.12 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.No](#rqsrs-009ldapexternaluserdirectoryconfigurationserverenabletlsoptionsno) + * 4.2.3.13 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Yes](#rqsrs-009ldapexternaluserdirectoryconfigurationserverenabletlsoptionsyes) + * 4.2.3.14 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.StartTLS](#rqsrs-009ldapexternaluserdirectoryconfigurationserverenabletlsoptionsstarttls) + * 4.2.3.15 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsminimumprotocolversion) + * 4.2.3.16 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Values](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsminimumprotocolversionvalues) + * 4.2.3.17 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsminimumprotocolversiondefault) + * 4.2.3.18 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecert) + * 4.2.3.19 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Default](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecertoptionsdefault) + * 4.2.3.20 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Demand](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecertoptionsdemand) + * 4.2.3.21 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Allow](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecertoptionsallow) + * 4.2.3.22 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Try](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecertoptionstry) + * 4.2.3.23 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Never](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsrequirecertoptionsnever) + * 4.2.3.24 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCertFile](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlscertfile) + * 4.2.3.25 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSKeyFile](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlskeyfile) + * 4.2.3.26 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertDir](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlscacertdir) + * 4.2.3.27 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertFile](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlscacertfile) + * 4.2.3.28 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCipherSuite](#rqsrs-009ldapexternaluserdirectoryconfigurationservertlsciphersuite) + * 4.2.3.29 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationserversyntax) + * 4.2.3.30 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectory) + * 4.2.3.31 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersldapuserdirectorymorethanone) + * 4.2.3.32 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax](#rqsrs-009ldapexternaluserdirectoryconfigurationuserssyntax) + * 4.2.3.33 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserver) + * 4.2.3.34 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverempty) + * 4.2.3.35 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermissing) + * 4.2.3.36 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersservermorethanone) + * 4.2.3.37 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersserverinvalid) + * 4.2.3.38 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersroles) + * 4.2.3.39 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmorethanone) + * 4.2.3.40 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesinvalid) + * 4.2.3.41 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesempty) + * 4.2.3.42 [RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing](#rqsrs-009ldapexternaluserdirectoryconfigurationusersparametersrolesmissing) + * 4.2.4 [Authentication](#authentication) + * 4.2.4.1 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Empty](#rqsrs-009ldapexternaluserdirectoryauthenticationusernameempty) + * 4.2.4.2 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Long](#rqsrs-009ldapexternaluserdirectoryauthenticationusernamelong) + * 4.2.4.3 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.UTF8](#rqsrs-009ldapexternaluserdirectoryauthenticationusernameutf8) + * 4.2.4.4 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Empty](#rqsrs-009ldapexternaluserdirectoryauthenticationpasswordempty) + * 4.2.4.5 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Long](#rqsrs-009ldapexternaluserdirectoryauthenticationpasswordlong) + * 4.2.4.6 [RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.UTF8](#rqsrs-009ldapexternaluserdirectoryauthenticationpasswordutf8) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Revision History]. + +## Introduction + +The [QA-SRS007 ClickHouse Authentication of Users via LDAP] enables support for authenticating +users using an [LDAP] server. This requirements specifications add addition functionality +for integrating [LDAP] with [ClickHouse]. + +This document will cover requirements to allow authenticatoin of users stored in the +external user discovery using an [LDAP] server without having to explicitly define users in [ClickHouse]'s +`users.xml` configuration file. + +## Terminology + +### LDAP + +* Lightweight Directory Access Protocol + +## Requirements + +### Generic + +#### User Authentication + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication +version: 1.0 + +[ClickHouse] SHALL support authenticating users that are defined only on the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories +version: 1.0 + +[ClickHouse] SHALL support authenticating users using multiple [LDAP] external user directories. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories.Lookup +version: 1.0 + +[ClickHouse] SHALL attempt to authenticate external [LDAP] user +using [LDAP] external user directory in the same order +in which user directories are specified in the `config.xml` file. +If a user cannot be authenticated using the first [LDAP] external user directory +then the next user directory in the list SHALL be used. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Authentication.NewUsers +version: 1.0 + +[ClickHouse] SHALL support authenticating users that are defined only on the [LDAP] server +as soon as they are added to the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.DeletedUsers +version: 1.0 + +[ClickHouse] SHALL not allow authentication of users that +were previously defined only on the [LDAP] server but were removed +from the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Valid +version: 1.0 + +[ClickHouse] SHALL only allow user authentication using [LDAP] server if and only if +user name and password match [LDAP] server records for the user +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if either user name or password +do not match [LDAP] server records for the user +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.UsernameChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the username is changed +on the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.PasswordChanged +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication if the password +for the user is changed on the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.LDAPServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after [LDAP] server is restarted +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.ClickHouseServerRestart +version: 1.0 + +[ClickHouse] SHALL support authenticating users after server is restarted +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of users using [LDAP] server +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.ValidAndInvalid +version: 1.0 + +[ClickHouse] SHALL support authentication of valid users and +prohibit authentication of invalid users using [LDAP] server +in parallel without having invalid attempts affecting valid authentications +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.MultipleServers +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of external [LDAP] users +authenticated using multiple [LDAP] external user directories. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalOnly +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of users defined only locally +when one or more [LDAP] external user directories are specified in the configuration file. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalAndMultipleLDAP +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of local and external [LDAP] users +authenticated using multiple [LDAP] external user directories. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.SameUser +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of the same external [LDAP] user +authenticated using the same [LDAP] external user directory. + +#### Connection + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.PlainText +version: 1.0 + +[ClickHouse] SHALL support user authentication using plain text `ldap://` non secure protocol +while connecting to the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using `SSL/TLS` `ldaps://` secure protocol +while connecting to the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.StartTLS +version: 1.0 + +[ClickHouse] SHALL support user authentication using legacy `StartTLS` protocol which is a +plain text `ldap://` protocol that is upgraded to [TLS] when connecting to the [LDAP] server +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.Validation +version: 1.0 + +[ClickHouse] SHALL support certificate validation used for [TLS] connections +to the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SelfSigned +version: 1.0 + +[ClickHouse] SHALL support self-signed certificates for [TLS] connections +to the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.TLS.Certificate.SpecificCertificationAuthority +version: 1.0 + +[ClickHouse] SHALL support certificates signed by specific Certification Authority for [TLS] connections +to the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Anonymous +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Anonymous Authentication Mechanism of Simple Bind] +authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.Unauthenticated +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit authentication using [Unauthenticated Authentication Mechanism of Simple Bind] +authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.Mechanism.NamePassword +version: 1.0 + +[ClickHouse] SHALL allow authentication using only [Name/Password Authentication Mechanism of Simple Bind] +authentication mechanism when connecting to the [LDAP] server when using [LDAP] external server directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Authentication.UnreachableServer +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server is unreachable +when using [LDAP] external user directory. + +### Specific + +#### User Discovery + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Lookup.Priority +version: 2.0 + +[ClickHouse] SHALL lookup user presence in the same order +as user directories are defined in the `config.xml`. + +#### Roles + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed +version: 1.0 + +[ClickHouse] SHALL reject authentication attempt if any of the roles that are specified in the configuration +of the external user directory are not defined at the time of the authentication attempt +with an exception that if a user was able to authenticate in past and its internal user object was created and cached +then the user SHALL be able to authenticate again, even if one of the roles is missing. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Removed.Privileges +version: 1.0 + +[ClickHouse] SHALL remove the privileges provided by the role from all the LDAP +users authenticated using external user directory if it is removed +including currently cached users that are still able to authenticated where the removed +role is specified in the configuration of the external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.Readded.Privileges +version: 1.0 + +[ClickHouse] SHALL reassign the role and add the privileges provided by the role +when it is re-added after removal for all LDAP users authenticated using external user directory +including any cached users where the re-added role was specified in the configuration of the external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.New +version: 1.0 + +[ClickHouse] SHALL not allow any new roles to be assigned to any LDAP +users authenticated using external user directory unless the role is specified +in the configuration of the external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.NewPrivilege +version: 1.0 + +[ClickHouse] SHALL add new privilege to all the LDAP users authenticated using external user directory +including cached users when new privilege is added to one of the roles specified +in the configuration of the external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Role.RemovedPrivilege +version: 1.0 + +[ClickHouse] SHALL remove privilege from all the LDAP users authenticated using external user directory +including cached users when privilege is removed from all the roles specified +in the configuration of the external user directory. + +#### Configuration + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error and prohibit user login if [LDAP] server configuration is not valid. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Definition +version: 1.0 + +[ClickHouse] SHALL support using the [LDAP] servers defined in the +`ldap_servers` section of the `config.xml` as the server to be used +for a external user directory that uses an [LDAP] server as a source of user definitions. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Name +version: 1.0 + +[ClickHouse] SHALL not support empty string as a server name. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Host +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] +server hostname or IP, this parameter SHALL be mandatory and SHALL not be empty. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [LDAP] server port. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Port.Default +version: 1.0 + +[ClickHouse] SHALL use default port number `636` if `enable_tls` is set to `yes` or `389` otherwise. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Prefix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the prefix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Suffix +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify the suffix +of value used to construct the DN to bound to during authentication via [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.AuthDN.Value +version: 1.0 + +[ClickHouse] SHALL construct DN as `auth_dn_prefix + escape(user_name) + auth_dn_suffix` string. + +> This implies that auth_dn_suffix should usually have comma ',' as its first non-space character. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS +version: 1.0 + +[ClickHouse] SHALL support `` parameter to trigger the use of secure connection to the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `yes` value as the default for `` parameter +to enable SSL/TLS `ldaps://` protocol. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.No +version: 1.0 + +[ClickHouse] SHALL support specifying `no` as the value of `` parameter to enable +plain text `ldap://` protocol. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.Yes +version: 1.0 + +[ClickHouse] SHALL support specifying `yes` as the value of `` parameter to enable +SSL/TLS `ldaps://` protocol. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.EnableTLS.Options.StartTLS +version: 1.0 + +[ClickHouse] SHALL support specifying `starttls` as the value of `` parameter to enable +legacy `StartTLS` protocol that used plain text `ldap://` protocol, upgraded to [TLS]. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify +the minimum protocol version of SSL/TLS. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Values +version: 1.0 + +[ClickHouse] SHALL support specifying `ssl2`, `ssl3`, `tls1.0`, `tls1.1`, and `tls1.2` +as a value of the `` parameter. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSMinimumProtocolVersion.Default +version: 1.0 + +[ClickHouse] SHALL set `tls1.2` as the default value of the `` parameter. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify [TLS] peer +certificate verification behavior. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Default +version: 1.0 + +[ClickHouse] SHALL use `demand` value as the default for the `` parameter. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Demand +version: 1.0 + +[ClickHouse] SHALL support specifying `demand` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, or a bad certificate is +provided, the session SHALL be immediately terminated. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Allow +version: 1.0 + +[ClickHouse] SHALL support specifying `allow` as the value of `` parameter to +enable requesting of client certificate. If no +certificate is provided, the session SHALL proceed normally. +If a bad certificate is provided, it SHALL be ignored and the session SHALL proceed normally. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Try +version: 1.0 + +[ClickHouse] SHALL support specifying `try` as the value of `` parameter to +enable requesting of client certificate. If no certificate is provided, the session +SHALL proceed normally. If a bad certificate is provided, the session SHALL be +immediately terminated. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSRequireCert.Options.Never +version: 1.0 + +[ClickHouse] SHALL support specifying `never` as the value of `` parameter to +disable requesting of client certificate. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCertFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to certificate file used by +[ClickHouse] to establish connection with the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSKeyFile +version: 1.0 + +[ClickHouse] SHALL support `` to specify the path to key file for the certificate +specified by the `` parameter. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertDir +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify to a path to +the directory containing [CA] certificates used to verify certificates provided by the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCACertFile +version: 1.0 + +[ClickHouse] SHALL support `` parameter to specify a path to a specific +[CA] certificate file used to verify certificates provided by the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.TLSCipherSuite +version: 1.0 + +[ClickHouse] SHALL support `tls_cipher_suite` parameter to specify allowed cipher suites. +The value SHALL use the same format as the `ciphersuites` in the [OpenSSL Ciphers]. + +For example, + +```xml +ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 +``` + +The available suites SHALL depend on the [OpenSSL] library version and variant used to build +[ClickHouse] and therefore might change. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Server.Syntax +version: 1.0 + +[ClickHouse] SHALL support the following example syntax to create an entry for an [LDAP] server inside the `config.xml` +configuration file or of any configuration file inside the `config.d` directory. + +```xml + + + localhost + 636 + cn= + , ou=users, dc=example, dc=com + yes + tls1.2 + demand + /path/to/tls_cert_file + /path/to/tls_key_file + /path/to/tls_ca_cert_file + /path/to/tls_ca_cert_dir + ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:AES256-GCM-SHA384 + + +``` + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory +version: 1.0 + +[ClickHouse] SHALL support `` sub-section in the `` section of the `config.xml` +that SHALL define a external user directory that uses an [LDAP] server as a source of user definitions. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne +version: 2.0 + +[ClickHouse] SHALL support more than one `` sub-sections in the `` section of the `config.xml` +that SHALL allow to define more than one external user directory that use an [LDAP] server as a source +of user definitions. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Syntax +version: 1.0 + +[ClickHouse] SHALL support `` section with the following syntax + +```xml + + + + my_ldap_server + + + + + + + +``` + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server +version: 1.0 + +[ClickHouse] SHALL support `server` parameter in the `` sub-section in the `` +section of the `config.xml` that SHALL specify one of LDAP server names +defined in `` section. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Empty +version: 1.0 + +[ClickHouse] SHALL return an error if the `server` parameter in the `` sub-section in the `` +is empty. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Missing +version: 1.0 + +[ClickHouse] SHALL return an error if the `server` parameter in the `` sub-section in the `` +is missing. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.MoreThanOne +version: 1.0 + +[ClickHouse] SHALL only use the first definitition of the `server` parameter in the `` sub-section in the `` +if more than one `server` parameter is defined in the configuration. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Server.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the server specified as the value of the `` +parameter is not defined. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles +version: 1.0 + +[ClickHouse] SHALL support `roles` parameter in the `` sub-section in the `` +section of the `config.xml` that SHALL specify the names of a locally defined roles that SHALL +be assigned to all users retrieved from the [LDAP] server. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.MoreThanOne +version: 1.0 + +[ClickHouse] SHALL only use the first definitition of the `roles` parameter +in the `` sub-section in the `` +if more than one `roles` parameter is defined in the configuration. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Invalid +version: 1.0 + +[ClickHouse] SHALL return an error if the role specified in the `` +parameter does not exist locally. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Empty +version: 1.0 + +[ClickHouse] SHALL not allow users authenticated using LDAP external user directory +to perform any action if the `roles` parameter in the `` sub-section in the `` +section is empty. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.Parameters.Roles.Missing +version: 1.0 + +[ClickHouse] SHALL not allow users authenticated using LDAP external user directory +to perform any action if the `roles` parameter in the `` sub-section in the `` +section is missing. + +#### Authentication + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty username +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.Long +version: 1.0 + +[ClickHouse] SHALL support authenticating users with a long username of at least 256 bytes +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Username.UTF8 +version: 1.0 + +[ClickHouse] SHALL support authentication users with a username that contains [UTF-8] characters +when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Empty +version: 1.0 + +[ClickHouse] SHALL not support authenticating users with empty passwords +even if an empty password is valid for the user and +is allowed by the [LDAP] server when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.Long +version: 1.0 + +[ClickHouse] SHALL support long password of at least 256 bytes +that can be used to authenticate users when using [LDAP] external user directory. + +##### RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Password.UTF8 +version: 1.0 + +[ClickHouse] SHALL support [UTF-8] characters in passwords +used to authenticate users when using [LDAP] external user directory. + +## References + +* **Access Control and Account Management**: https://clickhouse.tech/docs/en/operations/access-rights/ +* **LDAP**: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol +* **ClickHouse:** https://clickhouse.tech + +[SRS]: #srs +[Access Control and Account Management]: https://clickhouse.tech/docs/en/operations/access-rights/ +[SRS-007 ClickHouse Authentication of Users via LDAP]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/authentication/requirements/requirements.md +[LDAP]: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol +[ClickHouse]: https://clickhouse.tech +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/ldap/external_user_directory/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/ldap/external_user_directory/requirements/requirements.md +[Git]: https://git-scm.com/ +[GitHub]: https://github.com diff --git a/tests/testflows/ldap/external_user_directory/requirements/requirements.py b/tests/testflows/ldap/external_user_directory/requirements/requirements.py index 7e3ced037fa..bb21f7d3eb9 100644 --- a/tests/testflows/ldap/external_user_directory/requirements/requirements.py +++ b/tests/testflows/ldap/external_user_directory/requirements/requirements.py @@ -1,6 +1,6 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.200827.1211600. +# document by TestFlows v1.6.200929.1033606. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. from testflows.core import Requirement @@ -18,6 +18,36 @@ RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication = Requirement( link=None ) +RQ_SRS_009_LDAP_ExternalUserDirectory_MultipleUserDirectories = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support authenticating users using multiple [LDAP] external user directories.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_MultipleUserDirectories_Lookup = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.MultipleUserDirectories.Lookup', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL attempt to authenticate external [LDAP] user\n' + 'using [LDAP] external user directory in the same order\n' + 'in which user directories are specified in the `config.xml` file.\n' + 'If a user cannot be authenticated using the first [LDAP] external user directory\n' + 'then the next user directory in the list SHALL be used.\n' + ), + link=None + ) + RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Authentication_NewUsers = Requirement( name='RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Authentication.NewUsers', version='1.0', @@ -163,6 +193,62 @@ RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid = link=None ) +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_MultipleServers = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.MultipleServers', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support parallel authentication of external [LDAP] users\n' + 'authenticated using multiple [LDAP] external user directories.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalOnly = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalOnly', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support parallel authentication of users defined only locally\n' + 'when one or more [LDAP] external user directories are specified in the configuration file.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalAndMultipleLDAP = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.LocalAndMultipleLDAP', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support parallel authentication of local and external [LDAP] users\n' + 'authenticated using multiple [LDAP] external user directories.\n' + ), + link=None + ) + +RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_SameUser = Requirement( + name='RQ.SRS-009.LDAP.ExternalUserDirectory.Authentication.Parallel.SameUser', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support parallel authentication of the same external [LDAP] user\n' + 'authenticated using the same [LDAP] external user directory.\n' + ), + link=None + ) + RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Protocol_PlainText = Requirement( name='RQ.SRS-009.LDAP.ExternalUserDirectory.Connection.Protocol.PlainText', version='1.0', @@ -306,17 +392,14 @@ RQ_SRS_009_LDAP_ExternalUserDirectory_Connection_Authentication_UnreachableServe RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Lookup_Priority = Requirement( name='RQ.SRS-009.LDAP.ExternalUserDirectory.Users.Lookup.Priority', - version='1.0', + version='2.0', priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL lookup user presence in the following priority:\n' - '\n' - '1. access control\n' - '2. `users.xml`\n' - '3. LDAP\n' + '[ClickHouse] SHALL lookup user presence in the same order\n' + 'as user directories are defined in the `config.xml`.\n' ), link=None ) @@ -863,14 +946,15 @@ RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory = Re RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory_MoreThanOne = Requirement( name='RQ.SRS-009.LDAP.ExternalUserDirectory.Configuration.Users.LDAPUserDirectory.MoreThanOne', - version='1.0', + version='2.0', priority=None, group=None, type=None, uid=None, description=( - '[ClickHouse] SHALL only use the first `` sub-section in the `` section of the `config.xml`\n' - 'if more than one `` sub-sections are present.\n' + '[ClickHouse] SHALL support more than one `` sub-sections in the `` section of the `config.xml`\n' + 'that SHALL allow to define more than one external user directory that use an [LDAP] server as a source\n' + 'of user definitions.\n' ), link=None ) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index e4df466b1ea..55290d1ada9 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -450,7 +450,7 @@ def empty_username_and_empty_password(self, server=None): @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Lookup_Priority("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Lookup_Priority("2.0") ) def user_lookup_priority(self, server): """Check that users are looked up in the same priority diff --git a/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py b/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py index d95a4a674a1..b5677eba4b2 100644 --- a/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py +++ b/tests/testflows/ldap/external_user_directory/tests/external_user_directory_config.py @@ -5,7 +5,7 @@ from ldap.external_user_directory.requirements import * @TestScenario @Requirements( - RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory_MoreThanOne("1.0") + RQ_SRS_009_LDAP_ExternalUserDirectory_Configuration_Users_LDAPUserDirectory_MoreThanOne("2.0") ) def more_than_one_user_directory(self, timeout=20): """Check when more than one LDAP user directory is From 8a707b1bb42013c3b83d225289c3937098520906 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Wed, 30 Sep 2020 21:40:07 -0400 Subject: [PATCH 105/411] Adding `paralle login of rbac users` test. --- .../tests/authentications.py | 105 ++++++++++++------ .../external_user_directory/tests/common.py | 23 +++- 2 files changed, 89 insertions(+), 39 deletions(-) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index 55290d1ada9..cdace3007fa 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -52,57 +52,92 @@ def add_user_to_ldap_and_login(self, server, user=None, ch_user=None, login=None login_and_execute_query(username=username, password=password, exitcode=exitcode, message=message) +def login_with_valid_username_and_password(users, i, iterations=10): + """Login with valid username and password. + """ + with When(f"valid users try to login #{i}"): + for i in range(iterations): + random_user = users[random.randint(0, len(users)-1)] + login_and_execute_query(username=random_user["cn"], password=random_user["userpassword"], steps=False) + +def login_with_valid_username_and_invalid_password(users, i, iterations=10): + """Login with valid username and invalid password. + """ + with When(f"users try to login with valid username and invalid password #{i}"): + for i in range(iterations): + random_user = users[random.randint(0, len(users)-1)] + login_and_execute_query(username=random_user["cn"], + password=(random_user["userpassword"] + randomword(1)), + exitcode=4, + message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", + steps=False) + +def login_with_invalid_username_and_valid_password(users, i, iterations=10): + """Login with invalid username and valid password. + """ + with When(f"users try to login with invalid username and valid password #{i}"): + for i in range(iterations): + random_user = dict(users[random.randint(0, len(users)-1)]) + random_user["cn"] += randomword(1) + login_and_execute_query(username=random_user["cn"], + password=random_user["userpassword"], + exitcode=4, + message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", + steps=False) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel("1.0"), RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") ) def parallel_login(self, server, user_count=10, timeout=200): - """Check that login of valid and invalid LDAP authenticated users works in parallel.""" + """Check that login of valid and invalid LDAP authenticated users works in parallel. + """ self.context.ldap_node = self.context.cluster.node(server) user = None users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] with ldap_users(*users): - def login_with_valid_username_and_password(users, i, iterations=10): - with When(f"valid users try to login #{i}"): - for i in range(iterations): - random_user = users[random.randint(0, len(users)-1)] - login_and_execute_query(username=random_user["cn"], password=random_user["userpassword"], steps=False) + tasks = [] + try: + with When("I login in parallel"): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) - def login_with_valid_username_and_invalid_password(users, i, iterations=10): - with When(f"users try to login with valid username and invalid password #{i}"): - for i in range(iterations): - random_user = users[random.randint(0, len(users)-1)] - login_and_execute_query(username=random_user["cn"], - password=(random_user["userpassword"] + randomword(1)), - exitcode=4, - message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", - steps=False) + finally: + with Then("it should work"): + join(tasks, timeout) - def login_with_invalid_username_and_valid_password(users, i, iterations=10): - with When(f"users try to login with invalid username and valid password #{i}"): - for i in range(iterations): - random_user = dict(users[random.randint(0, len(users)-1)]) - random_user["cn"] += randomword(1) - login_and_execute_query(username=random_user["cn"], - password=random_user["userpassword"], - exitcode=4, - message=f"DB::Exception: {random_user['cn']}: Authentication failed: password is incorrect or there is no user with such name", - steps=False) +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalOnly("1.0") +) +def parallel_login_of_rbac_users(self, server, user_count=10, timeout=200): + """Check that login of only valid and invalid local users created using RBAC + works in parallel when server configuration includes LDAP external user directory. + """ + self.context.ldap_node = self.context.cluster.node(server) + user = None - with When("I login in parallel"): - p = Pool(15) - tasks = [] - for i in range(5): - tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) - tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] - with Then("it should work"): - for task in tasks: - task.get(timeout=timeout) + with rbac_users(*users): + tasks = [] + + try: + with When("I login in parallel"): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + finally: + with Then("it should work"): + join(tasks, timeout) @TestScenario @Requirements( diff --git a/tests/testflows/ldap/external_user_directory/tests/common.py b/tests/testflows/ldap/external_user_directory/tests/common.py index 4c7877035c8..57e5cc34a94 100644 --- a/tests/testflows/ldap/external_user_directory/tests/common.py +++ b/tests/testflows/ldap/external_user_directory/tests/common.py @@ -11,6 +11,21 @@ from ldap.authentication.tests.common import ldap_user, ldap_users, add_user_to_ from ldap.authentication.tests.common import change_user_password_in_ldap, change_user_cn_in_ldap from ldap.authentication.tests.common import randomword +def join(tasks, timeout): + """Join async tasks by waiting for their completion. + """ + task_exc = None + + for task in tasks: + try: + task.get(timeout=timeout) + except Exception as exc: + if task_exc is None: + task_exc = exc + + if task_exc is not None: + raise task_exc + @contextmanager def table(name, create_statement, on_cluster=False): node = current().context.node @@ -31,14 +46,14 @@ def rbac_users(*users): try: with Given("I have local users"): for user in users: - with By(f"creating user {user}"): - node.query(f"CREATE USER OR REPLACE {user} IDENTIFIED WITH PLAINTEXT_PASSWORD BY '{user}'") + with By(f"creating user {user['cn']}", format_name=False): + node.query(f"CREATE USER OR REPLACE {user['cn']} IDENTIFIED WITH PLAINTEXT_PASSWORD BY '{user['userpassword']}'") yield users finally: with Finally("I drop local users"): for user in users: - with By(f"dropping user {user}", flags=TE): - node.query(f"DROP USER IF EXISTS {user}") + with By(f"dropping user {user['cn']}", flags=TE, format_name=False): + node.query(f"DROP USER IF EXISTS {user['cn']}") @contextmanager def rbac_roles(*roles): From 74f8e41b55a2504c3419cb33e7c2429a2c1e116a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 1 Oct 2020 13:56:56 +0300 Subject: [PATCH 106/411] calculate on all nodes --- docker/test/performance-comparison/compare.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index 2f03ecc9ad7..08b18758874 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -462,7 +462,10 @@ wait unset IFS ) -parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log +# The comparison script might be bound to one NUMA node for better test +# stability, and the calculation runs out of memory because of this. Use +# all nodes. +numactl --all parallel --joblog analyze/parallel-log.txt --null < analyze/commands.txt 2>> analyze/errors.log clickhouse-local --query " -- Join the metric names back to the metric statistics we've calculated, and make From 80c334a5ace304b17b39a32a128911188ea6c23e Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 1 Oct 2020 07:21:38 -0400 Subject: [PATCH 107/411] Fixing user_lookup_priority test. --- .../ldap/external_user_directory/tests/authentications.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index cdace3007fa..971243b1d48 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -127,7 +127,6 @@ def parallel_login_of_rbac_users(self, server, user_count=10, timeout=200): with rbac_users(*users): tasks = [] - try: with When("I login in parallel"): p = Pool(15) @@ -509,7 +508,7 @@ def user_lookup_priority(self, server): } with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users.values()]): - with rbac_users("local"): + with rbac_users({"cn": "local", "userpassword": "local"}): with When("I try to login as 'default' user which is also defined in users.xml it should fail"): login_and_execute_query(**users["default"], exitcode=exitcode, message=message.format(username="default")) From ab2c37cead84d29815e1620a7414b98600527cdb Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sat, 3 Oct 2020 00:31:14 +0400 Subject: [PATCH 108/411] Serialize all calls to ldap lib --- src/Access/LDAPClient.cpp | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/Access/LDAPClient.cpp b/src/Access/LDAPClient.cpp index a85e96ab86c..d6580b89c68 100644 --- a/src/Access/LDAPClient.cpp +++ b/src/Access/LDAPClient.cpp @@ -2,6 +2,8 @@ #include #include +#include + #include #include @@ -27,16 +29,13 @@ LDAPClient::~LDAPClient() closeConnection(); } -void LDAPClient::openConnection() -{ - const bool graceful_bind_failure = false; - diag(openConnection(graceful_bind_failure)); -} - #if USE_LDAP namespace { + + std::recursive_mutex ldap_global_mutex; + auto escapeForLDAP(const String & src) { String dest; @@ -63,10 +62,13 @@ namespace return dest; } + } void LDAPClient::diag(const int rc) { + std::scoped_lock lock(ldap_global_mutex); + if (rc != LDAP_SUCCESS) { String text; @@ -100,8 +102,18 @@ void LDAPClient::diag(const int rc) } } +void LDAPClient::openConnection() +{ + std::scoped_lock lock(ldap_global_mutex); + + const bool graceful_bind_failure = false; + diag(openConnection(graceful_bind_failure)); +} + int LDAPClient::openConnection(const bool graceful_bind_failure) { + std::scoped_lock lock(ldap_global_mutex); + closeConnection(); { @@ -258,6 +270,8 @@ int LDAPClient::openConnection(const bool graceful_bind_failure) void LDAPClient::closeConnection() noexcept { + std::scoped_lock lock(ldap_global_mutex); + if (!handle) return; @@ -267,6 +281,8 @@ void LDAPClient::closeConnection() noexcept bool LDAPSimpleAuthClient::check() { + std::scoped_lock lock(ldap_global_mutex); + if (params.user.empty()) throw Exception("LDAP authentication of a user with an empty name is not allowed", ErrorCodes::BAD_ARGUMENTS); @@ -312,6 +328,11 @@ void LDAPClient::diag(const int) throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); } +void LDAPClient::openConnection() +{ + throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); +} + int LDAPClient::openConnection(const bool) { throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); From 68ccd59a7495f5e1592269cd6d12f31cb1de2f93 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sat, 3 Oct 2020 00:32:13 +0400 Subject: [PATCH 109/411] Synch with internal memory_storage Fix exception message --- src/Access/LDAPAccessStorage.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index ae61d912ebe..38922eeac55 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -69,6 +69,7 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr & entity) { + std::scoped_lock lock(mutex); auto role_ptr = typeid_cast>(entity); if (role_ptr) { @@ -123,6 +124,7 @@ const char * LDAPAccessStorage::getStorageType() const String LDAPAccessStorage::getStorageParamsJSON() const { + std::scoped_lock lock(mutex); Poco::JSON::Object params_json; params_json.set("server", ldap_server); @@ -137,30 +139,35 @@ String LDAPAccessStorage::getStorageParamsJSON() const std::optional LDAPAccessStorage::findImpl(EntityType type, const String & name) const { + std::scoped_lock lock(mutex); return memory_storage.find(type, name); } std::vector LDAPAccessStorage::findAllImpl(EntityType type) const { + std::scoped_lock lock(mutex); return memory_storage.findAll(type); } bool LDAPAccessStorage::existsImpl(const UUID & id) const { + std::scoped_lock lock(mutex); return memory_storage.exists(id); } AccessEntityPtr LDAPAccessStorage::readImpl(const UUID & id) const { + std::scoped_lock lock(mutex); return memory_storage.read(id); } String LDAPAccessStorage::readNameImpl(const UUID & id) const { + std::scoped_lock lock(mutex); return memory_storage.readName(id); } @@ -179,6 +186,7 @@ UUID LDAPAccessStorage::insertImpl(const AccessEntityPtr & entity, bool) void LDAPAccessStorage::removeImpl(const UUID & id) { + std::scoped_lock lock(mutex); auto entity = read(id); throwReadonlyCannotRemove(entity->getType(), entity->getName()); } @@ -186,6 +194,7 @@ void LDAPAccessStorage::removeImpl(const UUID & id) void LDAPAccessStorage::updateImpl(const UUID & id, const UpdateFunc &) { + std::scoped_lock lock(mutex); auto entity = read(id); throwReadonlyCannotUpdate(entity->getType(), entity->getName()); } @@ -193,24 +202,28 @@ void LDAPAccessStorage::updateImpl(const UUID & id, const UpdateFunc &) ext::scope_guard LDAPAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const { + std::scoped_lock lock(mutex); return memory_storage.subscribeForChanges(id, handler); } ext::scope_guard LDAPAccessStorage::subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const { + std::scoped_lock lock(mutex); return memory_storage.subscribeForChanges(type, handler); } bool LDAPAccessStorage::hasSubscriptionImpl(const UUID & id) const { + std::scoped_lock lock(mutex); return memory_storage.hasSubscription(id); } bool LDAPAccessStorage::hasSubscriptionImpl(EntityType type) const { + std::scoped_lock lock(mutex); return memory_storage.hasSubscription(type); } @@ -261,7 +274,7 @@ UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & passw } catch (...) { - tryLogCurrentException(getLogger(), "Authentication failed for user '" + user_name + "' from access storage '" + access_control_manager->getStorageName() + "'"); + tryLogCurrentException(getLogger(), "Authentication failed for user '" + user_name + "' from access storage '" + getStorageName() + "'"); } throwCannotAuthenticate(user_name); } From 8ee10a179037b08cde164a070d1f154386c13e64 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sat, 3 Oct 2020 00:49:51 +0400 Subject: [PATCH 110/411] Merge artefact --- contrib/grpc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/grpc b/contrib/grpc index 8aea4e168e7..a6570b863cf 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit 8aea4e168e78f3eb9828080740fc8cb73d53bf79 +Subproject commit a6570b863cf76c9699580ba51c7827d5bffaac43 From 92ec9e871684c4403d4d7c3888b59f31f1c06cd1 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sat, 3 Oct 2020 01:40:23 +0400 Subject: [PATCH 111/411] Merge artefact --- contrib/cyrus-sasl | 2 +- contrib/poco | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/cyrus-sasl b/contrib/cyrus-sasl index 6054630889f..9995bf9d8e1 160000 --- a/contrib/cyrus-sasl +++ b/contrib/cyrus-sasl @@ -1 +1 @@ -Subproject commit 6054630889fd1cd8d0659573d69badcee1e23a00 +Subproject commit 9995bf9d8e14f58934d9313ac64f13780d6dd3c9 diff --git a/contrib/poco b/contrib/poco index 297fc905e16..757d947235b 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 297fc905e166392156f83b96aaa5f44e8a6a35c4 +Subproject commit 757d947235b307675cff964f29b19d388140a9eb From eb1b3e39e7edf834758ae9713da26aa1b52ee901 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Fri, 2 Oct 2020 17:40:24 -0400 Subject: [PATCH 112/411] Adding additional parallel login tests to the external user directory feature. --- .../ldap/authentication/tests/common.py | 85 ++++--- .../tests/authentications.py | 209 +++++++++++++++++- 2 files changed, 254 insertions(+), 40 deletions(-) diff --git a/tests/testflows/ldap/authentication/tests/common.py b/tests/testflows/ldap/authentication/tests/common.py index 9e8e3afcd1d..979711652db 100644 --- a/tests/testflows/ldap/authentication/tests/common.py +++ b/tests/testflows/ldap/authentication/tests/common.py @@ -54,6 +54,48 @@ def add_config(config, timeout=20, restart=False): :param config: configuration file description :param timeout: timeout, default: 20 sec """ + def check_preprocessed_config_is_updated(): + """Check that preprocessed config is updated. + """ + started = time.time() + command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + + def wait_for_config_to_be_loaded(): + """Wait for config to be loaded. + """ + if restart: + with When("I close terminal to the node to be restarted"): + bash.close() + + with And("I get the current log size"): + logsize = \ + node.command("ls -s --block-size=1 /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[ + 0].strip() + + with And("I restart ClickHouse to apply the config changes"): + node.restart(safe=False) + + with Then("I tail the log file from using previous log size as the offset"): + bash.prompt = bash.__class__.prompt + bash.open() + bash.send(f"tail -c +{logsize} -f /var/log/clickhouse-server/clickhouse-server.log") + + with Then("I wait for config reload message in the log file"): + if restart: + bash.expect( + f"ConfigReloader: Loaded config '/etc/clickhouse-server/config.xml', performed update on configuration", + timeout=timeout) + else: + bash.expect( + f"ConfigReloader: Loaded config '/etc/clickhouse-server/{config.preprocessed_name}', performed update on configuration", + timeout=timeout) + node = current().context.node try: with Given(f"{config.name}"): @@ -70,29 +112,10 @@ def add_config(config, timeout=20, restart=False): node.command(command, steps=False, exitcode=0) with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): - started = time.time() - command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep {config.uid}{' > /dev/null' if not settings.debug else ''}" - while time.time() - started < timeout: - exitcode = node.command(command, steps=False).exitcode - if exitcode == 0: - break - time.sleep(1) - assert exitcode == 0, error() + check_preprocessed_config_is_updated() - if restart: - bash.close() - logsize = node.command("ls -s --block-size=1 /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[0].strip() - with When("I restart ClickHouse to apply the config changes"): - node.restart(safe=False) - bash.prompt = bash.__class__.prompt - bash.open() - bash.send(f"tail -c +{logsize} -f /var/log/clickhouse-server/clickhouse-server.log") - - with When("I wait for config to be loaded"): - if restart: - bash.expect(f"ConfigReloader: Loaded config '/etc/clickhouse-server/config.xml', performed update on configuration", timeout=timeout) - else: - bash.expect(f"ConfigReloader: Loaded config '/etc/clickhouse-server/{config.preprocessed_name}', performed update on configuration", timeout=timeout) + with And("I wait for config to be reloaded"): + wait_for_config_to_be_loaded() yield finally: with Finally(f"I remove {config.name}"): @@ -103,20 +126,11 @@ def add_config(config, timeout=20, restart=False): with By("removing the config file", description=config.path): node.command(f"rm -rf {config.path}", exitcode=0) - with Then(f"{config.preprocessed_name} should be updated"): - started = time.time() - command = f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name} | grep '{config.uid}'{' > /dev/null' if not settings.debug else ''}" - while time.time() - started < timeout: - exitcode = node.command(command, steps=False).exitcode - if exitcode == 1: - break - time.sleep(1) - assert exitcode == 1, error() - - with When("I wait for config to be loaded"): - started = time.time() - bash.expect(f"ConfigReloader: Loaded config '/etc/clickhouse-server/{config.preprocessed_name}', performed update on configuration", timeout=timeout) + with Then(f"{config.preprocessed_name} should be updated", description=f"timeout {timeout}"): + check_preprocessed_config_is_updated() + with And("I wait for config to be reloaded"): + wait_for_config_to_be_loaded() def create_ldap_servers_config_content(servers, config_d_dir="/etc/clickhouse-server/config.d", config_file="ldap_servers.xml"): """Create LDAP servers configuration content. @@ -288,6 +302,7 @@ def add_user_to_ldap(cn, userpassword, givenname=None, homedirectory=None, sn=No } lines = [] + for key, value in list(user.items()): if key.startswith("_"): continue diff --git a/tests/testflows/ldap/external_user_directory/tests/authentications.py b/tests/testflows/ldap/external_user_directory/tests/authentications.py index 971243b1d48..cf8620b3dee 100644 --- a/tests/testflows/ldap/external_user_directory/tests/authentications.py +++ b/tests/testflows/ldap/external_user_directory/tests/authentications.py @@ -96,12 +96,17 @@ def parallel_login(self, server, user_count=10, timeout=200): self.context.ldap_node = self.context.cluster.node(server) user = None - users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] + with Given("a group of LDAP users"): + users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] with ldap_users(*users): tasks = [] try: - with When("I login in parallel"): + with When("users try to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): p = Pool(15) for i in range(25): tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) @@ -112,11 +117,207 @@ def parallel_login(self, server, user_count=10, timeout=200): with Then("it should work"): join(tasks, timeout) +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_SameUser("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") +) +def parallel_login_with_the_same_user(self, server, timeout=200): + """Check that valid and invalid logins of the same + LDAP authenticated user works in parallel. + """ + self.context.ldap_node = self.context.cluster.node(server) + user = None + + with Given("only one LDAP user"): + users = [{"cn": f"parallel_user1", "userpassword": randomword(20)}] + + with ldap_users(*users): + tasks = [] + try: + with When("the same user tries to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + + finally: + with Then("it should work"): + join(tasks, timeout) + +@TestScenario +def login_after_ldap_external_user_directory_is_removed(self, server): + """Check that ClickHouse stops authenticating LDAP users + after LDAP external user directory is removed. + """ + with When("I attempt to login after LDAP external user directory is added"): + with ldap_external_user_directory(server="openldap2", roles=[], restart=True): + login_and_execute_query(username="user2", password="user2") + + with When("I attempt to login after LDAP external user directory is removed"): + exitcode = 4 + message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" + login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_SameUser("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") +) +def parallel_login_with_the_same_user_multiple_servers(self, server, timeout=200): + """Check that valid and invalid logins of the same + user defined in multiple LDAP external user directories + works in parallel. + """ + with Given("I have two LDAP servers"): + entries = [ + (["openldap1"], []), + (["openldap2"], []) + ] + + with Given("I define only one LDAP user"): + users = [{"cn": f"parallel_user1", "userpassword": randomword(20)}] + + with And("I create config file to define LDAP external user directory for each LDAP server"): + config = create_entries_ldap_external_user_directory_config_content(entries) + + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with ldap_users(*users, node=self.context.cluster.node("openldap1")): + with ldap_users(*users, node=self.context.cluster.node("openldap2")): + tasks = [] + try: + with When("the same user tries to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): + p = Pool(15) + for i in range(25): + tasks.append(p.apply_async(login_with_valid_username_and_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_valid_username_and_invalid_password, (users, i, 50,))) + tasks.append(p.apply_async(login_with_invalid_username_and_valid_password, (users, i, 50,))) + + finally: + with Then("it should work"): + join(tasks, timeout) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_MultipleServers("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") +) +def parallel_login_with_multiple_servers(self, server, user_count=10, timeout=200): + """Check that login of valid and invalid LDAP authenticated users works in parallel + using multiple LDAP external user directories. + """ + with Given("I have two LDAP servers"): + entries = [ + (["openldap1"], []), + (["openldap2"], []) + ] + + with And("I define a group of users to be created on each LDAP server"): + user_groups = { + "openldap1_users": [{"cn": f"openldap1_parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)], + "openldap2_users": [{"cn": f"openldap2_parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] + } + + with And("I have a list of checks that I want to run for each user group"): + checks = [ + login_with_valid_username_and_password, + login_with_valid_username_and_invalid_password, + login_with_invalid_username_and_valid_password + ] + + with And("I create config file to define LDAP external user directory for each LDAP server"): + config = create_entries_ldap_external_user_directory_config_content(entries) + + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with ldap_users(*user_groups["openldap1_users"], node=self.context.cluster.node("openldap1")): + with ldap_users(*user_groups["openldap2_users"], node=self.context.cluster.node("openldap2")): + tasks = [] + + try: + with When("users in each group try to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): + p = Pool(15) + for i in range(25): + for users in user_groups.values(): + for check in checks: + tasks.append(p.apply_async(check, (users, i, 50,))) + + finally: + with Then("it should work"): + join(tasks, timeout) + +@TestScenario +@Requirements( + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalAndMultipleLDAP("1.0"), + RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_ValidAndInvalid("1.0") +) +def parallel_login_with_rbac_and_multiple_servers(self, server, user_count=10, timeout=200): + """Check that login of valid and invalid users works in parallel + using local users defined using RBAC and LDAP users authenticated using + multiple LDAP external user directories. + """ + with Given("I have two LDAP servers"): + entries = [ + (["openldap1"], []), + (["openldap2"], []) + ] + + with And("I define a group of users to be created on each LDAP server"): + user_groups = { + "openldap1_users": [{"cn": f"openldap1_parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)], + "openldap2_users": [{"cn": f"openldap2_parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)], + "local_users": [{"cn": f"local_parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)] + } + + with And("I have a list of checks that I want to run for each user group"): + checks = [ + login_with_valid_username_and_password, + login_with_valid_username_and_invalid_password, + login_with_invalid_username_and_valid_password + ] + + with And("I create config file to define LDAP external user directory for each LDAP server"): + config = create_entries_ldap_external_user_directory_config_content(entries) + + with ldap_external_user_directory(server=None, roles=None, restart=True, config=config): + with ldap_users(*user_groups["openldap1_users"], node=self.context.cluster.node("openldap1")): + with ldap_users(*user_groups["openldap2_users"], node=self.context.cluster.node("openldap2")): + with rbac_users(*user_groups["local_users"]): + tasks = [] + + try: + with When("users in each group try to login in parallel", description=""" + * with valid username and password + * with invalid username and valid password + * with valid username and invalid password + """): + p = Pool(15) + for i in range(25): + for users in user_groups.values(): + for check in checks: + tasks.append(p.apply_async(check, (users, i, 50,))) + + finally: + with Then("it should work"): + join(tasks, timeout) + @TestScenario @Requirements( RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Parallel_LocalOnly("1.0") ) -def parallel_login_of_rbac_users(self, server, user_count=10, timeout=200): +def parallel_login_with_rbac_users(self, server, user_count=10, timeout=200): """Check that login of only valid and invalid local users created using RBAC works in parallel when server configuration includes LDAP external user directory. """ @@ -362,8 +563,6 @@ def valid_username_and_password_invalid_server(self, server=None): password but for a different server.""" self.context.ldap_node = self.context.cluster.node("openldap1") - user = {"username": "user2", "userpassword": "user2", "server": "openldap1"} - exitcode = 4 message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" From 82475088f9779b17e28498376b9f02d2e2103c79 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sat, 3 Oct 2020 17:31:02 +0400 Subject: [PATCH 113/411] Fix "user has been dropped" issue --- src/Access/LDAPAccessStorage.cpp | 19 ++++--------------- src/Access/LDAPClient.cpp | 2 +- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 38922eeac55..0142b98f1bb 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -29,11 +29,11 @@ LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_, AccessControl void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix) { + std::scoped_lock lock(mutex); + // TODO: switch to passing config as a ConfigurationView and remove this extra prefix once a version of Poco with proper implementation is available. const String prefix_str = (prefix.empty() ? "" : prefix + "."); - std::scoped_lock lock(mutex); - const bool has_server = config.has(prefix_str + "server"); const bool has_roles = config.has(prefix_str + "roles"); @@ -235,20 +235,9 @@ UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & passw auto id = memory_storage.find(user_name); if (id) { - // We try to re-authenticate the existing user, and if not successful, we will remove it, since that would mean - // something changed and the user we authenticated previously cannot be authenticated anymore. auto user = memory_storage.tryRead(*id); - try - { - if (user && isAddressAllowedImpl(*user, address) && isPasswordCorrectImpl(*user, password, external_authenticators)) - return *id; - } - catch (...) - { - memory_storage.remove(*id); - throw; - } - memory_storage.remove(*id); + if (user && isAddressAllowedImpl(*user, address) && isPasswordCorrectImpl(*user, password, external_authenticators)) + return *id; } else { diff --git a/src/Access/LDAPClient.cpp b/src/Access/LDAPClient.cpp index d6580b89c68..d3231f62f3b 100644 --- a/src/Access/LDAPClient.cpp +++ b/src/Access/LDAPClient.cpp @@ -284,7 +284,7 @@ bool LDAPSimpleAuthClient::check() std::scoped_lock lock(ldap_global_mutex); if (params.user.empty()) - throw Exception("LDAP authentication of a user with an empty name is not allowed", ErrorCodes::BAD_ARGUMENTS); + throw Exception("LDAP authentication of a user with empty name is not allowed", ErrorCodes::BAD_ARGUMENTS); if (params.password.empty()) return false; // Silently reject authentication attempt if the password is empty as if it didn't match. From 00a354cd376a0fbb0fcf6aca1e6e1d7fbebadc34 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sun, 4 Oct 2020 22:00:56 +0400 Subject: [PATCH 114/411] Manually remove storages in reverse order in MultipleAccessStorage d-tor --- src/Access/MultipleAccessStorage.cpp | 9 +++++++++ src/Access/MultipleAccessStorage.h | 1 + 2 files changed, 10 insertions(+) diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 8ddc7410d8d..792f4973a78 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -27,6 +28,14 @@ MultipleAccessStorage::MultipleAccessStorage(const String & storage_name_) { } +MultipleAccessStorage::~MultipleAccessStorage() +{ + auto storages = getStoragesPtr(); + for (auto storage : *storages | boost::adaptors::reversed) + { + removeStorage(storage); + } +} void MultipleAccessStorage::setStorages(const std::vector & storages) { diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index 36551f1cbc8..8844de8c029 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -18,6 +18,7 @@ public: using ConstStoragePtr = std::shared_ptr; MultipleAccessStorage(const String & storage_name_ = STORAGE_TYPE); + ~MultipleAccessStorage() override; const char * getStorageType() const override { return STORAGE_TYPE; } From 7f477197681d152ac64a7cff55726e672d8010d2 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sun, 4 Oct 2020 23:55:58 +0400 Subject: [PATCH 115/411] Refactor exception handling in login() et al. Simplify LDAPClient and LDAPAccessStorage --- src/Access/IAccessStorage.cpp | 31 ++++++++++-- src/Access/IAccessStorage.h | 2 + src/Access/LDAPAccessStorage.cpp | 70 ++++++++++++++++------------ src/Access/LDAPAccessStorage.h | 2 + src/Access/LDAPClient.cpp | 59 ++++------------------- src/Access/LDAPClient.h | 1 - src/Access/LDAPParams.h | 1 + src/Access/MultipleAccessStorage.cpp | 2 +- 8 files changed, 82 insertions(+), 86 deletions(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index e5170221e18..cb490d488c8 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -14,6 +14,8 @@ namespace ErrorCodes extern const int ACCESS_ENTITY_ALREADY_EXISTS; extern const int ACCESS_ENTITY_NOT_FOUND; extern const int ACCESS_STORAGE_READONLY; + extern const int WRONG_PASSWORD; + extern const int IP_ADDRESS_NOT_ALLOWED; extern const int AUTHENTICATION_FAILED; extern const int LOGICAL_ERROR; } @@ -420,7 +422,14 @@ UUID IAccessStorage::login( const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const { - return loginImpl(user_name, password, address, external_authenticators); + try { + return loginImpl(user_name, password, address, external_authenticators); + } + catch (...) + { + tryLogCurrentException(getLogger(), user_name + ": Authentication failed"); + throwCannotAuthenticate(user_name); + } } @@ -434,11 +443,16 @@ UUID IAccessStorage::loginImpl( { if (auto user = tryRead(*id)) { - if (isPasswordCorrectImpl(*user, password, external_authenticators) && isAddressAllowedImpl(*user, address)) - return *id; + if (!isPasswordCorrectImpl(*user, password, external_authenticators)) + throwInvalidPassword(); + + if (!isAddressAllowedImpl(*user, address)) + throwAddressNotAllowed(address); + + return *id; } } - throwCannotAuthenticate(user_name); + throwNotFound(EntityType::USER, user_name); } @@ -554,6 +568,15 @@ void IAccessStorage::throwReadonlyCannotRemove(EntityType type, const String & n ErrorCodes::ACCESS_STORAGE_READONLY); } +void IAccessStorage::throwAddressNotAllowed(const Poco::Net::IPAddress & address) +{ + throw Exception("Connections from " + address.toString() + " are not allowed", ErrorCodes::IP_ADDRESS_NOT_ALLOWED); +} + +void IAccessStorage::throwInvalidPassword() +{ + throw Exception("Invalid password", ErrorCodes::WRONG_PASSWORD); +} void IAccessStorage::throwCannotAuthenticate(const String & user_name) { diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 5a86e817fb2..059c6103f6a 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -182,6 +182,8 @@ protected: [[noreturn]] void throwReadonlyCannotInsert(EntityType type, const String & name) const; [[noreturn]] void throwReadonlyCannotUpdate(EntityType type, const String & name) const; [[noreturn]] void throwReadonlyCannotRemove(EntityType type, const String & name) const; + [[noreturn]] static void throwAddressNotAllowed(const Poco::Net::IPAddress & address); + [[noreturn]] static void throwInvalidPassword(); [[noreturn]] static void throwCannotAuthenticate(const String & user_name); using Notification = std::tuple; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 0142b98f1bb..9b1c6a48a13 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -230,42 +230,50 @@ bool LDAPAccessStorage::hasSubscriptionImpl(EntityType type) const UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const { std::scoped_lock lock(mutex); - try + auto id = memory_storage.find(user_name); + if (id) { - auto id = memory_storage.find(user_name); - if (id) - { - auto user = memory_storage.tryRead(*id); - if (user && isAddressAllowedImpl(*user, address) && isPasswordCorrectImpl(*user, password, external_authenticators)) - return *id; - } - else - { - // User does not exist, so we create one, and will add it if authentication is successful. - auto user = std::make_shared(); - user->setName(user_name); - user->authentication = Authentication(Authentication::Type::LDAP_SERVER); - user->authentication.setServerName(ldap_server); + auto user = memory_storage.read(*id); - if (isAddressAllowedImpl(*user, address) && isPasswordCorrectImpl(*user, password, external_authenticators)) - { - for (const auto& role_name : default_role_names) - { - std::optional role_id = access_control_manager->find(role_name); - if (!role_id) - throw Exception("One of the default roles, the role '" + role_name + "', is not found", IAccessEntity::TypeInfo::get(IAccessEntity::Type::ROLE).not_found_error_code); - roles_of_interest.insert(role_id.value()); - user->granted_roles.grant(role_id.value()); - } - return memory_storage.insert(user); - } - } + if (!isPasswordCorrectImpl(*user, password, external_authenticators)) + throwInvalidPassword(); + + if (!isAddressAllowedImpl(*user, address)) + throwAddressNotAllowed(address); + + return *id; } - catch (...) + else { - tryLogCurrentException(getLogger(), "Authentication failed for user '" + user_name + "' from access storage '" + getStorageName() + "'"); + // User does not exist, so we create one, and will add it if authentication is successful. + auto user = std::make_shared(); + user->setName(user_name); + user->authentication = Authentication(Authentication::Type::LDAP_SERVER); + user->authentication.setServerName(ldap_server); + + if (!isPasswordCorrectImpl(*user, password, external_authenticators)) + throwInvalidPassword(); + + if (!isAddressAllowedImpl(*user, address)) + throwAddressNotAllowed(address); + + for (const auto& role_name : default_role_names) + { + std::optional role_id = access_control_manager->find(role_name); + if (!role_id) + throwDefaultRoleNotFound(role_name); + + roles_of_interest.insert(role_id.value()); + user->granted_roles.grant(role_id.value()); + } + + return memory_storage.insert(user); } - throwCannotAuthenticate(user_name); +} + +void LDAPAccessStorage::throwDefaultRoleNotFound(const String & role_name) +{ + throw Exception("One of the default roles, the role '" + role_name + "', is not found", IAccessEntity::TypeInfo::get(IAccessEntity::Type::ROLE).not_found_error_code); } } diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index 35444944ec6..02c44a8d400 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -55,6 +55,8 @@ private: void setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix); void processRoleChange(const UUID & id, const AccessEntityPtr & entity); + [[noreturn]] static void throwDefaultRoleNotFound(const String & role_name); + mutable std::recursive_mutex mutex; AccessControlManager * access_control_manager = nullptr; String ldap_server; diff --git a/src/Access/LDAPClient.cpp b/src/Access/LDAPClient.cpp index d3231f62f3b..a3223902361 100644 --- a/src/Access/LDAPClient.cpp +++ b/src/Access/LDAPClient.cpp @@ -106,14 +106,6 @@ void LDAPClient::openConnection() { std::scoped_lock lock(ldap_global_mutex); - const bool graceful_bind_failure = false; - diag(openConnection(graceful_bind_failure)); -} - -int LDAPClient::openConnection(const bool graceful_bind_failure) -{ - std::scoped_lock lock(ldap_global_mutex); - closeConnection(); { @@ -244,8 +236,6 @@ int LDAPClient::openConnection(const bool graceful_bind_failure) if (params.enable_tls == LDAPServerParams::TLSEnable::YES_STARTTLS) diag(ldap_start_tls_s(handle, nullptr, nullptr)); - int rc = LDAP_OTHER; - switch (params.sasl_mechanism) { case LDAPServerParams::SASLMechanism::SIMPLE: @@ -256,16 +246,15 @@ int LDAPClient::openConnection(const bool graceful_bind_failure) cred.bv_val = const_cast(params.password.c_str()); cred.bv_len = params.password.size(); - rc = ldap_sasl_bind_s(handle, dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr); - - if (!graceful_bind_failure) - diag(rc); + diag(ldap_sasl_bind_s(handle, dn.c_str(), LDAP_SASL_SIMPLE, &cred, nullptr, nullptr, nullptr)); break; } + default: + { + throw Exception("Unknown SASL mechanism", ErrorCodes::LDAP_ERROR); + } } - - return rc; } void LDAPClient::closeConnection() noexcept @@ -286,39 +275,16 @@ bool LDAPSimpleAuthClient::check() if (params.user.empty()) throw Exception("LDAP authentication of a user with empty name is not allowed", ErrorCodes::BAD_ARGUMENTS); + // Silently reject authentication attempt if the password is empty as if it didn't match. if (params.password.empty()) - return false; // Silently reject authentication attempt if the password is empty as if it didn't match. + return false; SCOPE_EXIT({ closeConnection(); }); - const bool graceful_bind_failure = true; - const auto rc = openConnection(graceful_bind_failure); + // Will throw on any error, including invalid credentials. + openConnection(); - bool result = false; - - switch (rc) - { - case LDAP_SUCCESS: - { - result = true; - break; - } - - case LDAP_INVALID_CREDENTIALS: - { - result = false; - break; - } - - default: - { - result = false; - diag(rc); - break; - } - } - - return result; + return true; } #else // USE_LDAP @@ -333,11 +299,6 @@ void LDAPClient::openConnection() throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); } -int LDAPClient::openConnection(const bool) -{ - throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); -} - void LDAPClient::closeConnection() noexcept { } diff --git a/src/Access/LDAPClient.h b/src/Access/LDAPClient.h index b117ed9a026..777c87c5b94 100644 --- a/src/Access/LDAPClient.h +++ b/src/Access/LDAPClient.h @@ -32,7 +32,6 @@ public: protected: MAYBE_NORETURN void diag(const int rc); MAYBE_NORETURN void openConnection(); - int openConnection(const bool graceful_bind_failure = false); void closeConnection() noexcept; protected: diff --git a/src/Access/LDAPParams.h b/src/Access/LDAPParams.h index 2168ce45203..eeadba6bc01 100644 --- a/src/Access/LDAPParams.h +++ b/src/Access/LDAPParams.h @@ -42,6 +42,7 @@ struct LDAPServerParams enum class SASLMechanism { + UNKNOWN, SIMPLE }; diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 792f4973a78..eb9b142e112 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -425,7 +425,7 @@ UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & p throw; } } - throwCannotAuthenticate(user_name); + throwNotFound(EntityType::USER, user_name); } From 1eb8ecf050d071506304c92fcd295167abc8a0bc Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Sun, 4 Oct 2020 23:56:25 +0400 Subject: [PATCH 116/411] Fix compilation --- src/Access/MultipleAccessStorage.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index eb9b142e112..516042b5af5 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -30,8 +30,8 @@ MultipleAccessStorage::MultipleAccessStorage(const String & storage_name_) MultipleAccessStorage::~MultipleAccessStorage() { - auto storages = getStoragesPtr(); - for (auto storage : *storages | boost::adaptors::reversed) + const auto storages = getStoragesPtr(); + for (const auto & storage : *storages | boost::adaptors::reversed) { removeStorage(storage); } From 2fc6a4ea9c0bd1af1ce08c88ca3113928c705929 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 5 Oct 2020 00:24:09 +0400 Subject: [PATCH 117/411] Add log_and_mask_exceptions flag to login() --- src/Access/IAccessStorage.cpp | 6 +++++- src/Access/IAccessStorage.h | 2 +- src/Access/MultipleAccessStorage.cpp | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index cb490d488c8..b21527e48f1 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -420,13 +420,17 @@ UUID IAccessStorage::login( const String & user_name, const String & password, const Poco::Net::IPAddress & address, - const ExternalAuthenticators & external_authenticators) const + const ExternalAuthenticators & external_authenticators, + bool log_and_mask_exceptions) const { try { return loginImpl(user_name, password, address, external_authenticators); } catch (...) { + if (!log_and_mask_exceptions) + throw; + tryLogCurrentException(getLogger(), user_name + ": Authentication failed"); throwCannotAuthenticate(user_name); } diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 059c6103f6a..93c97144cda 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -144,7 +144,7 @@ public: /// Finds an user, check its password and returns the ID of the user. /// Throws an exception if no such user or password is incorrect. - UUID login(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const; + UUID login(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool log_and_mask_exceptions = true) const; /// Returns the ID of an user who has logged in (maybe on another node). /// The function assumes that the password has been already checked somehow, so we can skip checking it now. diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 516042b5af5..32aa8c50159 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -409,7 +409,7 @@ UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & p { try { - auto id = storage->login(user_name, password, address, external_authenticators); + auto id = storage->login(user_name, password, address, external_authenticators, false); std::lock_guard lock{mutex}; ids_cache.set(id, storage); return id; From d98366ed79fcf99323ab8f05b8d984540b546760 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Tue, 6 Oct 2020 11:20:47 +0300 Subject: [PATCH 118/411] Better initialization of S3 storage. --- src/IO/S3/PocoHTTPClient.cpp | 21 +++++++++++++++++++++ src/IO/S3/PocoHTTPClient.h | 2 ++ src/IO/S3Common.cpp | 28 ++++++++-------------------- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index c34d7719131..a649b76730b 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -17,6 +17,9 @@ #include #include #include +#include + +#include namespace ProfileEvents @@ -51,6 +54,24 @@ PocoHTTPClientConfiguration::PocoHTTPClientConfiguration( { } +void PocoHTTPClientConfiguration::updateSchemeAndRegion() +{ + if (!endpointOverride.empty()) + { + static const RE2 region_pattern(R"(^s3[.\-]([a-z0-9\-]+)\.amazonaws\.)"); + Poco::URI uri(endpointOverride); + if (uri.getScheme() == "http") + scheme = Aws::Http::Scheme::HTTP; + + String matched_region; + if (re2::RE2::PartialMatch(uri.getHost(), region_pattern, &matched_region)) + { + boost::algorithm::to_lower(matched_region); + region = matched_region; + } + } +} + PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & clientConfiguration) : per_request_configuration(clientConfiguration.perRequestConfiguration) diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index eefc85fae70..25055754519 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -19,6 +19,8 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration const RemoteHostFilter & remote_host_filter; PocoHTTPClientConfiguration(const Aws::Client::ClientConfiguration & cfg, const RemoteHostFilter & remote_host_filter_); + + void updateSchemeAndRegion(); }; class PocoHTTPClient : public Aws::Http::HttpClient diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index db7aaf1549b..1304b6b5054 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -13,7 +13,6 @@ # include # include # include -# include # include # include # include @@ -186,20 +185,7 @@ namespace S3 PocoHTTPClientConfiguration client_configuration(cfg, remote_host_filter); - if (!client_configuration.endpointOverride.empty()) - { - static const RE2 region_pattern(R"(^s3[.\-]([a-z0-9\-]+)\.amazonaws\.)"); - Poco::URI uri(client_configuration.endpointOverride); - if (uri.getScheme() == "http") - client_configuration.scheme = Aws::Http::Scheme::HTTP; - - String region; - if (re2::RE2::PartialMatch(uri.getHost(), region_pattern, ®ion)) - { - boost::algorithm::to_lower(region); - client_configuration.region = region; - } - } + client_configuration.updateSchemeAndRegion(); return std::make_shared( credentials, // Aws credentials. @@ -217,16 +203,18 @@ namespace S3 HeaderCollection headers, const RemoteHostFilter & remote_host_filter) { - PocoHTTPClientConfiguration cfg({}, remote_host_filter); + PocoHTTPClientConfiguration client_configuration({}, remote_host_filter); if (!endpoint.empty()) - cfg.endpointOverride = endpoint; + client_configuration.endpointOverride = endpoint; + + client_configuration.updateSchemeAndRegion(); Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key); return std::make_shared( - std::make_shared(cfg, std::move(credentials), std::move(headers)), - std::move(cfg), // Client configuration. - is_virtual_hosted_style || cfg.endpointOverride.empty() // Use virtual addressing only if endpoint is not specified. + std::make_shared(client_configuration, std::move(credentials), std::move(headers)), + std::move(client_configuration), // Client configuration. + is_virtual_hosted_style || client_configuration.endpointOverride.empty() // Use virtual addressing only if endpoint is not specified. ); } From 950a07835f287fba38b4c1e018a0588157bcfe9f Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 6 Oct 2020 19:23:08 +0400 Subject: [PATCH 119/411] Stylistic changes --- src/Access/IAccessStorage.cpp | 7 ++++--- src/Access/IAccessStorage.h | 2 +- src/Access/MultipleAccessStorage.cpp | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index b21527e48f1..8dd219e07d7 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -421,14 +421,15 @@ UUID IAccessStorage::login( const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, - bool log_and_mask_exceptions) const + bool replace_exception_with_cannot_authenticate) const { - try { + try + { return loginImpl(user_name, password, address, external_authenticators); } catch (...) { - if (!log_and_mask_exceptions) + if (!replace_exception_with_cannot_authenticate) throw; tryLogCurrentException(getLogger(), user_name + ": Authentication failed"); diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index 93c97144cda..ecf6b260712 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -144,7 +144,7 @@ public: /// Finds an user, check its password and returns the ID of the user. /// Throws an exception if no such user or password is incorrect. - UUID login(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool log_and_mask_exceptions = true) const; + UUID login(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool replace_exception_with_cannot_authenticate = true) const; /// Returns the ID of an user who has logged in (maybe on another node). /// The function assumes that the password has been already checked somehow, so we can skip checking it now. diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 32aa8c50159..6f888f2f150 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -409,7 +409,7 @@ UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & p { try { - auto id = storage->login(user_name, password, address, external_authenticators, false); + auto id = storage->login(user_name, password, address, external_authenticators, /* replace_exception_with_cannot_authenticate = */ false); std::lock_guard lock{mutex}; ids_cache.set(id, storage); return id; From e2f444ae8559f972e89d62409a84b98aea1b3986 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 6 Oct 2020 19:37:35 +0400 Subject: [PATCH 120/411] Simplify loginImpl() and getIDOfLoggedUserImpl() --- src/Access/MultipleAccessStorage.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 6f888f2f150..86790a23933 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -30,6 +30,7 @@ MultipleAccessStorage::MultipleAccessStorage(const String & storage_name_) MultipleAccessStorage::~MultipleAccessStorage() { + /// It's better to remove the storages in the reverse order because they could depend on each other somehow. const auto storages = getStoragesPtr(); for (const auto & storage : *storages | boost::adaptors::reversed) { @@ -414,9 +415,9 @@ UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & p ids_cache.set(id, storage); return id; } - catch (...) + catch (const Exception & e) { - if (!storage->find(EntityType::USER, user_name)) + if (e.code() == EntityTypeInfo::get(EntityType::USER).not_found_error_code) { /// The authentication failed because there no users with such name in the `storage` /// thus we can try to search in other nested storages. @@ -441,9 +442,9 @@ UUID MultipleAccessStorage::getIDOfLoggedUserImpl(const String & user_name) cons ids_cache.set(id, storage); return id; } - catch (...) + catch (const Exception & e) { - if (!storage->find(EntityType::USER, user_name)) + if (e.code() == EntityTypeInfo::get(EntityType::USER).not_found_error_code) { /// The authentication failed because there no users with such name in the `storage` /// thus we can try to search in other nested storages. From dfc13ca8e7d68a1cfb9fb8ccca494a44c7d82b19 Mon Sep 17 00:00:00 2001 From: Vasily Kozhukhovskiy Date: Tue, 6 Oct 2020 18:37:54 +0300 Subject: [PATCH 121/411] Enable parsing enum values by their ids for CSV, TSV and JSON input formats * for CSV and TSV input formats input_format_csv_enum_as_number, input_format_tsv_enum_as_number settings should be enabled in order to treat input value as enum id --- docs/en/operations/settings/settings.md | 8 +++ src/Core/Settings.h | 2 + src/DataTypes/DataTypeEnum.cpp | 67 ++++++++++++++----- src/Formats/FormatFactory.cpp | 2 + src/Formats/FormatSettings.h | 2 + ...ormat_csv_enum_as_number_setting.reference | 0 ...nput_format_csv_enum_as_number_setting.sql | 14 ++++ ...input_format_json_enum_as_number.reference | 0 ...01514_input_format_json_enum_as_number.sql | 10 +++ ...ormat_tsv_enum_as_number_setting.reference | 0 ...nput_format_tsv_enum_as_number_setting.sql | 14 ++++ 11 files changed, 104 insertions(+), 15 deletions(-) create mode 100644 tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.reference create mode 100644 tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.sql create mode 100644 tests/queries/0_stateless/01514_input_format_json_enum_as_number.reference create mode 100644 tests/queries/0_stateless/01514_input_format_json_enum_as_number.sql create mode 100644 tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.reference create mode 100644 tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index decaf6b9029..b5fda4c5699 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -276,6 +276,10 @@ When enabled, replace empty input fields in TSV with default values. For complex Disabled by default. +## input\_format\_tsv\_enum\_as\_number {#settings-input_format_tsv_enum_as_number} + +For TSV input format switches to parsing enum values as enum ids. + ## input\_format\_null\_as\_default {#settings-input-format-null-as-default} Enables or disables using default values if input data contain `NULL`, but data type of the corresponding column in not `Nullable(T)` (for text input formats). @@ -1107,6 +1111,10 @@ The character interpreted as a delimiter in the CSV data. By default, the delimi For CSV input format enables or disables parsing of unquoted `NULL` as literal (synonym for `\N`). +## input\_format\_csv\_enum\_as\_number {#settings-input_format_csv_enum_as_number} + +For CSV input format switches to parsing enum values as enum ids. + ## output\_format\_csv\_crlf\_end\_of\_line {#settings-output-format-csv-crlf-end-of-line} Use DOS/Windows-style line separator (CRLF) in CSV instead of Unix style (LF). diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 3ecb79c3fce..7c13cf2cf58 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -412,12 +412,14 @@ class IColumn; M(Bool, format_csv_allow_double_quotes, 1, "If it is set to true, allow strings in double quotes.", 0) \ M(Bool, output_format_csv_crlf_end_of_line, false, "If it is set true, end of line in CSV format will be \\r\\n instead of \\n.", 0) \ M(Bool, input_format_csv_unquoted_null_literal_as_null, false, "Consider unquoted NULL literal as \\N", 0) \ + M(Bool, input_format_csv_enum_as_number, false, "Treat inserted enum values in CSV formats as enum indices \\N", 0) \ M(Bool, input_format_skip_unknown_fields, false, "Skip columns with unknown names from input data (it works for JSONEachRow, CSVWithNames, TSVWithNames and TSKV formats).", 0) \ M(Bool, input_format_with_names_use_header, true, "For TSVWithNames and CSVWithNames input formats this controls whether format parser is to assume that column data appear in the input exactly as they are specified in the header.", 0) \ M(Bool, input_format_import_nested_json, false, "Map nested JSON data to nested tables (it works for JSONEachRow format).", 0) \ M(Bool, optimize_aggregators_of_group_by_keys, true, "Eliminates min/max/any/anyLast aggregators of GROUP BY keys in SELECT section", 0) \ M(Bool, input_format_defaults_for_omitted_fields, true, "For input data calculate default expressions for omitted fields (it works for JSONEachRow, CSV and TSV formats).", IMPORTANT) \ M(Bool, input_format_tsv_empty_as_default, false, "Treat empty fields in TSV input as default values.", 0) \ + M(Bool, input_format_tsv_enum_as_number, false, "Treat inserted enum values in TSV formats as enum indices \\N", 0) \ M(Bool, input_format_null_as_default, false, "For text input formats initialize null fields with default values if data type of this field is not nullable", 0) \ \ M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic' and 'best_effort'.", 0) \ diff --git a/src/DataTypes/DataTypeEnum.cpp b/src/DataTypes/DataTypeEnum.cpp index 9ad6a9cb690..86049e41e68 100644 --- a/src/DataTypes/DataTypeEnum.cpp +++ b/src/DataTypes/DataTypeEnum.cpp @@ -146,12 +146,22 @@ void DataTypeEnum::serializeTextEscaped(const IColumn & column, size_t row } template -void DataTypeEnum::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void DataTypeEnum::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - /// NOTE It would be nice to do without creating a temporary object - at least extract std::string out. - std::string field_name; - readEscapedString(field_name, istr); - assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + if (settings.tsv.input_format_enum_as_number) + { + FieldType x; + readText(x, istr); + static_cast(getNameForValue(x)); + assert_cast(column).getData().push_back(x); + } + else + { + /// NOTE It would be nice to do without creating a temporary object - at least extract std::string out. + std::string field_name; + readEscapedString(field_name, istr); + assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + } } template @@ -169,11 +179,20 @@ void DataTypeEnum::deserializeTextQuoted(IColumn & column, ReadBuffer & is } template -void DataTypeEnum::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void DataTypeEnum::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - std::string field_name; - readString(field_name, istr); - assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + if (settings.tsv.input_format_enum_as_number) { + FieldType x; + readText(x, istr); + static_cast(getNameForValue(x)); + assert_cast(column).getData().push_back(x); + } + else + { + std::string field_name; + readString(field_name, istr); + assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + } } template @@ -191,9 +210,18 @@ void DataTypeEnum::serializeTextXML(const IColumn & column, size_t row_num template void DataTypeEnum::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { - std::string field_name; - readJSONString(field_name, istr); - assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + if (*istr.position() != '"') { + FieldType x; + readText(x, istr); + static_cast(getNameForValue(x)); + assert_cast(column).getData().push_back(x); + } + else + { + std::string field_name; + readJSONString(field_name, istr); + assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + } } template @@ -205,9 +233,18 @@ void DataTypeEnum::serializeTextCSV(const IColumn & column, size_t row_num template void DataTypeEnum::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - std::string field_name; - readCSVString(field_name, istr, settings.csv); - assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + if (settings.csv.input_format_enum_as_number) { + FieldType x; + readText(x, istr); + static_cast(getNameForValue(x)); + assert_cast(column).getData().push_back(x); + } + else + { + std::string field_name; + readCSVString(field_name, istr, settings.csv); + assert_cast(column).getData().push_back(getValue(StringRef(field_name))); + } } template diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 065b14f86b7..a8b074df2a0 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -48,6 +48,7 @@ static FormatSettings getInputFormatSetting(const Settings & settings, const Con format_settings.csv.allow_double_quotes = settings.format_csv_allow_double_quotes; format_settings.csv.unquoted_null_literal_as_null = settings.input_format_csv_unquoted_null_literal_as_null; format_settings.csv.empty_as_default = settings.input_format_defaults_for_omitted_fields; + format_settings.csv.input_format_enum_as_number = settings.input_format_csv_enum_as_number; format_settings.null_as_default = settings.input_format_null_as_default; format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions; format_settings.values.deduce_templates_of_expressions = settings.input_format_values_deduce_templates_of_expressions; @@ -62,6 +63,7 @@ static FormatSettings getInputFormatSetting(const Settings & settings, const Con format_settings.template_settings.row_format = settings.format_template_row; format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter; format_settings.tsv.empty_as_default = settings.input_format_tsv_empty_as_default; + format_settings.tsv.input_format_enum_as_number = settings.input_format_tsv_enum_as_number; format_settings.schema.format_schema = settings.format_schema; format_settings.schema.format_schema_path = context.getFormatSchemaPath(); format_settings.schema.is_server = context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER); diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index a97bd9bf6c6..1ba9949a6b3 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -34,6 +34,7 @@ struct FormatSettings bool unquoted_null_literal_as_null = false; bool empty_as_default = false; bool crlf_end_of_line = false; + bool input_format_enum_as_number = false; }; CSV csv; @@ -81,6 +82,7 @@ struct FormatSettings bool empty_as_default = false; bool crlf_end_of_line = false; String null_representation = "\\N"; + bool input_format_enum_as_number = false; }; TSV tsv; diff --git a/tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.reference b/tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.sql b/tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.sql new file mode 100644 index 00000000000..f4f2b278ac3 --- /dev/null +++ b/tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS table_with_enum_column_for_csv_insert; + +CREATE TABLE table_with_enum_column_for_csv_insert ( + Id Int32, + Value Enum('ef' = 1, 'es' = 2) +) ENGINE=Memory(); + +SET input_format_csv_enum_as_number = 1; + +INSERT INTO table_with_enum_column_for_csv_insert FORMAT CSV 102,2 + +SET input_format_csv_enum_as_number = 0; + +DROP TABLE IF EXISTS table_with_enum_column_for_csv_insert; diff --git a/tests/queries/0_stateless/01514_input_format_json_enum_as_number.reference b/tests/queries/0_stateless/01514_input_format_json_enum_as_number.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01514_input_format_json_enum_as_number.sql b/tests/queries/0_stateless/01514_input_format_json_enum_as_number.sql new file mode 100644 index 00000000000..a07cf87e11d --- /dev/null +++ b/tests/queries/0_stateless/01514_input_format_json_enum_as_number.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS table_with_enum_column_for_json_insert; + +CREATE TABLE table_with_enum_column_for_json_insert ( + Id Int32, + Value Enum('ef' = 1, 'es' = 2) +) ENGINE=Memory(); + +INSERT INTO table_with_enum_column_for_json_insert FORMAT JSONEachRow {"Id":102,"Value":2} + +DROP TABLE IF EXISTS table_with_enum_column_for_json_insert; diff --git a/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.reference b/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql b/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql new file mode 100644 index 00000000000..f514c0f4dc5 --- /dev/null +++ b/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS table_with_enum_column_for_tsv_insert; + +CREATE TABLE table_with_enum_column_for_tsv_insert ( + Id Int32, + Value Enum('ef' = 1, 'es' = 2) +) ENGINE=Memory(); + +SET input_format_tsv_enum_as_number = 1; + +INSERT INTO table_with_enum_column_for_tsv_insert FORMAT TSV 102 2 + +SET input_format_tsv_enum_as_number = 0; + +DROP TABLE IF EXISTS table_with_enum_column_for_tsv_insert; From fed6080273eb7d1e837dcf6ca8be8e5ed1d3a10c Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 6 Oct 2020 19:54:22 +0400 Subject: [PATCH 122/411] Implement custom getIDOfLoggedUserImpl() --- src/Access/LDAPAccessStorage.cpp | 30 ++++++++++++++++++++++++++++++ src/Access/LDAPAccessStorage.h | 1 + 2 files changed, 31 insertions(+) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 9b1c6a48a13..d1dbf10cfbc 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -271,6 +271,36 @@ UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & passw } } +UUID LDAPAccessStorage::getIDOfLoggedUserImpl(const String & user_name) const +{ + std::scoped_lock lock(mutex); + auto id = memory_storage.find(user_name); + if (id) + { + return *id; + } + else + { + // User does not exist, so we create one, and add it pretending that the authentication is successful. + auto user = std::make_shared(); + user->setName(user_name); + user->authentication = Authentication(Authentication::Type::LDAP_SERVER); + user->authentication.setServerName(ldap_server); + + for (const auto& role_name : default_role_names) + { + std::optional role_id = access_control_manager->find(role_name); + if (!role_id) + throwDefaultRoleNotFound(role_name); + + roles_of_interest.insert(role_id.value()); + user->granted_roles.grant(role_id.value()); + } + + return memory_storage.insert(user); + } +} + void LDAPAccessStorage::throwDefaultRoleNotFound(const String & role_name) { throw Exception("One of the default roles, the role '" + role_name + "', is not found", IAccessEntity::TypeInfo::get(IAccessEntity::Type::ROLE).not_found_error_code); diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index 02c44a8d400..1e6e0713568 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -50,6 +50,7 @@ private: // IAccessStorage implementations. virtual bool hasSubscriptionImpl(const UUID & id) const override; virtual bool hasSubscriptionImpl(EntityType type) const override; virtual UUID loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const override; + virtual UUID getIDOfLoggedUserImpl(const String & user_name) const override; private: void setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix); From 54446eeec6e6f6c4449a45c6d78a015a0fb0f54c Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 6 Oct 2020 20:00:29 +0400 Subject: [PATCH 123/411] Use ErrorCodes::UNKNOWN_USER --- src/Access/MultipleAccessStorage.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 86790a23933..170819922ec 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -417,7 +417,7 @@ UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & p } catch (const Exception & e) { - if (e.code() == EntityTypeInfo::get(EntityType::USER).not_found_error_code) + if (e.code() == ErrorCodes::UNKNOWN_USER) { /// The authentication failed because there no users with such name in the `storage` /// thus we can try to search in other nested storages. @@ -444,7 +444,7 @@ UUID MultipleAccessStorage::getIDOfLoggedUserImpl(const String & user_name) cons } catch (const Exception & e) { - if (e.code() == EntityTypeInfo::get(EntityType::USER).not_found_error_code) + if (e.code() == ErrorCodes::UNKNOWN_USER) { /// The authentication failed because there no users with such name in the `storage` /// thus we can try to search in other nested storages. From 555f056a4c13c29d7f7a541bc135a1a7c98b59e8 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Tue, 6 Oct 2020 20:32:06 +0400 Subject: [PATCH 124/411] Revert "user not found" detection in loginImpl() and getIDOfLoggedUserImpl() --- src/Access/MultipleAccessStorage.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 170819922ec..a8ce3f602ed 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -415,9 +415,9 @@ UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & p ids_cache.set(id, storage); return id; } - catch (const Exception & e) + catch (...) { - if (e.code() == ErrorCodes::UNKNOWN_USER) + if (!storage->find(EntityType::USER, user_name)) { /// The authentication failed because there no users with such name in the `storage` /// thus we can try to search in other nested storages. @@ -442,9 +442,9 @@ UUID MultipleAccessStorage::getIDOfLoggedUserImpl(const String & user_name) cons ids_cache.set(id, storage); return id; } - catch (const Exception & e) + catch (...) { - if (e.code() == ErrorCodes::UNKNOWN_USER) + if (!storage->find(EntityType::USER, user_name)) { /// The authentication failed because there no users with such name in the `storage` /// thus we can try to search in other nested storages. From b8aa440912787b58be368ff51c7d7f6e67295b09 Mon Sep 17 00:00:00 2001 From: Vasily Kozhukhovskiy Date: Wed, 7 Oct 2020 12:24:54 +0300 Subject: [PATCH 125/411] style fixes --- src/DataTypes/DataTypeEnum.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/DataTypes/DataTypeEnum.cpp b/src/DataTypes/DataTypeEnum.cpp index 86049e41e68..f445b342e76 100644 --- a/src/DataTypes/DataTypeEnum.cpp +++ b/src/DataTypes/DataTypeEnum.cpp @@ -181,7 +181,8 @@ void DataTypeEnum::deserializeTextQuoted(IColumn & column, ReadBuffer & is template void DataTypeEnum::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - if (settings.tsv.input_format_enum_as_number) { + if (settings.tsv.input_format_enum_as_number) + { FieldType x; readText(x, istr); static_cast(getNameForValue(x)); @@ -210,7 +211,8 @@ void DataTypeEnum::serializeTextXML(const IColumn & column, size_t row_num template void DataTypeEnum::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { - if (*istr.position() != '"') { + if (*istr.position() != '"') + { FieldType x; readText(x, istr); static_cast(getNameForValue(x)); @@ -233,7 +235,8 @@ void DataTypeEnum::serializeTextCSV(const IColumn & column, size_t row_num template void DataTypeEnum::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - if (settings.csv.input_format_enum_as_number) { + if (settings.csv.input_format_enum_as_number) + { FieldType x; readText(x, istr); static_cast(getNameForValue(x)); From 04c71314897056e18fb781b27b7f429a11e3bc1d Mon Sep 17 00:00:00 2001 From: Vasily Kozhukhovskiy Date: Wed, 7 Oct 2020 12:25:22 +0300 Subject: [PATCH 126/411] add test for inserting enum values by ids for TabSeparatedRaw format (input_format_tsv_enum_as_number setting) --- .../01514_input_format_tsv_enum_as_number_setting.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql b/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql index f514c0f4dc5..c1f0ba9167e 100644 --- a/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql +++ b/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql @@ -8,6 +8,7 @@ CREATE TABLE table_with_enum_column_for_tsv_insert ( SET input_format_tsv_enum_as_number = 1; INSERT INTO table_with_enum_column_for_tsv_insert FORMAT TSV 102 2 +INSERT INTO table_with_enum_column_for_tsv_insert FORMAT TabSeparatedRaw 103 1 SET input_format_tsv_enum_as_number = 0; From 69c126f1f12ebe27b8f48c22996e4a2a683bb40a Mon Sep 17 00:00:00 2001 From: Pavel Kovalenko Date: Wed, 7 Oct 2020 14:35:28 +0300 Subject: [PATCH 127/411] Possibility to move part to another disk/volume if first attempt was failed. --- src/Disks/DiskDecorator.cpp | 5 ++ src/Disks/DiskDecorator.h | 1 + src/Disks/IDisk.h | 4 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 5 +- .../configs/config.d/instant_moves.xml | 4 ++ .../configs/config.d/part_log.xml | 8 +++ .../configs/config.d/storage_conf.xml | 11 ++++ .../test_merge_tree_s3_failover/test.py | 62 ++++++++++++++++++- 8 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_merge_tree_s3_failover/configs/config.d/instant_moves.xml create mode 100644 tests/integration/test_merge_tree_s3_failover/configs/config.d/part_log.xml diff --git a/src/Disks/DiskDecorator.cpp b/src/Disks/DiskDecorator.cpp index 7f2ea58d7cf..aaa54005f6f 100644 --- a/src/Disks/DiskDecorator.cpp +++ b/src/Disks/DiskDecorator.cpp @@ -180,4 +180,9 @@ void DiskDecorator::sync(int fd) const delegate->sync(fd); } +Executor & DiskDecorator::getExecutor() +{ + return delegate->getExecutor(); +} + } diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index f1ddfff4952..ffaf0919776 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -46,6 +46,7 @@ public: void close(int fd) const override; void sync(int fd) const override; const String getType() const override { return delegate->getType(); } + Executor & getExecutor() override; protected: DiskPtr delegate; diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 688c1dfad42..ac0f5a2ae8f 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -195,10 +195,10 @@ public: /// Invoked when Global Context is shutdown. virtual void shutdown() { } -private: /// Returns executor to perform asynchronous operations. - Executor & getExecutor() { return *executor; } + virtual Executor & getExecutor() { return *executor; } +private: std::unique_ptr executor; }; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 40f12428561..1f68c08b6e6 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -953,7 +953,10 @@ void IMergeTreeDataPart::makeCloneOnDiskDetached(const ReservationPtr & reservat String path_to_clone = storage.relative_data_path + "detached/"; if (reserved_disk->exists(path_to_clone + relative_path)) - throw Exception("Path " + fullPath(reserved_disk, path_to_clone + relative_path) + " already exists. Can not clone ", ErrorCodes::DIRECTORY_ALREADY_EXISTS); + { + LOG_WARNING(storage.log, "Path " + fullPath(reserved_disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again."); + reserved_disk->removeRecursive(path_to_clone + relative_path + '/'); + } reserved_disk->createDirectory(path_to_clone); volume->getDisk()->copy(getFullRelativePath(), reserved_disk, path_to_clone); diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/instant_moves.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/instant_moves.xml new file mode 100644 index 00000000000..7b68c6946ca --- /dev/null +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/instant_moves.xml @@ -0,0 +1,4 @@ + + 0.5 + 0.5 + diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/part_log.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/part_log.xml new file mode 100644 index 00000000000..fb449ee4ad5 --- /dev/null +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/part_log.xml @@ -0,0 +1,8 @@ + + + + system + part_log
+ 7500 + + diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml index d4d53ab5efe..bcd5ef97a09 100644 --- a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml @@ -12,6 +12,7 @@ 0 + @@ -21,6 +22,16 @@ + + +
+ default +
+ + s3 + + + diff --git a/tests/integration/test_merge_tree_s3_failover/test.py b/tests/integration/test_merge_tree_s3_failover/test.py index 59006e2e99a..8e37164c721 100644 --- a/tests/integration/test_merge_tree_s3_failover/test.py +++ b/tests/integration/test_merge_tree_s3_failover/test.py @@ -45,7 +45,10 @@ def cluster(): try: cluster = ClickHouseCluster(__file__) cluster.add_instance("node", - main_configs=["configs/config.d/log_conf.xml", "configs/config.d/storage_conf.xml"], + main_configs=["configs/config.d/log_conf.xml", + "configs/config.d/storage_conf.xml", + "configs/config.d/instant_moves.xml", + "configs/config.d/part_log.xml"], with_minio=True) logging.info("Starting cluster...") cluster.start() @@ -115,3 +118,60 @@ def test_write_failover(cluster, min_bytes_for_wide_part, request_count): assert node.query("CHECK TABLE s3_failover_test") == '1\n' assert node.query("SELECT * FROM s3_failover_test FORMAT Values") == data + + +# Check that second data part move is ended successfully if first attempt was failed. +def test_move_failover(cluster): + node = cluster.instances["node"] + + node.query( + """ + CREATE TABLE s3_failover_test ( + dt DateTime, + id Int64, + data String + ) ENGINE=MergeTree() + ORDER BY id + TTL dt + INTERVAL 3 SECOND TO VOLUME 'external' + SETTINGS storage_policy='s3_cold' + """ + ) + + # Fail a request to S3 to break first TTL move. + fail_request(cluster, 1) + + node.query("INSERT INTO s3_failover_test VALUES (now() - 2, 0, 'data'), (now() - 2, 1, 'data')") + + # Wait for part move to S3. + max_attempts = 10 + for attempt in range(max_attempts + 1): + disk = node.query("SELECT disk_name FROM system.parts WHERE table='s3_failover_test' LIMIT 1") + if disk != "s3\n": + if attempt == max_attempts: + assert disk == "s3\n", "Expected move to S3 while part still on disk " + disk + else: + time.sleep(1) + else: + break + + # Ensure part_log is created. + node.query("SYSTEM FLUSH LOGS") + + # There should be 2 attempts to move part. + assert node.query(""" + SELECT count(*) FROM system.part_log + WHERE event_type='MovePart' AND table='s3_failover_test' + """) == '2\n' + + # First attempt should be failed with expected error. + exception = node.query(""" + SELECT exception FROM system.part_log + WHERE event_type='MovePart' AND table='s3_failover_test' + ORDER BY event_time + LIMIT 1 + """) + assert exception.find("Expected Error") != -1, exception + + # Ensure data is not corrupted. + assert node.query("CHECK TABLE s3_failover_test") == '1\n' + assert node.query("SELECT id,data FROM s3_failover_test FORMAT Values") == "(0,'data'),(1,'data')" From 867216103f92fd60f566610e829146c694349147 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 21 Sep 2020 18:13:01 +0800 Subject: [PATCH 128/411] Extend trivial count optimization. --- src/Interpreters/InterpreterSelectQuery.cpp | 13 +++++- src/Interpreters/TreeRewriter.cpp | 20 ++++++++- src/Storages/IStorage.h | 3 ++ src/Storages/MergeTree/KeyCondition.cpp | 33 +++++++++++--- src/Storages/MergeTree/KeyCondition.h | 12 ++++- src/Storages/StorageMergeTree.cpp | 33 ++++++++++++++ src/Storages/StorageMergeTree.h | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 30 +++++++++++++ src/Storages/StorageReplicatedMergeTree.h | 1 + .../00636_partition_key_parts_pruning.sh | 14 +++--- ...l_count_with_partition_predicate.reference | 9 ++++ ...trivial_count_with_partition_predicate.sql | 45 +++++++++++++++++++ 12 files changed, 196 insertions(+), 18 deletions(-) create mode 100644 tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.reference create mode 100644 tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 823808759a2..78223e56d2a 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1115,6 +1115,7 @@ void InterpreterSelectQuery::executeFetchColumns( /// Optimization for trivial query like SELECT count() FROM table. bool optimize_trivial_count = syntax_analyzer_result->optimize_trivial_count + && (settings.max_parallel_replicas <= 1) && storage && !filter_info && processing_stage == QueryProcessingStage::FetchColumns @@ -1126,7 +1127,17 @@ void InterpreterSelectQuery::executeFetchColumns( { const auto & desc = query_analyzer->aggregates()[0]; const auto & func = desc.function; - std::optional num_rows = storage->totalRows(); + std::optional num_rows{}; + if (!query.prewhere() && !query.where()) + num_rows = storage->totalRows(); + else // It's possible to optimize count() given only partition predicates + { + SelectQueryInfo temp_query_info; + temp_query_info.query = query_ptr; + temp_query_info.syntax_analyzer_result = syntax_analyzer_result; + temp_query_info.sets = query_analyzer->getPreparedSets(); + num_rows = storage->totalRowsByPartitionPredicate(temp_query_info, *context); + } if (num_rows) { AggregateFunctionCount & agg_count = static_cast(*func); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 0b2f8ac3eb7..b45533f7a7b 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -478,6 +478,24 @@ void TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select /// If we have no information about columns sizes, choose a column of minimum size of its data type. required.insert(ExpressionActions::getSmallestColumn(source_columns)); } + else if (is_select && metadata_snapshot) + { + const auto & partition_desc = metadata_snapshot->getPartitionKey(); + if (partition_desc.expression) + { + const auto & partition_source_columns = partition_desc.expression->getRequiredColumns(); + optimize_trivial_count = true; + for (const auto & required_column : required) + { + if (std::find(partition_source_columns.begin(), partition_source_columns.end(), required_column) + == partition_source_columns.end()) + { + optimize_trivial_count = false; + break; + } + } + } + } NameSet unknown_required_source_columns = required; @@ -620,7 +638,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( if (result.optimize_trivial_count) result.optimize_trivial_count = settings.optimize_trivial_count_query && - !select_query->where() && !select_query->prewhere() && !select_query->groupBy() && !select_query->having() && + !select_query->groupBy() && !select_query->having() && !select_query->sampleSize() && !select_query->sampleOffset() && !select_query->final() && (tables_with_columns.size() < 2 || isLeft(result.analyzed_join->kind())); diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 4a2e70aa84b..836e2d7dcf1 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -463,6 +463,9 @@ public: /// Does takes underlying Storage (if any) into account. virtual std::optional totalRows() const { return {}; } + /// Same as above but also take partition predicate into account. + virtual std::optional totalRowsByPartitionPredicate(const SelectQueryInfo &, const Context &) const { return {}; } + /// If it is possible to quickly determine exact number of bytes for the table on storage: /// - memory (approximated, resident) /// - disk (compressed) diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index bd45d970a7c..aae5cf3becf 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -368,8 +368,10 @@ KeyCondition::KeyCondition( const SelectQueryInfo & query_info, const Context & context, const Names & key_column_names, - const ExpressionActionsPtr & key_expr_) - : key_expr(key_expr_), prepared_sets(query_info.sets) + const ExpressionActionsPtr & key_expr_, + bool single_point_, + bool strict_) + : key_expr(key_expr_), prepared_sets(query_info.sets), single_point(single_point_), strict(strict_) { for (size_t i = 0, size = key_column_names.size(); i < size; ++i) { @@ -551,6 +553,18 @@ bool KeyCondition::canConstantBeWrappedByMonotonicFunctions( Field & out_value, DataTypePtr & out_type) { + /// We don't look for inversed key transformations when strict is true, which is required for trivial count(). + /// Consider the following test case: + /// + /// create table test1(p DateTime, k int) engine MergeTree partition by toDate(p) order by k; + /// insert into test1 values ('2020-09-01 00:01:02', 1), ('2020-09-01 20:01:03', 2), ('2020-09-02 00:01:03', 3); + /// select count() from test1 where p > toDateTime('2020-09-01 10:00:00'); + /// + /// toDate(DateTime) is always monotonic, but we cannot relaxing the predicates to be + /// >= toDate(toDateTime('2020-09-01 10:00:00')), which returns 3 instead of the right count: 2. + if (strict) + return false; + String expr_name = node->getColumnName(); const auto & sample_block = key_expr->getSampleBlock(); if (!sample_block.has(expr_name)) @@ -734,7 +748,8 @@ bool KeyCondition::isKeyPossiblyWrappedByMonotonicFunctions( arguments.push_back({ nullptr, key_column_type, "" }); auto func = func_builder->build(arguments); - if (!func || !func->hasInformationAboutMonotonicity()) + /// If we know the given range only contains one value, then we treat all functions as positive monotonic. + if (!func || (!single_point && !func->hasInformationAboutMonotonicity())) return false; key_column_type = func->getReturnType(); @@ -1163,13 +1178,16 @@ BoolMask KeyCondition::checkInRange( std::optional KeyCondition::applyMonotonicFunctionsChainToRange( Range key_range, const MonotonicFunctionsChain & functions, - DataTypePtr current_type) + DataTypePtr current_type, + bool single_point) { for (const auto & func : functions) { /// We check the monotonicity of each function on a specific range. - IFunction::Monotonicity monotonicity = func->getMonotonicityForRange( - *current_type.get(), key_range.left, key_range.right); + /// If we know the given range only contains one value, then we treat all functions as positive monotonic. + IFunction::Monotonicity monotonicity = single_point + ? IFunction::Monotonicity{true} + : func->getMonotonicityForRange(*current_type.get(), key_range.left, key_range.right); if (!monotonicity.is_monotonic) { @@ -1299,7 +1317,8 @@ BoolMask KeyCondition::checkInHyperrectangle( std::optional new_range = applyMonotonicFunctionsChainToRange( *key_range, element.monotonic_functions_chain, - data_types[element.key_column] + data_types[element.key_column], + single_point ); if (!new_range) diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h index fdae7335646..04591f197bb 100644 --- a/src/Storages/MergeTree/KeyCondition.h +++ b/src/Storages/MergeTree/KeyCondition.h @@ -232,7 +232,9 @@ public: const SelectQueryInfo & query_info, const Context & context, const Names & key_column_names, - const ExpressionActionsPtr & key_expr); + const ExpressionActionsPtr & key_expr, + bool single_point_ = false, + bool strict_ = false); /// Whether the condition and its negation are feasible in the direct product of single column ranges specified by `hyperrectangle`. BoolMask checkInHyperrectangle( @@ -307,7 +309,8 @@ public: static std::optional applyMonotonicFunctionsChainToRange( Range key_range, const MonotonicFunctionsChain & functions, - DataTypePtr current_type); + DataTypePtr current_type, + bool single_point = false); bool matchesExactContinuousRange() const; @@ -413,6 +416,11 @@ private: ColumnIndices key_columns; ExpressionActionsPtr key_expr; PreparedSets prepared_sets; + + // If true, always allow key_expr to be wrapped by function + bool single_point; + // If true, do not use always_monotonic information to transform constants + bool strict; }; } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 55fb42b550e..e2d5daecada 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -189,6 +189,39 @@ std::optional StorageMergeTree::totalRows() const return getTotalActiveSizeInRows(); } +std::optional StorageMergeTree::totalRowsByPartitionPredicate(const SelectQueryInfo & query_info, const Context & context) const +{ + auto metadata_snapshot = getInMemoryMetadataPtr(); + const auto & partition_key = metadata_snapshot->getPartitionKey(); + Names partition_key_columns = partition_key.column_names; + KeyCondition key_condition( + query_info, context, partition_key_columns, partition_key.expression, true /* single_point */, true /* strict */); + if (key_condition.alwaysUnknownOrTrue()) + return {}; + std::unordered_map partition_filter_map; + size_t res = 0; + auto lock = lockParts(); + for (const auto & part : getDataPartsStateRange(DataPartState::Committed)) + { + if (part->isEmpty()) + continue; + const auto & partition_id = part->info.partition_id; + bool is_valid; + if (auto it = partition_filter_map.find(partition_id); it != partition_filter_map.end()) + is_valid = it->second; + else + { + const auto & partition_value = part->partition.value; + std::vector index_value(partition_value.begin(), partition_value.end()); + is_valid = key_condition.mayBeTrueInRange(partition_value.size(), index_value.data(), index_value.data(), partition_key.data_types); + partition_filter_map.emplace(partition_id, is_valid); + } + if (is_valid) + res += part->rows_count; + } + return res; +} + std::optional StorageMergeTree::totalBytes() const { return getTotalActiveSizeInBytes(); diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 5662f9e0088..7d092602703 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -47,6 +47,7 @@ public: unsigned num_streams) override; std::optional totalRows() const override; + std::optional totalRowsByPartitionPredicate(const SelectQueryInfo &, const Context &) const override; std::optional totalBytes() const override; BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) override; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 9613bd5111d..9bd749a32ff 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3563,6 +3563,36 @@ std::optional StorageReplicatedMergeTree::totalRows() const return res; } +std::optional StorageReplicatedMergeTree::totalRowsByPartitionPredicate(const SelectQueryInfo & query_info, const Context & context) const +{ + auto metadata_snapshot = getInMemoryMetadataPtr(); + const auto & partition_key = metadata_snapshot->getPartitionKey(); + Names partition_key_columns = partition_key.column_names; + KeyCondition key_condition( + query_info, context, partition_key_columns, partition_key.expression, true /* single_point */, true /* strict */); + if (key_condition.alwaysUnknownOrTrue()) + return {}; + std::unordered_map partition_filter_map; + size_t res = 0; + foreachCommittedParts([&](auto & part) + { + const auto & partition_id = part->info.partition_id; + bool is_valid; + if (auto it = partition_filter_map.find(partition_id); it != partition_filter_map.end()) + is_valid = it->second; + else + { + const auto & partition_value = part->partition.value; + std::vector index_value(partition_value.begin(), partition_value.end()); + is_valid = key_condition.mayBeTrueInRange(partition_value.size(), index_value.data(), index_value.data(), partition_key.data_types); + partition_filter_map.emplace(partition_id, is_valid); + } + if (is_valid) + res += part->rows_count; + }); + return res; +} + std::optional StorageReplicatedMergeTree::totalBytes() const { UInt64 res = 0; diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index d851082d5c2..1ed2066f221 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -97,6 +97,7 @@ public: unsigned num_streams) override; std::optional totalRows() const override; + std::optional totalRowsByPartitionPredicate(const SelectQueryInfo & query_info, const Context & context) const override; std::optional totalBytes() const override; BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) override; diff --git a/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh b/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh index aad54fc0a73..8150d52abc9 100755 --- a/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh +++ b/tests/queries/0_stateless/00636_partition_key_parts_pruning.sh @@ -30,14 +30,14 @@ ${CLICKHOUSE_CLIENT} --query="INSERT INTO composite_partition_key VALUES \ ${CLICKHOUSE_CLIENT} --query="INSERT INTO composite_partition_key VALUES \ (301, 20, 3), (302, 21, 3), (303, 22, 3)" -${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a > 400 FORMAT XML" | grep -F rows_read | sed 's/^[ \t]*//g' -${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE b = 11 FORMAT XML" | grep -F rows_read | sed 's/^[ \t]*//g' -${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE c = 4 FORMAT XML" | grep -F rows_read | sed 's/^[ \t]*//g' +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a > 400 FORMAT XML SETTINGS optimize_trivial_count_query = 0" | grep -F rows_read | sed 's/^[ \t]*//g' +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE b = 11 FORMAT XML SETTINGS optimize_trivial_count_query = 0" | grep -F rows_read | sed 's/^[ \t]*//g' +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE c = 4 FORMAT XML SETTINGS optimize_trivial_count_query = 0" | grep -F rows_read | sed 's/^[ \t]*//g' -${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a < 200 AND c = 2 FORMAT XML" | grep -F rows_read | sed 's/^[ \t]*//g' -${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a = 301 AND b < 20 FORMAT XML" | grep -F rows_read | sed 's/^[ \t]*//g' -${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE b >= 12 AND c = 2 FORMAT XML" | grep -F rows_read | sed 's/^[ \t]*//g' +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a < 200 AND c = 2 FORMAT XML SETTINGS optimize_trivial_count_query = 0" | grep -F rows_read | sed 's/^[ \t]*//g' +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a = 301 AND b < 20 FORMAT XML SETTINGS optimize_trivial_count_query = 0" | grep -F rows_read | sed 's/^[ \t]*//g' +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE b >= 12 AND c = 2 FORMAT XML SETTINGS optimize_trivial_count_query = 0" | grep -F rows_read | sed 's/^[ \t]*//g' -${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a = 301 AND b = 21 AND c = 3 FORMAT XML" | grep -F rows_read | sed 's/^[ \t]*//g' +${CLICKHOUSE_CLIENT} --query="SELECT count() FROM composite_partition_key WHERE a = 301 AND b = 21 AND c = 3 FORMAT XML SETTINGS optimize_trivial_count_query = 0" | grep -F rows_read | sed 's/^[ \t]*//g' ${CLICKHOUSE_CLIENT} --query="DROP TABLE composite_partition_key" diff --git a/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.reference b/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.reference new file mode 100644 index 00000000000..4fe8dbf8cfb --- /dev/null +++ b/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.reference @@ -0,0 +1,9 @@ +0 +0 +2 +1 +1 +0 +2 +0 +3 diff --git a/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql b/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql new file mode 100644 index 00000000000..110e883803d --- /dev/null +++ b/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql @@ -0,0 +1,45 @@ +drop table if exists test1; + +create table test1(p DateTime, k int) engine MergeTree partition by toDate(p) order by k; +insert into test1 values ('2020-09-01 00:01:02', 1), ('2020-09-01 20:01:03', 2), ('2020-09-02 00:01:03', 3); + +set max_rows_to_read = 1; +-- non-optimized +select count() from test1 settings max_parallel_replicas = 3; -- { serverError 158; } +-- optimized (toYear is monotonic and we provide the partition expr as is) +select count() from test1 where toYear(toDate(p)) = 1999; +-- non-optimized (toDate(DateTime) is always monotonic, but we cannot relaxing the predicates to do trivial count()) +select count() from test1 where p > toDateTime('2020-09-01 10:00:00'); -- { serverError 158; } +-- optimized (partition expr wrapped with non-monotonic functions) +select count() FROM test1 where toDate(p) = '2020-09-01' and sipHash64(toString(toDate(p))) % 2 = 1; +select count() FROM test1 where toDate(p) = '2020-09-01' and sipHash64(toString(toDate(p))) % 2 = 0; +-- non-optimized (some predicate depends on non-partition_expr columns) +select count() FROM test1 where toDate(p) = '2020-09-01' and k = 2; -- { serverError 158; } +-- optimized +select count() from test1 where toDate(p) > '2020-09-01'; + +create table test_tuple(p DateTime, i int, j int) engine MergeTree partition by (toDate(p), i) order by j; + +insert into test_tuple values ('2020-09-01 00:01:02', 1, 2), ('2020-09-01 00:01:03', 2, 3), ('2020-09-02 00:01:03', 3, 4); + +-- optimized +select count() from test_tuple where toDate(p) > '2020-09-01'; +-- optimized +select count() from test_tuple where toDate(p) > '2020-09-01' and i = 1; +-- optimized +select count() from test_tuple where i > 1; +-- optimized +select count() from test_tuple where i < 1; + +create table test_two_args(i int, j int, k int) engine MergeTree partition by i + j order by k; + +insert into test_two_args values (1, 2, 3), (2, 1, 3), (0, 3, 4); + +-- optimized +select count() from test_two_args where i + j = 3; +-- non-optimized +select count() from test_two_args where i = 1; -- { serverError 158; } + +drop table test1; +drop table test_tuple; +drop table test_two_args; From 1103ce0d785fc9e2ba17b3cdbc15e616ad783c1a Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Thu, 8 Oct 2020 15:36:03 -0400 Subject: [PATCH 129/411] Updating exit codes and messages in failing tests. --- .../authentication/tests/server_config.py | 32 ++++++++++++------- .../ldap/authentication/tests/user_config.py | 4 +-- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/testflows/ldap/authentication/tests/server_config.py b/tests/testflows/ldap/authentication/tests/server_config.py index 5658f7a9399..80f2a496b0e 100644 --- a/tests/testflows/ldap/authentication/tests/server_config.py +++ b/tests/testflows/ldap/authentication/tests/server_config.py @@ -28,7 +28,8 @@ def invalid_host(self): servers = {"foo": {"host": "foo", "port": "389", "enable_tls": "no"}} users = [{ "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Can't contact LDAP server" + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -44,7 +45,8 @@ def empty_host(self): servers = {"foo": {"host": "", "port": "389", "enable_tls": "no"}} users = [{ "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 36, "message": "DB::Exception: LDAP server 'foo' is not configured." + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -60,7 +62,8 @@ def missing_host(self): servers = {"foo": {"port": "389", "enable_tls": "no"}} users = [{ "server": "foo", "username": "user1", "password": "user1", "login": True, - "exitcode": 36, "message": "DB::Exception: LDAP server 'foo' is not configured." + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -75,7 +78,8 @@ def invalid_port(self): servers = {"openldap1": {"host": "openldap1", "port": "3890", "enable_tls": "no"}} users = [{ "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Can't contact LDAP server." + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -93,7 +97,8 @@ def invalid_auth_dn_prefix(self): }} users = [{ "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Invalid DN syntax: invalid DN" + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -110,7 +115,8 @@ def invalid_auth_dn_suffix(self): }} users = [{ "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 20, "message": "DB::Exception: Invalid DN syntax: invalid DN" + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -127,7 +133,8 @@ def invalid_enable_tls_value(self): }} users = [{ "server": "openldap1", "username": "user1", "password": "user1", "login": True, - "exitcode": 36, "message": "DB::Exception: LDAP server 'openldap1' is not configured" + "exitcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -148,7 +155,8 @@ def invalid_tls_require_cert_value(self): }} users = [{ "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 36, "message": "DB::Exception: LDAP server 'openldap2' is not configured" + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -167,8 +175,8 @@ def empty_ca_cert_dir(self): }} users = [{ "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 20, - "message": "DB::Exception: Can't contact LDAP server: error:14000086:SSL routines::certificate verify failed (self signed certificate in certificate chain" + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) @@ -187,8 +195,8 @@ def empty_ca_cert_file(self): }} users = [{ "server": "openldap2", "username": "user2", "password": "user2", "login": True, - "exitcode": 20, - "message": "Received from localhost:9000. DB::Exception: Can't contact LDAP server: error:14000086:SSL routines::certificate verify failed (self signed certificate in certificate chain)" + "exitcode": 4, + "message": "DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) diff --git a/tests/testflows/ldap/authentication/tests/user_config.py b/tests/testflows/ldap/authentication/tests/user_config.py index 391e4ee24c5..36ed33ed17a 100644 --- a/tests/testflows/ldap/authentication/tests/user_config.py +++ b/tests/testflows/ldap/authentication/tests/user_config.py @@ -54,8 +54,8 @@ def empty_server_not_defined(self, timeout=20): "auth_dn_prefix": "cn=", "auth_dn_suffix": ",ou=users,dc=company,dc=com" }} users = [{"server": "foo", "username": "user1", "password": "user1", "login": True, - "errorcode": 36, - "message": "DB::Exception: LDAP server 'foo' is not configured" + "errorcode": 4, + "message": "DB::Exception: user1: Authentication failed: password is incorrect or there is no user with such name" }] login(servers, *users) From e348ec17b2ae9afb7838ad131049966c2a16e75a Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 9 Oct 2020 00:57:23 +0400 Subject: [PATCH 130/411] Refactor role handling --- src/Access/LDAPAccessStorage.cpp | 110 ++++++++++++++++--------------- src/Access/LDAPAccessStorage.h | 3 +- 2 files changed, 59 insertions(+), 54 deletions(-) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index d1dbf10cfbc..a3945686ff7 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include @@ -64,54 +66,66 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m return this->processRoleChange(id, entity); } ); + + /// Update `roles_of_interests` with initial values. + for (const auto & role_name : default_role_names) + { + if (auto role_id = access_control_manager->find(role_name)) + roles_of_interest.emplace(*role_id, role_name); + } } void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr & entity) { std::scoped_lock lock(mutex); - auto role_ptr = typeid_cast>(entity); - if (role_ptr) - { - if (default_role_names.find(role_ptr->getName()) != default_role_names.end()) - { - auto update_func = [&id](const AccessEntityPtr & cached_entity) -> AccessEntityPtr - { - auto user_ptr = typeid_cast>(cached_entity); - if (user_ptr && user_ptr->granted_roles.roles.find(id) == user_ptr->granted_roles.roles.end()) - { - auto clone = user_ptr->clone(); - auto user_clone_ptr = typeid_cast>(clone); - user_clone_ptr->granted_roles.grant(id); - return user_clone_ptr; - } - return cached_entity; - }; - memory_storage.update(memory_storage.findAll(), update_func); - roles_of_interest.insert(id); - } + /// Update `roles_of_interests`. + auto role = typeid_cast>(entity); + bool need_to_update_users = false; + + if (role && default_role_names.contains(role->getName())) + { + /// If a role was created with one of the `default_role_names` or renamed to one of the `default_role_names`, + /// then set `need_to_update_users`. + need_to_update_users = roles_of_interest.insert_or_assign(id, role->getName()).second; } else { - if (roles_of_interest.find(id) != roles_of_interest.end()) - { - auto update_func = [&id](const AccessEntityPtr & cached_entity) -> AccessEntityPtr - { - auto user_ptr = typeid_cast>(cached_entity); - if (user_ptr && user_ptr->granted_roles.roles.find(id) != user_ptr->granted_roles.roles.end()) - { - auto clone = user_ptr->clone(); - auto user_clone_ptr = typeid_cast>(clone); - user_clone_ptr->granted_roles.revoke(id); - return user_clone_ptr; - } - return cached_entity; - }; + /// If a role was removed or renamed to a name which isn't contained in the `default_role_names`, + /// then set `need_to_update_users`. + need_to_update_users = roles_of_interest.erase(id) > 0; + } - memory_storage.update(memory_storage.findAll(), update_func); - roles_of_interest.erase(id); - } + /// Update users which have been created. + if (need_to_update_users) + { + auto update_func = [this] (const AccessEntityPtr & entity_) -> AccessEntityPtr + { + if (auto user = typeid_cast>(entity_)) + { + auto changed_user = typeid_cast>(user->clone()); + auto & granted_roles = changed_user->granted_roles.roles; + granted_roles.clear(); + boost::range::copy(roles_of_interest | boost::adaptors::map_keys, std::inserter(granted_roles, granted_roles.end())); + return changed_user; + } + return entity_; + }; + memory_storage.update(memory_storage.findAll(), update_func); + } +} + + +void LDAPAccessStorage::checkAllDefaultRoleNamesFoundNoLock() const +{ + boost::container::flat_set role_names_of_interest; + boost::range::copy(roles_of_interest | boost::adaptors::map_values, std::inserter(role_names_of_interest, role_names_of_interest.end())); + + for (const auto & role_name : default_role_names) + { + if (!role_names_of_interest.contains(role_name)) + throwDefaultRoleNotFound(role_name); } } @@ -257,15 +271,10 @@ UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & passw if (!isAddressAllowedImpl(*user, address)) throwAddressNotAllowed(address); - for (const auto& role_name : default_role_names) - { - std::optional role_id = access_control_manager->find(role_name); - if (!role_id) - throwDefaultRoleNotFound(role_name); + checkAllDefaultRoleNamesFoundNoLock(); - roles_of_interest.insert(role_id.value()); - user->granted_roles.grant(role_id.value()); - } + auto & granted_roles = user->granted_roles.roles; + boost::range::copy(roles_of_interest | boost::adaptors::map_keys, std::inserter(granted_roles, granted_roles.end())); return memory_storage.insert(user); } @@ -287,15 +296,10 @@ UUID LDAPAccessStorage::getIDOfLoggedUserImpl(const String & user_name) const user->authentication = Authentication(Authentication::Type::LDAP_SERVER); user->authentication.setServerName(ldap_server); - for (const auto& role_name : default_role_names) - { - std::optional role_id = access_control_manager->find(role_name); - if (!role_id) - throwDefaultRoleNotFound(role_name); + checkAllDefaultRoleNamesFoundNoLock(); - roles_of_interest.insert(role_id.value()); - user->granted_roles.grant(role_id.value()); - } + auto & granted_roles = user->granted_roles.roles; + boost::range::copy(roles_of_interest | boost::adaptors::map_keys, std::inserter(granted_roles, granted_roles.end())); return memory_storage.insert(user); } diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index 1e6e0713568..7ac37b9142c 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -55,6 +55,7 @@ private: // IAccessStorage implementations. private: void setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix); void processRoleChange(const UUID & id, const AccessEntityPtr & entity); + void checkAllDefaultRoleNamesFoundNoLock() const; [[noreturn]] static void throwDefaultRoleNotFound(const String & role_name); @@ -62,7 +63,7 @@ private: AccessControlManager * access_control_manager = nullptr; String ldap_server; std::set default_role_names; - mutable std::set roles_of_interest; + std::map roles_of_interest; ext::scope_guard role_change_subscription; mutable MemoryAccessStorage memory_storage; }; From f4f79aa84a73397ef93331fc9432527fb367899f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 3 Oct 2020 22:55:02 +0300 Subject: [PATCH 131/411] Add HashTable::reserve() (for preallocated hashed/sparse_hashed dictionaries) --- src/Common/HashTable/HashTable.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Common/HashTable/HashTable.h b/src/Common/HashTable/HashTable.h index baad5d40764..897c84fe951 100644 --- a/src/Common/HashTable/HashTable.h +++ b/src/Common/HashTable/HashTable.h @@ -850,6 +850,11 @@ protected: public: + void reserve(size_t num_elements) + { + resize(num_elements); + } + /// Insert a value. In the case of any more complex values, it is better to use the `emplace` function. std::pair ALWAYS_INLINE insert(const value_type & x) { From 064f901ea8977f0f164f7c369a32574bb2e5bf99 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 24 Sep 2020 08:01:00 +0300 Subject: [PATCH 132/411] Add ability to preallocate hashtables for hashed/sparsehashed dictionaries preallocation can be used only when we know number of rows, and for this we need: - source clickhouse - no filtering (i.e. lack of ), since filtering can filter too much rows and eventually it may allocate memory that will never be used. For sparse_hash the difference is quite significant, preallocated sparse_hash hashtable allocates ~33% faster (7.5 seconds vs 5 seconds for insert, and the difference is more significant for higher number of elements): $ ninja bench-sparse_hash-run [1/1] cd /src/ch/hashtable-bench/.cmake && ...ch/hashtable-bench/.cmake/bench-sparse_hash sparse_hash/insert: 7.574 GroupingAggregated --> ResizeProcessor --> MergingAggregatedBucket --> SortingAggregated --> /// --> --> MergingAggregatedBucket --> - pipe.addTransform(std::make_shared(Block(), 1, num_merging_processors)); + pipe.resize(num_merging_processors); pipe.addSimpleTransform([params](const Block &) { diff --git a/src/Storages/StorageMemory.h b/src/Storages/StorageMemory.h index 6b525cd6dbb..25f23d131b6 100644 --- a/src/Storages/StorageMemory.h +++ b/src/Storages/StorageMemory.h @@ -21,7 +21,6 @@ namespace DB */ class StorageMemory final : public ext::shared_ptr_helper, public IStorage { -friend class MemoryBlockInputStream; friend class MemoryBlockOutputStream; friend struct ext::shared_ptr_helper; From 1c3fe943e0242b8a64c6b7f17563e9b41ab994ff Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 13 Oct 2020 10:18:30 +0800 Subject: [PATCH 151/411] Fix DelayedSource --- src/Processors/Sources/DelayedSource.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Processors/Sources/DelayedSource.cpp b/src/Processors/Sources/DelayedSource.cpp index ce23fbd8fbd..054453c4b5a 100644 --- a/src/Processors/Sources/DelayedSource.cpp +++ b/src/Processors/Sources/DelayedSource.cpp @@ -30,12 +30,12 @@ DelayedSource::DelayedSource(const Block & header, Creator processors_creator, b IProcessor::Status DelayedSource::prepare() { - /// At first, wait for main input is needed and expand pipeline. + /// At first, wait for main output is needed and expand pipeline. if (inputs.empty()) { auto & first_output = outputs.front(); - /// If main port was finished before callback was called, stop execution. + /// If main output port was finished before callback was called, stop execution. if (first_output.isFinished()) { for (auto & output : outputs) @@ -75,7 +75,7 @@ IProcessor::Status DelayedSource::prepare() input->setNeeded(); if (!input->hasData()) - return Status::PortFull; + return Status::NeedData; output->pushData(input->pullData(true)); return Status::PortFull; From dbaada559782898e6a992b048e9f6fc58df22f2a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Sep 2020 15:15:47 +0300 Subject: [PATCH 152/411] Another test (cherry picked from commit da87861285e63369bd79e176ce375a8d6ea18b85) --- .../01502_log_tinylog_deadlock_race.reference | 6 ++ .../01502_log_tinylog_deadlock_race.sh | 85 +++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference create mode 100755 tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh diff --git a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference new file mode 100644 index 00000000000..4bf85ae79f3 --- /dev/null +++ b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference @@ -0,0 +1,6 @@ +Testing TinyLog +Done TinyLog +Testing StripeLog +Done StripeLog +Testing Log +Done Log diff --git a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh new file mode 100755 index 00000000000..a5b2ff6db8f --- /dev/null +++ b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +set -e + +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + + +function thread_create { + while true; do + $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS $1 (x UInt64, s Array(Nullable(String))) ENGINE = $2" + sleep 0.0$RANDOM + done +} + +function thread_drop { + while true; do + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $1" + sleep 0.0$RANDOM + done +} + +function thread_rename { + while true; do + $CLICKHOUSE_CLIENT --query "RENAME TABLE $1 TO $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|57)' + sleep 0.0$RANDOM + done +} + +function thread_select { + while true; do + $CLICKHOUSE_CLIENT --query "SELECT * FROM $1 FORMAT Null" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done +} + +function thread_insert { + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT rand64(1), [toString(rand64(2))] FROM numbers($2)" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done +} + +function thread_insert_select { + while true; do + $CLICKHOUSE_CLIENT --query "INSERT INTO $1 SELECT * FROM $2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (60|218)' + sleep 0.0$RANDOM + done +} + +export -f thread_create +export -f thread_drop +export -f thread_rename +export -f thread_select +export -f thread_insert +export -f thread_insert_select + + +# Do randomized queries and expect nothing extraordinary happens. + +function test_with_engine { + echo "Testing $1" + + timeout 10 bash -c "thread_create t1 $1" & + timeout 10 bash -c "thread_create t2 $1" & + timeout 10 bash -c 'thread_drop t1' & + timeout 10 bash -c 'thread_drop t2' & + timeout 10 bash -c 'thread_rename t1 t2' & + timeout 10 bash -c 'thread_rename t2 t1' & + timeout 10 bash -c 'thread_select t1' & + timeout 10 bash -c 'thread_select t2' & + timeout 10 bash -c 'thread_insert t1 5' & + timeout 10 bash -c 'thread_insert t2 10' & + timeout 10 bash -c 'thread_insert_select t1 t2' & + timeout 10 bash -c 'thread_insert_select t2 t1' & + + wait + echo "Done $1" +} + +test_with_engine TinyLog +test_with_engine StripeLog +test_with_engine Log From 3e001e7ef1799b4829f3bca854b2b60b9cdddd97 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 13 Oct 2020 15:19:38 +0300 Subject: [PATCH 153/411] Update Settings.h --- src/Core/Settings.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index a7a7bd6d4b7..17c01270683 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -403,7 +403,7 @@ class IColumn; M(Bool, enable_global_with_statement, false, "Propagate WITH statements to UNION queries and all subqueries", 0) // End of COMMON_SETTINGS -// !!! please add settings related to formats into the FORMAT_FACTORY_SETTINGS below !!! +// Please add settings related to formats into the FORMAT_FACTORY_SETTINGS below. #define FORMAT_FACTORY_SETTINGS(M) \ M(Char, format_csv_delimiter, ',', "The character to be considered as a delimiter in CSV data. If setting with a string, a string has to have a length of 1.", 0) \ @@ -470,7 +470,7 @@ class IColumn; M(Bool, output_format_pretty_row_numbers, false, "Add row numbers before each row for pretty output format", 0) // End of FORMAT_FACTORY_SETTINGS -// !!! please add settings non-related to formats into the COMMON_SETTINGS above !!! +// Please add settings non-related to formats into the COMMON_SETTINGS above. #define LIST_OF_SETTINGS(M) \ COMMON_SETTINGS(M) \ From b3fc6d9b3385a47cc9553c32f1181797b64486e2 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 13 Oct 2020 16:31:02 +0300 Subject: [PATCH 154/411] fix IF [NOT] EXISTS failure --- src/Interpreters/InterpreterDropQuery.cpp | 24 ++++++++----------- src/Interpreters/InterpreterDropQuery.h | 2 +- .../01502_log_tinylog_deadlock_race.reference | 8 ++----- .../01502_log_tinylog_deadlock_race.sh | 7 +++--- 4 files changed, 17 insertions(+), 24 deletions(-) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index c70431e5238..a9bc738f614 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -47,7 +47,7 @@ BlockIO InterpreterDropQuery::execute() if (!drop.table.empty()) { if (!drop.is_dictionary) - return executeToTable({drop.database, drop.table, drop.uuid}, drop); + return executeToTable(drop); else return executeToDictionary(drop.database, drop.table, drop.kind, drop.if_exists, drop.temporary, drop.no_ddl_lock); } @@ -58,29 +58,25 @@ BlockIO InterpreterDropQuery::execute() } -BlockIO InterpreterDropQuery::executeToTable( - const StorageID & table_id_, - const ASTDropQuery & query) +BlockIO InterpreterDropQuery::executeToTable(const ASTDropQuery & query) { - if (query.temporary || table_id_.database_name.empty()) + auto table_id = StorageID(query); + if (query.temporary || table_id.database_name.empty()) { - if (context.tryResolveStorageID(table_id_, Context::ResolveExternal)) - return executeToTemporaryTable(table_id_.getTableName(), query.kind); + if (context.tryResolveStorageID(table_id, Context::ResolveExternal)) + return executeToTemporaryTable(table_id.getTableName(), query.kind); + else + table_id.database_name = context.getCurrentDatabase(); } if (query.temporary) { if (query.if_exists) return {}; - throw Exception("Temporary table " + backQuoteIfNeed(table_id_.table_name) + " doesn't exist", + throw Exception("Temporary table " + backQuoteIfNeed(table_id.table_name) + " doesn't exist", ErrorCodes::UNKNOWN_TABLE); } - auto table_id = query.if_exists ? context.tryResolveStorageID(table_id_, Context::ResolveOrdinary) - : context.resolveStorageID(table_id_, Context::ResolveOrdinary); - if (!table_id) - return {}; - auto ddl_guard = (!query.no_ddl_lock ? DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name) : nullptr); /// If table was already dropped by anyone, an exception will be thrown @@ -255,7 +251,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next()) { query.table = iterator->name(); - executeToTable({query.database, query.table}, query); + executeToTable(query); } } diff --git a/src/Interpreters/InterpreterDropQuery.h b/src/Interpreters/InterpreterDropQuery.h index b54736b5c21..c5d9aacdfd5 100644 --- a/src/Interpreters/InterpreterDropQuery.h +++ b/src/Interpreters/InterpreterDropQuery.h @@ -31,7 +31,7 @@ private: BlockIO executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists); - BlockIO executeToTable(const StorageID & table_id, const ASTDropQuery & query); + BlockIO executeToTable(const ASTDropQuery & query); BlockIO executeToDictionary(const String & database_name, const String & dictionary_name, ASTDropQuery::Kind kind, bool if_exists, bool is_temporary, bool no_ddl_lock); diff --git a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference index 4bf85ae79f3..c62a2b18918 100644 --- a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference +++ b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.reference @@ -1,6 +1,2 @@ -Testing TinyLog -Done TinyLog -Testing StripeLog -Done StripeLog -Testing Log -Done Log +Testing Memory +Done Memory diff --git a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh index a5b2ff6db8f..f0b5f0a3568 100755 --- a/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh +++ b/tests/queries/0_stateless/01502_log_tinylog_deadlock_race.sh @@ -80,6 +80,7 @@ function test_with_engine { echo "Done $1" } -test_with_engine TinyLog -test_with_engine StripeLog -test_with_engine Log +#test_with_engine TinyLog +#test_with_engine StripeLog +#test_with_engine Log +test_with_engine Memory From 2bbdfc77eaf0cc479188baa515e0d377e88d7ad1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 13 Oct 2020 16:48:00 +0300 Subject: [PATCH 155/411] Update creating sets. --- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- src/Processors/QueryPlan/CreatingSetsStep.cpp | 2 +- src/Processors/QueryPlan/CreatingSetsStep.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 8422b4c9878..1cbb9e81a47 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1907,7 +1907,7 @@ void InterpreterSelectQuery::executeSubqueriesInSetsAndJoins(QueryPlan & query_p const Settings & settings = context->getSettingsRef(); SizeLimits limits(settings.max_rows_to_transfer, settings.max_bytes_to_transfer, settings.transfer_overflow_mode); - addCreatingSetsStep(query_plan, std::move(subqueries_for_sets), limits, *context); + addCreatingSetsStep(query_plan, subqueries_for_sets, limits, *context); } diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 5868a7045f7..63df70a0201 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -108,7 +108,7 @@ void CreatingSetsStep::describePipeline(FormatSettings & settings) const } void addCreatingSetsStep( - QueryPlan & query_plan, SubqueriesForSets subqueries_for_sets, const SizeLimits & limits, const Context & context) + QueryPlan & query_plan, SubqueriesForSets & subqueries_for_sets, const SizeLimits & limits, const Context & context) { DataStreams input_streams; input_streams.emplace_back(query_plan.getCurrentDataStream()); diff --git a/src/Processors/QueryPlan/CreatingSetsStep.h b/src/Processors/QueryPlan/CreatingSetsStep.h index ec13ab2052e..c20aba4a392 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.h +++ b/src/Processors/QueryPlan/CreatingSetsStep.h @@ -48,7 +48,7 @@ private: void addCreatingSetsStep( QueryPlan & query_plan, - SubqueriesForSets subqueries_for_sets, + SubqueriesForSets & subqueries_for_sets, const SizeLimits & limits, const Context & context); From 1af0792f6770fd0b4f9d9176d489b25766c6ed10 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 13 Oct 2020 17:33:34 +0300 Subject: [PATCH 156/411] Update InterpreterSelectQuery. --- src/Interpreters/InterpreterSelectQuery.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 1cbb9e81a47..8bd0af66f25 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -304,6 +304,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (storage) view = dynamic_cast(storage.get()); + SubqueriesForSets subquery_for_sets; + auto analyze = [&] (bool try_move_to_prewhere) { /// Allow push down and other optimizations for VIEW: replace with subquery and rewrite it. @@ -346,6 +348,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( NameSet(required_result_column_names.begin(), required_result_column_names.end()), !options.only_analyze, options); + query_analyzer->getSubqueriesForSets() = std::move(subquery_for_sets); + if (!options.only_analyze) { if (query.sampleSize() && (input || input_pipe || !storage || !storage->supportsSampling())) @@ -430,6 +434,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (need_analyze_again) { + subquery_for_sets = std::move(query_analyzer->getSubqueriesForSets()); /// Do not try move conditions to PREWHERE for the second time. /// Otherwise, we won't be able to fallback from inefficient PREWHERE to WHERE later. analyze(/* try_move_to_prewhere = */ false); From 23afd74e1c403b1f2b32d3015a398c0d87847b69 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 13 Oct 2020 17:55:22 +0300 Subject: [PATCH 157/411] Update InterpreterSelectQuery. --- src/Interpreters/ExpressionAnalyzer.cpp | 5 ++++- src/Interpreters/ExpressionAnalyzer.h | 10 ++++++---- src/Interpreters/InterpreterSelectQuery.cpp | 4 +--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 2f0dee58141..d5ad79e7670 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -117,11 +117,14 @@ ExpressionAnalyzer::ExpressionAnalyzer( const TreeRewriterResultPtr & syntax_analyzer_result_, const Context & context_, size_t subquery_depth_, - bool do_global) + bool do_global, + SubqueriesForSets subqueries_for_sets_) : query(query_), context(context_), settings(context.getSettings()) , subquery_depth(subquery_depth_) , syntax(syntax_analyzer_result_) { + subqueries_for_sets = std::move(subqueries_for_sets_); + /// external_tables, subqueries_for_sets for global subqueries. /// Replaces global subqueries with the generated names of temporary tables that will be sent to remote servers. initGlobalSubqueriesAndExternalTables(do_global); diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index 0790c9f9bfb..6389d8a142c 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -93,7 +93,7 @@ public: const ASTPtr & query_, const TreeRewriterResultPtr & syntax_analyzer_result_, const Context & context_) - : ExpressionAnalyzer(query_, syntax_analyzer_result_, context_, 0, false) + : ExpressionAnalyzer(query_, syntax_analyzer_result_, context_, 0, false, {}) {} void appendExpression(ExpressionActionsChain & chain, const ASTPtr & expr, bool only_types); @@ -124,7 +124,8 @@ protected: const TreeRewriterResultPtr & syntax_analyzer_result_, const Context & context_, size_t subquery_depth_, - bool do_global_); + bool do_global_, + SubqueriesForSets subqueries_for_sets_); ASTPtr query; const Context & context; @@ -244,8 +245,9 @@ public: const StorageMetadataPtr & metadata_snapshot_, const NameSet & required_result_columns_ = {}, bool do_global_ = false, - const SelectQueryOptions & options_ = {}) - : ExpressionAnalyzer(query_, syntax_analyzer_result_, context_, options_.subquery_depth, do_global_) + const SelectQueryOptions & options_ = {}, + SubqueriesForSets subqueries_for_sets_ = {}) + : ExpressionAnalyzer(query_, syntax_analyzer_result_, context_, options_.subquery_depth, do_global_, std::move(subqueries_for_sets_)) , metadata_snapshot(metadata_snapshot_) , required_result_columns(required_result_columns_) , query_options(options_) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 8bd0af66f25..a55dc3ede24 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -346,9 +346,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( query_analyzer = std::make_unique( query_ptr, syntax_analyzer_result, *context, metadata_snapshot, NameSet(required_result_column_names.begin(), required_result_column_names.end()), - !options.only_analyze, options); - - query_analyzer->getSubqueriesForSets() = std::move(subquery_for_sets); + !options.only_analyze, options, std::move(subquery_for_sets)); if (!options.only_analyze) { From cb8d132cca22da7531f345371f75ad6d4e793d61 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 13 Oct 2020 18:00:36 +0300 Subject: [PATCH 158/411] fix deadlock with DDLGuard --- src/Interpreters/DatabaseCatalog.cpp | 27 +++++++++--- src/Interpreters/DatabaseCatalog.h | 4 +- src/Interpreters/InterpreterDropQuery.cpp | 1 + .../0_stateless/01150_ddl_guard_rwr.reference | 0 .../0_stateless/01150_ddl_guard_rwr.sh | 43 +++++++++++++++++++ 5 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/01150_ddl_guard_rwr.reference create mode 100755 tests/queries/0_stateless/01150_ddl_guard_rwr.sh diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 203e2292c08..03f0e057821 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -530,7 +530,7 @@ std::unique_ptr DatabaseCatalog::getDDLGuard(const String & database, std::unique_lock lock(ddl_guards_mutex); auto db_guard_iter = ddl_guards.try_emplace(database).first; DatabaseGuard & db_guard = db_guard_iter->second; - return std::make_unique(db_guard.first, db_guard.second, std::move(lock), table); + return std::make_unique(db_guard.first, db_guard.second, std::move(lock), table, database); } std::unique_lock DatabaseCatalog::getExclusiveDDLGuardForDatabase(const String & database) @@ -832,7 +832,7 @@ void DatabaseCatalog::waitTableFinallyDropped(const UUID & uuid) } -DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock guards_lock_, const String & elem) +DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock guards_lock_, const String & elem, const String & database_name) : map(map_), db_mutex(db_mutex_), guards_lock(std::move(guards_lock_)) { it = map.emplace(elem, Entry{std::make_unique(), 0}).first; @@ -841,14 +841,19 @@ DDLGuard::DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_locksecond.mutex); bool is_database = elem.empty(); if (!is_database) - db_mutex.lock_shared(); + { + + bool locked_database_for_read = db_mutex.try_lock_shared(); + if (!locked_database_for_read) + { + removeTableLock(); + throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} is currently dropped or renamed", database_name); + } + } } -DDLGuard::~DDLGuard() +void DDLGuard::removeTableLock() { - bool is_database = it->first.empty(); - if (!is_database) - db_mutex.unlock_shared(); guards_lock.lock(); --it->second.counter; if (!it->second.counter) @@ -858,4 +863,12 @@ DDLGuard::~DDLGuard() } } +DDLGuard::~DDLGuard() +{ + bool is_database = it->first.empty(); + if (!is_database) + db_mutex.unlock_shared(); + removeTableLock(); +} + } diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 7bc6923bde4..c6f50117564 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -51,7 +51,7 @@ public: /// NOTE: using std::map here (and not std::unordered_map) to avoid iterator invalidation on insertion. using Map = std::map; - DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock guards_lock_, const String & elem); + DDLGuard(Map & map_, std::shared_mutex & db_mutex_, std::unique_lock guards_lock_, const String & elem, const String & database_name); ~DDLGuard(); private: @@ -60,6 +60,8 @@ private: Map::iterator it; std::unique_lock guards_lock; std::unique_lock table_lock; + + void removeTableLock(); }; diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index a9bc738f614..b29f2893db9 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -247,6 +247,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS ASTDropQuery query; query.kind = kind; + query.if_exists = true; query.database = database_name; for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next()) { diff --git a/tests/queries/0_stateless/01150_ddl_guard_rwr.reference b/tests/queries/0_stateless/01150_ddl_guard_rwr.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01150_ddl_guard_rwr.sh b/tests/queries/0_stateless/01150_ddl_guard_rwr.sh new file mode 100755 index 00000000000..c14e4c38f54 --- /dev/null +++ b/tests/queries/0_stateless/01150_ddl_guard_rwr.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS test_01150" +$CLICKHOUSE_CLIENT --query "CREATE DATABASE test_01150" + +$CLICKHOUSE_CLIENT --query "CREATE TABLE test_01150.t1 (x UInt64, s Array(Nullable(String))) ENGINE = Memory" +$CLICKHOUSE_CLIENT --query "CREATE TABLE test_01150.t2 (x UInt64, s Array(Nullable(String))) ENGINE = Memory" + +function thread_detach_attach { + while true; do + $CLICKHOUSE_CLIENT --query "DETACH DATABASE test_01150" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (219)' + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "ATTACH DATABASE test_01150" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (82)' + sleep 0.0$RANDOM + done +} + +function thread_rename { + while true; do + $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t1 TO test_01150.t2_tmp, test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (81|60|57|521)' + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2 TO test_01150.t1, test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (81|60|57|521)' + sleep 0.0$RANDOM + $CLICKHOUSE_CLIENT --query "RENAME TABLE test_01150.t2_tmp TO test_01150.t2" 2>&1 | grep -v -F 'Received exception from server' | grep -v -P 'Code: (81|60|57|521)' + sleep 0.0$RANDOM + done +} + +export -f thread_detach_attach +export -f thread_rename + +timeout 20 bash -c "thread_detach_attach" & +timeout 20 bash -c 'thread_rename' & +wait +sleep 1 + +$CLICKHOUSE_CLIENT --query "ATTACH DATABASE IF NOT EXISTS test_01150" +$CLICKHOUSE_CLIENT --query "DROP DATABASE test_01150"; From 534b60e24252455260c9ee5a40abe63106b30ca6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 13 Oct 2020 18:10:30 +0300 Subject: [PATCH 159/411] Fix build. --- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- src/Processors/QueryPlan/CreatingSetsStep.cpp | 2 +- src/Processors/QueryPlan/CreatingSetsStep.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index a55dc3ede24..b1188ffed4f 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1910,7 +1910,7 @@ void InterpreterSelectQuery::executeSubqueriesInSetsAndJoins(QueryPlan & query_p const Settings & settings = context->getSettingsRef(); SizeLimits limits(settings.max_rows_to_transfer, settings.max_bytes_to_transfer, settings.transfer_overflow_mode); - addCreatingSetsStep(query_plan, subqueries_for_sets, limits, *context); + addCreatingSetsStep(query_plan, std::move(subqueries_for_sets), limits, *context); } diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 63df70a0201..5868a7045f7 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -108,7 +108,7 @@ void CreatingSetsStep::describePipeline(FormatSettings & settings) const } void addCreatingSetsStep( - QueryPlan & query_plan, SubqueriesForSets & subqueries_for_sets, const SizeLimits & limits, const Context & context) + QueryPlan & query_plan, SubqueriesForSets subqueries_for_sets, const SizeLimits & limits, const Context & context) { DataStreams input_streams; input_streams.emplace_back(query_plan.getCurrentDataStream()); diff --git a/src/Processors/QueryPlan/CreatingSetsStep.h b/src/Processors/QueryPlan/CreatingSetsStep.h index c20aba4a392..ec13ab2052e 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.h +++ b/src/Processors/QueryPlan/CreatingSetsStep.h @@ -48,7 +48,7 @@ private: void addCreatingSetsStep( QueryPlan & query_plan, - SubqueriesForSets & subqueries_for_sets, + SubqueriesForSets subqueries_for_sets, const SizeLimits & limits, const Context & context); From 5aba92a6bf17f8cdf4f77d58e3048910c6156b4d Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 13 Oct 2020 18:19:35 +0300 Subject: [PATCH 160/411] Added test. --- .../01521_global_in_prewhere_15792.reference | 1 + .../0_stateless/01521_global_in_prewhere_15792.sql | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 tests/queries/0_stateless/01521_global_in_prewhere_15792.reference create mode 100644 tests/queries/0_stateless/01521_global_in_prewhere_15792.sql diff --git a/tests/queries/0_stateless/01521_global_in_prewhere_15792.reference b/tests/queries/0_stateless/01521_global_in_prewhere_15792.reference new file mode 100644 index 00000000000..f7393e847d3 --- /dev/null +++ b/tests/queries/0_stateless/01521_global_in_prewhere_15792.reference @@ -0,0 +1 @@ +100000 diff --git a/tests/queries/0_stateless/01521_global_in_prewhere_15792.sql b/tests/queries/0_stateless/01521_global_in_prewhere_15792.sql new file mode 100644 index 00000000000..adb7bccd0df --- /dev/null +++ b/tests/queries/0_stateless/01521_global_in_prewhere_15792.sql @@ -0,0 +1,12 @@ +drop table if exists xp; +drop table if exists xp_d; + +create table xp(A Date, B Int64, S String) Engine=MergeTree partition by toYYYYMM(A) order by B; +insert into xp select '2020-01-01', number , '' from numbers(100000); + +create table xp_d as xp Engine=Distributed(test_shard_localhost, currentDatabase(), xp); + +select count() from xp_d prewhere toYYYYMM(A) global in (select toYYYYMM(min(A)) from xp_d) where B > -1; + +drop table if exists xp; +drop table if exists xp_d; From 8a64b65e51713d429bd744ca7288ef858041a2bd Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 13 Oct 2020 20:45:59 +0300 Subject: [PATCH 161/411] fix --- src/Interpreters/InterpreterDropQuery.cpp | 2 ++ .../0_stateless/01516_drop_table_stress.sh | 18 +++++++----------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index b29f2893db9..a250ab1afd4 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -88,6 +88,8 @@ BlockIO InterpreterDropQuery::executeToTable(const ASTDropQuery & query) if (query_ptr->as().is_view && !table->isView()) throw Exception("Table " + table_id.getNameForLogs() + " is not a View", ErrorCodes::LOGICAL_ERROR); + table_id = table->getStorageID(); + if (query.kind == ASTDropQuery::Kind::Detach) { context.checkAccess(table->isView() ? AccessType::DROP_VIEW : AccessType::DROP_TABLE, table_id); diff --git a/tests/queries/0_stateless/01516_drop_table_stress.sh b/tests/queries/0_stateless/01516_drop_table_stress.sh index 3d6218c4549..3e2fd613a36 100755 --- a/tests/queries/0_stateless/01516_drop_table_stress.sh +++ b/tests/queries/0_stateless/01516_drop_table_stress.sh @@ -12,21 +12,17 @@ function drop_database() function drop_table() { - ${CLICKHOUSE_CLIENT} -nm <&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db_01516.data1;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db_01516.data2;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" } function create() { - ${CLICKHOUSE_CLIENT} -nm <&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS db_01516.data2 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS db_01516.data3 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" } for _ in {1..100}; do From 744013d4b8e1624c528697b594982d3b696345d6 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Tue, 13 Oct 2020 21:46:15 +0300 Subject: [PATCH 162/411] test in comparison with scipy --- .../AggregateFunctionStudentTTest.h | 56 ++++++++++++++----- .../AggregateFunctionWelchTTest.h | 43 ++++++++++++-- src/AggregateFunctions/ya.make | 4 +- .../0_stateless/01322_student_ttest.reference | 4 ++ .../0_stateless/01322_student_ttest.sql | 19 +++++++ .../0_stateless/01322_welch_ttest.reference | 10 +++- .../queries/0_stateless/01322_welch_ttest.sql | 27 +++++++-- 7 files changed, 137 insertions(+), 26 deletions(-) create mode 100644 tests/queries/0_stateless/01322_student_ttest.reference create mode 100644 tests/queries/0_stateless/01322_student_ttest.sql diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.h b/src/AggregateFunctions/AggregateFunctionStudentTTest.h index b03f9178709..ac05a11d334 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.h +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.h @@ -115,18 +115,22 @@ struct AggregateFunctionStudentTTestData final throw Exception("Division by zero encountered in Aggregate function StudentTTest", ErrorCodes::BAD_ARGUMENTS); } - if (mean_x - mean_y < 1e-8) - { - return static_cast(0.0); - } - return std::pow(mean_x - mean_y, 2) / getStandartErrorSquared(); } + Float64 getTStatistic() const + { + if (size_x == 0 || size_y == 0) + { + throw Exception("Division by zero encountered in Aggregate function StudentTTest", ErrorCodes::BAD_ARGUMENTS); + } + + return (mean_x - mean_y) / std::sqrt(getStandartErrorSquared()); + } Float64 getStandartErrorSquared() const { - return getSSquared() * (1 / size_x + 1 / size_y); + return getSSquared() * (1.0 / static_cast(size_x) + 1.0 / static_cast(size_y)); } Float64 getDegreesOfFreedom() const @@ -150,20 +154,23 @@ struct AggregateFunctionStudentTTestData final { const Float64 v = getDegreesOfFreedom(); const Float64 t = getTStatisticSquared(); - std::cout << "getDegreesOfFreedom " << v << " getTStatisticSquared " << t << std::endl; + std::cout << "getDegreesOfFreedom() " << getDegreesOfFreedom() << std::endl; + std::cout << "getTStatisticSquared() " << getTStatisticSquared() << std::endl; auto f = [&v] (double x) { return std::pow(x, v/2 - 1) / std::sqrt(1 - x); }; Float64 numenator = integrateSimpson(0, v / (t + v), f); Float64 denominator = std::exp(std::lgammal(v/2) + std::lgammal(0.5) - std::lgammal(v/2 + 0.5)); + std::cout << "numenator " << numenator << std::endl; + std::cout << "denominator " << denominator << std::endl; return numenator / denominator; } - Float64 getResult() const + std::pair getResult() const { - return getPValue(); + return std::make_pair(getTStatistic(), getPValue()); } }; -/// Returns p-value +/// Returns tuple of (t-statistic, p-value) /// https://cpb-us-w2.wpmucdn.com/voices.uchicago.edu/dist/9/1193/files/2016/01/05b-TandP.pdf template class AggregateFunctionStudentTTest : @@ -182,7 +189,22 @@ public: DataTypePtr getReturnType() const override { - return std::make_shared>(); + DataTypes types + { + std::make_shared>(), + std::make_shared>(), + }; + + Strings names + { + "t-statistic", + "p-value" + }; + + return std::make_shared( + std::move(types), + std::move(names) + ); } void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override @@ -221,8 +243,16 @@ public: throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS); } - auto & column = static_cast &>(to); - column.getData().push_back(this->data(place).getResult()); + Float64 t_statistic = 0.0; + Float64 p_value = 0.0; + std::tie(t_statistic, p_value) = this->data(place).getResult(); + + auto & column_tuple = assert_cast(to); + auto & column_stat = assert_cast &>(column_tuple.getColumn(0)); + auto & column_value = assert_cast &>(column_tuple.getColumn(1)); + + column_stat.getData().push_back(t_statistic); + column_value.getData().push_back(p_value); } }; diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.h b/src/AggregateFunctions/AggregateFunctionWelchTTest.h index e445278e9e7..36641b826b1 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.h +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.h @@ -122,6 +122,16 @@ struct AggregateFunctionWelchTTestData final return std::pow(mean_x - mean_y, 2) / (getSxSquared() / size_x + getSySquared() / size_y); } + Float64 getTStatistic() const + { + if (size_x == 0 || size_y == 0) + { + throw Exception("Division by zero encountered in Aggregate function WelchTTest", ErrorCodes::BAD_ARGUMENTS); + } + + return (mean_x - mean_y) / std::sqrt(getSxSquared() / size_x + getSySquared() / size_y); + } + Float64 getDegreesOfFreedom() const { auto sx = getSxSquared(); @@ -154,9 +164,9 @@ struct AggregateFunctionWelchTTestData final return numenator / denominator; } - Float64 getResult() const + std::pair getResult() const { - return getPValue(); + return std::make_pair(getTStatistic(), getPValue()); } }; @@ -178,7 +188,22 @@ public: DataTypePtr getReturnType() const override { - return std::make_shared>(); + DataTypes types + { + std::make_shared>(), + std::make_shared>(), + }; + + Strings names + { + "t-statistic", + "p-value" + }; + + return std::make_shared( + std::move(types), + std::move(names) + ); } void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override @@ -217,8 +242,16 @@ public: throw Exception("Aggregate function " + getName() + " requires samples to be of size > 1", ErrorCodes::BAD_ARGUMENTS); } - auto & column = static_cast &>(to); - column.getData().push_back(this->data(place).getResult()); + Float64 t_statistic = 0.0; + Float64 p_value = 0.0; + std::tie(t_statistic, p_value) = this->data(place).getResult(); + + auto & column_tuple = assert_cast(to); + auto & column_stat = assert_cast &>(column_tuple.getColumn(0)); + auto & column_value = assert_cast &>(column_tuple.getColumn(1)); + + column_stat.getData().push_back(t_statistic); + column_value.getData().push_back(p_value); } }; diff --git a/src/AggregateFunctions/ya.make b/src/AggregateFunctions/ya.make index f5a869b2f78..1578e0c80ea 100644 --- a/src/AggregateFunctions/ya.make +++ b/src/AggregateFunctions/ya.make @@ -42,6 +42,7 @@ SRCS( AggregateFunctionState.cpp AggregateFunctionStatistics.cpp AggregateFunctionStatisticsSimple.cpp + AggregateFunctionStudentTTest.cpp AggregateFunctionSum.cpp AggregateFunctionSumMap.cpp AggregateFunctionTimeSeriesGroupSum.cpp @@ -49,12 +50,13 @@ SRCS( AggregateFunctionUniqCombined.cpp AggregateFunctionUniq.cpp AggregateFunctionUniqUpTo.cpp + AggregateFunctionWelchTTest.cpp AggregateFunctionWindowFunnel.cpp parseAggregateFunctionParameters.cpp registerAggregateFunctions.cpp UniqCombinedBiasData.cpp UniqVariadicHash.cpp - AggregateFunctionWelchTTest.cpp + ) END() diff --git a/tests/queries/0_stateless/01322_student_ttest.reference b/tests/queries/0_stateless/01322_student_ttest.reference new file mode 100644 index 00000000000..02e44744629 --- /dev/null +++ b/tests/queries/0_stateless/01322_student_ttest.reference @@ -0,0 +1,4 @@ +-2.610898982580138 0.00916587538237954 +-2.610898982580134 0.0091658753823792 +-28.740781574102936 7.667329672103986e-133 +-28.74078157410298 0 diff --git a/tests/queries/0_stateless/01322_student_ttest.sql b/tests/queries/0_stateless/01322_student_ttest.sql new file mode 100644 index 00000000000..3636e239fe8 --- /dev/null +++ b/tests/queries/0_stateless/01322_student_ttest.sql @@ -0,0 +1,19 @@ +DROP TABLE IF EXISTS student_ttest; + +/*Check t-stat and p-value and compare it with scipy.stat implementation + First: a=1, sigma (not sigma^2)=5, size=500 + Second: a=1, sigma = 5, size = 500 */ +CREATE TABLE student_ttest (left Float64, right Float64) ENGINE = Memory; +INSERT INTO student_ttest VALUES (0.88854,-2.90702), (-5.76966,3.61651), (6.76618,4.27458), (3.55546,4.82133), (-9.76948,9.59483), (4.92323,1.00424), (-0.36352,2.04147), (0.97018,-3.58214), (4.61656,6.59543), (-6.78292,-1.00532), (4.02008,-3.59794), (12.41838,-2.82434), (5.14417,-3.13194), (3.86836,9.90977), (-1.26199,0.523), (12.44106,4.62779), (3.28349,-2.56872), (1.77261,2.25807), (-8.94748,1.04044), (-1.01449,-2.35744), (-1.26377,10.81531), (6.79682,-9.68469), (6.32333,3.80885), (-8.21214,12.70435), (-1.68565,-6.01112), (9.7557,1.89065), (3.66694,5.08892), (1.39967,3.45254), (-5.52035,11.58151), (-10.95601,0.85035), (0.93877,8.38397), (1.45933,1.17169), (-5.40551,4.74621), (-0.83857,-1.66614), (8.50794,4.2414), (-6.68686,1.68765), (5.03099,1.85223), (1.56251,9.10111), (4.17381,-2.38085), (-2.92644,-14.79595), (5.11068,-3.8938), (2.09617,-3.41864), (11.7787,-3.15282), (6.50336,-0.56684), (0.62098,12.87997), (-7.97121,6.89115), (3.81902,12.921), (0.33151,-7.94908), (10.68584,2.45687), (0.56007,2.14957), (-7.38621,7.55081), (5.05882,-3.71534), (2.34616,-2.41064), (11.3806,-0.80734), (5.95276,-4.75651), (-3.01429,2.05241), (5.98169,-5.44523), (0.96985,-2.75054), (-1.15932,-13.00131), (2.11547,-2.74451), (2.49668,-1.39004), (-12.49569,-3.02854), (-4.94667,7.65112), (-3.64215,1.1245), (-8.35595,6.74117), (3.211,-0.75777), (2.33805,8.93451), (2.38608,-8.85559), (-3.2862,-0.36405), (-0.80454,4.02742), (-0.53483,6.88718), (10.66445,-1.05124), (-0.37619,3.04085), (0.48246,3.32368), (7.41919,1.147), (0.42414,3.41554), (-2.32335,-3.47851), (-0.70223,-0.47684), (-5.9332,-0.55605), (-1.20561,-0.17006), (3.39865,2.26218), (9.61739,12.45494), (-0.78651,-1.84097), (-4.00256,1.64934), (-7.99646,-7.07496), (8.72923,-9.99462), (0.71859,6.09954), (-1.62726,-1.05319), (5.11234,3.04757), (-0.95625,0.93899), (-3.75573,-4.63243), (1.03141,-7.43322), (-3.33588,-7.298), (1.51804,-6.59016), (-3.30935,-6.11649), (-1.97507,0.56682), (4.06456,2.00661), (3.27195,-2.79814), (-7.81761,2.84482), (-3.81785,3.65348), (-4.18311,-4.22807), (-11.33313,-4.54336), (-0.25221,-3.63343), (7.2514,2.96878), (5.30301,6.11661), (2.46762,-1.70919), (4.22716,-4.71133), (0.33916,6.09652), (9.7638,-6.83454), (-7.58684,0.18006), (-4.09888,1.51676), (4.26617,-5.31646), (-0.56744,-3.21215), (4.65125,-5.07599), (-1.30301,-2.36591), (4.53771,3.55724), (9.96929,4.8904), (3.72939,-3.22586), (-2.29818,-1.74928), (3.09417,5.73458), (0.82251,1.41188), (5.29975,2.86255), (2.8685,2.90179), (-5.73321,-2.19949), (-1.85651,1.72727), (-1.07984,1.76939), (9.78342,-0.12848), (-13.49652,-0.52), (3.68791,3.48333), (1.9998,7.8262), (1.11674,0.09099), (9.43869,7.77017), (4.07029,9.49484), (5.32715,1.42825), (7.16504,1.99624), (6.66096,4.00419), (-5.7111,1.07925), (-0.38575,-0.09987), (4.49165,-5.48733), (-3.36489,-1.83517), (7.71814,2.38059), (-1.58966,1.42075), (-1.61063,-1.11968), (-0.91602,-6.46035), (0.73459,7.66576), (-3.24463,4.6307), (6.3947,5.55989), (-2.77845,3.16684), (4.45899,5.07671), (-8.84186,-10.20566), (2.62276,-4.73386), (1.774,1.28353), (4.3692,6.75679), (0.05942,12.09895), (-1.44042,7.0049), (-2.53594,7.16156), (-2.24752,-0.64311), (4.98874,-0.66747), (4.05434,3.99996), (-2.56483,9.07298), (-6.79286,-4.60971), (-2.06165,0.70744), (-0.26056,2.56774), (1.89567,9.32424), (-3.15145,3.95087), (-7.31321,7.11372), (0.28936,-0.89284), (-0.63111,8.6155), (0.22611,-0.14141), (-9.3377,-4.86319), (-5.76638,-6.95801), (3.87306,4.44883), (6.7011,4.6156), (9.03915,-2.3579), (-1.21835,-5.1186), (0.82892,8.12819), (2.80656,2.78392), (-1.34746,-4.30221), (-1.99912,-1.47506), (0.6036,6.8598), (-3.46117,0.47636), (5.23732,0.95383), (-1.86702,7.79779), (-5.86115,-2.61767), (6.48523,-10.5087), (-7.40158,-2.74299), (-1.38913,3.87369), (4.94613,-1.07093), (-2.07818,4.98864), (2.39808,-7.50772), (4.89238,6.41316), (4.39481,1.39061), (5.20425,-3.1747), (13.62598,-2.13621), (-2.86293,-0.02203), (-3.62396,0.89025), (-4.28695,-5.87746), (4.66425,3.60026), (2.20871,-0.23178), (1.60382,-2.1897), (-9.87024,-5.85101), (-7.37302,-1.6053), (-4.17814,3.6184), (2.5148,-8.53795), (3.21708,-0.35987), (-11.48089,2.15301), (1.19821,-6.60692), (-0.07436,9.54341), (-1.10652,1.11511), (4.03395,2.94025), (-4.35883,12.05657), (2.04013,3.75156), (0.52264,7.95597), (8.14004,-0.99449), (-8.86949,0.90597), (-0.35807,-7.90627), (-10.71113,3.50863), (-2.13755,-1.47493), (0.50715,4.11671), (6.30826,10.06325), (2.37527,-1.06059), (0.20872,-1.37737), (-5.85729,-0.42542), (-4.97217,-3.90267), (-9.78434,9.35037), (-1.53277,-7.91219), (0.14827,-4.69945), (-1.053,3.63776), (1.74558,3.46492), (11.17194,2.84518), (9.35487,-3.04301), (-9.17209,8.82764), (10.41814,7.80134), (7.41206,7.87755), (3.71775,7.01035), (-2.04674,2.43271), (6.18037,11.36418), (5.6383,-6.92659), (-0.90058,5.95541), (-1.27073,3.59436), (-2.3473,5.18429), (-8.44271,4.20225), (2.75551,0.5029), (-1.15521,4.03074), (4.08722,5.23152), (-1.70399,10.65409), (7.24114,-0.69845), (-8.43976,11.70096), (-1.53052,5.80692), (-0.00526,-8.1819), (-4.04813,4.31485), (-2.84299,5.7227), (-5.201,5.67398), (7.75774,-1.75826), (-2.85791,7.54164), (-3.86071,-1.79026), (-1.80029,-1.7395), (-5.26015,5.65042), (-3.158,0.38765), (7.71014,-4.64719), (-4.84866,-10.22048), (-8.38785,-2.05447), (7.67021,-2.43441), (4.96521,-5.38551), (-0.40919,5.47764), (-3.25711,8.26637), (3.07685,-3.6421), (2.89376,-11.66269), (-10.47331,3.972), (-3.48942,5.46642), (1.13906,-3.72304), (-8.57454,5.75251), (-3.38963,5.12841), (-2.3195,0.59067), (-1.60694,5.21138), (-5.57406,-4.58702), (-0.93075,-8.737), (-11.76579,-2.12737), (10.68283,0.22888), (8.74324,-1.46448), (7.66409,2.40311), (4.76715,-5.21814), (0.44539,13.94749), (-1.35941,-2.77448), (4.18849,-3.7867), (-6.17097,3.4954), (0.27977,3.12586), (-1.45006,-7.01485), (-4.81694,-3.20727), (-3.0297,6.31415), (0.02145,2.37521), (2.46883,8.13787), (9.60317,2.15956), (-9.93898,-0.40842), (1.05549,-7.27283), (5.55366,4.27575), (-3.80722,-2.89126), (-4.18851,6.84344), (1.00351,7.0869), (3.11385,-5.18837), (-5.17623,2.67648), (-3.18396,-6.57021), (-6.65302,0.60429), (-0.50832,-1.04921), (-4.04375,7.12873), (4.52707,1.68973), (6.63124,-2.58404), (-3.72082,-3.83114), (5.79825,-7.26546), (-2.0158,-5.07153), (-2.78369,-0.80395), (-1.91821,2.09455), (6.31714,4.33374), (-1.80869,8.54335), (8.55586,0.80566), (2.40826,-8.38085), (-8.46361,7.54812), (5.04452,8.78007), (-0.84665,1.5857), (2.30903,8.43855), (-3.71837,-1.90846), (-0.69419,-1.2434), (3.6733,7.16172), (-1.96098,-3.44129), (2.36747,-6.37542), (-12.03622,-4.99486), (4.38481,4.99033), (2.93955,-1.83734), (2.16804,-2.83289), (-0.08218,-4.13997), (-3.97934,1.40163), (-7.43985,8.57867), (0.91666,-1.87639), (7.23432,3.41667), (-6.13303,6.31762), (-10.23217,1.58473), (-6.21681,1.63625), (-0.80934,-6.93618), (0.17914,3.58046), (2.13338,-6.8097), (6.97656,4.69978), (6.90455,-1.72912), (6.25943,5.29491), (-6.04019,-1.63062), (-7.30909,5.83818), (1.4589,17.0769), (12.00208,4.54301), (2.22457,-1.33801), (-2.45912,5.64339), (-6.92213,1.26913), (4.05547,-1.01553), (0.04709,4.8316), (-7.70952,3.08635), (-1.47883,-2.27738), (1.3701,-1.13761), (-4.92928,10.08698), (-2.75872,5.33827), (-0.09178,2.84345), (2.62642,-1.51132), (-1.14623,13.46078), (2.76609,8.58965), (4.94404,-2.36683), (-7.01764,-1.8217), (-10.91568,1.96981), (-2.49738,2.31718), (0.73576,3.66493), (2.25436,1.93104), (-1.72956,5.20332), (2.41054,3.20519), (5.72149,3.34631), (-6.41371,7.0087), (3.38217,-7.96126), (1.24133,-0.62182), (10.03634,-4.65227), (-2.37303,10.6572), (-1.35543,4.50891), (-1.4387,9.74298), (-4.0976,3.85707), (-0.82501,6.41144), (-1.93498,1.48649), (5.59955,2.28076), (5.46656,2.75342), (2.43568,-5.40401), (-0.23926,7.11389), (-4.9945,5.74368), (-4.96655,6.78345), (-0.59258,3.83773), (2.02497,0.70959), (0.67583,0.57434), (3.16522,1.5888), (-1.9673,3.94889), (-6.75319,5.8234), (-6.69723,7.78366), (0.81148,9.08354), (4.44531,-7.99182), (-4.43522,-2.77033), (-5.28602,-10.29342), (-3.58829,1.76251), (-7.97395,2.09266), (-2.84891,4.20614), (-3.95112,-3.63064), (3.54945,-2.17794), (12.12376,-2.66225), (-3.12347,-2.74707), (3.65209,-1.93431), (9.34031,1.38629), (-0.26348,4.12816), (-5.23968,-1.58902), (2.22336,-5.08864), (-10.70405,-2.30491), (-4.41319,2.64605), (-5.94912,1.16158), (1.8147,2.63534), (7.69287,1.4956), (9.46125,-4.60768), (4.72497,0.60771), (-0.57565,3.29549), (-1.12303,-1.42592), (2.90272,0.8883), (-4.4584,-1.10612), (4.28819,-2.57296), (11.64512,5.88085), (-1.80395,7.40745), (2.51605,13.48116), (-3.18439,5.53539), (-0.70213,-1.46014), (-7.68383,3.73304), (-8.32268,3.5435), (-8.71115,-3.89151), (9.96933,4.16265), (0.95675,2.32663), (3.35114,5.31735), (-2.66008,6.33485), (7.75456,2.1339), (0.73568,0.82708), (0.3483,-2.95155), (-1.09203,-6.76019), (-7.76963,-4.20179), (5.81902,8.78354), (-3.41424,1.41863), (-0.39209,7.65689), (4.67608,-6.52601), (0.68753,-4.4426), (5.17179,-4.49483), (4.98983,-3.91479), (-0.12659,-2.84562), (3.25267,2.58974), (1.50184,2.24424), (2.94507,-4.65846), (-0.42333,8.4062), (-3.66227,8.20262), (8.90812,-8.63752), (4.74411,4.97966), (2.22018,-0.35563), (-2.07976,-4.72116), (4.8711,-2.95997), (0.5023,2.73959), (6.31569,-0.23956), (-4.36903,10.13915), (3.82146,11.83775), (-6.99477,-2.50332), (3.61225,-0.58181), (14.69335,-7.62836), (0.58368,2.26478), (4.65341,-3.50179), (-3.14272,-2.08023), (2.67048,4.07256), (4.64963,-1.40826), (-2.70828,-2.33644), (1.42923,3.00197), (5.84498,4.23668), (-4.76568,-2.24647), (0.19907,1.0445), (1.67486,-0.31901), (5.32145,8.62657), (-8.03477,3.92817), (3.46776,0.08462), (4.66374,10.15884), (-5.37394,0.4113), (5.39045,4.45847), (-1.44756,5.82941), (-1.64419,6.59202), (3.39699,-3.73441), (-2.94659,-5.86969), (-2.38437,-4.56543), (-0.23958,-1.32636), (6.88389,-0.17884), (-2.7172,-3.56181), (-1.53419,-0.66932), (7.38841,6.87538), (-5.44178,0.73527), (-0.89287,-0.24177), (2.93546,-0.8657), (-0.26901,-0.22977), (-4.70044,1.02095), (2.25846,6.16311), (-9.28813,-5.68027), (6.04268,-3.7619), (4.41693,4.22959), (1.75714,-1.5249); +SELECT '-2.610898982580138', '0.00916587538237954'; +SELECT roundBankers(StudentTTest(left, right).1, 16) as t_stat, roundBankers(StudentTTest(left, right).2, 16) as p_value from student_ttest; +DROP TABLE IF EXISTS student_ttest; + +/*Check t-stat and p-value and compare it with scipy.stat implementation + First: a=1, sigma (not sigma^2)=5, size=500 + Second: a=1, sigma = 5, size = 500 */ +CREATE TABLE student_ttest (left Float64, right Float64) ENGINE = Memory; +INSERT INTO student_ttest VALUES (4.52546,8.69444), (3.73628,3.81414), (-0.39478,12.38442), (5.15633,8.9738), (0.50539,9.19594), (-5.34036,7.21009), (0.19336,4.97743), (8.35729,4.94756), (6.95818,19.80911), (-2.93812,13.75358), (8.30807,16.56373), (-3.3517,9.72882), (4.16279,4.64509), (-3.17231,17.76854), (1.93545,4.80693), (11.06606,8.79505), (-4.22678,10.88868), (-1.99975,6.21932), (-4.51178,15.11614), (-4.50711,13.24703), (1.89786,14.76476), (-6.19638,-0.6117), (-3.70188,17.48993), (5.01334,12.11847), (1.79036,4.87439), (2.14435,18.56479), (3.0282,1.23712), (2.35528,5.41596), (-12.18535,4.54994), (5.59709,11.37668), (-12.92336,9.5982), (-0.04281,6.59822), (-0.16923,1.16703), (0.88924,8.88418), (-4.68414,10.95047), (8.01099,5.52787), (2.61686,-1.11647), (-2.76895,14.49946), (3.32165,3.27585), (-0.85135,-0.42025), (1.21368,6.37906), (4.38673,2.5242), (6.20964,8.1405), (-1.23172,6.46732), (4.65516,9.89332), (-1.87143,10.4374), (0.86429,-1.06465), (2.51184,6.84902), (-1.88822,10.96576), (-1.61802,7.83319), (1.93653,14.39823), (-3.66631,7.02594), (-1.05294,13.46629), (-10.74718,10.39531), (16.49295,11.27348), (-7.65494,9.32187), (-3.39303,12.32667), (-4.89418,8.98905), (3.2521,9.54757), (0.05831,5.98325), (-3.00409,3.47248), (5.76702,9.26966), (2.67674,5.77816), (10.52623,6.32966), (-0.54501,9.49313), (-4.89835,6.21337), (3.52457,10.00242), (-0.0451,6.25167), (-6.61226,15.64671), (9.02391,2.78968), (5.52571,6.55442), (4.54352,3.68819), (-3.8394,9.55934), (-7.75295,4.166), (5.91167,12.32471), (1.38897,7.10969), (6.24166,16.31723), (5.58536,12.99482), (4.7591,10.11585), (-2.58336,10.29455), (-1.91263,18.27524), (3.31575,12.84435), (5.3507,13.11954), (-15.22081,12.84147), (-0.84775,15.55658), (-4.538,11.45329), (6.71177,7.50912), (0.52882,8.56226), (2.0242,8.63104), (5.69146,15.68026), (4.63328,21.6361), (0.22984,6.23925), (-2.84052,8.65714), (7.91867,9.9423), (1.11001,12.28213), (-0.11251,3.11279), (-0.20905,13.58128), (0.03287,16.51407), (-1.59397,16.60476), (-5.39405,12.02022), (-7.1233,12.11035), (4.51517,9.47832), (-0.70967,6.40742), (5.67299,8.87252), (-0.33835,15.14265), (-1.83047,2.23572), (-0.62877,11.57144), (-7.23148,18.87737), (0.1802,12.1833), (11.73325,11.17519), (2.17603,16.80422), (-0.11683,6.81423), (-1.29102,12.12546), (-0.23201,8.06153), (-6.8643,10.97228), (-6.85153,7.30596), (-4.77163,15.44026), (6.11721,8.00993), (5.96406,12.60196), (3.59135,13.96832), (-0.60095,14.03207), (3.11163,4.53758), (-0.18831,8.08297), (0.67657,4.90451), (-3.16117,8.14253), (0.26957,19.88605), (2.18653,13.85254), (-5.94611,23.01839), (-4.39352,6.02084), (-3.71525,9.60319), (5.11103,1.90511), (1.33998,10.35237), (1.01629,16.27082), (-3.36917,12.52379), (-3.99661,11.37435), (8.19336,13.61823), (2.89168,15.77622), (-11.10373,15.17254), (11.68005,6.711), (3.08282,4.74205), (-6.81506,10.09812), (-2.34587,6.61722), (-2.68725,10.34164), (0.3577,8.96602), (-3.05682,12.32157), (9.08062,11.75711), (-0.77913,13.49499), (10.35215,8.57713), (6.82565,11.50313), (-1.24674,1.13097), (5.18822,7.83205), (-3.70743,5.77957), (1.40319,15.5519), (5.89432,10.82676), (1.43152,11.51218), (6.70638,9.29779), (9.76613,9.77021), (4.27604,9.94114), (-2.63141,15.54513), (-7.8133,19.10736), (-0.06668,15.04205), (1.05391,9.03114), (4.41797,24.0104), (0.09337,9.94205), (6.16075,2.5925), (7.49413,8.82726), (-3.52872,10.0209), (-2.17126,8.1635), (-3.87605,4.24074), (3.26607,7.67291), (-3.28045,5.21642), (2.1429,11.2808), (1.53386,6.88172), (0.21169,5.98743), (-0.63674,17.97249), (5.84893,6.46323), (-0.63498,15.37416), (8.29526,2.89957), (-1.08358,17.13044), (-2.306,11.06355), (2.86991,3.09625), (-0.76074,-2.33019), (5.49191,7.42675), (1.82883,15.06792), (-3.70497,8.81116), (-0.53232,19.17446), (-11.49722,18.77181), (3.44877,14.06443), (-1.8596,12.81241), (-10.34851,2.72299), (1.13093,18.67739), (-10.93389,11.63275), (-3.39703,2.23891), (0.19749,13.01195), (-3.68389,7.43402), (-4.67863,8.14599), (10.78916,16.65328), (0.37675,1.362), (3.98094,3.87957), (-3.64775,11.16134), (-4.8443,6.25357), (1.102,4.21945), (8.72112,12.50047), (-1.47361,6.45486), (6.24183,18.99924), (6.83569,18.09508), (-3.11684,13.59528), (4.91306,3.39681), (-0.03628,13.33157), (5.1282,5.8945), (-2.38558,5.61212), (2.33351,8.41149), (-0.97191,13.78608), (-0.05588,6.08609), (-4.70019,12.76962), (-5.12371,3.26206), (0.65606,0.25528), (-0.11574,11.9083), (4.4238,4.35071), (6.93399,11.19855), (3.68712,13.87404), (-0.01187,6.87986), (1.8332,8.32566), (5.81322,22.51334), (-4.04709,2.5226), (-8.26397,16.84498), (-2.11273,6.26108), (5.28396,13.84824), (0.73054,6.03262), (6.43559,14.12668), (4.35565,16.01939), (-1.05545,8.19237), (5.00087,18.01595), (-2.72239,9.45609), (7.32313,6.90459), (2.11548,12.83115), (-3.40953,10.603), (6.97051,13.70439), (-0.45567,6.1633), (1.31699,4.1151), (-1.49871,8.20499), (7.14772,11.67903), (0.79277,7.30851), (6.9698,6.50941), (2.08733,7.3949), (-3.55962,12.80075), (0.75601,5.62043), (1.21,18.2542), (-2.17877,17.9393), (1.83206,16.4569), (5.72463,8.78811), (7.42257,4.85949), (0.97829,-3.36394), (7.54238,5.38683), (9.91081,12.26083), (-4.61743,10.27907), (-4.40799,11.5144), (9.99854,11.57335), (8.53725,1.94203), (3.2905,7.78228), (0.38634,11.79385), (-2.53374,10.18415), (4.94758,14.67613), (4.79624,4.70301), (5.57664,12.72151), (-6.44871,-3.35508), (3.34431,17.63775), (0.14209,2.53883), (10.88431,14.01483), (0.31846,12.4387), (-0.54703,11.15408), (-4.67791,7.74882), (-5.68011,13.60956), (-4.93362,7.81991), (1.2271,10.90969), (5.27512,8.19828), (-3.84611,-1.18523), (6.81706,0.5916), (10.33033,0.35805), (5.13979,12.98364), (3.66534,11.38628), (-2.07219,13.94644), (10.65442,2.03781), (-3.31751,10.74447), (-1.82011,12.35656), (-0.39886,7.08701), (1.77052,2.69871), (1.29049,19.66653), (7.92344,7.88636), (-2.92595,10.36916), (-2.67107,1.632), (5.64708,11.86081), (0.34639,13.47602), (-3.04356,6.60204), (3.98828,7.01303), (-1.36695,20.19992), (-8.48462,18.88249), (-4.04669,11.34367), (9.84561,12.97305), (-6.1537,9.5776), (0.82433,17.91364), (1.92449,18.3247), (2.51288,9.9211), (0.40965,7.14257), (2.89183,6.59133), (3.84347,12.35274), (0.66829,10.57523), (-3.45094,12.12859), (1.3544,9.47177), (-9.85456,0.60659), (5.25689,4.72996), (-5.26018,4.51121), (-6.16912,13.28893), (-1.77163,8.09014), (3.96687,8.02511), (0.70893,13.85406), (-5.45342,1.75412), (-3.89706,6.00641), (3.11868,6.35554), (4.41714,7.11293), (7.64841,8.30442), (0.00489,12.63024), (3.2263,12.38966), (-5.33042,7.6801), (2.52189,11.33744), (-7.40308,4.67713), (0.67891,7.62276), (2.49343,2.14478), (5.43133,15.32988), (-0.67541,1.52299), (-0.60299,17.00017), (-6.32903,8.29701), (-3.44336,10.92961), (-0.23963,6.78449), (6.94686,7.02698), (6.59442,11.51719), (-4.18532,9.97926), (-1.8228,7.44251), (-0.29443,7.58541), (2.99821,4.76058), (2.51942,12.88959), (-3.49176,9.974), (-0.57979,17.03689), (8.69471,11.14554), (-1.19427,11.7392), (-3.17119,11.50029), (-2.99566,19.41759), (-3.34493,9.65127), (-2.33826,9.87673), (-5.04164,14.13485), (-0.48214,9.78034), (7.45097,1.57826), (3.04787,3.72091), (2.92632,9.4054), (1.39694,23.22816), (4.38686,-0.12571), (3.25753,6.97343), (7.14218,10.09049), (-4.04341,11.78393), (-9.19352,3.01909), (2.78473,16.09448), (0.33331,6.25485), (9.89238,7.13164), (6.00566,7.75879), (-1.7511,9.56834), (4.77815,6.14824), (5.07457,13.53454), (2.56132,8.26364), (2.38317,8.7095), (-1.63486,10.61607), (-1.46871,10.64418), (-5.8681,23.9106), (-2.96227,11.38978), (-1.90638,11.4383), (-13.3052,18.41498), (-2.14705,3.70959), (-9.62069,19.95918), (2.29313,9.53847), (0.22162,14.04957), (-1.83956,13.70151), (4.1853,5.45046), (6.05965,10.95061), (-0.23737,9.55156), (6.07452,17.92345), (4.34629,6.23976), (4.02922,8.71029), (3.62622,13.58736), (-3.95825,8.78527), (-1.63412,11.14213), (-1.25727,12.23717), (5.06323,16.44557), (-0.66176,0.47144), (2.36606,9.7198), (-5.77792,13.50981), (4.535,14.27806), (1.02031,13.50793), (4.49345,7.47381), (-4.99791,11.07844), (2.46716,9.89844), (3.65471,21.48548), (11.2283,6.92085), (6.69743,4.44074), (-5.60375,19.98074), (0.28683,7.92826), (-0.85737,16.6313), (4.26726,17.17618), (-3.4322,13.80807), (-2.07039,5.37083), (-2.26798,9.73962), (-0.99818,10.66273), (0.41335,8.90639), (5.18124,12.24596), (-5.01858,16.89203), (2.05561,12.69184), (-0.12117,15.59077), (0.99471,6.94287), (6.89979,-0.1801), (-4.18527,3.25318), (-6.35104,8.08804), (3.89734,13.78384), (-1.979,0.46434), (3.15404,7.78224), (3.52672,9.10987), (2.48372,-0.89391), (-6.13089,14.3696), (2.2968,3.01763), (-2.74324,8.03559), (-0.12876,7.24609), (-1.51135,11.86271), (-3.92434,6.28196), (-1.71254,8.9725), (-1.25878,14.46114), (2.03021,9.50216), (4.31726,16.30413), (-3.02908,1.02795), (9.7093,1.88717), (-3.36284,9.80106), (6.70938,4.53487), (0.42762,16.34543), (5.04726,7.71098), (2.78386,2.74639), (6.83022,6.51875), (-3.02109,10.42308), (-0.65382,13.57901), (-15.58675,0.52784), (5.89746,4.4708), (-4.11598,6.39619), (-1.37208,14.57666), (10.08082,2.71602), (5.35686,12.53905), (1.93331,11.4292), (10.47444,12.44641), (-2.36872,14.50894), (6.50752,17.64374), (2.54603,11.03218), (-0.4332,9.82789), (5.26572,10.11104), (2.09016,2.16137), (1.15513,10.24054), (14.95941,12.86909), (-3.85505,15.22845), (-2.36239,5.05411), (1.64338,10.84836), (-4.25074,11.15717), (7.29744,0.91782), (-1.18964,13.29961), (5.60612,15.11314), (-3.77011,11.54004), (6.67642,-0.94238), (-0.06862,19.32581), (5.60514,10.20744), (3.7341,6.54857), (9.59001,8.69108), (3.30093,8.2296), (-2.75658,8.4474), (4.71994,6.81178), (0.74699,5.99415), (2.91095,13.99336), (-7.36829,8.7469), (-5.29487,8.62349), (3.31079,1.84212), (1.06974,4.4762), (-1.18424,9.25421), (-7.415,10.44229), (3.40595,12.21649), (-7.63085,10.45968), (1.13336,15.34722), (-0.0096,5.50868), (0.8928,10.93609), (-0.5943,2.78631), (7.48306,11.86145), (10.11943,18.67385), (5.60459,10.64051), (4.00189,12.75565), (2.35823,6.63666), (0.33475,12.19343), (3.47072,9.08636), (-6.68867,11.67256), (3.31031,20.31392), (2.17159,11.66443); +SELECT -28.740781574102936, 7.667329672103986e-133; +SELECT roundBankers(StudentTTest(left, right).1, 16) as t_stat, roundBankers(StudentTTest(left, right).2, 16) as p_value from student_ttest; +DROP TABLE IF EXISTS student_ttest; diff --git a/tests/queries/0_stateless/01322_welch_ttest.reference b/tests/queries/0_stateless/01322_welch_ttest.reference index 015dd503b7e..d06853a0a5e 100644 --- a/tests/queries/0_stateless/01322_welch_ttest.reference +++ b/tests/queries/0_stateless/01322_welch_ttest.reference @@ -1,6 +1,10 @@ 0.021378001462867 -0.021378 +0.0213780014628671 0.090773324285671 -0.09077332 +0.0907733242891952 0.00339907162713746 -0.00339907 +0.0033990715715539 +-0.5028215369186904 0.6152361677168877 +-0.5028215369187079 0.6152361677170834 +14.971190998235835 5.898143508382202e-44 +14.971190998235837 0 diff --git a/tests/queries/0_stateless/01322_welch_ttest.sql b/tests/queries/0_stateless/01322_welch_ttest.sql index 073e71f69fe..2a045e70b32 100644 --- a/tests/queries/0_stateless/01322_welch_ttest.sql +++ b/tests/queries/0_stateless/01322_welch_ttest.sql @@ -1,18 +1,37 @@ +/*Check only p-value first*/ DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (27.5,27.1), (21.0,22.0), (19.0,20.8), (23.6,23.4), (17.0,23.4), (17.9,23.5), (16.9,25.8), (20.1,22.0), (21.9,24.8), (22.6,20.2), (23.1,21.9), (19.6,22.1), (19.0,22.9), (21.7,20.5), (21.4,24.4); SELECT '0.021378001462867'; -SELECT roundBankers(WelchTTest(left, right), 8) from welch_ttest; +SELECT roundBankers(WelchTTest(left, right).2, 16) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (30.02,29.89), (29.99,29.93), (30.11,29.72), (29.97,29.98), (30.01,30.02), (29.99,29.98); SELECT '0.090773324285671'; -SELECT roundBankers(WelchTTest(left, right), 8) from welch_ttest; +SELECT roundBankers(WelchTTest(left, right).2, 16) from welch_ttest; DROP TABLE IF EXISTS welch_ttest; CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (0.010268,0.159258), (0.000167,0.136278), (0.000167,0.122389); SELECT '0.00339907162713746'; -SELECT roundBankers(WelchTTest(left, right), 8) from welch_ttest; -DROP TABLE IF EXISTS welch_ttest; \ No newline at end of file +SELECT roundBankers(WelchTTest(left, right).2, 16) from welch_ttest; +DROP TABLE IF EXISTS welch_ttest; + +/*Check t-stat and p-value and compare it with scipy.stat implementation + First: a=10, sigma (not sigma^2)=5, size=500 + Second: a=10, sigma = 10, size = 500 */ +CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; +INSERT INTO welch_ttest VALUES (14.72789,-8.65656), (9.61661,22.98234), (13.57615,23.80821), (3.98392,13.33939), (11.98889,-4.05537), (10.99422,23.5155), (5.44792,-6.45272), (20.29346,17.7903), (7.05926,11.463), (9.22732,5.28021), (12.06847,8.39157), (13.52612,6.02464), (8.24597,14.43732), (9.35245,15.76584), (10.12297,1.54391), (15.80624,1.24897), (13.68613,27.1507), (10.72729,7.71091), (5.62078,15.71846), (6.12229,32.97808), (6.03801,-1.79334), (8.95585,-9.23439), (24.04613,11.27838), (9.04757,0.72703), (2.68263,18.51557), (15.43935,9.16619), (2.89423,17.29624), (4.01423,-1.30208), (4.30568,-3.48018), (11.99948,10.12082), (8.40574,-8.01318), (10.86642,-14.22264), (9.4266,16.58174), (-8.12752,-0.55975), (7.91634,5.61449), (7.3967,1.44626), (2.26431,7.89158), (14.20118,1.13369), (6.68233,-0.82609), (15.46221,12.23365), (7.88467,12.45443), (11.20011,14.46915), (8.92027,13.72627), (10.27926,18.41459), (5.14395,29.66702), (5.62178,1.51619), (12.84383,10.40078), (9.98009,3.33266), (-0.69789,6.12036), (11.41386,11.86553), (7.76863,6.59422), (7.21743,22.0948), (1.81176,1.79623), (9.43762,14.29513), (19.22117,19.69162), (2.97128,-7.98033), (14.32851,5.48433), (7.54959,-2.28474), (3.81545,9.91876), (10.1281,10.64097), (2.48596,0.22523), (10.0461,17.01773), (3.59714,22.37388), (9.73522,14.04215), (18.8077,23.1244), (3.15148,18.96958), (12.26062,8.42663), (5.66707,3.7165), (6.58623,14.29366), (17.30902,23.50886), (9.91391,26.33722), (5.36946,26.72396), (15.73637,13.26287), (16.96281,12.97607), (11.54063,17.41838), (18.37358,8.63875), (11.38255,17.08943), (10.53256,23.15356), (8.08833,-4.4965), (16.27556,7.58895), (2.42969,26.04074), (9.56127,6.84245), (7.32998,20.56287), (9.19511,3.84735), (9.66903,-2.76304), (4.15029,13.1615), (8.83511,8.21954), (14.60617,-3.49943), (14.06143,22.12419), (5.39556,7.08323), (10.11871,16.12937), (10.56619,-0.32672), (14.4462,16.5942), (10.42106,7.68977), (7.75551,11.39484), (11.00418,-5.11987), (4.47226,20.87404), (16.35461,8.01007), (18.55174,3.26497), (11.82044,5.61253), (7.39454,20.69182), (11.27767,0.0296), (6.83827,21.904), (7.76858,22.46572), (15.97614,3.63685), (14.53781,-5.10846), (12.99546,14.86389), (16.91151,5.47188), (9.65012,18.44095), (14.25487,16.71368), (14.03618,6.36704), (2.57382,8.82663), (2.50779,14.6727), (14.24787,7.98383), (13.34666,2.65568), (7.31102,21.45827), (10.22981,11.77948), (17.4435,4.71979), (21.2074,3.17951), (6.64191,13.90226), (18.7086,15.50578), (14.78686,10.8026), (9.85287,16.91369), (4.48263,9.90552), (14.17469,13.87322), (14.4342,4.12366), (19.2481,-3.78985), (3.47165,1.7599), (8.28712,3.43715), (8.81657,-3.45246), (0.92319,23.64571), (20.41106,-4.96877), (6.76127,3.93514), (22.00242,1.49914), (8.66129,12.71519), (10.9929,5.11521), (17.95494,4.79872), (17.20996,20.89391), (12.18888,5.363), (12.14257,8.02765), (15.81243,14.30804), (4.43362,11.49002), (1.17567,14.25281), (15.60881,7.6573), (9.34833,15.49686), (6.33513,3.29327), (-0.83095,2.27236), (12.43268,12.58104), (6.63207,19.19128), (11.96877,15.25901), (14.81029,6.5221), (21.84876,10.10965), (3.75896,12.75249), (6.91307,16.50977), (13.73015,-8.6697), (8.63753,8.28553), (15.71679,1.44315), (1.74565,4.65869), (9.16895,0.98149), (5.70685,0.16623), (5.00117,17.66332), (13.06888,4.35346), (7.51204,6.52742), (15.34885,-1.06631), (5.20264,-5.28454), (8.59043,14.25583), (6.45619,8.74058), (14.61979,1.89553), (11.7075,-0.92959), (14.04901,10.30289), (4.20525,-6.3744), (15.1733,-8.1706), (3.12934,10.95369), (8.08049,4.94384), (15.41273,28.40568), (16.90751,3.7004), (5.86893,2.52363), (7.1086,4.07997), (4.418,7.8849), (12.0614,17.95409), (7.07887,16.67021), (3.61585,11.34377), (11.73001,-0.07446), (10.80449,22.00223), (8.40311,3.31778), (9.91276,18.50719), (16.4164,-3.58655), (5.25034,6.5394), (15.20283,12.40459), (10.42909,16.59866), (9.53888,7.54176), (14.68939,-1.51044), (6.60007,12.69758), (18.31058,2.9842), (7.01885,2.49187), (18.71631,2.04113), (10.50002,-2.46544), (10.7517,15.18368), (4.23224,-0.04058), (2.28924,-0.4127), (8.56059,10.5526), (8.25095,12.03982), (9.15673,12.10923), (13.28409,11.54954), (8.4513,-1.18613), (2.83911,11.30984), (2.79676,23.54105), (9.11055,10.67321), (7.18529,24.09196), (-4.1258,7.5008), (5.28306,12.52233), (6.82757,4.30673), (10.89035,9.35793), (5.24822,4.44472), (11.935,-7.00679), (6.45675,8.56241), (10.18088,23.73891), (4.9932,15.62708), (18.09939,16.09205), (8.11738,12.52074), (5.37883,14.58927), (10.50339,-4.80187), (16.64093,8.47964), (14.77263,7.75477), (13.71385,12.6893), (6.98746,7.14147), (10.74635,12.12654), (5.49432,12.32334), (13.46078,7.98909), (10.67565,3.26652), (9.0291,20.53684), (11.51417,32.3369), (13.07118,19.74911), (9.5049,-4.62897), (8.50611,8.26483), (6.47606,20.88451), (13.06526,-2.12982), (19.08658,25.61459), (9.49741,5.32091), (10.60865,-4.1196), (2.28996,7.57937), (8.12846,21.15847), (5.62241,6.46355), (4.07712,7.74846), (17.98526,19.62636), (9.466,28.34629), (11.38904,26.73919), (5.91826,20.40427), (1.52059,3.03378), (18.79161,10.2537), (18.20669,7.47745), (-1.67829,10.79184), (18.01586,3.91962), (16.31577,19.97973), (7.88281,18.87711), (8.46179,12.56157), (10.31113,11.46033), (14.88377,3.78661), (1.31835,-9.45748), (2.53176,12.06033), (9.48625,-0.74615), (3.97936,13.2815), (11.52319,24.78052), (13.24178,5.83337), (7.58739,17.4111), (10.00959,19.70331), (9.73361,11.78446), (8.35716,-1.366), (1.65491,1.37458), (11.11521,16.31483), (6.08355,32.63464), (10.04582,-3.79736), (11.58237,19.17984), (16.40249,-0.27705), (1.9691,-3.69456), (13.22776,28.38058), (2.67059,-1.36876), (9.83651,-25.63301), (2.12539,3.58644), (9.27114,-6.85667), (9.0699,13.42225), (2.78179,12.04671), (12.49311,28.99468), (12.97662,7.87662), (15.06359,2.61119), (16.91565,-3.56022), (5.92011,1.50022), (5.81304,14.55836), (8.46425,9.35831), (9.48705,16.9366), (4.68191,29.23126), (5.70028,15.31386), (-0.78798,13.46112), (10.03442,7.39667), (15.45433,11.15599), (9.43845,9.80499), (3.05825,22.64923), (6.92126,8.67693), (14.05905,18.67335), (19.71579,-3.19127), (15.0131,22.94716), (4.50386,17.86834), (1.31061,16.98267), (10.81197,15.91653), (14.32942,11.79718), (9.26469,18.50208), (7.27679,8.90755), (22.69295,10.44843), (12.03763,4.67433), (7.34876,6.82287), (16.60689,10.82228), (7.48786,-4.18631), (15.78602,20.3872), (17.21048,11.84735), (13.93482,21.25376), (9.69911,10.55032), (12.24315,12.19023), (10.58131,0.63369), (19.57006,7.92381), (9.8856,17.90933), (11.70302,15.30781), (7.89864,10.01877), (12.24831,0.88744), (16.93707,22.20967), (9.65467,-4.23117), (4.221,21.50819), (15.45229,11.27421), (12.83088,-16.23179), (7.58313,33.43085), (12.895,5.15093), (10.02471,1.34505), (13.36059,6.027), (5.07864,-10.43035), (9.72017,27.45998), (11.05809,19.24886), (15.28528,-4.44761), (13.99834,5.453), (19.26989,12.73758), (9.41846,11.2897), (11.65425,31.032), (8.49638,7.39168), (6.38592,11.95245), (-4.69837,26.279), (12.22061,-1.0255), (9.41331,10.36675), (13.2075,11.58439), (12.97005,27.8405), (11.44352,13.1707), (9.79805,31.39133), (6.93116,27.08301), (10.07691,-2.14368), (22.05892,4.08476), (7.80353,21.5573), (-2.17276,16.69822), (0.61509,7.69955), (8.35842,8.32793), (17.77108,6.49235), (14.70841,-7.3284), (1.27992,10.58264), (15.62699,-6.17006), (9.32914,34.55782), (15.41866,10.93221), (10.82009,44.24299), (3.29902,14.6224), (9.21998,-7.42798), (7.93845,15.52351), (10.33344,11.33982), (12.06399,10.46716), (5.5308,13.0986), (8.38727,-4.25988), (18.11104,9.55316), (8.86565,0.75489), (19.41825,25.99212), (9.52376,-0.81401), (3.94552,3.49551), (9.37587,22.99402), (15.44954,10.99628), (15.90527,23.70223), (13.18927,2.71482), (7.01646,22.82309), (9.06005,31.25686), (9.06431,4.86318), (5.76006,-1.06476), (9.18705,15.10298), (-3.48446,-0.61015), (15.89817,17.81246), (12.94719,-1.55788), (23.69426,18.09709), (17.47755,9.11271), (15.61528,9.94682), (0.54832,-7.33194), (14.32916,-4.67293), (9.55305,21.81717), (13.79891,7.16318), (0.82544,13.25649), (13.34875,13.88776), (9.07614,4.95793), (5.19621,17.65303), (2.1451,14.47382), (9.87726,13.19373), (8.45439,31.86093), (-1.41842,5.73161), (7.93598,10.96492), (11.23151,6.97951), (17.84458,1.75136), (7.02237,10.96144), (10.7842,15.08137), (4.42832,9.95311), (4.45044,7.07729), (1.50938,3.08148), (21.21651,22.37954), (6.2097,8.51951), (6.84354,2.88746), (18.53804,26.73509), (12.01072,-2.88939), (4.8345,-2.82367), (20.41587,-0.35783), (14.48353,14.22076), (8.71116,11.50295), (12.42818,7.10171), (14.89244,8.28488), (8.03033,0.54178), (5.25917,13.8022), (2.30092,15.62157), (10.22504,10.79173), (15.37573,28.18946), (7.13666,30.43524), (4.45018,2.54914), (10.18405,9.89421), (3.91025,13.08631), (14.52304,4.68761), (13.14771,5.61516), (11.99219,22.88072), (9.21345,7.4735), (8.85106,11.27382), (12.91887,2.39559), (15.62308,-3.31889), (11.88034,9.61957), (15.12097,23.01381), (11.58168,-1.23467), (16.83051,9.07691), (5.25405,15.78056), (2.19976,12.28421), (4.56716,9.44888), (16.46053,13.16928), (5.61995,4.33357), (8.67704,2.21737), (5.62789,33.17833), (9.84815,13.25407), (13.05834,-2.47961), (11.74205,6.41401), (3.88393,18.8439), (16.15321,-4.63375), (4.83925,-8.2909), (13.00334,12.18221), (4.4028,-2.95356), (4.35794,19.61659), (4.47478,12.45056), (2.38713,-4.17198), (4.25235,21.9641), (10.87509,11.96416), (9.82411,12.74573), (13.61518,10.47873), (10.25507,12.73295), (4.0335,11.31373), (10.69881,9.9827), (5.70321,5.87138), (6.96244,4.24372), (9.35874,-23.72256), (6.28076,28.41337), (8.29015,4.88103), (6.88653,3.61902), (7.70687,8.93586), (8.2001,16.40759), (6.73415,27.84494), (3.82052,5.6001), (3.94469,14.51379), (15.82384,13.5576), (2.54004,12.92213), (10.74876,3.90686), (12.60517,17.07104), (17.7024,15.84268), (4.6722,17.38777), (13.67341,16.54766), (6.4565,5.94487), (12.95699,17.02804), (4.56912,7.66386), (5.58464,10.43088), (4.0638,6.16059), (13.05559,20.46178), (5.38269,20.02888), (0.16354,20.95949), (7.23962,6.50808), (7.38577,7.22366), (8.50951,8.06659), (13.72574,16.08241), (17.80421,13.83514), (3.01135,-0.33454), (8.02608,12.98848), (14.23847,12.99024); +SELECT -0.5028215369186904, 0.6152361677168877; +SELECT roundBankers(WelchTTest(left, right).1, 16) as t_stat, roundBankers(WelchTTest(left, right).2, 16) as p_value from welch_ttest; +DROP TABLE IF EXISTS welch_ttest; + +/*Check t-stat and p-value and compare it with scipy.stat implementation + First: a=10, sigma (not sigma^2)=5, size=500 + Second: a=1, sigma = 12, size = 500 */ +CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; +INSERT INTO welch_ttest VALUES (4.82025,-2.69857), (6.13896,15.80943), (15.20277,7.31555), (14.15351,3.96517), (7.21338,4.77809), (8.55506,9.6472), (13.80816,-26.41717), (11.28411,-10.85635), (7.4612,-1.4376), (7.43759,-0.96308), (12.9832,2.84315), (-5.74783,5.79467), (12.47114,-3.06091), (15.14223,-14.62902), (3.40603,22.08022), (9.27323,-2.11982), (7.88547,-4.84824), (8.56456,-10.50447), (4.59731,2.4891), (7.91213,9.90324), (7.33894,-22.66866), (21.74811,-0.97103), (11.92111,-16.57608), (0.18828,-3.78749), (10.47314,25.84511), (20.37396,5.30797), (11.04991,-18.19466), (13.30083,11.72708), (14.28065,0.2891), (2.86942,-9.83474), (24.96072,6.69942), (14.20164,18.09604), (18.28769,18.52651), (10.50949,1.38201), (9.22273,7.64615), (11.77608,17.66598), (8.56872,-2.44141), (13.74535,-9.01598), (11.65209,27.69142), (12.51894,4.06946), (17.76256,-15.0077), (13.52122,-10.49648), (8.70796,-4.88322), (6.04749,-25.09805), (16.33064,-4.64024), (8.35636,20.94434), (14.03496,24.12126), (11.05834,-14.10962), (14.49261,10.6512), (2.59383,14.50687), (8.01022,-19.88081), (4.05458,-11.55271), (13.26384,13.16921), (14.62058,16.63864), (10.52489,-24.08114), (8.46357,-9.09949), (6.4147,-10.54702), (9.70071,0.20813), (12.47581,8.19066), (4.38333,-2.70523), (17.54172,-0.23954), (10.12109,7.19398), (7.73186,-7.1618), (14.0279,-7.44322), (11.6621,-17.92031), (17.47045,-1.58146), (15.50223,9.18338), (15.46034,3.25838), (13.39964,-14.30234), (14.98025,1.84695), (15.87912,31.13794), (17.67374,-0.85067), (9.64073,19.02787), (12.84904,-3.09594), (7.70278,13.45584), (13.03156,-5.48104), (9.04512,-22.74928), (15.97014,-8.03697), (8.96389,17.31143), (11.48009,-16.65231), (9.71153,-18.58713), (13.00084,-16.52641), (12.39803,14.95261), (13.08188,12.56762), (5.82244,15.00188), (10.81871,1.85858), (8.2539,2.1926), (7.52114,-2.4095), (9.11488,21.56873), (8.37482,3.35509), (14.48652,-4.98672), (11.42152,35.08603), (16.03111,-10.01602), (13.14057,-3.85153), (-2.26351,-6.81974), (15.50394,19.56525), (14.88603,-9.35488), (13.37257,0.24268), (11.84026,-3.51488), (7.66558,-0.37066), (6.24584,24.20888), (3.6312,-11.73537), (2.7018,0.01282), (5.63656,0.03963), (5.82643,-9.65589), (10.06745,-0.37429), (-0.5831,5.61255), (14.84202,0.49984), (9.5524,-10.15066), (19.71713,-14.54314), (14.23109,16.56889), (8.69105,-7.73873), (5.33742,-3.76422), (7.30372,1.40722), (7.93342,2.28818), (15.20884,-13.12643), (7.53839,5.17082), (13.45311,4.79089), (11.04473,-17.42643), (10.76673,8.72548), (15.44145,-3.70285), (14.06596,16.77893), (9.14873,13.382), (12.88372,19.98418), (8.74994,0.00483), (10.53263,-4.75951), (16.16694,2.35391), (8.37197,21.65809), (3.43739,-9.2714), (4.72799,-18.38253), (9.08802,7.23097), (11.2531,14.97927), (5.16115,-4.02197), (10.20895,-29.8189), (18.70884,-12.8554), (15.88924,-7.60124), (3.38758,-14.90158), (6.46449,-3.31486), (10.21088,31.38144), (14.08458,-8.61288), (15.74508,15.31895), (19.31896,-10.19488), (13.19641,13.796), (11.95409,-0.32912), (10.70718,-0.0684), (1.05245,-30.06834), (10.04772,24.93912), (17.01369,-3.26506), (10.2286,-8.29751), (19.58323,-5.39189), (7.02892,-25.08603), (4.16866,-1.45318), (8.94326,16.72724), (4.99854,-3.38467), (8.88352,-26.00478), (18.65422,7.28369), (17.32328,16.96226), (9.33492,16.5858), (14.94788,10.46583), (8.05863,3.84345), (14.6737,-2.99382), (10.93801,1.42078), (0.54036,-11.0123), (-0.34242,2.09909), (5.89076,1.21064), (3.15189,15.36079), (1.94421,-21.61349), (6.38698,22.7726), (10.50654,10.50512), (8.95362,-6.95825), (6.23711,9.20036), (11.75359,15.66902), (12.42155,3.28098), (-1.55472,-9.05692), (4.6688,0.32882), (10.48087,-1.64934), (11.74615,-4.81406), (9.26822,-5.06006), (7.55517,19.97493), (12.76005,2.88646), (16.47102,-0.34552), (11.31297,7.55186), (14.37437,-22.96115), (2.38799,31.29166), (6.44577,6.18798), (5.07471,-2.52715), (11.55123,-11.58799), (7.76795,14.13596), (10.60116,13.45069), (14.40885,12.15179), (11.58158,3.44491), (8.81648,-8.78006), (12.92299,18.32087), (11.26939,11.91757), (17.95014,-2.00179), (2.95002,10.88411), (17.41959,9.09327), (11.12455,6.62484), (8.78541,8.87178), (14.36413,11.52254), (12.98554,-14.15988), (12.58505,-17.19515), (15.49789,14.03089), (11.70999,-2.4095), (0.65596,-16.83575), (11.08202,2.71469), (14.75752,4.84351), (6.84385,-1.17651), (9.27245,-3.37529), (13.78243,-19.92137), (17.4863,4.48952), (4.01777,-12.4906), (11.82861,-5.65277), (13.86551,8.50819), (6.16591,-19.61261), (8.71589,12.54156), (16.77195,11.06784), (17.23243,-12.59285), (-2.12941,3.43683), (5.66629,-3.00325), (12.45153,12.49082), (1.63971,7.20955), (13.84031,17.6547), (4.6144,15.8619), (5.26169,24.3048), (9.27769,-8.05434), (9.14288,-6.06901), (9.71953,-15.69515), (9.38446,-11.13917), (1.64788,-3.90757), (11.72922,-2.57038), (13.68926,5.14065), (9.42952,17.8497), (12.05574,-8.64665), (9.09148,-18.68331), (5.32273,5.8567), (20.25258,-20.93884), (10.14599,4.40583), (10.82156,14.35985), (5.75736,4.18134), (7.13567,4.3635), (9.29746,9.35428), (5.1618,2.8908), (10.076,16.01017), (21.65669,-1.48499), (13.35486,-9.97949), (6.79957,1.03055), (8.76243,-2.79697), (14.59294,6.85977), (16.90609,4.73213), (10.50337,2.7815), (-0.07923,-2.46866), (13.51648,18.39425), (12.0676,-0.80378), (0.86482,-0.22982), (9.03563,-16.11608), (5.38751,3.0862), (17.16866,3.20779), (2.78702,10.50146), (11.15548,-0.21305), (12.30843,11.21012), (8.04897,-0.99825), (9.95814,18.39633), (11.29308,-3.39003), (14.13032,-0.64411), (21.05877,-1.39932), (3.57386,15.45319), (7.96631,-0.66044), (3.30484,-15.2223), (18.61856,-34.39907), (16.35184,-3.57836), (7.65236,16.82828), (18.02895,1.66624), (9.79458,15.43475), (16.7274,8.17776), (8.84453,5.50486), (13.05709,10.43082), (10.91447,-6.63332), (8.40171,2.28008), (16.95211,16.37203), (11.82194,5.16313), (19.87978,-8.85281), (12.88455,13.26692), (-0.00947,-7.46842), (12.28109,8.43091), (6.96462,-13.18172), (13.75282,-0.72401), (14.39141,22.3881), (11.07193,10.65448), (12.88039,2.81289), (11.38253,10.92405), (21.02707,-8.95358), (7.51955,19.80653), (6.31984,-12.86527), (15.6543,5.38826), (14.80315,-6.83501), (8.38024,-15.7647), (21.7516,-27.67412), (14.31336,8.6499), (15.04703,-4.89542), (5.73787,16.76167), (13.16911,12.84284), (12.40695,-17.27324), (9.88968,-4.18726), (8.46703,-14.62366), (8.70637,-5.49863), (8.03551,-16.22846), (5.9757,10.60329), (12.22951,6.46781), (3.14736,1.70458), (10.51266,10.77448), (18.593,0.8463), (10.82213,13.0482), (7.14216,-4.36264), (6.81154,3.22647), (-0.6486,2.38828), (20.56136,6.7946), (11.35367,-0.25254), (11.38205,1.2497), (17.14,1.6544), (14.91215,4.1019), (15.50207,11.27839), (5.93162,-5.04127), (3.74869,18.11674), (14.11532,0.51231), (7.38954,-0.51029), (5.45764,13.52556), (18.33733,16.10171), (9.91923,5.68197), (2.38991,-2.85904), (14.16756,-8.89167), (2.39791,6.24489), (6.92586,10.85319), (5.32474,-0.39816), (2.28812,3.87079), (5.71718,-3.1867), (5.84197,1.55322), (2.76206,16.86779), (19.05928,-14.60321), (11.51788,-1.81952), (6.56648,-3.11624), (3.35735,1.24193), (7.55948,10.18179), (19.99908,4.69796), (13.00634,0.69032), (18.36886,11.7723), (11.14675,7.62896), (16.72931,9.89741), (12.50106,9.11484), (6.00605,-3.84676), (23.06653,-0.4777), (5.39694,0.95958), (9.53167,-7.95056), (12.76944,-10.97474), (7.20604,-6.54861), (13.25391,34.74933), (13.7341,27.39463), (10.85292,4.18299), (-7.75835,6.02476), (10.29728,-1.99397), (13.70099,1.26478), (10.17959,23.37106), (9.98399,10.49682), (12.69389,-11.04354), (-0.28848,-12.22284), (-2.18319,-9.87635), (13.36378,28.90511), (10.09232,6.77613), (5.49489,0.55352), (5.46156,0.37031), (0.94225,7.1418), (12.79205,3.24897), (10.09593,-1.60918), (6.06218,3.1675), (0.89463,-17.97072), (11.88986,-5.61743), (10.79733,14.1422), (1.51371,14.87695), (2.20967,-4.65961), (15.45732,-0.99174), (16.5262,-2.96623), (5.99724,-9.02263), (8.3613,-17.2088), (15.68183,2.78608), (15.32117,6.74239), (14.15674,4.8524), (6.64553,7.46731), (4.20777,1.04894), (-0.10521,-12.8023), (-0.88169,-17.18188), (1.85913,-5.08801), (9.73673,22.13942), (0.30926,-0.36384), (6.17559,17.80564), (11.76602,7.67504), (5.68385,1.59779), (14.57088,4.10942), (12.81509,0.61074), (9.85682,-14.40767), (12.06376,10.59906), (6.08874,16.57017), (11.63921,-15.17526), (14.86722,-6.98549), (10.41035,-0.64548), (2.93794,3.23756), (12.21841,14.65504), (0.23804,4.583), (3.14845,12.72378), (7.29748,5.26547), (3.06134,0.81781), (13.77684,9.38273), (16.21992,10.37636), (5.33511,10.70325), (9.68959,-0.83043), (9.44169,-7.53149), (18.08012,-9.09147), (4.04224,-19.51381), (8.77918,-28.44508), (10.18324,6.44392), (9.38914,11.10201), (11.76995,-2.86184), (14.19963,8.30673), (6.88817,8.8797), (16.56123,10.68053), (15.39885,15.62919), (5.21241,8.00579), (4.44408,6.4651), (17.87587,-4.50029), (12.53337,18.04514), (13.60916,11.12996), (6.60104,-5.14007), (7.35453,9.43857), (18.61572,3.13476), (6.10437,4.9772), (13.08682,-17.45782), (12.15404,0.05552), (4.90789,-1.90283), (2.13353,2.67908), (12.49593,-2.62243), (11.93056,-3.22767), (13.29408,-8.70222), (5.70038,-23.11605), (8.40271,21.6757), (5.19456,12.70076), (-5.51028,4.4322), (14.0329,11.69344), (10.38365,9.18052), (6.56812,-2.2549), (4.21129,-2.15615), (9.7157,20.29765), (9.88553,-0.29536), (13.45346,15.50109), (4.97752,8.79187), (12.77595,5.11533), (8.56465,-20.44436), (4.27703,-3.00909), (18.12502,-4.48291), (12.45735,21.84462), (12.42912,1.94225), (12.08125,-2.81908), (10.85779,17.19418), (4.36013,-9.33528), (11.85062,-0.17346), (8.47776,0.03958), (9.60822,-35.17786), (11.3069,8.36887), (14.25525,-9.02292), (1.55168,-10.98804), (14.57782,0.29335), (7.84786,4.29634), (9.87774,3.87718), (14.75575,-9.08532), (3.68774,7.13922), (9.37667,-7.62463), (20.28676,-10.5666), (12.10027,4.68165), (8.01819,-3.30172), (18.78158,13.04852), (20.85402,13.45616), (18.98069,2.41043), (16.1429,-0.36501), (9.24047,-15.67383), (14.12487,17.92217), (10.18841,8.42106), (-3.04478,3.22063), (5.7552,-7.31753), (9.30376,21.99596), (11.42837,-36.8273), (6.02364,-20.46391), (8.86984,5.74179), (10.91177,-15.83178), (10.04418,14.90454), (18.10774,-8.84645), (7.49384,3.72036), (9.11556,4.6877), (9.7051,16.35418), (5.23268,3.15441), (9.04647,2.39907), (8.81547,-17.58664), (2.65098,-13.18269); +SELECT 14.971190998235835, 5.898143508382202e-44; +SELECT roundBankers(WelchTTest(left, right).1, 16) as t_stat, roundBankers(WelchTTest(left, right).2, 16) as p_value from welch_ttest; +DROP TABLE IF EXISTS welch_ttest; From a750b76818b6fc6273a6d7fc8e5785d9ff0068c7 Mon Sep 17 00:00:00 2001 From: nikitamikhaylov Date: Tue, 13 Oct 2020 21:50:04 +0300 Subject: [PATCH 163/411] better --- tests/queries/0_stateless/01322_student_ttest.sql | 2 +- tests/queries/0_stateless/01322_welch_ttest.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01322_student_ttest.sql b/tests/queries/0_stateless/01322_student_ttest.sql index 3636e239fe8..babc8457bc0 100644 --- a/tests/queries/0_stateless/01322_student_ttest.sql +++ b/tests/queries/0_stateless/01322_student_ttest.sql @@ -14,6 +14,6 @@ DROP TABLE IF EXISTS student_ttest; Second: a=1, sigma = 5, size = 500 */ CREATE TABLE student_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO student_ttest VALUES (4.52546,8.69444), (3.73628,3.81414), (-0.39478,12.38442), (5.15633,8.9738), (0.50539,9.19594), (-5.34036,7.21009), (0.19336,4.97743), (8.35729,4.94756), (6.95818,19.80911), (-2.93812,13.75358), (8.30807,16.56373), (-3.3517,9.72882), (4.16279,4.64509), (-3.17231,17.76854), (1.93545,4.80693), (11.06606,8.79505), (-4.22678,10.88868), (-1.99975,6.21932), (-4.51178,15.11614), (-4.50711,13.24703), (1.89786,14.76476), (-6.19638,-0.6117), (-3.70188,17.48993), (5.01334,12.11847), (1.79036,4.87439), (2.14435,18.56479), (3.0282,1.23712), (2.35528,5.41596), (-12.18535,4.54994), (5.59709,11.37668), (-12.92336,9.5982), (-0.04281,6.59822), (-0.16923,1.16703), (0.88924,8.88418), (-4.68414,10.95047), (8.01099,5.52787), (2.61686,-1.11647), (-2.76895,14.49946), (3.32165,3.27585), (-0.85135,-0.42025), (1.21368,6.37906), (4.38673,2.5242), (6.20964,8.1405), (-1.23172,6.46732), (4.65516,9.89332), (-1.87143,10.4374), (0.86429,-1.06465), (2.51184,6.84902), (-1.88822,10.96576), (-1.61802,7.83319), (1.93653,14.39823), (-3.66631,7.02594), (-1.05294,13.46629), (-10.74718,10.39531), (16.49295,11.27348), (-7.65494,9.32187), (-3.39303,12.32667), (-4.89418,8.98905), (3.2521,9.54757), (0.05831,5.98325), (-3.00409,3.47248), (5.76702,9.26966), (2.67674,5.77816), (10.52623,6.32966), (-0.54501,9.49313), (-4.89835,6.21337), (3.52457,10.00242), (-0.0451,6.25167), (-6.61226,15.64671), (9.02391,2.78968), (5.52571,6.55442), (4.54352,3.68819), (-3.8394,9.55934), (-7.75295,4.166), (5.91167,12.32471), (1.38897,7.10969), (6.24166,16.31723), (5.58536,12.99482), (4.7591,10.11585), (-2.58336,10.29455), (-1.91263,18.27524), (3.31575,12.84435), (5.3507,13.11954), (-15.22081,12.84147), (-0.84775,15.55658), (-4.538,11.45329), (6.71177,7.50912), (0.52882,8.56226), (2.0242,8.63104), (5.69146,15.68026), (4.63328,21.6361), (0.22984,6.23925), (-2.84052,8.65714), (7.91867,9.9423), (1.11001,12.28213), (-0.11251,3.11279), (-0.20905,13.58128), (0.03287,16.51407), (-1.59397,16.60476), (-5.39405,12.02022), (-7.1233,12.11035), (4.51517,9.47832), (-0.70967,6.40742), (5.67299,8.87252), (-0.33835,15.14265), (-1.83047,2.23572), (-0.62877,11.57144), (-7.23148,18.87737), (0.1802,12.1833), (11.73325,11.17519), (2.17603,16.80422), (-0.11683,6.81423), (-1.29102,12.12546), (-0.23201,8.06153), (-6.8643,10.97228), (-6.85153,7.30596), (-4.77163,15.44026), (6.11721,8.00993), (5.96406,12.60196), (3.59135,13.96832), (-0.60095,14.03207), (3.11163,4.53758), (-0.18831,8.08297), (0.67657,4.90451), (-3.16117,8.14253), (0.26957,19.88605), (2.18653,13.85254), (-5.94611,23.01839), (-4.39352,6.02084), (-3.71525,9.60319), (5.11103,1.90511), (1.33998,10.35237), (1.01629,16.27082), (-3.36917,12.52379), (-3.99661,11.37435), (8.19336,13.61823), (2.89168,15.77622), (-11.10373,15.17254), (11.68005,6.711), (3.08282,4.74205), (-6.81506,10.09812), (-2.34587,6.61722), (-2.68725,10.34164), (0.3577,8.96602), (-3.05682,12.32157), (9.08062,11.75711), (-0.77913,13.49499), (10.35215,8.57713), (6.82565,11.50313), (-1.24674,1.13097), (5.18822,7.83205), (-3.70743,5.77957), (1.40319,15.5519), (5.89432,10.82676), (1.43152,11.51218), (6.70638,9.29779), (9.76613,9.77021), (4.27604,9.94114), (-2.63141,15.54513), (-7.8133,19.10736), (-0.06668,15.04205), (1.05391,9.03114), (4.41797,24.0104), (0.09337,9.94205), (6.16075,2.5925), (7.49413,8.82726), (-3.52872,10.0209), (-2.17126,8.1635), (-3.87605,4.24074), (3.26607,7.67291), (-3.28045,5.21642), (2.1429,11.2808), (1.53386,6.88172), (0.21169,5.98743), (-0.63674,17.97249), (5.84893,6.46323), (-0.63498,15.37416), (8.29526,2.89957), (-1.08358,17.13044), (-2.306,11.06355), (2.86991,3.09625), (-0.76074,-2.33019), (5.49191,7.42675), (1.82883,15.06792), (-3.70497,8.81116), (-0.53232,19.17446), (-11.49722,18.77181), (3.44877,14.06443), (-1.8596,12.81241), (-10.34851,2.72299), (1.13093,18.67739), (-10.93389,11.63275), (-3.39703,2.23891), (0.19749,13.01195), (-3.68389,7.43402), (-4.67863,8.14599), (10.78916,16.65328), (0.37675,1.362), (3.98094,3.87957), (-3.64775,11.16134), (-4.8443,6.25357), (1.102,4.21945), (8.72112,12.50047), (-1.47361,6.45486), (6.24183,18.99924), (6.83569,18.09508), (-3.11684,13.59528), (4.91306,3.39681), (-0.03628,13.33157), (5.1282,5.8945), (-2.38558,5.61212), (2.33351,8.41149), (-0.97191,13.78608), (-0.05588,6.08609), (-4.70019,12.76962), (-5.12371,3.26206), (0.65606,0.25528), (-0.11574,11.9083), (4.4238,4.35071), (6.93399,11.19855), (3.68712,13.87404), (-0.01187,6.87986), (1.8332,8.32566), (5.81322,22.51334), (-4.04709,2.5226), (-8.26397,16.84498), (-2.11273,6.26108), (5.28396,13.84824), (0.73054,6.03262), (6.43559,14.12668), (4.35565,16.01939), (-1.05545,8.19237), (5.00087,18.01595), (-2.72239,9.45609), (7.32313,6.90459), (2.11548,12.83115), (-3.40953,10.603), (6.97051,13.70439), (-0.45567,6.1633), (1.31699,4.1151), (-1.49871,8.20499), (7.14772,11.67903), (0.79277,7.30851), (6.9698,6.50941), (2.08733,7.3949), (-3.55962,12.80075), (0.75601,5.62043), (1.21,18.2542), (-2.17877,17.9393), (1.83206,16.4569), (5.72463,8.78811), (7.42257,4.85949), (0.97829,-3.36394), (7.54238,5.38683), (9.91081,12.26083), (-4.61743,10.27907), (-4.40799,11.5144), (9.99854,11.57335), (8.53725,1.94203), (3.2905,7.78228), (0.38634,11.79385), (-2.53374,10.18415), (4.94758,14.67613), (4.79624,4.70301), (5.57664,12.72151), (-6.44871,-3.35508), (3.34431,17.63775), (0.14209,2.53883), (10.88431,14.01483), (0.31846,12.4387), (-0.54703,11.15408), (-4.67791,7.74882), (-5.68011,13.60956), (-4.93362,7.81991), (1.2271,10.90969), (5.27512,8.19828), (-3.84611,-1.18523), (6.81706,0.5916), (10.33033,0.35805), (5.13979,12.98364), (3.66534,11.38628), (-2.07219,13.94644), (10.65442,2.03781), (-3.31751,10.74447), (-1.82011,12.35656), (-0.39886,7.08701), (1.77052,2.69871), (1.29049,19.66653), (7.92344,7.88636), (-2.92595,10.36916), (-2.67107,1.632), (5.64708,11.86081), (0.34639,13.47602), (-3.04356,6.60204), (3.98828,7.01303), (-1.36695,20.19992), (-8.48462,18.88249), (-4.04669,11.34367), (9.84561,12.97305), (-6.1537,9.5776), (0.82433,17.91364), (1.92449,18.3247), (2.51288,9.9211), (0.40965,7.14257), (2.89183,6.59133), (3.84347,12.35274), (0.66829,10.57523), (-3.45094,12.12859), (1.3544,9.47177), (-9.85456,0.60659), (5.25689,4.72996), (-5.26018,4.51121), (-6.16912,13.28893), (-1.77163,8.09014), (3.96687,8.02511), (0.70893,13.85406), (-5.45342,1.75412), (-3.89706,6.00641), (3.11868,6.35554), (4.41714,7.11293), (7.64841,8.30442), (0.00489,12.63024), (3.2263,12.38966), (-5.33042,7.6801), (2.52189,11.33744), (-7.40308,4.67713), (0.67891,7.62276), (2.49343,2.14478), (5.43133,15.32988), (-0.67541,1.52299), (-0.60299,17.00017), (-6.32903,8.29701), (-3.44336,10.92961), (-0.23963,6.78449), (6.94686,7.02698), (6.59442,11.51719), (-4.18532,9.97926), (-1.8228,7.44251), (-0.29443,7.58541), (2.99821,4.76058), (2.51942,12.88959), (-3.49176,9.974), (-0.57979,17.03689), (8.69471,11.14554), (-1.19427,11.7392), (-3.17119,11.50029), (-2.99566,19.41759), (-3.34493,9.65127), (-2.33826,9.87673), (-5.04164,14.13485), (-0.48214,9.78034), (7.45097,1.57826), (3.04787,3.72091), (2.92632,9.4054), (1.39694,23.22816), (4.38686,-0.12571), (3.25753,6.97343), (7.14218,10.09049), (-4.04341,11.78393), (-9.19352,3.01909), (2.78473,16.09448), (0.33331,6.25485), (9.89238,7.13164), (6.00566,7.75879), (-1.7511,9.56834), (4.77815,6.14824), (5.07457,13.53454), (2.56132,8.26364), (2.38317,8.7095), (-1.63486,10.61607), (-1.46871,10.64418), (-5.8681,23.9106), (-2.96227,11.38978), (-1.90638,11.4383), (-13.3052,18.41498), (-2.14705,3.70959), (-9.62069,19.95918), (2.29313,9.53847), (0.22162,14.04957), (-1.83956,13.70151), (4.1853,5.45046), (6.05965,10.95061), (-0.23737,9.55156), (6.07452,17.92345), (4.34629,6.23976), (4.02922,8.71029), (3.62622,13.58736), (-3.95825,8.78527), (-1.63412,11.14213), (-1.25727,12.23717), (5.06323,16.44557), (-0.66176,0.47144), (2.36606,9.7198), (-5.77792,13.50981), (4.535,14.27806), (1.02031,13.50793), (4.49345,7.47381), (-4.99791,11.07844), (2.46716,9.89844), (3.65471,21.48548), (11.2283,6.92085), (6.69743,4.44074), (-5.60375,19.98074), (0.28683,7.92826), (-0.85737,16.6313), (4.26726,17.17618), (-3.4322,13.80807), (-2.07039,5.37083), (-2.26798,9.73962), (-0.99818,10.66273), (0.41335,8.90639), (5.18124,12.24596), (-5.01858,16.89203), (2.05561,12.69184), (-0.12117,15.59077), (0.99471,6.94287), (6.89979,-0.1801), (-4.18527,3.25318), (-6.35104,8.08804), (3.89734,13.78384), (-1.979,0.46434), (3.15404,7.78224), (3.52672,9.10987), (2.48372,-0.89391), (-6.13089,14.3696), (2.2968,3.01763), (-2.74324,8.03559), (-0.12876,7.24609), (-1.51135,11.86271), (-3.92434,6.28196), (-1.71254,8.9725), (-1.25878,14.46114), (2.03021,9.50216), (4.31726,16.30413), (-3.02908,1.02795), (9.7093,1.88717), (-3.36284,9.80106), (6.70938,4.53487), (0.42762,16.34543), (5.04726,7.71098), (2.78386,2.74639), (6.83022,6.51875), (-3.02109,10.42308), (-0.65382,13.57901), (-15.58675,0.52784), (5.89746,4.4708), (-4.11598,6.39619), (-1.37208,14.57666), (10.08082,2.71602), (5.35686,12.53905), (1.93331,11.4292), (10.47444,12.44641), (-2.36872,14.50894), (6.50752,17.64374), (2.54603,11.03218), (-0.4332,9.82789), (5.26572,10.11104), (2.09016,2.16137), (1.15513,10.24054), (14.95941,12.86909), (-3.85505,15.22845), (-2.36239,5.05411), (1.64338,10.84836), (-4.25074,11.15717), (7.29744,0.91782), (-1.18964,13.29961), (5.60612,15.11314), (-3.77011,11.54004), (6.67642,-0.94238), (-0.06862,19.32581), (5.60514,10.20744), (3.7341,6.54857), (9.59001,8.69108), (3.30093,8.2296), (-2.75658,8.4474), (4.71994,6.81178), (0.74699,5.99415), (2.91095,13.99336), (-7.36829,8.7469), (-5.29487,8.62349), (3.31079,1.84212), (1.06974,4.4762), (-1.18424,9.25421), (-7.415,10.44229), (3.40595,12.21649), (-7.63085,10.45968), (1.13336,15.34722), (-0.0096,5.50868), (0.8928,10.93609), (-0.5943,2.78631), (7.48306,11.86145), (10.11943,18.67385), (5.60459,10.64051), (4.00189,12.75565), (2.35823,6.63666), (0.33475,12.19343), (3.47072,9.08636), (-6.68867,11.67256), (3.31031,20.31392), (2.17159,11.66443); -SELECT -28.740781574102936, 7.667329672103986e-133; +SELECT '-28.740781574102936', '7.667329672103986e-133'; SELECT roundBankers(StudentTTest(left, right).1, 16) as t_stat, roundBankers(StudentTTest(left, right).2, 16) as p_value from student_ttest; DROP TABLE IF EXISTS student_ttest; diff --git a/tests/queries/0_stateless/01322_welch_ttest.sql b/tests/queries/0_stateless/01322_welch_ttest.sql index 2a045e70b32..5a5b52ab612 100644 --- a/tests/queries/0_stateless/01322_welch_ttest.sql +++ b/tests/queries/0_stateless/01322_welch_ttest.sql @@ -23,7 +23,7 @@ DROP TABLE IF EXISTS welch_ttest; Second: a=10, sigma = 10, size = 500 */ CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (14.72789,-8.65656), (9.61661,22.98234), (13.57615,23.80821), (3.98392,13.33939), (11.98889,-4.05537), (10.99422,23.5155), (5.44792,-6.45272), (20.29346,17.7903), (7.05926,11.463), (9.22732,5.28021), (12.06847,8.39157), (13.52612,6.02464), (8.24597,14.43732), (9.35245,15.76584), (10.12297,1.54391), (15.80624,1.24897), (13.68613,27.1507), (10.72729,7.71091), (5.62078,15.71846), (6.12229,32.97808), (6.03801,-1.79334), (8.95585,-9.23439), (24.04613,11.27838), (9.04757,0.72703), (2.68263,18.51557), (15.43935,9.16619), (2.89423,17.29624), (4.01423,-1.30208), (4.30568,-3.48018), (11.99948,10.12082), (8.40574,-8.01318), (10.86642,-14.22264), (9.4266,16.58174), (-8.12752,-0.55975), (7.91634,5.61449), (7.3967,1.44626), (2.26431,7.89158), (14.20118,1.13369), (6.68233,-0.82609), (15.46221,12.23365), (7.88467,12.45443), (11.20011,14.46915), (8.92027,13.72627), (10.27926,18.41459), (5.14395,29.66702), (5.62178,1.51619), (12.84383,10.40078), (9.98009,3.33266), (-0.69789,6.12036), (11.41386,11.86553), (7.76863,6.59422), (7.21743,22.0948), (1.81176,1.79623), (9.43762,14.29513), (19.22117,19.69162), (2.97128,-7.98033), (14.32851,5.48433), (7.54959,-2.28474), (3.81545,9.91876), (10.1281,10.64097), (2.48596,0.22523), (10.0461,17.01773), (3.59714,22.37388), (9.73522,14.04215), (18.8077,23.1244), (3.15148,18.96958), (12.26062,8.42663), (5.66707,3.7165), (6.58623,14.29366), (17.30902,23.50886), (9.91391,26.33722), (5.36946,26.72396), (15.73637,13.26287), (16.96281,12.97607), (11.54063,17.41838), (18.37358,8.63875), (11.38255,17.08943), (10.53256,23.15356), (8.08833,-4.4965), (16.27556,7.58895), (2.42969,26.04074), (9.56127,6.84245), (7.32998,20.56287), (9.19511,3.84735), (9.66903,-2.76304), (4.15029,13.1615), (8.83511,8.21954), (14.60617,-3.49943), (14.06143,22.12419), (5.39556,7.08323), (10.11871,16.12937), (10.56619,-0.32672), (14.4462,16.5942), (10.42106,7.68977), (7.75551,11.39484), (11.00418,-5.11987), (4.47226,20.87404), (16.35461,8.01007), (18.55174,3.26497), (11.82044,5.61253), (7.39454,20.69182), (11.27767,0.0296), (6.83827,21.904), (7.76858,22.46572), (15.97614,3.63685), (14.53781,-5.10846), (12.99546,14.86389), (16.91151,5.47188), (9.65012,18.44095), (14.25487,16.71368), (14.03618,6.36704), (2.57382,8.82663), (2.50779,14.6727), (14.24787,7.98383), (13.34666,2.65568), (7.31102,21.45827), (10.22981,11.77948), (17.4435,4.71979), (21.2074,3.17951), (6.64191,13.90226), (18.7086,15.50578), (14.78686,10.8026), (9.85287,16.91369), (4.48263,9.90552), (14.17469,13.87322), (14.4342,4.12366), (19.2481,-3.78985), (3.47165,1.7599), (8.28712,3.43715), (8.81657,-3.45246), (0.92319,23.64571), (20.41106,-4.96877), (6.76127,3.93514), (22.00242,1.49914), (8.66129,12.71519), (10.9929,5.11521), (17.95494,4.79872), (17.20996,20.89391), (12.18888,5.363), (12.14257,8.02765), (15.81243,14.30804), (4.43362,11.49002), (1.17567,14.25281), (15.60881,7.6573), (9.34833,15.49686), (6.33513,3.29327), (-0.83095,2.27236), (12.43268,12.58104), (6.63207,19.19128), (11.96877,15.25901), (14.81029,6.5221), (21.84876,10.10965), (3.75896,12.75249), (6.91307,16.50977), (13.73015,-8.6697), (8.63753,8.28553), (15.71679,1.44315), (1.74565,4.65869), (9.16895,0.98149), (5.70685,0.16623), (5.00117,17.66332), (13.06888,4.35346), (7.51204,6.52742), (15.34885,-1.06631), (5.20264,-5.28454), (8.59043,14.25583), (6.45619,8.74058), (14.61979,1.89553), (11.7075,-0.92959), (14.04901,10.30289), (4.20525,-6.3744), (15.1733,-8.1706), (3.12934,10.95369), (8.08049,4.94384), (15.41273,28.40568), (16.90751,3.7004), (5.86893,2.52363), (7.1086,4.07997), (4.418,7.8849), (12.0614,17.95409), (7.07887,16.67021), (3.61585,11.34377), (11.73001,-0.07446), (10.80449,22.00223), (8.40311,3.31778), (9.91276,18.50719), (16.4164,-3.58655), (5.25034,6.5394), (15.20283,12.40459), (10.42909,16.59866), (9.53888,7.54176), (14.68939,-1.51044), (6.60007,12.69758), (18.31058,2.9842), (7.01885,2.49187), (18.71631,2.04113), (10.50002,-2.46544), (10.7517,15.18368), (4.23224,-0.04058), (2.28924,-0.4127), (8.56059,10.5526), (8.25095,12.03982), (9.15673,12.10923), (13.28409,11.54954), (8.4513,-1.18613), (2.83911,11.30984), (2.79676,23.54105), (9.11055,10.67321), (7.18529,24.09196), (-4.1258,7.5008), (5.28306,12.52233), (6.82757,4.30673), (10.89035,9.35793), (5.24822,4.44472), (11.935,-7.00679), (6.45675,8.56241), (10.18088,23.73891), (4.9932,15.62708), (18.09939,16.09205), (8.11738,12.52074), (5.37883,14.58927), (10.50339,-4.80187), (16.64093,8.47964), (14.77263,7.75477), (13.71385,12.6893), (6.98746,7.14147), (10.74635,12.12654), (5.49432,12.32334), (13.46078,7.98909), (10.67565,3.26652), (9.0291,20.53684), (11.51417,32.3369), (13.07118,19.74911), (9.5049,-4.62897), (8.50611,8.26483), (6.47606,20.88451), (13.06526,-2.12982), (19.08658,25.61459), (9.49741,5.32091), (10.60865,-4.1196), (2.28996,7.57937), (8.12846,21.15847), (5.62241,6.46355), (4.07712,7.74846), (17.98526,19.62636), (9.466,28.34629), (11.38904,26.73919), (5.91826,20.40427), (1.52059,3.03378), (18.79161,10.2537), (18.20669,7.47745), (-1.67829,10.79184), (18.01586,3.91962), (16.31577,19.97973), (7.88281,18.87711), (8.46179,12.56157), (10.31113,11.46033), (14.88377,3.78661), (1.31835,-9.45748), (2.53176,12.06033), (9.48625,-0.74615), (3.97936,13.2815), (11.52319,24.78052), (13.24178,5.83337), (7.58739,17.4111), (10.00959,19.70331), (9.73361,11.78446), (8.35716,-1.366), (1.65491,1.37458), (11.11521,16.31483), (6.08355,32.63464), (10.04582,-3.79736), (11.58237,19.17984), (16.40249,-0.27705), (1.9691,-3.69456), (13.22776,28.38058), (2.67059,-1.36876), (9.83651,-25.63301), (2.12539,3.58644), (9.27114,-6.85667), (9.0699,13.42225), (2.78179,12.04671), (12.49311,28.99468), (12.97662,7.87662), (15.06359,2.61119), (16.91565,-3.56022), (5.92011,1.50022), (5.81304,14.55836), (8.46425,9.35831), (9.48705,16.9366), (4.68191,29.23126), (5.70028,15.31386), (-0.78798,13.46112), (10.03442,7.39667), (15.45433,11.15599), (9.43845,9.80499), (3.05825,22.64923), (6.92126,8.67693), (14.05905,18.67335), (19.71579,-3.19127), (15.0131,22.94716), (4.50386,17.86834), (1.31061,16.98267), (10.81197,15.91653), (14.32942,11.79718), (9.26469,18.50208), (7.27679,8.90755), (22.69295,10.44843), (12.03763,4.67433), (7.34876,6.82287), (16.60689,10.82228), (7.48786,-4.18631), (15.78602,20.3872), (17.21048,11.84735), (13.93482,21.25376), (9.69911,10.55032), (12.24315,12.19023), (10.58131,0.63369), (19.57006,7.92381), (9.8856,17.90933), (11.70302,15.30781), (7.89864,10.01877), (12.24831,0.88744), (16.93707,22.20967), (9.65467,-4.23117), (4.221,21.50819), (15.45229,11.27421), (12.83088,-16.23179), (7.58313,33.43085), (12.895,5.15093), (10.02471,1.34505), (13.36059,6.027), (5.07864,-10.43035), (9.72017,27.45998), (11.05809,19.24886), (15.28528,-4.44761), (13.99834,5.453), (19.26989,12.73758), (9.41846,11.2897), (11.65425,31.032), (8.49638,7.39168), (6.38592,11.95245), (-4.69837,26.279), (12.22061,-1.0255), (9.41331,10.36675), (13.2075,11.58439), (12.97005,27.8405), (11.44352,13.1707), (9.79805,31.39133), (6.93116,27.08301), (10.07691,-2.14368), (22.05892,4.08476), (7.80353,21.5573), (-2.17276,16.69822), (0.61509,7.69955), (8.35842,8.32793), (17.77108,6.49235), (14.70841,-7.3284), (1.27992,10.58264), (15.62699,-6.17006), (9.32914,34.55782), (15.41866,10.93221), (10.82009,44.24299), (3.29902,14.6224), (9.21998,-7.42798), (7.93845,15.52351), (10.33344,11.33982), (12.06399,10.46716), (5.5308,13.0986), (8.38727,-4.25988), (18.11104,9.55316), (8.86565,0.75489), (19.41825,25.99212), (9.52376,-0.81401), (3.94552,3.49551), (9.37587,22.99402), (15.44954,10.99628), (15.90527,23.70223), (13.18927,2.71482), (7.01646,22.82309), (9.06005,31.25686), (9.06431,4.86318), (5.76006,-1.06476), (9.18705,15.10298), (-3.48446,-0.61015), (15.89817,17.81246), (12.94719,-1.55788), (23.69426,18.09709), (17.47755,9.11271), (15.61528,9.94682), (0.54832,-7.33194), (14.32916,-4.67293), (9.55305,21.81717), (13.79891,7.16318), (0.82544,13.25649), (13.34875,13.88776), (9.07614,4.95793), (5.19621,17.65303), (2.1451,14.47382), (9.87726,13.19373), (8.45439,31.86093), (-1.41842,5.73161), (7.93598,10.96492), (11.23151,6.97951), (17.84458,1.75136), (7.02237,10.96144), (10.7842,15.08137), (4.42832,9.95311), (4.45044,7.07729), (1.50938,3.08148), (21.21651,22.37954), (6.2097,8.51951), (6.84354,2.88746), (18.53804,26.73509), (12.01072,-2.88939), (4.8345,-2.82367), (20.41587,-0.35783), (14.48353,14.22076), (8.71116,11.50295), (12.42818,7.10171), (14.89244,8.28488), (8.03033,0.54178), (5.25917,13.8022), (2.30092,15.62157), (10.22504,10.79173), (15.37573,28.18946), (7.13666,30.43524), (4.45018,2.54914), (10.18405,9.89421), (3.91025,13.08631), (14.52304,4.68761), (13.14771,5.61516), (11.99219,22.88072), (9.21345,7.4735), (8.85106,11.27382), (12.91887,2.39559), (15.62308,-3.31889), (11.88034,9.61957), (15.12097,23.01381), (11.58168,-1.23467), (16.83051,9.07691), (5.25405,15.78056), (2.19976,12.28421), (4.56716,9.44888), (16.46053,13.16928), (5.61995,4.33357), (8.67704,2.21737), (5.62789,33.17833), (9.84815,13.25407), (13.05834,-2.47961), (11.74205,6.41401), (3.88393,18.8439), (16.15321,-4.63375), (4.83925,-8.2909), (13.00334,12.18221), (4.4028,-2.95356), (4.35794,19.61659), (4.47478,12.45056), (2.38713,-4.17198), (4.25235,21.9641), (10.87509,11.96416), (9.82411,12.74573), (13.61518,10.47873), (10.25507,12.73295), (4.0335,11.31373), (10.69881,9.9827), (5.70321,5.87138), (6.96244,4.24372), (9.35874,-23.72256), (6.28076,28.41337), (8.29015,4.88103), (6.88653,3.61902), (7.70687,8.93586), (8.2001,16.40759), (6.73415,27.84494), (3.82052,5.6001), (3.94469,14.51379), (15.82384,13.5576), (2.54004,12.92213), (10.74876,3.90686), (12.60517,17.07104), (17.7024,15.84268), (4.6722,17.38777), (13.67341,16.54766), (6.4565,5.94487), (12.95699,17.02804), (4.56912,7.66386), (5.58464,10.43088), (4.0638,6.16059), (13.05559,20.46178), (5.38269,20.02888), (0.16354,20.95949), (7.23962,6.50808), (7.38577,7.22366), (8.50951,8.06659), (13.72574,16.08241), (17.80421,13.83514), (3.01135,-0.33454), (8.02608,12.98848), (14.23847,12.99024); -SELECT -0.5028215369186904, 0.6152361677168877; +SELECT '-0.5028215369186904', '0.6152361677168877'; SELECT roundBankers(WelchTTest(left, right).1, 16) as t_stat, roundBankers(WelchTTest(left, right).2, 16) as p_value from welch_ttest; DROP TABLE IF EXISTS welch_ttest; @@ -32,6 +32,6 @@ DROP TABLE IF EXISTS welch_ttest; Second: a=1, sigma = 12, size = 500 */ CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (4.82025,-2.69857), (6.13896,15.80943), (15.20277,7.31555), (14.15351,3.96517), (7.21338,4.77809), (8.55506,9.6472), (13.80816,-26.41717), (11.28411,-10.85635), (7.4612,-1.4376), (7.43759,-0.96308), (12.9832,2.84315), (-5.74783,5.79467), (12.47114,-3.06091), (15.14223,-14.62902), (3.40603,22.08022), (9.27323,-2.11982), (7.88547,-4.84824), (8.56456,-10.50447), (4.59731,2.4891), (7.91213,9.90324), (7.33894,-22.66866), (21.74811,-0.97103), (11.92111,-16.57608), (0.18828,-3.78749), (10.47314,25.84511), (20.37396,5.30797), (11.04991,-18.19466), (13.30083,11.72708), (14.28065,0.2891), (2.86942,-9.83474), (24.96072,6.69942), (14.20164,18.09604), (18.28769,18.52651), (10.50949,1.38201), (9.22273,7.64615), (11.77608,17.66598), (8.56872,-2.44141), (13.74535,-9.01598), (11.65209,27.69142), (12.51894,4.06946), (17.76256,-15.0077), (13.52122,-10.49648), (8.70796,-4.88322), (6.04749,-25.09805), (16.33064,-4.64024), (8.35636,20.94434), (14.03496,24.12126), (11.05834,-14.10962), (14.49261,10.6512), (2.59383,14.50687), (8.01022,-19.88081), (4.05458,-11.55271), (13.26384,13.16921), (14.62058,16.63864), (10.52489,-24.08114), (8.46357,-9.09949), (6.4147,-10.54702), (9.70071,0.20813), (12.47581,8.19066), (4.38333,-2.70523), (17.54172,-0.23954), (10.12109,7.19398), (7.73186,-7.1618), (14.0279,-7.44322), (11.6621,-17.92031), (17.47045,-1.58146), (15.50223,9.18338), (15.46034,3.25838), (13.39964,-14.30234), (14.98025,1.84695), (15.87912,31.13794), (17.67374,-0.85067), (9.64073,19.02787), (12.84904,-3.09594), (7.70278,13.45584), (13.03156,-5.48104), (9.04512,-22.74928), (15.97014,-8.03697), (8.96389,17.31143), (11.48009,-16.65231), (9.71153,-18.58713), (13.00084,-16.52641), (12.39803,14.95261), (13.08188,12.56762), (5.82244,15.00188), (10.81871,1.85858), (8.2539,2.1926), (7.52114,-2.4095), (9.11488,21.56873), (8.37482,3.35509), (14.48652,-4.98672), (11.42152,35.08603), (16.03111,-10.01602), (13.14057,-3.85153), (-2.26351,-6.81974), (15.50394,19.56525), (14.88603,-9.35488), (13.37257,0.24268), (11.84026,-3.51488), (7.66558,-0.37066), (6.24584,24.20888), (3.6312,-11.73537), (2.7018,0.01282), (5.63656,0.03963), (5.82643,-9.65589), (10.06745,-0.37429), (-0.5831,5.61255), (14.84202,0.49984), (9.5524,-10.15066), (19.71713,-14.54314), (14.23109,16.56889), (8.69105,-7.73873), (5.33742,-3.76422), (7.30372,1.40722), (7.93342,2.28818), (15.20884,-13.12643), (7.53839,5.17082), (13.45311,4.79089), (11.04473,-17.42643), (10.76673,8.72548), (15.44145,-3.70285), (14.06596,16.77893), (9.14873,13.382), (12.88372,19.98418), (8.74994,0.00483), (10.53263,-4.75951), (16.16694,2.35391), (8.37197,21.65809), (3.43739,-9.2714), (4.72799,-18.38253), (9.08802,7.23097), (11.2531,14.97927), (5.16115,-4.02197), (10.20895,-29.8189), (18.70884,-12.8554), (15.88924,-7.60124), (3.38758,-14.90158), (6.46449,-3.31486), (10.21088,31.38144), (14.08458,-8.61288), (15.74508,15.31895), (19.31896,-10.19488), (13.19641,13.796), (11.95409,-0.32912), (10.70718,-0.0684), (1.05245,-30.06834), (10.04772,24.93912), (17.01369,-3.26506), (10.2286,-8.29751), (19.58323,-5.39189), (7.02892,-25.08603), (4.16866,-1.45318), (8.94326,16.72724), (4.99854,-3.38467), (8.88352,-26.00478), (18.65422,7.28369), (17.32328,16.96226), (9.33492,16.5858), (14.94788,10.46583), (8.05863,3.84345), (14.6737,-2.99382), (10.93801,1.42078), (0.54036,-11.0123), (-0.34242,2.09909), (5.89076,1.21064), (3.15189,15.36079), (1.94421,-21.61349), (6.38698,22.7726), (10.50654,10.50512), (8.95362,-6.95825), (6.23711,9.20036), (11.75359,15.66902), (12.42155,3.28098), (-1.55472,-9.05692), (4.6688,0.32882), (10.48087,-1.64934), (11.74615,-4.81406), (9.26822,-5.06006), (7.55517,19.97493), (12.76005,2.88646), (16.47102,-0.34552), (11.31297,7.55186), (14.37437,-22.96115), (2.38799,31.29166), (6.44577,6.18798), (5.07471,-2.52715), (11.55123,-11.58799), (7.76795,14.13596), (10.60116,13.45069), (14.40885,12.15179), (11.58158,3.44491), (8.81648,-8.78006), (12.92299,18.32087), (11.26939,11.91757), (17.95014,-2.00179), (2.95002,10.88411), (17.41959,9.09327), (11.12455,6.62484), (8.78541,8.87178), (14.36413,11.52254), (12.98554,-14.15988), (12.58505,-17.19515), (15.49789,14.03089), (11.70999,-2.4095), (0.65596,-16.83575), (11.08202,2.71469), (14.75752,4.84351), (6.84385,-1.17651), (9.27245,-3.37529), (13.78243,-19.92137), (17.4863,4.48952), (4.01777,-12.4906), (11.82861,-5.65277), (13.86551,8.50819), (6.16591,-19.61261), (8.71589,12.54156), (16.77195,11.06784), (17.23243,-12.59285), (-2.12941,3.43683), (5.66629,-3.00325), (12.45153,12.49082), (1.63971,7.20955), (13.84031,17.6547), (4.6144,15.8619), (5.26169,24.3048), (9.27769,-8.05434), (9.14288,-6.06901), (9.71953,-15.69515), (9.38446,-11.13917), (1.64788,-3.90757), (11.72922,-2.57038), (13.68926,5.14065), (9.42952,17.8497), (12.05574,-8.64665), (9.09148,-18.68331), (5.32273,5.8567), (20.25258,-20.93884), (10.14599,4.40583), (10.82156,14.35985), (5.75736,4.18134), (7.13567,4.3635), (9.29746,9.35428), (5.1618,2.8908), (10.076,16.01017), (21.65669,-1.48499), (13.35486,-9.97949), (6.79957,1.03055), (8.76243,-2.79697), (14.59294,6.85977), (16.90609,4.73213), (10.50337,2.7815), (-0.07923,-2.46866), (13.51648,18.39425), (12.0676,-0.80378), (0.86482,-0.22982), (9.03563,-16.11608), (5.38751,3.0862), (17.16866,3.20779), (2.78702,10.50146), (11.15548,-0.21305), (12.30843,11.21012), (8.04897,-0.99825), (9.95814,18.39633), (11.29308,-3.39003), (14.13032,-0.64411), (21.05877,-1.39932), (3.57386,15.45319), (7.96631,-0.66044), (3.30484,-15.2223), (18.61856,-34.39907), (16.35184,-3.57836), (7.65236,16.82828), (18.02895,1.66624), (9.79458,15.43475), (16.7274,8.17776), (8.84453,5.50486), (13.05709,10.43082), (10.91447,-6.63332), (8.40171,2.28008), (16.95211,16.37203), (11.82194,5.16313), (19.87978,-8.85281), (12.88455,13.26692), (-0.00947,-7.46842), (12.28109,8.43091), (6.96462,-13.18172), (13.75282,-0.72401), (14.39141,22.3881), (11.07193,10.65448), (12.88039,2.81289), (11.38253,10.92405), (21.02707,-8.95358), (7.51955,19.80653), (6.31984,-12.86527), (15.6543,5.38826), (14.80315,-6.83501), (8.38024,-15.7647), (21.7516,-27.67412), (14.31336,8.6499), (15.04703,-4.89542), (5.73787,16.76167), (13.16911,12.84284), (12.40695,-17.27324), (9.88968,-4.18726), (8.46703,-14.62366), (8.70637,-5.49863), (8.03551,-16.22846), (5.9757,10.60329), (12.22951,6.46781), (3.14736,1.70458), (10.51266,10.77448), (18.593,0.8463), (10.82213,13.0482), (7.14216,-4.36264), (6.81154,3.22647), (-0.6486,2.38828), (20.56136,6.7946), (11.35367,-0.25254), (11.38205,1.2497), (17.14,1.6544), (14.91215,4.1019), (15.50207,11.27839), (5.93162,-5.04127), (3.74869,18.11674), (14.11532,0.51231), (7.38954,-0.51029), (5.45764,13.52556), (18.33733,16.10171), (9.91923,5.68197), (2.38991,-2.85904), (14.16756,-8.89167), (2.39791,6.24489), (6.92586,10.85319), (5.32474,-0.39816), (2.28812,3.87079), (5.71718,-3.1867), (5.84197,1.55322), (2.76206,16.86779), (19.05928,-14.60321), (11.51788,-1.81952), (6.56648,-3.11624), (3.35735,1.24193), (7.55948,10.18179), (19.99908,4.69796), (13.00634,0.69032), (18.36886,11.7723), (11.14675,7.62896), (16.72931,9.89741), (12.50106,9.11484), (6.00605,-3.84676), (23.06653,-0.4777), (5.39694,0.95958), (9.53167,-7.95056), (12.76944,-10.97474), (7.20604,-6.54861), (13.25391,34.74933), (13.7341,27.39463), (10.85292,4.18299), (-7.75835,6.02476), (10.29728,-1.99397), (13.70099,1.26478), (10.17959,23.37106), (9.98399,10.49682), (12.69389,-11.04354), (-0.28848,-12.22284), (-2.18319,-9.87635), (13.36378,28.90511), (10.09232,6.77613), (5.49489,0.55352), (5.46156,0.37031), (0.94225,7.1418), (12.79205,3.24897), (10.09593,-1.60918), (6.06218,3.1675), (0.89463,-17.97072), (11.88986,-5.61743), (10.79733,14.1422), (1.51371,14.87695), (2.20967,-4.65961), (15.45732,-0.99174), (16.5262,-2.96623), (5.99724,-9.02263), (8.3613,-17.2088), (15.68183,2.78608), (15.32117,6.74239), (14.15674,4.8524), (6.64553,7.46731), (4.20777,1.04894), (-0.10521,-12.8023), (-0.88169,-17.18188), (1.85913,-5.08801), (9.73673,22.13942), (0.30926,-0.36384), (6.17559,17.80564), (11.76602,7.67504), (5.68385,1.59779), (14.57088,4.10942), (12.81509,0.61074), (9.85682,-14.40767), (12.06376,10.59906), (6.08874,16.57017), (11.63921,-15.17526), (14.86722,-6.98549), (10.41035,-0.64548), (2.93794,3.23756), (12.21841,14.65504), (0.23804,4.583), (3.14845,12.72378), (7.29748,5.26547), (3.06134,0.81781), (13.77684,9.38273), (16.21992,10.37636), (5.33511,10.70325), (9.68959,-0.83043), (9.44169,-7.53149), (18.08012,-9.09147), (4.04224,-19.51381), (8.77918,-28.44508), (10.18324,6.44392), (9.38914,11.10201), (11.76995,-2.86184), (14.19963,8.30673), (6.88817,8.8797), (16.56123,10.68053), (15.39885,15.62919), (5.21241,8.00579), (4.44408,6.4651), (17.87587,-4.50029), (12.53337,18.04514), (13.60916,11.12996), (6.60104,-5.14007), (7.35453,9.43857), (18.61572,3.13476), (6.10437,4.9772), (13.08682,-17.45782), (12.15404,0.05552), (4.90789,-1.90283), (2.13353,2.67908), (12.49593,-2.62243), (11.93056,-3.22767), (13.29408,-8.70222), (5.70038,-23.11605), (8.40271,21.6757), (5.19456,12.70076), (-5.51028,4.4322), (14.0329,11.69344), (10.38365,9.18052), (6.56812,-2.2549), (4.21129,-2.15615), (9.7157,20.29765), (9.88553,-0.29536), (13.45346,15.50109), (4.97752,8.79187), (12.77595,5.11533), (8.56465,-20.44436), (4.27703,-3.00909), (18.12502,-4.48291), (12.45735,21.84462), (12.42912,1.94225), (12.08125,-2.81908), (10.85779,17.19418), (4.36013,-9.33528), (11.85062,-0.17346), (8.47776,0.03958), (9.60822,-35.17786), (11.3069,8.36887), (14.25525,-9.02292), (1.55168,-10.98804), (14.57782,0.29335), (7.84786,4.29634), (9.87774,3.87718), (14.75575,-9.08532), (3.68774,7.13922), (9.37667,-7.62463), (20.28676,-10.5666), (12.10027,4.68165), (8.01819,-3.30172), (18.78158,13.04852), (20.85402,13.45616), (18.98069,2.41043), (16.1429,-0.36501), (9.24047,-15.67383), (14.12487,17.92217), (10.18841,8.42106), (-3.04478,3.22063), (5.7552,-7.31753), (9.30376,21.99596), (11.42837,-36.8273), (6.02364,-20.46391), (8.86984,5.74179), (10.91177,-15.83178), (10.04418,14.90454), (18.10774,-8.84645), (7.49384,3.72036), (9.11556,4.6877), (9.7051,16.35418), (5.23268,3.15441), (9.04647,2.39907), (8.81547,-17.58664), (2.65098,-13.18269); -SELECT 14.971190998235835, 5.898143508382202e-44; +SELECT '14.971190998235835', '5.898143508382202e-44'; SELECT roundBankers(WelchTTest(left, right).1, 16) as t_stat, roundBankers(WelchTTest(left, right).2, 16) as p_value from welch_ttest; DROP TABLE IF EXISTS welch_ttest; From e3ef8bbc48dd7a46d11c9159247c64099eb57355 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Wed, 14 Oct 2020 05:29:25 +0400 Subject: [PATCH 164/411] GCC 9/10 compilation fix --- src/Access/LDAPAccessStorage.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index 7ac37b9142c..a845279841c 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include From f26b7573a21e2d849614e48511896a8d55d47cb3 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Wed, 14 Oct 2020 16:58:54 +0400 Subject: [PATCH 165/411] Fix pre-C++20 compiler builds --- src/Access/LDAPAccessStorage.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index a3945686ff7..fc97ff24e69 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -84,7 +84,7 @@ void LDAPAccessStorage::processRoleChange(const UUID & id, const AccessEntityPtr auto role = typeid_cast>(entity); bool need_to_update_users = false; - if (role && default_role_names.contains(role->getName())) + if (role && default_role_names.find(role->getName()) != default_role_names.end()) { /// If a role was created with one of the `default_role_names` or renamed to one of the `default_role_names`, /// then set `need_to_update_users`. @@ -124,7 +124,7 @@ void LDAPAccessStorage::checkAllDefaultRoleNamesFoundNoLock() const for (const auto & role_name : default_role_names) { - if (!role_names_of_interest.contains(role_name)) + if (role_names_of_interest.find(role_name) == role_names_of_interest.end()) throwDefaultRoleNotFound(role_name); } } From 1bcacde8d53fe25d5a2d79521402305091bbe033 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 14 Oct 2020 20:51:38 +0800 Subject: [PATCH 166/411] Eagerly move data out of sink --- src/Processors/Formats/IOutputFormat.cpp | 2 +- src/Processors/ISink.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Processors/Formats/IOutputFormat.cpp b/src/Processors/Formats/IOutputFormat.cpp index d6f2cf75f58..88649d9ca25 100644 --- a/src/Processors/Formats/IOutputFormat.cpp +++ b/src/Processors/Formats/IOutputFormat.cpp @@ -30,7 +30,7 @@ IOutputFormat::Status IOutputFormat::prepare() if (!input.hasData()) return Status::NeedData; - current_chunk = input.pull(); + current_chunk = input.pull(true); current_block_kind = kind; has_input = true; return Status::Ready; diff --git a/src/Processors/ISink.cpp b/src/Processors/ISink.cpp index 5ace49b0824..bfe015f876c 100644 --- a/src/Processors/ISink.cpp +++ b/src/Processors/ISink.cpp @@ -24,7 +24,7 @@ ISink::Status ISink::prepare() if (!input.hasData()) return Status::NeedData; - current_chunk = input.pull(); + current_chunk = input.pull(true); has_input = true; return Status::Ready; } From 966b1d6cf5bddf95beefb2eba5fd7345ef9b7ebb Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 14 Oct 2020 16:09:11 +0300 Subject: [PATCH 167/411] Rename Block to ColumnsWithTypeAndName. --- src/Functions/FunctionBase64Conversion.h | 2 +- src/Functions/FunctionBinaryArithmetic.h | 18 ++--- src/Functions/FunctionBitTestMany.h | 10 +-- src/Functions/FunctionCustomWeekToSomething.h | 2 +- .../FunctionDateOrDateTimeAddInterval.h | 2 +- .../FunctionDateOrDateTimeToSomething.h | 2 +- src/Functions/FunctionFQDN.cpp | 2 +- src/Functions/FunctionJoinGet.cpp | 4 +- src/Functions/FunctionJoinGet.h | 4 +- src/Functions/FunctionMathBinaryFloat64.h | 6 +- src/Functions/FunctionMathConstFloat64.h | 2 +- src/Functions/FunctionMathUnary.h | 6 +- src/Functions/FunctionNumericPredicate.h | 4 +- src/Functions/FunctionStartsEndsWith.h | 2 +- src/Functions/FunctionStringOrArrayToT.h | 2 +- src/Functions/FunctionStringReplace.h | 2 +- src/Functions/FunctionStringToString.h | 2 +- src/Functions/FunctionUnaryArithmetic.h | 2 +- src/Functions/FunctionUnixTimestamp64.h | 2 +- src/Functions/FunctionsBitmap.h | 32 ++++---- src/Functions/FunctionsCoding.h | 34 ++++----- src/Functions/FunctionsComparison.h | 68 ++++++++--------- src/Functions/FunctionsConsistentHashing.h | 4 +- src/Functions/FunctionsConversion.h | 54 ++++++------- src/Functions/FunctionsEmbeddedDictionaries.h | 8 +- src/Functions/FunctionsExternalDictionaries.h | 72 +++++++++--------- src/Functions/FunctionsExternalModels.cpp | 2 +- src/Functions/FunctionsExternalModels.h | 2 +- src/Functions/FunctionsFormatting.h | 12 +-- src/Functions/FunctionsHashing.h | 14 ++-- src/Functions/FunctionsJSON.h | 2 +- src/Functions/FunctionsLogical.cpp | 4 +- src/Functions/FunctionsLogical.h | 4 +- src/Functions/FunctionsMiscellaneous.h | 8 +- .../FunctionsMultiStringFuzzySearch.h | 2 +- src/Functions/FunctionsMultiStringPosition.h | 2 +- src/Functions/FunctionsMultiStringSearch.h | 2 +- src/Functions/FunctionsRandom.h | 2 +- src/Functions/FunctionsRound.h | 6 +- src/Functions/FunctionsStringArray.h | 4 +- src/Functions/FunctionsStringSearch.h | 2 +- src/Functions/FunctionsStringSearchToString.h | 2 +- src/Functions/FunctionsStringSimilarity.h | 2 +- src/Functions/IFunctionAdaptors.h | 6 +- src/Functions/IFunctionImpl.h | 16 ++-- src/Functions/LeastGreatestGeneric.h | 2 +- src/Functions/URL/port.cpp | 2 +- src/Functions/abtesting.cpp | 2 +- src/Functions/addressToLine.cpp | 2 +- src/Functions/addressToSymbol.cpp | 2 +- src/Functions/appendTrailingCharIfAbsent.cpp | 2 +- src/Functions/array/FunctionArrayMapped.h | 2 +- src/Functions/array/array.cpp | 2 +- src/Functions/array/arrayConcat.cpp | 2 +- src/Functions/array/arrayDistinct.cpp | 4 +- src/Functions/array/arrayElement.cpp | 76 +++++++++---------- src/Functions/array/arrayEnumerate.cpp | 2 +- src/Functions/array/arrayEnumerateExtended.h | 4 +- src/Functions/array/arrayEnumerateRanked.h | 4 +- src/Functions/array/arrayFlatten.cpp | 2 +- src/Functions/array/arrayIndex.h | 18 ++--- src/Functions/array/arrayIntersect.cpp | 10 +-- src/Functions/array/arrayJoin.cpp | 2 +- src/Functions/array/arrayPop.h | 2 +- src/Functions/array/arrayPush.h | 2 +- src/Functions/array/arrayReduce.cpp | 4 +- src/Functions/array/arrayReduceInRanges.cpp | 4 +- src/Functions/array/arrayResize.cpp | 2 +- src/Functions/array/arrayReverse.cpp | 4 +- src/Functions/array/arrayScalarProduct.h | 6 +- src/Functions/array/arraySlice.cpp | 2 +- src/Functions/array/arrayUniq.cpp | 4 +- src/Functions/array/arrayWithConstant.cpp | 2 +- src/Functions/array/arrayZip.cpp | 2 +- src/Functions/array/emptyArray.cpp | 2 +- src/Functions/array/emptyArrayToSingle.cpp | 4 +- src/Functions/array/hasAllAny.h | 2 +- src/Functions/array/mapOp.cpp | 6 +- src/Functions/array/mapPopulateSeries.cpp | 6 +- src/Functions/array/range.cpp | 14 ++-- src/Functions/assumeNotNull.cpp | 2 +- src/Functions/bar.cpp | 4 +- src/Functions/blockNumber.cpp | 2 +- src/Functions/blockSerializedSize.cpp | 2 +- src/Functions/blockSize.cpp | 2 +- src/Functions/buildId.cpp | 2 +- src/Functions/caseWithExpression.cpp | 2 +- src/Functions/coalesce.cpp | 2 +- src/Functions/concat.cpp | 6 +- src/Functions/convertCharset.cpp | 2 +- src/Functions/countDigits.cpp | 2 +- src/Functions/currentDatabase.cpp | 2 +- src/Functions/currentUser.cpp | 2 +- src/Functions/dateDiff.cpp | 2 +- src/Functions/date_trunc.cpp | 2 +- src/Functions/defaultValueOfArgumentType.cpp | 2 +- src/Functions/defaultValueOfTypeName.cpp | 2 +- src/Functions/demange.cpp | 2 +- src/Functions/dumpColumnStructure.cpp | 2 +- src/Functions/equals.cpp | 2 +- src/Functions/evalMLMethod.cpp | 2 +- src/Functions/extractAllGroups.h | 2 +- src/Functions/extractGroups.cpp | 2 +- src/Functions/filesystem.cpp | 2 +- src/Functions/finalizeAggregation.cpp | 2 +- src/Functions/formatDateTime.cpp | 4 +- src/Functions/formatRow.cpp | 2 +- src/Functions/formatString.cpp | 2 +- src/Functions/fuzzBits.cpp | 2 +- src/Functions/generateUUIDv4.cpp | 4 +- src/Functions/geoToH3.cpp | 2 +- src/Functions/geohashDecode.cpp | 2 +- src/Functions/geohashEncode.cpp | 2 +- src/Functions/geohashesInBox.cpp | 2 +- src/Functions/getMacro.cpp | 2 +- src/Functions/getScalar.cpp | 2 +- src/Functions/getSetting.cpp | 2 +- src/Functions/getSizeOfEnumType.cpp | 2 +- src/Functions/globalVariable.cpp | 2 +- src/Functions/greatCircleDistance.cpp | 2 +- src/Functions/greater.cpp | 2 +- src/Functions/greaterOrEquals.cpp | 2 +- src/Functions/h3EdgeAngle.cpp | 2 +- src/Functions/h3EdgeLengthM.cpp | 2 +- src/Functions/h3GetBaseCell.cpp | 2 +- src/Functions/h3GetResolution.cpp | 2 +- src/Functions/h3HexAreaM2.cpp | 2 +- src/Functions/h3IndexesAreNeighbors.cpp | 2 +- src/Functions/h3IsValid.cpp | 2 +- src/Functions/h3ToChildren.cpp | 2 +- src/Functions/h3ToParent.cpp | 2 +- src/Functions/h3ToString.cpp | 2 +- src/Functions/h3kRing.cpp | 2 +- src/Functions/hasColumnInTable.cpp | 4 +- src/Functions/hasThreadFuzzer.cpp | 2 +- src/Functions/hostName.cpp | 2 +- src/Functions/identity.cpp | 2 +- src/Functions/if.cpp | 44 +++++------ src/Functions/ifNotFinite.cpp | 2 +- src/Functions/ifNull.cpp | 2 +- src/Functions/ignore.cpp | 2 +- src/Functions/in.cpp | 2 +- src/Functions/initializeAggregation.cpp | 4 +- src/Functions/isConstant.cpp | 2 +- src/Functions/isDecimalOverflow.cpp | 2 +- src/Functions/isNotNull.cpp | 2 +- src/Functions/isNull.cpp | 2 +- src/Functions/isZeroOrNull.cpp | 2 +- src/Functions/less.cpp | 2 +- src/Functions/lessOrEquals.cpp | 2 +- src/Functions/logTrace.cpp | 2 +- src/Functions/lowCardinalityIndices.cpp | 2 +- src/Functions/lowCardinalityKeys.cpp | 2 +- src/Functions/materialize.cpp | 2 +- src/Functions/multiIf.cpp | 2 +- src/Functions/neighbor.cpp | 2 +- src/Functions/normalizedQueryHash.cpp | 2 +- src/Functions/notEquals.cpp | 2 +- src/Functions/now.cpp | 4 +- src/Functions/now64.cpp | 2 +- src/Functions/nullIf.cpp | 2 +- src/Functions/pointInEllipses.cpp | 2 +- src/Functions/pointInPolygon.cpp | 8 +- src/Functions/randConstant.cpp | 4 +- src/Functions/randomFixedString.cpp | 4 +- src/Functions/randomPrintableASCII.cpp | 2 +- src/Functions/randomString.cpp | 4 +- src/Functions/randomStringUTF8.cpp | 2 +- src/Functions/regexpQuoteMeta.cpp | 2 +- src/Functions/reinterpretAsFixedString.cpp | 2 +- src/Functions/reinterpretAsString.cpp | 2 +- src/Functions/reinterpretStringAs.cpp | 2 +- src/Functions/repeat.cpp | 2 +- src/Functions/replicate.cpp | 2 +- src/Functions/reverse.cpp | 2 +- src/Functions/rowNumberInAllBlocks.cpp | 4 +- src/Functions/rowNumberInBlock.cpp | 2 +- src/Functions/runningAccumulate.cpp | 2 +- src/Functions/runningDifference.h | 2 +- src/Functions/sleep.h | 2 +- src/Functions/stringToH3.cpp | 2 +- src/Functions/substring.cpp | 8 +- src/Functions/throwIf.cpp | 4 +- src/Functions/tid.cpp | 2 +- src/Functions/timeSlots.cpp | 2 +- src/Functions/timezone.cpp | 2 +- src/Functions/toColumnTypeName.cpp | 2 +- src/Functions/toFixedString.h | 4 +- src/Functions/toLowCardinality.cpp | 2 +- src/Functions/toNullable.cpp | 2 +- src/Functions/toStartOfInterval.cpp | 2 +- src/Functions/toTimeZone.cpp | 2 +- src/Functions/toTypeName.cpp | 4 +- src/Functions/today.cpp | 4 +- src/Functions/transform.cpp | 6 +- src/Functions/tuple.cpp | 2 +- src/Functions/tupleElement.cpp | 2 +- src/Functions/uptime.cpp | 2 +- src/Functions/version.cpp | 2 +- src/Functions/visibleWidth.cpp | 2 +- src/Functions/yesterday.cpp | 4 +- src/Interpreters/ExpressionJIT.cpp | 4 +- src/Interpreters/ExpressionJIT.h | 2 +- 203 files changed, 492 insertions(+), 498 deletions(-) diff --git a/src/Functions/FunctionBase64Conversion.h b/src/Functions/FunctionBase64Conversion.h index 56624fc5ca9..6fa9197b4b5 100644 --- a/src/Functions/FunctionBase64Conversion.h +++ b/src/Functions/FunctionBase64Conversion.h @@ -91,7 +91,7 @@ public: return std::make_shared(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { const ColumnPtr column_string = block[arguments[0]].column; const ColumnString * input = checkAndGetColumn(column_string.get()); diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index c1f5f0ffcdb..0dd54dd601d 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -613,7 +613,7 @@ class FunctionBinaryArithmetic : public IFunction } /// Multiply aggregation state by integer constant: by merging it with itself specified number of times. - void executeAggregateMultiply(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const + void executeAggregateMultiply(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const { ColumnNumbers new_arguments = arguments; if (WhichDataType(block[new_arguments[1]].type).isAggregateFunction()) @@ -680,7 +680,7 @@ class FunctionBinaryArithmetic : public IFunction } /// Merge two aggregation states together. - void executeAggregateAddition(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const + void executeAggregateAddition(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const { const IColumn & lhs_column = *block[arguments[0]].column; const IColumn & rhs_column = *block[arguments[1]].column; @@ -712,8 +712,8 @@ class FunctionBinaryArithmetic : public IFunction block[result].column = std::move(column_to); } - void executeDateTimeIntervalPlusMinus(Block & block, const ColumnNumbers & arguments, - size_t result, size_t input_rows_count, const FunctionOverloadResolverPtr & function_builder) const + void executeDateTimeIntervalPlusMinus(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, + size_t result, size_t input_rows_count, const FunctionOverloadResolverPtr & function_builder) const { ColumnNumbers new_arguments = arguments; @@ -722,7 +722,7 @@ class FunctionBinaryArithmetic : public IFunction std::swap(new_arguments[0], new_arguments[1]); /// Change interval argument type to its representation - Block new_block = block; + ColumnsWithTypeAndName new_block = block; new_block[new_arguments[1]].type = std::make_shared>(); ColumnsWithTypeAndName new_arguments_with_type_and_name = @@ -851,7 +851,7 @@ public: return type_res; } - bool executeFixedString(Block & block, const ColumnNumbers & arguments, size_t result) const + bool executeFixedString(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result) const { using OpImpl = FixedStringOperationImpl>; @@ -929,7 +929,7 @@ public: } template - bool executeNumeric(Block & block, const ColumnNumbers & arguments, size_t result [[maybe_unused]], const A & left, const B & right) const + bool executeNumeric(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result [[maybe_unused]], const A & left, const B & right) const { using LeftDataType = std::decay_t; using RightDataType = std::decay_t; @@ -1056,7 +1056,7 @@ public: return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { /// Special case when multiply aggregate function state if (isAggregateMultiply(block[arguments[0]].type, block[arguments[1]].type)) @@ -1171,7 +1171,7 @@ class FunctionBinaryArithmeticWithConstants : public FunctionBinaryArithmetic; using Monotonicity = typename Base::Monotonicity; - using Block = typename Base::Block; + using Block = typename Base::ColumnsWithTypeAndName; static FunctionPtr create( const ColumnWithTypeAndName & left_, diff --git a/src/Functions/FunctionBitTestMany.h b/src/Functions/FunctionBitTestMany.h index 54587b5b65e..75a4b7ed1fc 100644 --- a/src/Functions/FunctionBitTestMany.h +++ b/src/Functions/FunctionBitTestMany.h @@ -54,7 +54,7 @@ public: return std::make_shared(); } - void executeImpl(Block & block , const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block , const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const auto value_col = block[arguments.front()].column.get(); @@ -72,8 +72,8 @@ public: private: template bool execute( - Block & block, const ColumnNumbers & arguments, const size_t result, - const IColumn * const value_col_untyped) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, + const IColumn * const value_col_untyped) const { if (const auto value_col = checkAndGetColumn>(value_col_untyped)) { @@ -132,7 +132,7 @@ private: } template - ValueType createConstMaskIfConst(const Block & block, const ColumnNumbers & arguments, bool & out_is_const) const + ValueType createConstMaskIfConst(const ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, bool & out_is_const) const { out_is_const = true; ValueType mask = 0; @@ -156,7 +156,7 @@ private: } template - PaddedPODArray createMask(const size_t size, const Block & block, const ColumnNumbers & arguments) const + PaddedPODArray createMask(const size_t size, const ColumnsWithTypeAndName & block, const ColumnNumbers & arguments) const { PaddedPODArray mask(size, ValueType{}); diff --git a/src/Functions/FunctionCustomWeekToSomething.h b/src/Functions/FunctionCustomWeekToSomething.h index fa142dd7a13..19a0fdf3767 100644 --- a/src/Functions/FunctionCustomWeekToSomething.h +++ b/src/Functions/FunctionCustomWeekToSomething.h @@ -96,7 +96,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { const IDataType * from_type = block[arguments[0]].type.get(); WhichDataType which(from_type); diff --git a/src/Functions/FunctionDateOrDateTimeAddInterval.h b/src/Functions/FunctionDateOrDateTimeAddInterval.h index 92a8858bfeb..806ea199b5b 100644 --- a/src/Functions/FunctionDateOrDateTimeAddInterval.h +++ b/src/Functions/FunctionDateOrDateTimeAddInterval.h @@ -463,7 +463,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {2}; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const IDataType * from_type = block[arguments[0]].type.get(); WhichDataType which(from_type); diff --git a/src/Functions/FunctionDateOrDateTimeToSomething.h b/src/Functions/FunctionDateOrDateTimeToSomething.h index cf1a82ea194..eee98a11123 100644 --- a/src/Functions/FunctionDateOrDateTimeToSomething.h +++ b/src/Functions/FunctionDateOrDateTimeToSomething.h @@ -95,7 +95,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { const IDataType * from_type = block[arguments[0]].type.get(); WhichDataType which(from_type); diff --git a/src/Functions/FunctionFQDN.cpp b/src/Functions/FunctionFQDN.cpp index 8daf3545788..ecc0d417711 100644 --- a/src/Functions/FunctionFQDN.cpp +++ b/src/Functions/FunctionFQDN.cpp @@ -34,7 +34,7 @@ public: return std::make_shared(); } - void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers &, size_t result, size_t input_rows_count) const override { block[result].column = block[result].type->createColumnConst( input_rows_count, getFQDNOrHostName())->convertToFullColumnIfConst(); diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index e44b43c4b3f..ba05d52ad33 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -17,7 +17,7 @@ namespace ErrorCodes } template -void ExecutableFunctionJoinGet::execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t) +void ExecutableFunctionJoinGet::execute(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t) { ColumnsWithTypeAndName keys; for (size_t i = 2; i < arguments.size(); ++i) @@ -29,7 +29,7 @@ void ExecutableFunctionJoinGet::execute(Block & block, const ColumnNumb } template -ExecutableFunctionImplPtr FunctionJoinGet::prepare(const Block &, const ColumnNumbers &, size_t) const +ExecutableFunctionImplPtr FunctionJoinGet::prepare(const ColumnsWithTypeAndName &, const ColumnNumbers &, size_t) const { return std::make_unique>(join, DB::Block{{return_type->createColumn(), return_type, attr_name}}); } diff --git a/src/Functions/FunctionJoinGet.h b/src/Functions/FunctionJoinGet.h index 482d9ae0092..1e8e51b3bfd 100644 --- a/src/Functions/FunctionJoinGet.h +++ b/src/Functions/FunctionJoinGet.h @@ -24,7 +24,7 @@ public: bool useDefaultImplementationForLowCardinalityColumns() const override { return true; } bool useDefaultImplementationForConstants() const override { return true; } - void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override; + void execute(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override; String getName() const override { return name; } @@ -56,7 +56,7 @@ public: const DataTypes & getArgumentTypes() const override { return argument_types; } const DataTypePtr & getReturnType() const override { return return_type; } - ExecutableFunctionImplPtr prepare(const Block & sample_block, const ColumnNumbers & arguments, size_t result) const override; + ExecutableFunctionImplPtr prepare(const ColumnsWithTypeAndName & sample_block, const ColumnNumbers & arguments, size_t result) const override; private: TableLockHolder table_lock; diff --git a/src/Functions/FunctionMathBinaryFloat64.h b/src/Functions/FunctionMathBinaryFloat64.h index c02bbba94c7..ee061075cbf 100644 --- a/src/Functions/FunctionMathBinaryFloat64.h +++ b/src/Functions/FunctionMathBinaryFloat64.h @@ -54,7 +54,7 @@ private: } template - bool executeTyped(Block & block, const size_t result, const ColumnConst * left_arg, const IColumn * right_arg) const + bool executeTyped(ColumnsWithTypeAndName & block, const size_t result, const ColumnConst * left_arg, const IColumn * right_arg) const { if (const auto right_arg_typed = checkAndGetColumn>(right_arg)) { @@ -103,7 +103,7 @@ private: } template - bool executeTyped(Block & block, const size_t result, const ColumnVector * left_arg, const IColumn * right_arg) const + bool executeTyped(ColumnsWithTypeAndName & block, const size_t result, const ColumnVector * left_arg, const IColumn * right_arg) const { if (const auto right_arg_typed = checkAndGetColumn>(right_arg)) { @@ -207,7 +207,7 @@ private: return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const ColumnWithTypeAndName & col_left = block[arguments[0]]; const ColumnWithTypeAndName & col_right = block[arguments[1]]; diff --git a/src/Functions/FunctionMathConstFloat64.h b/src/Functions/FunctionMathConstFloat64.h index 838b85babcf..ea776c159b2 100644 --- a/src/Functions/FunctionMathConstFloat64.h +++ b/src/Functions/FunctionMathConstFloat64.h @@ -25,7 +25,7 @@ private: return std::make_shared(); } - void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers &, size_t result, size_t input_rows_count) const override { block[result].column = block[result].type->createColumnConst(input_rows_count, Impl::value); } diff --git a/src/Functions/FunctionMathUnary.h b/src/Functions/FunctionMathUnary.h index 26c8c975d91..8ced676342c 100644 --- a/src/Functions/FunctionMathUnary.h +++ b/src/Functions/FunctionMathUnary.h @@ -113,7 +113,7 @@ private: } template - static bool execute(Block & block, const ColumnVector * col, const size_t result) + static bool execute(ColumnsWithTypeAndName & block, const ColumnVector * col, const size_t result) { const auto & src_data = col->getData(); const size_t size = src_data.size(); @@ -129,7 +129,7 @@ private: } template - static bool execute(Block & block, const ColumnDecimal * col, const size_t result) + static bool execute(ColumnsWithTypeAndName & block, const ColumnDecimal * col, const size_t result) { const auto & src_data = col->getData(); const size_t size = src_data.size(); @@ -150,7 +150,7 @@ private: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const ColumnWithTypeAndName & col = block[arguments[0]]; diff --git a/src/Functions/FunctionNumericPredicate.h b/src/Functions/FunctionNumericPredicate.h index ad07da84271..6065a4dffb0 100644 --- a/src/Functions/FunctionNumericPredicate.h +++ b/src/Functions/FunctionNumericPredicate.h @@ -46,7 +46,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const auto in = block[arguments.front()].column.get(); @@ -64,7 +64,7 @@ public: } template - bool execute(Block & block, const IColumn * in_untyped, const size_t result) const + bool execute(ColumnsWithTypeAndName & block, const IColumn * in_untyped, const size_t result) const { if (const auto in = checkAndGetColumn>(in_untyped)) { diff --git a/src/Functions/FunctionStartsEndsWith.h b/src/Functions/FunctionStartsEndsWith.h index 35e53613f58..7adaca8bc8f 100644 --- a/src/Functions/FunctionStartsEndsWith.h +++ b/src/Functions/FunctionStartsEndsWith.h @@ -63,7 +63,7 @@ public: return std::make_shared(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { const IColumn * haystack_column = block[arguments[0]].column.get(); const IColumn * needle_column = block[arguments[1]].column.get(); diff --git a/src/Functions/FunctionStringOrArrayToT.h b/src/Functions/FunctionStringOrArrayToT.h index c0270aa6a23..fb159eb6642 100644 --- a/src/Functions/FunctionStringOrArrayToT.h +++ b/src/Functions/FunctionStringOrArrayToT.h @@ -50,7 +50,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const ColumnPtr column = block[arguments[0]].column; if (const ColumnString * col = checkAndGetColumn(column.get())) diff --git a/src/Functions/FunctionStringReplace.h b/src/Functions/FunctionStringReplace.h index a4b654fc355..ca7b610f34b 100644 --- a/src/Functions/FunctionStringReplace.h +++ b/src/Functions/FunctionStringReplace.h @@ -52,7 +52,7 @@ public: return std::make_shared(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const ColumnPtr column_src = block[arguments[0]].column; const ColumnPtr column_needle = block[arguments[1]].column; diff --git a/src/Functions/FunctionStringToString.h b/src/Functions/FunctionStringToString.h index b4bfe219a07..6a10842d76f 100644 --- a/src/Functions/FunctionStringToString.h +++ b/src/Functions/FunctionStringToString.h @@ -52,7 +52,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const ColumnPtr column = block[arguments[0]].column; if (const ColumnString * col = checkAndGetColumn(column.get())) diff --git a/src/Functions/FunctionUnaryArithmetic.h b/src/Functions/FunctionUnaryArithmetic.h index 1706f6ab645..412e2a14b29 100644 --- a/src/Functions/FunctionUnaryArithmetic.h +++ b/src/Functions/FunctionUnaryArithmetic.h @@ -154,7 +154,7 @@ public: return result; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { bool valid = castType(block[arguments[0]].type.get(), [&](const auto & type) { diff --git a/src/Functions/FunctionUnixTimestamp64.h b/src/Functions/FunctionUnixTimestamp64.h index cf66ad5fcb0..5b5262bd6d0 100644 --- a/src/Functions/FunctionUnixTimestamp64.h +++ b/src/Functions/FunctionUnixTimestamp64.h @@ -65,7 +65,7 @@ public: } } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { using SourceColumnType = typename SourceDataType::ColumnType; using ResultColumnType = typename ResultDataType::ColumnType; diff --git a/src/Functions/FunctionsBitmap.h b/src/Functions/FunctionsBitmap.h index d98eca7c05f..b4a8d1b7a5e 100644 --- a/src/Functions/FunctionsBitmap.h +++ b/src/Functions/FunctionsBitmap.h @@ -122,7 +122,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /* input_rows_count */) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /* input_rows_count */) const override { const IDataType * from_type = block[arguments[0]].type.get(); auto array_type = typeid_cast(from_type); @@ -146,7 +146,7 @@ public: private: template - void executeBitmapData(Block & block, DataTypes & argument_types, const ColumnNumbers & arguments, size_t result) const + void executeBitmapData(ColumnsWithTypeAndName & block, DataTypes & argument_types, const ColumnNumbers & arguments, size_t result) const { // input data const ColumnArray * array = typeid_cast(block[arguments[0]].column.get()); @@ -207,7 +207,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { // input data const auto & return_type = block[result].type; @@ -240,7 +240,7 @@ private: template void executeIntType( - Block & block, const ColumnNumbers & arguments, size_t input_rows_count, IColumn & res_data_col, ColumnArray::Offsets & res_offsets) + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t input_rows_count, IColumn & res_data_col, ColumnArray::Offsets & res_offsets) const { const ColumnAggregateFunction * column @@ -299,7 +299,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { const IDataType * from_type = block[arguments[0]].type.get(); const DataTypeAggregateFunction * aggr_type = typeid_cast(from_type); @@ -322,7 +322,7 @@ private: template void executeIntType( - Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const { const IColumn * columns[3]; @@ -435,7 +435,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { const IDataType * from_type = block[arguments[0]].type.get(); const DataTypeAggregateFunction * aggr_type = typeid_cast(from_type); @@ -458,7 +458,7 @@ private: template void executeIntType( - Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const { const IColumn * columns[3]; bool is_column_const[3]; @@ -568,7 +568,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { auto col_to = ColumnVector::create(input_rows_count); typename ColumnVector::Container & vec_to = col_to->getData(); @@ -596,7 +596,7 @@ private: template void executeIntType( - Block & block, const ColumnNumbers & arguments, size_t input_rows_count, typename ColumnVector::Container & vec_to) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t input_rows_count, typename ColumnVector::Container & vec_to) const { const ColumnAggregateFunction * column = typeid_cast(block[arguments[0]].column.get()); @@ -738,7 +738,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { auto col_to = ColumnVector::create(input_rows_count); typename ColumnVector::Container & vec_to = col_to->getData(); @@ -764,7 +764,7 @@ public: private: template void executeIntType( - Block & block, const ColumnNumbers & arguments, size_t input_rows_count, typename ColumnVector::Container & vec_to) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t input_rows_count, typename ColumnVector::Container & vec_to) const { const IColumn * columns[2]; bool is_column_const[2]; @@ -835,7 +835,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { auto col_to = ColumnVector::create(input_rows_count); typename ColumnVector::Container & vec_to = col_to->getData(); @@ -861,7 +861,7 @@ public: private: template void executeIntType( - Block & block, const ColumnNumbers & arguments, size_t input_rows_count, typename ColumnVector::Container & vec_to) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t input_rows_count, typename ColumnVector::Container & vec_to) const { const ColumnAggregateFunction * columns[2]; bool is_column_const[2]; @@ -970,7 +970,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { const IDataType * from_type = block[arguments[0]].type.get(); const DataTypeAggregateFunction * aggr_type = typeid_cast(from_type); @@ -990,7 +990,7 @@ public: private: template - void executeBitmapData(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const + void executeBitmapData(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const { const ColumnAggregateFunction * columns[2]; bool is_column_const[2]; diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index c854f079775..d01060b9e30 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -88,7 +88,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const auto & col_type_name = block[arguments[0]]; const ColumnPtr & column = col_type_name.column; @@ -168,7 +168,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const auto & col_type_name = block[arguments[0]]; const ColumnPtr & column = col_type_name.column; @@ -277,7 +277,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const ColumnPtr & column = block[arguments[0]].column; @@ -339,7 +339,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const ColumnPtr & column = block[arguments[0]].column; @@ -407,7 +407,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const ColumnPtr & column = block[arguments[0]].column; @@ -460,7 +460,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const auto & col_type_name = block[arguments[0]]; const ColumnPtr & column = col_type_name.column; @@ -578,7 +578,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const ColumnPtr & column = block[arguments[0]].column; @@ -688,7 +688,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const ColumnPtr & column = block[arguments[0]].column; @@ -755,7 +755,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const ColumnWithTypeAndName & col_type_name = block[arguments[0]]; const ColumnPtr & column = col_type_name.column; @@ -857,7 +857,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const ColumnWithTypeAndName & col_type_name = block[arguments[0]]; const ColumnPtr & column = col_type_name.column; @@ -1187,7 +1187,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const IColumn * column = block[arguments[0]].column.get(); ColumnPtr & res_column = block[result].column; @@ -1255,7 +1255,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const ColumnPtr & column = block[arguments[0]].column; @@ -1335,7 +1335,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { auto col_str = ColumnString::create(); ColumnString::Chars & out_vec = col_str->getChars(); @@ -1461,7 +1461,7 @@ public: } } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const IColumn * in_column = block[arguments[0]].column.get(); ColumnPtr & out_column = block[result].column; @@ -1599,7 +1599,7 @@ public: } } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const IColumn * column = block[arguments[0]].column.get(); ColumnPtr & res_column = block[result].column; @@ -1668,7 +1668,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { const auto & col_type_name_ip = block[arguments[0]]; const ColumnPtr & column_ip = col_type_name_ip.column; @@ -1772,7 +1772,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { const auto & col_type_name_ip = block[arguments[0]]; const ColumnPtr & column_ip = col_type_name_ip.column; diff --git a/src/Functions/FunctionsComparison.h b/src/Functions/FunctionsComparison.h index 78ef9f28b67..cf8c2479d71 100644 --- a/src/Functions/FunctionsComparison.h +++ b/src/Functions/FunctionsComparison.h @@ -571,7 +571,7 @@ private: bool check_decimal_overflow = true; template - bool executeNumRightType(Block & block, size_t result, const ColumnVector * col_left, const IColumn * col_right_untyped) const + bool executeNumRightType(ColumnsWithTypeAndName & block, size_t result, const ColumnVector * col_left, const IColumn * col_right_untyped) const { if (const ColumnVector * col_right = checkAndGetColumn>(col_right_untyped)) { @@ -600,7 +600,7 @@ private: } template - bool executeNumConstRightType(Block & block, size_t result, const ColumnConst * col_left, const IColumn * col_right_untyped) const + bool executeNumConstRightType(ColumnsWithTypeAndName & block, size_t result, const ColumnConst * col_left, const IColumn * col_right_untyped) const { if (const ColumnVector * col_right = checkAndGetColumn>(col_right_untyped)) { @@ -626,7 +626,7 @@ private: } template - bool executeNumLeftType(Block & block, size_t result, const IColumn * col_left_untyped, const IColumn * col_right_untyped) const + bool executeNumLeftType(ColumnsWithTypeAndName & block, size_t result, const IColumn * col_left_untyped, const IColumn * col_right_untyped) const { if (const ColumnVector * col_left = checkAndGetColumn>(col_left_untyped)) { @@ -676,7 +676,7 @@ private: return false; } - void executeDecimal(Block & block, size_t result, const ColumnWithTypeAndName & col_left, const ColumnWithTypeAndName & col_right) const + void executeDecimal(ColumnsWithTypeAndName & block, size_t result, const ColumnWithTypeAndName & col_left, const ColumnWithTypeAndName & col_right) const { TypeIndex left_number = col_left.type->getTypeId(); TypeIndex right_number = col_right.type->getTypeId(); @@ -699,7 +699,7 @@ private: ErrorCodes::LOGICAL_ERROR); } - bool executeString(Block & block, size_t result, const IColumn * c0, const IColumn * c1) const + bool executeString(ColumnsWithTypeAndName & block, size_t result, const IColumn * c0, const IColumn * c1) const { const ColumnString * c0_string = checkAndGetColumn(c0); const ColumnString * c1_string = checkAndGetColumn(c1); @@ -824,8 +824,8 @@ private: } bool executeWithConstString( - Block & block, size_t result, const IColumn * col_left_untyped, const IColumn * col_right_untyped, - const DataTypePtr & left_type, const DataTypePtr & right_type, size_t input_rows_count) const + ColumnsWithTypeAndName & block, size_t result, const IColumn * col_left_untyped, const IColumn * col_right_untyped, + const DataTypePtr & left_type, const DataTypePtr & right_type, size_t input_rows_count) const { /// To compare something with const string, we cast constant to appropriate type and compare as usual. /// It is ok to throw exception if value is not convertible. @@ -867,8 +867,8 @@ private: return true; } - void executeTuple(Block & block, size_t result, const ColumnWithTypeAndName & c0, const ColumnWithTypeAndName & c1, - size_t input_rows_count) const + void executeTuple(ColumnsWithTypeAndName & block, size_t result, const ColumnWithTypeAndName & c0, const ColumnWithTypeAndName & c1, + size_t input_rows_count) const { /** We will lexicographically compare the tuples. This is done as follows: * x == y : x1 == y1 && x2 == y2 ... @@ -930,19 +930,19 @@ private: executeTupleImpl(block, result, x, y, tuple_size, input_rows_count); } - void executeTupleImpl(Block & block, size_t result, const ColumnsWithTypeAndName & x, - const ColumnsWithTypeAndName & y, size_t tuple_size, - size_t input_rows_count) const; + void executeTupleImpl(ColumnsWithTypeAndName & block, size_t result, const ColumnsWithTypeAndName & x, + const ColumnsWithTypeAndName & y, size_t tuple_size, + size_t input_rows_count) const; void executeTupleEqualityImpl( - std::shared_ptr func_compare, - std::shared_ptr func_convolution, - Block & block, - size_t result, - const ColumnsWithTypeAndName & x, - const ColumnsWithTypeAndName & y, - size_t tuple_size, - size_t input_rows_count) const + std::shared_ptr func_compare, + std::shared_ptr func_convolution, + ColumnsWithTypeAndName & block, + size_t result, + const ColumnsWithTypeAndName & x, + const ColumnsWithTypeAndName & y, + size_t tuple_size, + size_t input_rows_count) const { if (0 == tuple_size) throw Exception("Comparison of zero-sized tuples is not implemented.", ErrorCodes::NOT_IMPLEMENTED); @@ -984,17 +984,17 @@ private: } void executeTupleLessGreaterImpl( - std::shared_ptr func_compare_head, - std::shared_ptr func_compare_tail, - std::shared_ptr func_and, - std::shared_ptr func_or, - std::shared_ptr func_equals, - Block & block, - size_t result, - const ColumnsWithTypeAndName & x, - const ColumnsWithTypeAndName & y, - size_t tuple_size, - size_t input_rows_count) const + std::shared_ptr func_compare_head, + std::shared_ptr func_compare_tail, + std::shared_ptr func_and, + std::shared_ptr func_or, + std::shared_ptr func_equals, + ColumnsWithTypeAndName & block, + size_t result, + const ColumnsWithTypeAndName & x, + const ColumnsWithTypeAndName & y, + size_t tuple_size, + size_t input_rows_count) const { ColumnsWithTypeAndName tmp_block; @@ -1066,7 +1066,7 @@ private: block[result].column = tmp_block[tmp_block.size() - 1].column; } - void executeGenericIdenticalTypes(Block & block, size_t result, const IColumn * c0, const IColumn * c1) const + void executeGenericIdenticalTypes(ColumnsWithTypeAndName & block, size_t result, const IColumn * c0, const IColumn * c1) const { bool c0_const = isColumnConst(*c0); bool c1_const = isColumnConst(*c1); @@ -1094,7 +1094,7 @@ private: } } - void executeGeneric(Block & block, size_t result, const ColumnWithTypeAndName & c0, const ColumnWithTypeAndName & c1) const + void executeGeneric(ColumnsWithTypeAndName & block, size_t result, const ColumnWithTypeAndName & c0, const ColumnWithTypeAndName & c1) const { DataTypePtr common_type = getLeastSupertype({c0.type, c1.type}); @@ -1173,7 +1173,7 @@ public: return std::make_shared(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { const auto & col_with_type_and_name_left = block[arguments[0]]; const auto & col_with_type_and_name_right = block[arguments[1]]; diff --git a/src/Functions/FunctionsConsistentHashing.h b/src/Functions/FunctionsConsistentHashing.h index d39c72e364e..484110989fa 100644 --- a/src/Functions/FunctionsConsistentHashing.h +++ b/src/Functions/FunctionsConsistentHashing.h @@ -65,7 +65,7 @@ public: return {1}; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { if (isColumnConst(*block[arguments[1]].column)) executeConstBuckets(block, arguments, result); @@ -93,7 +93,7 @@ private: return static_cast(buckets); } - void executeConstBuckets(Block & block, const ColumnNumbers & arguments, size_t result) const + void executeConstBuckets(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result) const { Field buckets_field = (*block[arguments[1]].column)[0]; BucketsType num_buckets; diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index cfb8da70ebd..417d2da427d 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -1132,7 +1132,7 @@ public: ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } bool canBeExecutedOnDefaultArguments() const override { return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { try { @@ -1177,7 +1177,7 @@ public: } private: - void executeInternal(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const + void executeInternal(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const { if (!arguments.size()) throw Exception{"Function " + getName() + " expects at least 1 arguments", @@ -1394,7 +1394,7 @@ public: } template - bool executeInternal(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count, UInt32 scale = 0) const + bool executeInternal(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count, UInt32 scale = 0) const { const IDataType * from_type = block[arguments[0]].type.get(); @@ -1414,7 +1414,7 @@ public: return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { bool ok = true; @@ -1856,7 +1856,7 @@ using FunctionParseDateTime64BestEffortOrNull = FunctionConvertFromString< class ExecutableFunctionCast : public IExecutableFunctionImpl { public: - using WrapperType = std::function; + using WrapperType = std::function; explicit ExecutableFunctionCast(WrapperType && wrapper_function_, const char * name_) : wrapper_function(std::move(wrapper_function_)), name(name_) {} @@ -1864,7 +1864,7 @@ public: String getName() const override { return name; } protected: - void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + void execute(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { /// drop second argument, pass others ColumnNumbers new_arguments{arguments.front()}; @@ -1890,7 +1890,7 @@ struct NameCast { static constexpr auto name = "CAST"; }; class FunctionCast final : public IFunctionBaseImpl { public: - using WrapperType = std::function; + using WrapperType = std::function; using MonotonicityForRange = std::function; FunctionCast(const char * name_, MonotonicityForRange && monotonicity_for_range_ @@ -1903,7 +1903,7 @@ public: const DataTypes & getArgumentTypes() const override { return argument_types; } const DataTypePtr & getReturnType() const override { return return_type; } - ExecutableFunctionImplPtr prepare(const Block & /*sample_block*/, const ColumnNumbers & /*arguments*/, size_t /*result*/) const override + ExecutableFunctionImplPtr prepare(const ColumnsWithTypeAndName & /*sample_block*/, const ColumnNumbers & /*arguments*/, size_t /*result*/) const override { return std::make_unique( prepareUnpackDictionaries(getArgumentTypes()[0], getReturnType()), name); @@ -1950,7 +1950,7 @@ private: FunctionOverloadResolverAdaptor(std::make_unique(function)) .build({ColumnWithTypeAndName{nullptr, from_type, ""}}); - return [function_adaptor] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) + return [function_adaptor] (ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { function_adaptor->execute(block, arguments, result, input_rows_count); }; @@ -1964,7 +1964,7 @@ private: FunctionOverloadResolverAdaptor(std::make_unique(function)) .build({ColumnWithTypeAndName{nullptr, from_type, ""}}); - return [function_adaptor] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) + return [function_adaptor] (ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { function_adaptor->execute(block, arguments, result, input_rows_count); }; @@ -1975,7 +1975,7 @@ private: if (!isStringOrFixedString(from_type)) throw Exception{"CAST AS FixedString is only implemented for types String and FixedString", ErrorCodes::NOT_IMPLEMENTED}; - return [N] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t /*input_rows_count*/) + return [N] (ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t /*input_rows_count*/) { FunctionToFixedString::executeForN(block, arguments, result, N); }; @@ -1992,7 +1992,7 @@ private: FunctionOverloadResolverAdaptor(std::make_unique(function)) .build({ColumnWithTypeAndName{nullptr, from_type, ""}}); - return [function_adaptor] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) + return [function_adaptor] (ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { function_adaptor->execute(block, arguments, result, input_rows_count); }; @@ -2016,7 +2016,7 @@ private: throw Exception{"Conversion from " + from_type->getName() + " to " + to_type->getName() + " is not supported", ErrorCodes::CANNOT_CONVERT_TYPE}; - return [type_index, scale, to_type] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) + return [type_index, scale, to_type] (ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { auto res = callOnIndexAndDataType(type_index, [&](const auto & types) -> bool { @@ -2042,7 +2042,7 @@ private: /// Conversion from String through parsing. if (checkAndGetDataType(from_type_untyped.get())) { - return [] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t /*input_rows_count*/) + return [] (ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t /*input_rows_count*/) { ConvertImplGenericFromString::execute(block, arguments, result); }; @@ -2057,7 +2057,7 @@ private: /// Conversion from String through parsing. if (checkAndGetDataType(from_type_untyped.get())) { - return [] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t /*input_rows_count*/) + return [] (ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t /*input_rows_count*/) { ConvertImplGenericFromString::execute(block, arguments, result); }; @@ -2086,7 +2086,7 @@ private: const auto nested_function = prepareUnpackDictionaries(from_nested_type, to_nested_type); return [nested_function, from_nested_type, to_nested_type]( - Block & block, const ColumnNumbers & arguments, const size_t result, size_t /*input_rows_count*/) + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t /*input_rows_count*/) { const auto & array_arg = block[arguments.front()]; @@ -2098,7 +2098,7 @@ private: { col_array->getDataPtr(), from_nested_type, "" }, { nullptr, to_nested_type, "" } }; - Block nested_block(nested_block_columns); + ColumnsWithTypeAndName nested_block(nested_block_columns); /// convert nested column nested_function(nested_block, {0}, 1, nested_block_columns.front().column->size()); @@ -2116,7 +2116,7 @@ private: /// Conversion from String through parsing. if (checkAndGetDataType(from_type_untyped.get())) { - return [] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t /*input_rows_count*/) + return [] (ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t /*input_rows_count*/) { ConvertImplGenericFromString::execute(block, arguments, result); }; @@ -2141,7 +2141,7 @@ private: element_wrappers.push_back(prepareUnpackDictionaries(idx_type.second, to_element_types[idx_type.first])); return [element_wrappers, from_element_types, to_element_types] - (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) + (ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { const auto col = block[arguments.front()].column.get(); @@ -2196,7 +2196,7 @@ private: auto func_or_adaptor = FunctionOverloadResolverAdaptor(std::make_unique(function)) .build(ColumnsWithTypeAndName{{nullptr, from_type, "" }}); - return [func_or_adaptor] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) + return [func_or_adaptor] (ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { func_or_adaptor->execute(block, arguments, result, input_rows_count); }; @@ -2235,7 +2235,7 @@ private: WrapperType createStringToEnumWrapper(ssize_t source_is_nullable) const { const char * function_name = name; - return [function_name, source_is_nullable] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t /*input_rows_count*/) + return [function_name, source_is_nullable] (ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t /*input_rows_count*/) { const auto first_col = block[arguments.front()].column.get(); @@ -2287,7 +2287,7 @@ private: WrapperType createIdentityWrapper(const DataTypePtr &) const { - return [] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t /*input_rows_count*/) + return [] (ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t /*input_rows_count*/) { block[result].column = block[arguments.front()].column; }; @@ -2296,7 +2296,7 @@ private: WrapperType createNothingWrapper(const IDataType * to_type) const { ColumnPtr res = to_type->createColumnConstWithDefaultValue(1); - return [res] (Block & block, const ColumnNumbers &, const size_t result, size_t input_rows_count) + return [res] (ColumnsWithTypeAndName & block, const ColumnNumbers &, const size_t result, size_t input_rows_count) { /// Column of Nothing type is trivially convertible to any other column block[result].column = res->cloneResized(input_rows_count)->convertToFullColumnIfConst(); @@ -2315,7 +2315,7 @@ private: if (!to_nested->isNullable()) throw Exception{"Cannot convert NULL to a non-nullable type", ErrorCodes::CANNOT_CONVERT_TYPE}; - return [](Block & block, const ColumnNumbers &, const size_t result, size_t input_rows_count) + return [](ColumnsWithTypeAndName & block, const ColumnNumbers &, const size_t result, size_t input_rows_count) { auto & res = block[result]; res.column = res.type->createColumnConstWithDefaultValue(input_rows_count)->convertToFullColumnIfConst(); @@ -2333,7 +2333,7 @@ private: return wrapper; return [wrapper, from_low_cardinality, to_low_cardinality, skip_not_null_check] - (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) + (ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { auto & arg = block[arguments[0]]; auto & res = block[result]; @@ -2415,7 +2415,7 @@ private: if (result_is_nullable) { return [wrapper, source_is_nullable] - (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) + (ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { /// Create a temporary block on which to perform the operation. auto & res = block[result]; @@ -2456,7 +2456,7 @@ private: { /// Conversion from Nullable to non-Nullable. - return [wrapper, skip_not_null_check] (Block & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) + return [wrapper, skip_not_null_check] (ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, size_t input_rows_count) { auto tmp_block_columns = createBlockWithNestedColumns(block, arguments, result); diff --git a/src/Functions/FunctionsEmbeddedDictionaries.h b/src/Functions/FunctionsEmbeddedDictionaries.h index f47cb1c2e0b..ed0b462683e 100644 --- a/src/Functions/FunctionsEmbeddedDictionaries.h +++ b/src/Functions/FunctionsEmbeddedDictionaries.h @@ -183,7 +183,7 @@ public: bool isDeterministic() const override { return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { /// The dictionary key that defines the "point of view". std::string dict_key; @@ -279,7 +279,7 @@ public: bool isDeterministic() const override { return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { /// The dictionary key that defines the "point of view". std::string dict_key; @@ -415,7 +415,7 @@ public: bool isDeterministic() const override { return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { /// The dictionary key that defines the "point of view". std::string dict_key; @@ -620,7 +620,7 @@ public: bool isDeterministic() const override { return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { RegionsNames::Language language = RegionsNames::Language::ru; diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index ad692ed9ebb..3e5e2c1f1e6 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -161,7 +161,7 @@ private: bool isDeterministic() const override { return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { /** Do not require existence of the dictionary if the function is called for empty block. * This is needed to allow successful query analysis on a server, @@ -203,7 +203,7 @@ private: template bool executeDispatchSimple( - Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const { const auto dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -226,7 +226,7 @@ private: template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const { const auto dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -326,7 +326,7 @@ private: bool isDeterministic() const override { return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { if (input_rows_count == 0) { @@ -362,7 +362,7 @@ private: template bool executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const { const auto dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -393,7 +393,7 @@ private: template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const { const auto dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -430,7 +430,7 @@ private: template bool executeDispatchRange( - Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const { const auto dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -512,7 +512,7 @@ private: bool isDeterministic() const override { return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { if (input_rows_count == 0) { @@ -547,7 +547,7 @@ private: template bool executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const { const auto dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -572,8 +572,8 @@ private: template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, - const std::string & attr_name, const ColumnUInt64 * id_col) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, + const std::string & attr_name, const ColumnUInt64 * id_col) const { const auto default_col_untyped = block[arguments[3]].column.get(); @@ -600,8 +600,8 @@ private: template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, - const std::string & attr_name, const ColumnConst * id_col) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, + const std::string & attr_name, const ColumnConst * id_col) const { const auto default_col_untyped = block[arguments[3]].column.get(); @@ -635,7 +635,7 @@ private: template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const { const auto dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -854,7 +854,7 @@ private: bool isDeterministic() const override { return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { if (input_rows_count == 0) { @@ -889,7 +889,7 @@ private: } template - bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const + bool executeDispatch(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const { const auto dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -945,7 +945,7 @@ private: template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const { const auto dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -988,7 +988,7 @@ private: template bool executeDispatchRange( - Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const { const auto dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -1117,7 +1117,7 @@ private: bool isDeterministic() const override { return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { if (input_rows_count == 0) { @@ -1151,7 +1151,7 @@ private: } template - bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const + bool executeDispatch(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const { const auto dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -1176,8 +1176,8 @@ private: template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, - const std::string & attr_name, const ColumnUInt64 * id_col) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, + const std::string & attr_name, const ColumnUInt64 * id_col) const { const auto default_col_untyped = block[arguments[3]].column.get(); @@ -1215,8 +1215,8 @@ private: template void executeDispatch( - Block & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, - const std::string & attr_name, const ColumnConst * id_col) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const DictionaryType * dict, + const std::string & attr_name, const ColumnConst * id_col) const { const auto default_col_untyped = block[arguments[3]].column.get(); @@ -1274,7 +1274,7 @@ private: template bool executeDispatchComplex( - Block & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const { const auto dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -1501,7 +1501,7 @@ private: bool isDeterministic() const override { return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { impl->executeImpl(block, arguments, result, input_rows_count); } @@ -1643,7 +1643,7 @@ private: bool isDeterministic() const override { return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { impl->executeImpl(block, arguments, result, input_rows_count); } @@ -1692,7 +1692,7 @@ private: bool isDeterministic() const override { return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { if (input_rows_count == 0) { @@ -1711,8 +1711,8 @@ private: } template - bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, - const std::shared_ptr & dict_ptr) const + bool executeDispatch(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, + const std::shared_ptr & dict_ptr) const { const auto dict = typeid_cast(dict_ptr.get()); if (!dict) @@ -1849,7 +1849,7 @@ private: bool isDeterministic() const override { return false; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { if (input_rows_count == 0) { @@ -1868,7 +1868,7 @@ private: } template - bool executeDispatch(Block & block, const ColumnNumbers & arguments, const size_t result, + bool executeDispatch(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result, const std::shared_ptr & dict_ptr) const { const auto dict = typeid_cast(dict_ptr.get()); @@ -1893,8 +1893,8 @@ private: } template - bool execute(Block & block, const size_t result, const DictionaryType * dict, - const ColumnUInt64 * child_id_col, const IColumn * ancestor_id_col_untyped) const + bool execute(ColumnsWithTypeAndName & block, const size_t result, const DictionaryType * dict, + const ColumnUInt64 * child_id_col, const IColumn * ancestor_id_col_untyped) const { if (const auto ancestor_id_col = checkAndGetColumn(ancestor_id_col_untyped)) { @@ -1932,8 +1932,8 @@ private: } template - bool execute(Block & block, const size_t result, const DictionaryType * dict, - const ColumnConst * child_id_col, const IColumn * ancestor_id_col_untyped) const + bool execute(ColumnsWithTypeAndName & block, const size_t result, const DictionaryType * dict, + const ColumnConst * child_id_col, const IColumn * ancestor_id_col_untyped) const { if (const auto ancestor_id_col = checkAndGetColumn(ancestor_id_col_untyped)) { diff --git a/src/Functions/FunctionsExternalModels.cpp b/src/Functions/FunctionsExternalModels.cpp index 9504ab97392..c80120958ee 100644 --- a/src/Functions/FunctionsExternalModels.cpp +++ b/src/Functions/FunctionsExternalModels.cpp @@ -69,7 +69,7 @@ DataTypePtr FunctionModelEvaluate::getReturnTypeImpl(const ColumnsWithTypeAndNam return type; } -void FunctionModelEvaluate::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const +void FunctionModelEvaluate::executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const { const auto * name_col = checkAndGetColumnConst(block[arguments[0]].column.get()); if (!name_col) diff --git a/src/Functions/FunctionsExternalModels.h b/src/Functions/FunctionsExternalModels.h index 5c26dd36d85..411642c99ab 100644 --- a/src/Functions/FunctionsExternalModels.h +++ b/src/Functions/FunctionsExternalModels.h @@ -32,7 +32,7 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override; - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override; + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override; private: const ExternalModelsLoader & models_loader; diff --git a/src/Functions/FunctionsFormatting.h b/src/Functions/FunctionsFormatting.h index 1ff93b7fe61..1f4a7512f88 100644 --- a/src/Functions/FunctionsFormatting.h +++ b/src/Functions/FunctionsFormatting.h @@ -57,7 +57,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { if (!(executeType(block, arguments, result) || executeType(block, arguments, result) @@ -93,7 +93,7 @@ private: } template - bool executeType(Block & block, const ColumnNumbers & arguments, size_t result) const + bool executeType(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result) const { if (const ColumnVector * col_from = checkAndGetColumn>(block[arguments[0]].column.get())) { @@ -153,7 +153,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { if (!(executeType(block, arguments, result) || executeType(block, arguments, result) @@ -172,7 +172,7 @@ public: private: template - bool executeType(Block & block, const ColumnNumbers & arguments, size_t result) const + bool executeType(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result) const { if (const ColumnVector * col_from = checkAndGetColumn>(block[arguments[0]].column.get())) { @@ -229,7 +229,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { if (!(executeType(block, arguments, result) || executeType(block, arguments, result) @@ -248,7 +248,7 @@ public: private: template - bool executeType(Block & block, const ColumnNumbers & arguments, size_t result) const + bool executeType(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result) const { if (const ColumnVector * col_from = checkAndGetColumn>(block[arguments[0]].column.get())) { diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h index a54bb44254d..1f06c902962 100644 --- a/src/Functions/FunctionsHashing.h +++ b/src/Functions/FunctionsHashing.h @@ -543,7 +543,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { if (const ColumnString * col_from = checkAndGetColumn(block[arguments[0]].column.get())) { @@ -604,7 +604,7 @@ private: using ToType = typename Impl::ReturnType; template - void executeType(Block & block, const ColumnNumbers & arguments, size_t result) const + void executeType(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result) const { using ColVecType = std::conditional_t, ColumnDecimal, ColumnVector>; @@ -647,7 +647,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const IDataType * from_type = block[arguments[0]].type.get(); WhichDataType which(from_type); @@ -1041,7 +1041,7 @@ public: return std::make_shared>(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override { size_t rows = input_rows_count; auto col_to = ColumnVector::create(rows); @@ -1209,7 +1209,7 @@ public: bool useDefaultImplementationForConstants() const override { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) const override { const auto arg_count = arguments.size(); @@ -1222,7 +1222,7 @@ public: } private: - void executeSingleArg(Block & block, const ColumnNumbers & arguments, const size_t result) const + void executeSingleArg(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result) const { const auto col_untyped = block[arguments.front()].column.get(); @@ -1252,7 +1252,7 @@ private: " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN}; } - void executeTwoArgs(Block & block, const ColumnNumbers & arguments, const size_t result) const + void executeTwoArgs(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, const size_t result) const { const auto level_col = block[arguments.back()].column.get(); if (!isColumnConst(*level_col)) diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index be7050e91eb..b6abf976a38 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -286,7 +286,7 @@ public: return Impl::getReturnType(Name::name, arguments); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result_pos, size_t input_rows_count) const override + void executeImpl(ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result_pos, size_t input_rows_count) const override { /// Choose JSONParser. #if USE_SIMDJSON diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index fee5f36ac53..54279fae5f1 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -512,7 +512,7 @@ DataTypePtr FunctionAnyArityLogical::getReturnTypeImpl(const DataTyp template void FunctionAnyArityLogical::executeImpl( - Block & block, const ColumnNumbers & arguments, size_t result_index, size_t input_rows_count) const + ColumnsWithTypeAndName & block, const ColumnNumbers & arguments, size_t result_index, size_t input_rows_count) const { ColumnRawPtrs args_in; for (const auto arg_index : arguments) @@ -572,7 +572,7 @@ bool functionUnaryExecuteType(ColumnsWithTypeAndName & block, const ColumnNumber } template