From bb4ad9ed6bf3b7faf2ff01ded8eec9ff6ebb38d6 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Fri, 30 Dec 2022 14:48:58 +0800 Subject: [PATCH 01/79] improve count if rewrite --- .../RewriteSumIfFunctionVisitor.cpp | 50 +++++++++++++++---- .../RewriteSumIfFunctionVisitor.h | 3 ++ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/src/Interpreters/RewriteSumIfFunctionVisitor.cpp b/src/Interpreters/RewriteSumIfFunctionVisitor.cpp index 50e6bec28f3..88f77606414 100644 --- a/src/Interpreters/RewriteSumIfFunctionVisitor.cpp +++ b/src/Interpreters/RewriteSumIfFunctionVisitor.cpp @@ -33,18 +33,25 @@ void RewriteSumIfFunctionMatcher::visit(const ASTFunction & func, ASTPtr & ast, if (lower_name == "sumif") { - /// sumIf(1, cond) -> countIf(cond) const auto * literal = func_arguments[0]->as(); if (!literal || !DB::isInt64OrUInt64FieldType(literal->value.getType())) return; + std::shared_ptr new_func; if (func_arguments.size() == 2 && literal->value.get() == 1) { - auto new_func = makeASTFunction("countIf", func_arguments[1]); - new_func->setAlias(func.alias); - ast = std::move(new_func); - return; + /// sumIf(1, cond) -> countIf(cond) + new_func = makeASTFunction("countIf", func_arguments[1]); } + else + { + /// sumIf(123, cond) -> 123 * countIf(cond) + auto count_if_func = makeASTFunction("countIf", func_arguments[1]); + new_func = makeASTFunction("multiply", func_arguments[0], std::move(count_if_func)); + } + new_func->setAlias(func.alias); + ast = std::move(new_func); + return; } else { @@ -65,19 +72,40 @@ void RewriteSumIfFunctionMatcher::visit(const ASTFunction & func, ASTPtr & ast, auto first_value = first_literal->value.get(); auto second_value = second_literal->value.get(); - /// sum(if(cond, 1, 0)) -> countIf(cond) - if (first_value == 1 && second_value == 0) + + std::shared_ptr new_func; + if (second_value == 0) { - auto new_func = makeASTFunction("countIf", if_arguments[0]); + if (first_value == 1) + { + /// sum(if(cond, 1, 0)) -> countIf(cond) + new_func = makeASTFunction("countIf", if_arguments[0]); + } + else + { + /// sum(if(cond, 123, 0)) -> 123 * countIf(cond) + auto count_if_func = makeASTFunction("countIf", if_arguments[0]); + new_func = makeASTFunction("multiply", if_arguments[1], std::move(count_if_func)); + } new_func->setAlias(func.alias); ast = std::move(new_func); return; } - /// sum(if(cond, 0, 1)) -> countIf(not(cond)) - if (first_value == 0 && second_value == 1) + + if (first_value == 0) { auto not_func = makeASTFunction("not", if_arguments[0]); - auto new_func = makeASTFunction("countIf", not_func); + if (second_value == 1) + { + /// sum(if(cond, 0, 1)) -> countIf(not(cond)) + new_func = makeASTFunction("countIf", std::move(not_func)); + } + else + { + /// sum(if(cond, 0, 123)) -> 123 * countIf(not(cond)) + auto count_if_func = makeASTFunction("countIf", std::move(not_func)); + new_func = makeASTFunction("multiply", if_arguments[2], std::move(count_if_func)); + } new_func->setAlias(func.alias); ast = std::move(new_func); return; diff --git a/src/Interpreters/RewriteSumIfFunctionVisitor.h b/src/Interpreters/RewriteSumIfFunctionVisitor.h index 86aeef65377..b3280e0c7c6 100644 --- a/src/Interpreters/RewriteSumIfFunctionVisitor.h +++ b/src/Interpreters/RewriteSumIfFunctionVisitor.h @@ -12,8 +12,11 @@ class ASTFunction; /// Rewrite 'sum(if())' and 'sumIf' functions to counIf. /// sumIf(1, cond) -> countIf(1, cond) +/// sumIf(123, cond) -> 123 * countIf(1, cond) /// sum(if(cond, 1, 0)) -> countIf(cond) +/// sum(if(cond, 123, 0)) -> 123 * countIf(cond) /// sum(if(cond, 0, 1)) -> countIf(not(cond)) +/// sum(if(cond, 0, 123)) -> 123 * countIf(not(cond)) class RewriteSumIfFunctionMatcher { public: From c3531455fcc2d76220743465643f258b5b02d44b Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Fri, 30 Dec 2022 15:43:54 +0800 Subject: [PATCH 02/79] commit again --- .../0_stateless/01646_rewrite_sum_if.reference | 12 ++++++++++++ tests/queries/0_stateless/01646_rewrite_sum_if.sql | 8 ++++++++ 2 files changed, 20 insertions(+) diff --git a/tests/queries/0_stateless/01646_rewrite_sum_if.reference b/tests/queries/0_stateless/01646_rewrite_sum_if.reference index 0f315b0812a..0e37c91578a 100644 --- a/tests/queries/0_stateless/01646_rewrite_sum_if.reference +++ b/tests/queries/0_stateless/01646_rewrite_sum_if.reference @@ -10,6 +10,12 @@ 50 50 50 50 1 0 50 +SELECT sumIf(123, (number % 2) = 0) +FROM numbers(100) +SELECT sum(if((number % 2) = 0, 123, 0)) +FROM numbers(100) +SELECT sum(if((number % 2) = 0, 0, 123)) +FROM numbers(100) 0 0 0 1 0 @@ -22,3 +28,9 @@ 50 50 50 50 1 0 50 +SELECT 123 * countIf((number % 2) = 0) +FROM numbers(100) +SELECT 123 * countIf((number % 2) = 0) +FROM numbers(100) +SELECT 123 * countIf(NOT ((number % 2) = 0)) +FROM numbers(100) diff --git a/tests/queries/0_stateless/01646_rewrite_sum_if.sql b/tests/queries/0_stateless/01646_rewrite_sum_if.sql index 07fb90c0eb7..9abe7f35005 100644 --- a/tests/queries/0_stateless/01646_rewrite_sum_if.sql +++ b/tests/queries/0_stateless/01646_rewrite_sum_if.sql @@ -16,6 +16,10 @@ SELECT sum(if(number % 2 == 0, 0, 1)) FROM numbers(100); SELECT sum(if(number % 2 == 0 as cond_expr, 0 as zero_expr, 1 as one_expr) as if_expr), sum(cond_expr), sum(if_expr), one_expr, zero_expr FROM numbers(100); SELECT countIf(number % 2 != 0) FROM numbers(100); +EXPLAIN SYNTAX SELECT sumIf(123, number % 2 == 0) FROM numbers(100); +EXPLAIN SYNTAX SELECT sum(if(number % 2 == 0, 123, 0)) FROM numbers(100); +EXPLAIN SYNTAX SELECT sum(if(number % 2 == 0, 0, 123)) FROM numbers(100); + SET optimize_rewrite_sum_if_to_count_if = 1; SELECT sumIf(1, number % 2 > 2) FROM numbers(100); @@ -33,3 +37,7 @@ SELECT countIf(number % 2 == 0) FROM numbers(100); SELECT sum(if(number % 2 == 0, 0, 1)) FROM numbers(100); SELECT sum(if(number % 2 == 0 as cond_expr, 0 as zero_expr, 1 as one_expr) as if_expr), sum(cond_expr), sum(if_expr), one_expr, zero_expr FROM numbers(100); SELECT countIf(number % 2 != 0) FROM numbers(100); + +EXPLAIN SYNTAX SELECT sumIf(123, number % 2 == 0) FROM numbers(100); +EXPLAIN SYNTAX SELECT sum(if(number % 2 == 0, 123, 0)) FROM numbers(100); +EXPLAIN SYNTAX SELECT sum(if(number % 2 == 0, 0, 123)) FROM numbers(100); From ba0edb310f297aeb02b4c56f30d80aabd2fdd0a0 Mon Sep 17 00:00:00 2001 From: Meena Renganathan Date: Thu, 5 Jan 2023 12:16:26 -0500 Subject: [PATCH 03/79] Changes to support the HDFS in PowerPC --- contrib/libhdfs3-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index 15d7a4df424..0a142908735 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -1,4 +1,4 @@ -if(NOT ARCH_AARCH64 AND NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE) +if(NOT ARCH_AARCH64 AND NOT OS_FREEBSD AND NOT APPLE) option(ENABLE_HDFS "Enable HDFS" ${ENABLE_LIBRARIES}) elseif(ENABLE_HDFS) message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use HDFS3 with current configuration") From 06e44827a37fe0cd791e27389a5e46900de58e07 Mon Sep 17 00:00:00 2001 From: Meena Renganathan Date: Thu, 5 Jan 2023 14:18:05 -0500 Subject: [PATCH 04/79] Added the file DataReader.cpp in source list when the HDFS submodule has been updated to the latest commit --- contrib/libhdfs3-cmake/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index 0a142908735..bd48772dd33 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -47,6 +47,7 @@ set(SRCS "${HDFS3_SOURCE_DIR}/network/TcpSocket.cpp" "${HDFS3_SOURCE_DIR}/network/DomainSocket.cpp" "${HDFS3_SOURCE_DIR}/network/BufferedSocketReader.cpp" + "${HDFS3_SOURCE_DIR}/client/DataReader.cpp" "${HDFS3_SOURCE_DIR}/client/ReadShortCircuitInfo.cpp" "${HDFS3_SOURCE_DIR}/client/Pipeline.cpp" "${HDFS3_SOURCE_DIR}/client/Hdfs.cpp" From 082e25b88e047b6d1c38527dbff35e21049f527c Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Thu, 12 Jan 2023 17:39:50 +0800 Subject: [PATCH 05/79] fix failed tests --- .../RewriteSumIfFunctionVisitor.cpp | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/Interpreters/RewriteSumIfFunctionVisitor.cpp b/src/Interpreters/RewriteSumIfFunctionVisitor.cpp index 88f77606414..c598fc1a6b7 100644 --- a/src/Interpreters/RewriteSumIfFunctionVisitor.cpp +++ b/src/Interpreters/RewriteSumIfFunctionVisitor.cpp @@ -37,21 +37,24 @@ void RewriteSumIfFunctionMatcher::visit(const ASTFunction & func, ASTPtr & ast, if (!literal || !DB::isInt64OrUInt64FieldType(literal->value.getType())) return; - std::shared_ptr new_func; - if (func_arguments.size() == 2 && literal->value.get() == 1) + if (func_arguments.size() == 2) { - /// sumIf(1, cond) -> countIf(cond) - new_func = makeASTFunction("countIf", func_arguments[1]); + std::shared_ptr new_func; + if (literal->value.get() == 1) + { + /// sumIf(1, cond) -> countIf(cond) + new_func = makeASTFunction("countIf", func_arguments[1]); + } + else + { + /// sumIf(123, cond) -> 123 * countIf(cond) + auto count_if_func = makeASTFunction("countIf", func_arguments[1]); + new_func = makeASTFunction("multiply", func_arguments[0], std::move(count_if_func)); + } + new_func->setAlias(func.alias); + ast = std::move(new_func); + return; } - else - { - /// sumIf(123, cond) -> 123 * countIf(cond) - auto count_if_func = makeASTFunction("countIf", func_arguments[1]); - new_func = makeASTFunction("multiply", func_arguments[0], std::move(count_if_func)); - } - new_func->setAlias(func.alias); - ast = std::move(new_func); - return; } else { From 4448bfe8a78bda2838aac6c9b914b1a7691e65e1 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Mon, 16 Jan 2023 15:08:12 +0800 Subject: [PATCH 06/79] try to improve performance of function multiif --- src/Core/Settings.h | 2 + src/Functions/multiIf.cpp | 105 +++++++++++++++++++++++++++++++------- 2 files changed, 89 insertions(+), 18 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b8d46244b6c..6f101164672 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -454,6 +454,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(Bool, allow_simdjson, true, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.", 0) \ M(Bool, allow_introspection_functions, false, "Allow functions for introspection of ELF and DWARF for query profiling. These functions are slow and may impose security considerations.", 0) \ \ + M(Bool, allow_execute_multiif_columnar, true, "Allow execute multiIf function columnar", 0) \ + \ M(UInt64, max_partitions_per_insert_block, 100, "Limit maximum number of partitions in single INSERTed block. Zero means unlimited. Throw exception if the block contains too many partitions. This setting is a safety threshold, because using large number of partitions is a common misconception.", 0) \ M(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited.", 0) \ M(Bool, check_query_single_value_result, true, "Return check query result as single 1/0 value", 0) \ diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index 37301037c0e..feb8abb1185 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -36,7 +37,9 @@ class FunctionMultiIf final : public FunctionIfBase { public: static constexpr auto name = "multiIf"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + + explicit FunctionMultiIf(ContextPtr context_) : context(context_) { } String getName() const override { return name; } bool isVariadic() const override { return true; } @@ -117,6 +120,21 @@ public: return getLeastSupertype(types_of_branches); } + struct Instruction + { + IColumn::Ptr condition = nullptr; + IColumn::Ptr source = nullptr; + + bool condition_always_true = false; + bool condition_is_nullable = false; + bool source_is_constant = false; + + bool condition_is_short = false; + bool source_is_short = false; + size_t condition_index = 0; + size_t source_index = 0; + }; + ColumnPtr executeImpl(const ColumnsWithTypeAndName & args, const DataTypePtr & result_type, size_t input_rows_count) const override { ColumnsWithTypeAndName arguments = args; @@ -124,26 +142,13 @@ public: /** We will gather values from columns in branches to result column, * depending on values of conditions. */ - struct Instruction - { - IColumn::Ptr condition = nullptr; - IColumn::Ptr source = nullptr; - bool condition_always_true = false; - bool condition_is_nullable = false; - bool source_is_constant = false; - - bool condition_is_short = false; - bool source_is_short = false; - size_t condition_index = 0; - size_t source_index = 0; - }; std::vector instructions; instructions.reserve(arguments.size() / 2 + 1); Columns converted_columns_holder; - converted_columns_holder.reserve(instructions.size()); + converted_columns_holder.reserve(instructions.capacity()); const DataTypePtr & return_type = result_type; @@ -225,6 +230,25 @@ public: size_t rows = input_rows_count; + /* + executeInstructionsColumnar(instructions, rows, res) || executeInstructionsColumnar(instructions, rows, res) || + executeInstructionsColumnar(instructions, rows, res) || executeInstructionsColumnar(instructions, rows, res) || + executeInstructionsColumnar(instructions, rows, res) || executeInstructionsColumnar(instructions, rows, res) || + executeInstructionsColumnar(instructions, rows, res) || executeInstructionsColumnar(instructions, rows, res) || + executeInstructionsColumnar(instructions, rows, res) || executeInstructionsColumnar(instructions, rows, res) || + executeInstructionsColumnar(instructions, rows, res) || executeInstructionsColumnar(instructions, rows, res) || + */ + const auto & settings = context->getSettingsRef(); + if (settings.allow_execute_multiif_columnar) + executeInstructionsColumnar(instructions, rows, res); + else + executeInstructions(instructions, rows, res); + return res; + } + +private: + static void executeInstructions(std::vector & instructions, size_t rows, const MutableColumnPtr & res) + { for (size_t i = 0; i < rows; ++i) { for (auto & instruction : instructions) @@ -257,11 +281,54 @@ public: } } } - - return res; } -private: + // template + static void executeInstructionsColumnar(std::vector & instructions, size_t rows, const MutableColumnPtr & res) + { + /// We should read source from which instruction on each row? + PaddedPODArray inserts(rows, instructions.size()); + // PaddedPODArray & res_data = assert_cast &>(*res).getData(); + for (Int64 i = instructions.size() - 1; i >= 0; --i) + { + auto & instruction = instructions[i]; + if (instruction.condition_always_true) + { + for (size_t row_i = 0; row_i < rows; ++row_i) + inserts[row_i] = i; + } + else if (!instruction.condition_is_nullable) + { + const auto & cond_data = assert_cast(*instruction.condition).getData(); + for (size_t row_i = 0; row_i < rows; ++row_i) + { + size_t condition_index = instruction.condition_is_short ? instruction.condition_index++ : row_i; + if (cond_data[condition_index]) + inserts[row_i] = i; + } + } + else + { + const ColumnNullable & condition_nullable = assert_cast(*instruction.condition); + const ColumnUInt8 & condition_nested = assert_cast(condition_nullable.getNestedColumn()); + const NullMap & condition_null_map = condition_nullable.getNullMapData(); + + for (size_t row_i = 0; row_i < rows; ++row_i) + { + size_t condition_index = instruction.condition_is_short ? instruction.condition_index++ : row_i; + if (!condition_null_map[condition_index] && condition_nested.getData()[condition_index]) + inserts[row_i] = i; + } + } + } + + for (size_t row_i = 0; row_i < rows; ++row_i) + { + auto & instruction = instructions[inserts[row_i]]; + (*res)[row_i] = (*instruction.source)[row_i]; + } + } + static void executeShortCircuitArguments(ColumnsWithTypeAndName & arguments) { int last_short_circuit_argument_index = checkShortCircuitArguments(arguments); @@ -330,6 +397,8 @@ private: for (; i <= last_short_circuit_argument_index; ++i) executeColumnIfNeeded(arguments[i], true); } + + ContextPtr context; }; } From 5b3f9eb62e1133560a0040cb6b562e846c2017c3 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Mon, 16 Jan 2023 20:35:26 +0800 Subject: [PATCH 07/79] vectorize function calculateInserts --- src/Functions/multiIf.cpp | 85 ++++++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 24 deletions(-) diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index feb8abb1185..8d3e10b480a 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -18,6 +18,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int LOGICAL_ERROR; } namespace @@ -217,37 +218,60 @@ public: break; } - MutableColumnPtr res = return_type->createColumn(); /// Special case if first instruction condition is always true and source is constant if (instructions.size() == 1 && instructions.front().source_is_constant && instructions.front().condition_always_true) { + MutableColumnPtr res = return_type->createColumn(); auto & instruction = instructions.front(); res->insertFrom(assert_cast(*instruction.source).getDataColumn(), 0); return ColumnConst::create(std::move(res), instruction.source->size()); } size_t rows = input_rows_count; - - /* - executeInstructionsColumnar(instructions, rows, res) || executeInstructionsColumnar(instructions, rows, res) || - executeInstructionsColumnar(instructions, rows, res) || executeInstructionsColumnar(instructions, rows, res) || - executeInstructionsColumnar(instructions, rows, res) || executeInstructionsColumnar(instructions, rows, res) || - executeInstructionsColumnar(instructions, rows, res) || executeInstructionsColumnar(instructions, rows, res) || - executeInstructionsColumnar(instructions, rows, res) || executeInstructionsColumnar(instructions, rows, res) || - executeInstructionsColumnar(instructions, rows, res) || executeInstructionsColumnar(instructions, rows, res) || - */ const auto & settings = context->getSettingsRef(); - if (settings.allow_execute_multiif_columnar) - executeInstructionsColumnar(instructions, rows, res); + const WhichDataType which(result_type); + if (settings.allow_execute_multiif_columnar && (which.isInt() || which.isUInt() || which.isFloat())) + { +#define EXECUTE_INSTRUCTIONS_COLUMNAR(TYPE) \ + if (which.is##TYPE()) \ + { \ + MutableColumnPtr res = ColumnVector::create(rows); \ + executeInstructionsColumnar(instructions, rows, res); \ + return std::move(res); \ + } + +#define ENUMERATE_NUMERIC_TYPES(M) \ + M(UInt8) \ + M(UInt16) \ + M(UInt32) \ + M(UInt64) \ + M(Int8) \ + M(Int16) \ + M(Int32) \ + M(Int64) \ + M(UInt128) \ + M(UInt256) \ + M(Int128) \ + M(Int256) \ + M(Float32) \ + M(Float64) + + ENUMERATE_NUMERIC_TYPES(EXECUTE_INSTRUCTIONS_COLUMNAR) + } else + { + MutableColumnPtr res = return_type->createColumn(); executeInstructions(instructions, rows, res); - return res; + return std::move(res); + } + + return {}; } private: - static void executeInstructions(std::vector & instructions, size_t rows, const MutableColumnPtr & res) + static NO_INLINE void executeInstructions(std::vector & instructions, size_t rows, const MutableColumnPtr & res) { for (size_t i = 0; i < rows; ++i) { @@ -283,12 +307,9 @@ private: } } - // template - static void executeInstructionsColumnar(std::vector & instructions, size_t rows, const MutableColumnPtr & res) + /// We should read source from which instruction on each row? + static NO_INLINE void calculateInserts(std::vector & instructions, size_t rows, PaddedPODArray & inserts) { - /// We should read source from which instruction on each row? - PaddedPODArray inserts(rows, instructions.size()); - // PaddedPODArray & res_data = assert_cast &>(*res).getData(); for (Int64 i = instructions.size() - 1; i >= 0; --i) { auto & instruction = instructions[i]; @@ -303,29 +324,45 @@ private: for (size_t row_i = 0; row_i < rows; ++row_i) { size_t condition_index = instruction.condition_is_short ? instruction.condition_index++ : row_i; - if (cond_data[condition_index]) - inserts[row_i] = i; + + /// Equivalent to below code. But it is able to utilize SIMD instructions. + /// if (cond_data[condition_index]) + /// inserts[row_i] = i; + inserts[row_i] += cond_data[condition_index] * (i - inserts[row_i]); } } else { const ColumnNullable & condition_nullable = assert_cast(*instruction.condition); const ColumnUInt8 & condition_nested = assert_cast(condition_nullable.getNestedColumn()); + const auto & condition_nested_data = condition_nested.getData(); const NullMap & condition_null_map = condition_nullable.getNullMapData(); for (size_t row_i = 0; row_i < rows; ++row_i) { size_t condition_index = instruction.condition_is_short ? instruction.condition_index++ : row_i; - if (!condition_null_map[condition_index] && condition_nested.getData()[condition_index]) - inserts[row_i] = i; + /// Equivalent to below code. But it is able to utilize SIMD instructions. + /// if (!condition_null_map[condition_index] && condition_nested.getData()[condition_index]) + /// inserts[row_i] = i; + inserts[row_i] += (~condition_null_map[condition_index] & condition_nested_data[condition_index]) + * (i - inserts[row_i]); } } } + } + template + static NO_INLINE void executeInstructionsColumnar(std::vector & instructions, size_t rows, const MutableColumnPtr & res) + { + PaddedPODArray inserts(rows, -1); + calculateInserts(instructions, rows, inserts); + + PaddedPODArray & res_data = assert_cast &>(*res).getData(); for (size_t row_i = 0; row_i < rows; ++row_i) { auto & instruction = instructions[inserts[row_i]]; - (*res)[row_i] = (*instruction.source)[row_i]; + auto str_ref = instruction.source->getDataAt(row_i); + memcpy(&res_data[row_i], str_ref.data, str_ref.size); } } From 87f869a12924d10fc1bd2022cd5b924d89305998 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Mon, 16 Jan 2023 21:02:33 +0800 Subject: [PATCH 08/79] reorder branches --- src/Functions/multiIf.cpp | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index 8d3e10b480a..d3b15a2c200 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -18,7 +18,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; } namespace @@ -232,8 +232,14 @@ public: size_t rows = input_rows_count; const auto & settings = context->getSettingsRef(); const WhichDataType which(result_type); - if (settings.allow_execute_multiif_columnar && (which.isInt() || which.isUInt() || which.isFloat())) + bool execute_multiif_columnar = settings.allow_execute_multiif_columnar && (which.isInt() || which.isUInt() || which.isFloat()); + if (!execute_multiif_columnar) { + MutableColumnPtr res = return_type->createColumn(); + executeInstructions(instructions, rows, res); + return std::move(res); + } + #define EXECUTE_INSTRUCTIONS_COLUMNAR(TYPE) \ if (which.is##TYPE()) \ { \ @@ -258,20 +264,12 @@ public: M(Float32) \ M(Float64) - ENUMERATE_NUMERIC_TYPES(EXECUTE_INSTRUCTIONS_COLUMNAR) - } - else - { - MutableColumnPtr res = return_type->createColumn(); - executeInstructions(instructions, rows, res); - return std::move(res); - } - - return {}; - } + ENUMERATE_NUMERIC_TYPES(EXECUTE_INSTRUCTIONS_COLUMNAR) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Columnar multiIf not implemented for type {}", result_type->getName()); +} private: - static NO_INLINE void executeInstructions(std::vector & instructions, size_t rows, const MutableColumnPtr & res) + static void executeInstructions(std::vector & instructions, size_t rows, const MutableColumnPtr & res) { for (size_t i = 0; i < rows; ++i) { @@ -308,7 +306,7 @@ private: } /// We should read source from which instruction on each row? - static NO_INLINE void calculateInserts(std::vector & instructions, size_t rows, PaddedPODArray & inserts) + static void calculateInserts(std::vector & instructions, size_t rows, PaddedPODArray & inserts) { for (Int64 i = instructions.size() - 1; i >= 0; --i) { @@ -352,7 +350,7 @@ private: } template - static NO_INLINE void executeInstructionsColumnar(std::vector & instructions, size_t rows, const MutableColumnPtr & res) + static void executeInstructionsColumnar(std::vector & instructions, size_t rows, const MutableColumnPtr & res) { PaddedPODArray inserts(rows, -1); calculateInserts(instructions, rows, inserts); @@ -361,8 +359,8 @@ private: for (size_t row_i = 0; row_i < rows; ++row_i) { auto & instruction = instructions[inserts[row_i]]; - auto str_ref = instruction.source->getDataAt(row_i); - memcpy(&res_data[row_i], str_ref.data, str_ref.size); + auto ref = instruction.source->getDataAt(row_i); + memcpy(&res_data[row_i], ref.data, ref.size); } } From 9f571d3e930150497cad368df5e14ffcc0381a48 Mon Sep 17 00:00:00 2001 From: MeenaRenganathan22 Date: Mon, 16 Jan 2023 08:28:03 -0500 Subject: [PATCH 09/79] Updating the submodules libhdfs3 and libgsasl --- contrib/libgsasl | 2 +- contrib/libhdfs3 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/libgsasl b/contrib/libgsasl index 0324680f13f..f4e7bf0bb06 160000 --- a/contrib/libgsasl +++ b/contrib/libgsasl @@ -1 +1 @@ -Subproject commit 0324680f13f22bb43df5353a08e26453d7d640ac +Subproject commit f4e7bf0bb068030d57266f87ccac4c8c012fb5c4 diff --git a/contrib/libhdfs3 b/contrib/libhdfs3 index 9194af44588..d18a7bb366b 160000 --- a/contrib/libhdfs3 +++ b/contrib/libhdfs3 @@ -1 +1 @@ -Subproject commit 9194af44588633c1b2dae44bf945804401ff883e +Subproject commit d18a7bb366b2979032857c0950e3bf3b5fc846ed From 3b4ac825ba24e4c51a0072ff87f8243433abf83d Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Tue, 17 Jan 2023 11:27:28 +0800 Subject: [PATCH 10/79] another optimization: adjuest type of inserts according to the size of instructions, which makes more parallalism --- src/Functions/multiIf.cpp | 68 +++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index d3b15a2c200..ec116466dbf 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -19,6 +19,7 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int NOT_IMPLEMENTED; + extern const int LOGICAL_ERROR; } namespace @@ -240,33 +241,49 @@ public: return std::move(res); } -#define EXECUTE_INSTRUCTIONS_COLUMNAR(TYPE) \ +#define EXECUTE_INSTRUCTIONS_COLUMNAR(TYPE, INDEX) \ if (which.is##TYPE()) \ { \ MutableColumnPtr res = ColumnVector::create(rows); \ - executeInstructionsColumnar(instructions, rows, res); \ + executeInstructionsColumnar(instructions, rows, res); \ return std::move(res); \ } -#define ENUMERATE_NUMERIC_TYPES(M) \ - M(UInt8) \ - M(UInt16) \ - M(UInt32) \ - M(UInt64) \ - M(Int8) \ - M(Int16) \ - M(Int32) \ - M(Int64) \ - M(UInt128) \ - M(UInt256) \ - M(Int128) \ - M(Int256) \ - M(Float32) \ - M(Float64) +#define ENUMERATE_NUMERIC_TYPES(M, INDEX) \ + M(UInt8, INDEX) \ + M(UInt16, INDEX) \ + M(UInt32, INDEX) \ + M(UInt64, INDEX) \ + M(Int8, INDEX) \ + M(Int16, INDEX) \ + M(Int32, INDEX) \ + M(Int64, INDEX) \ + M(UInt128, INDEX) \ + M(UInt256, INDEX) \ + M(Int128, INDEX) \ + M(Int256, INDEX) \ + M(Float32, INDEX) \ + M(Float64, INDEX) \ + throw Exception( \ + ErrorCodes::NOT_IMPLEMENTED, "Columnar execution of function {} not implemented for type {}", getName(), result_type->getName()); - ENUMERATE_NUMERIC_TYPES(EXECUTE_INSTRUCTIONS_COLUMNAR) - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Columnar multiIf not implemented for type {}", result_type->getName()); -} + size_t num_instructions = instructions.size(); + if (num_instructions <= std::numeric_limits::max()) + { + ENUMERATE_NUMERIC_TYPES(EXECUTE_INSTRUCTIONS_COLUMNAR, Int16) + } + else if (num_instructions <= std::numeric_limits::max()) + { + ENUMERATE_NUMERIC_TYPES(EXECUTE_INSTRUCTIONS_COLUMNAR, Int32) + } + else if (num_instructions <= std::numeric_limits::max()) + { + ENUMERATE_NUMERIC_TYPES(EXECUTE_INSTRUCTIONS_COLUMNAR, Int64) + } + else + throw Exception( + ErrorCodes::LOGICAL_ERROR, "Instruction size({}) of function {} is out of range", getName(), result_type->getName()); + } private: static void executeInstructions(std::vector & instructions, size_t rows, const MutableColumnPtr & res) @@ -306,9 +323,10 @@ private: } /// We should read source from which instruction on each row? - static void calculateInserts(std::vector & instructions, size_t rows, PaddedPODArray & inserts) + template + static NO_INLINE void calculateInserts(std::vector & instructions, size_t rows, PaddedPODArray & inserts) { - for (Int64 i = instructions.size() - 1; i >= 0; --i) + for (S i = static_cast(instructions.size() - 1); i >= 0; --i) { auto & instruction = instructions[i]; if (instruction.condition_always_true) @@ -349,10 +367,10 @@ private: } } - template - static void executeInstructionsColumnar(std::vector & instructions, size_t rows, const MutableColumnPtr & res) + template + static NO_INLINE void executeInstructionsColumnar(std::vector & instructions, size_t rows, const MutableColumnPtr & res) { - PaddedPODArray inserts(rows, -1); + PaddedPODArray inserts(rows, static_cast(instructions.size())); calculateInserts(instructions, rows, inserts); PaddedPODArray & res_data = assert_cast &>(*res).getData(); From 9d2844e68975b551cf0257b8573228212adef8b3 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Tue, 17 Jan 2023 11:40:06 +0800 Subject: [PATCH 11/79] last optimization: convert memcpy to assign operation of fixed type T --- src/Functions/multiIf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index ec116466dbf..a9aa1955fec 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -378,7 +378,7 @@ private: { auto & instruction = instructions[inserts[row_i]]; auto ref = instruction.source->getDataAt(row_i); - memcpy(&res_data[row_i], ref.data, ref.size); + res_data[row_i] = *reinterpret_cast(ref.data); } } From d23e456f9ee409cdfbc330fb9d168b84af2be133 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Tue, 17 Jan 2023 11:41:49 +0800 Subject: [PATCH 12/79] remove no_inline --- src/Functions/multiIf.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index a9aa1955fec..65da8b31727 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -324,7 +324,7 @@ private: /// We should read source from which instruction on each row? template - static NO_INLINE void calculateInserts(std::vector & instructions, size_t rows, PaddedPODArray & inserts) + static void calculateInserts(std::vector & instructions, size_t rows, PaddedPODArray & inserts) { for (S i = static_cast(instructions.size() - 1); i >= 0; --i) { @@ -368,7 +368,7 @@ private: } template - static NO_INLINE void executeInstructionsColumnar(std::vector & instructions, size_t rows, const MutableColumnPtr & res) + static void executeInstructionsColumnar(std::vector & instructions, size_t rows, const MutableColumnPtr & res) { PaddedPODArray inserts(rows, static_cast(instructions.size())); calculateInserts(instructions, rows, inserts); From 54d2ae4a78afc3e65872ece85b6b72666e73d936 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Tue, 17 Jan 2023 14:44:33 +0800 Subject: [PATCH 13/79] add some debug logs --- src/Functions/multiIf.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index 65da8b31727..b0a75477c7a 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -219,6 +219,15 @@ public: break; } + for (size_t i = 0; i < instructions.size(); ++i) + { + auto & instruction = instructions[i]; + auto source = instruction.source; + for (size_t row_i = 0; row_i < input_rows_count; ++row_i) + { + std::cout << "number:" << row_i << ",instruction:" << i << ",field" << toString((*source)[row_i]) << std::endl; + } + } /// Special case if first instruction condition is always true and source is constant if (instructions.size() == 1 && instructions.front().source_is_constant @@ -241,6 +250,7 @@ public: return std::move(res); } + #define EXECUTE_INSTRUCTIONS_COLUMNAR(TYPE, INDEX) \ if (which.is##TYPE()) \ { \ @@ -373,12 +383,15 @@ private: PaddedPODArray inserts(rows, static_cast(instructions.size())); calculateInserts(instructions, rows, inserts); + + PaddedPODArray & res_data = assert_cast &>(*res).getData(); for (size_t row_i = 0; row_i < rows; ++row_i) { auto & instruction = instructions[inserts[row_i]]; auto ref = instruction.source->getDataAt(row_i); res_data[row_i] = *reinterpret_cast(ref.data); + std::cout << "number:" << row_i << ",instruction:" << inserts[row_i] << std::endl; } } From 1093a42f685b46025f2b1acfff16ea5d3ab053fa Mon Sep 17 00:00:00 2001 From: Suzy Wang Date: Tue, 17 Jan 2023 12:04:01 -0800 Subject: [PATCH 14/79] window view fix --- src/Interpreters/Aggregator.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index c81cfa2c0a2..7dec8c9c8a4 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -209,7 +209,8 @@ struct AggregationMethodOneNumber // Insert the key from the hash table into columns. static void insertKeyIntoColumns(const Key & key, std::vector & key_columns, const Sizes & /*key_sizes*/) { - const auto * key_holder = reinterpret_cast(&key); + FieldType casted_key = static_cast(key); + const auto * key_holder = reinterpret_cast(&casted_key); auto * column = static_cast(key_columns[0]); column->insertRawData(key_holder); } From f66dbf6372cadb605ec46f6da0742285abf6f43c Mon Sep 17 00:00:00 2001 From: MeenaRenganathan22 Date: Wed, 18 Jan 2023 13:47:16 -0500 Subject: [PATCH 15/79] Committing the CMakeLists.txt with IntelAsmCrc32c.cpp and perf_checksum.cpp --- contrib/libhdfs3-cmake/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index bd48772dd33..6b405ad628d 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -100,6 +100,8 @@ set(SRCS "${HDFS3_SOURCE_DIR}/common/Hash.cpp" "${HDFS3_SOURCE_DIR}/common/SWCrc32c.cpp" "${HDFS3_SOURCE_DIR}/common/Thread.cpp" + "${HDFS3_SOURCE_DIR}/common/IntelAsmCrc32c.cpp" + "${HDFS3_SOURCE_DIR}/common/perf_checksum.cpp" ${PROTO_SOURCES} ) From 359775a1cf0a0a66e5e65d67c6f4b2b31a5014fb Mon Sep 17 00:00:00 2001 From: MeenaRenganathan22 Date: Tue, 24 Jan 2023 20:31:33 -0500 Subject: [PATCH 16/79] Updated yhe libhdfs3 submoule --- contrib/libhdfs3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libhdfs3 b/contrib/libhdfs3 index d18a7bb366b..f938c73b401 160000 --- a/contrib/libhdfs3 +++ b/contrib/libhdfs3 @@ -1 +1 @@ -Subproject commit d18a7bb366b2979032857c0950e3bf3b5fc846ed +Subproject commit f938c73b401967df45ae65bb791b601123aac5a4 From 6f76ecfdb5f90a2b0368c19e87c36a0da384eaa1 Mon Sep 17 00:00:00 2001 From: MeenaRenganathan22 Date: Tue, 24 Jan 2023 20:33:25 -0500 Subject: [PATCH 17/79] Committing the libhdfs3-cmake/CMakeLists.txt to include the IntelAsmCrc32c.cpp and perf_checksum.cpp --- contrib/libhdfs3-cmake/CMakeLists.txt | 28 +++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index 6b405ad628d..f9c59f0f016 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -101,10 +101,18 @@ set(SRCS "${HDFS3_SOURCE_DIR}/common/SWCrc32c.cpp" "${HDFS3_SOURCE_DIR}/common/Thread.cpp" "${HDFS3_SOURCE_DIR}/common/IntelAsmCrc32c.cpp" - "${HDFS3_SOURCE_DIR}/common/perf_checksum.cpp" ${PROTO_SOURCES} ) +if (ARCH_AMD64) + find_program(YASM_PATH NAMES yasm) + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/crc_iscsi_v_pcl.o + COMMAND ${YASM_PATH} -f x64 -f elf64 -X gnu -g dwarf2 -D LINUX -o ${CMAKE_CURRENT_BINARY_DIR}/crc_iscsi_v_pcl.o ${HDFS3_SOURCE_DIR}/common/crc_iscsi_v_pcl.asm + COMMENT "Translating Intel CRC assembly to object code") + LIST(APPEND SRCS ${CMAKE_CURRENT_BINARY_DIR}/crc_iscsi_v_pcl.o) +endif () + # old kernels (< 3.17) doesn't have SYS_getrandom. Always use POSIX implementation to have better compatibility set_source_files_properties("${HDFS3_SOURCE_DIR}/rpc/RpcClient.cpp" PROPERTIES COMPILE_FLAGS "-DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1") @@ -113,9 +121,16 @@ add_library(_hdfs3 ${SRCS}) add_dependencies(_hdfs3 protoc) -target_include_directories(_hdfs3 PRIVATE ${HDFS3_SOURCE_DIR}) -target_include_directories(_hdfs3 PRIVATE ${HDFS3_COMMON_DIR}) -target_include_directories(_hdfs3 PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +if (ENABLE_CLICKHOUSE_BENCHMARK) + message(STATUS "enable perf checksum") + target_include_directories(_hdfs3 PUBLIC ${HDFS3_SOURCE_DIR}) + target_include_directories(_hdfs3 PUBLIC ${HDFS3_COMMON_DIR}) + target_include_directories(_hdfs3 PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) +else () + target_include_directories(_hdfs3 PRIVATE ${HDFS3_SOURCE_DIR}) + target_include_directories(_hdfs3 PRIVATE ${HDFS3_COMMON_DIR}) + target_include_directories(_hdfs3 PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +endif () target_include_directories(_hdfs3 SYSTEM BEFORE INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include") @@ -132,3 +147,8 @@ if (TARGET OpenSSL::SSL) endif() add_library(ch_contrib::hdfs ALIAS _hdfs3) + +if (ENABLE_CLICKHOUSE_BENCHMARK) + add_executable(perf_checksum ${HDFS3_SOURCE_DIR}/common/perf_checksum.cpp) + target_link_libraries(perf_checksum PRIVATE ch_contrib::hdfs) +endif () From 4a0202a07422b8328e6432adf38141ee4633f1f0 Mon Sep 17 00:00:00 2001 From: MeenaRenganathan22 Date: Tue, 24 Jan 2023 20:41:43 -0500 Subject: [PATCH 18/79] Committing the docker with the changes to install the ysam --- docker/packager/binary/Dockerfile | 1 + docker/server/Dockerfile.ubuntu | 1 + 2 files changed, 2 insertions(+) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index cd9eb133f62..f750dbf4f74 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -23,6 +23,7 @@ RUN apt-get update && \ libc6 \ libc6-dev \ libc6-dev-arm64-cross \ + yasm \ zstd && \ apt-get clean diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index b6a93c808c3..6f8858feff5 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -16,6 +16,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list gnupg \ locales \ wget \ + yasm \ tzdata \ && apt-get clean From f4d38b82e6e19c6e321523a516fbb48a9abfa3f4 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Wed, 25 Jan 2023 10:38:51 +0100 Subject: [PATCH 19/79] RegExpTreeDict use re2 engines when processing heavy regexps --- src/Dictionaries/RegExpTreeDictionary.cpp | 119 +- src/Dictionaries/RegExpTreeDictionary.h | 4 +- tests/metadata/INFORMATION_SCHEMA.sql | 2 + tests/metadata/information_schema.sql | 2 + tests/metadata/system | 1 + tests/metadata/system.sql | 2 + ...2504_regexp_dictionary_ua_parser.reference | 793 +++++ .../02504_regexp_dictionary_ua_parser.sh | 87 + ...04_regexp_dictionary_yaml_source.reference | 6 +- .../02504_regexp_dictionary_yaml_source.sh | 4 +- .../0_stateless/data_ua_parser/browser.yaml | 1409 +++++++++ .../0_stateless/data_ua_parser/device.yaml | 2651 +++++++++++++++++ .../0_stateless/data_ua_parser/os.yaml | 1075 +++++++ .../0_stateless/data_ua_parser/useragents.txt | 793 +++++ 14 files changed, 6926 insertions(+), 22 deletions(-) create mode 100644 tests/metadata/INFORMATION_SCHEMA.sql create mode 100644 tests/metadata/information_schema.sql create mode 120000 tests/metadata/system create mode 100644 tests/metadata/system.sql create mode 100644 tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.reference create mode 100755 tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh create mode 100644 tests/queries/0_stateless/data_ua_parser/browser.yaml create mode 100644 tests/queries/0_stateless/data_ua_parser/device.yaml create mode 100644 tests/queries/0_stateless/data_ua_parser/os.yaml create mode 100644 tests/queries/0_stateless/data_ua_parser/useragents.txt diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index 9410944c71b..21c61173e92 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -86,8 +87,15 @@ struct RegExpTreeDictionary::RegexTreeNode UInt64 parent_id; std::string regex; re2_st::RE2 searcher; + RegexTreeNode(UInt64 id_, UInt64 parent_id_, const String & regex_, const re2_st::RE2::Options & regexp_options): id(id_), parent_id(parent_id_), regex(regex_), searcher(regex_, regexp_options) {} + + bool match(const char * haystack, size_t size) const + { + return searcher.Match(haystack, 0, size, re2_st::RE2::Anchor::UNANCHORED, nullptr, 0); + } + struct AttributeValue { Field field; @@ -142,6 +150,52 @@ void RegExpTreeDictionary::calculateBytesAllocated() bytes_allocated += 2 * sizeof(UInt64) * topology_order.size(); } +namespace +{ + /// hyper scan is not good at processing regex containing {0, 200} + /// This will make re compilation slow and failed. So we select this heavy regular expressions and + /// process it with re2. + struct ComplexRegexChecker + { + re2_st::RE2 searcher; + ComplexRegexChecker() : searcher(R"(\{([\d]+),([\d]+)\})") {} + + static bool isFigureLargerThanFifty(const String & str) + try + { + auto number = std::stoi(str); + return number > 50; + } + catch (std::exception &) + { + return false; + } + + bool check(const String & data) const + { + + re2_st::StringPiece haystack(data.data(), data.size()); + re2_st::StringPiece matches[10]; + size_t start_pos = 0; + while (start_pos < data.size()) + { + if (searcher.Match(haystack, start_pos, data.size(), re2_st::RE2::Anchor::UNANCHORED, matches, 10)) + { + const auto & match = matches[0]; + start_pos += match.length(); + const auto & match1 = matches[1]; + const auto & match2 = matches[2]; + if (isFigureLargerThanFifty(match1.ToString()) || isFigureLargerThanFifty(match2.ToString())) + return true; + } + else + break; + } + return false; + } + }; +} + void RegExpTreeDictionary::initRegexNodes(Block & block) { auto id_column = block.getByName(kId).column; @@ -150,6 +204,8 @@ void RegExpTreeDictionary::initRegexNodes(Block & block) auto keys_column = block.getByName(kKeys).column; auto values_column = block.getByName(kValues).column; + ComplexRegexChecker checker; + size_t size = block.rows(); for (size_t i = 0; i < size; i++) { @@ -163,12 +219,10 @@ void RegExpTreeDictionary::initRegexNodes(Block & block) if (id == 0) throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION, "There are invalid id {}", id); - regexps.push_back(regex); - regexp_ids.push_back(id); re2_st::RE2::Options regexp_options; regexp_options.set_log_errors(false); - RegexTreeNodePtr node = std::make_unique(id, parent_id, regex, regexp_options); + RegexTreeNodePtr node = std::make_shared(id, parent_id, regex, regexp_options); int num_captures = std::min(node->searcher.NumberOfCapturingGroups() + 1, 10); @@ -194,7 +248,17 @@ void RegExpTreeDictionary::initRegexNodes(Block & block) } } } - regex_nodes.emplace(id, std::move(node)); + regex_nodes.emplace(id, node); + if (checker.check(regex)) + { + complex_regexp_nodes.push_back(node); + LOG_INFO(logger, "Found complex regexp {} in the config. This will slow down the queries", regex); + } + else + { + regexps.push_back(regex); + regexp_ids.push_back(id); + } } } @@ -245,6 +309,7 @@ void RegExpTreeDictionary::loadData() initGraph(); if (regexps.empty()) throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION, "There are no available regular expression. Please check your config"); + LOG_INFO(logger, "There are {} simple regexps and {} complex regexps", regexps.size(), complex_regexp_nodes.size()); #if USE_VECTORSCAN try { @@ -287,12 +352,15 @@ RegExpTreeDictionary::RegExpTreeDictionary( calculateBytesAllocated(); } -String processBackRefs(const String & data, const re2_st::RE2 & searcher, const std::vector & pieces) +std::pair processBackRefs(const String & data, const re2_st::RE2 & searcher, const std::vector & pieces) { re2_st::StringPiece haystack(data.data(), data.size()); re2_st::StringPiece matches[10]; String result; searcher.Match(haystack, 0, data.size(), re2_st::RE2::Anchor::UNANCHORED, matches, 10); + /// if the pattern is a single '$1' but fails to match, we would use the default value. + if (pieces.size() == 1 && pieces[0].ref_num >= 0 && pieces[0].ref_num < 9 && matches[pieces[0].ref_num].empty()) + return std::make_pair(result, true); for (const auto & item : pieces) { if (item.ref_num >= 0 && item.ref_num < 10) @@ -300,7 +368,7 @@ String processBackRefs(const String & data, const re2_st::RE2 & searcher, const else result += item.literal; } - return result; + return std::make_pair(result, false); } // walk towards root and collect attributes. @@ -323,8 +391,9 @@ bool RegExpTreeDictionary::setAttributes( continue; if (value.containsBackRefs()) { - String updated_str = processBackRefs(data, regex_nodes.at(id)->searcher, value.pieces); - attributes_to_set[name] = parseStringToField(updated_str, attributes.at(name).type); + auto [updated_str, use_default] = processBackRefs(data, regex_nodes.at(id)->searcher, value.pieces); + if (!use_default) + attributes_to_set[name] = parseStringToField(updated_str, attributes.at(name).type); } else attributes_to_set[name] = value.field; @@ -352,12 +421,19 @@ namespace MatchContext(const std::vector & regexp_ids_, const std::unordered_map & topology_order_) : regexp_ids(regexp_ids_), topology_order(topology_order_) {} - void insert(unsigned int id) + void insertIdx(unsigned int idx) { - UInt64 idx = regexp_ids[id-1]; - UInt64 topological_order = topology_order.at(idx); - matched_idx_set.emplace(idx); - matched_idx_sorted_list.push_back(std::make_pair(topological_order, idx)); + UInt64 node_id = regexp_ids[idx-1]; + UInt64 topological_order = topology_order.at(node_id); + matched_idx_set.emplace(node_id); + matched_idx_sorted_list.push_back(std::make_pair(topological_order, node_id)); + } + + void insertNodeID(UInt64 id) + { + UInt64 topological_order = topology_order.at(id); + matched_idx_set.emplace(id); + matched_idx_sorted_list.push_back(std::make_pair(topological_order, id)); } void sort() @@ -406,7 +482,7 @@ std::unordered_map RegExpTreeDictionary::matchSearchAllIndice unsigned int /* flags */, void * context) -> int { - static_cast(context)->insert(id); + static_cast(context)->insertIdx(id); return 0; }; @@ -430,13 +506,25 @@ std::unordered_map RegExpTreeDictionary::matchSearchAllIndice if (err != HS_SUCCESS) throw Exception(ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT, "Failed to scan data with vectorscan"); + if (!match_result.matched_idx_set.empty()) + { + for (const auto & node_ptr : complex_regexp_nodes) + { + if (node_ptr->match(reinterpret_cast(keys_data.data()) + offset, length)) + { + match_result.insertNodeID(node_ptr->id); + break; + } + } + } + match_result.sort(); // Walk through the regex tree util all attributes are set; std::unordered_map attributes_to_set; std::unordered_set visited_nodes; - // check if it is a valid id + // Some node matches but its parents cannot match. In this case we must regard this node unmatched. auto is_invalid = [&](UInt64 id) { while (id) @@ -466,7 +554,6 @@ std::unordered_map RegExpTreeDictionary::matchSearchAllIndice if (attributes_to_set.contains(name)) continue; - /// TODO: default value might be a back-reference, that is useful in lib ua-core DefaultValueProvider default_value(attr.null_value, defaults.at(name)); columns[name]->insert(default_value.getDefaultValue(key_idx)); } diff --git a/src/Dictionaries/RegExpTreeDictionary.h b/src/Dictionaries/RegExpTreeDictionary.h index 8fdf5cff5e6..f3de05239b5 100644 --- a/src/Dictionaries/RegExpTreeDictionary.h +++ b/src/Dictionaries/RegExpTreeDictionary.h @@ -149,7 +149,9 @@ private: const std::unordered_map & attributes) const; struct RegexTreeNode; - using RegexTreeNodePtr = std::unique_ptr; + using RegexTreeNodePtr = std::shared_ptr; + + std::vector complex_regexp_nodes; std::map regex_nodes; std::unordered_map topology_order; diff --git a/tests/metadata/INFORMATION_SCHEMA.sql b/tests/metadata/INFORMATION_SCHEMA.sql new file mode 100644 index 00000000000..291582fd1eb --- /dev/null +++ b/tests/metadata/INFORMATION_SCHEMA.sql @@ -0,0 +1,2 @@ +ATTACH DATABASE INFORMATION_SCHEMA +ENGINE = Memory diff --git a/tests/metadata/information_schema.sql b/tests/metadata/information_schema.sql new file mode 100644 index 00000000000..6cea934b49d --- /dev/null +++ b/tests/metadata/information_schema.sql @@ -0,0 +1,2 @@ +ATTACH DATABASE information_schema +ENGINE = Memory diff --git a/tests/metadata/system b/tests/metadata/system new file mode 120000 index 00000000000..9a11b7e3458 --- /dev/null +++ b/tests/metadata/system @@ -0,0 +1 @@ +/home/hanfei/ClickHouse/tests/store/069/0693727d-7be1-48e3-9ff3-aa0bba736b34/ \ No newline at end of file diff --git a/tests/metadata/system.sql b/tests/metadata/system.sql new file mode 100644 index 00000000000..941c12b0a66 --- /dev/null +++ b/tests/metadata/system.sql @@ -0,0 +1,2 @@ +ATTACH DATABASE _ UUID '0693727d-7be1-48e3-9ff3-aa0bba736b34' +ENGINE = Atomic diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.reference b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.reference new file mode 100644 index 00000000000..d6dd2adae74 --- /dev/null +++ b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.reference @@ -0,0 +1,793 @@ +AppleTV Other 0.0 ATV OS X 8.1.3 +LG-M150 Firefox Mobile 68.0 Android 7.0.0 +Other Firefox Mobile 68.0 Android 8.0.0 +Other Firefox Mobile 68.0 Android 8.1.0 +Other Firefox Mobile 68.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 10.0.0 +Other Chrome Mobile 77.0 Android 10.0.0 +Other Chrome Mobile 77.0 Android 10.0.0 +Other Facebook 240.0 Android 10.0.0 +Other Chrome Mobile WebView 77.0 Android 10.0.0 +Other Chrome Mobile 77.0 Android 10.0.0 +HTC Sensation 4G Chrome Mobile 42.0 Android 4.0.3 +Kindle Amazon Silk 73.7 Android 4.0.3 +Samsung GT-I9152 Chrome Mobile 42.0 Android 4.2.2 +Samsung GT-N5110 Chrome 76.0 Android 4.4.2 +RCT6773W22 Chrome 77.0 Android 4.4.2 +Samsung SM-T217S Chrome 77.0 Android 4.4.2 +Samsung SM-T530NU Chrome 77.0 Android 4.4.2 +Other Chrome 69.0 Android 4.4.2 +Kindle Amazon Silk 76.3 Android 4.4.3 +Samsung SM-A500H Chrome Mobile 73.0 Android 5.0.2 +Samsung SM-T357T Chrome 77.0 Android 5.0.2 +Samsung SM-T530NU Chrome 76.0 Android 5.0.2 +Samsung SM-T530NU Chrome 77.0 Android 5.0.2 +RCT6213W87DK Yandex Browser 19.4 Android 5.0.0 +Samsung SM-N900T Facebook 229.0 Android 5.0.0 +Other Chrome Mobile WebView 70.0 Android 5.1.1 +Kindle Amazon Silk 76.3 Android 5.1.1 +Other Chrome Mobile WebView 70.0 Android 5.1.1 +Kindle Amazon Silk 76.3 Android 5.1.1 +Kindle Amazon Silk 76.3 Android 5.1.1 +Kindle Amazon Silk 71.2 Android 5.1.1 +Kindle Amazon Silk 76.3 Android 5.1.1 +Kindle Amazon Silk 76.3 Android 5.1.1 +Kindle Amazon Silk 76.3 Android 5.1.1 +Kindle Amazon Silk 76.3 Android 5.1.1 +Kindle Amazon Silk 77.1 Android 5.1.1 +LG-AS330 Chrome Mobile 77.0 Android 5.1.1 +Other Chrome Mobile 77.0 Android 5.1.1 +Samsung SM-G530R7 Samsung Internet 9.2 Android 5.1.1 +Samsung SM-T377P Samsung Internet 10.1 Android 5.1.1 +Samsung SM-T900 Samsung Internet 10.1 Android 5.1.1 +Samsung SM-T337A Chrome 69.0 Android 5.1.1 +Samsung SM-G360T1 Chrome Mobile 67.0 Android 5.1.1 +Samsung SM-J320FN Chrome Mobile 74.0 Android 5.1.1 +Other Chrome 74.0 Android 5.1.1 +Samsung SM-T330NU Chrome 71.0 Android 5.1.1 +Other Chrome 76.0 Android 5.1.1 +Other Chrome 77.0 Android 5.1.1 +Other Chrome Mobile WebView 74.0 Android 5.1.1 +Other Chrome Mobile 66.0 Android 5.1.0 +HTC Desire 626s Chrome Mobile 77.0 Android 5.1.0 +HUAWEI LUA-L22 Chrome Mobile 50.0 Android 5.1.0 +Other Chrome 77.0 Android 5.1.0 +Other Chrome Mobile 73.0 Android 5.1.0 +Oppo CPH1613 Chrome Mobile 77.0 Android 6.0.1 +LG-M153 Chrome Mobile WebView 55.0 Android 6.0.1 +LG-M153 Chrome Mobile 77.0 Android 6.0.1 +Other Chrome Mobile 77.0 Android 6.0.1 +Other Chrome Mobile 74.0 Android 6.0.1 +Other Chrome 44.0 Android 6.0.1 +Samsung SM-G900I Samsung Internet 10.1 Android 6.0.1 +Samsung SM-G900P Samsung Internet 7.2 Android 6.0.1 +Samsung SM-J700M Samsung Internet 10.1 Android 6.0.1 +Samsung SM-S327VL Samsung Internet 10.1 Android 6.0.1 +Samsung SM-T377A Chrome 77.0 Android 6.0.1 +Samsung SM-G532M Chrome Mobile 55.0 Android 6.0.1 +Samsung SM-G532M Facebook 240.0 Android 6.0.1 +Samsung SM-G532M Chrome Mobile 77.0 Android 6.0.1 +Samsung SM-G550T Chrome Mobile 76.0 Android 6.0.1 +Samsung SM-G550T Chrome Mobile 77.0 Android 6.0.1 +Samsung SM-G550T1 Chrome Mobile 76.0 Android 6.0.1 +Samsung SM-G900V Chrome Mobile 73.0 Android 6.0.1 +Samsung SM-G920A Chrome Mobile 77.0 Android 6.0.1 +Samsung SM-J327P Chrome Mobile 77.0 Android 6.0.1 +Samsung SM-N910S Chrome Mobile 75.0 Android 6.0.1 +Samsung SM-N920V Chrome Mobile 76.0 Android 6.0.1 +Samsung SM-T350 Chrome 59.0 Android 6.0.1 +Samsung SM-T560NU Chrome 77.0 Android 6.0.1 +Other Chrome 77.0 Android 6.0.1 +Other Chrome Mobile 77.0 Android 6.0.1 +Other Chrome Mobile 67.0 Android 6.0.1 +Other Chrome Mobile WebView 45.0 Android 6.0.1 +Other Chrome Mobile 76.0 Android 6.0.0 +Huawei CAM-L21 Chrome Mobile 77.0 Android 6.0.0 +Other Chrome Mobile 77.0 Android 6.0.0 +RCT6603W47M7 Chrome 77.0 Android 6.0.0 +Other Chrome Mobile 56.0 Android 7.0.0 +Asus A002A Chrome Mobile 77.0 Android 7.0.0 +Other Chrome Mobile 77.0 Android 7.0.0 +Other Chrome Mobile WebView 59.0 Android 7.0.0 +Other Chrome Mobile 77.0 Android 7.0.0 +LG-H872 Chrome Mobile 64.0 Android 7.0.0 +LG-K425 Chrome Mobile 55.0 Android 7.0.0 +LG-LS777 Chrome Mobile 77.0 Android 7.0.0 +LG-M210 Chrome Mobile 77.0 Android 7.0.0 +LG-M430 Chrome Mobile 77.0 Android 7.0.0 +LG-TP260 Chrome Mobile WebView 64.0 Android 7.0.0 +LG-TP260 Chrome Mobile 77.0 Android 7.0.0 +LG-TP450 Chrome Mobile 64.0 Android 7.0.0 +LG-V521 Chrome 75.0 Android 7.0.0 +LG-V521 Chrome 77.0 Android 7.0.0 +Other Chrome Mobile 58.0 Android 7.0.0 +Other Chrome Mobile 55.0 Android 7.0.0 +Other Chrome Mobile 77.0 Android 7.0.0 +Other Chrome 77.0 Android 7.0.0 +Other Chrome Mobile 77.0 Android 7.0.0 +Samsung SM-J701F Samsung Internet 10.1 Android 7.0.0 +Samsung SM-J710F Samsung Internet 10.1 Android 7.0.0 +Samsung SM-N920T Samsung Internet 9.2 Android 7.0.0 +Samsung SM-G920A Chrome Mobile 77.0 Android 7.0.0 +Samsung SM-G920P Flipboard 4.2 Android 7.0.0 +Samsung SM-G920V Chrome Mobile 76.0 Android 7.0.0 +Samsung SM-G928V Chrome Mobile 77.0 Android 7.0.0 +Samsung SM-G950U Chrome Mobile 77.0 Android 7.0.0 +Samsung SM-G955U Chrome Mobile 77.0 Android 7.0.0 +Samsung SM-J327T Chrome Mobile 74.0 Android 7.0.0 +Samsung SM-J327T Chrome Mobile 77.0 Android 7.0.0 +Samsung SM-J327T1 Chrome Mobile 64.0 Android 7.0.0 +Samsung SM-J327T1 Chrome Mobile 75.0 Android 7.0.0 +Samsung SM-J327T1 Chrome Mobile 77.0 Android 7.0.0 +Samsung SM-N9208 Chrome Mobile 73.0 Android 7.0.0 +Samsung SM-N920P Chrome Mobile 74.0 Android 7.0.0 +Samsung SM-N920T Chrome Mobile 77.0 Android 7.0.0 +Other Chrome 77.0 Android 7.0.0 +Other Chrome 75.0 Android 7.0.0 +Other Chrome 76.0 Android 7.0.0 +Other Chrome 77.0 Android 7.0.0 +Other Chrome 76.0 Android 7.0.0 +Other Chrome 76.0 Android 7.0.0 +Other Chrome 75.0 Android 7.0.0 +Other Chrome Mobile 77.0 Android 7.0.0 +Generic Smartphone Chrome Mobile 58.0 Android 7.0.0 +Other Chrome Mobile WebView 77.0 Android 7.1.1 +Other Chrome Mobile 77.0 Android 7.1.1 +Oppo CPH1729 Facebook 240.0 Android 7.1.1 +Other Chrome Mobile 74.0 Android 7.1.1 +Other Chrome Mobile 77.0 Android 7.1.1 +Other Chrome Mobile WebView 76.0 Android 7.1.1 +Other Chrome Mobile 70.0 Android 7.1.1 +Other Chrome Mobile 76.0 Android 7.1.1 +Other Chrome Mobile 77.0 Android 7.1.1 +Other Chrome Mobile 77.0 Android 7.1.1 +Other Chrome Mobile 77.0 Android 7.1.1 +Other Chrome Mobile 64.0 Android 7.1.1 +Samsung SM-J320A Samsung Internet 10.1 Android 7.1.1 +Samsung SM-T550 Samsung Internet 10.1 Android 7.1.1 +Samsung SM-T377A Chrome 64.0 Android 7.1.1 +Samsung SM-J250F Chrome Mobile 76.0 Android 7.1.1 +Samsung SM-J700T Chrome Mobile 77.0 Android 7.1.1 +Other Chrome 77.0 Android 7.1.1 +Samsung SM-T377T Chrome 77.0 Android 7.1.1 +Samsung SM-T550 Chrome 69.0 Android 7.1.1 +Other Chrome 77.0 Android 7.1.1 +Samsung SM-T560NU Chrome 77.0 Android 7.1.1 +X20 Chrome Mobile WebView 52.0 Android 7.1.1 +Other Chrome Mobile 58.0 Android 7.1.1 +Other Chrome Mobile WebView 74.0 Android 7.1.1 +Other Chrome Mobile WebView 75.0 Android 7.1.1 +Other Chrome Mobile 77.0 Android 7.1.1 +Other Chrome Mobile WebView 70.0 Android 7.1.2 +Other Chrome Mobile WebView 70.0 Android 7.1.2 +Kindle Amazon Silk 76.3 Android 7.1.2 +Other Chrome Mobile WebView 70.0 Android 7.1.2 +Other Chrome Mobile WebView 70.0 Android 7.1.2 +Other Chrome Mobile WebView 59.0 Android 7.1.2 +Kindle Amazon Silk 76.3 Android 7.1.2 +Kindle Amazon Silk 76.3 Android 7.1.2 +LG-SP200 Chrome Mobile 75.0 Android 7.1.2 +LG-SP200 Chrome Mobile 76.0 Android 7.1.2 +Other Chrome Mobile 76.0 Android 7.1.2 +Other Chrome Mobile 76.0 Android 7.1.2 +RCT6973W43R Chrome 77.0 Android 7.1.2 +XiaoMi Redmi 4 Chrome Mobile 77.0 Android 7.1.2 +Other Chrome Mobile WebView 76.0 Android 8.0.0 +Asus Z01FD Chrome Mobile 77.0 Android 8.0.0 +Huawei AUM-L29 Chrome Mobile 77.0 Android 8.0.0 +Other Chrome Mobile WebView 77.0 Android 8.0.0 +Other Chrome 77.0 Android 8.0.0 +Other Chrome Mobile 77.0 Android 8.0.0 +Other Chrome Mobile 77.0 Android 8.0.0 +LG-H910 Chrome Mobile 77.0 Android 8.0.0 +LG-H931 Chrome Mobile 76.0 Android 8.0.0 +LG-H932 Chrome Mobile 77.0 Android 8.0.0 +Samsung SM-A520F Samsung Internet 10.1 Android 8.0.0 +Samsung SM-G891A Samsung Internet 8.2 Android 8.0.0 +Samsung SM-G935T Samsung Internet 10.1 Android 8.0.0 +Samsung SM-G955U Samsung Internet 10.1 Android 8.0.0 +Samsung SM-J337T Samsung Internet 9.2 Android 8.0.0 +Samsung SM-J737P Samsung Internet 10.1 Android 8.0.0 +Samsung SM-N950F Samsung Internet 10.1 Android 8.0.0 +Samsung SM-G891A Chrome Mobile 72.0 Android 8.0.0 +Samsung SM-G935A Chrome Mobile 76.0 Android 8.0.0 +Samsung SM-A720F Chrome Mobile 77.0 Android 8.0.0 +Samsung SM-G570F Facebook 231.0 Android 8.0.0 +Samsung SM-G570Y Chrome Mobile 77.0 Android 8.0.0 +Samsung SM-G930T Chrome Mobile WebView 77.0 Android 8.0.0 +Samsung SM-G930V Chrome Mobile 64.0 Android 8.0.0 +Samsung SM-G930VL Chrome Mobile 74.0 Android 8.0.0 +Samsung SM-G935F Chrome Mobile 75.0 Android 8.0.0 +Samsung SM-G935P Chrome Mobile 77.0 Android 8.0.0 +Samsung SM-G935T Facebook 240.0 Android 8.0.0 +Samsung SM-G935T Chrome Mobile 77.0 Android 8.0.0 +Samsung SM-G950U Chrome Mobile 77.0 Android 8.0.0 +Samsung SM-G955U Chrome Mobile 74.0 Android 8.0.0 +Samsung SM-G955U Chrome Mobile 77.0 Android 8.0.0 +Samsung SM-J330G Chrome Mobile 77.0 Android 8.0.0 +Samsung SM-J337T Chrome Mobile 77.0 Android 8.0.0 +Samsung SM-J737A Chrome Mobile 77.0 Android 8.0.0 +Samsung SM-J737T1 Chrome Mobile 66.0 Android 8.0.0 +Samsung SM-J737T1 Chrome Mobile 77.0 Android 8.0.0 +Samsung SM-N950F Chrome Mobile 66.0 Android 8.0.0 +Samsung SM-N950U Chrome Mobile 76.0 Android 8.0.0 +Samsung SM-N950U Chrome Mobile 77.0 Android 8.0.0 +Samsung SM-N950U1 Chrome Mobile 77.0 Android 8.0.0 +Samsung SM-S367VL Chrome Mobile 77.0 Android 8.0.0 +Other Chrome Mobile 77.0 Android 8.0.0 +Other Chrome Mobile 77.0 Android 8.0.0 +Other Chrome Mobile 76.0 Android 8.0.0 +Other Chrome Mobile 77.0 Android 8.0.0 +Other Chrome Mobile 76.0 Android 8.0.0 +Other Chrome Mobile 77.0 Android 8.0.0 +Other Chrome Mobile 77.0 Android 8.1.0 +Other Chrome Mobile 77.0 Android 8.1.0 +Other Chrome Mobile 67.0 Android 8.1.0 +Other Chrome Mobile 76.0 Android 8.1.0 +Huawei BKK-LX2 Chrome Mobile 76.0 Android 8.1.0 +Other Chrome Mobile 70.0 Android 8.1.0 +Other Chrome Mobile 77.0 Android 8.1.0 +Other Chrome Mobile 64.0 Android 8.1.0 +Other Chrome Mobile 77.0 Android 8.1.0 +Other Chrome Mobile 77.0 Android 8.1.0 +LG-Q710AL Chrome Mobile 77.0 Android 8.1.0 +Other Chrome Mobile 77.0 Android 8.1.0 +Other Facebook 235.0 Android 8.1.0 +Other Chrome Mobile 70.0 Android 8.1.0 +Other Chrome Mobile 76.0 Android 8.1.0 +Other Chrome Mobile 76.0 Android 8.1.0 +Other Chrome Mobile 77.0 Android 8.1.0 +Other Chrome Mobile 77.0 Android 8.1.0 +Other UC Browser 11.6 Android 8.1.0 +Other Chrome Mobile 70.0 Android 8.1.0 +Other Chrome Mobile 72.0 Android 8.1.0 +Other Chrome Mobile 77.0 Android 8.1.0 +Other Chrome Mobile 77.0 Android 8.1.0 +Other Chrome Mobile 70.0 Android 8.1.0 +Other Chrome Mobile 76.0 Android 8.1.0 +Other Chrome Mobile WebView 77.0 Android 8.1.0 +Other Chrome Mobile 70.0 Android 8.1.0 +Other Chrome Mobile 76.0 Android 8.1.0 +Other Chrome Mobile 77.0 Android 8.1.0 +Other Chrome Mobile 68.0 Android 8.1.0 +Other Chrome Mobile 76.0 Android 8.1.0 +Other Chrome Mobile 77.0 Android 8.1.0 +Other Chrome Mobile 77.0 Android 8.1.0 +HTC One Chrome Mobile WebView 70.0 Android 8.1.0 +RCT6873W42BMF8KC Chrome Mobile 77.0 Android 8.1.0 +Other Chrome Mobile 67.0 Android 8.1.0 +Other Chrome Mobile 76.0 Android 8.1.0 +Samsung SM-J727T Samsung Internet 10.1 Android 8.1.0 +Samsung SM-J727T1 Samsung Internet 9.4 Android 8.1.0 +Samsung SM-J727T1 Samsung Internet 10.1 Android 8.1.0 +Samsung SM-T580 Samsung Internet 9.4 Android 8.1.0 +Samsung SM-J727A Facebook 240.0 Android 8.1.0 +Samsung SM-G610F Chrome Mobile 77.0 Android 8.1.0 +Samsung SM-J260T1 Chrome Mobile 76.0 Android 8.1.0 +Samsung SM-J260T1 Chrome Mobile 76.0 Android 8.1.0 +Samsung SM-J260T1 Chrome Mobile 77.0 Android 8.1.0 +Samsung SM-J410F Chrome Mobile 77.0 Android 8.1.0 +Samsung SM-J727P Chrome Mobile 68.0 Android 8.1.0 +Samsung SM-J727T Chrome Mobile 66.0 Android 8.1.0 +Samsung SM-J727T1 Chrome Mobile 76.0 Android 8.1.0 +Samsung SM-J727T1 Chrome Mobile 77.0 Android 8.1.0 +Samsung SM-J727T1 Chrome Mobile 77.0 Android 8.1.0 +Samsung SM-J727V Chrome Mobile 70.0 Android 8.1.0 +Samsung SM-J727V Chrome Mobile 77.0 Android 8.1.0 +Other Chrome 77.0 Android 8.1.0 +Other Chrome 75.0 Android 8.1.0 +Other Edge Mobile 42.0 Android 8.1.0 +Other Chrome 76.0 Android 8.1.0 +Other Chrome 76.0 Android 8.1.0 +Other Chrome 77.0 Android 8.1.0 +Samsung SM-T837T Chrome 77.0 Android 8.1.0 +Other Facebook 239.0 Android 8.1.0 +Other Chrome Mobile 75.0 Android 8.1.0 +Other Chrome Mobile 68.0 Android 8.1.0 +Other Chrome Mobile 76.0 Android 8.1.0 +Other Chrome Mobile 77.0 Android 8.1.0 +Other DuckDuckGo Mobile 5.0 Android 9.0.0 +Other Chrome Mobile 70.0 Android 9.0.0 +Other Facebook 236.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 9.0.0 +Huawei CLT-L04 Chrome Mobile 77.0 Android 9.0.0 +Oppo CPH1911 Facebook 239.0 Android 9.0.0 +Oppo CPH1923 Chrome Mobile WebView 76.0 Android 9.0.0 +Huawei ELE-L29 Chrome Mobile 77.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 9.0.0 +Other Chrome Mobile 76.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 9.0.0 +Huawei INE-LX2 Chrome Mobile 76.0 Android 9.0.0 +Other Chrome Mobile WebView 77.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 9.0.0 +Other Chrome Mobile WebView 77.0 Android 9.0.0 +Other Chrome Mobile 76.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 9.0.0 +Other Chrome Mobile 72.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 9.0.0 +XiaoMi MI 9 Chrome Mobile 77.0 Android 9.0.0 +XiaoMi Mi A2 Chrome Mobile 77.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 9.0.0 +Nokia 6 Chrome Mobile 77.0 Android 9.0.0 +OnePlus ONEPLUS A6000 Chrome Mobile 77.0 Android 9.0.0 +OnePlus ONEPLUS A6003 Chrome Mobile 77.0 Android 9.0.0 +OnePlus ONEPLUS A6013 Chrome Mobile WebView 77.0 Android 9.0.0 +OnePlus ONEPLUS A6013 Chrome Mobile 74.0 Android 9.0.0 +OnePlus ONEPLUS A6013 Chrome Mobile 77.0 Android 9.0.0 +Other Facebook 235.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 9.0.0 +Other Chrome Mobile WebView 77.0 Android 9.0.0 +Other Chrome Mobile 76.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 9.0.0 +Other Chrome Mobile 73.0 Android 9.0.0 +Oppo RMX1801 Chrome Mobile 75.0 Android 9.0.0 +XiaoMi Redmi 7 Chrome Mobile 77.0 Android 9.0.0 +XiaoMi Redmi Note 7 Chrome Mobile 76.0 Android 9.0.0 +Samsung SM-A102U Samsung Internet 10.1 Android 9.0.0 +Samsung SM-A505FN Samsung Internet 10.1 Android 9.0.0 +Samsung SM-A505GN Samsung Internet 10.1 Android 9.0.0 +Samsung SM-G892U Samsung Internet 10.1 Android 9.0.0 +Samsung SM-G950U Samsung Internet 10.1 Android 9.0.0 +Samsung SM-G955F Samsung Internet 9.4 Android 9.0.0 +Samsung SM-G955U Samsung Internet 10.1 Android 9.0.0 +Samsung SM-G9600 Samsung Internet 9.4 Android 9.0.0 +Samsung SM-G960U Samsung Internet 10.1 Android 9.0.0 +Samsung SM-G965U Samsung Internet 10.1 Android 9.0.0 +Samsung SM-G970F Samsung Internet 10.1 Android 9.0.0 +Samsung SM-G970U Samsung Internet 10.1 Android 9.0.0 +Samsung SM-G973U Samsung Internet 9.4 Android 9.0.0 +Samsung SM-G973U Samsung Internet 10.1 Android 9.0.0 +Samsung SM-G975U Samsung Internet 10.1 Android 9.0.0 +Samsung SM-J415F Samsung Internet 10.1 Android 9.0.0 +Samsung SM-J730F Samsung Internet 10.1 Android 9.0.0 +Samsung SM-J737P Samsung Internet 10.1 Android 9.0.0 +Samsung SM-J737T Samsung Internet 9.0 Android 9.0.0 +Samsung SM-N950U Samsung Internet 10.1 Android 9.0.0 +Samsung SM-N960F Samsung Internet 10.1 Android 9.0.0 +Samsung SM-N960U Samsung Internet 10.1 Android 9.0.0 +Samsung SM-N960U1 Samsung Internet 9.2 Android 9.0.0 +Samsung SM-N970U Samsung Internet 10.1 Android 9.0.0 +Samsung SM-N975U Samsung Internet 10.1 Android 9.0.0 +Samsung SM-N975U1 Samsung Internet 10.1 Android 9.0.0 +Samsung SM-T510 Samsung Internet 10.1 Android 9.0.0 +Samsung SM-T720 Samsung Internet 10.1 Android 9.0.0 +Other Chrome Mobile WebView 77.0 Android 9.0.0 +Samsung SM-A102U Chrome Mobile 72.0 Android 9.0.0 +Samsung SM-A102U Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-A105M Facebook 237.0 Android 9.0.0 +Samsung SM-A205G Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-A205U Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-A505F Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-A530F Facebook 240.0 Android 9.0.0 +Samsung SM-A530N Chrome Mobile WebView 77.0 Android 9.0.0 +Samsung SM-A600T Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-A605F Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-A920F Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G892A Chrome Mobile 74.0 Android 9.0.0 +Samsung SM-G950F Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G950U Chrome Mobile WebView 77.0 Android 9.0.0 +Samsung SM-G950U Chrome Mobile 71.0 Android 9.0.0 +Samsung SM-G950U Chrome Mobile 76.0 Android 9.0.0 +Samsung SM-G950U Chrome Mobile 76.0 Android 9.0.0 +Samsung SM-G950U Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G950U1 Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G955F Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G955U Facebook 240.0 Android 9.0.0 +Samsung SM-G955U Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G9600 Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G960U Facebook 233.0 Android 9.0.0 +Samsung SM-G960U Chrome Mobile WebView 77.0 Android 9.0.0 +Samsung SM-G960U Chrome Mobile 71.0 Android 9.0.0 +Samsung SM-G960U Chrome Mobile 74.0 Android 9.0.0 +Samsung SM-G960U Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G960U1 Facebook 240.0 Android 9.0.0 +Samsung SM-G960U1 Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G965F Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G965U Chrome Mobile 74.0 Android 9.0.0 +Samsung SM-G965U Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G965U Chrome Mobile 79.0 Android 9.0.0 +Samsung SM-G965U1 Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G970U Facebook 240.0 Android 9.0.0 +Samsung SM-G970U Chrome Mobile 75.0 Android 9.0.0 +Samsung SM-G970U Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G970U1 Chrome Mobile WebView 77.0 Android 9.0.0 +Samsung SM-G973U Chrome Mobile 74.0 Android 9.0.0 +Samsung SM-G973U Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G973U1 Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G975U Chrome Mobile 75.0 Android 9.0.0 +Samsung SM-G975U Chrome Mobile 76.0 Android 9.0.0 +Samsung SM-G975U Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-G975U1 Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-J260A Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-J337P Chrome Mobile 76.0 Android 9.0.0 +Samsung SM-J600FN Chrome Mobile 75.0 Android 9.0.0 +Samsung SM-J600G Facebook 238.0 Android 9.0.0 +Samsung SM-J730F Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-J737A Chrome Mobile WebView 77.0 Android 9.0.0 +Samsung SM-J737A Chrome Mobile 74.0 Android 9.0.0 +Samsung SM-J737V Pinterest 75.0 Android 9.0.0 +Samsung SM-J737V Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-J810M Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-N950U Facebook 240.0 Android 9.0.0 +Samsung SM-N950U Chrome Mobile 72.0 Android 9.0.0 +Samsung SM-N950U Chrome Mobile 75.0 Android 9.0.0 +Samsung SM-N950U Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-N950U Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-N960F Chrome Mobile 76.0 Android 9.0.0 +Samsung SM-N960U Facebook 240.0 Android 9.0.0 +Samsung SM-N960U Chrome Mobile WebView 77.0 Android 9.0.0 +Samsung SM-N960U Chrome Mobile 74.0 Android 9.0.0 +Samsung SM-N960U Chrome Mobile 75.0 Android 9.0.0 +Samsung SM-N960U Chrome Mobile 76.0 Android 9.0.0 +Samsung SM-N960U Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-N960U1 Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-N975U Chrome Mobile WebView 77.0 Android 9.0.0 +Samsung SM-N975U Chrome Mobile WebView 77.0 Android 9.0.0 +Samsung SM-N975U Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-N976V Facebook 240.0 Android 9.0.0 +Samsung SM-S367VL Chrome Mobile 77.0 Android 9.0.0 +Samsung SM-S767VL Chrome Mobile 76.0 Android 9.0.0 +Samsung SM-T597P Chrome 77.0 Android 9.0.0 +Other Chrome 77.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 9.0.0 +Huawei VOG-L29 Chrome Mobile 77.0 Android 9.0.0 +Other Chrome Mobile 74.0 Android 9.0.0 +Other Chrome Mobile WebView 77.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 9.0.0 +Other Facebook 235.0 Android 9.0.0 +Other Chrome Mobile 70.0 Android 9.0.0 +Other Chrome Mobile 75.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 9.0.0 +Other Chrome Mobile 73.0 Android 9.0.0 +Other Chrome Mobile 77.0 Android 9.0.0 +Samsung GT-P3113 Android 4.1 Android 4.1.1 +Samsung GT-I8160 Android 4.1 Android 4.1.2 +Other Android 4.2 Android 4.2.2 +Samsung SM-E500H Android 4.4 Android 4.4.0 +Other Chrome Mobile WebView 43.0 Android 6.0.1 +Samsung SM-J737T1 Chrome Mobile WebView 43.0 Android 6.0.1 +Other Opera Mobile 5.3 Android 7.0.0 +XiaoMi Redmi 5A MiuiBrowser 9.5 Android 7.1.2 +Oppo CPH1911 Chrome Mobile WebView 70.0 Android 9.0.0 +Other Opera Mobile 44.1 Android 9.0.0 +Mac Firefox 68.0 Mac OS X 10.11.0 +Mac Firefox 69.0 Mac OS X 10.13.0 +Mac Firefox 67.0 Mac OS X 10.14.0 +Mac Firefox 68.0 Mac OS X 10.14.0 +Mac Firefox 69.0 Mac OS X 10.14.0 +Mac Firefox 70.0 Mac OS X 10.14.0 +Mac Chrome 76.0 Mac OS X 10.10.5 +Mac Chrome 77.0 Mac OS X 10.10.5 +Mac Safari 10.1 Mac OS X 10.10.5 +Mac Chrome 76.0 Mac OS X 10.11.4 +Mac Chrome 72.0 Mac OS X 10.11.6 +Mac Chrome 76.0 Mac OS X 10.11.6 +Mac Chrome 76.0 Mac OS X 10.11.6 +Mac Chrome 77.0 Mac OS X 10.11.6 +Mac Safari 9.1 Mac OS X 10.11.6 +Mac Safari 10.0 Mac OS X 10.11.6 +Mac Safari 11.1 Mac OS X 10.11.6 +Mac Chrome 77.0 Mac OS X 10.12.1 +Mac Safari 10.0 Mac OS X 10.12.3 +Mac Chrome 75.0 Mac OS X 10.12.6 +Mac Chrome 76.0 Mac OS X 10.12.6 +Mac Chrome 76.0 Mac OS X 10.12.6 +Mac Chrome 77.0 Mac OS X 10.12.6 +Mac Safari 12.1 Mac OS X 10.12.6 +Mac Safari 11.0 Mac OS X 10.13.0 +Mac Chrome 77.0 Mac OS X 10.13.1 +Mac Chrome 77.0 Mac OS X 10.13.2 +Mac Chrome 76.0 Mac OS X 10.13.4 +Mac Chrome 76.0 Mac OS X 10.13.4 +Mac Chrome 76.0 Mac OS X 10.13.5 +Mac Chrome 75.0 Mac OS X 10.13.6 +Mac Chrome 76.0 Mac OS X 10.13.6 +Mac Chrome 77.0 Mac OS X 10.13.6 +Mac Safari 12.0 Mac OS X 10.13.6 +Mac Safari 12.1 Mac OS X 10.13.6 +Mac Safari 12.1 Mac OS X 10.13.6 +Mac Safari 13.0 Mac OS X 10.13.6 +Mac Safari 13.0 Mac OS X 10.13.6 +Mac Chrome 75.0 Mac OS X 10.14.0 +Mac Chrome 76.0 Mac OS X 10.14.0 +Mac Chrome 77.0 Mac OS X 10.14.0 +Mac Chrome 77.0 Mac OS X 10.14.1 +Mac Chrome 76.0 Mac OS X 10.14.2 +Mac Chrome 69.0 Mac OS X 10.14.3 +Mac Safari 12.0 Mac OS X 10.14.3 +Mac Chrome 75.0 Mac OS X 10.14.4 +Mac Chrome 77.0 Mac OS X 10.14.4 +Mac Safari 12.1 Mac OS X 10.14.4 +Mac Chrome 76.0 Mac OS X 10.14.5 +Mac Chrome 77.0 Mac OS X 10.14.5 +Mac Safari 12.1 Mac OS X 10.14.5 +Mac Chrome 75.0 Mac OS X 10.14.6 +Mac Chrome 76.0 Mac OS X 10.14.6 +Mac Chrome 76.0 Mac OS X 10.14.6 +Mac Chrome 77.0 Mac OS X 10.14.6 +Mac Chrome 77.0 Mac OS X 10.14.6 +Mac Safari 12.1 Mac OS X 10.14.6 +Mac Safari 13.0 Mac OS X 10.14.6 +Mac Chrome 65.0 Mac OS X 10.9.5 +Mac Chrome 66.0 Mac OS X 10.9.5 +Mac Chrome 67.0 Mac OS X 10.9.5 +PlayStation 4 Apple Mail 605.1 Other 0.0.0 +Other Safari 3.0 Tizen 3.0.0 +Other Samsung Internet 2.0 Tizen 3.0.0 +Other Samsung Internet 2.1 Tizen 4.0.0 +Other Samsung Internet 2.2 Tizen 5.0.0 +Other Edge 17.17134 Windows 10.0.0 +Other Edge 18.17763 Windows 10.0.0 +Other Chrome 77.0 Windows 10.0.0 +Other Maxthon 5.2 Windows 10.0.0 +Other Chrome 73.1 Windows 10.0.0 +Other Chrome 76.0 Windows 10.0.0 +Other Opera 63.0 Windows 10.0.0 +Other Chrome 77.0 Windows 10.0.0 +Other Chrome 77.0 Windows 10.0.0 +Other Coc Coc 82.0 Windows 10.0.0 +Other IE 11.0 Windows 10.0.0 +Other Firefox 59.0 Windows 10.0.0 +Other Firefox 60.0 Windows 10.0.0 +Other Edge 15.15063 Windows 10.0.0 +Other Edge 16.16299 Windows 10.0.0 +Other Edge 17.17134 Windows 10.0.0 +Other Edge 18.17763 Windows 10.0.0 +Other Chrome 65.0 Windows 10.0.0 +Other Chrome 70.0 Windows 10.0.0 +Other Edge 18.18362 Windows 10.0.0 +Other Edge 18.18995 Windows 10.0.0 +Other Edge 18.19493 Windows 10.0.0 +Other Chrome 70.0 Windows 10.0.0 +Other Chrome 71.0 Windows 10.0.0 +Other Chrome 73.0 Windows 10.0.0 +Other Chrome 74.0 Windows 10.0.0 +Other Chrome 75.0 Windows 10.0.0 +Other Chrome 76.0 Windows 10.0.0 +Other Vivaldi 2.7 Windows 10.0.0 +Other Chrome 76.0 Windows 10.0.0 +Other Opera 63.0 Windows 10.0.0 +Other Chrome 77.0 Windows 10.0.0 +Other Chrome 77.0 Windows 10.0.0 +Other Edge 79.0 Windows 10.0.0 +Other Edge 18.18362 Windows 10.0.0 +Other Edge 18.18363 Windows 10.0.0 +Other Edge 18.18362 Windows 10.0.0 +Other Firefox 61.0 Windows 10.0.0 +Other Firefox 63.0 Windows 10.0.0 +Other Firefox 67.0 Windows 10.0.0 +Other Firefox 68.0 Windows 10.0.0 +Other Firefox 69.0 Windows 10.0.0 +Other Firefox 69.0 Windows 10.0.0 +Other Chrome 49.0 Windows XP.0.0 +Other Chrome 49.0 Windows Vista.0.0 +Other Chrome 49.0 Windows Vista.0.0 +Other Chrome 76.0 Windows 7.0.0 +Other Chrome 77.0 Windows 7.0.0 +Other Chrome 77.0 Windows 7.0.0 +Other Coc Coc 80.0 Windows 7.0.0 +Other Coc Coc 82.0 Windows 7.0.0 +Other IE 11.0 Windows 7.0.0 +Other Chrome 67.0 Windows 7.0.0 +Other Chrome 70.0 Windows 7.0.0 +Other Chrome 72.0 Windows 7.0.0 +Other Chrome 74.0 Windows 7.0.0 +Other Chrome 75.0 Windows 7.0.0 +Other Chrome 76.0 Windows 7.0.0 +Other Chrome 76.0 Windows 7.0.0 +Other Chrome 77.0 Windows 7.0.0 +Other Waterfox 56.2 Windows 7.0.0 +Other Firefox 60.0 Windows 7.0.0 +Other Firefox 63.0 Windows 7.0.0 +Other Firefox 68.0 Windows 7.0.0 +Other Firefox 69.0 Windows 7.0.0 +Other Firefox 69.0 Windows 7.0.0 +Other Chrome 77.0 Windows 8.0.0 +Other Firefox 69.0 Windows 8.0.0 +Other Chrome 77.0 Windows 8.1.0 +Other IE 11.0 Windows RT 8.1.0 +Other IE 11.0 Windows 8.1.0 +Other IE 11.0 Windows 8.1.0 +Other Chrome 63.0 Windows 8.1.0 +Other Chrome 64.0 Windows 8.1.0 +Other Chrome 76.0 Windows 8.1.0 +Other Chrome 76.0 Windows 8.1.0 +Other Chrome 77.0 Windows 8.1.0 +Other Firefox 69.0 Windows 8.1.0 +Other Firefox 69.0 Windows 8.1.0 +Other Chrome 72.0 Windows 10.0.0 +Other Chrome 77.0 Chrome OS 12371.75.0 +Other Chrome 76.0 Chrome OS 12239.92.0 +Other Chrome 69.0 Chrome OS 10895.78.0 +Other Chrome 70.0 Chrome OS 11021.81.0 +Other Chrome 74.0 Chrome OS 11895.118.0 +Other Chrome 76.0 Chrome OS 12239.92.0 +Other Chrome 76.0 Chrome OS 12239.92.1 +Other Chrome 76.0 Chrome OS 12239.92.4 +Other Chrome 77.0 Chrome OS 12371.46.0 +Other Chrome 77.0 Chrome OS 12371.65.0 +Other Chrome 75.0 Linux 0.0.0 +Other Chrome 77.0 Linux 0.0.0 +Other Samsung Internet 10.1 Linux 0.0.0 +Other Chrome 66.0 Linux 0.0.0 +Other Chrome 66.0 Linux 0.0.0 +Other Chrome 66.0 Linux 0.0.0 +Other Chrome 66.0 Linux 0.0.0 +Other Chrome 66.0 Linux 0.0.0 +Other Firefox 65.0 Ubuntu 0.0.0 +Other Firefox 66.0 Ubuntu 0.0.0 +Other Firefox 67.0 Ubuntu 0.0.0 +iPad Google 22.0 iOS 10.3.3 +iPad Chrome Mobile iOS 71.0 iOS 10.3.3 +iPad Firefox iOS 14.0 iOS 10.3.3 +iPad Mobile Safari UI/WKWebView 603.3 iOS 10.3.3 +iPad Facebook 240.0 iOS 10.3.3 +iPad Mobile Safari 10.0 iOS 10.3.3 +iPad Mobile Safari 10.0 iOS 10.3.4 +iPad Chrome Mobile iOS 76.0 iOS 11.1.0 +iPad Chrome Mobile iOS 76.0 iOS 11.1.2 +iPad Mobile Safari 11.0 iOS 11.2.1 +iPad Mobile Safari 11.0 iOS 11.2.2 +iPad Mobile Safari 11.0 iOS 11.2.6 +iPad Mobile Safari 11.0 iOS 11.3.0 +iPad Mobile Safari 11.0 iOS 11.4.0 +iPad Mobile Safari UI/WKWebView 605.1 iOS 11.4.1 +iPad Mobile Safari 11.0 iOS 11.4.1 +iPad Google 83.0 iOS 12.0.0 +iPad Mobile Safari 12.0 iOS 12.0.0 +iPad Chrome Mobile iOS 75.0 iOS 12.1.0 +iPad Chrome Mobile iOS 76.0 iOS 12.1.0 +iPad Mobile Safari UI/WKWebView 605.1 iOS 12.1.0 +iPad Mobile Safari 12.0 iOS 12.1.0 +iPad Mobile Safari 12.0 iOS 12.1.1 +iPad Google 48.0 iOS 12.1.4 +iPad Mobile Safari UI/WKWebView 605.1 iOS 12.1.4 +iPad Mobile Safari 12.0 iOS 12.1.4 +iPad Chrome Mobile iOS 76.0 iOS 12.2.0 +iPad Mobile Safari UI/WKWebView 605.1 iOS 12.2.0 +iPad Mobile Safari 12.1 iOS 12.2.0 +iPad Chrome Mobile iOS 77.0 iOS 12.3.0 +iPad Google 83.0 iOS 12.3.0 +iPad Mobile Safari 12.1 iOS 12.3.0 +iPad Mobile Safari UI/WKWebView 605.1 iOS 12.3.1 +iPad Mobile Safari 12.1 iOS 12.3.1 +iPad Chrome Mobile iOS 76.0 iOS 12.4.0 +iPad Chrome Mobile iOS 76.0 iOS 12.4.0 +iPad Chrome Mobile iOS 77.0 iOS 12.4.0 +iPad Chrome Mobile iOS 77.0 iOS 12.4.0 +iPad Chrome Mobile iOS 77.0 iOS 12.4.0 +iPad Google 74.0 iOS 12.4.0 +iPad Google 83.0 iOS 12.4.0 +iPad Mobile Safari UI/WKWebView 605.1 iOS 12.4.0 +iPad Mobile Safari 12.1 iOS 12.4.0 +iPad Chrome Mobile iOS 67.0 iOS 12.4.1 +iPad Firefox iOS 19.0 iOS 12.4.1 +iPad Mobile Safari UI/WKWebView 605.1 iOS 12.4.1 +iPad Facebook 605.1 iOS 12.4.1 +iPad Facebook 605.1 iOS 12.4.1 +iPad Facebook 605.1 iOS 12.4.1 +iPad Facebook 605.1 iOS 12.4.1 +iPad Mobile Safari 12.1 iOS 12.4.1 +iPad Mobile Safari 6.0 iOS 6.1.3 +iPad Mobile Safari 8.0 iOS 8.0.0 +iPad Mobile Safari 8.0 iOS 8.2.0 +iPad Google 23.1 iOS 8.4.0 +iPad Mobile Safari 9.0 iOS 9.3.2 +iPad Mobile Safari 9.0 iOS 9.3.5 +iPhone Mobile Safari 10.0 iOS 10.2.0 +iPhone Facebook 603.3 iOS 10.3.3 +iPhone Google 68.0 iOS 10.3.4 +iPhone Mobile Safari 10.0 iOS 10.3.4 +iPhone Mobile Safari 11.0 iOS 11.0.3 +iPhone Mobile Safari 11.0 iOS 11.1.1 +iPhone Mobile Safari 11.0 iOS 11.1.2 +iPhone Mobile Safari 11.0 iOS 11.2.1 +iPhone Facebook 207.0 iOS 11.2.6 +iPhone Chrome Mobile iOS 76.0 iOS 11.3.0 +iPhone Facebook 605.1 iOS 11.3.0 +iPhone Mobile Safari 11.0 iOS 11.3.0 +iPhone Google 83.0 iOS 11.4.0 +iPhone Mobile Safari 11.0 iOS 11.4.0 +iPhone Google 74.1 iOS 11.4.1 +iPhone Mobile Safari 11.0 iOS 11.4.1 +iPhone Mobile Safari 12.0 iOS 12.0.0 +iPhone Mobile Safari 12.0 iOS 12.1.0 +iPhone Mobile Safari 12.0 iOS 12.1.1 +iPhone Google 74.1 iOS 12.1.2 +iPhone Facebook 605.1 iOS 12.1.2 +iPhone Mobile Safari 12.0 iOS 12.1.2 +iPhone Mobile Safari 12.0 iOS 12.1.3 +iPhone Google 74.1 iOS 12.1.4 +iPhone Mobile Safari 12.0 iOS 12.1.4 +iPhone Chrome Mobile iOS 72.0 iOS 12.2.0 +iPhone Chrome Mobile iOS 76.0 iOS 12.2.0 +iPhone Chrome Mobile iOS 77.0 iOS 12.2.0 +iPhone Facebook 605.1 iOS 12.2.0 +iPhone Facebook 605.1 iOS 12.2.0 +iPhone Mobile Safari 12.1 iOS 12.2.0 +iPhone Chrome Mobile iOS 77.0 iOS 12.3.0 +iPhone Google 83.0 iOS 12.3.0 +iPhone Mobile Safari 12.1 iOS 12.3.0 +iPhone Google 79.0 iOS 12.3.1 +iPhone Mobile Safari UI/WKWebView 605.1 iOS 12.3.1 +iPhone DuckDuckGo Mobile 7.1 iOS 12.3.1 +iPhone Facebook 605.1 iOS 12.3.1 +iPhone Facebook 605.1 iOS 12.3.1 +iPhone Facebook 605.1 iOS 12.3.1 +iPhone Mobile Safari 12.1 iOS 12.3.1 +iPhone Mobile Safari 12.1 iOS 12.3.2 +iPhone Chrome Mobile iOS 69.0 iOS 12.4.0 +iPhone Chrome Mobile iOS 73.0 iOS 12.4.0 +iPhone Chrome Mobile iOS 75.0 iOS 12.4.0 +iPhone Chrome Mobile iOS 76.0 iOS 12.4.0 +iPhone Chrome Mobile iOS 77.0 iOS 12.4.0 +iPhone Chrome Mobile iOS 77.0 iOS 12.4.0 +iPhone Google 81.0 iOS 12.4.0 +iPhone Google 82.1 iOS 12.4.0 +iPhone Google 83.0 iOS 12.4.0 +iPhone Facebook 605.1 iOS 12.4.0 +iPhone Facebook 605.1 iOS 12.4.0 +iPhone Facebook 605.1 iOS 12.4.0 +iPhone Facebook 605.1 iOS 12.4.0 +iPhone Mobile Safari 12.1 iOS 12.4.0 +iPhone Google 74.1 iOS 12.4.1 +iPhone Mobile Safari UI/WKWebView 605.1 iOS 12.4.1 +iPhone Instagram 89.0 iOS 12.4.1 +iPhone Facebook 240.0 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 605.1 iOS 12.4.1 +iPhone Mobile Safari 12.1 iOS 12.4.1 +iPhone Mobile Safari 12.4 iOS 12.4.1 +iPhone Mobile Safari UI/WKWebView 605.1 iOS 12.4.2 +iPhone Mobile Safari 12.1 iOS 12.4.2 +iPhone Chrome Mobile iOS 77.0 iOS 13.0.0 +iPhone Facebook 605.1 iOS 13.0.0 +iPhone Facebook 605.1 iOS 13.0.0 +iPhone Facebook 605.1 iOS 13.0.0 +iPhone Facebook 605.1 iOS 13.0.0 +iPhone Facebook 605.1 iOS 13.0.0 +iPhone Facebook 605.1 iOS 13.0.0 +iPhone Mobile Safari 13.0 iOS 13.0.0 +iPhone Chrome Mobile iOS 76.0 iOS 13.1.0 +iPhone Chrome Mobile iOS 77.0 iOS 13.1.0 +iPhone Chrome Mobile iOS 77.0 iOS 13.1.0 +iPhone Firefox iOS 8.1 iOS 13.1.0 +iPhone Google 83.0 iOS 13.1.0 +iPhone Mobile Safari UI/WKWebView 605.1 iOS 13.1.0 +iPhone DuckDuckGo Mobile 7.1 iOS 13.1.0 +iPhone Facebook 605.1 iOS 13.1.0 +iPhone Facebook 605.1 iOS 13.1.0 +iPhone Facebook 605.1 iOS 13.1.0 +iPhone Facebook 605.1 iOS 13.1.0 +iPhone Facebook 605.1 iOS 13.1.0 +iPhone Mobile Safari 13.0 iOS 13.1.0 +iPhone Mobile Safari UI/WKWebView 605.1 iOS 13.1.1 +iPhone Facebook 605.1 iOS 13.1.1 +iPhone Facebook 605.1 iOS 13.1.1 +iPhone Facebook 605.1 iOS 13.1.1 +iPhone Facebook 605.1 iOS 13.1.1 +iPhone Mobile Safari 13.0 iOS 13.1.1 +iPhone Mobile Safari UI/WKWebView 605.1 iOS 13.1.2 +iPhone Facebook 605.1 iOS 13.1.2 +iPhone Mobile Safari 13.0 iOS 13.1.2 diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh new file mode 100755 index 00000000000..f783a0db60f --- /dev/null +++ b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +# Tags: use-vectorscan, no-fasttest, no-parallel + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +mkdir -p $user_files_path/test_02504 + +cp $CURDIR/data_ua_parser/os.yaml ${user_files_path}/test_02504/ +cp $CURDIR/data_ua_parser/browser.yaml ${user_files_path}/test_02504/ +cp $CURDIR/data_ua_parser/device.yaml ${user_files_path}/test_02504/ + +$CLICKHOUSE_CLIENT -n --query=" +drop dictionary if exists regexp_os; +drop dictionary if exists regexp_browser; +drop dictionary if exists regexp_device; +drop table if exists user_agents; +create dictionary regexp_os +( + regex String, + os_replacement String default 'Other', + os_v1_replacement String default '0', + os_v2_replacement String default '0', + os_v3_replacement String default '0', + os_v4_replacement String default '0' +) +PRIMARY KEY(regex) +SOURCE(YAMLRegExpTree(PATH '${user_files_path}/test_02504/os.yaml')) +LIFETIME(0) +LAYOUT(regexp_tree); + +create dictionary regexp_browser +( + regex String, + family_replacement String default 'Other', + v1_replacement String default '0', + v2_replacement String default '0' +) +PRIMARY KEY(regex) +SOURCE(YAMLRegExpTree(PATH '${user_files_path}/test_02504/browser.yaml')) +LIFETIME(0) +LAYOUT(regexp_tree); + +create dictionary regexp_device +( + regex String, + device_replacement String default 'Other', + brand_replacement String, + model_replacement String +) +PRIMARY KEY(regex) +SOURCE(YAMLRegExpTree(PATH '${user_files_path}/test_02504/device.yaml')) +LIFETIME(0) +LAYOUT(regexp_tree); + +create table user_agents +( + ua String +) +Engine = Log(); +" + +$CLICKHOUSE_CLIENT -n --query=" +insert into user_agents select ua from input('ua String') FORMAT LineAsString" < $CURDIR/data_ua_parser/useragents.txt + +$CLICKHOUSE_CLIENT -n --query=" +select device, +concat(tupleElement(browser, 1), ' ', tupleElement(browser, 2), '.', tupleElement(browser, 3)) as browser , +concat(tupleElement(os, 1), ' ', tupleElement(os, 2), '.', tupleElement(os, 3), '.', tupleElement(os, 4)) as os +from ( + select dictGet('regexp_os', ('os_replacement', 'os_v1_replacement', 'os_v2_replacement', 'os_v3_replacement'), ua) os, + dictGet('regexp_browser', ('family_replacement', 'v1_replacement', 'v2_replacement'), ua) as browser, + dictGet('regexp_device', 'device_replacement', ua) device from user_agents); +" + +$CLICKHOUSE_CLIENT -n --query=" +drop dictionary if exists regexp_os; +drop dictionary if exists regexp_browser; +drop dictionary if exists regexp_device; +drop table if exists user_agents; +" + +rm -rf "$USER_FILES_PATH/test_02504" diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.reference b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.reference index d4e23bf981c..6d5372cc484 100644 --- a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.reference +++ b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.reference @@ -1,5 +1,5 @@ ('TencentOS','123.45.67') -('Andriod','12') -('Andriod','default') -('Andriod','default') +('Android','12') +('Android','default') +('Android','default') (true,'61f0c404-5cb3-11e7-907b-a6006ad3dba0','2023-01-01','2023-01-01 01:01:01',[1,2,3,-1,-2,-3]) diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh index f09409bb90b..8d80f9f1fea 100755 --- a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh +++ b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh @@ -18,7 +18,7 @@ cat > "$yaml" < "$yaml" < Date: Wed, 25 Jan 2023 23:57:20 +0100 Subject: [PATCH 20/79] remove trivial logs --- src/Dictionaries/RegExpTreeDictionary.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index 21c61173e92..e6321e26a7b 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -252,7 +252,6 @@ void RegExpTreeDictionary::initRegexNodes(Block & block) if (checker.check(regex)) { complex_regexp_nodes.push_back(node); - LOG_INFO(logger, "Found complex regexp {} in the config. This will slow down the queries", regex); } else { From 2f172a3f8befe4ce649fbd9b3e2b1fde6cd7b93d Mon Sep 17 00:00:00 2001 From: Han Fei Date: Thu, 26 Jan 2023 09:07:34 +0100 Subject: [PATCH 21/79] fix check style --- tests/metadata/INFORMATION_SCHEMA.sql | 2 -- tests/metadata/information_schema.sql | 2 -- tests/metadata/system | 1 - tests/metadata/system.sql | 2 -- tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh | 2 +- 5 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 tests/metadata/INFORMATION_SCHEMA.sql delete mode 100644 tests/metadata/information_schema.sql delete mode 120000 tests/metadata/system delete mode 100644 tests/metadata/system.sql diff --git a/tests/metadata/INFORMATION_SCHEMA.sql b/tests/metadata/INFORMATION_SCHEMA.sql deleted file mode 100644 index 291582fd1eb..00000000000 --- a/tests/metadata/INFORMATION_SCHEMA.sql +++ /dev/null @@ -1,2 +0,0 @@ -ATTACH DATABASE INFORMATION_SCHEMA -ENGINE = Memory diff --git a/tests/metadata/information_schema.sql b/tests/metadata/information_schema.sql deleted file mode 100644 index 6cea934b49d..00000000000 --- a/tests/metadata/information_schema.sql +++ /dev/null @@ -1,2 +0,0 @@ -ATTACH DATABASE information_schema -ENGINE = Memory diff --git a/tests/metadata/system b/tests/metadata/system deleted file mode 120000 index 9a11b7e3458..00000000000 --- a/tests/metadata/system +++ /dev/null @@ -1 +0,0 @@ -/home/hanfei/ClickHouse/tests/store/069/0693727d-7be1-48e3-9ff3-aa0bba736b34/ \ No newline at end of file diff --git a/tests/metadata/system.sql b/tests/metadata/system.sql deleted file mode 100644 index 941c12b0a66..00000000000 --- a/tests/metadata/system.sql +++ /dev/null @@ -1,2 +0,0 @@ -ATTACH DATABASE _ UUID '0693727d-7be1-48e3-9ff3-aa0bba736b34' -ENGINE = Atomic diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh index f783a0db60f..4d372bf1ce6 100755 --- a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh +++ b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh @@ -84,4 +84,4 @@ drop dictionary if exists regexp_device; drop table if exists user_agents; " -rm -rf "$USER_FILES_PATH/test_02504" +rm -rf "$user_files_path/test_02504" From fb3f7409fae9824f2081b08887ba0fc486b532c8 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 26 Jan 2023 22:17:22 +0000 Subject: [PATCH 22/79] allow vertical merges from compact to wide parts --- src/Storages/MergeTree/IMergeTreeDataPart.h | 2 - src/Storages/MergeTree/MergeTask.cpp | 6 +-- .../MergeTree/MergeTreeDataPartWide.h | 2 - ...539_vertical_merge_compact_parts.reference | 2 + .../02539_vertical_merge_compact_parts.sql | 41 +++++++++++++++++++ 5 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/02539_vertical_merge_compact_parts.reference create mode 100644 tests/queries/0_stateless/02539_vertical_merge_compact_parts.sql diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 68d5147362b..7a44770a05d 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -116,8 +116,6 @@ public: virtual bool isStoredOnRemoteDiskWithZeroCopySupport() const = 0; - virtual bool supportsVerticalMerge() const { return false; } - /// NOTE: Returns zeros if column files are not found in checksums. /// Otherwise return information about column size on disk. ColumnSize getColumnSize(const String & column_name) const; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 35e1b82dae0..d5bba5d4ab5 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -955,10 +955,8 @@ MergeAlgorithm MergeTask::ExecuteAndFinalizeHorizontalPart::chooseMergeAlgorithm return MergeAlgorithm::Horizontal; if (ctx->need_remove_expired_values) return MergeAlgorithm::Horizontal; - - for (const auto & part : global_ctx->future_part->parts) - if (!part->supportsVerticalMerge()) - return MergeAlgorithm::Horizontal; + if (global_ctx->future_part->type != MergeTreeDataPartType::Wide) + return MergeAlgorithm::Horizontal; bool is_supported_storage = ctx->merging_params.mode == MergeTreeData::MergingParams::Ordinary || diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.h b/src/Storages/MergeTree/MergeTreeDataPartWide.h index 601bdff51a1..3596d6172fc 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -53,8 +53,6 @@ public: bool isStoredOnRemoteDiskWithZeroCopySupport() const override; - bool supportsVerticalMerge() const override { return true; } - String getFileNameForColumn(const NameAndTypePair & column) const override; ~MergeTreeDataPartWide() override; diff --git a/tests/queries/0_stateless/02539_vertical_merge_compact_parts.reference b/tests/queries/0_stateless/02539_vertical_merge_compact_parts.reference new file mode 100644 index 00000000000..cb19dda2d60 --- /dev/null +++ b/tests/queries/0_stateless/02539_vertical_merge_compact_parts.reference @@ -0,0 +1,2 @@ +1 2 all_1_2_1 MergeParts Horizontal Compact +1 3 all_1_3_2 MergeParts Vertical Wide diff --git a/tests/queries/0_stateless/02539_vertical_merge_compact_parts.sql b/tests/queries/0_stateless/02539_vertical_merge_compact_parts.sql new file mode 100644 index 00000000000..45c7151b29b --- /dev/null +++ b/tests/queries/0_stateless/02539_vertical_merge_compact_parts.sql @@ -0,0 +1,41 @@ +DROP TABLE IF EXISTS t_compact_vertical_merge; + +CREATE TABLE t_compact_vertical_merge (id UInt64, s LowCardinality(String), arr Array(UInt64)) +ENGINE MergeTree ORDER BY id +SETTINGS + index_granularity = 16, + min_bytes_for_wide_part = 0, + min_rows_for_wide_part = 100, + vertical_merge_algorithm_min_rows_to_activate = 1, + vertical_merge_algorithm_min_columns_to_activate = 1; + +INSERT INTO t_compact_vertical_merge SELECT number, toString(number), range(number % 10) FROM numbers(40); +INSERT INTO t_compact_vertical_merge SELECT number, toString(number), range(number % 10) FROM numbers(40); + +OPTIMIZE TABLE t_compact_vertical_merge FINAL; +SYSTEM FLUSH LOGS; + +WITH splitByChar('_', part_name) AS name_parts, + name_parts[2]::UInt64 AS min_block, + name_parts[3]::UInt64 AS max_block +SELECT min_block, max_block, part_name, event_type, merge_algorithm, part_type FROM system.part_log +WHERE + database = currentDatabase() AND + table = 't_compact_vertical_merge' AND + min_block = 1 AND max_block = 2; + +INSERT INTO t_compact_vertical_merge SELECT number, toString(number), range(number % 10) FROM numbers(40); + +OPTIMIZE TABLE t_compact_vertical_merge FINAL; +SYSTEM FLUSH LOGS; + +WITH splitByChar('_', part_name) AS name_parts, + name_parts[2]::UInt64 AS min_block, + name_parts[3]::UInt64 AS max_block +SELECT min_block, max_block, part_name, event_type, merge_algorithm, part_type FROM system.part_log +WHERE + database = currentDatabase() AND + table = 't_compact_vertical_merge' AND + min_block = 1 AND max_block = 3; + +DROP TABLE t_compact_vertical_merge; From 9c1717b42d7897cd7f1c020cc3a967160307a6e7 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 27 Jan 2023 13:35:04 +0000 Subject: [PATCH 23/79] fix computation of granularity in vertical merges --- .../MergeTree/MergeTreeDataPartWriterOnDisk.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp index 1f40177d0fa..61cb9a4e7bd 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp @@ -135,6 +135,7 @@ static size_t computeIndexGranularityImpl( { size_t rows_in_block = block.rows(); size_t index_granularity_for_block; + if (!can_use_adaptive_index_granularity) { index_granularity_for_block = fixed_index_granularity_rows; @@ -143,7 +144,9 @@ static size_t computeIndexGranularityImpl( { size_t block_size_in_memory = block.bytes(); if (blocks_are_granules) + { index_granularity_for_block = rows_in_block; + } else if (block_size_in_memory >= index_granularity_bytes) { size_t granules_in_block = block_size_in_memory / index_granularity_bytes; @@ -155,10 +158,14 @@ static size_t computeIndexGranularityImpl( index_granularity_for_block = index_granularity_bytes / size_of_row_in_bytes; } } - /// We should be less or equal than fixed index granularity - index_granularity_for_block = std::min(fixed_index_granularity_rows, index_granularity_for_block); - /// very rare case when index granularity bytes less then single row + /// We should be less or equal than fixed index granularity. + /// But if block size is a granule size then do no ajust it. + /// Granularity greater than fixed granularity might come from compact part. + if (!blocks_are_granules) + index_granularity_for_block = std::min(fixed_index_granularity_rows, index_granularity_for_block); + + /// Very rare case when index granularity bytes less than single row. if (index_granularity_for_block == 0) index_granularity_for_block = 1; From e2324c9f153a1685629affc6470d389c5a049d45 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 27 Jan 2023 14:54:10 +0000 Subject: [PATCH 24/79] better test --- .../0_stateless/02539_vertical_merge_compact_parts.reference | 4 ++-- .../0_stateless/02539_vertical_merge_compact_parts.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02539_vertical_merge_compact_parts.reference b/tests/queries/0_stateless/02539_vertical_merge_compact_parts.reference index cb19dda2d60..685d3f3140d 100644 --- a/tests/queries/0_stateless/02539_vertical_merge_compact_parts.reference +++ b/tests/queries/0_stateless/02539_vertical_merge_compact_parts.reference @@ -1,2 +1,2 @@ -1 2 all_1_2_1 MergeParts Horizontal Compact -1 3 all_1_3_2 MergeParts Vertical Wide +1 2 MergeParts Horizontal Compact +1 3 MergeParts Vertical Wide diff --git a/tests/queries/0_stateless/02539_vertical_merge_compact_parts.sql b/tests/queries/0_stateless/02539_vertical_merge_compact_parts.sql index 45c7151b29b..4140a73b8bc 100644 --- a/tests/queries/0_stateless/02539_vertical_merge_compact_parts.sql +++ b/tests/queries/0_stateless/02539_vertical_merge_compact_parts.sql @@ -18,7 +18,7 @@ SYSTEM FLUSH LOGS; WITH splitByChar('_', part_name) AS name_parts, name_parts[2]::UInt64 AS min_block, name_parts[3]::UInt64 AS max_block -SELECT min_block, max_block, part_name, event_type, merge_algorithm, part_type FROM system.part_log +SELECT min_block, max_block, event_type, merge_algorithm, part_type FROM system.part_log WHERE database = currentDatabase() AND table = 't_compact_vertical_merge' AND @@ -32,7 +32,7 @@ SYSTEM FLUSH LOGS; WITH splitByChar('_', part_name) AS name_parts, name_parts[2]::UInt64 AS min_block, name_parts[3]::UInt64 AS max_block -SELECT min_block, max_block, part_name, event_type, merge_algorithm, part_type FROM system.part_log +SELECT min_block, max_block, event_type, merge_algorithm, part_type FROM system.part_log WHERE database = currentDatabase() AND table = 't_compact_vertical_merge' AND From eeef2dae7748107f1b62c83e654ff2ea84056ddc Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Sun, 29 Jan 2023 19:48:46 +0800 Subject: [PATCH 25/79] fix cores --- src/Analyzer/IQueryTreeNode.h | 3 + src/Analyzer/Passes/SumIfToCountIfPass.cpp | 45 +++++++++-- .../01646_rewrite_sum_if.reference | 81 +++++++++++++++++++ .../0_stateless/01646_rewrite_sum_if.sql | 6 ++ 4 files changed, 128 insertions(+), 7 deletions(-) diff --git a/src/Analyzer/IQueryTreeNode.h b/src/Analyzer/IQueryTreeNode.h index 8aa834e60b7..719f92d773e 100644 --- a/src/Analyzer/IQueryTreeNode.h +++ b/src/Analyzer/IQueryTreeNode.h @@ -72,6 +72,9 @@ class IQueryTreeNode : public TypePromotion public: virtual ~IQueryTreeNode() = default; + IQueryTreeNode & operator=(const IQueryTreeNode &) = default; + IQueryTreeNode & operator=(IQueryTreeNode &&) = default; + /// Get query tree node type virtual QueryTreeNodeType getNodeType() const = 0; diff --git a/src/Analyzer/Passes/SumIfToCountIfPass.cpp b/src/Analyzer/Passes/SumIfToCountIfPass.cpp index 4462131ed7a..d55af278152 100644 --- a/src/Analyzer/Passes/SumIfToCountIfPass.cpp +++ b/src/Analyzer/Passes/SumIfToCountIfPass.cpp @@ -58,13 +58,20 @@ public: if (!isInt64OrUInt64FieldType(constant_value_literal.getType())) return; - if (constant_value_literal.get() != 1 || getSettings().aggregate_functions_null_for_empty) + if (getSettings().aggregate_functions_null_for_empty) return; + /// Rewrite `sumIf(1, cond)` into `countIf(cond)` + auto multiplier_node = function_node_arguments_nodes[0]; function_node_arguments_nodes[0] = std::move(function_node_arguments_nodes[1]); function_node_arguments_nodes.resize(1); - resolveAsCountIfAggregateFunction(*function_node, function_node_arguments_nodes[0]->getResultType()); + + if (constant_value_literal.get() != 1) + { + /// Rewrite `sumIf(123, cond)` into `123 * countIf(cond)` + node = getMultiplyFunction(std::move(multiplier_node), node); + } return; } @@ -79,7 +86,7 @@ public: if (!nested_function || nested_function->getFunctionName() != "if") return; - const auto & nested_if_function_arguments_nodes = nested_function->getArguments().getNodes(); + const auto nested_if_function_arguments_nodes = nested_function->getArguments().getNodes(); if (nested_if_function_arguments_nodes.size() != 3) return; @@ -100,19 +107,25 @@ public: auto if_true_condition_value = if_true_condition_constant_value_literal.get(); auto if_false_condition_value = if_false_condition_constant_value_literal.get(); - /// Rewrite `sum(if(cond, 1, 0))` into `countIf(cond)`. - if (if_true_condition_value == 1 && if_false_condition_value == 0) + if (if_false_condition_value == 0) { + /// Rewrite `sum(if(cond, 1, 0))` into `countIf(cond)`. function_node_arguments_nodes[0] = nested_if_function_arguments_nodes[0]; function_node_arguments_nodes.resize(1); resolveAsCountIfAggregateFunction(*function_node, function_node_arguments_nodes[0]->getResultType()); + + if (if_true_condition_value != 1) + { + /// Rewrite `sum(if(cond, 123, 0))` into `123 * countIf(cond)`. + node = getMultiplyFunction(nested_if_function_arguments_nodes[1], node); + } return; } - /// Rewrite `sum(if(cond, 0, 1))` into `countIf(not(cond))` if condition is not Nullable (otherwise the result can be different). - if (if_true_condition_value == 0 && if_false_condition_value == 1 && !cond_argument->getResultType()->isNullable()) + if (if_true_condition_value == 0 && !cond_argument->getResultType()->isNullable()) { + /// Rewrite `sum(if(cond, 0, 1))` into `countIf(not(cond))` if condition is not Nullable (otherwise the result can be different). DataTypePtr not_function_result_type = std::make_shared(); const auto & condition_result_type = nested_if_function_arguments_nodes[0]->getResultType(); @@ -130,6 +143,12 @@ public: function_node_arguments_nodes.resize(1); resolveAsCountIfAggregateFunction(*function_node, function_node_arguments_nodes[0]->getResultType()); + + if (if_false_condition_value != 1) + { + /// Rewrite `sum(if(cond, 0, 123))` into `123 * countIf(not(cond))` if condition is not Nullable (otherwise the result can be different). + node = getMultiplyFunction(nested_if_function_arguments_nodes[2], node); + } return; } } @@ -145,6 +164,18 @@ private: function_node.resolveAsAggregateFunction(std::move(aggregate_function)); } + + inline QueryTreeNodePtr getMultiplyFunction(QueryTreeNodePtr left, QueryTreeNodePtr right) + { + auto multiply_function_node = std::make_shared("multiply"); + auto & multiply_arguments_nodes = multiply_function_node->getArguments().getNodes(); + multiply_arguments_nodes.push_back(std::move(left)); + multiply_arguments_nodes.push_back(std::move(right)); + + auto multiply_function_base = FunctionFactory::instance().get("multiply", getContext())->build(multiply_function_node->getArgumentColumns()); + multiply_function_node->resolveAsFunction(std::move(multiply_function_base)); + return std::move(multiply_function_node); + } }; } diff --git a/tests/queries/0_stateless/01646_rewrite_sum_if.reference b/tests/queries/0_stateless/01646_rewrite_sum_if.reference index 0e37c91578a..871c75737c6 100644 --- a/tests/queries/0_stateless/01646_rewrite_sum_if.reference +++ b/tests/queries/0_stateless/01646_rewrite_sum_if.reference @@ -34,3 +34,84 @@ SELECT 123 * countIf((number % 2) = 0) FROM numbers(100) SELECT 123 * countIf(NOT ((number % 2) = 0)) FROM numbers(100) +QUERY id: 0 + PROJECTION COLUMNS + sumIf(123, equals(modulo(number, 2), 0)) UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: multiply, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 3, nodes: 2 + CONSTANT id: 4, constant_value: UInt64_123, constant_value_type: UInt8 + FUNCTION id: 5, function_name: countIf, function_type: aggregate, result_type: UInt64 + ARGUMENTS + LIST id: 6, nodes: 1 + FUNCTION id: 7, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 2 + FUNCTION id: 9, function_name: modulo, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 10, nodes: 2 + COLUMN id: 11, column_name: number, result_type: UInt64, source_id: 12 + CONSTANT id: 13, constant_value: UInt64_2, constant_value_type: UInt8 + CONSTANT id: 14, constant_value: UInt64_0, constant_value_type: UInt8 + JOIN TREE + TABLE_FUNCTION id: 12, table_function_name: numbers + ARGUMENTS + LIST id: 15, nodes: 1 + CONSTANT id: 16, constant_value: UInt64_100, constant_value_type: UInt8 +QUERY id: 0 + PROJECTION COLUMNS + sum(if(equals(modulo(number, 2), 0), 123, 0)) UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: multiply, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 3, nodes: 2 + CONSTANT id: 4, constant_value: UInt64_123, constant_value_type: UInt8 + FUNCTION id: 5, function_name: countIf, function_type: aggregate, result_type: UInt64 + ARGUMENTS + LIST id: 6, nodes: 1 + FUNCTION id: 7, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 2 + FUNCTION id: 9, function_name: modulo, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 10, nodes: 2 + COLUMN id: 11, column_name: number, result_type: UInt64, source_id: 12 + CONSTANT id: 13, constant_value: UInt64_2, constant_value_type: UInt8 + CONSTANT id: 14, constant_value: UInt64_0, constant_value_type: UInt8 + JOIN TREE + TABLE_FUNCTION id: 12, table_function_name: numbers + ARGUMENTS + LIST id: 15, nodes: 1 + CONSTANT id: 16, constant_value: UInt64_100, constant_value_type: UInt8 +QUERY id: 0 + PROJECTION COLUMNS + sum(if(equals(modulo(number, 2), 0), 0, 123)) UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: multiply, function_type: ordinary, result_type: UInt64 + ARGUMENTS + LIST id: 3, nodes: 2 + CONSTANT id: 4, constant_value: UInt64_123, constant_value_type: UInt8 + FUNCTION id: 5, function_name: countIf, function_type: aggregate, result_type: UInt64 + ARGUMENTS + LIST id: 6, nodes: 1 + FUNCTION id: 7, function_name: not, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 1 + FUNCTION id: 9, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 10, nodes: 2 + FUNCTION id: 11, function_name: modulo, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 12, nodes: 2 + COLUMN id: 13, column_name: number, result_type: UInt64, source_id: 14 + CONSTANT id: 15, constant_value: UInt64_2, constant_value_type: UInt8 + CONSTANT id: 16, constant_value: UInt64_0, constant_value_type: UInt8 + JOIN TREE + TABLE_FUNCTION id: 14, table_function_name: numbers + ARGUMENTS + LIST id: 17, nodes: 1 + CONSTANT id: 18, constant_value: UInt64_100, constant_value_type: UInt8 diff --git a/tests/queries/0_stateless/01646_rewrite_sum_if.sql b/tests/queries/0_stateless/01646_rewrite_sum_if.sql index 9abe7f35005..b2de98e9e07 100644 --- a/tests/queries/0_stateless/01646_rewrite_sum_if.sql +++ b/tests/queries/0_stateless/01646_rewrite_sum_if.sql @@ -41,3 +41,9 @@ SELECT countIf(number % 2 != 0) FROM numbers(100); EXPLAIN SYNTAX SELECT sumIf(123, number % 2 == 0) FROM numbers(100); EXPLAIN SYNTAX SELECT sum(if(number % 2 == 0, 123, 0)) FROM numbers(100); EXPLAIN SYNTAX SELECT sum(if(number % 2 == 0, 0, 123)) FROM numbers(100); + +set allow_experimental_analyzer = true; + +EXPLAIN QUERY TREE run_passes=1 SELECT sumIf(123, number % 2 == 0) FROM numbers(100); +EXPLAIN QUERY TREE run_passes=1 SELECT sum(if(number % 2 == 0, 123, 0)) FROM numbers(100); +EXPLAIN QUERY TREE run_passes=1 SELECT sum(if(number % 2 == 0, 0, 123)) FROM numbers(100); \ No newline at end of file From fa6dea8b6b08b5ec00e04a300923a84cb11560ae Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Sun, 29 Jan 2023 19:50:30 +0800 Subject: [PATCH 26/79] remove useless code --- src/Analyzer/IQueryTreeNode.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Analyzer/IQueryTreeNode.h b/src/Analyzer/IQueryTreeNode.h index 719f92d773e..8aa834e60b7 100644 --- a/src/Analyzer/IQueryTreeNode.h +++ b/src/Analyzer/IQueryTreeNode.h @@ -72,9 +72,6 @@ class IQueryTreeNode : public TypePromotion public: virtual ~IQueryTreeNode() = default; - IQueryTreeNode & operator=(const IQueryTreeNode &) = default; - IQueryTreeNode & operator=(IQueryTreeNode &&) = default; - /// Get query tree node type virtual QueryTreeNodeType getNodeType() const = 0; From 2d7c357d61b7dc2b61a84ae736daaad1130d8832 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Tue, 31 Jan 2023 18:00:31 +0800 Subject: [PATCH 27/79] implement columnar version for multiIf when instructions contains short --- src/Functions/multiIf.cpp | 160 ++++++++++++++++++++++++++++++++------ 1 file changed, 136 insertions(+), 24 deletions(-) diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index 247787f3a3a..c8be9e7b44e 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -217,16 +217,6 @@ public: break; } - for (size_t i = 0; i < instructions.size(); ++i) - { - auto & instruction = instructions[i]; - auto source = instruction.source; - for (size_t row_i = 0; row_i < input_rows_count; ++row_i) - { - std::cout << "number:" << row_i << ",instruction:" << i << ",field" << toString((*source)[row_i]) << std::endl; - } - } - /// Special case if first instruction condition is always true and source is constant if (instructions.size() == 1 && instructions.front().source_is_constant && instructions.front().condition_always_true) @@ -237,10 +227,45 @@ public: return ColumnConst::create(std::move(res), instruction.source->size()); } - size_t rows = input_rows_count; + bool contains_short = false; + // size_t i = 0; + for (const auto & instruction : instructions) + { + if (instruction.condition_is_short || instruction.source_is_short) + { + contains_short = true; + } + + /* + Field field; + for (size_t row_i = 0; row_i < instruction.source->size(); ++row_i) + { + instruction.source->get(row_i, field); + std::cout << "source" << i << ",row:" << row_i << ",value:" << toString(field) << std::endl; + } + + if (instruction.condition_always_true) + { + std::cout << "cond" << i << " is always true" << std::endl; + continue; + } + + for (size_t row_i=0; row_i < instruction.condition->size(); ++row_i) + { + instruction.condition->get(row_i, field); + std::cout << "cond" << i << ",row:" << row_i << ",value:" << toString(field) << std::endl; + } + ++i; + */ + } + // std::cout << "contains_short:" << contains_short << std::endl; + const auto & settings = context->getSettingsRef(); const WhichDataType which(result_type); bool execute_multiif_columnar = settings.allow_execute_multiif_columnar && (which.isInt() || which.isUInt() || which.isFloat()); + // std::cout << "execute_multiif_columnar:" << execute_multiif_columnar << std::endl; + + size_t rows = input_rows_count; if (!execute_multiif_columnar) { MutableColumnPtr res = return_type->createColumn(); @@ -248,12 +273,14 @@ public: return std::move(res); } - #define EXECUTE_INSTRUCTIONS_COLUMNAR(TYPE, INDEX) \ if (which.is##TYPE()) \ { \ MutableColumnPtr res = ColumnVector::create(rows); \ - executeInstructionsColumnar(instructions, rows, res); \ + if (!contains_short) \ + executeInstructionsColumnar(instructions, rows, res); \ + else \ + executeInstructionsColumnarNew(instructions, rows, res); \ return std::move(res); \ } @@ -347,12 +374,11 @@ private: const auto & cond_data = assert_cast(*instruction.condition).getData(); for (size_t row_i = 0; row_i < rows; ++row_i) { - size_t condition_index = instruction.condition_is_short ? instruction.condition_index++ : row_i; - /// Equivalent to below code. But it is able to utilize SIMD instructions. - /// if (cond_data[condition_index]) + /// if (cond_data[row_i]) /// inserts[row_i] = i; - inserts[row_i] += cond_data[condition_index] * (i - inserts[row_i]); + + inserts[row_i] += (!!cond_data[row_i]) * (i - inserts[row_i]); } } else @@ -364,12 +390,10 @@ private: for (size_t row_i = 0; row_i < rows; ++row_i) { - size_t condition_index = instruction.condition_is_short ? instruction.condition_index++ : row_i; /// Equivalent to below code. But it is able to utilize SIMD instructions. - /// if (!condition_null_map[condition_index] && condition_nested.getData()[condition_index]) + /// if (!condition_null_map[row_i] && condition_nested_data[row_i]) /// inserts[row_i] = i; - inserts[row_i] += (~condition_null_map[condition_index] & condition_nested_data[condition_index]) - * (i - inserts[row_i]); + inserts[row_i] += (~condition_null_map[row_i] & (!!condition_nested_data[row_i])) * (i - inserts[row_i]); } } } @@ -381,15 +405,103 @@ private: PaddedPODArray inserts(rows, static_cast(instructions.size())); calculateInserts(instructions, rows, inserts); - - PaddedPODArray & res_data = assert_cast &>(*res).getData(); for (size_t row_i = 0; row_i < rows; ++row_i) { auto & instruction = instructions[inserts[row_i]]; auto ref = instruction.source->getDataAt(row_i); res_data[row_i] = *reinterpret_cast(ref.data); - std::cout << "number:" << row_i << ",instruction:" << inserts[row_i] << std::endl; + // std::cout << "row:" << row_i << ",instruction:" << inserts[row_i] << std::endl; + } + } + + /// We should read source from which instruction on each row? + template + static NO_INLINE void calculateInsertsNew(std::vector & instructions, size_t rows, PaddedPODArray & instruction_indexes, PaddedPODArray & source_indexes) + { + S instruction_size = static_cast(instructions.size()); + for (S i = 0; i < static_cast(instructions.size()); ++i) + { + auto & instruction = instructions[i]; + if (instruction.condition_always_true) + { + for (size_t row_i = 0; row_i < rows; ++row_i) + { + if (instruction_indexes[row_i] != instruction_size) + continue; + + instruction_indexes[row_i] = i; + + size_t source_index = instruction.source_is_short ? instruction.source_index++ : row_i; + source_indexes[row_i] = source_index; + } + } + else if (!instruction.condition_is_nullable) + { + const auto & cond_data = assert_cast(*instruction.condition).getData(); + for (size_t row_i = 0; row_i < rows; ++row_i) + { + if (instruction_indexes[row_i] != instruction_size) + continue; + + size_t condition_index = instruction.condition_is_short ? instruction.condition_index++ : row_i; + if (cond_data[condition_index]) + { + instruction_indexes[row_i] = i; + + size_t source_index = instruction.source_is_short ? instruction.source_index++ : row_i; + source_indexes[row_i] = source_index; + } + } + } + else + { + const ColumnNullable & condition_nullable = assert_cast(*instruction.condition); + const ColumnUInt8 & condition_nested = assert_cast(condition_nullable.getNestedColumn()); + const auto & condition_nested_data = condition_nested.getData(); + const NullMap & condition_null_map = condition_nullable.getNullMapData(); + + for (size_t row_i = 0; row_i < rows; ++row_i) + { + if (instruction_indexes[row_i] != instruction_size) + continue; + + size_t condition_index = instruction.condition_is_short ? instruction.condition_index++ : row_i; + if (!condition_null_map[condition_index] && condition_nested_data[condition_index]) + { + instruction_indexes[row_i] = i; + + size_t source_index = instruction.source_is_short ? instruction.source_index++ : row_i; + source_indexes[row_i] = source_index; + } + } + } + + /* + std::cout << "after instruction: " << i << std::endl; + for (size_t row_i = 0; row_i < rows; ++row_i) + { + std::cout << "instruction indexes[" << row_i << "]:" << instruction_indexes[row_i] << std::endl; + std::cout << "source indexes[" << row_i << "]:" << source_indexes[row_i] << std::endl; + } + */ + } + } + + template + static NO_INLINE void executeInstructionsColumnarNew(std::vector & instructions, size_t rows, const MutableColumnPtr & res) + { + PaddedPODArray instruction_indexes(rows, static_cast(instructions.size())); + PaddedPODArray source_indexes(rows, rows); + calculateInsertsNew(instructions, rows, instruction_indexes, source_indexes); + + PaddedPODArray & res_data = assert_cast &>(*res).getData(); + for (size_t row_i = 0; row_i < rows; ++row_i) + { + // std::cout << "row:" << row_i << ", instruction:" << instruction_indexes[row_i] << ", source_row:" << source_indexes[row_i] << std::endl; + auto & instruction = instructions[instruction_indexes[row_i]]; + auto ref = instruction.source->getDataAt(source_indexes[row_i]); + res_data[row_i] = *reinterpret_cast(ref.data); } } From d2e97fb94dd53ec972cb0cc8f00a2601250a19f0 Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Tue, 31 Jan 2023 18:13:01 +0800 Subject: [PATCH 28/79] remove columnar impl when instructions contains short --- src/Functions/multiIf.cpp | 125 ++------------------------------------ 1 file changed, 4 insertions(+), 121 deletions(-) diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index c8be9e7b44e..cdb9ca061c3 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -228,42 +228,19 @@ public: } bool contains_short = false; - // size_t i = 0; for (const auto & instruction : instructions) { if (instruction.condition_is_short || instruction.source_is_short) { contains_short = true; + break; } - - /* - Field field; - for (size_t row_i = 0; row_i < instruction.source->size(); ++row_i) - { - instruction.source->get(row_i, field); - std::cout << "source" << i << ",row:" << row_i << ",value:" << toString(field) << std::endl; - } - - if (instruction.condition_always_true) - { - std::cout << "cond" << i << " is always true" << std::endl; - continue; - } - - for (size_t row_i=0; row_i < instruction.condition->size(); ++row_i) - { - instruction.condition->get(row_i, field); - std::cout << "cond" << i << ",row:" << row_i << ",value:" << toString(field) << std::endl; - } - ++i; - */ } - // std::cout << "contains_short:" << contains_short << std::endl; const auto & settings = context->getSettingsRef(); const WhichDataType which(result_type); - bool execute_multiif_columnar = settings.allow_execute_multiif_columnar && (which.isInt() || which.isUInt() || which.isFloat()); - // std::cout << "execute_multiif_columnar:" << execute_multiif_columnar << std::endl; + bool execute_multiif_columnar + = settings.allow_execute_multiif_columnar && !contains_short && (which.isInt() || which.isUInt() || which.isFloat()); size_t rows = input_rows_count; if (!execute_multiif_columnar) @@ -277,10 +254,7 @@ public: if (which.is##TYPE()) \ { \ MutableColumnPtr res = ColumnVector::create(rows); \ - if (!contains_short) \ - executeInstructionsColumnar(instructions, rows, res); \ - else \ - executeInstructionsColumnarNew(instructions, rows, res); \ + executeInstructionsColumnar(instructions, rows, res); \ return std::move(res); \ } @@ -411,97 +385,6 @@ private: auto & instruction = instructions[inserts[row_i]]; auto ref = instruction.source->getDataAt(row_i); res_data[row_i] = *reinterpret_cast(ref.data); - // std::cout << "row:" << row_i << ",instruction:" << inserts[row_i] << std::endl; - } - } - - /// We should read source from which instruction on each row? - template - static NO_INLINE void calculateInsertsNew(std::vector & instructions, size_t rows, PaddedPODArray & instruction_indexes, PaddedPODArray & source_indexes) - { - S instruction_size = static_cast(instructions.size()); - for (S i = 0; i < static_cast(instructions.size()); ++i) - { - auto & instruction = instructions[i]; - if (instruction.condition_always_true) - { - for (size_t row_i = 0; row_i < rows; ++row_i) - { - if (instruction_indexes[row_i] != instruction_size) - continue; - - instruction_indexes[row_i] = i; - - size_t source_index = instruction.source_is_short ? instruction.source_index++ : row_i; - source_indexes[row_i] = source_index; - } - } - else if (!instruction.condition_is_nullable) - { - const auto & cond_data = assert_cast(*instruction.condition).getData(); - for (size_t row_i = 0; row_i < rows; ++row_i) - { - if (instruction_indexes[row_i] != instruction_size) - continue; - - size_t condition_index = instruction.condition_is_short ? instruction.condition_index++ : row_i; - if (cond_data[condition_index]) - { - instruction_indexes[row_i] = i; - - size_t source_index = instruction.source_is_short ? instruction.source_index++ : row_i; - source_indexes[row_i] = source_index; - } - } - } - else - { - const ColumnNullable & condition_nullable = assert_cast(*instruction.condition); - const ColumnUInt8 & condition_nested = assert_cast(condition_nullable.getNestedColumn()); - const auto & condition_nested_data = condition_nested.getData(); - const NullMap & condition_null_map = condition_nullable.getNullMapData(); - - for (size_t row_i = 0; row_i < rows; ++row_i) - { - if (instruction_indexes[row_i] != instruction_size) - continue; - - size_t condition_index = instruction.condition_is_short ? instruction.condition_index++ : row_i; - if (!condition_null_map[condition_index] && condition_nested_data[condition_index]) - { - instruction_indexes[row_i] = i; - - size_t source_index = instruction.source_is_short ? instruction.source_index++ : row_i; - source_indexes[row_i] = source_index; - } - } - } - - /* - std::cout << "after instruction: " << i << std::endl; - for (size_t row_i = 0; row_i < rows; ++row_i) - { - std::cout << "instruction indexes[" << row_i << "]:" << instruction_indexes[row_i] << std::endl; - std::cout << "source indexes[" << row_i << "]:" << source_indexes[row_i] << std::endl; - } - */ - } - } - - template - static NO_INLINE void executeInstructionsColumnarNew(std::vector & instructions, size_t rows, const MutableColumnPtr & res) - { - PaddedPODArray instruction_indexes(rows, static_cast(instructions.size())); - PaddedPODArray source_indexes(rows, rows); - calculateInsertsNew(instructions, rows, instruction_indexes, source_indexes); - - PaddedPODArray & res_data = assert_cast &>(*res).getData(); - for (size_t row_i = 0; row_i < rows; ++row_i) - { - // std::cout << "row:" << row_i << ", instruction:" << instruction_indexes[row_i] << ", source_row:" << source_indexes[row_i] << std::endl; - auto & instruction = instructions[instruction_indexes[row_i]]; - auto ref = instruction.source->getDataAt(source_indexes[row_i]); - res_data[row_i] = *reinterpret_cast(ref.data); } } From 90153c11fcff270de69567229f197db1efee300f Mon Sep 17 00:00:00 2001 From: Han Fei Date: Wed, 1 Feb 2023 15:09:04 +0100 Subject: [PATCH 29/79] fix matching priority --- src/Dictionaries/RegExpTreeDictionary.cpp | 31 +- src/Dictionaries/RegExpTreeDictionary.h | 4 +- ...2504_regexp_dictionary_ua_parser.reference | 478 +++++++++--------- 3 files changed, 260 insertions(+), 253 deletions(-) diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index 7feb3c50779..0cf7d579f96 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -289,7 +289,7 @@ void RegExpTreeDictionary::initTopologyOrder(UInt64 node_idx, std::set & visited.insert(node_idx); for (UInt64 child_idx : regex_nodes[node_idx]->children) if (visited.contains(child_idx)) - throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION, "Invalid Regex tree"); + throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION, "Invalid Regex tree. The input tree is cyclical"); else initTopologyOrder(child_idx, visited, topology_id); topology_order[node_idx] = topology_id++; @@ -360,7 +360,7 @@ std::pair processBackRefs(const String & data, const re2_st::RE2 & String result; searcher.Match(haystack, 0, data.size(), re2_st::RE2::Anchor::UNANCHORED, matches, 10); /// if the pattern is a single '$1' but fails to match, we would use the default value. - if (pieces.size() == 1 && pieces[0].ref_num >= 0 && pieces[0].ref_num < 9 && matches[pieces[0].ref_num].empty()) + if (pieces.size() == 1 && pieces[0].ref_num >= 0 && pieces[0].ref_num < 10 && matches[pieces[0].ref_num].empty()) return std::make_pair(result, true); for (const auto & item : pieces) { @@ -379,7 +379,9 @@ bool RegExpTreeDictionary::setAttributes( std::unordered_map & attributes_to_set, const String & data, std::unordered_set & visited_nodes, - const std::unordered_map & attributes) const + const std::unordered_map & attributes, + const std::unordered_map & defaults, + size_t key_index) const { if (visited_nodes.contains(id)) @@ -393,7 +395,12 @@ bool RegExpTreeDictionary::setAttributes( if (value.containsBackRefs()) { auto [updated_str, use_default] = processBackRefs(data, regex_nodes.at(id)->searcher, value.pieces); - if (!use_default) + if (use_default) + { + DefaultValueProvider default_value(attributes.at(name).null_value, defaults.at(name)); + attributes_to_set[name] = default_value.getDefaultValue(key_index); + } + else attributes_to_set[name] = parseStringToField(updated_str, attributes.at(name).type); } else @@ -402,7 +409,7 @@ bool RegExpTreeDictionary::setAttributes( auto parent_id = regex_nodes.at(id)->parent_id; if (parent_id > 0) - setAttributes(parent_id, attributes_to_set, data, visited_nodes, attributes); + setAttributes(parent_id, attributes_to_set, data, visited_nodes, attributes, defaults, key_index); // if all the attributes have set, the walking through can be stopped. return attributes_to_set.size() == attributes.size(); @@ -437,6 +444,7 @@ namespace matched_idx_sorted_list.push_back(std::make_pair(topological_order, id)); } + /// Sort by topological order, which indicates the matching priorities. void sort() { std::sort(matched_idx_sorted_list.begin(), matched_idx_sorted_list.end()); @@ -507,15 +515,12 @@ std::unordered_map RegExpTreeDictionary::matchSearchAllIndice if (err != HS_SUCCESS) throw Exception(ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT, "Failed to scan data with vectorscan"); - if (!match_result.matched_idx_set.empty()) + for (const auto & node_ptr : complex_regexp_nodes) { - for (const auto & node_ptr : complex_regexp_nodes) + if (node_ptr->match(reinterpret_cast(keys_data.data()) + offset, length)) { - if (node_ptr->match(reinterpret_cast(keys_data.data()) + offset, length)) - { - match_result.insertNodeID(node_ptr->id); - break; - } + match_result.insertNodeID(node_ptr->id); + break; } } @@ -546,7 +551,7 @@ std::unordered_map RegExpTreeDictionary::matchSearchAllIndice continue; if (visited_nodes.contains(id)) continue; - if (setAttributes(id, attributes_to_set, str, visited_nodes, attributes)) + if (setAttributes(id, attributes_to_set, str, visited_nodes, attributes, defaults, key_idx)) break; } diff --git a/src/Dictionaries/RegExpTreeDictionary.h b/src/Dictionaries/RegExpTreeDictionary.h index f3de05239b5..29c8b461324 100644 --- a/src/Dictionaries/RegExpTreeDictionary.h +++ b/src/Dictionaries/RegExpTreeDictionary.h @@ -146,7 +146,9 @@ private: std::unordered_map & attributes_to_set, const String & data, std::unordered_set & visited_nodes, - const std::unordered_map & attributes) const; + const std::unordered_map & attributes, + const std::unordered_map & defaults, + size_t key_index) const; struct RegexTreeNode; using RegexTreeNodePtr = std::shared_ptr; diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.reference b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.reference index d6dd2adae74..b161b099eef 100644 --- a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.reference +++ b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.reference @@ -1,14 +1,14 @@ -AppleTV Other 0.0 ATV OS X 8.1.3 +AppleTV Other 0.0 ATV OS X 0.0.0 LG-M150 Firefox Mobile 68.0 Android 7.0.0 -Other Firefox Mobile 68.0 Android 8.0.0 -Other Firefox Mobile 68.0 Android 8.1.0 -Other Firefox Mobile 68.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 10.0.0 -Other Chrome Mobile 77.0 Android 10.0.0 -Other Chrome Mobile 77.0 Android 10.0.0 -Other Facebook 240.0 Android 10.0.0 -Other Chrome Mobile WebView 77.0 Android 10.0.0 -Other Chrome Mobile 77.0 Android 10.0.0 +Generic Smartphone Firefox Mobile 68.0 Android 8.0.0 +Generic Tablet Firefox Mobile 68.0 Android 8.1.0 +Generic Smartphone Firefox Mobile 68.0 Android 9.0.0 +PH-1 Chrome Mobile 77.0 Android 10.0.0 +Pixel 2 XL Chrome Mobile 77.0 Android 10.0.0 +Pixel 2 Chrome Mobile 77.0 Android 10.0.0 +Pixel 3 Facebook 240.0 Android 10.0.0 +Pixel XL Chrome Mobile WebView 77.0 Android 10.0.0 +Pixel XL Chrome Mobile 77.0 Android 10.0.0 HTC Sensation 4G Chrome Mobile 42.0 Android 4.0.3 Kindle Amazon Silk 73.7 Android 4.0.3 Samsung GT-I9152 Chrome Mobile 42.0 Android 4.2.2 @@ -16,7 +16,7 @@ Samsung GT-N5110 Chrome 76.0 Android 4.4.2 RCT6773W22 Chrome 77.0 Android 4.4.2 Samsung SM-T217S Chrome 77.0 Android 4.4.2 Samsung SM-T530NU Chrome 77.0 Android 4.4.2 -Other Chrome 69.0 Android 4.4.2 +TegraNote-P1640 Chrome 69.0 Android 4.4.2 Kindle Amazon Silk 76.3 Android 4.4.3 Samsung SM-A500H Chrome Mobile 73.0 Android 5.0.2 Samsung SM-T357T Chrome 77.0 Android 5.0.2 @@ -24,9 +24,9 @@ Samsung SM-T530NU Chrome 76.0 Android 5.0.2 Samsung SM-T530NU Chrome 77.0 Android 5.0.2 RCT6213W87DK Yandex Browser 19.4 Android 5.0.0 Samsung SM-N900T Facebook 229.0 Android 5.0.0 -Other Chrome Mobile WebView 70.0 Android 5.1.1 +Generic Smartphone Chrome Mobile WebView 70.0 Android 5.1.1 Kindle Amazon Silk 76.3 Android 5.1.1 -Other Chrome Mobile WebView 70.0 Android 5.1.1 +AFTT Chrome Mobile WebView 70.0 Android 5.1.1 Kindle Amazon Silk 76.3 Android 5.1.1 Kindle Amazon Silk 76.3 Android 5.1.1 Kindle Amazon Silk 71.2 Android 5.1.1 @@ -36,29 +36,29 @@ Kindle Amazon Silk 76.3 Android 5.1.1 Kindle Amazon Silk 76.3 Android 5.1.1 Kindle Amazon Silk 77.1 Android 5.1.1 LG-AS330 Chrome Mobile 77.0 Android 5.1.1 -Other Chrome Mobile 77.0 Android 5.1.1 +LGL43AL Chrome Mobile 77.0 Android 5.1.1 Samsung SM-G530R7 Samsung Internet 9.2 Android 5.1.1 Samsung SM-T377P Samsung Internet 10.1 Android 5.1.1 Samsung SM-T900 Samsung Internet 10.1 Android 5.1.1 Samsung SM-T337A Chrome 69.0 Android 5.1.1 Samsung SM-G360T1 Chrome Mobile 67.0 Android 5.1.1 Samsung SM-J320FN Chrome Mobile 74.0 Android 5.1.1 -Other Chrome 74.0 Android 5.1.1 +SM-T280 Chrome 74.0 Android 5.1.1 Samsung SM-T330NU Chrome 71.0 Android 5.1.1 -Other Chrome 76.0 Android 5.1.1 -Other Chrome 77.0 Android 5.1.1 -Other Chrome Mobile WebView 74.0 Android 5.1.1 -Other Chrome Mobile 66.0 Android 5.1.0 +SM-T670 Chrome 76.0 Android 5.1.1 +SM-T670 Chrome 77.0 Android 5.1.1 +Vodafone Smart ultra 6 Chrome Mobile WebView 74.0 Android 5.1.1 +BLU Advance 5.0 Chrome Mobile 66.0 Android 5.1.0 HTC Desire 626s Chrome Mobile 77.0 Android 5.1.0 HUAWEI LUA-L22 Chrome Mobile 50.0 Android 5.1.0 -Other Chrome 77.0 Android 5.1.0 -Other Chrome Mobile 73.0 Android 5.1.0 +NX16A11264 Chrome 77.0 Android 5.1.0 +XT1526 Chrome Mobile 73.0 Android 5.1.0 Oppo CPH1613 Chrome Mobile 77.0 Android 6.0.1 LG-M153 Chrome Mobile WebView 55.0 Android 6.0.1 LG-M153 Chrome Mobile 77.0 Android 6.0.1 -Other Chrome Mobile 77.0 Android 6.0.1 -Other Chrome Mobile 74.0 Android 6.0.1 -Other Chrome 44.0 Android 6.0.1 +LGLS676 Chrome Mobile 77.0 Android 6.0.1 +N9136 Chrome Mobile 74.0 Android 6.0.1 +Asus Nexus 7 Chrome 44.0 Android 6.0.1 Samsung SM-G900I Samsung Internet 10.1 Android 6.0.1 Samsung SM-G900P Samsung Internet 7.2 Android 6.0.1 Samsung SM-J700M Samsung Internet 10.1 Android 6.0.1 @@ -77,19 +77,19 @@ Samsung SM-N910S Chrome Mobile 75.0 Android 6.0.1 Samsung SM-N920V Chrome Mobile 76.0 Android 6.0.1 Samsung SM-T350 Chrome 59.0 Android 6.0.1 Samsung SM-T560NU Chrome 77.0 Android 6.0.1 -Other Chrome 77.0 Android 6.0.1 -Other Chrome Mobile 77.0 Android 6.0.1 -Other Chrome Mobile 67.0 Android 6.0.1 -Other Chrome Mobile WebView 45.0 Android 6.0.1 -Other Chrome Mobile 76.0 Android 6.0.0 +SM-T800 Chrome 77.0 Android 6.0.1 +XT1254 Chrome Mobile 77.0 Android 6.0.1 +Z798BL Chrome Mobile 67.0 Android 6.0.1 +Z799VL Chrome Mobile WebView 45.0 Android 6.0.1 +5010X Chrome Mobile 76.0 Android 6.0.0 Huawei CAM-L21 Chrome Mobile 77.0 Android 6.0.0 -Other Chrome Mobile 77.0 Android 6.0.0 +F3313 Chrome Mobile 77.0 Android 6.0.0 RCT6603W47M7 Chrome 77.0 Android 6.0.0 -Other Chrome Mobile 56.0 Android 7.0.0 +5049Z Chrome Mobile 56.0 Android 7.0.0 Asus A002A Chrome Mobile 77.0 Android 7.0.0 -Other Chrome Mobile 77.0 Android 7.0.0 -Other Chrome Mobile WebView 59.0 Android 7.0.0 -Other Chrome Mobile 77.0 Android 7.0.0 +Alcatel_5044C Chrome Mobile 77.0 Android 7.0.0 +Astra Young Pro Chrome Mobile WebView 59.0 Android 7.0.0 +Infinix X571 Chrome Mobile 77.0 Android 7.0.0 LG-H872 Chrome Mobile 64.0 Android 7.0.0 LG-K425 Chrome Mobile 55.0 Android 7.0.0 LG-LS777 Chrome Mobile 77.0 Android 7.0.0 @@ -100,11 +100,11 @@ LG-TP260 Chrome Mobile 77.0 Android 7.0.0 LG-TP450 Chrome Mobile 64.0 Android 7.0.0 LG-V521 Chrome 75.0 Android 7.0.0 LG-V521 Chrome 77.0 Android 7.0.0 -Other Chrome Mobile 58.0 Android 7.0.0 -Other Chrome Mobile 55.0 Android 7.0.0 -Other Chrome Mobile 77.0 Android 7.0.0 -Other Chrome 77.0 Android 7.0.0 -Other Chrome Mobile 77.0 Android 7.0.0 +LGMP260 Chrome Mobile 58.0 Android 7.0.0 +LGMS210 Chrome Mobile 55.0 Android 7.0.0 +LGMS210 Chrome Mobile 77.0 Android 7.0.0 +P00I Chrome 77.0 Android 7.0.0 +RS988 Chrome Mobile 77.0 Android 7.0.0 Samsung SM-J701F Samsung Internet 10.1 Android 7.0.0 Samsung SM-J710F Samsung Internet 10.1 Android 7.0.0 Samsung SM-N920T Samsung Internet 9.2 Android 7.0.0 @@ -122,63 +122,63 @@ Samsung SM-J327T1 Chrome Mobile 77.0 Android 7.0.0 Samsung SM-N9208 Chrome Mobile 73.0 Android 7.0.0 Samsung SM-N920P Chrome Mobile 74.0 Android 7.0.0 Samsung SM-N920T Chrome Mobile 77.0 Android 7.0.0 -Other Chrome 77.0 Android 7.0.0 -Other Chrome 75.0 Android 7.0.0 -Other Chrome 76.0 Android 7.0.0 -Other Chrome 77.0 Android 7.0.0 -Other Chrome 76.0 Android 7.0.0 -Other Chrome 76.0 Android 7.0.0 -Other Chrome 75.0 Android 7.0.0 -Other Chrome Mobile 77.0 Android 7.0.0 +SM-T585 Chrome 77.0 Android 7.0.0 +SM-T810 Chrome 75.0 Android 7.0.0 +SM-T810 Chrome 76.0 Android 7.0.0 +SM-T810 Chrome 77.0 Android 7.0.0 +SM-T813 Chrome 76.0 Android 7.0.0 +SM-T813 Chrome 76.0 Android 7.0.0 +Trekstor ST1009X Chrome 75.0 Android 7.0.0 +XT1663 Chrome Mobile 77.0 Android 7.0.0 Generic Smartphone Chrome Mobile 58.0 Android 7.0.0 -Other Chrome Mobile WebView 77.0 Android 7.1.1 -Other Chrome Mobile 77.0 Android 7.1.1 +A574BL Chrome Mobile WebView 77.0 Android 7.1.1 +A574BL Chrome Mobile 77.0 Android 7.1.1 Oppo CPH1729 Facebook 240.0 Android 7.1.1 -Other Chrome Mobile 74.0 Android 7.1.1 -Other Chrome Mobile 77.0 Android 7.1.1 -Other Chrome Mobile WebView 76.0 Android 7.1.1 -Other Chrome Mobile 70.0 Android 7.1.1 -Other Chrome Mobile 76.0 Android 7.1.1 -Other Chrome Mobile 77.0 Android 7.1.1 -Other Chrome Mobile 77.0 Android 7.1.1 -Other Chrome Mobile 77.0 Android 7.1.1 -Other Chrome Mobile 64.0 Android 7.1.1 +3632A Chrome Mobile 74.0 Android 7.1.1 +General Mobile 4G Dual Chrome Mobile 77.0 Android 7.1.1 +Moto E (4) Plus Chrome Mobile WebView 76.0 Android 7.1.1 +Moto E (4) Chrome Mobile 70.0 Android 7.1.1 +Moto E (4) Chrome Mobile 76.0 Android 7.1.1 +Moto E (4) Chrome Mobile 77.0 Android 7.1.1 +Moto E (4) Chrome Mobile 77.0 Android 7.1.1 +NX591J Chrome Mobile 77.0 Android 7.1.1 +REVVLPLUS C3701A Chrome Mobile 64.0 Android 7.1.1 Samsung SM-J320A Samsung Internet 10.1 Android 7.1.1 Samsung SM-T550 Samsung Internet 10.1 Android 7.1.1 Samsung SM-T377A Chrome 64.0 Android 7.1.1 Samsung SM-J250F Chrome Mobile 76.0 Android 7.1.1 Samsung SM-J700T Chrome Mobile 77.0 Android 7.1.1 -Other Chrome 77.0 Android 7.1.1 +SM-T350 Chrome 77.0 Android 7.1.1 Samsung SM-T377T Chrome 77.0 Android 7.1.1 Samsung SM-T550 Chrome 69.0 Android 7.1.1 -Other Chrome 77.0 Android 7.1.1 +SM-T550 Chrome 77.0 Android 7.1.1 Samsung SM-T560NU Chrome 77.0 Android 7.1.1 X20 Chrome Mobile WebView 52.0 Android 7.1.1 -Other Chrome Mobile 58.0 Android 7.1.1 -Other Chrome Mobile WebView 74.0 Android 7.1.1 -Other Chrome Mobile WebView 75.0 Android 7.1.1 -Other Chrome Mobile 77.0 Android 7.1.1 -Other Chrome Mobile WebView 70.0 Android 7.1.2 -Other Chrome Mobile WebView 70.0 Android 7.1.2 +Z851M Chrome Mobile 58.0 Android 7.1.1 +Z899VL Chrome Mobile WebView 74.0 Android 7.1.1 +Z982 Chrome Mobile WebView 75.0 Android 7.1.1 +Z982 Chrome Mobile 77.0 Android 7.1.1 +Generic Smartphone Chrome Mobile WebView 70.0 Android 7.1.2 +AFTKMST12 Chrome Mobile WebView 70.0 Android 7.1.2 Kindle Amazon Silk 76.3 Android 7.1.2 -Other Chrome Mobile WebView 70.0 Android 7.1.2 -Other Chrome Mobile WebView 70.0 Android 7.1.2 -Other Chrome Mobile WebView 59.0 Android 7.1.2 +AFTMM Chrome Mobile WebView 70.0 Android 7.1.2 +AFTN Chrome Mobile WebView 70.0 Android 7.1.2 +KFKAWI Chrome Mobile WebView 59.0 Android 7.1.2 Kindle Amazon Silk 76.3 Android 7.1.2 Kindle Amazon Silk 76.3 Android 7.1.2 LG-SP200 Chrome Mobile 75.0 Android 7.1.2 LG-SP200 Chrome Mobile 76.0 Android 7.1.2 -Other Chrome Mobile 76.0 Android 7.1.2 -Other Chrome Mobile 76.0 Android 7.1.2 +LM-X210(G) Chrome Mobile 76.0 Android 7.1.2 +LM-X210 Chrome Mobile 76.0 Android 7.1.2 RCT6973W43R Chrome 77.0 Android 7.1.2 XiaoMi Redmi 4 Chrome Mobile 77.0 Android 7.1.2 -Other Chrome Mobile WebView 76.0 Android 8.0.0 +Generic Smartphone Chrome Mobile WebView 76.0 Android 8.0.0 Asus Z01FD Chrome Mobile 77.0 Android 8.0.0 Huawei AUM-L29 Chrome Mobile 77.0 Android 8.0.0 -Other Chrome Mobile WebView 77.0 Android 8.0.0 -Other Chrome 77.0 Android 8.0.0 -Other Chrome Mobile 77.0 Android 8.0.0 -Other Chrome Mobile 77.0 Android 8.0.0 +BRAVIA 4K GB Chrome Mobile WebView 77.0 Android 8.0.0 +CMR-W09 Chrome 77.0 Android 8.0.0 +EVA-AL00 Chrome Mobile 77.0 Android 8.0.0 +G3223 Chrome Mobile 77.0 Android 8.0.0 LG-H910 Chrome Mobile 77.0 Android 8.0.0 LG-H931 Chrome Mobile 76.0 Android 8.0.0 LG-H932 Chrome Mobile 77.0 Android 8.0.0 @@ -214,49 +214,49 @@ Samsung SM-N950U Chrome Mobile 76.0 Android 8.0.0 Samsung SM-N950U Chrome Mobile 77.0 Android 8.0.0 Samsung SM-N950U1 Chrome Mobile 77.0 Android 8.0.0 Samsung SM-S367VL Chrome Mobile 77.0 Android 8.0.0 -Other Chrome Mobile 77.0 Android 8.0.0 -Other Chrome Mobile 77.0 Android 8.0.0 -Other Chrome Mobile 76.0 Android 8.0.0 -Other Chrome Mobile 77.0 Android 8.0.0 -Other Chrome Mobile 76.0 Android 8.0.0 -Other Chrome Mobile 77.0 Android 8.0.0 -Other Chrome Mobile 77.0 Android 8.1.0 -Other Chrome Mobile 77.0 Android 8.1.0 -Other Chrome Mobile 67.0 Android 8.1.0 -Other Chrome Mobile 76.0 Android 8.1.0 +VS995 Chrome Mobile 77.0 Android 8.0.0 +XT1635-02 Chrome Mobile 77.0 Android 8.0.0 +moto e5 play Chrome Mobile 76.0 Android 8.0.0 +moto e5 play Chrome Mobile 77.0 Android 8.0.0 +moto e5 supra Chrome Mobile 76.0 Android 8.0.0 +moto g(6) Chrome Mobile 77.0 Android 8.0.0 +5041C Chrome Mobile 77.0 Android 8.1.0 +6062W Chrome Mobile 77.0 Android 8.1.0 +A502DL Chrome Mobile 67.0 Android 8.1.0 +A502DL Chrome Mobile 76.0 Android 8.1.0 Huawei BKK-LX2 Chrome Mobile 76.0 Android 8.1.0 -Other Chrome Mobile 70.0 Android 8.1.0 -Other Chrome Mobile 77.0 Android 8.1.0 -Other Chrome Mobile 64.0 Android 8.1.0 -Other Chrome Mobile 77.0 Android 8.1.0 -Other Chrome Mobile 77.0 Android 8.1.0 +C4 Chrome Mobile 70.0 Android 8.1.0 +3310A Chrome Mobile 77.0 Android 8.1.0 +Infinix X604 Chrome Mobile 64.0 Android 8.1.0 +Joy 1 Chrome Mobile 77.0 Android 8.1.0 +LAVA LE9820 Chrome Mobile 77.0 Android 8.1.0 LG-Q710AL Chrome Mobile 77.0 Android 8.1.0 -Other Chrome Mobile 77.0 Android 8.1.0 -Other Facebook 235.0 Android 8.1.0 -Other Chrome Mobile 70.0 Android 8.1.0 -Other Chrome Mobile 76.0 Android 8.1.0 -Other Chrome Mobile 76.0 Android 8.1.0 -Other Chrome Mobile 77.0 Android 8.1.0 -Other Chrome Mobile 77.0 Android 8.1.0 -Other UC Browser 11.6 Android 8.1.0 -Other Chrome Mobile 70.0 Android 8.1.0 -Other Chrome Mobile 72.0 Android 8.1.0 -Other Chrome Mobile 77.0 Android 8.1.0 -Other Chrome Mobile 77.0 Android 8.1.0 -Other Chrome Mobile 70.0 Android 8.1.0 -Other Chrome Mobile 76.0 Android 8.1.0 -Other Chrome Mobile WebView 77.0 Android 8.1.0 -Other Chrome Mobile 70.0 Android 8.1.0 -Other Chrome Mobile 76.0 Android 8.1.0 -Other Chrome Mobile 77.0 Android 8.1.0 -Other Chrome Mobile 68.0 Android 8.1.0 -Other Chrome Mobile 76.0 Android 8.1.0 -Other Chrome Mobile 77.0 Android 8.1.0 -Other Chrome Mobile 77.0 Android 8.1.0 +LM-Q610(FGN) Chrome Mobile 77.0 Android 8.1.0 +LM-Q710(FGN) Facebook 235.0 Android 8.1.0 +LM-Q710(FGN) Chrome Mobile 70.0 Android 8.1.0 +LM-Q710(FGN) Chrome Mobile 76.0 Android 8.1.0 +LM-Q710(FGN) Chrome Mobile 76.0 Android 8.1.0 +LM-Q710(FGN) Chrome Mobile 77.0 Android 8.1.0 +LM-V405 Chrome Mobile 77.0 Android 8.1.0 +LM-X210(G) UC Browser 11.6 Android 8.1.0 +LM-X210(G) Chrome Mobile 70.0 Android 8.1.0 +LM-X210(G) Chrome Mobile 72.0 Android 8.1.0 +LM-X210(G) Chrome Mobile 77.0 Android 8.1.0 +LM-X212(G) Chrome Mobile 77.0 Android 8.1.0 +LM-X220 Chrome Mobile 70.0 Android 8.1.0 +LM-X220 Chrome Mobile 76.0 Android 8.1.0 +LM-X220PM Chrome Mobile WebView 77.0 Android 8.1.0 +LM-X410(FG) Chrome Mobile 70.0 Android 8.1.0 +LM-X410(FG) Chrome Mobile 76.0 Android 8.1.0 +LM-X410(FG) Chrome Mobile 77.0 Android 8.1.0 +LM-X410.FGN Chrome Mobile 68.0 Android 8.1.0 +LML414DL Chrome Mobile 76.0 Android 8.1.0 +LML713DL Chrome Mobile 77.0 Android 8.1.0 +Moto G (5S) Plus Chrome Mobile 77.0 Android 8.1.0 HTC One Chrome Mobile WebView 70.0 Android 8.1.0 RCT6873W42BMF8KC Chrome Mobile 77.0 Android 8.1.0 -Other Chrome Mobile 67.0 Android 8.1.0 -Other Chrome Mobile 76.0 Android 8.1.0 +REVVL 2 Chrome Mobile 67.0 Android 8.1.0 +REVVL 2 Chrome Mobile 76.0 Android 8.1.0 Samsung SM-J727T Samsung Internet 10.1 Android 8.1.0 Samsung SM-J727T1 Samsung Internet 9.4 Android 8.1.0 Samsung SM-J727T1 Samsung Internet 10.1 Android 8.1.0 @@ -274,54 +274,54 @@ Samsung SM-J727T1 Chrome Mobile 77.0 Android 8.1.0 Samsung SM-J727T1 Chrome Mobile 77.0 Android 8.1.0 Samsung SM-J727V Chrome Mobile 70.0 Android 8.1.0 Samsung SM-J727V Chrome Mobile 77.0 Android 8.1.0 -Other Chrome 77.0 Android 8.1.0 -Other Chrome 75.0 Android 8.1.0 -Other Edge Mobile 42.0 Android 8.1.0 -Other Chrome 76.0 Android 8.1.0 -Other Chrome 76.0 Android 8.1.0 -Other Chrome 77.0 Android 8.1.0 +SM-P580 Chrome 77.0 Android 8.1.0 +SM-T380 Chrome 75.0 Android 8.1.0 +SM-T580 Edge Mobile 42.0 Android 8.1.0 +SM-T580 Chrome 76.0 Android 8.1.0 +SM-T580 Chrome 76.0 Android 8.1.0 +SM-T580 Chrome 77.0 Android 8.1.0 Samsung SM-T837T Chrome 77.0 Android 8.1.0 -Other Facebook 239.0 Android 8.1.0 -Other Chrome Mobile 75.0 Android 8.1.0 -Other Chrome Mobile 68.0 Android 8.1.0 -Other Chrome Mobile 76.0 Android 8.1.0 -Other Chrome Mobile 77.0 Android 8.1.0 -Other DuckDuckGo Mobile 5.0 Android 9.0.0 -Other Chrome Mobile 70.0 Android 9.0.0 -Other Facebook 236.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 9.0.0 +TECNO CF8 Facebook 239.0 Android 8.1.0 +V1818CA Chrome Mobile 75.0 Android 8.1.0 +meizu C9 Chrome Mobile 68.0 Android 8.1.0 +vivo 1724 Chrome Mobile 76.0 Android 8.1.0 +vivo 1814 Chrome Mobile 77.0 Android 8.1.0 +Generic Smartphone DuckDuckGo Mobile 5.0 Android 9.0.0 +1825 Chrome Mobile 70.0 Android 9.0.0 +ANE-LX2 Facebook 236.0 Android 9.0.0 +BLA-A09 Chrome Mobile 77.0 Android 9.0.0 Huawei CLT-L04 Chrome Mobile 77.0 Android 9.0.0 Oppo CPH1911 Facebook 239.0 Android 9.0.0 Oppo CPH1923 Chrome Mobile WebView 76.0 Android 9.0.0 Huawei ELE-L29 Chrome Mobile 77.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 9.0.0 -Other Chrome Mobile 76.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 9.0.0 +G8142 Chrome Mobile 77.0 Android 9.0.0 +GM1911 Chrome Mobile 76.0 Android 9.0.0 +GM1917 Chrome Mobile 77.0 Android 9.0.0 Huawei INE-LX2 Chrome Mobile 76.0 Android 9.0.0 -Other Chrome Mobile WebView 77.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 9.0.0 -Other Chrome Mobile WebView 77.0 Android 9.0.0 -Other Chrome Mobile 76.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 9.0.0 -Other Chrome Mobile 72.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 9.0.0 +LM-G710 Chrome Mobile WebView 77.0 Android 9.0.0 +LM-Q720 Chrome Mobile 77.0 Android 9.0.0 +LM-V405 Chrome Mobile WebView 77.0 Android 9.0.0 +LM-V405 Chrome Mobile 76.0 Android 9.0.0 +LM-V500N Chrome Mobile 77.0 Android 9.0.0 +LM-X420 Chrome Mobile 72.0 Android 9.0.0 +LM-X420 Chrome Mobile 77.0 Android 9.0.0 +MAR-LX1A Chrome Mobile 77.0 Android 9.0.0 XiaoMi MI 9 Chrome Mobile 77.0 Android 9.0.0 XiaoMi Mi A2 Chrome Mobile 77.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 9.0.0 +Moto Z (2) Chrome Mobile 77.0 Android 9.0.0 Nokia 6 Chrome Mobile 77.0 Android 9.0.0 OnePlus ONEPLUS A6000 Chrome Mobile 77.0 Android 9.0.0 OnePlus ONEPLUS A6003 Chrome Mobile 77.0 Android 9.0.0 OnePlus ONEPLUS A6013 Chrome Mobile WebView 77.0 Android 9.0.0 OnePlus ONEPLUS A6013 Chrome Mobile 74.0 Android 9.0.0 OnePlus ONEPLUS A6013 Chrome Mobile 77.0 Android 9.0.0 -Other Facebook 235.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 9.0.0 -Other Chrome Mobile WebView 77.0 Android 9.0.0 -Other Chrome Mobile 76.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 9.0.0 -Other Chrome Mobile 73.0 Android 9.0.0 +PAR-AL00 Facebook 235.0 Android 9.0.0 +Pixel 2 XL Chrome Mobile 77.0 Android 9.0.0 +Pixel 3 Chrome Mobile WebView 77.0 Android 9.0.0 +Pixel 3 Chrome Mobile 76.0 Android 9.0.0 +Pixel 3 Chrome Mobile 77.0 Android 9.0.0 +Pixel 3a XL Chrome Mobile 77.0 Android 9.0.0 +REVVLRY Chrome Mobile 73.0 Android 9.0.0 Oppo RMX1801 Chrome Mobile 75.0 Android 9.0.0 XiaoMi Redmi 7 Chrome Mobile 77.0 Android 9.0.0 XiaoMi Redmi Note 7 Chrome Mobile 76.0 Android 9.0.0 @@ -353,7 +353,7 @@ Samsung SM-N975U Samsung Internet 10.1 Android 9.0.0 Samsung SM-N975U1 Samsung Internet 10.1 Android 9.0.0 Samsung SM-T510 Samsung Internet 10.1 Android 9.0.0 Samsung SM-T720 Samsung Internet 10.1 Android 9.0.0 -Other Chrome Mobile WebView 77.0 Android 9.0.0 +SHIELD Android TV Chrome Mobile WebView 77.0 Android 9.0.0 Samsung SM-A102U Chrome Mobile 72.0 Android 9.0.0 Samsung SM-A102U Chrome Mobile 77.0 Android 9.0.0 Samsung SM-A105M Facebook 237.0 Android 9.0.0 @@ -407,7 +407,7 @@ Samsung SM-J600G Facebook 238.0 Android 9.0.0 Samsung SM-J730F Chrome Mobile 77.0 Android 9.0.0 Samsung SM-J737A Chrome Mobile WebView 77.0 Android 9.0.0 Samsung SM-J737A Chrome Mobile 74.0 Android 9.0.0 -Samsung SM-J737V Pinterest 75.0 Android 9.0.0 +Samsung SM-J737V Pinterest 0.0 Android 9.0.0 Samsung SM-J737V Chrome Mobile 77.0 Android 9.0.0 Samsung SM-J810M Chrome Mobile 77.0 Android 9.0.0 Samsung SM-N950U Facebook 240.0 Android 9.0.0 @@ -430,28 +430,28 @@ Samsung SM-N976V Facebook 240.0 Android 9.0.0 Samsung SM-S367VL Chrome Mobile 77.0 Android 9.0.0 Samsung SM-S767VL Chrome Mobile 76.0 Android 9.0.0 Samsung SM-T597P Chrome 77.0 Android 9.0.0 -Other Chrome 77.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 9.0.0 +SM-T720 Chrome 77.0 Android 9.0.0 +TECNO KC8 Chrome Mobile 77.0 Android 9.0.0 Huawei VOG-L29 Chrome Mobile 77.0 Android 9.0.0 -Other Chrome Mobile 74.0 Android 9.0.0 -Other Chrome Mobile WebView 77.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 9.0.0 -Other Facebook 235.0 Android 9.0.0 -Other Chrome Mobile 70.0 Android 9.0.0 -Other Chrome Mobile 75.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 9.0.0 -Other Chrome Mobile 73.0 Android 9.0.0 -Other Chrome Mobile 77.0 Android 9.0.0 +cp3705A Chrome Mobile 74.0 Android 9.0.0 +moto g(6) Chrome Mobile WebView 77.0 Android 9.0.0 +moto g(6) play Chrome Mobile 77.0 Android 9.0.0 +moto g(7) play Facebook 235.0 Android 9.0.0 +moto g(7) play Chrome Mobile 70.0 Android 9.0.0 +moto g(7) power Chrome Mobile 75.0 Android 9.0.0 +moto g(7) power Chrome Mobile 77.0 Android 9.0.0 +moto z4 Chrome Mobile 73.0 Android 9.0.0 +moto z4 Chrome Mobile 77.0 Android 9.0.0 Samsung GT-P3113 Android 4.1 Android 4.1.1 Samsung GT-I8160 Android 4.1 Android 4.1.2 -Other Android 4.2 Android 4.2.2 +Asus Nexus 7 Android 4.2 Android 4.2.2 Samsung SM-E500H Android 4.4 Android 4.4.0 -Other Chrome Mobile WebView 43.0 Android 6.0.1 +LGMS550 Chrome Mobile WebView 43.0 Android 6.0.1 Samsung SM-J737T1 Chrome Mobile WebView 43.0 Android 6.0.1 -Other Opera Mobile 5.3 Android 7.0.0 +TECNO CA6 Opera Mobile 5.3 Android 7.0.0 XiaoMi Redmi 5A MiuiBrowser 9.5 Android 7.1.2 Oppo CPH1911 Chrome Mobile WebView 70.0 Android 9.0.0 -Other Opera Mobile 44.1 Android 9.0.0 +vivo 1904 Opera Mobile 44.1 Android 9.0.0 Mac Firefox 68.0 Mac OS X 10.11.0 Mac Firefox 69.0 Mac OS X 10.13.0 Mac Firefox 67.0 Mac OS X 10.14.0 @@ -514,10 +514,10 @@ Mac Chrome 65.0 Mac OS X 10.9.5 Mac Chrome 66.0 Mac OS X 10.9.5 Mac Chrome 67.0 Mac OS X 10.9.5 PlayStation 4 Apple Mail 605.1 Other 0.0.0 -Other Safari 3.0 Tizen 3.0.0 -Other Samsung Internet 2.0 Tizen 3.0.0 -Other Samsung Internet 2.1 Tizen 4.0.0 -Other Samsung Internet 2.2 Tizen 5.0.0 +Samsung SMART-TV Safari 3.0 Tizen 3.0.0 +Samsung SMART-TV Samsung Internet 2.0 Tizen 3.0.0 +Samsung SMART-TV Samsung Internet 2.1 Tizen 4.0.0 +Samsung SMART-TV Samsung Internet 2.2 Tizen 5.0.0 Other Edge 17.17134 Windows 10.0.0 Other Edge 18.17763 Windows 10.0.0 Other Chrome 77.0 Windows 10.0.0 @@ -622,7 +622,7 @@ Other Firefox 67.0 Ubuntu 0.0.0 iPad Google 22.0 iOS 10.3.3 iPad Chrome Mobile iOS 71.0 iOS 10.3.3 iPad Firefox iOS 14.0 iOS 10.3.3 -iPad Mobile Safari UI/WKWebView 603.3 iOS 10.3.3 +iPad Mobile Safari UI/WKWebView 0.0 iOS 10.3.3 iPad Facebook 240.0 iOS 10.3.3 iPad Mobile Safari 10.0 iOS 10.3.3 iPad Mobile Safari 10.0 iOS 10.3.4 @@ -633,25 +633,25 @@ iPad Mobile Safari 11.0 iOS 11.2.2 iPad Mobile Safari 11.0 iOS 11.2.6 iPad Mobile Safari 11.0 iOS 11.3.0 iPad Mobile Safari 11.0 iOS 11.4.0 -iPad Mobile Safari UI/WKWebView 605.1 iOS 11.4.1 +iPad Mobile Safari UI/WKWebView 0.0 iOS 11.4.1 iPad Mobile Safari 11.0 iOS 11.4.1 iPad Google 83.0 iOS 12.0.0 iPad Mobile Safari 12.0 iOS 12.0.0 iPad Chrome Mobile iOS 75.0 iOS 12.1.0 iPad Chrome Mobile iOS 76.0 iOS 12.1.0 -iPad Mobile Safari UI/WKWebView 605.1 iOS 12.1.0 +iPad Mobile Safari UI/WKWebView 0.0 iOS 12.1.0 iPad Mobile Safari 12.0 iOS 12.1.0 iPad Mobile Safari 12.0 iOS 12.1.1 iPad Google 48.0 iOS 12.1.4 -iPad Mobile Safari UI/WKWebView 605.1 iOS 12.1.4 +iPad Mobile Safari UI/WKWebView 0.0 iOS 12.1.4 iPad Mobile Safari 12.0 iOS 12.1.4 iPad Chrome Mobile iOS 76.0 iOS 12.2.0 -iPad Mobile Safari UI/WKWebView 605.1 iOS 12.2.0 +iPad Mobile Safari UI/WKWebView 0.0 iOS 12.2.0 iPad Mobile Safari 12.1 iOS 12.2.0 iPad Chrome Mobile iOS 77.0 iOS 12.3.0 iPad Google 83.0 iOS 12.3.0 iPad Mobile Safari 12.1 iOS 12.3.0 -iPad Mobile Safari UI/WKWebView 605.1 iOS 12.3.1 +iPad Mobile Safari UI/WKWebView 0.0 iOS 12.3.1 iPad Mobile Safari 12.1 iOS 12.3.1 iPad Chrome Mobile iOS 76.0 iOS 12.4.0 iPad Chrome Mobile iOS 76.0 iOS 12.4.0 @@ -660,15 +660,15 @@ iPad Chrome Mobile iOS 77.0 iOS 12.4.0 iPad Chrome Mobile iOS 77.0 iOS 12.4.0 iPad Google 74.0 iOS 12.4.0 iPad Google 83.0 iOS 12.4.0 -iPad Mobile Safari UI/WKWebView 605.1 iOS 12.4.0 +iPad Mobile Safari UI/WKWebView 0.0 iOS 12.4.0 iPad Mobile Safari 12.1 iOS 12.4.0 iPad Chrome Mobile iOS 67.0 iOS 12.4.1 iPad Firefox iOS 19.0 iOS 12.4.1 -iPad Mobile Safari UI/WKWebView 605.1 iOS 12.4.1 -iPad Facebook 605.1 iOS 12.4.1 -iPad Facebook 605.1 iOS 12.4.1 -iPad Facebook 605.1 iOS 12.4.1 -iPad Facebook 605.1 iOS 12.4.1 +iPad Mobile Safari UI/WKWebView 0.0 iOS 12.4.1 +iPad Facebook 0.0 iOS 12.4.1 +iPad Facebook 0.0 iOS 12.4.1 +iPad Facebook 0.0 iOS 12.4.1 +iPad Facebook 0.0 iOS 12.4.1 iPad Mobile Safari 12.1 iOS 12.4.1 iPad Mobile Safari 6.0 iOS 6.1.3 iPad Mobile Safari 8.0 iOS 8.0.0 @@ -677,7 +677,7 @@ iPad Google 23.1 iOS 8.4.0 iPad Mobile Safari 9.0 iOS 9.3.2 iPad Mobile Safari 9.0 iOS 9.3.5 iPhone Mobile Safari 10.0 iOS 10.2.0 -iPhone Facebook 603.3 iOS 10.3.3 +iPhone Facebook 0.0 iOS 10.3.3 iPhone Google 68.0 iOS 10.3.4 iPhone Mobile Safari 10.0 iOS 10.3.4 iPhone Mobile Safari 11.0 iOS 11.0.3 @@ -686,7 +686,7 @@ iPhone Mobile Safari 11.0 iOS 11.1.2 iPhone Mobile Safari 11.0 iOS 11.2.1 iPhone Facebook 207.0 iOS 11.2.6 iPhone Chrome Mobile iOS 76.0 iOS 11.3.0 -iPhone Facebook 605.1 iOS 11.3.0 +iPhone Facebook 0.0 iOS 11.3.0 iPhone Mobile Safari 11.0 iOS 11.3.0 iPhone Google 83.0 iOS 11.4.0 iPhone Mobile Safari 11.0 iOS 11.4.0 @@ -696,7 +696,7 @@ iPhone Mobile Safari 12.0 iOS 12.0.0 iPhone Mobile Safari 12.0 iOS 12.1.0 iPhone Mobile Safari 12.0 iOS 12.1.1 iPhone Google 74.1 iOS 12.1.2 -iPhone Facebook 605.1 iOS 12.1.2 +iPhone Facebook 0.0 iOS 12.1.2 iPhone Mobile Safari 12.0 iOS 12.1.2 iPhone Mobile Safari 12.0 iOS 12.1.3 iPhone Google 74.1 iOS 12.1.4 @@ -704,18 +704,18 @@ iPhone Mobile Safari 12.0 iOS 12.1.4 iPhone Chrome Mobile iOS 72.0 iOS 12.2.0 iPhone Chrome Mobile iOS 76.0 iOS 12.2.0 iPhone Chrome Mobile iOS 77.0 iOS 12.2.0 -iPhone Facebook 605.1 iOS 12.2.0 -iPhone Facebook 605.1 iOS 12.2.0 +iPhone Facebook 0.0 iOS 12.2.0 +iPhone Facebook 0.0 iOS 12.2.0 iPhone Mobile Safari 12.1 iOS 12.2.0 iPhone Chrome Mobile iOS 77.0 iOS 12.3.0 iPhone Google 83.0 iOS 12.3.0 iPhone Mobile Safari 12.1 iOS 12.3.0 iPhone Google 79.0 iOS 12.3.1 -iPhone Mobile Safari UI/WKWebView 605.1 iOS 12.3.1 -iPhone DuckDuckGo Mobile 7.1 iOS 12.3.1 -iPhone Facebook 605.1 iOS 12.3.1 -iPhone Facebook 605.1 iOS 12.3.1 -iPhone Facebook 605.1 iOS 12.3.1 +iPhone Mobile Safari UI/WKWebView 0.0 iOS 12.3.1 +iPhone DuckDuckGo Mobile 7.0 iOS 12.3.1 +iPhone Facebook 0.0 iOS 12.3.1 +iPhone Facebook 0.0 iOS 12.3.1 +iPhone Facebook 0.0 iOS 12.3.1 iPhone Mobile Safari 12.1 iOS 12.3.1 iPhone Mobile Safari 12.1 iOS 12.3.2 iPhone Chrome Mobile iOS 69.0 iOS 12.4.0 @@ -727,67 +727,67 @@ iPhone Chrome Mobile iOS 77.0 iOS 12.4.0 iPhone Google 81.0 iOS 12.4.0 iPhone Google 82.1 iOS 12.4.0 iPhone Google 83.0 iOS 12.4.0 -iPhone Facebook 605.1 iOS 12.4.0 -iPhone Facebook 605.1 iOS 12.4.0 -iPhone Facebook 605.1 iOS 12.4.0 -iPhone Facebook 605.1 iOS 12.4.0 +iPhone Facebook 0.0 iOS 12.4.0 +iPhone Facebook 0.0 iOS 12.4.0 +iPhone Facebook 0.0 iOS 12.4.0 +iPhone Facebook 0.0 iOS 12.4.0 iPhone Mobile Safari 12.1 iOS 12.4.0 iPhone Google 74.1 iOS 12.4.1 -iPhone Mobile Safari UI/WKWebView 605.1 iOS 12.4.1 +iPhone Mobile Safari UI/WKWebView 0.0 iOS 12.4.1 iPhone Instagram 89.0 iOS 12.4.1 iPhone Facebook 240.0 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 -iPhone Facebook 605.1 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 +iPhone Facebook 0.0 iOS 12.4.1 iPhone Mobile Safari 12.1 iOS 12.4.1 iPhone Mobile Safari 12.4 iOS 12.4.1 -iPhone Mobile Safari UI/WKWebView 605.1 iOS 12.4.2 +iPhone Mobile Safari UI/WKWebView 0.0 iOS 12.4.2 iPhone Mobile Safari 12.1 iOS 12.4.2 iPhone Chrome Mobile iOS 77.0 iOS 13.0.0 -iPhone Facebook 605.1 iOS 13.0.0 -iPhone Facebook 605.1 iOS 13.0.0 -iPhone Facebook 605.1 iOS 13.0.0 -iPhone Facebook 605.1 iOS 13.0.0 -iPhone Facebook 605.1 iOS 13.0.0 -iPhone Facebook 605.1 iOS 13.0.0 +iPhone Facebook 0.0 iOS 13.0.0 +iPhone Facebook 0.0 iOS 13.0.0 +iPhone Facebook 0.0 iOS 13.0.0 +iPhone Facebook 0.0 iOS 13.0.0 +iPhone Facebook 0.0 iOS 13.0.0 +iPhone Facebook 0.0 iOS 13.0.0 iPhone Mobile Safari 13.0 iOS 13.0.0 iPhone Chrome Mobile iOS 76.0 iOS 13.1.0 iPhone Chrome Mobile iOS 77.0 iOS 13.1.0 iPhone Chrome Mobile iOS 77.0 iOS 13.1.0 iPhone Firefox iOS 8.1 iOS 13.1.0 iPhone Google 83.0 iOS 13.1.0 -iPhone Mobile Safari UI/WKWebView 605.1 iOS 13.1.0 -iPhone DuckDuckGo Mobile 7.1 iOS 13.1.0 -iPhone Facebook 605.1 iOS 13.1.0 -iPhone Facebook 605.1 iOS 13.1.0 -iPhone Facebook 605.1 iOS 13.1.0 -iPhone Facebook 605.1 iOS 13.1.0 -iPhone Facebook 605.1 iOS 13.1.0 +iPhone Mobile Safari UI/WKWebView 0.0 iOS 13.1.0 +iPhone DuckDuckGo Mobile 7.0 iOS 13.1.0 +iPhone Facebook 0.0 iOS 13.1.0 +iPhone Facebook 0.0 iOS 13.1.0 +iPhone Facebook 0.0 iOS 13.1.0 +iPhone Facebook 0.0 iOS 13.1.0 +iPhone Facebook 0.0 iOS 13.1.0 iPhone Mobile Safari 13.0 iOS 13.1.0 -iPhone Mobile Safari UI/WKWebView 605.1 iOS 13.1.1 -iPhone Facebook 605.1 iOS 13.1.1 -iPhone Facebook 605.1 iOS 13.1.1 -iPhone Facebook 605.1 iOS 13.1.1 -iPhone Facebook 605.1 iOS 13.1.1 +iPhone Mobile Safari UI/WKWebView 0.0 iOS 13.1.1 +iPhone Facebook 0.0 iOS 13.1.1 +iPhone Facebook 0.0 iOS 13.1.1 +iPhone Facebook 0.0 iOS 13.1.1 +iPhone Facebook 0.0 iOS 13.1.1 iPhone Mobile Safari 13.0 iOS 13.1.1 -iPhone Mobile Safari UI/WKWebView 605.1 iOS 13.1.2 -iPhone Facebook 605.1 iOS 13.1.2 +iPhone Mobile Safari UI/WKWebView 0.0 iOS 13.1.2 +iPhone Facebook 0.0 iOS 13.1.2 iPhone Mobile Safari 13.0 iOS 13.1.2 From 9f26e169633edf1a3bf28e0e85d39719b7127709 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 1 Feb 2023 15:31:06 +0000 Subject: [PATCH 30/79] loose the assertion --- src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index cce459c1ba8..cf19f958b4c 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -484,7 +484,7 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePai column->size(), mark_num, index_granularity.getMarksCount(), index_granularity_rows); } - if (index_granularity_rows > data_part->index_granularity_info.fixed_index_granularity) + if (!settings.blocks_are_granules_size && index_granularity_rows > data_part->index_granularity_info.fixed_index_granularity) { throw Exception(ErrorCodes::LOGICAL_ERROR, "Mark #{} has {} rows, but max fixed granularity is {}, index granularity size {}", @@ -493,6 +493,7 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePai } if (index_granularity_rows != index_granularity.getMarkRows(mark_num)) + { throw Exception( ErrorCodes::LOGICAL_ERROR, "Incorrect mark rows for part {} for mark #{}" @@ -501,6 +502,7 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePai mark_num, offset_in_compressed_file, offset_in_decompressed_block, index_granularity.getMarkRows(mark_num), index_granularity_rows, index_granularity.getMarksCount()); + } auto column = type->createColumn(); From 7c416477ac9532d2822d0b8e057fc0322577a514 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Wed, 1 Feb 2023 17:22:49 +0100 Subject: [PATCH 31/79] do not run wal on remote disks --- src/Storages/MergeTree/MergeTreeData.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index b2e0c14489a..7b2b23c6e6b 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7541,7 +7541,19 @@ MergeTreeData::WriteAheadLogPtr MergeTreeData::getWriteAheadLog() if (!write_ahead_log) { auto reservation = reserveSpace(getSettings()->write_ahead_log_max_bytes); - write_ahead_log = std::make_shared(*this, reservation->getDisk()); + for (auto disk: reservation->getDisks()) + { + if (!disk->isRemote()) + { + write_ahead_log = std::make_shared(*this, reservation->getDisk()); + break; + } + } + + if (!write_ahead_log) + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "Can't store write ahead log in remote disk. It makes no sense."); } return write_ahead_log; From 5b3054492ddad332393a73db547c301633afc1a9 Mon Sep 17 00:00:00 2001 From: Suzy Wang Date: Wed, 1 Feb 2023 15:21:34 -0500 Subject: [PATCH 32/79] Update src/Interpreters/Aggregator.h Co-authored-by: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com> --- src/Interpreters/Aggregator.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 7dec8c9c8a4..3e65d220406 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -209,10 +209,13 @@ struct AggregationMethodOneNumber // Insert the key from the hash table into columns. static void insertKeyIntoColumns(const Key & key, std::vector & key_columns, const Sizes & /*key_sizes*/) { - FieldType casted_key = static_cast(key); - const auto * key_holder = reinterpret_cast(&casted_key); + static_assert(sizeof(FieldType) <= sizeof(Key)); + const auto * key_holder = reinterpret_cast(&key); auto * column = static_cast(key_columns[0]); - column->insertRawData(key_holder); + if constexpr (sizeof(FieldType) < sizeof(Key) && std::endian::native == std::endian::big) + column->insertRawData(key_holder + (sizeof(Key) - sizeof(FieldType))); + else + column->insertRawData(key_holder); } }; From a2e43bc333645f3a437445b00393c3630495537a Mon Sep 17 00:00:00 2001 From: Han Fei Date: Thu, 2 Feb 2023 14:40:09 +0100 Subject: [PATCH 33/79] log for ci --- src/Dictionaries/RegExpTreeDictionary.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index 0cf7d579f96..5d8bb04623a 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -126,7 +126,7 @@ std::vector createStringPieces(const String & value, int num_captur } int ref_num = value[i+1]-'0'; if (ref_num >= num_captures) - LOG_DEBUG(logger, + LOG_TRACE(logger, "Reference Id {} in set string is invalid, the regexp {} only has {} capturing groups", ref_num, regex, num_captures-1); result.push_back(StringPiece(ref_num)); @@ -420,7 +420,7 @@ namespace { struct MatchContext { - std::unordered_set matched_idx_set; + std::set matched_idx_set; std::vector> matched_idx_sorted_list; const std::vector & regexp_ids ; @@ -520,10 +520,11 @@ std::unordered_map RegExpTreeDictionary::matchSearchAllIndice if (node_ptr->match(reinterpret_cast(keys_data.data()) + offset, length)) { match_result.insertNodeID(node_ptr->id); - break; } } + LOG_TRACE(logger, "match string is {}, first matched id is {}, match number is {}", std::string(reinterpret_cast(keys_data.data()) + offset, length), match_result.matched_idx_set.empty() ? -1 : *match_result.matched_idx_set.begin(), match_result.matched_idx_set.size()); + match_result.sort(); // Walk through the regex tree util all attributes are set; From a71ad718122e30d0890cf784e93c54c132b8d3cf Mon Sep 17 00:00:00 2001 From: MeenaRenganathan22 Date: Thu, 2 Feb 2023 10:26:34 -0500 Subject: [PATCH 34/79] Updating the libhdfs3 submoulde to point the branch HDSF_PowerPC --- .gitmodules | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index b4673f113b7..d917d138932 100644 --- a/.gitmodules +++ b/.gitmodules @@ -58,7 +58,8 @@ url = https://github.com/apache/thrift [submodule "contrib/libhdfs3"] path = contrib/libhdfs3 - url = https://github.com/ClickHouse/libhdfs3 + url = https://github.com/ClibMouse/libhdfs3.git + branch = HDFS_PowerPC [submodule "contrib/libxml2"] path = contrib/libxml2 url = https://github.com/GNOME/libxml2 From 7435ba1e3d1ebbdea2f8c2486595ac16fdf10325 Mon Sep 17 00:00:00 2001 From: MeenaRenganathan22 Date: Thu, 2 Feb 2023 17:36:03 -0800 Subject: [PATCH 35/79] Updating the submodule and CMakeLists.txt --- contrib/libhdfs3 | 2 +- contrib/libhdfs3-cmake/CMakeLists.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/contrib/libhdfs3 b/contrib/libhdfs3 index f938c73b401..f3f4b9bc823 160000 --- a/contrib/libhdfs3 +++ b/contrib/libhdfs3 @@ -1 +1 @@ -Subproject commit f938c73b401967df45ae65bb791b601123aac5a4 +Subproject commit f3f4b9bc82356e0d027eb05bbd727988501ee6c2 diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index f9c59f0f016..b270f2090c3 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -47,7 +47,6 @@ set(SRCS "${HDFS3_SOURCE_DIR}/network/TcpSocket.cpp" "${HDFS3_SOURCE_DIR}/network/DomainSocket.cpp" "${HDFS3_SOURCE_DIR}/network/BufferedSocketReader.cpp" - "${HDFS3_SOURCE_DIR}/client/DataReader.cpp" "${HDFS3_SOURCE_DIR}/client/ReadShortCircuitInfo.cpp" "${HDFS3_SOURCE_DIR}/client/Pipeline.cpp" "${HDFS3_SOURCE_DIR}/client/Hdfs.cpp" From 486c10bde1e405a09e9b0ef16f7b2a433358bcca Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 3 Feb 2023 13:11:21 +0100 Subject: [PATCH 36/79] Fix named collections access --- src/Access/Common/AccessType.h | 2 +- .../configs/users.d/users.xml | 3 - .../users.d/users_no_default_access.xml | 9 +++ .../users.d/users_no_default_access2.xml | 10 +++ .../test_named_collections/test.py | 67 ++++++++++++++++++- 5 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 tests/integration/test_named_collections/configs/users.d/users_no_default_access.xml create mode 100644 tests/integration/test_named_collections/configs/users.d/users_no_default_access2.xml diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 366667410d5..8b8284f1185 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -134,7 +134,7 @@ enum class AccessType M(SHOW_QUOTAS, "SHOW CREATE QUOTA", GLOBAL, SHOW_ACCESS) \ M(SHOW_SETTINGS_PROFILES, "SHOW PROFILES, SHOW CREATE SETTINGS PROFILE, SHOW CREATE PROFILE", GLOBAL, SHOW_ACCESS) \ M(SHOW_ACCESS, "", GROUP, ACCESS_MANAGEMENT) \ - M(SHOW_NAMED_COLLECTIONS, "SHOW NAMED COLLECTIONS", GROUP, ACCESS_MANAGEMENT) \ + M(SHOW_NAMED_COLLECTIONS, "SHOW NAMED COLLECTIONS", GLOBAL, ACCESS_MANAGEMENT) \ M(ACCESS_MANAGEMENT, "", GROUP, ALL) \ \ M(SYSTEM_SHUTDOWN, "SYSTEM KILL, SHUTDOWN", GLOBAL, SYSTEM) \ diff --git a/tests/integration/test_named_collections/configs/users.d/users.xml b/tests/integration/test_named_collections/configs/users.d/users.xml index ee38baa3df9..fb5e2028d6e 100644 --- a/tests/integration/test_named_collections/configs/users.d/users.xml +++ b/tests/integration/test_named_collections/configs/users.d/users.xml @@ -2,9 +2,6 @@ - - ::/0 - default default 1 diff --git a/tests/integration/test_named_collections/configs/users.d/users_no_default_access.xml b/tests/integration/test_named_collections/configs/users.d/users_no_default_access.xml new file mode 100644 index 00000000000..b8f38f04ca9 --- /dev/null +++ b/tests/integration/test_named_collections/configs/users.d/users_no_default_access.xml @@ -0,0 +1,9 @@ + + + + + default + default + + + diff --git a/tests/integration/test_named_collections/configs/users.d/users_no_default_access2.xml b/tests/integration/test_named_collections/configs/users.d/users_no_default_access2.xml new file mode 100644 index 00000000000..83dc04f03aa --- /dev/null +++ b/tests/integration/test_named_collections/configs/users.d/users_no_default_access2.xml @@ -0,0 +1,10 @@ + + + + + default + default + 1 + + + diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index ce5c8aaa62e..1b853fadaff 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -24,6 +24,26 @@ def cluster(): ], stay_alive=True, ) + cluster.add_instance( + "node_no_default_access", + main_configs=[ + "configs/config.d/named_collections.xml", + ], + user_configs=[ + "configs/users.d/users_no_default_access.xml", + ], + stay_alive=True, + ) + cluster.add_instance( + "node_no_default_access_but_with_access_management", + main_configs=[ + "configs/config.d/named_collections.xml", + ], + user_configs=[ + "configs/users.d/users_no_default_access2.xml", + ], + stay_alive=True, + ) logging.info("Starting cluster...") cluster.start() @@ -34,7 +54,7 @@ def cluster(): cluster.shutdown() -def replace_config(node, old, new): +def replace_in_server_config(node, old, new): node.replace_in_config( "/etc/clickhouse-server/config.d/named_collections.xml", old, @@ -42,6 +62,49 @@ def replace_config(node, old, new): ) +def replace_in_users_config(node, old, new): + node.replace_in_config( + "/etc/clickhouse-server/users.d/users.xml", + old, + new, + ) + + +def test_access(cluster): + node = cluster.instances["node_no_default_access"] + assert ( + "DB::Exception: default: Not enough privileges. To execute this query it's necessary to have grant SHOW NAMED COLLECTIONS ON *.*" + in node.query_and_get_error("select count() from system.named_collections") + ) + node = cluster.instances["node_no_default_access_but_with_access_management"] + assert ( + "DB::Exception: default: Not enough privileges. To execute this query it's necessary to have grant SHOW NAMED COLLECTIONS ON *.*" + in node.query_and_get_error("select count() from system.named_collections") + ) + + node = cluster.instances["node"] + assert int(node.query("select count() from system.named_collections")) > 0 + replace_in_users_config( + node, "show_named_collections>1", "show_named_collections>0" + ) + assert "show_named_collections>0" in node.exec_in_container( + ["bash", "-c", f"cat /etc/clickhouse-server/users.d/users.xml"] + ) + node.restart_clickhouse() + assert ( + "DB::Exception: default: Not enough privileges. To execute this query it's necessary to have grant SHOW NAMED COLLECTIONS ON *.*" + in node.query_and_get_error("select count() from system.named_collections") + ) + replace_in_users_config( + node, "show_named_collections>0", "show_named_collections>1" + ) + assert "show_named_collections>1" in node.exec_in_container( + ["bash", "-c", f"cat /etc/clickhouse-server/users.d/users.xml"] + ) + node.restart_clickhouse() + assert int(node.query("select count() from system.named_collections")) > 0 + + def test_config_reload(cluster): node = cluster.instances["node"] assert ( @@ -60,7 +123,7 @@ def test_config_reload(cluster): ).strip() ) - replace_config(node, "value1", "value2") + replace_in_server_config(node, "value1", "value2") node.query("SYSTEM RELOAD CONFIG") assert ( From a356c64d9c229a6ec8962a07ad37f4f1902e6766 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 2 Feb 2023 14:37:08 +0100 Subject: [PATCH 37/79] add no-s3-storage to stateless tests with inmemory parts --- tests/queries/0_stateless/01130_in_memory_parts.sql | 2 ++ tests/queries/0_stateless/01130_in_memory_parts_check.sql | 3 +++ tests/queries/0_stateless/01130_in_memory_parts_default.sql | 1 + tests/queries/0_stateless/01130_in_memory_parts_nested.sql | 1 + tests/queries/0_stateless/01130_in_memory_parts_partitons.sql | 2 +- tests/queries/0_stateless/01475_read_subcolumns_storages.sh | 1 + .../01508_race_condition_rename_clear_zookeeper_long.sh | 2 +- tests/queries/0_stateless/01600_parts_types_metrics_long.sh | 2 +- tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql | 2 ++ .../0_stateless/01643_replicated_merge_tree_fsync_smoke.sql | 2 +- tests/queries/0_stateless/02410_inmemory_wal_cleanup.sql | 2 ++ tests/queries/0_stateless/02423_drop_memory_parts.sql | 2 ++ 12 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01130_in_memory_parts.sql b/tests/queries/0_stateless/01130_in_memory_parts.sql index dca12a85841..2b15ae24763 100644 --- a/tests/queries/0_stateless/01130_in_memory_parts.sql +++ b/tests/queries/0_stateless/01130_in_memory_parts.sql @@ -1,3 +1,5 @@ +-- Tags: no-s3-storage + DROP TABLE IF EXISTS in_memory; CREATE TABLE in_memory (a UInt32, b UInt32) ENGINE = MergeTree ORDER BY a diff --git a/tests/queries/0_stateless/01130_in_memory_parts_check.sql b/tests/queries/0_stateless/01130_in_memory_parts_check.sql index 57cd1c83528..c2f5eba5949 100644 --- a/tests/queries/0_stateless/01130_in_memory_parts_check.sql +++ b/tests/queries/0_stateless/01130_in_memory_parts_check.sql @@ -1,4 +1,7 @@ +-- Tags: no-s3-storage + -- Part of 00961_check_table test, but with in-memory parts + SET check_query_single_value_result = 0; DROP TABLE IF EXISTS mt_table; CREATE TABLE mt_table (d Date, key UInt64, data String) ENGINE = MergeTree() PARTITION BY toYYYYMM(d) ORDER BY key diff --git a/tests/queries/0_stateless/01130_in_memory_parts_default.sql b/tests/queries/0_stateless/01130_in_memory_parts_default.sql index 61e20c84f3d..776d5f89fcf 100644 --- a/tests/queries/0_stateless/01130_in_memory_parts_default.sql +++ b/tests/queries/0_stateless/01130_in_memory_parts_default.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage -- Test 01266_default_prewhere_reqq, but with in-memory parts DROP TABLE IF EXISTS t1; diff --git a/tests/queries/0_stateless/01130_in_memory_parts_nested.sql b/tests/queries/0_stateless/01130_in_memory_parts_nested.sql index 76e5a6d7751..c503e8834b4 100644 --- a/tests/queries/0_stateless/01130_in_memory_parts_nested.sql +++ b/tests/queries/0_stateless/01130_in_memory_parts_nested.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage -- Test 00576_nested_and_prewhere, but with in-memory parts. DROP TABLE IF EXISTS nested; diff --git a/tests/queries/0_stateless/01130_in_memory_parts_partitons.sql b/tests/queries/0_stateless/01130_in_memory_parts_partitons.sql index b1ba8bc5560..18da2d2bd30 100644 --- a/tests/queries/0_stateless/01130_in_memory_parts_partitons.sql +++ b/tests/queries/0_stateless/01130_in_memory_parts_partitons.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-s3-storage DROP TABLE IF EXISTS t2; diff --git a/tests/queries/0_stateless/01475_read_subcolumns_storages.sh b/tests/queries/0_stateless/01475_read_subcolumns_storages.sh index be22b1b4185..d770d5118ac 100755 --- a/tests/queries/0_stateless/01475_read_subcolumns_storages.sh +++ b/tests/queries/0_stateless/01475_read_subcolumns_storages.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-s3-storage CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.sh b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.sh index 7db9f22951c..efe24aa3a88 100755 --- a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.sh +++ b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: race, zookeeper +# Tags: race, zookeeper, no-s3-storage CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01600_parts_types_metrics_long.sh b/tests/queries/0_stateless/01600_parts_types_metrics_long.sh index 6e4ccbdd325..05edf02f7ed 100755 --- a/tests/queries/0_stateless/01600_parts_types_metrics_long.sh +++ b/tests/queries/0_stateless/01600_parts_types_metrics_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long +# Tags: long, no-s3-storage CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql index 598e1ef3c34..ad0dfca0db2 100644 --- a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql +++ b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql @@ -1,3 +1,5 @@ +-- Tags: no-s3-storage + drop table if exists data_01643; select 'default'; diff --git a/tests/queries/0_stateless/01643_replicated_merge_tree_fsync_smoke.sql b/tests/queries/0_stateless/01643_replicated_merge_tree_fsync_smoke.sql index dadd7eaba6c..bcce87e11db 100644 --- a/tests/queries/0_stateless/01643_replicated_merge_tree_fsync_smoke.sql +++ b/tests/queries/0_stateless/01643_replicated_merge_tree_fsync_smoke.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-s3-storage -- no-parallel -- for flaky check and to avoid "Removing leftovers from table" (for other tables) -- Temporarily skip warning 'table was created by another server at the same moment, will retry' diff --git a/tests/queries/0_stateless/02410_inmemory_wal_cleanup.sql b/tests/queries/0_stateless/02410_inmemory_wal_cleanup.sql index 0228852a115..7f832d980ba 100644 --- a/tests/queries/0_stateless/02410_inmemory_wal_cleanup.sql +++ b/tests/queries/0_stateless/02410_inmemory_wal_cleanup.sql @@ -1,3 +1,5 @@ +-- Tags: no-s3-storage + -- { echo } DROP TABLE IF EXISTS in_memory; diff --git a/tests/queries/0_stateless/02423_drop_memory_parts.sql b/tests/queries/0_stateless/02423_drop_memory_parts.sql index fad81d46e68..9326f159b0c 100644 --- a/tests/queries/0_stateless/02423_drop_memory_parts.sql +++ b/tests/queries/0_stateless/02423_drop_memory_parts.sql @@ -1,3 +1,5 @@ +-- Tags: no-s3-storage + DROP TABLE IF EXISTS table_in_memory; CREATE TABLE table_in_memory From ed00db75801e5482ecaf7d0e7034e75789307f96 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 3 Feb 2023 13:15:06 +0000 Subject: [PATCH 38/79] Update sorting properties after reading in order applied --- src/Processors/QueryPlan/DistinctStep.cpp | 7 +++- .../Optimizations/optimizeReadInOrder.cpp | 41 ++++++++++++++----- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/Processors/QueryPlan/DistinctStep.cpp b/src/Processors/QueryPlan/DistinctStep.cpp index e3d29256c23..323ef0bbdab 100644 --- a/src/Processors/QueryPlan/DistinctStep.cpp +++ b/src/Processors/QueryPlan/DistinctStep.cpp @@ -103,7 +103,12 @@ void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const Buil return nullptr; return std::make_shared( - header, set_size_limits, limit_hint, distinct_sort_desc, columns, false); + header, + set_size_limits, + limit_hint, + distinct_sort_desc, + columns, + input_stream.sort_scope == DataStream::SortScope::Stream); }); return; } diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index bdf8f24f9d6..675436fc2d9 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -63,20 +63,25 @@ ISourceStep * checkSupportedReadingStep(IQueryPlanStep * step) return nullptr; } -QueryPlan::Node * findReadingStep(QueryPlan::Node & node) +QueryPlan::Node * findReadingStep(QueryPlan::Node & node, std::vector& backward_path) { IQueryPlanStep * step = node.step.get(); if (auto * reading = checkSupportedReadingStep(step)) + { + backward_path.push_back(node.step.get()); return &node; + } if (node.children.size() != 1) return nullptr; + backward_path.push_back(node.step.get()); + if (typeid_cast(step) || typeid_cast(step) || typeid_cast(step)) - return findReadingStep(*node.children.front()); + return findReadingStep(*node.children.front(), backward_path); if (auto * distinct = typeid_cast(step); distinct && distinct->isPreliminary()) - return findReadingStep(*node.children.front()); + return findReadingStep(*node.children.front(), backward_path); return nullptr; } @@ -987,9 +992,9 @@ AggregationInputOrder buildInputOrderInfo( return order_info; } -InputOrderInfoPtr buildInputOrderInfo(SortingStep & sorting, QueryPlan::Node & node) +InputOrderInfoPtr buildInputOrderInfo(SortingStep & sorting, QueryPlan::Node & node, std::vector& backward_path) { - QueryPlan::Node * reading_node = findReadingStep(node); + QueryPlan::Node * reading_node = findReadingStep(node, backward_path); if (!reading_node) return nullptr; @@ -1035,9 +1040,9 @@ InputOrderInfoPtr buildInputOrderInfo(SortingStep & sorting, QueryPlan::Node & n return nullptr; } -AggregationInputOrder buildInputOrderInfo(AggregatingStep & aggregating, QueryPlan::Node & node) +AggregationInputOrder buildInputOrderInfo(AggregatingStep & aggregating, QueryPlan::Node & node, std::vector& backward_path) { - QueryPlan::Node * reading_node = findReadingStep(node); + QueryPlan::Node * reading_node = findReadingStep(node, backward_path); if (!reading_node) return {}; @@ -1096,6 +1101,7 @@ void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes) if (sorting->getType() != SortingStep::Type::Full) return; + std::vector steps_to_update; if (typeid_cast(node.children.front()->step.get())) { auto & union_node = node.children.front(); @@ -1105,7 +1111,7 @@ void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes) infos.reserve(node.children.size()); for (auto * child : union_node->children) { - infos.push_back(buildInputOrderInfo(*sorting, *child)); + infos.push_back(buildInputOrderInfo(*sorting, *child, steps_to_update)); if (infos.back() && (!max_sort_descr || max_sort_descr->size() < infos.back()->sort_description_for_merging.size())) max_sort_descr = &infos.back()->sort_description_for_merging; @@ -1155,9 +1161,23 @@ void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes) sorting->convertToFinishSorting(*max_sort_descr); } - else if (auto order_info = buildInputOrderInfo(*sorting, *node.children.front())) + else if (auto order_info = buildInputOrderInfo(*sorting, *node.children.front(), steps_to_update)) { sorting->convertToFinishSorting(order_info->sort_description_for_merging); + + /// update data stream's sorting properties for found transforms + if (!steps_to_update.empty()) + { + const DataStream * input_stream = &steps_to_update.back()->getOutputStream(); + steps_to_update.pop_back(); + + while (!steps_to_update.empty()) + { + dynamic_cast(steps_to_update.back())->updateInputStream(*input_stream); + input_stream = &steps_to_update.back()->getOutputStream(); + steps_to_update.pop_back(); + } + } } } @@ -1174,7 +1194,8 @@ void optimizeAggregationInOrder(QueryPlan::Node & node, QueryPlan::Nodes &) return; /// TODO: maybe add support for UNION later. - if (auto order_info = buildInputOrderInfo(*aggregating, *node.children.front()); order_info.input_order) + std::vector steps_to_update; + if (auto order_info = buildInputOrderInfo(*aggregating, *node.children.front(), steps_to_update); order_info.input_order) { aggregating->applyOrder(std::move(order_info.sort_description_for_merging), std::move(order_info.group_by_sort_description)); } From 2656027c9fa47de2eb1223a642182dac9200b1c9 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Fri, 3 Feb 2023 14:25:53 +0100 Subject: [PATCH 39/79] make it work if we dont define use_vectorscan macro --- src/Dictionaries/RegExpTreeDictionary.cpp | 50 +++++++++---------- .../02504_regexp_dictionary_ua_parser.sh | 2 +- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index 5d8bb04623a..f094e5ebcd4 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -173,6 +173,7 @@ namespace return false; } + [[maybe_unused]] bool check(const String & data) const { @@ -251,15 +252,15 @@ void RegExpTreeDictionary::initRegexNodes(Block & block) } } regex_nodes.emplace(id, node); - if (checker.check(regex)) - { - complex_regexp_nodes.push_back(node); - } - else +#if USE_VECTORSCAN + if (!checker.check(regex)) { regexps.push_back(regex); regexp_ids.push_back(id); } + else +#endif + complex_regexp_nodes.push_back(node); } } @@ -308,7 +309,7 @@ void RegExpTreeDictionary::loadData() initRegexNodes(block); } initGraph(); - if (regexps.empty()) + if (regexps.empty() && complex_regexp_nodes.empty()) throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION, "There are no available regular expression. Please check your config"); LOG_INFO(logger, "There are {} simple regexps and {} complex regexps", regexps.size(), complex_regexp_nodes.size()); #if USE_VECTORSCAN @@ -415,7 +416,6 @@ bool RegExpTreeDictionary::setAttributes( return attributes_to_set.size() == attributes.size(); } -#if USE_VECTORSCAN namespace { struct MatchContext @@ -429,6 +429,7 @@ namespace MatchContext(const std::vector & regexp_ids_, const std::unordered_map & topology_order_) : regexp_ids(regexp_ids_), topology_order(topology_order_) {} + [[maybe_unused]] void insertIdx(unsigned int idx) { UInt64 node_id = regexp_ids[idx-1]; @@ -456,14 +457,14 @@ namespace } }; } -#endif // USE_VECTORSCAN std::unordered_map RegExpTreeDictionary::matchSearchAllIndices( - [[maybe_unused]] const ColumnString::Chars & keys_data, - [[maybe_unused]] const ColumnString::Offsets & keys_offsets, - [[maybe_unused]] const std::unordered_map & attributes, - [[maybe_unused]] const std::unordered_map & defaults) const + const ColumnString::Chars & keys_data, + const ColumnString::Offsets & keys_offsets, + const std::unordered_map & attributes, + const std::unordered_map & defaults) const { + #if USE_VECTORSCAN hs_scratch_t * scratch = nullptr; hs_error_t err = hs_clone_scratch(hyperscan_regex->get()->getScratch(), &scratch); @@ -474,6 +475,16 @@ std::unordered_map RegExpTreeDictionary::matchSearchAllIndice } MultiRegexps::ScratchPtr smart_scratch(scratch); + auto on_match = [](unsigned int id, + unsigned long long /* from */, // NOLINT + unsigned long long /* to */, // NOLINT + unsigned int /* flags */, + void * context) -> int + { + static_cast(context)->insertIdx(id); + return 0; + }; +#endif std::unordered_map columns; @@ -485,16 +496,6 @@ std::unordered_map RegExpTreeDictionary::matchSearchAllIndice columns[name] = std::move(col_ptr); } - auto on_match = [](unsigned int id, - unsigned long long /* from */, // NOLINT - unsigned long long /* to */, // NOLINT - unsigned int /* flags */, - void * context) -> int - { - static_cast(context)->insertIdx(id); - return 0; - }; - UInt64 offset = 0; for (size_t key_idx = 0; key_idx < keys_offsets.size(); ++key_idx) { @@ -503,6 +504,7 @@ std::unordered_map RegExpTreeDictionary::matchSearchAllIndice MatchContext match_result(regexp_ids, topology_order); +#if USE_VECTORSCAN err = hs_scan( hyperscan_regex->get()->getDB(), reinterpret_cast(keys_data.data()) + offset, @@ -514,6 +516,7 @@ std::unordered_map RegExpTreeDictionary::matchSearchAllIndice if (err != HS_SUCCESS) throw Exception(ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT, "Failed to scan data with vectorscan"); +#endif for (const auto & node_ptr : complex_regexp_nodes) { @@ -577,9 +580,6 @@ std::unordered_map RegExpTreeDictionary::matchSearchAllIndice result.emplace(name, std::move(mutable_ptr)); return result; -#else - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Multi search all indices is not implemented when USE_VECTORSCAN is off"); -#endif // USE_VECTORSCAN } Columns RegExpTreeDictionary::getColumns( diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh index 4d372bf1ce6..8e361a191d6 100755 --- a/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh +++ b/tests/queries/0_stateless/02504_regexp_dictionary_ua_parser.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Tags: use-vectorscan, no-fasttest, no-parallel +# Tags: no-fasttest, no-parallel CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From 0d77f29a990d63d6434e8018ac2ff604d60679ff Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Fri, 3 Feb 2023 15:22:02 +0100 Subject: [PATCH 40/79] Update .reference --- tests/queries/0_stateless/01271_show_privileges.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index f2c3e8eda9d..af4efb357d0 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -89,7 +89,7 @@ SHOW ROW POLICIES ['SHOW POLICIES','SHOW CREATE ROW POLICY','SHOW CREATE POLICY' SHOW QUOTAS ['SHOW CREATE QUOTA'] GLOBAL SHOW ACCESS SHOW SETTINGS PROFILES ['SHOW PROFILES','SHOW CREATE SETTINGS PROFILE','SHOW CREATE PROFILE'] GLOBAL SHOW ACCESS SHOW ACCESS [] \N ACCESS MANAGEMENT -SHOW NAMED COLLECTIONS ['SHOW NAMED COLLECTIONS'] \N ACCESS MANAGEMENT +SHOW NAMED COLLECTIONS ['SHOW NAMED COLLECTIONS'] GLOBAL ACCESS MANAGEMENT ACCESS MANAGEMENT [] \N ALL SYSTEM SHUTDOWN ['SYSTEM KILL','SHUTDOWN'] GLOBAL SYSTEM SYSTEM DROP DNS CACHE ['SYSTEM DROP DNS','DROP DNS CACHE','DROP DNS'] GLOBAL SYSTEM DROP CACHE From f49f5d7091128b92a7a1c82cf5e144e32e413b7b Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 3 Feb 2023 17:10:31 +0000 Subject: [PATCH 41/79] Update tests + updadte sorting properties after applying aggregation in order --- .../Optimizations/optimizeReadInOrder.cpp | 46 +++++++++++-------- ...ct_in_order_optimization_explain.reference | 6 +++ ..._input_stream_properties_explain.reference | 8 ++-- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index 675436fc2d9..e2feb66c823 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -63,7 +63,9 @@ ISourceStep * checkSupportedReadingStep(IQueryPlanStep * step) return nullptr; } -QueryPlan::Node * findReadingStep(QueryPlan::Node & node, std::vector& backward_path) +using StepStack = std::vector; + +QueryPlan::Node * findReadingStep(QueryPlan::Node & node, StepStack & backward_path) { IQueryPlanStep * step = node.step.get(); if (auto * reading = checkSupportedReadingStep(step)) @@ -86,6 +88,24 @@ QueryPlan::Node * findReadingStep(QueryPlan::Node & node, std::vectorgetOutputStream(); + chassert(dynamic_cast(steps_to_update.back())); + steps_to_update.pop_back(); + + while (!steps_to_update.empty()) + { + dynamic_cast(steps_to_update.back())->updateInputStream(*input_stream); + input_stream = &steps_to_update.back()->getOutputStream(); + steps_to_update.pop_back(); + } + } +} + /// FixedColumns are columns which values become constants after filtering. /// In a query "SELECT x, y, z FROM table WHERE x = 1 AND y = 'a' ORDER BY x, y, z" /// Fixed columns are 'x' and 'y'. @@ -992,7 +1012,7 @@ AggregationInputOrder buildInputOrderInfo( return order_info; } -InputOrderInfoPtr buildInputOrderInfo(SortingStep & sorting, QueryPlan::Node & node, std::vector& backward_path) +InputOrderInfoPtr buildInputOrderInfo(SortingStep & sorting, QueryPlan::Node & node, StepStack & backward_path) { QueryPlan::Node * reading_node = findReadingStep(node, backward_path); if (!reading_node) @@ -1040,7 +1060,7 @@ InputOrderInfoPtr buildInputOrderInfo(SortingStep & sorting, QueryPlan::Node & n return nullptr; } -AggregationInputOrder buildInputOrderInfo(AggregatingStep & aggregating, QueryPlan::Node & node, std::vector& backward_path) +AggregationInputOrder buildInputOrderInfo(AggregatingStep & aggregating, QueryPlan::Node & node, StepStack & backward_path) { QueryPlan::Node * reading_node = findReadingStep(node, backward_path); if (!reading_node) @@ -1101,7 +1121,7 @@ void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes) if (sorting->getType() != SortingStep::Type::Full) return; - std::vector steps_to_update; + StepStack steps_to_update; if (typeid_cast(node.children.front()->step.get())) { auto & union_node = node.children.front(); @@ -1164,20 +1184,8 @@ void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes) else if (auto order_info = buildInputOrderInfo(*sorting, *node.children.front(), steps_to_update)) { sorting->convertToFinishSorting(order_info->sort_description_for_merging); - - /// update data stream's sorting properties for found transforms - if (!steps_to_update.empty()) - { - const DataStream * input_stream = &steps_to_update.back()->getOutputStream(); - steps_to_update.pop_back(); - - while (!steps_to_update.empty()) - { - dynamic_cast(steps_to_update.back())->updateInputStream(*input_stream); - input_stream = &steps_to_update.back()->getOutputStream(); - steps_to_update.pop_back(); - } - } + /// update data stream's sorting properties + updateStepsDataStreams(steps_to_update); } } @@ -1198,6 +1206,8 @@ void optimizeAggregationInOrder(QueryPlan::Node & node, QueryPlan::Nodes &) if (auto order_info = buildInputOrderInfo(*aggregating, *node.children.front(), steps_to_update); order_info.input_order) { aggregating->applyOrder(std::move(order_info.sort_description_for_merging), std::move(order_info.group_by_sort_description)); + /// update data stream's sorting properties + updateStepsDataStreams(steps_to_update); } } diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.reference b/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.reference index f85b9cd9e86..2511c806e1b 100644 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.reference +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.reference @@ -59,6 +59,8 @@ Sorting (Stream): a ASC, b ASC -- check that reading in order optimization for ORDER BY and DISTINCT applied correctly in the same query -- disabled, check that sorting description for ReadFromMergeTree match ORDER BY columns Sorting (Stream): a ASC +Sorting (Stream): a ASC +Sorting (Stream): a ASC -- enabled, check that ReadFromMergeTree sorting description is overwritten by DISTINCT optimization i.e. it contains columns from DISTINCT clause Sorting (Stream): a ASC, b ASC Sorting (Stream): a ASC, b ASC @@ -69,8 +71,12 @@ Sorting (Stream): a DESC, b DESC Sorting (Stream): a DESC, b DESC -- enabled, check that ReadFromMergeTree sorting description is NOT overwritten by DISTINCT optimization (1), - it contains columns from ORDER BY clause Sorting (Stream): a ASC, b ASC +Sorting (Stream): a ASC, b ASC +Sorting (Stream): a ASC, b ASC -- enabled, check that ReadFromMergeTree sorting description is NOT overwritten by DISTINCT optimization (2), - direction used from ORDER BY clause Sorting (Stream): a DESC, b DESC +Sorting (Stream): a DESC, b DESC +Sorting (Stream): a DESC, b DESC -- enabled, check that disabling other 'read in order' optimizations do not disable distinct in order optimization Sorting (Stream): a ASC, b ASC Sorting (Stream): a ASC, b ASC diff --git a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.reference b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.reference index bafa70556e7..9d78707429e 100644 --- a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.reference +++ b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.reference @@ -25,7 +25,7 @@ PartialSortingTransform × 3 Sorting (Global): a ASC Sorting (Sorting for ORDER BY) Sorting (Global): a ASC -Sorting (Chunk): a ASC +Sorting (Stream): a ASC Sorting (Stream): a ASC -- QUERY: set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a FROM optimize_sorting ORDER BY a+1 Sorting (None) @@ -70,14 +70,14 @@ Sorting (Global): a ASC Sorting (None) Sorting (Sorting for ORDER BY) Sorting (Global): a ASC -Sorting (Chunk): a ASC +Sorting (Stream): a ASC Sorting (Stream): a ASC -- aliases DONT break sorting order -- QUERY: set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a, b FROM (SELECT x AS a, y AS b FROM (SELECT a AS x, b AS y FROM optimize_sorting) ORDER BY x, y) Sorting (Global): x ASC, y ASC Sorting (Sorting for ORDER BY) Sorting (Global): x ASC, y ASC -Sorting (Chunk): a ASC, b ASC +Sorting (Stream): a ASC, b ASC Sorting (Stream): a ASC, b ASC -- actions chain breaks sorting order: input(column a)->sipHash64(column a)->alias(sipHash64(column a), a)->plus(alias a, 1) -- QUERY: set optimize_read_in_order=1;set max_threads=3;set query_plan_remove_redundant_sorting=0;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a, z FROM (SELECT sipHash64(a) AS a, a + 1 AS z FROM (SELECT a FROM optimize_sorting ORDER BY a + 1)) ORDER BY a + 1 @@ -94,5 +94,5 @@ Sorting (Chunk): a ASC Sorting (Global): a ASC Sorting (Sorting for ORDER BY) Sorting (Global): a ASC -Sorting (Chunk): a ASC, b ASC +Sorting (Stream): a ASC Sorting (Stream): a ASC From 722e389847170c665ae0d9071dfdedf9251e79e3 Mon Sep 17 00:00:00 2001 From: MeenaRenganathan22 Date: Fri, 3 Feb 2023 17:45:48 -0800 Subject: [PATCH 42/79] Updating the submodule to the local branch HDFS_PowerPC --- contrib/libhdfs3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libhdfs3 b/contrib/libhdfs3 index f3f4b9bc823..30df3beedd2 160000 --- a/contrib/libhdfs3 +++ b/contrib/libhdfs3 @@ -1 +1 @@ -Subproject commit f3f4b9bc82356e0d027eb05bbd727988501ee6c2 +Subproject commit 30df3beedd226f598051126f72a5ed760232d662 From 9ea3de14ceb8f2a61fe0ea9ffac0159fdd977561 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Sat, 4 Feb 2023 10:53:54 +0100 Subject: [PATCH 43/79] use re2 by default --- src/Core/Settings.h | 1 + src/Dictionaries/RegExpTreeDictionary.cpp | 88 +++++++++++++---------- src/Dictionaries/RegExpTreeDictionary.h | 21 +++--- 3 files changed, 66 insertions(+), 44 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index dbe905ee962..5897b62defe 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -891,6 +891,7 @@ class IColumn; M(Bool, input_format_bson_skip_fields_with_unsupported_types_in_schema_inference, false, "Skip fields with unsupported types while schema inference for format BSON.", 0) \ \ M(Bool, regexp_dict_allow_other_sources, false, "Allow regexp_tree dictionary to use sources other than yaml source.", 0) \ + M(Bool, regexp_dict_allow_hyperscan, false, "Allow regexp_tree dictionary using Hyperscan library.", 0) \ // End of FORMAT_FACTORY_SETTINGS // Please add settings non-related to formats into the COMMON_SETTINGS above. diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index f094e5ebcd4..ecb7a009b67 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -145,7 +145,7 @@ std::vector createStringPieces(const String & value, int num_captur void RegExpTreeDictionary::calculateBytesAllocated() { - for (const String & regex : regexps) + for (const String & regex : simple_regexps) bytes_allocated += regex.size(); bytes_allocated += sizeof(UInt64) * regexp_ids.size(); bytes_allocated += (sizeof(RegexTreeNode) + sizeof(UInt64)) * regex_nodes.size(); @@ -253,9 +253,9 @@ void RegExpTreeDictionary::initRegexNodes(Block & block) } regex_nodes.emplace(id, node); #if USE_VECTORSCAN - if (!checker.check(regex)) + if (use_vectorscan && !checker.check(regex)) { - regexps.push_back(regex); + simple_regexps.push_back(regex); regexp_ids.push_back(id); } else @@ -309,13 +309,15 @@ void RegExpTreeDictionary::loadData() initRegexNodes(block); } initGraph(); - if (regexps.empty() && complex_regexp_nodes.empty()) + if (simple_regexps.empty() && complex_regexp_nodes.empty()) throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION, "There are no available regular expression. Please check your config"); - LOG_INFO(logger, "There are {} simple regexps and {} complex regexps", regexps.size(), complex_regexp_nodes.size()); + LOG_INFO(logger, "There are {} simple regexps and {} complex regexps", simple_regexps.size(), complex_regexp_nodes.size()); + if (!use_vectorscan) + return; #if USE_VECTORSCAN try { - std::vector regexps_views(regexps.begin(), regexps.end()); + std::vector regexps_views(simple_regexps.begin(), simple_regexps.end()); hyperscan_regex = MultiRegexps::getOrSet(regexps_views, std::nullopt); hyperscan_regex->get(); } @@ -323,7 +325,6 @@ void RegExpTreeDictionary::loadData() { /// Some compile errors will be thrown as LOGICAL ERROR and cause crash, e.g. empty expression or expressions are too large. /// We catch the error here and rethrow again. - /// TODO: fallback to other engine, like re2, when exceptions occur. throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION, "Error occurs when compiling regular expressions, reason: {}", e.message()); } #endif @@ -335,8 +336,17 @@ void RegExpTreeDictionary::loadData() } RegExpTreeDictionary::RegExpTreeDictionary( - const StorageID & id_, const DictionaryStructure & structure_, DictionarySourcePtr source_ptr_, Configuration configuration_) - : IDictionary(id_), structure(structure_), source_ptr(source_ptr_), configuration(configuration_), logger(&Poco::Logger::get("RegExpTreeDictionary")) + const StorageID & id_, + const DictionaryStructure & structure_, + DictionarySourcePtr source_ptr_, + Configuration configuration_, + bool use_vectorscan_) + : IDictionary(id_), + structure(structure_), + source_ptr(source_ptr_), + configuration(configuration_), + use_vectorscan(use_vectorscan_), + logger(&Poco::Logger::get("RegExpTreeDictionary")) { if (auto * ch_source = typeid_cast(source_ptr.get())) { @@ -370,7 +380,7 @@ std::pair processBackRefs(const String & data, const re2_st::RE2 & else result += item.literal; } - return std::make_pair(result, false); + return {result, false}; } // walk towards root and collect attributes. @@ -458,7 +468,7 @@ namespace }; } -std::unordered_map RegExpTreeDictionary::matchSearchAllIndices( +std::unordered_map RegExpTreeDictionary::match( const ColumnString::Chars & keys_data, const ColumnString::Offsets & keys_offsets, const std::unordered_map & attributes, @@ -467,23 +477,17 @@ std::unordered_map RegExpTreeDictionary::matchSearchAllIndice #if USE_VECTORSCAN hs_scratch_t * scratch = nullptr; - hs_error_t err = hs_clone_scratch(hyperscan_regex->get()->getScratch(), &scratch); - - if (err != HS_SUCCESS) + if (use_vectorscan) { - throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Could not clone scratch space for hyperscan"); + hs_error_t err = hs_clone_scratch(hyperscan_regex->get()->getScratch(), &scratch); + + if (err != HS_SUCCESS) + { + throw Exception(ErrorCodes::CANNOT_ALLOCATE_MEMORY, "Could not clone scratch space for hyperscan"); + } } MultiRegexps::ScratchPtr smart_scratch(scratch); - auto on_match = [](unsigned int id, - unsigned long long /* from */, // NOLINT - unsigned long long /* to */, // NOLINT - unsigned int /* flags */, - void * context) -> int - { - static_cast(context)->insertIdx(id); - return 0; - }; #endif std::unordered_map columns; @@ -505,17 +509,29 @@ std::unordered_map RegExpTreeDictionary::matchSearchAllIndice MatchContext match_result(regexp_ids, topology_order); #if USE_VECTORSCAN - err = hs_scan( - hyperscan_regex->get()->getDB(), - reinterpret_cast(keys_data.data()) + offset, - static_cast(length), - 0, - smart_scratch.get(), - on_match, - &match_result); + if (use_vectorscan) + { + auto on_match = [](unsigned int id, + unsigned long long /* from */, // NOLINT + unsigned long long /* to */, // NOLINT + unsigned int /* flags */, + void * context) -> int + { + static_cast(context)->insertIdx(id); + return 0; + }; + hs_error_t err = hs_scan( + hyperscan_regex->get()->getDB(), + reinterpret_cast(keys_data.data()) + offset, + static_cast(length), + 0, + smart_scratch.get(), + on_match, + &match_result); - if (err != HS_SUCCESS) - throw Exception(ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT, "Failed to scan data with vectorscan"); + if (err != HS_SUCCESS) + throw Exception(ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT, "Failed to scan data with vectorscan"); + } #endif for (const auto & node_ptr : complex_regexp_nodes) @@ -608,7 +624,7 @@ Columns RegExpTreeDictionary::getColumns( /// calculate matches const ColumnString * key_column = typeid_cast(key_columns[0].get()); - const auto & columns_map = matchSearchAllIndices( + const auto & columns_map = match( key_column->getChars(), key_column->getOffsets(), attributes, @@ -653,7 +669,7 @@ void registerDictionaryRegExpTree(DictionaryFactory & factory) "regexp_tree dictionary doesn't accept sources other than yaml source. " "To active it, please set regexp_dict_allow_other_sources=true"); - return std::make_unique(dict_id, dict_struct, std::move(source_ptr), configuration); + return std::make_unique(dict_id, dict_struct, std::move(source_ptr), configuration, context->getSettings().regexp_dict_allow_hyperscan); }; factory.registerLayout("regexp_tree", create_layout, true); diff --git a/src/Dictionaries/RegExpTreeDictionary.h b/src/Dictionaries/RegExpTreeDictionary.h index 29c8b461324..32206f25429 100644 --- a/src/Dictionaries/RegExpTreeDictionary.h +++ b/src/Dictionaries/RegExpTreeDictionary.h @@ -43,7 +43,11 @@ public: const std::string name = "RegExpTree"; RegExpTreeDictionary( - const StorageID & id_, const DictionaryStructure & structure_, DictionarySourcePtr source_ptr_, Configuration configuration_); + const StorageID & id_, + const DictionaryStructure & structure_, + DictionarySourcePtr source_ptr_, + Configuration configuration_, + bool use_vectorscan_); std::string getTypeName() const override { return name; } @@ -79,7 +83,7 @@ public: std::shared_ptr clone() const override { - return std::make_shared(getDictionaryID(), structure, source_ptr->clone(), configuration); + return std::make_shared(getDictionaryID(), structure, source_ptr->clone(), configuration, use_vectorscan); } ColumnUInt8::Ptr hasKeys(const Columns &, const DataTypes &) const override @@ -122,11 +126,6 @@ private: mutable std::atomic query_count{0}; mutable std::atomic found_count{0}; - std::vector regexps; - std::vector regexp_ids; - - Poco::Logger * logger; - void calculateBytesAllocated(); void loadData(); @@ -135,7 +134,7 @@ private: void initTopologyOrder(UInt64 node_idx, std::set & visited, UInt64 & topology_id); void initGraph(); - std::unordered_map matchSearchAllIndices( + std::unordered_map match( const ColumnString::Chars & keys_data, const ColumnString::Offsets & keys_offsets, const std::unordered_map & attributes, @@ -153,6 +152,10 @@ private: struct RegexTreeNode; using RegexTreeNodePtr = std::shared_ptr; + bool use_vectorscan; + + std::vector simple_regexps; + std::vector regexp_ids; std::vector complex_regexp_nodes; std::map regex_nodes; @@ -160,6 +163,8 @@ private: #if USE_VECTORSCAN MultiRegexps::DeferredConstructedRegexpsPtr hyperscan_regex; #endif + + Poco::Logger * logger; }; } From cb4757fc43e9a773b32dff5e6dfe189d01acfe67 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 4 Feb 2023 22:54:49 +0100 Subject: [PATCH 44/79] Fix tests --- .../test_create_query_constraints/configs/users.xml | 10 ++++++++++ .../integration/test_create_query_constraints/test.py | 8 ++++++-- tests/integration/test_grant_and_revoke/test.py | 8 +++++--- .../configs/users.d/users.xml | 10 ++++++++++ .../test_settings_constraints_distributed/test.py | 6 +++--- 5 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 tests/integration/test_create_query_constraints/configs/users.xml create mode 100644 tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml diff --git a/tests/integration/test_create_query_constraints/configs/users.xml b/tests/integration/test_create_query_constraints/configs/users.xml new file mode 100644 index 00000000000..fb5e2028d6e --- /dev/null +++ b/tests/integration/test_create_query_constraints/configs/users.xml @@ -0,0 +1,10 @@ + + + + + default + default + 1 + + + diff --git a/tests/integration/test_create_query_constraints/test.py b/tests/integration/test_create_query_constraints/test.py index 997671a6996..54173528aa3 100644 --- a/tests/integration/test_create_query_constraints/test.py +++ b/tests/integration/test_create_query_constraints/test.py @@ -7,8 +7,12 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import assert_eq_with_retry, TSV cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance("instance") - +instance = cluster.add_instance( + "instance", + user_configs=[ + "configs/users.xml", + ], +) @pytest.fixture(scope="module", autouse=True) def start_cluster(): diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index d0a2f2ab933..e5cb47cbc80 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -377,6 +377,8 @@ def test_introspection(): instance.query("GRANT SELECT ON test.table TO A") instance.query("GRANT CREATE ON *.* TO B WITH GRANT OPTION") + all_access_except_show_named_collections="SHOW, SELECT, INSERT, ALTER, CREATE, DROP, TRUNCATE, OPTIMIZE, BACKUP, KILL QUERY, KILL TRANSACTION, MOVE PARTITION BETWEEN SHARDS, CREATE USER, ALTER USER, DROP USER, CREATE ROLE, ALTER ROLE, DROP ROLE, ROLE ADMIN, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, CREATE QUOTA, ALTER QUOTA, DROP QUOTA, CREATE SETTINGS PROFILE, ALTER SETTINGS PROFILE, DROP SETTINGS PROFILE, SHOW ACCESS, SYSTEM, dictGet, INTROSPECTION, SOURCES, CLUSTER" + assert instance.query("SHOW USERS") == TSV(["A", "B", "default"]) assert instance.query("SHOW CREATE USERS A") == TSV(["CREATE USER A"]) assert instance.query("SHOW CREATE USERS B") == TSV(["CREATE USER B"]) @@ -413,7 +415,7 @@ def test_introspection(): [ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION", - "GRANT ALL ON *.* TO default WITH GRANT OPTION", + f"GRANT {all_access_except_show_named_collections} ON *.* TO default WITH GRANT OPTION", ] ) @@ -434,7 +436,7 @@ def test_introspection(): [ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION", - "GRANT ALL ON *.* TO default WITH GRANT OPTION", + f"GRANT {all_access_except_show_named_collections} ON *.* TO default WITH GRANT OPTION", ] ) @@ -449,7 +451,7 @@ def test_introspection(): expected_access2 = ( "GRANT SELECT ON test.table TO A\n" "GRANT CREATE ON *.* TO B WITH GRANT OPTION\n" - "GRANT ALL ON *.* TO default WITH GRANT OPTION\n" + f"GRANT {all_access_except_show_named_collections} ON *.* TO default WITH GRANT OPTION" ) assert expected_access1 in instance.query("SHOW ACCESS") assert expected_access2 in instance.query("SHOW ACCESS") diff --git a/tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml b/tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml new file mode 100644 index 00000000000..fb5e2028d6e --- /dev/null +++ b/tests/integration/test_settings_constraints_distributed/configs/users.d/users.xml @@ -0,0 +1,10 @@ + + + + + default + default + 1 + + + diff --git a/tests/integration/test_settings_constraints_distributed/test.py b/tests/integration/test_settings_constraints_distributed/test.py index aed522bb5d1..254a8562a03 100644 --- a/tests/integration/test_settings_constraints_distributed/test.py +++ b/tests/integration/test_settings_constraints_distributed/test.py @@ -8,17 +8,17 @@ cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", main_configs=["configs/config.d/remote_servers.xml"], - user_configs=["configs/users.d/allow_introspection_functions.xml"], + user_configs=["configs/users.d/allow_introspection_functions.xml", "configs/users.d/users.xml"], ) node2 = cluster.add_instance( "node2", main_configs=["configs/config.d/remote_servers.xml"], - user_configs=["configs/users.d/allow_introspection_functions.xml"], + user_configs=["configs/users.d/allow_introspection_functions.xml", "configs/users.d/users.xml"], ) distributed = cluster.add_instance( "distributed", main_configs=["configs/config.d/remote_servers.xml"], - user_configs=["configs/users.d/allow_introspection_functions.xml"], + user_configs=["configs/users.d/allow_introspection_functions.xml", "configs/users.d/users.xml"], stay_alive=True, ) From 2a48f3c88b4c9f2b2d83d135ecb8b4c30afbd6e6 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sat, 4 Feb 2023 22:47:26 +0000 Subject: [PATCH 45/79] Update tests --- .../02317_distinct_in_order_optimization_explain.reference | 4 ++-- .../0_stateless/02481_aggregation_in_order_plan.reference | 1 + tests/queries/0_stateless/02481_aggregation_in_order_plan.sql | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.reference b/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.reference index 2511c806e1b..da24918a208 100644 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.reference +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.reference @@ -14,10 +14,10 @@ DistinctSortedTransform DistinctSortedChunkTransform -- distinct with primary key prefix and order by the same columns -> pre-distinct and final distinct optimization DistinctSortedStreamTransform -DistinctSortedChunkTransform +DistinctSortedStreamTransform -- distinct with primary key prefix and order by columns are prefix of distinct columns -> pre-distinct and final distinct optimization DistinctSortedTransform -DistinctSortedChunkTransform +DistinctSortedStreamTransform -- distinct with primary key prefix and order by column in distinct but non-primary key prefix -> pre-distinct and final distinct optimization DistinctSortedTransform DistinctSortedChunkTransform diff --git a/tests/queries/0_stateless/02481_aggregation_in_order_plan.reference b/tests/queries/0_stateless/02481_aggregation_in_order_plan.reference index ea266a98c7e..d8b62435bfa 100644 --- a/tests/queries/0_stateless/02481_aggregation_in_order_plan.reference +++ b/tests/queries/0_stateless/02481_aggregation_in_order_plan.reference @@ -4,5 +4,6 @@ 0 1 2 200 Aggregating Order: a ASC, c ASC + Sorting (Stream): a ASC, b ASC, c ASC ReadFromMergeTree (default.tab) Sorting (Stream): a ASC, b ASC, c ASC diff --git a/tests/queries/0_stateless/02481_aggregation_in_order_plan.sql b/tests/queries/0_stateless/02481_aggregation_in_order_plan.sql index 1568e44dbd5..9c99609ca46 100644 --- a/tests/queries/0_stateless/02481_aggregation_in_order_plan.sql +++ b/tests/queries/0_stateless/02481_aggregation_in_order_plan.sql @@ -1,3 +1,4 @@ +drop table if exists tab; create table tab (a Int32, b Int32, c Int32, d Int32) engine = MergeTree order by (a, b, c); insert into tab select 0, number % 3, 2 - intDiv(number, 3), (number % 3 + 1) * 10 from numbers(6); From 3042e9ee6e0155a266f2389f13fb502a25717955 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 5 Feb 2023 13:03:03 +0100 Subject: [PATCH 46/79] Fix integration test --- .../test_overcommit_tracker/configs/users.d/users.xml | 10 ++++++++++ tests/integration/test_overcommit_tracker/test.py | 7 ++++++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_overcommit_tracker/configs/users.d/users.xml diff --git a/tests/integration/test_overcommit_tracker/configs/users.d/users.xml b/tests/integration/test_overcommit_tracker/configs/users.d/users.xml new file mode 100644 index 00000000000..fb5e2028d6e --- /dev/null +++ b/tests/integration/test_overcommit_tracker/configs/users.d/users.xml @@ -0,0 +1,10 @@ + + + + + default + default + 1 + + + diff --git a/tests/integration/test_overcommit_tracker/test.py b/tests/integration/test_overcommit_tracker/test.py index 50979526e6a..1f1f23de1f8 100644 --- a/tests/integration/test_overcommit_tracker/test.py +++ b/tests/integration/test_overcommit_tracker/test.py @@ -4,7 +4,12 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -node = cluster.add_instance("node") +node = cluster.add_instance( + "node", + user_configs=[ + "configs/users.d/users.xml", + ], +) @pytest.fixture(scope="module", autouse=True) From 089a0009ad859e829ec7b9c1a0a48deafbdaaa5e Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sun, 5 Feb 2023 13:38:20 +0000 Subject: [PATCH 47/79] Polishing + try to stabilize distinct in order perf test --- .../QueryPlan/Optimizations/optimizeReadInOrder.cpp | 6 +++++- tests/performance/distinct_in_order.xml | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index e2feb66c823..421a9bbe317 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -99,7 +99,11 @@ void updateStepsDataStreams(StepStack & steps_to_update) while (!steps_to_update.empty()) { - dynamic_cast(steps_to_update.back())->updateInputStream(*input_stream); + auto * transforming_step = dynamic_cast(steps_to_update.back()); + if (!transforming_step) + break; + + transforming_step->updateInputStream(*input_stream); input_stream = &steps_to_update.back()->getOutputStream(); steps_to_update.pop_back(); } diff --git a/tests/performance/distinct_in_order.xml b/tests/performance/distinct_in_order.xml index 78856b507e7..9e089ddc158 100644 --- a/tests/performance/distinct_in_order.xml +++ b/tests/performance/distinct_in_order.xml @@ -2,6 +2,7 @@ CREATE TABLE distinct_cardinality_high (high UInt64, medium UInt64, low UInt64) ENGINE MergeTree() ORDER BY (high, medium) INSERT INTO distinct_cardinality_high SELECT number % 1e6, number % 1e4, number % 1e2 FROM numbers_mt(1e8) + OPTIMIZE TABLE distinct_cardinality_high FINAL SELECT DISTINCT high FROM distinct_cardinality_high FORMAT Null SELECT DISTINCT high, medium FROM distinct_cardinality_high FORMAT Null @@ -14,6 +15,7 @@ CREATE TABLE distinct_cardinality_low (low UInt64, medium UInt64, high UInt64) ENGINE MergeTree() ORDER BY (low, medium) INSERT INTO distinct_cardinality_low SELECT number % 1e2, number % 1e4, number % 1e6 FROM numbers_mt(1e8) + OPTIMIZE TABLE distinct_cardinality_low FINAL SELECT DISTINCT low FROM distinct_cardinality_low FORMAT Null SELECT DISTINCT low, medium FROM distinct_cardinality_low FORMAT Null From baa345fa64bb36b7dea31885ca221db4d05352dc Mon Sep 17 00:00:00 2001 From: Han Fei Date: Sun, 5 Feb 2023 18:06:06 +0100 Subject: [PATCH 48/79] remove logs --- src/Dictionaries/RegExpTreeDictionary.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index ecb7a009b67..ca1fc4e48bc 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -422,7 +422,7 @@ bool RegExpTreeDictionary::setAttributes( if (parent_id > 0) setAttributes(parent_id, attributes_to_set, data, visited_nodes, attributes, defaults, key_index); - // if all the attributes have set, the walking through can be stopped. + /// if all the attributes have set, the walking through can be stopped. return attributes_to_set.size() == attributes.size(); } @@ -542,15 +542,12 @@ std::unordered_map RegExpTreeDictionary::match( } } - LOG_TRACE(logger, "match string is {}, first matched id is {}, match number is {}", std::string(reinterpret_cast(keys_data.data()) + offset, length), match_result.matched_idx_set.empty() ? -1 : *match_result.matched_idx_set.begin(), match_result.matched_idx_set.size()); - match_result.sort(); - - // Walk through the regex tree util all attributes are set; + /// Walk through the regex tree util all attributes are set; std::unordered_map attributes_to_set; std::unordered_set visited_nodes; - // Some node matches but its parents cannot match. In this case we must regard this node unmatched. + /// Some node matches but its parents cannot match. In this case we must regard this node unmatched. auto is_invalid = [&](UInt64 id) { while (id) @@ -584,7 +581,7 @@ std::unordered_map RegExpTreeDictionary::match( columns[name]->insert(default_value.getDefaultValue(key_idx)); } - // insert to columns + /// insert to columns for (const auto & [name, value] : attributes_to_set) columns[name]->insert(value); From 9937e73fa950af876549e74f3cb80a9218382741 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 5 Feb 2023 22:38:07 +0100 Subject: [PATCH 49/79] Fix integration test --- .../test_access_control_on_cluster/test.py | 21 ++++++++++++++++--- .../test_global_overcommit_tracker/test.py | 6 +++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_access_control_on_cluster/test.py b/tests/integration/test_access_control_on_cluster/test.py index db76233a35f..8dbb87c67d8 100644 --- a/tests/integration/test_access_control_on_cluster/test.py +++ b/tests/integration/test_access_control_on_cluster/test.py @@ -3,13 +3,28 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) ch1 = cluster.add_instance( - "ch1", main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True + "ch1", + main_configs=["configs/config.d/clusters.xml"], + user_configs=[ + "configs/users.d/users.xml", + ], + with_zookeeper=True, ) ch2 = cluster.add_instance( - "ch2", main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True + "ch2", + main_configs=["configs/config.d/clusters.xml"], + user_configs=[ + "configs/users.d/users.xml", + ], + with_zookeeper=True, ) ch3 = cluster.add_instance( - "ch3", main_configs=["configs/config.d/clusters.xml"], with_zookeeper=True + "ch3", + main_configs=["configs/config.d/clusters.xml"], + user_configs=[ + "configs/users.d/users.xml", + ], + with_zookeeper=True, ) diff --git a/tests/integration/test_global_overcommit_tracker/test.py b/tests/integration/test_global_overcommit_tracker/test.py index 6bedc03a30e..4c60ee8783c 100644 --- a/tests/integration/test_global_overcommit_tracker/test.py +++ b/tests/integration/test_global_overcommit_tracker/test.py @@ -5,7 +5,11 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance( - "node", main_configs=["configs/global_overcommit_tracker.xml"] + "node", + main_configs=["configs/global_overcommit_tracker.xml"], + user_configs=[ + "configs/users.xml", + ], ) From 631c460c967030e13d432e96c648dc1427059768 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 6 Feb 2023 11:34:46 +0100 Subject: [PATCH 50/79] Add forgitten config file --- .../configs/users.d/users.xml | 10 ++++++++++ .../test_global_overcommit_tracker/configs/users.xml | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/integration/test_access_control_on_cluster/configs/users.d/users.xml create mode 100644 tests/integration/test_global_overcommit_tracker/configs/users.xml diff --git a/tests/integration/test_access_control_on_cluster/configs/users.d/users.xml b/tests/integration/test_access_control_on_cluster/configs/users.d/users.xml new file mode 100644 index 00000000000..fb5e2028d6e --- /dev/null +++ b/tests/integration/test_access_control_on_cluster/configs/users.d/users.xml @@ -0,0 +1,10 @@ + + + + + default + default + 1 + + + diff --git a/tests/integration/test_global_overcommit_tracker/configs/users.xml b/tests/integration/test_global_overcommit_tracker/configs/users.xml new file mode 100644 index 00000000000..fb5e2028d6e --- /dev/null +++ b/tests/integration/test_global_overcommit_tracker/configs/users.xml @@ -0,0 +1,10 @@ + + + + + default + default + 1 + + + From dfb961db724999b0417b50b55a7a99b547ec7b28 Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Mon, 6 Feb 2023 12:06:06 +0100 Subject: [PATCH 51/79] increase block size in order to speed up test --- tests/queries/0_stateless/01130_in_memory_parts_nested.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01130_in_memory_parts_nested.sql b/tests/queries/0_stateless/01130_in_memory_parts_nested.sql index c503e8834b4..45e778b9f04 100644 --- a/tests/queries/0_stateless/01130_in_memory_parts_nested.sql +++ b/tests/queries/0_stateless/01130_in_memory_parts_nested.sql @@ -9,8 +9,8 @@ INSERT INTO nested SELECT number, number % 2, range(number % 10) FROM system.num ALTER TABLE nested ADD COLUMN n.b Array(UInt64); SELECT DISTINCT n.b FROM nested PREWHERE filter; -SELECT DISTINCT n.b FROM nested PREWHERE filter SETTINGS max_block_size = 10; SELECT DISTINCT n.b FROM nested PREWHERE filter SETTINGS max_block_size = 123; +SELECT DISTINCT n.b FROM nested PREWHERE filter SETTINGS max_block_size = 1234; ALTER TABLE nested ADD COLUMN n.c Array(UInt64) DEFAULT arrayMap(x -> x * 2, n.a); SELECT DISTINCT n.c FROM nested PREWHERE filter; From 734d3a318b2eacd092f4c5563937ae39ff7dc95a Mon Sep 17 00:00:00 2001 From: Sema Checherinda <104093494+CheSema@users.noreply.github.com> Date: Mon, 6 Feb 2023 12:09:33 +0100 Subject: [PATCH 52/79] const auto ref in the loop --- src/Storages/MergeTree/MergeTreeData.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 7b2b23c6e6b..84df58b6beb 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7541,11 +7541,11 @@ MergeTreeData::WriteAheadLogPtr MergeTreeData::getWriteAheadLog() if (!write_ahead_log) { auto reservation = reserveSpace(getSettings()->write_ahead_log_max_bytes); - for (auto disk: reservation->getDisks()) + for (const auto & disk: reservation->getDisks()) { if (!disk->isRemote()) { - write_ahead_log = std::make_shared(*this, reservation->getDisk()); + write_ahead_log = std::make_shared(*this, disk); break; } } From e8f45ddf596cceee8373a2e7f198a724cef0b256 Mon Sep 17 00:00:00 2001 From: MeenaRenganathan22 Date: Mon, 6 Feb 2023 10:22:29 -0500 Subject: [PATCH 53/79] Updating the submodule to reflect the latest changes i.e revert of PR#24 --- .gitmodules | 3 +-- contrib/libhdfs3 | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index d917d138932..039d4f042f8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -58,8 +58,7 @@ url = https://github.com/apache/thrift [submodule "contrib/libhdfs3"] path = contrib/libhdfs3 - url = https://github.com/ClibMouse/libhdfs3.git - branch = HDFS_PowerPC + url = https://github.com/ClickHouse/libhdfs3 [submodule "contrib/libxml2"] path = contrib/libxml2 url = https://github.com/GNOME/libxml2 diff --git a/contrib/libhdfs3 b/contrib/libhdfs3 index 30df3beedd2..9e286249a2a 160000 --- a/contrib/libhdfs3 +++ b/contrib/libhdfs3 @@ -1 +1 @@ -Subproject commit 30df3beedd226f598051126f72a5ed760232d662 +Subproject commit 9e286249a2a423f3b2f2759b3702dc29713eb020 From ab0dedf0c88cbfb732d7c0c71922dce49a227607 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 6 Feb 2023 15:36:47 +0100 Subject: [PATCH 54/79] Simplify code around storage s3 configuration --- src/IO/S3/URI.h | 1 + .../ExternalDataSourceConfiguration.h | 12 -- src/Storages/StorageConfiguration.h | 15 +++ src/Storages/StorageDeltaLake.cpp | 69 ++++------- src/Storages/StorageDeltaLake.h | 12 +- src/Storages/StorageHudi.cpp | 58 ++++------ src/Storages/StorageHudi.h | 8 +- src/Storages/StorageS3.cpp | 109 ++++++++---------- src/Storages/StorageS3.h | 61 ++++------ src/Storages/StorageS3Cluster.cpp | 10 +- src/Storages/StorageS3Cluster.h | 10 +- src/Storages/StorageURL.h | 8 +- src/TableFunctions/TableFunctionDeltaLake.cpp | 6 +- src/TableFunctions/TableFunctionDeltaLake.h | 4 +- src/TableFunctions/TableFunctionHudi.cpp | 6 +- src/TableFunctions/TableFunctionHudi.h | 4 +- src/TableFunctions/TableFunctionS3.cpp | 8 +- src/TableFunctions/TableFunctionS3.h | 6 +- src/TableFunctions/TableFunctionS3Cluster.cpp | 5 +- src/TableFunctions/TableFunctionS3Cluster.h | 4 +- 20 files changed, 181 insertions(+), 235 deletions(-) create mode 100644 src/Storages/StorageConfiguration.h diff --git a/src/IO/S3/URI.h b/src/IO/S3/URI.h index f21d30929ee..f8f40cf9108 100644 --- a/src/IO/S3/URI.h +++ b/src/IO/S3/URI.h @@ -30,6 +30,7 @@ struct URI bool is_virtual_hosted_style; + URI() = default; explicit URI(const std::string & uri_); static void validateBucket(const std::string & bucket, const Poco::URI & uri); diff --git a/src/Storages/ExternalDataSourceConfiguration.h b/src/Storages/ExternalDataSourceConfiguration.h index 65ef7d28b1b..4475daaa8ff 100644 --- a/src/Storages/ExternalDataSourceConfiguration.h +++ b/src/Storages/ExternalDataSourceConfiguration.h @@ -109,18 +109,6 @@ struct URLBasedDataSourceConfiguration void set(const URLBasedDataSourceConfiguration & conf); }; -struct StorageS3Configuration : URLBasedDataSourceConfiguration -{ - S3::AuthSettings auth_settings; - S3Settings::RequestSettings request_settings; -}; - - -struct StorageS3ClusterConfiguration : StorageS3Configuration -{ - String cluster_name; -}; - struct URLBasedDataSourceConfig { URLBasedDataSourceConfiguration configuration; diff --git a/src/Storages/StorageConfiguration.h b/src/Storages/StorageConfiguration.h new file mode 100644 index 00000000000..cab95203c81 --- /dev/null +++ b/src/Storages/StorageConfiguration.h @@ -0,0 +1,15 @@ +#pragma once +#include + +namespace DB +{ + +/// A base class for stateless table engines configurations. +struct StatelessTableEngineConfiguration +{ + String format = "auto"; + String compression_method = "auto"; + String structure = "auto"; +}; + +} diff --git a/src/Storages/StorageDeltaLake.cpp b/src/Storages/StorageDeltaLake.cpp index c74e37a207c..6127db79525 100644 --- a/src/Storages/StorageDeltaLake.cpp +++ b/src/Storages/StorageDeltaLake.cpp @@ -57,8 +57,8 @@ std::vector DeltaLakeMetadata::listCurrentFiles() && return keys; } -JsonMetadataGetter::JsonMetadataGetter(StorageS3::S3Configuration & configuration_, const String & table_path_, ContextPtr context) - : base_configuration(configuration_), table_path(table_path_) +JsonMetadataGetter::JsonMetadataGetter(const StorageS3::Configuration & configuration_, ContextPtr context) + : base_configuration(configuration_) { init(context); } @@ -96,15 +96,15 @@ void JsonMetadataGetter::init(ContextPtr context) std::vector JsonMetadataGetter::getJsonLogFiles() { - std::vector keys; - const auto & client = base_configuration.client; + const auto table_path = base_configuration.url.key; + const auto bucket = base_configuration.url.bucket; + std::vector keys; S3::ListObjectsV2Request request; Aws::S3::Model::ListObjectsV2Outcome outcome; bool is_finished{false}; - const auto bucket{base_configuration.uri.bucket}; request.SetBucket(bucket); @@ -154,9 +154,9 @@ std::shared_ptr JsonMetadataGetter::createS3ReadBuffer(const String request_settings.max_single_read_retries = 10; return std::make_shared( base_configuration.client, - base_configuration.uri.bucket, + base_configuration.url.bucket, key, - base_configuration.uri.version_id, + base_configuration.url.version_id, request_settings, context->getReadSettings()); } @@ -182,11 +182,6 @@ void JsonMetadataGetter::handleJSON(const JSON & json) namespace { -StorageS3::S3Configuration getBaseConfiguration(const StorageS3Configuration & configuration) -{ - return {configuration.url, configuration.auth_settings, configuration.request_settings, configuration.headers}; -} - // DeltaLake stores data in parts in different files // keys is vector of parts with latest version // generateQueryFromKeys constructs query from parts filenames for @@ -198,35 +193,25 @@ String generateQueryFromKeys(const std::vector & keys) } -StorageS3Configuration getAdjustedS3Configuration( - const ContextPtr & context, - StorageS3::S3Configuration & base_configuration, - const StorageS3Configuration & configuration, - const std::string & table_path, - Poco::Logger * log) +StorageS3::Configuration getAdjustedS3Configuration( + const ContextPtr & context, const StorageS3::Configuration & configuration, Poco::Logger * log) { - JsonMetadataGetter getter{base_configuration, table_path, context}; - - auto keys = getter.getFiles(); - auto new_uri = base_configuration.uri.uri.toString() + generateQueryFromKeys(keys); - - LOG_DEBUG(log, "New uri: {}", new_uri); - LOG_DEBUG(log, "Table path: {}", table_path); + JsonMetadataGetter getter{configuration, context}; + const auto keys = getter.getFiles(); + const auto new_uri = configuration.url.uri.toString() + generateQueryFromKeys(keys); // set new url in configuration - StorageS3Configuration new_configuration; - new_configuration.url = new_uri; - new_configuration.auth_settings.access_key_id = configuration.auth_settings.access_key_id; - new_configuration.auth_settings.secret_access_key = configuration.auth_settings.secret_access_key; - new_configuration.format = configuration.format; + StorageS3::Configuration new_configuration(configuration); + new_configuration.url = S3::URI(new_uri); + LOG_DEBUG(log, "Table path: {}, new uri: {}", configuration.url.key, new_uri); return new_configuration; } } StorageDeltaLake::StorageDeltaLake( - const StorageS3Configuration & configuration_, + const StorageS3::Configuration & configuration_, const StorageID & table_id_, ColumnsDescription columns_, const ConstraintsDescription & constraints_, @@ -234,14 +219,13 @@ StorageDeltaLake::StorageDeltaLake( ContextPtr context_, std::optional format_settings_) : IStorage(table_id_) - , base_configuration{getBaseConfiguration(configuration_)} + , base_configuration{configuration_} , log(&Poco::Logger::get("StorageDeltaLake (" + table_id_.table_name + ")")) - , table_path(base_configuration.uri.key) + , table_path(base_configuration.url.key) { StorageInMemoryMetadata storage_metadata; StorageS3::updateS3Configuration(context_, base_configuration); - - auto new_configuration = getAdjustedS3Configuration(context_, base_configuration, configuration_, table_path, log); + auto new_configuration = getAdjustedS3Configuration(context_, base_configuration, log); if (columns_.empty()) { @@ -284,14 +268,11 @@ Pipe StorageDeltaLake::read( } ColumnsDescription StorageDeltaLake::getTableStructureFromData( - const StorageS3Configuration & configuration, const std::optional & format_settings, ContextPtr ctx) + StorageS3::Configuration & configuration, const std::optional & format_settings, ContextPtr ctx) { - auto base_configuration = getBaseConfiguration(configuration); - StorageS3::updateS3Configuration(ctx, base_configuration); - auto new_configuration = getAdjustedS3Configuration( - ctx, base_configuration, configuration, base_configuration.uri.key, &Poco::Logger::get("StorageDeltaLake")); - return StorageS3::getTableStructureFromData( - new_configuration, /*distributed processing*/ false, format_settings, ctx, /*object_infos*/ nullptr); + StorageS3::updateS3Configuration(ctx, configuration); + auto new_configuration = getAdjustedS3Configuration(ctx, configuration, &Poco::Logger::get("StorageDeltaLake")); + return StorageS3::getTableStructureFromData(new_configuration, /*distributed processing*/ false, format_settings, ctx, /*object_infos*/ nullptr); } void registerStorageDeltaLake(StorageFactory & factory) @@ -306,9 +287,9 @@ void registerStorageDeltaLake(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Storage DeltaLake requires 3 to 4 arguments: table_url, access_key, secret_access_key, [format]"); - StorageS3Configuration configuration; + StorageS3::Configuration configuration; - configuration.url = checkAndGetLiteralArgument(engine_args[0], "url"); + configuration.url = S3::URI(checkAndGetLiteralArgument(engine_args[0], "url")); configuration.auth_settings.access_key_id = checkAndGetLiteralArgument(engine_args[1], "access_key_id"); configuration.auth_settings.secret_access_key = checkAndGetLiteralArgument(engine_args[2], "secret_access_key"); diff --git a/src/Storages/StorageDeltaLake.h b/src/Storages/StorageDeltaLake.h index 5e3fdd8ad71..3495a061b14 100644 --- a/src/Storages/StorageDeltaLake.h +++ b/src/Storages/StorageDeltaLake.h @@ -37,7 +37,7 @@ private: class JsonMetadataGetter { public: - JsonMetadataGetter(StorageS3::S3Configuration & configuration_, const String & table_path_, ContextPtr context); + JsonMetadataGetter(const StorageS3::Configuration & configuration_, ContextPtr context); std::vector getFiles() { return std::move(metadata).listCurrentFiles(); } @@ -50,8 +50,7 @@ private: void handleJSON(const JSON & json); - StorageS3::S3Configuration base_configuration; - String table_path; + StorageS3::Configuration base_configuration; DeltaLakeMetadata metadata; }; @@ -62,7 +61,7 @@ public: // 2. Finds out parts with latest version // 3. Creates url for underlying StorageS3 enigne to handle reads StorageDeltaLake( - const StorageS3Configuration & configuration_, + const StorageS3::Configuration & configuration_, const StorageID & table_id_, ColumnsDescription columns_, const ConstraintsDescription & constraints_, @@ -83,11 +82,12 @@ public: size_t num_streams) override; static ColumnsDescription getTableStructureFromData( - const StorageS3Configuration & configuration, + StorageS3::Configuration & configuration, const std::optional & format_settings, ContextPtr ctx); + private: - StorageS3::S3Configuration base_configuration; + StorageS3::Configuration base_configuration; std::shared_ptr s3engine; Poco::Logger * log; String table_path; diff --git a/src/Storages/StorageHudi.cpp b/src/Storages/StorageHudi.cpp index 1b0de3c3ed2..b50759d1c2f 100644 --- a/src/Storages/StorageHudi.cpp +++ b/src/Storages/StorageHudi.cpp @@ -30,11 +30,6 @@ namespace ErrorCodes namespace { -StorageS3::S3Configuration getBaseConfiguration(const StorageS3Configuration & configuration) -{ - return {configuration.url, configuration.auth_settings, configuration.request_settings, configuration.headers}; -} - /// Apache Hudi store parts of data in different files. /// Every part file has timestamp in it. /// Every partition(directory) in Apache Hudi has different versions of part. @@ -91,17 +86,17 @@ String generateQueryFromKeys(const std::vector & keys, const String return "{" + list_of_keys + "}"; } -std::vector getKeysFromS3(const StorageS3::S3Configuration & base_configuration, const std::string & table_path, Poco::Logger * log) +std::vector getKeysFromS3(const StorageS3::Configuration & configuration, Poco::Logger * log) { + const auto & client = configuration.client; + const auto & table_path = configuration.url.key; + const auto & bucket = configuration.url.bucket; + std::vector keys; - - const auto & client = base_configuration.client; - S3::ListObjectsV2Request request; Aws::S3::Model::ListObjectsV2Outcome outcome; bool is_finished{false}; - const auto bucket{base_configuration.uri.bucket}; request.SetBucket(bucket); request.SetPrefix(table_path); @@ -134,31 +129,22 @@ std::vector getKeysFromS3(const StorageS3::S3Configuration & base_c } -StorageS3Configuration getAdjustedS3Configuration( - StorageS3::S3Configuration & base_configuration, - const StorageS3Configuration & configuration, - const std::string & table_path, - Poco::Logger * log) +StorageS3::Configuration getAdjustedS3Configuration(const StorageS3::Configuration & configuration, Poco::Logger * log) { - auto keys = getKeysFromS3(base_configuration, table_path, log); - auto new_uri = base_configuration.uri.uri.toString() + generateQueryFromKeys(keys, configuration.format); + const auto keys = getKeysFromS3(configuration, log); + const auto new_uri = configuration.url.uri.toString() + generateQueryFromKeys(keys, configuration.format); - LOG_DEBUG(log, "New uri: {}", new_uri); - LOG_DEBUG(log, "Table path: {}", table_path); - - StorageS3Configuration new_configuration; - new_configuration.url = new_uri; - new_configuration.auth_settings.access_key_id = configuration.auth_settings.access_key_id; - new_configuration.auth_settings.secret_access_key = configuration.auth_settings.secret_access_key; - new_configuration.format = configuration.format; + StorageS3::Configuration new_configuration(configuration); + new_configuration.url = S3::URI(new_uri); + LOG_DEBUG(log, "Table path: {}, new uri: {}", configuration.url.key, new_uri); return new_configuration; } } StorageHudi::StorageHudi( - const StorageS3Configuration & configuration_, + const StorageS3::Configuration & configuration_, const StorageID & table_id_, ColumnsDescription columns_, const ConstraintsDescription & constraints_, @@ -166,14 +152,13 @@ StorageHudi::StorageHudi( ContextPtr context_, std::optional format_settings_) : IStorage(table_id_) - , base_configuration{getBaseConfiguration(configuration_)} + , base_configuration{configuration_} , log(&Poco::Logger::get("StorageHudi (" + table_id_.table_name + ")")) - , table_path(base_configuration.uri.key) { StorageInMemoryMetadata storage_metadata; StorageS3::updateS3Configuration(context_, base_configuration); - auto new_configuration = getAdjustedS3Configuration(base_configuration, configuration_, table_path, log); + auto new_configuration = getAdjustedS3Configuration(base_configuration, log); if (columns_.empty()) { @@ -214,14 +199,11 @@ Pipe StorageHudi::read( } ColumnsDescription StorageHudi::getTableStructureFromData( - const StorageS3Configuration & configuration, const std::optional & format_settings, ContextPtr ctx) + StorageS3::Configuration & configuration, const std::optional & format_settings, ContextPtr ctx) { - auto base_configuration = getBaseConfiguration(configuration); - StorageS3::updateS3Configuration(ctx, base_configuration); - auto new_configuration = getAdjustedS3Configuration( - base_configuration, configuration, base_configuration.uri.key, &Poco::Logger::get("StorageDeltaLake")); - return StorageS3::getTableStructureFromData( - new_configuration, /*distributed processing*/ false, format_settings, ctx, /*object_infos*/ nullptr); + StorageS3::updateS3Configuration(ctx, configuration); + auto new_configuration = getAdjustedS3Configuration(configuration, &Poco::Logger::get("StorageDeltaLake")); + return StorageS3::getTableStructureFromData(new_configuration, /*distributed processing*/ false, format_settings, ctx, /*object_infos*/ nullptr); } void registerStorageHudi(StorageFactory & factory) @@ -236,9 +218,9 @@ void registerStorageHudi(StorageFactory & factory) ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Storage Hudi requires 3 to 4 arguments: table_url, access_key, secret_access_key, [format]"); - StorageS3Configuration configuration; + StorageS3::Configuration configuration; - configuration.url = checkAndGetLiteralArgument(engine_args[0], "url"); + configuration.url = S3::URI(checkAndGetLiteralArgument(engine_args[0], "url")); configuration.auth_settings.access_key_id = checkAndGetLiteralArgument(engine_args[1], "access_key_id"); configuration.auth_settings.secret_access_key = checkAndGetLiteralArgument(engine_args[2], "secret_access_key"); diff --git a/src/Storages/StorageHudi.h b/src/Storages/StorageHudi.h index d4a9aa7ff09..6de94c97f83 100644 --- a/src/Storages/StorageHudi.h +++ b/src/Storages/StorageHudi.h @@ -22,7 +22,7 @@ public: /// 2. Finds out parts with latest version. /// 3. Creates url for underlying StorageS3 enigne to handle reads. StorageHudi( - const StorageS3Configuration & configuration_, + const StorageS3::Configuration & configuration_, const StorageID & table_id_, ColumnsDescription columns_, const ConstraintsDescription & constraints_, @@ -44,14 +44,14 @@ public: size_t num_streams) override; static ColumnsDescription getTableStructureFromData( - const StorageS3Configuration & configuration, + StorageS3::Configuration & configuration, const std::optional & format_settings, ContextPtr ctx); + private: - StorageS3::S3Configuration base_configuration; + StorageS3::Configuration base_configuration; std::shared_ptr s3engine; Poco::Logger * log; - String table_path; }; } diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index f6419430746..8ad02cb7319 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -764,7 +764,7 @@ public: ContextPtr context, std::optional format_settings_, const CompressionMethod compression_method, - const StorageS3::S3Configuration & s3_configuration_, + const StorageS3::Configuration & s3_configuration_, const String & bucket, const String & key) : SinkToStorage(sample_block_) @@ -855,7 +855,7 @@ public: ContextPtr context_, std::optional format_settings_, const CompressionMethod compression_method_, - const StorageS3::S3Configuration & s3_configuration_, + const StorageS3::Configuration & s3_configuration_, const String & bucket_, const String & key_) : PartitionedSink(partition_by, context_, sample_block_) @@ -895,7 +895,7 @@ private: const Block sample_block; ContextPtr context; const CompressionMethod compression_method; - const StorageS3::S3Configuration & s3_configuration; + const StorageS3::Configuration & s3_configuration; const String bucket; const String key; std::optional format_settings; @@ -930,7 +930,7 @@ private: StorageS3::StorageS3( - const StorageS3Configuration & configuration_, + const StorageS3::Configuration & configuration_, const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, @@ -940,18 +940,18 @@ StorageS3::StorageS3( bool distributed_processing_, ASTPtr partition_by_) : IStorage(table_id_) - , s3_configuration{configuration_.url, configuration_.auth_settings, configuration_.request_settings, configuration_.headers} - , keys({s3_configuration.uri.key}) + , s3_configuration{configuration_} + , keys({s3_configuration.url.key}) , format_name(configuration_.format) , compression_method(configuration_.compression_method) - , name(s3_configuration.uri.storage_name) + , name(s3_configuration.url.storage_name) , distributed_processing(distributed_processing_) , format_settings(format_settings_) , partition_by(partition_by_) - , is_key_with_globs(s3_configuration.uri.key.find_first_of("*?{") != std::string::npos) + , is_key_with_globs(s3_configuration.url.key.find_first_of("*?{") != std::string::npos) { FormatFactory::instance().checkFormatName(format_name); - context_->getGlobalContext()->getRemoteHostFilter().checkURL(s3_configuration.uri.uri); + context_->getGlobalContext()->getRemoteHostFilter().checkURL(s3_configuration.url.uri); StorageInMemoryMetadata storage_metadata; updateS3Configuration(context_, s3_configuration); @@ -987,7 +987,7 @@ StorageS3::StorageS3( } std::shared_ptr StorageS3::createFileIterator( - const S3Configuration & s3_configuration, + const Configuration & s3_configuration, const std::vector & keys, bool is_key_with_globs, bool distributed_processing, @@ -1005,14 +1005,14 @@ std::shared_ptr StorageS3::createFileIterator( { /// Iterate through disclosed globs and make a source for each file return std::make_shared( - *s3_configuration.client, s3_configuration.uri, query, virtual_block, + *s3_configuration.client, s3_configuration.url, query, virtual_block, local_context, object_infos, read_keys, s3_configuration.request_settings); } else { return std::make_shared( - *s3_configuration.client, s3_configuration.uri.version_id, keys, - s3_configuration.uri.bucket, s3_configuration.request_settings, query, virtual_block, local_context, + *s3_configuration.client, s3_configuration.url.version_id, keys, + s3_configuration.url.bucket, s3_configuration.request_settings, query, virtual_block, local_context, object_infos, read_keys); } } @@ -1036,7 +1036,7 @@ Pipe StorageS3::read( size_t max_block_size, size_t num_streams) { - bool has_wildcards = s3_configuration.uri.bucket.find(PARTITION_ID_WILDCARD) != String::npos + bool has_wildcards = s3_configuration.url.bucket.find(PARTITION_ID_WILDCARD) != String::npos || keys.back().find(PARTITION_ID_WILDCARD) != String::npos; if (partition_by && has_wildcards) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Reading from a partitioned S3 storage is not implemented yet"); @@ -1102,8 +1102,8 @@ Pipe StorageS3::read( s3_configuration.request_settings, compression_method, s3_configuration.client, - s3_configuration.uri.bucket, - s3_configuration.uri.version_id, + s3_configuration.url.bucket, + s3_configuration.url.version_id, iterator_wrapper, max_download_threads)); } @@ -1120,7 +1120,7 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr auto sample_block = metadata_snapshot->getSampleBlock(); auto chosen_compression_method = chooseCompressionMethod(keys.back(), compression_method); - bool has_wildcards = s3_configuration.uri.bucket.find(PARTITION_ID_WILDCARD) != String::npos || keys.back().find(PARTITION_ID_WILDCARD) != String::npos; + bool has_wildcards = s3_configuration.url.bucket.find(PARTITION_ID_WILDCARD) != String::npos || keys.back().find(PARTITION_ID_WILDCARD) != String::npos; auto insert_query = std::dynamic_pointer_cast(query); auto partition_by_ast = insert_query ? (insert_query->partition_by ? insert_query->partition_by : partition_by) : nullptr; @@ -1136,18 +1136,18 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr format_settings, chosen_compression_method, s3_configuration, - s3_configuration.uri.bucket, + s3_configuration.url.bucket, keys.back()); } else { if (is_key_with_globs) throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, - "S3 key '{}' contains globs, so the table is in readonly mode", s3_configuration.uri.key); + "S3 key '{}' contains globs, so the table is in readonly mode", s3_configuration.url.key); bool truncate_in_insert = local_context->getSettingsRef().s3_truncate_on_insert; - if (!truncate_in_insert && S3::objectExists(*s3_configuration.client, s3_configuration.uri.bucket, keys.back(), s3_configuration.uri.version_id, s3_configuration.request_settings)) + if (!truncate_in_insert && S3::objectExists(*s3_configuration.client, s3_configuration.url.bucket, keys.back(), s3_configuration.url.version_id, s3_configuration.request_settings)) { if (local_context->getSettingsRef().s3_create_new_file_on_insert) { @@ -1159,7 +1159,7 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr new_key = keys[0].substr(0, pos) + "." + std::to_string(index) + (pos == std::string::npos ? "" : keys[0].substr(pos)); ++index; } - while (S3::objectExists(*s3_configuration.client, s3_configuration.uri.bucket, new_key, s3_configuration.uri.version_id, s3_configuration.request_settings)); + while (S3::objectExists(*s3_configuration.client, s3_configuration.url.bucket, new_key, s3_configuration.url.version_id, s3_configuration.request_settings)); keys.push_back(new_key); } else @@ -1168,7 +1168,7 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr "Object in bucket {} with key {} already exists. " "If you want to overwrite it, enable setting s3_truncate_on_insert, if you " "want to create a new file on each insert, enable setting s3_create_new_file_on_insert", - s3_configuration.uri.bucket, + s3_configuration.url.bucket, keys.back()); } @@ -1179,7 +1179,7 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr format_settings, chosen_compression_method, s3_configuration, - s3_configuration.uri.bucket, + s3_configuration.url.bucket, keys.back()); } } @@ -1190,7 +1190,7 @@ void StorageS3::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, if (is_key_with_globs) throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, - "S3 key '{}' contains globs, so the table is in readonly mode", s3_configuration.uri.key); + "S3 key '{}' contains globs, so the table is in readonly mode", s3_configuration.url.key); Aws::S3::Model::Delete delkeys; @@ -1203,7 +1203,7 @@ void StorageS3::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, ProfileEvents::increment(ProfileEvents::S3DeleteObjects); S3::DeleteObjectsRequest request; - request.SetBucket(s3_configuration.uri.bucket); + request.SetBucket(s3_configuration.url.bucket); request.SetDelete(delkeys); auto response = s3_configuration.client->DeleteObjects(request); @@ -1218,9 +1218,9 @@ void StorageS3::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, } -void StorageS3::updateS3Configuration(ContextPtr ctx, StorageS3::S3Configuration & upd) +void StorageS3::updateS3Configuration(ContextPtr ctx, StorageS3::Configuration & upd) { - auto settings = ctx->getStorageS3Settings().getSettings(upd.uri.uri.toString()); + auto settings = ctx->getStorageS3Settings().getSettings(upd.url.uri.toString()); upd.request_settings = settings.request_settings; upd.request_settings.updateFromSettings(ctx->getSettings()); @@ -1244,7 +1244,7 @@ void StorageS3::updateS3Configuration(ContextPtr ctx, StorageS3::S3Configuration upd.request_settings.get_request_throttler, upd.request_settings.put_request_throttler); - client_configuration.endpointOverride = upd.uri.endpoint; + client_configuration.endpointOverride = upd.url.endpoint; client_configuration.maxConnections = static_cast(upd.request_settings.max_connections); auto credentials = Aws::Auth::AWSCredentials(upd.auth_settings.access_key_id, upd.auth_settings.secret_access_key); @@ -1254,7 +1254,7 @@ void StorageS3::updateS3Configuration(ContextPtr ctx, StorageS3::S3Configuration upd.client = S3::ClientFactory::instance().create( client_configuration, - upd.uri.is_virtual_hosted_style, + upd.url.is_virtual_hosted_style, credentials.GetAWSAccessKeyId(), credentials.GetAWSSecretKey(), upd.auth_settings.server_side_encryption_customer_key_base64, @@ -1263,15 +1263,15 @@ void StorageS3::updateS3Configuration(ContextPtr ctx, StorageS3::S3Configuration upd.auth_settings.use_insecure_imds_request.value_or(ctx->getConfigRef().getBool("s3.use_insecure_imds_request", false))); } -void StorageS3::processNamedCollectionResult(StorageS3Configuration & configuration, const NamedCollection & collection) +void StorageS3::processNamedCollectionResult(StorageS3::Configuration & configuration, const NamedCollection & collection) { validateNamedCollection(collection, required_configuration_keys, optional_configuration_keys); - configuration.url = collection.get("url"); - auto filename = collection.getOrDefault("filename", ""); if (!filename.empty()) - configuration.url = std::filesystem::path(configuration.url) / filename; + configuration.url = S3::URI(std::filesystem::path(collection.get("url")) / filename); + else + configuration.url = S3::URI(collection.get("url")); configuration.auth_settings.access_key_id = collection.getOrDefault("access_key_id", ""); configuration.auth_settings.secret_access_key = collection.getOrDefault("secret_access_key", ""); @@ -1284,9 +1284,9 @@ void StorageS3::processNamedCollectionResult(StorageS3Configuration & configurat configuration.request_settings = S3Settings::RequestSettings(collection); } -StorageS3Configuration StorageS3::getConfiguration(ASTs & engine_args, ContextPtr local_context) +StorageS3::Configuration StorageS3::getConfiguration(ASTs & engine_args, ContextPtr local_context) { - StorageS3Configuration configuration; + StorageS3::Configuration configuration; if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) { @@ -1307,14 +1307,14 @@ StorageS3Configuration StorageS3::getConfiguration(ASTs & engine_args, ContextPt "Storage S3 requires 1 to 5 arguments: " "url, [access_key_id, secret_access_key], name of used format and [compression_method]."); - auto * header_it = StorageURL::collectHeaders(engine_args, configuration.headers, local_context); + auto * header_it = StorageURL::collectHeaders(engine_args, configuration.headers_from_ast, local_context); if (header_it != engine_args.end()) engine_args.erase(header_it); for (auto & engine_arg : engine_args) engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, local_context); - configuration.url = checkAndGetLiteralArgument(engine_args[0], "url"); + configuration.url = S3::URI(checkAndGetLiteralArgument(engine_args[0], "url")); if (engine_args.size() >= 4) { configuration.auth_settings.access_key_id = checkAndGetLiteralArgument(engine_args[1], "access_key_id"); @@ -1334,34 +1334,27 @@ StorageS3Configuration StorageS3::getConfiguration(ASTs & engine_args, ContextPt } if (configuration.format == "auto") - configuration.format = FormatFactory::instance().getFormatFromFileName(configuration.url, true); + configuration.format = FormatFactory::instance().getFormatFromFileName(configuration.url.key, true); return configuration; } ColumnsDescription StorageS3::getTableStructureFromData( - const StorageS3Configuration & configuration, + StorageS3::Configuration & configuration, bool distributed_processing, const std::optional & format_settings, ContextPtr ctx, ObjectInfos * object_infos) { - S3Configuration s3_configuration{ - configuration.url, - configuration.auth_settings, - S3Settings::RequestSettings(ctx->getSettingsRef()), - configuration.headers}; - - updateS3Configuration(ctx, s3_configuration); - + updateS3Configuration(ctx, configuration); return getTableStructureFromDataImpl( - configuration.format, s3_configuration, configuration.compression_method, distributed_processing, - s3_configuration.uri.key.find_first_of("*?{") != std::string::npos, format_settings, ctx, object_infos); + configuration.format, configuration, configuration.compression_method, distributed_processing, + configuration.url.key.find_first_of("*?{") != std::string::npos, format_settings, ctx, object_infos); } ColumnsDescription StorageS3::getTableStructureFromDataImpl( const String & format, - const S3Configuration & s3_configuration, + const Configuration & s3_configuration, const String & compression_method, bool distributed_processing, bool is_key_with_globs, @@ -1373,7 +1366,7 @@ ColumnsDescription StorageS3::getTableStructureFromDataImpl( auto file_iterator = createFileIterator( s3_configuration, - {s3_configuration.uri.key}, + {s3_configuration.url.key}, is_key_with_globs, distributed_processing, ctx, nullptr, @@ -1415,7 +1408,7 @@ ColumnsDescription StorageS3::getTableStructureFromDataImpl( int zstd_window_log_max = static_cast(ctx->getSettingsRef().zstd_window_log_max); return wrapReadBufferWithCompressionMethod( std::make_unique( - s3_configuration.client, s3_configuration.uri.bucket, key, s3_configuration.uri.version_id, s3_configuration.request_settings, ctx->getReadSettings()), + s3_configuration.client, s3_configuration.url.bucket, key, s3_configuration.url.version_id, s3_configuration.request_settings, ctx->getReadSettings()), chooseCompressionMethod(key, compression_method), zstd_window_log_max); }; @@ -1525,7 +1518,7 @@ SchemaCache & StorageS3::getSchemaCache(const ContextPtr & ctx) std::optional StorageS3::tryGetColumnsFromCache( const Strings::const_iterator & begin, const Strings::const_iterator & end, - const S3Configuration & s3_configuration, + const Configuration & s3_configuration, ObjectInfos * object_infos, const String & format_name, const std::optional & format_settings, @@ -1534,7 +1527,7 @@ std::optional StorageS3::tryGetColumnsFromCache( auto & schema_cache = getSchemaCache(ctx); for (auto it = begin; it < end; ++it) { - String path = fs::path(s3_configuration.uri.bucket) / *it; + String path = fs::path(s3_configuration.url.bucket) / *it; auto get_last_mod_time = [&]() -> std::optional { S3::ObjectInfo info; @@ -1547,7 +1540,7 @@ std::optional StorageS3::tryGetColumnsFromCache( /// Note that in case of exception in getObjectInfo returned info will be empty, /// but schema cache will handle this case and won't return columns from cache /// because we can't say that it's valid without last modification time. - info = S3::getObjectInfo(*s3_configuration.client, s3_configuration.uri.bucket, *it, s3_configuration.uri.version_id, s3_configuration.request_settings, + info = S3::getObjectInfo(*s3_configuration.client, s3_configuration.url.bucket, *it, s3_configuration.url.version_id, s3_configuration.request_settings, {}, {}, /* throw_on_error= */ false); if (object_infos) (*object_infos)[path] = info; @@ -1559,7 +1552,7 @@ std::optional StorageS3::tryGetColumnsFromCache( return std::nullopt; }; - String source = fs::path(s3_configuration.uri.uri.getHost() + std::to_string(s3_configuration.uri.uri.getPort())) / path; + String source = fs::path(s3_configuration.url.uri.getHost() + std::to_string(s3_configuration.url.uri.getPort())) / path; auto cache_key = getKeyForSchemaCache(source, format_name, format_settings, ctx); auto columns = schema_cache.tryGet(cache_key, get_last_mod_time); if (columns) @@ -1571,13 +1564,13 @@ std::optional StorageS3::tryGetColumnsFromCache( void StorageS3::addColumnsToCache( const Strings & keys, - const S3Configuration & s3_configuration, + const Configuration & s3_configuration, const ColumnsDescription & columns, const String & format_name, const std::optional & format_settings, const ContextPtr & ctx) { - auto host_and_bucket = fs::path(s3_configuration.uri.uri.getHost() + std::to_string(s3_configuration.uri.uri.getPort())) / s3_configuration.uri.bucket; + auto host_and_bucket = fs::path(s3_configuration.url.uri.getHost() + std::to_string(s3_configuration.url.uri.getPort())) / s3_configuration.url.bucket; Strings sources; sources.reserve(keys.size()); std::transform(keys.begin(), keys.end(), std::back_inserter(sources), [&](const String & key){ return host_and_bucket / key; }); diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 65c6928906d..0531a3c08d3 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -21,6 +21,7 @@ #include #include #include +#include namespace Aws::S3 { @@ -238,8 +239,21 @@ private: class StorageS3 : public IStorage, WithContext { public: + struct Configuration : public StatelessTableEngineConfiguration + { + S3::URI url; + std::shared_ptr client; + S3::AuthSettings auth_settings; + S3Settings::RequestSettings request_settings; + /// If s3 configuration was passed from ast, then it is static. + /// If from config - it can be changed with config reload. + bool static_configuration = true; + /// Headers from ast is a part of static configuration. + HTTPHeaderEntries headers_from_ast; + }; + StorageS3( - const StorageS3Configuration & configuration_, + const StorageS3::Configuration & configuration_, const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, @@ -271,45 +285,18 @@ public: bool supportsPartitionBy() const override; - static StorageS3Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context); + static StorageS3::Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context); using ObjectInfos = StorageS3Source::ObjectInfos; static ColumnsDescription getTableStructureFromData( - const StorageS3Configuration & configuration, + StorageS3::Configuration & configuration, bool distributed_processing, const std::optional & format_settings, ContextPtr ctx, ObjectInfos * object_infos = nullptr); - static void processNamedCollectionResult(StorageS3Configuration & configuration, const NamedCollection & collection); - - struct S3Configuration - { - const S3::URI uri; - std::shared_ptr client; - - S3::AuthSettings auth_settings; - S3Settings::RequestSettings request_settings; - - /// If s3 configuration was passed from ast, then it is static. - /// If from config - it can be changed with config reload. - bool static_configuration = true; - - /// Headers from ast is a part of static configuration. - HTTPHeaderEntries headers_from_ast; - - S3Configuration( - const String & url_, - const S3::AuthSettings & auth_settings_, - const S3Settings::RequestSettings & request_settings_, - const HTTPHeaderEntries & headers_from_ast_) - : uri(S3::URI(url_)) - , auth_settings(auth_settings_) - , request_settings(request_settings_) - , static_configuration(!auth_settings_.access_key_id.empty()) - , headers_from_ast(headers_from_ast_) {} - }; + static void processNamedCollectionResult(StorageS3::Configuration & configuration, const NamedCollection & collection); static SchemaCache & getSchemaCache(const ContextPtr & ctx); @@ -319,7 +306,7 @@ private: friend class StorageHudi; friend class StorageDeltaLake; - S3Configuration s3_configuration; + Configuration s3_configuration; std::vector keys; NamesAndTypesList virtual_columns; Block virtual_block; @@ -334,10 +321,10 @@ private: ObjectInfos object_infos; - static void updateS3Configuration(ContextPtr, S3Configuration &); + static void updateS3Configuration(ContextPtr, Configuration &); static std::shared_ptr createFileIterator( - const S3Configuration & s3_configuration, + const Configuration & s3_configuration, const std::vector & keys, bool is_key_with_globs, bool distributed_processing, @@ -349,7 +336,7 @@ private: static ColumnsDescription getTableStructureFromDataImpl( const String & format, - const S3Configuration & s3_configuration, + const Configuration & s3_configuration, const String & compression_method, bool distributed_processing, bool is_key_with_globs, @@ -364,7 +351,7 @@ private: static std::optional tryGetColumnsFromCache( const Strings::const_iterator & begin, const Strings::const_iterator & end, - const S3Configuration & s3_configuration, + const Configuration & s3_configuration, ObjectInfos * object_infos, const String & format_name, const std::optional & format_settings, @@ -372,7 +359,7 @@ private: static void addColumnsToCache( const Strings & keys, - const S3Configuration & s3_configuration, + const Configuration & s3_configuration, const ColumnsDescription & columns, const String & format_name, const std::optional & format_settings, diff --git a/src/Storages/StorageS3Cluster.cpp b/src/Storages/StorageS3Cluster.cpp index e158aff60a0..65353d6a521 100644 --- a/src/Storages/StorageS3Cluster.cpp +++ b/src/Storages/StorageS3Cluster.cpp @@ -41,26 +41,26 @@ namespace DB { StorageS3Cluster::StorageS3Cluster( - const StorageS3ClusterConfiguration & configuration_, + const Configuration & configuration_, const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, ContextPtr context_, bool structure_argument_was_provided_) : IStorageCluster(table_id_) - , s3_configuration{configuration_.url, configuration_.auth_settings, configuration_.request_settings, configuration_.headers} - , filename(configuration_.url) + , s3_configuration{configuration_} , cluster_name(configuration_.cluster_name) , format_name(configuration_.format) , compression_method(configuration_.compression_method) , structure_argument_was_provided(structure_argument_was_provided_) { - context_->getGlobalContext()->getRemoteHostFilter().checkURL(Poco::URI{filename}); + context_->getGlobalContext()->getRemoteHostFilter().checkURL(configuration_.url.uri); StorageInMemoryMetadata storage_metadata; StorageS3::updateS3Configuration(context_, s3_configuration); if (columns_.empty()) { + const auto & filename = configuration_.url.uri.getPath(); const bool is_key_with_globs = filename.find_first_of("*?{") != std::string::npos; /// `distributed_processing` is set to false, because this code is executed on the initiator, so there is no callback set @@ -173,7 +173,7 @@ ClusterPtr StorageS3Cluster::getCluster(ContextPtr context) const RemoteQueryExecutor::Extension StorageS3Cluster::getTaskIteratorExtension(ASTPtr query, ContextPtr context) const { auto iterator = std::make_shared( - *s3_configuration.client, s3_configuration.uri, query, virtual_block, context); + *s3_configuration.client, s3_configuration.url, query, virtual_block, context); auto callback = std::make_shared>([iterator]() mutable -> String { return iterator->next().key; }); return RemoteQueryExecutor::Extension{ .task_iterator = std::move(callback) }; } diff --git a/src/Storages/StorageS3Cluster.h b/src/Storages/StorageS3Cluster.h index d9b69b4913b..e55382c66b2 100644 --- a/src/Storages/StorageS3Cluster.h +++ b/src/Storages/StorageS3Cluster.h @@ -21,8 +21,13 @@ class Context; class StorageS3Cluster : public IStorageCluster { public: + struct Configuration : public StorageS3::Configuration + { + std::string cluster_name; + }; + StorageS3Cluster( - const StorageS3ClusterConfiguration & configuration_, + const Configuration & configuration_, const StorageID & table_id_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, @@ -43,8 +48,7 @@ public: ClusterPtr getCluster(ContextPtr context) const override; private: - StorageS3::S3Configuration s3_configuration; - String filename; + StorageS3::Configuration s3_configuration; String cluster_name; String format_name; String compression_method; diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 6666e2a716e..acf49f3cb71 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB @@ -181,15 +182,10 @@ public: static FormatSettings getFormatSettingsFromArgs(const StorageFactory::Arguments & args); - struct Configuration + struct Configuration : public StatelessTableEngineConfiguration { std::string url; std::string http_method; - - std::string format = "auto"; - std::string compression_method = "auto"; - std::string structure = "auto"; - HTTPHeaderEntries headers; }; diff --git a/src/TableFunctions/TableFunctionDeltaLake.cpp b/src/TableFunctions/TableFunctionDeltaLake.cpp index 4a285445016..f8339e06820 100644 --- a/src/TableFunctions/TableFunctionDeltaLake.cpp +++ b/src/TableFunctions/TableFunctionDeltaLake.cpp @@ -28,12 +28,12 @@ namespace ErrorCodes void TableFunctionDeltaLake::parseArgumentsImpl( - const String & error_message, ASTs & args, ContextPtr context, StorageS3Configuration & base_configuration) + const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & base_configuration) { if (args.empty() || args.size() > 6) throw Exception::createDeprecated(error_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - auto * header_it = StorageURL::collectHeaders(args, base_configuration.headers, context); + auto * header_it = StorageURL::collectHeaders(args, base_configuration.headers_from_ast, context); if (header_it != args.end()) args.erase(header_it); @@ -77,7 +77,7 @@ void TableFunctionDeltaLake::parseArgumentsImpl( } /// This argument is always the first - base_configuration.url = checkAndGetLiteralArgument(args[0], "url"); + base_configuration.url = S3::URI(checkAndGetLiteralArgument(args[0], "url")); if (args_to_idx.contains("format")) base_configuration.format = checkAndGetLiteralArgument(args[args_to_idx["format"]], "format"); diff --git a/src/TableFunctions/TableFunctionDeltaLake.h b/src/TableFunctions/TableFunctionDeltaLake.h index e36ffc3847f..657a8b02d90 100644 --- a/src/TableFunctions/TableFunctionDeltaLake.h +++ b/src/TableFunctions/TableFunctionDeltaLake.h @@ -34,9 +34,9 @@ protected: ColumnsDescription getActualTableStructure(ContextPtr context) const override; void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; - static void parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3Configuration & configuration); + static void parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & configuration); - StorageS3Configuration configuration; + mutable StorageS3::Configuration configuration; }; } diff --git a/src/TableFunctions/TableFunctionHudi.cpp b/src/TableFunctions/TableFunctionHudi.cpp index 781518ad77c..3f89fdb640a 100644 --- a/src/TableFunctions/TableFunctionHudi.cpp +++ b/src/TableFunctions/TableFunctionHudi.cpp @@ -28,12 +28,12 @@ namespace ErrorCodes void TableFunctionHudi::parseArgumentsImpl( - const String & error_message, ASTs & args, ContextPtr context, StorageS3Configuration & base_configuration) + const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & base_configuration) { if (args.empty() || args.size() > 6) throw Exception::createDeprecated(error_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - auto * header_it = StorageURL::collectHeaders(args, base_configuration.headers, context); + auto * header_it = StorageURL::collectHeaders(args, base_configuration.headers_from_ast, context); if (header_it != args.end()) args.erase(header_it); @@ -77,7 +77,7 @@ void TableFunctionHudi::parseArgumentsImpl( } /// This argument is always the first - base_configuration.url = checkAndGetLiteralArgument(args[0], "url"); + base_configuration.url = S3::URI(checkAndGetLiteralArgument(args[0], "url")); if (args_to_idx.contains("format")) base_configuration.format = checkAndGetLiteralArgument(args[args_to_idx["format"]], "format"); diff --git a/src/TableFunctions/TableFunctionHudi.h b/src/TableFunctions/TableFunctionHudi.h index a370bca8c45..519d95e074a 100644 --- a/src/TableFunctions/TableFunctionHudi.h +++ b/src/TableFunctions/TableFunctionHudi.h @@ -34,9 +34,9 @@ protected: ColumnsDescription getActualTableStructure(ContextPtr context) const override; void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; - static void parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3Configuration & configuration); + static void parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & configuration); - StorageS3Configuration configuration; + mutable StorageS3::Configuration configuration; }; } diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index f7d5e9c9df2..0abddfef3e4 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -29,7 +29,7 @@ namespace ErrorCodes /// This is needed to avoid copy-pase. Because s3Cluster arguments only differ in additional argument (first) - cluster name -void TableFunctionS3::parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3Configuration & s3_configuration) +void TableFunctionS3::parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & s3_configuration) { if (auto named_collection = tryGetNamedCollectionWithOverrides(args)) { @@ -40,7 +40,7 @@ void TableFunctionS3::parseArgumentsImpl(const String & error_message, ASTs & ar if (args.empty() || args.size() > 6) throw Exception::createDeprecated(error_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - auto * header_it = StorageURL::collectHeaders(args, s3_configuration.headers, context); + auto * header_it = StorageURL::collectHeaders(args, s3_configuration.headers_from_ast, context); if (header_it != args.end()) args.erase(header_it); @@ -87,7 +87,7 @@ void TableFunctionS3::parseArgumentsImpl(const String & error_message, ASTs & ar } /// This argument is always the first - s3_configuration.url = checkAndGetLiteralArgument(args[0], "url"); + s3_configuration.url = S3::URI(checkAndGetLiteralArgument(args[0], "url")); if (args_to_idx.contains("format")) s3_configuration.format = checkAndGetLiteralArgument(args[args_to_idx["format"]], "format"); @@ -106,7 +106,7 @@ void TableFunctionS3::parseArgumentsImpl(const String & error_message, ASTs & ar } if (s3_configuration.format == "auto") - s3_configuration.format = FormatFactory::instance().getFormatFromFileName(s3_configuration.url, true); + s3_configuration.format = FormatFactory::instance().getFormatFromFileName(s3_configuration.url.uri.getPath(), true); } void TableFunctionS3::parseArguments(const ASTPtr & ast_function, ContextPtr context) diff --git a/src/TableFunctions/TableFunctionS3.h b/src/TableFunctions/TableFunctionS3.h index 125238fa7db..2adddef538c 100644 --- a/src/TableFunctions/TableFunctionS3.h +++ b/src/TableFunctions/TableFunctionS3.h @@ -5,7 +5,7 @@ #if USE_AWS_S3 #include -#include +#include namespace DB @@ -51,9 +51,9 @@ protected: ColumnsDescription getActualTableStructure(ContextPtr context) const override; void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; - static void parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3Configuration & configuration); + static void parseArgumentsImpl(const String & error_message, ASTs & args, ContextPtr context, StorageS3::Configuration & configuration); - StorageS3Configuration configuration; + mutable StorageS3::Configuration configuration; ColumnsDescription structure_hint; }; diff --git a/src/TableFunctions/TableFunctionS3Cluster.cpp b/src/TableFunctions/TableFunctionS3Cluster.cpp index 9572d67c720..9dc49d3c4f8 100644 --- a/src/TableFunctions/TableFunctionS3Cluster.cpp +++ b/src/TableFunctions/TableFunctionS3Cluster.cpp @@ -2,7 +2,6 @@ #if USE_AWS_S3 -#include #include #include @@ -73,8 +72,8 @@ void TableFunctionS3Cluster::parseArguments(const ASTPtr & ast_function, Context clipped_args.reserve(args.size()); std::copy(args.begin() + 1, args.end(), std::back_inserter(clipped_args)); - /// StorageS3ClusterConfiguration inherints from StorageS3Configuration, so it is safe to upcast it. - TableFunctionS3::parseArgumentsImpl(message.text, clipped_args, context, static_cast(configuration)); + /// StorageS3ClusterConfiguration inherints from StorageS3::Configuration, so it is safe to upcast it. + TableFunctionS3::parseArgumentsImpl(message.text, clipped_args, context, static_cast(configuration)); } diff --git a/src/TableFunctions/TableFunctionS3Cluster.h b/src/TableFunctions/TableFunctionS3Cluster.h index 42dbfe2ec23..17df8999470 100644 --- a/src/TableFunctions/TableFunctionS3Cluster.h +++ b/src/TableFunctions/TableFunctionS3Cluster.h @@ -5,7 +5,7 @@ #if USE_AWS_S3 #include -#include +#include namespace DB @@ -50,7 +50,7 @@ protected: ColumnsDescription getActualTableStructure(ContextPtr) const override; void parseArguments(const ASTPtr &, ContextPtr) override; - StorageS3ClusterConfiguration configuration; + mutable StorageS3Cluster::Configuration configuration; ColumnsDescription structure_hint; }; From b90411f323ca7c9980e48d44ec5eea0873411b45 Mon Sep 17 00:00:00 2001 From: MeenaRenganathan22 Date: Mon, 6 Feb 2023 10:24:54 -0500 Subject: [PATCH 55/79] Fix the indentation --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 039d4f042f8..b4673f113b7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -58,7 +58,7 @@ url = https://github.com/apache/thrift [submodule "contrib/libhdfs3"] path = contrib/libhdfs3 - url = https://github.com/ClickHouse/libhdfs3 + url = https://github.com/ClickHouse/libhdfs3 [submodule "contrib/libxml2"] path = contrib/libxml2 url = https://github.com/GNOME/libxml2 From eb760413125fbfc1679c19e8cb26647cb1847374 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Mon, 6 Feb 2023 17:26:20 +0100 Subject: [PATCH 56/79] address comments and add one more test --- src/Dictionaries/RegExpTreeDictionary.cpp | 19 +++++++++++-------- ...04_regexp_dictionary_yaml_source.reference | 2 ++ .../02504_regexp_dictionary_yaml_source.sh | 19 ++++++++++++++++++- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index ca1fc4e48bc..8f82e642462 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -157,10 +157,10 @@ namespace /// hyper scan is not good at processing regex containing {0, 200} /// This will make re compilation slow and failed. So we select this heavy regular expressions and /// process it with re2. - struct ComplexRegexChecker + struct RegexChecker { re2_st::RE2 searcher; - ComplexRegexChecker() : searcher(R"(\{([\d]+),([\d]+)\})") {} + RegexChecker() : searcher(R"(\{([\d]+),([\d]+)\})") {} static bool isFigureLargerThanFifty(const String & str) try @@ -174,15 +174,15 @@ namespace } [[maybe_unused]] - bool check(const String & data) const + bool isSimpleRegex(const String & regex) const { - re2_st::StringPiece haystack(data.data(), data.size()); + re2_st::StringPiece haystack(regex.data(), regex.size()); re2_st::StringPiece matches[10]; size_t start_pos = 0; - while (start_pos < data.size()) + while (start_pos < regex.size()) { - if (searcher.Match(haystack, start_pos, data.size(), re2_st::RE2::Anchor::UNANCHORED, matches, 10)) + if (searcher.Match(haystack, start_pos, regex.size(), re2_st::RE2::Anchor::UNANCHORED, matches, 10)) { const auto & match = matches[0]; start_pos += match.length(); @@ -207,7 +207,7 @@ void RegExpTreeDictionary::initRegexNodes(Block & block) auto keys_column = block.getByName(kKeys).column; auto values_column = block.getByName(kValues).column; - ComplexRegexChecker checker; + RegexChecker checker; size_t size = block.rows(); for (size_t i = 0; i < size; i++) @@ -253,7 +253,7 @@ void RegExpTreeDictionary::initRegexNodes(Block & block) } regex_nodes.emplace(id, node); #if USE_VECTORSCAN - if (use_vectorscan && !checker.check(regex)) + if (use_vectorscan && !checker.isSimpleRegex(regex)) { simple_regexps.push_back(regex); regexp_ids.push_back(id); @@ -312,6 +312,9 @@ void RegExpTreeDictionary::loadData() if (simple_regexps.empty() && complex_regexp_nodes.empty()) throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION, "There are no available regular expression. Please check your config"); LOG_INFO(logger, "There are {} simple regexps and {} complex regexps", simple_regexps.size(), complex_regexp_nodes.size()); + /// If all the regexps cannot work with hyperscan, we should set this flag off to avoid exceptions. + if (simple_regexps.empty()) + use_vectorscan = false; if (!use_vectorscan) return; #if USE_VECTORSCAN diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.reference b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.reference index 6d5372cc484..dfcd170e8f4 100644 --- a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.reference +++ b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.reference @@ -2,4 +2,6 @@ ('Android','12') ('Android','default') ('Android','default') +('BlackBerry WebKit','10.0') +('BlackBerry WebKit','1.0') (true,'61f0c404-5cb3-11e7-907b-a6006ad3dba0','2023-01-01','2023-01-01 01:01:01',[1,2,3,-1,-2,-3]) diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh index 8d80f9f1fea..1b5a9cdeea4 100755 --- a/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh +++ b/tests/queries/0_stateless/02504_regexp_dictionary_yaml_source.sh @@ -42,7 +42,8 @@ create dictionary regexp_dict1 PRIMARY KEY(regexp) SOURCE(YAMLRegExpTree(PATH '$yaml')) LIFETIME(0) -LAYOUT(regexp_tree); +LAYOUT(regexp_tree) +SETTINGS(regexp_dict_allow_hyperscan = true); select dictGet('regexp_dict1', ('name', 'version'), 'Linux/123.45.67 tlinux'); select dictGet('regexp_dict1', ('name', 'version'), '31/tclwebkit1024'); @@ -84,6 +85,22 @@ EOL $CLICKHOUSE_CLIENT -n --query=" system reload dictionary regexp_dict1; -- { serverError 318 } " + +cat > "$yaml" < "$yaml" < Date: Mon, 6 Feb 2023 16:30:18 +0000 Subject: [PATCH 57/79] allow bloom filter for IPv4 and IPv6 --- src/Interpreters/BloomFilterHash.h | 4 ++++ .../MergeTree/MergeTreeIndexBloomFilter.cpp | 3 ++- .../0_stateless/02559_ip_types_bloom.reference | 1 + .../0_stateless/02559_ip_types_bloom.sql | 18 ++++++++++++++++++ 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02559_ip_types_bloom.reference create mode 100644 tests/queries/0_stateless/02559_ip_types_bloom.sql diff --git a/src/Interpreters/BloomFilterHash.h b/src/Interpreters/BloomFilterHash.h index 88feeb5b3df..a340fe29ab4 100644 --- a/src/Interpreters/BloomFilterHash.h +++ b/src/Interpreters/BloomFilterHash.h @@ -94,6 +94,8 @@ struct BloomFilterHash else if (which.isFloat32()) return build_hash_column(getNumberTypeHash(field)); else if (which.isFloat64()) return build_hash_column(getNumberTypeHash(field)); else if (which.isUUID()) return build_hash_column(getNumberTypeHash(field)); + else if (which.isIPv4()) return build_hash_column(getNumberTypeHash(field)); + else if (which.isIPv6()) return build_hash_column(getNumberTypeHash(field)); else if (which.isString()) return build_hash_column(getStringTypeHash(field)); else if (which.isFixedString()) return build_hash_column(getFixedStringTypeHash(field, data_type)); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected type {} of bloom filter index.", data_type->getName()); @@ -156,6 +158,8 @@ struct BloomFilterHash else if (which.isFloat32()) getNumberTypeHash(column, vec, pos); else if (which.isFloat64()) getNumberTypeHash(column, vec, pos); else if (which.isUUID()) getNumberTypeHash(column, vec, pos); + else if (which.isIPv4()) getNumberTypeHash(column, vec, pos); + else if (which.isIPv6()) getNumberTypeHash(column, vec, pos); else if (which.isString()) getStringTypeHash(column, vec, pos); else if (which.isFixedString()) getStringTypeHash(column, vec, pos); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected type {} of bloom filter index.", data_type->getName()); diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp index 7a8a28b24aa..b3e6ec6ec10 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp @@ -88,7 +88,8 @@ static void assertIndexColumnsType(const Block & header) WhichDataType which(actual_type); if (!which.isUInt() && !which.isInt() && !which.isString() && !which.isFixedString() && !which.isFloat() && - !which.isDate() && !which.isDateTime() && !which.isDateTime64() && !which.isEnum() && !which.isUUID()) + !which.isDate() && !which.isDateTime() && !which.isDateTime64() && !which.isEnum() && !which.isUUID() && + !which.isIPv4() && !which.isIPv6()) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Unexpected type {} of bloom filter index.", type->getName()); } } diff --git a/tests/queries/0_stateless/02559_ip_types_bloom.reference b/tests/queries/0_stateless/02559_ip_types_bloom.reference new file mode 100644 index 00000000000..7a76aeeff8e --- /dev/null +++ b/tests/queries/0_stateless/02559_ip_types_bloom.reference @@ -0,0 +1 @@ +1 1.1.1.1 ::1 diff --git a/tests/queries/0_stateless/02559_ip_types_bloom.sql b/tests/queries/0_stateless/02559_ip_types_bloom.sql new file mode 100644 index 00000000000..b3fc16debd9 --- /dev/null +++ b/tests/queries/0_stateless/02559_ip_types_bloom.sql @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS ip_bloom; + +CREATE TABLE ip_bloom +( + `a` UInt32, + `ip4` Nullable(IPv4), + `ip6` Nullable(IPv6), + INDEX x4 ip4 TYPE bloom_filter(0.1) GRANULARITY 3, + INDEX x6 ip6 TYPE bloom_filter(0.1) GRANULARITY 3 +) +ENGINE = MergeTree +ORDER BY a; + +INSERT INTO ip_bloom VALUES (1, '1.1.1.1', '::1'); + +SELECT * FROM ip_bloom; + +DROP TABLE ip_bloom; From d1d893275a862830546e34ae567cb253bfbd5bb3 Mon Sep 17 00:00:00 2001 From: Han Fei Date: Mon, 6 Feb 2023 18:46:23 +0100 Subject: [PATCH 58/79] fix --- src/Dictionaries/RegExpTreeDictionary.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Dictionaries/RegExpTreeDictionary.cpp b/src/Dictionaries/RegExpTreeDictionary.cpp index 8f82e642462..fdda2cd407d 100644 --- a/src/Dictionaries/RegExpTreeDictionary.cpp +++ b/src/Dictionaries/RegExpTreeDictionary.cpp @@ -189,12 +189,12 @@ namespace const auto & match1 = matches[1]; const auto & match2 = matches[2]; if (isFigureLargerThanFifty(match1.ToString()) || isFigureLargerThanFifty(match2.ToString())) - return true; + return false; } else break; } - return false; + return true; } }; } @@ -253,7 +253,7 @@ void RegExpTreeDictionary::initRegexNodes(Block & block) } regex_nodes.emplace(id, node); #if USE_VECTORSCAN - if (use_vectorscan && !checker.isSimpleRegex(regex)) + if (use_vectorscan && checker.isSimpleRegex(regex)) { simple_regexps.push_back(regex); regexp_ids.push_back(id); From ad3b4da4884a8b29a5a7afc089b06ad4f9fac77b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 6 Feb 2023 19:45:02 +0100 Subject: [PATCH 59/79] Add also last messages from stdout/stderr/debuglog in clickhouse-test First lines not very helpful usually, since the problem usually happens at the end [1], though some tricky stuff may happens at the beginning, so let's output both. [1]: https://s3.amazonaws.com/clickhouse-test-reports/45654/1716af465d376f5335720b5045bcccdd9e1823aa/fast_test.html Signed-off-by: Azat Khuzhin --- tests/clickhouse-test | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 4361b64b62f..b52c6c58946 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -71,6 +71,13 @@ def stringhash(s): return zlib.crc32(s.encode("utf-8")) +def trim_for_log(s): + if not s: + return s + lines = s.splitlines() + return "\n".join(lines[:50] + ["#" * 100] + lines[-50:]) + + class HTTPError(Exception): def __init__(self, message=None, code=None): self.message = message @@ -870,8 +877,7 @@ class TestCase: ): description = "" - if debug_log: - debug_log = "\n".join(debug_log.splitlines()[:100]) + debug_log = trim_for_log(debug_log) if proc: if proc.returncode is None: @@ -921,9 +927,7 @@ class TestCase: if os.path.isfile(self.stdout_file): description += ", result:\n\n" - description += "\n".join( - open(self.stdout_file).read().splitlines()[:100] - ) + description += trim_for_log(open(self.stdout_file).read()) description += "\n" description += f"\nstdout:\n{stdout}\n" @@ -932,7 +936,9 @@ class TestCase: ) if stderr: - description += "\n{}\n".format("\n".join(stderr.splitlines()[:100])) + description += "\n" + description += trim_for_log(stderr) + description += "\n" description += f"\nstdout:\n{stdout}\n" if debug_log: description += "\n" @@ -946,7 +952,9 @@ class TestCase: ) if "Exception" in stdout: - description += "\n{}\n".format("\n".join(stdout.splitlines()[:100])) + description += "\n" + description += trim_for_log(stdout) + description += "\n" if debug_log: description += "\n" description += debug_log From 6a389eef32cc7ae3668e1af5e87f2c9550d1b5f7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 6 Feb 2023 22:27:46 +0300 Subject: [PATCH 60/79] Add a comment --- tests/clickhouse-test | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index b52c6c58946..a65662440f6 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -71,6 +71,7 @@ def stringhash(s): return zlib.crc32(s.encode("utf-8")) +# First and last lines of the log def trim_for_log(s): if not s: return s From 7dfea59267eb6ec837753676ac7cd98de8d624cd Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 7 Feb 2023 00:50:41 +0000 Subject: [PATCH 61/79] remove an assertion --- src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index cf19f958b4c..f9fe6f2c8ab 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -484,14 +484,6 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePai column->size(), mark_num, index_granularity.getMarksCount(), index_granularity_rows); } - if (!settings.blocks_are_granules_size && index_granularity_rows > data_part->index_granularity_info.fixed_index_granularity) - { - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Mark #{} has {} rows, but max fixed granularity is {}, index granularity size {}", - mark_num, index_granularity_rows, data_part->index_granularity_info.fixed_index_granularity, - index_granularity.getMarksCount()); - } - if (index_granularity_rows != index_granularity.getMarkRows(mark_num)) { throw Exception( From 427f2d02578007ad92a5a87f40979ff76c1890f4 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 7 Feb 2023 12:36:25 +0100 Subject: [PATCH 62/79] Try fix rabbit test --- tests/integration/test_storage_rabbitmq/test.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index f354ae01714..030d9507d4f 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -1031,9 +1031,9 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): rabbitmq_exchange_name = 'over', rabbitmq_queue_base = 'over', rabbitmq_exchange_type = 'direct', - rabbitmq_num_consumers = 3, + rabbitmq_num_consumers = 2, rabbitmq_flush_interval_ms=1000, - rabbitmq_max_block_size = 100, + rabbitmq_max_block_size = 1000, rabbitmq_num_queues = 2, rabbitmq_routing_key_list = 'over', rabbitmq_format = 'TSV', @@ -1045,13 +1045,12 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): rabbitmq_exchange_type = 'direct', rabbitmq_routing_key_list = 'over', rabbitmq_flush_interval_ms=1000, - rabbitmq_max_block_size = 100, + rabbitmq_max_block_size = 1000, rabbitmq_format = 'TSV', rabbitmq_row_delimiter = '\\n'; CREATE TABLE test.view_overload (key UInt64, value UInt64) ENGINE = MergeTree - ORDER BY key - SETTINGS old_parts_lifetime=5, cleanup_delay_period=2, cleanup_delay_period_random_add=3; + ORDER BY key; CREATE MATERIALIZED VIEW test.consumer_overload TO test.view_overload AS SELECT * FROM test.rabbitmq_consume; """ @@ -1080,7 +1079,7 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): raise threads = [] - threads_num = 3 + threads_num = 2 for _ in range(threads_num): threads.append(threading.Thread(target=insert)) for thread in threads: @@ -1089,11 +1088,11 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): while True: result = instance.query("SELECT count() FROM test.view_overload") - time.sleep(1) expected = messages_num * threads_num if int(result) == expected: break print(f"Result: {result} / {expected}") + time.sleep(1) instance.query( """ From 9485873a2f4697000a00230ec11332c8d8a97235 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 7 Feb 2023 12:45:23 +0100 Subject: [PATCH 63/79] Fix integration test --- src/Storages/StorageS3.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 8ad02cb7319..40ddc09adaf 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -1332,6 +1332,7 @@ StorageS3::Configuration StorageS3::getConfiguration(ASTs & engine_args, Context configuration.format = checkAndGetLiteralArgument(engine_args.back(), "format"); } } + configuration.static_configuration = !configuration.auth_settings.access_key_id.empty(); if (configuration.format == "auto") configuration.format = FormatFactory::instance().getFormatFromFileName(configuration.url.key, true); From fe832c4418d4753fa461da80b5e84dae739215e0 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 7 Feb 2023 12:04:16 +0100 Subject: [PATCH 64/79] Fix TOCTOU while checking for container existence in azure_blob_storage This is a simple time-of-check vs time-of-use, since between checking does such container exist and creating one, there is a time window. Signed-off-by: Azat Khuzhin --- .../AzureBlobStorage/AzureBlobStorageAuth.cpp | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp index 4756a2b11f1..61fa4af100e 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp @@ -2,6 +2,7 @@ #if USE_AZURE_BLOB_STORAGE +#include #include #include #include @@ -125,21 +126,22 @@ std::unique_ptr getAzureBlobContainerClient( auto blob_service_client = getAzureBlobStorageClientWithAuth(endpoint.storage_account_url, container_name, config, config_prefix); - if (!endpoint.container_already_exists.has_value()) + try { - ListBlobContainersOptions blob_containers_list_options; - blob_containers_list_options.Prefix = container_name; - blob_containers_list_options.PageSizeHint = 1; - auto blob_containers = blob_service_client->ListBlobContainers().BlobContainers; - for (const auto & blob_container : blob_containers) - { - if (blob_container.Name == endpoint.container_name) - return getAzureBlobStorageClientWithAuth(final_url, container_name, config, config_prefix); - } + return std::make_unique( + blob_service_client->CreateBlobContainer(container_name).Value); + } + catch (const Azure::Storage::StorageException & e) + { + /// If container_already_exists is not set (in config), ignore already exists error. + /// (Conflict - The specified container already exists) + if (!endpoint.container_already_exists.has_value() && e.StatusCode == Azure::Core::Http::HttpStatusCode::Conflict) + { + tryLogCurrentException("Container already exist, return existing container"); + return getAzureBlobStorageClientWithAuth(final_url, container_name, config, config_prefix); + } + throw; } - - return std::make_unique( - blob_service_client->CreateBlobContainer(container_name).Value); } std::unique_ptr getAzureBlobStorageSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr /*context*/) From bca85c685286f526843c85adba03ddc0a8ee7da3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 7 Feb 2023 12:27:23 +0100 Subject: [PATCH 65/79] Fix test_merge_tree_azure_blob_storage::test_zero_copy_replication test Here is an example of an error [1]: 2023.02.06 21:59:09.076849 [ 200 ] {} DiskObjectStorage(DiskAzureBlobStorage): Starting up disk blob_storage_disk 2023.02.06 21:59:09.076897 [ 200 ] {} DiskObjectStorage(DiskAzureBlobStorage): Disk blob_storage_disk started up 2023.02.06 21:59:09.077263 [ 200 ] {} StoragePolicy (blob_storage_policy): Storage policy blob_storage_policy created, total volumes 1 2023.02.06 21:59:09.077318 [ 200 ] {} StoragePolicySelector: Storage policy `blob_storage_policy` loaded ... 2023.02.06 21:59:09.227326 [ 1 ] {} Application: std::exception. Code: 1001, type: Azure::Storage::StorageException, e.what() = 409 The specified container already exists. The specified container already exists. RequestId:73d54dcf-0c66-489c-b9b8-2b5398df9384 Time:2023-02-06T21:59:08.225Z Request ID: 73d54dcf-0c66-489c-b9b8-2b5398df9384, Stack trace (when copying this message, always include the lines below): 0. ./build_docker/../contrib/llvm-project/libcxx/include/exception:134: std::runtime_error::runtime_error() @ 0x44297b8d in /usr/bin/clickhouse ... 5. ./build_docker/../contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:714: DB::getAzureBlobContainerClient() @ 0x2f5b225f in /usr/bin/clickhouse 6. ./build_docker/../src/Disks/ObjectStorages/AzureBlobStorage/registerDiskAzureBlobStorage.cpp:0: 10. ./build_docker/../src/Interpreters/Context.cpp:0: DB::Context::getDisksMap() const @ 0x2f95af47 in /usr/bin/clickhouse ... 13. ./build_docker/../contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:714: DB::AsynchronousMetrics::start() @ 0x1f4d6161 in /usr/bin/clickhouse 14. ./build_docker/../programs/server/Server.cpp:0: DB::Server::main() @ 0x1f3a5bfb in /usr/bin/clickhouse 15. ./build_docker/../contrib/poco/Util/src/Application.cpp:0: Poco::Util::Application::run() @ 0x3c7546bf in /usr/bin/clickhouse 16. ./build_docker/../programs/server/Server.cpp:476: DB::Server::run() @ 0x1f381f7b in /usr/bin/clickhouse 17. ./build_docker/../contrib/poco/Util/src/ServerApplication.cpp:612: Poco::Util::ServerApplication::run(int, char**) @ 0x3c797ee0 in /usr/bin/clickhouse 18. ./build_docker/../programs/server/Server.cpp:0: mainEntryClickHouseServer(int, char**) @ 0x1f37a262 in /usr/bin/clickhouse 19. ./build_docker/../programs/main.cpp:0: main @ 0xe9d21ae in /usr/bin/clickhouse 20. __libc_start_main @ 0x7f58271a1083 in ? 21. _start @ 0xe90ef6e in /usr/bin/clickhouse (version 23.2.1.1) 2023.02.06 21:59:09.227971 [ 1 ] {} Application: 409 The specified container already exists. The specified container already exists. RequestId:73d54dcf-0c66-489c-b9b8-2b5398df9384 Time:2023-02-06T21:59:08.225Z Request ID: 73d54dcf-0c66-489c-b9b8-2b5398df9384 [1]: https://s3.amazonaws.com/clickhouse-test-reports/46045/bd4170e03c6af583a51d12d2c39fa775dcb9997b/integration_tests__asan__[6/6].html Signed-off-by: Azat Khuzhin --- .../configs/config.d/storage_conf.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_merge_tree_azure_blob_storage/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_azure_blob_storage/configs/config.d/storage_conf.xml index 1440160b857..07f57c08e32 100644 --- a/tests/integration/test_merge_tree_azure_blob_storage/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_azure_blob_storage/configs/config.d/storage_conf.xml @@ -5,12 +5,15 @@ azure_blob_storage http://azurite1:10000/devstoreaccount1 cont - false false devstoreaccount1 Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== 100000 + local From 022d4308ced7d429f8526e7a1edc5b37e2c2c2af Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 7 Feb 2023 13:01:15 +0100 Subject: [PATCH 66/79] Address review comments --- .../configs/users.d/users.xml | 10 ++++++++++ tests/integration/test_grant_and_revoke/test.py | 15 +++++++++------ ..._no_default_access_with_access_management.xml} | 0 tests/integration/test_named_collections/test.py | 2 +- 4 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 tests/integration/test_grant_and_revoke/configs/users.d/users.xml rename tests/integration/test_named_collections/configs/users.d/{users_no_default_access2.xml => users_no_default_access_with_access_management.xml} (100%) diff --git a/tests/integration/test_grant_and_revoke/configs/users.d/users.xml b/tests/integration/test_grant_and_revoke/configs/users.d/users.xml new file mode 100644 index 00000000000..fb5e2028d6e --- /dev/null +++ b/tests/integration/test_grant_and_revoke/configs/users.d/users.xml @@ -0,0 +1,10 @@ + + + + + default + default + 1 + + + diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index a04002b2ddc..6011c4379fe 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -3,7 +3,12 @@ from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance("instance") +instance = cluster.add_instance( + "instance", + user_configs=[ + "configs/users.d/users.xml", + ], +) @pytest.fixture(scope="module", autouse=True) @@ -377,8 +382,6 @@ def test_introspection(): instance.query("GRANT SELECT ON test.table TO A") instance.query("GRANT CREATE ON *.* TO B WITH GRANT OPTION") - all_access_except_show_named_collections = "SHOW, SELECT, INSERT, ALTER, CREATE, DROP, TRUNCATE, OPTIMIZE, BACKUP, KILL QUERY, KILL TRANSACTION, MOVE PARTITION BETWEEN SHARDS, CREATE USER, ALTER USER, DROP USER, CREATE ROLE, ALTER ROLE, DROP ROLE, ROLE ADMIN, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, CREATE QUOTA, ALTER QUOTA, DROP QUOTA, CREATE SETTINGS PROFILE, ALTER SETTINGS PROFILE, DROP SETTINGS PROFILE, SHOW ACCESS, SYSTEM, dictGet, INTROSPECTION, SOURCES, CLUSTER" - assert instance.query("SHOW USERS") == TSV(["A", "B", "default"]) assert instance.query("SHOW CREATE USERS A") == TSV(["CREATE USER A"]) assert instance.query("SHOW CREATE USERS B") == TSV(["CREATE USER B"]) @@ -415,7 +418,7 @@ def test_introspection(): [ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION", - f"GRANT {all_access_except_show_named_collections} ON *.* TO default WITH GRANT OPTION", + "GRANT ALL ON *.* TO default WITH GRANT OPTION", ] ) @@ -436,7 +439,7 @@ def test_introspection(): [ "GRANT SELECT ON test.table TO A", "GRANT CREATE ON *.* TO B WITH GRANT OPTION", - f"GRANT {all_access_except_show_named_collections} ON *.* TO default WITH GRANT OPTION", + "GRANT ALL ON *.* TO default WITH GRANT OPTION", ] ) @@ -451,7 +454,7 @@ def test_introspection(): expected_access2 = ( "GRANT SELECT ON test.table TO A\n" "GRANT CREATE ON *.* TO B WITH GRANT OPTION\n" - f"GRANT {all_access_except_show_named_collections} ON *.* TO default WITH GRANT OPTION" + "GRANT ALL ON *.* TO default WITH GRANT OPTION\n" ) assert expected_access1 in instance.query("SHOW ACCESS") assert expected_access2 in instance.query("SHOW ACCESS") diff --git a/tests/integration/test_named_collections/configs/users.d/users_no_default_access2.xml b/tests/integration/test_named_collections/configs/users.d/users_no_default_access_with_access_management.xml similarity index 100% rename from tests/integration/test_named_collections/configs/users.d/users_no_default_access2.xml rename to tests/integration/test_named_collections/configs/users.d/users_no_default_access_with_access_management.xml diff --git a/tests/integration/test_named_collections/test.py b/tests/integration/test_named_collections/test.py index 1b853fadaff..3b102f1aa70 100644 --- a/tests/integration/test_named_collections/test.py +++ b/tests/integration/test_named_collections/test.py @@ -40,7 +40,7 @@ def cluster(): "configs/config.d/named_collections.xml", ], user_configs=[ - "configs/users.d/users_no_default_access2.xml", + "configs/users.d/users_no_default_access_with_access_management.xml", ], stay_alive=True, ) From 2445a391fbaf7e798c6eb9b65863ac6d6b1c60d8 Mon Sep 17 00:00:00 2001 From: MeenaRenganathan22 Date: Tue, 7 Feb 2023 09:05:12 -0500 Subject: [PATCH 67/79] Updating the libhdfs3 to reflect the revert of PR#21 --- contrib/libhdfs3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libhdfs3 b/contrib/libhdfs3 index 9e286249a2a..9ee3ce77215 160000 --- a/contrib/libhdfs3 +++ b/contrib/libhdfs3 @@ -1 +1 @@ -Subproject commit 9e286249a2a423f3b2f2759b3702dc29713eb020 +Subproject commit 9ee3ce77215fca83b7fdfcfe2186a3db0d0bdb74 From 40deaf1661812d6d89fd19ff3efdfc867175d1ac Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 7 Feb 2023 16:30:04 +0100 Subject: [PATCH 68/79] Fix clang-tidy --- src/Storages/StorageDeltaLake.cpp | 2 +- src/Storages/StorageDeltaLake.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/StorageDeltaLake.cpp b/src/Storages/StorageDeltaLake.cpp index 6127db79525..9f86eaf70d0 100644 --- a/src/Storages/StorageDeltaLake.cpp +++ b/src/Storages/StorageDeltaLake.cpp @@ -94,7 +94,7 @@ void JsonMetadataGetter::init(ContextPtr context) } } -std::vector JsonMetadataGetter::getJsonLogFiles() +std::vector JsonMetadataGetter::getJsonLogFiles() const { const auto & client = base_configuration.client; const auto table_path = base_configuration.url.key; diff --git a/src/Storages/StorageDeltaLake.h b/src/Storages/StorageDeltaLake.h index 3495a061b14..33a15dcf115 100644 --- a/src/Storages/StorageDeltaLake.h +++ b/src/Storages/StorageDeltaLake.h @@ -44,7 +44,7 @@ public: private: void init(ContextPtr context); - std::vector getJsonLogFiles(); + std::vector getJsonLogFiles() const; std::shared_ptr createS3ReadBuffer(const String & key, ContextPtr context); From b43ffb98e865b367bbd634928dc3dded757abe75 Mon Sep 17 00:00:00 2001 From: Zhiguo Zhou Date: Tue, 7 Feb 2023 23:33:01 +0800 Subject: [PATCH 69/79] Vectorize filter generation of ColumnNullable in FilterDescription (#45962) This commit achieved the data parallelism for filter generations of the nullable columns by replacing the logical AND operator with the bitwise one, which could be auto-vectorized by the compiler. --- src/Columns/FilterDescription.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Columns/FilterDescription.cpp b/src/Columns/FilterDescription.cpp index a87c70dcdad..9e967a85c59 100644 --- a/src/Columns/FilterDescription.cpp +++ b/src/Columns/FilterDescription.cpp @@ -82,7 +82,12 @@ FilterDescription::FilterDescription(const IColumn & column_) const auto size = res.size(); assert(size == null_map.size()); for (size_t i = 0; i < size; ++i) - res[i] = res[i] && !null_map[i]; + { + auto has_val = static_cast(!!res[i]); + auto not_null = static_cast(!null_map[i]); + /// Instead of the logical AND operator(&&), the bitwise one(&) is utilized for the auto vectorization. + res[i] = has_val & not_null; + } data = &res; data_holder = std::move(mutable_holder); From 18d2a99761db4c4fd73a7eb8aa0352c114e8a0cc Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 7 Feb 2023 16:53:45 +0100 Subject: [PATCH 70/79] Use comments for variables that clang can parse Signed-off-by: Azat Khuzhin --- programs/copier/ClusterCopier.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 2fc0eb27213..69f87d9bbf3 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -1452,7 +1452,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( *zookeeper, host_id); // Maybe we are the first worker - ASTPtr query_select_ast = get_select_query(split_table_for_current_piece, "count()", /*enable_splitting*/ true); + ASTPtr query_select_ast = get_select_query(split_table_for_current_piece, "count()", /* enable_splitting= */ true); UInt64 count; { auto local_context = Context::createCopy(context); @@ -1520,7 +1520,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( } // Select all fields - ASTPtr query_select_ast = get_select_query(task_shard.table_read_shard, "*", /*enable_splitting*/ true, inject_fault ? "1" : ""); + ASTPtr query_select_ast = get_select_query(task_shard.table_read_shard, "*", /* enable_splitting= */ true, inject_fault ? "1" : ""); LOG_INFO(log, "Executing SELECT query and pull from {}: {}", task_shard.getDescription(), queryToString(query_select_ast)); From ff324fe81af22d04fd4bbec7958449197c1f755d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 7 Feb 2023 16:53:59 +0100 Subject: [PATCH 71/79] Fix fault injection in copier (wrong query syntax) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are very frequent flakiness of `test_cluster_copier` test, here is an example of copier failures on CI [1]: AssertionError: Instance: s0_1_0 (172.16.29.9). Info: {'ID': '5d68dcb46fdb4b0c54b7c7ba1ddde83b8f34d483bbb32abcb0c52b966444ce82', 'Running': False, 'ExitCode': 85, 'ProcessConfig': {'tty': False, 'entrypoint': '/usr/bin/clickhouse', 'arguments': ['copier', '--config', '/etc/clickhouse-server/config-copier.xml', '--task-path', '/clickhouse-copier/task_simple_4DFWYTDD49', '--task-file', '/task0_description.xml', '--task-upload-force', 'true', '--base-dir', '/var/log/clickhouse-server/copier', '--copy-fault-probability', '0.2', '--experimental-use-sample-offset', '1'], 'privileged': False, 'user': '0'}, 'OpenStdin': False, 'OpenStderr': True, 'OpenStdout': True, 'CanRemove': False, 'ContainerID': 'f356df6694b3cc09ee9830c623681626f8e8d999677c188b9fe911aa702784ca', 'DetachKeys': '', 'Pid': 84332} assert 85 == 0 But let's look what the error it is, apparently it is UNFINISHED: SELECT name, code FROM system.errors WHERE ((code % 256) = 85) AND (NOT remote) SETTINGS system_events_show_zero_values = 1 ┌─name─────────────────────────────┬─code─┐ │ FORMAT_IS_NOT_SUITABLE_FOR_INPUT │ 85 │ │ UNFINISHED │ 341 │ │ NO_SUCH_ERROR_CODE │ 597 │ └──────────────────────────────────┴──────┘ Let's verify: $ grep -r UNFINISHED ./test_cluster_copier/_instances_0/s0_1_0/logs/copier/clickhouse-copier_* ./test_cluster_copier/_instances_0/s0_1_0/logs/copier/clickhouse-copier_20230206220846_368/log.log:2023.02.06 22:09:19.015251 [ 368 ] {} : virtual int DB::ClusterCopierApp::main(const std::vector &): Code: 341. DB::Exception: Too many tries to process table cluster1.default.hits. Abort remaining execution. (UNFINISHED), Stack trace (when copying this message, always include the lines below): And apparently that it is due to query error with fault injection: 2023.02.06 22:09:15.654724 [ 368 ] {} Application: An error occurred while processing partition 0: Code: 62. DB::Exception: Syntax error (Query): failed at position 168 ('Native'): Native. Expected one of: token, Dot, OR, AND, BETWEEN, NOT BETWEEN, LIKE, ILIKE, NOT LIKE, NOT ILIKE, IN, NOT IN, GLOBAL IN, GLOBAL NOT IN, MOD, DIV, IS NULL, IS NOT NULL, alias, AS, Comma, OFFSET, WITH TIES, BY, LIMIT, SETTINGS, UNION, EXCEPT, INTERSECT, INTO OUTFILE, FORMAT, end of query. (SYNTAX_ERROR), Stack trace (when copying this message, always include the lines below): Example: select x from x limit 1FORMAT Native Syntax error: failed at position 32 ('Native'): So fixing this should fix test_cluster_copier flakiness. [1]: https://s3.amazonaws.com/clickhouse-test-reports/46045/bd4170e03c6af583a51d12d2c39fa775dcb9997b/integration_tests__release__[4/4].html Signed-off-by: Azat Khuzhin --- programs/copier/ClusterCopier.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp index 69f87d9bbf3..c4e48bf7eb0 100644 --- a/programs/copier/ClusterCopier.cpp +++ b/programs/copier/ClusterCopier.cpp @@ -1306,7 +1306,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl( if (!limit.empty()) query += " LIMIT " + limit; - query += "FORMAT Native"; + query += " FORMAT Native"; ParserQuery p_query(query.data() + query.size()); From e5b5fce970f80502f3e28a7c8eb6c1c4a543746a Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> Date: Tue, 7 Feb 2023 18:11:43 +0100 Subject: [PATCH 72/79] Update liburing CmakeLists.txt --- contrib/liburing-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/liburing-cmake/CMakeLists.txt b/contrib/liburing-cmake/CMakeLists.txt index 02bc116c660..300cc02c864 100644 --- a/contrib/liburing-cmake/CMakeLists.txt +++ b/contrib/liburing-cmake/CMakeLists.txt @@ -50,4 +50,4 @@ configure_file (io_uring_version.h.in ${LIBURING_VERSION_HEADER}) add_library (_liburing ${SRCS}) add_library (ch_contrib::liburing ALIAS _liburing) -target_include_directories (_liburing SYSTEM PUBLIC ${LIBURING_COMPAT_INCLUDE_DIR} ${LIBURING_GENERATED_INCLUDE_DIR} "${LIBURING_SOURCE_DIR}/include") +target_include_directories (_liburing SYSTEM PUBLIC ${LIBURING_COMPAT_INCLUDE_DIR} ${LIBURING_GENERATED_INCLUDE_DIR} ${LIBURING_INCLUDE_DIR}) From fe5bc0195135426a33c51b350a3d9491af38ee9a Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Tue, 7 Feb 2023 14:43:36 -0500 Subject: [PATCH 73/79] merged Mermaid support today --- docs/en/development/architecture.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/en/development/architecture.md b/docs/en/development/architecture.md index 4d4271ab79f..50b338844df 100644 --- a/docs/en/development/architecture.md +++ b/docs/en/development/architecture.md @@ -198,7 +198,19 @@ Each slot can be seen as an independent state machine with the following states: Note that `allocated` slot can be in two different states: `granted` and `acquired`. The former is a transitional state, that actually should be short (from the instant when a slot is allocated to a query till the moment when the up-scaling procedure is run by any thread of that query). -![state diagram](@site/docs/en/development/images/concurrency.png) +```mermaid +stateDiagram-v2 + direction LR + [*] --> free + free --> allocated: allocate + state allocated { + direction LR + [*] --> granted + granted --> acquired: acquire + acquired --> [*] + } + allocated --> free: release +``` API of `ConcurrencyControl` consists of the following functions: 1. Create a resource allocation for a query: `auto slots = ConcurrencyControl::instance().allocate(1, max_threads);`. It will allocate at least 1 and at most `max_threads` slots. Note that the first slot is granted immediately, but the remaining slots may be granted later. Thus limit is soft, because every query will obtain at least one thread. From a60a3726e88395fad98d277dd00e7580cd06a1d1 Mon Sep 17 00:00:00 2001 From: Tyler Hannan Date: Tue, 7 Feb 2023 23:06:32 +0100 Subject: [PATCH 74/79] Adding FOSDEM --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bcf2643c33d..4cb18c5b6b2 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ ClickHouse® is an open-source column-oriented database management system that a * [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlight, powered by github.dev. * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any. -## Upcoming events +## Events +* **FOSDEM 2023**: In the "Fast and Streaming Data" room Alexey gave a talk entitled "Building Analytical Apps With ClickHouse" that looks at the landscape of data tools, an interesting data set, and how you can interact with data quickly. Check out the recording on [youtube](https://www.youtube.com/watch?v=JlcI2Vfz_uk) * **Recording available**: [**v23.1 Release Webinar**](https://www.youtube.com/watch?v=zYSZXBnTMSE) 23.1 is the ClickHouse New Year release. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release. Inverted indices, query cache, and so -- very -- much more. * **Recording available**: [**ClickHouse Meetup at the CHEQ office in Tel Aviv**](https://www.meetup.com/clickhouse-tel-aviv-user-group/events/289599423/) - We are very excited to be holding our next in-person ClickHouse meetup at the CHEQ office in Tel Aviv! Hear from CHEQ, ServiceNow and Contentsquare, as well as a deep dive presentation from ClickHouse CTO Alexey Milovidov. Join us for a fun evening of talks, food and discussion! From 924f19e94a0b5381d88fb7b24096095b21bcc75b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 8 Feb 2023 04:12:00 +0300 Subject: [PATCH 75/79] Update AzureBlobStorageAuth.cpp --- .../ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp index 61fa4af100e..1e06490b5bc 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp @@ -137,7 +137,7 @@ std::unique_ptr getAzureBlobContainerClient( /// (Conflict - The specified container already exists) if (!endpoint.container_already_exists.has_value() && e.StatusCode == Azure::Core::Http::HttpStatusCode::Conflict) { - tryLogCurrentException("Container already exist, return existing container"); + tryLogCurrentException("Container already exists, returning the existing container"); return getAzureBlobStorageClientWithAuth(final_url, container_name, config, config_prefix); } throw; From 66e9dd3c9475b9d9103b67c5d3280966f3f625d1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 8 Feb 2023 04:53:47 +0300 Subject: [PATCH 76/79] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4cb18c5b6b2..75e0fa1bc4d 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,6 @@ ClickHouse® is an open-source column-oriented database management system that a * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any. ## Events -* **FOSDEM 2023**: In the "Fast and Streaming Data" room Alexey gave a talk entitled "Building Analytical Apps With ClickHouse" that looks at the landscape of data tools, an interesting data set, and how you can interact with data quickly. Check out the recording on [youtube](https://www.youtube.com/watch?v=JlcI2Vfz_uk) +* **FOSDEM 2023**: In the "Fast and Streaming Data" room Alexey gave a talk entitled "Building Analytical Apps With ClickHouse" that looks at the landscape of data tools, an interesting data set, and how you can interact with data quickly. Check out the recording on **[YouTube](https://www.youtube.com/watch?v=JlcI2Vfz_uk)**. * **Recording available**: [**v23.1 Release Webinar**](https://www.youtube.com/watch?v=zYSZXBnTMSE) 23.1 is the ClickHouse New Year release. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release. Inverted indices, query cache, and so -- very -- much more. * **Recording available**: [**ClickHouse Meetup at the CHEQ office in Tel Aviv**](https://www.meetup.com/clickhouse-tel-aviv-user-group/events/289599423/) - We are very excited to be holding our next in-person ClickHouse meetup at the CHEQ office in Tel Aviv! Hear from CHEQ, ServiceNow and Contentsquare, as well as a deep dive presentation from ClickHouse CTO Alexey Milovidov. Join us for a fun evening of talks, food and discussion! From a2df6e950ed3568ba09246e0f2cd009b91926de3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 8 Feb 2023 03:38:14 +0100 Subject: [PATCH 77/79] Whitespace --- tests/performance/decimal_casts.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/decimal_casts.xml b/tests/performance/decimal_casts.xml index 6dd38b6a06a..f385a0362cc 100644 --- a/tests/performance/decimal_casts.xml +++ b/tests/performance/decimal_casts.xml @@ -17,7 +17,7 @@ SELECT toUInt256(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t LIMIT 10000000 FORMAT Null SELECT toFloat32(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t FORMAT Null SELECT toFloat64(x) y, toDecimal32(y, 1), toDecimal64(y, 5), toDecimal128(y, 6) FROM t FORMAT Null - + SELECT toInt32(d32), toInt64(d32), toInt128(d32) FROM t FORMAT Null SELECT toInt32(d64), toInt64(d64), toInt128(d64) FROM t FORMAT Null SELECT toInt32(d128), toInt64(d128), toInt128(d128) FROM t FORMAT Null From a69f9c05a981a6c95af496b47413e540c87e2c09 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 8 Feb 2023 05:35:19 +0100 Subject: [PATCH 78/79] Remove flaky test --- ...nd_exception_messages_formatting.reference | 14 ----- ..._log_and_exception_messages_formatting.sql | 61 ------------------- 2 files changed, 75 deletions(-) delete mode 100644 tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference delete mode 100644 tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference deleted file mode 100644 index d3b69939e66..00000000000 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference +++ /dev/null @@ -1,14 +0,0 @@ -10 0.001 -20 0.05 -30 10 -40 40 -50 125 -60 0.3 -70 0.16 -80 0.08 -90 0.04 -100 0.005 -110 0.01 -120 3 -130 10 -140 15 diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql deleted file mode 100644 index 686d04013fe..00000000000 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql +++ /dev/null @@ -1,61 +0,0 @@ --- Tags: no-parallel --- no-parallel because we want to run this test when most of the other tests already passed - --- If this test fails, see the "Top patterns of log messages" diagnostics in the end of run.log - -system flush logs; -drop table if exists logs; -create view logs as select * from system.text_log where now() - toIntervalMinute(120) < event_time; - --- Check that we don't have too many messages formatted with fmt::runtime or strings concatenation. --- 0.001 threshold should be always enough, the value was about 0.00025 -select 10, max2(sum(length(message_format_string) = 0) / count(), 0.001) from logs; - --- Check the same for exceptions. The value was 0.03 -select 20, max2(sum(length(message_format_string) = 0) / count(), 0.05) from logs where message like '%DB::Exception%'; - --- Check that we don't have too many short meaningless message patterns. -select 30, max2(countDistinct(message_format_string), 10) from logs where length(message_format_string) < 10; - --- Same as above. Feel free to update the threshold or remove this query if really necessary -select 40, max2(countDistinct(message_format_string), 40) from logs where length(message_format_string) < 16; - --- Same as above, but exceptions must be more informative. Feel free to update the threshold or remove this query if really necessary -select 50, max2(countDistinct(message_format_string), 125) from logs where length(message_format_string) < 30 and message ilike '%DB::Exception%'; - - --- Avoid too noisy messages: top 1 message frequency must be less than 30%. We should reduce the threshold -select 60, max2((select count() from logs group by message_format_string order by count() desc limit 1) / (select count() from logs), 0.30); - --- Same as above, but excluding Test level (actually finds top 1 Trace message) -with ('Access granted: {}{}', '{} -> {}') as frequent_in_tests -select 70, max2((select count() from logs where level!='Test' and message_format_string not in frequent_in_tests - group by message_format_string order by count() desc limit 1) / (select count() from logs), 0.16); - --- Same as above for Debug -select 80, max2((select count() from logs where level <= 'Debug' group by message_format_string order by count() desc limit 1) / (select count() from logs), 0.08); - --- Same as above for Info -select 90, max2((select count() from logs where level <= 'Information' group by message_format_string order by count() desc limit 1) / (select count() from logs), 0.04); - --- Same as above for Warning -with ('Not enabled four letter command {}') as frequent_in_tests -select 100, max2((select count() from logs where level = 'Warning' and message_format_string not in frequent_in_tests - group by message_format_string order by count() desc limit 1) / (select count() from logs), 0.005); - --- Same as above for Error -select 110, max2((select count() from logs where level = 'Error' group by message_format_string order by count() desc limit 1) / (select count() from logs), 0.01); - --- Avoid too noisy messages: limit the number of messages with high frequency -select 120, max2(count(), 3) from (select count() / (select count() from logs) as freq, message_format_string from logs group by message_format_string having freq > 0.10); -select 130, max2(count(), 10) from (select count() / (select count() from logs) as freq, message_format_string from logs group by message_format_string having freq > 0.05); - --- Each message matches its pattern (returns 0 rows) --- FIXME maybe we should make it stricter ('Code:%Exception: '||s||'%'), but it's not easy because of addMessage -select 140, max2(countDistinct(message_format_string), 15) from ( - select message_format_string, any(message) as any_message from logs - where message not like (replaceRegexpAll(message_format_string, '{[:.0-9dfx]*}', '%') as s) - and message not like ('%Exception: '||s||'%') group by message_format_string -) where any_message not like '%Poco::Exception%'; - -drop table logs; From b79ead9c847b9cf5077145611a1c13b8c45daa5b Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 8 Feb 2023 12:04:11 +0100 Subject: [PATCH 79/79] Move poco to base/poco/ (#46075) * Replicate poco into base/poco/ * De-register poco submodule * Build poco from ClickHouse * Exclude poco from stylecheck * Exclude poco from whitespace check * Exclude poco from typo check * Remove x bit from sources/headers (the style check complained) * Exclude poco from duplicate include check * Fix fasttest * Remove contrib/poco-cmake/* * Simplify poco build descriptions * Remove poco stuff not used by ClickHouse * Glob poco sources * Exclude poco from clang-tidy --- .gitmodules | 4 - base/CMakeLists.txt | 5 +- base/base/CMakeLists.txt | 4 + base/pcg-random/CMakeLists.txt | 4 + base/poco/CHANGELOG | 2791 ++++ .../poco-cmake => base/poco}/CMakeLists.txt | 5 +- base/poco/CONTRIBUTORS | 52 + base/poco/Crypto/CMakeLists.txt | 17 + base/poco/Crypto/include/Poco/Crypto/Cipher.h | 138 + .../include/Poco/Crypto/CipherFactory.h | 75 + .../Crypto/include/Poco/Crypto/CipherImpl.h | 69 + .../Crypto/include/Poco/Crypto/CipherKey.h | 201 + .../include/Poco/Crypto/CipherKeyImpl.h | 168 + base/poco/Crypto/include/Poco/Crypto/Crypto.h | 195 + .../include/Poco/Crypto/CryptoException.h | 56 + .../Crypto/include/Poco/Crypto/CryptoStream.h | 192 + .../include/Poco/Crypto/CryptoTransform.h | 87 + .../Crypto/include/Poco/Crypto/DigestEngine.h | 80 + .../include/Poco/Crypto/ECDSADigestEngine.h | 101 + base/poco/Crypto/include/Poco/Crypto/ECKey.h | 136 + .../Crypto/include/Poco/Crypto/ECKeyImpl.h | 174 + .../poco/Crypto/include/Poco/Crypto/EVPPKey.h | 354 + .../poco/Crypto/include/Poco/Crypto/KeyPair.h | 133 + .../Crypto/include/Poco/Crypto/KeyPairImpl.h | 107 + .../include/Poco/Crypto/OpenSSLInitializer.h | 115 + .../include/Poco/Crypto/PKCS12Container.h | 159 + .../include/Poco/Crypto/RSACipherImpl.h | 77 + .../include/Poco/Crypto/RSADigestEngine.h | 111 + base/poco/Crypto/include/Poco/Crypto/RSAKey.h | 125 + .../Crypto/include/Poco/Crypto/RSAKeyImpl.h | 141 + .../include/Poco/Crypto/X509Certificate.h | 245 + base/poco/Crypto/src/Cipher.cpp | 140 + base/poco/Crypto/src/CipherFactory.cpp | 65 + base/poco/Crypto/src/CipherImpl.cpp | 272 + base/poco/Crypto/src/CipherKey.cpp | 49 + base/poco/Crypto/src/CipherKeyImpl.cpp | 222 + base/poco/Crypto/src/CryptoException.cpp | 108 + base/poco/Crypto/src/CryptoStream.cpp | 355 + base/poco/Crypto/src/CryptoTransform.cpp | 38 + base/poco/Crypto/src/DigestEngine.cpp | 80 + base/poco/Crypto/src/ECDSADigestEngine.cpp | 100 + base/poco/Crypto/src/ECKey.cpp | 75 + base/poco/Crypto/src/ECKeyImpl.cpp | 258 + base/poco/Crypto/src/EVPPKey.cpp | 321 + base/poco/Crypto/src/KeyPair.cpp | 34 + base/poco/Crypto/src/KeyPairImpl.cpp | 35 + base/poco/Crypto/src/OpenSSLInitializer.cpp | 190 + base/poco/Crypto/src/PKCS12Container.cpp | 191 + base/poco/Crypto/src/RSACipherImpl.cpp | 342 + base/poco/Crypto/src/RSADigestEngine.cpp | 96 + base/poco/Crypto/src/RSAKey.cpp | 87 + base/poco/Crypto/src/RSAKeyImpl.cpp | 386 + base/poco/Crypto/src/X509Certificate.cpp | 386 + base/poco/Data/CMakeLists.txt | 7 + base/poco/Data/ODBC/CMakeLists.txt | 39 + .../Data/ODBC/include/Poco/Data/ODBC/Binder.h | 1516 ++ .../include/Poco/Data/ODBC/ConnectionHandle.h | 96 + .../ODBC/include/Poco/Data/ODBC/Connector.h | 96 + .../ODBC/include/Poco/Data/ODBC/Diagnostics.h | 239 + .../Poco/Data/ODBC/EnvironmentHandle.h | 94 + .../Data/ODBC/include/Poco/Data/ODBC/Error.h | 124 + .../ODBC/include/Poco/Data/ODBC/Extractor.h | 728 + .../Data/ODBC/include/Poco/Data/ODBC/Handle.h | 113 + .../Data/ODBC/include/Poco/Data/ODBC/ODBC.h | 69 + .../include/Poco/Data/ODBC/ODBCException.h | 150 + .../include/Poco/Data/ODBC/ODBCMetaColumn.h | 92 + .../Poco/Data/ODBC/ODBCStatementImpl.h | 206 + .../ODBC/include/Poco/Data/ODBC/Parameter.h | 111 + .../ODBC/include/Poco/Data/ODBC/Preparator.h | 1271 ++ .../ODBC/include/Poco/Data/ODBC/SessionImpl.h | 290 + .../ODBC/include/Poco/Data/ODBC/TypeInfo.h | 117 + .../ODBC/include/Poco/Data/ODBC/Unicode.h | 985 ++ .../include/Poco/Data/ODBC/Unicode_UNIXODBC.h | 51 + .../include/Poco/Data/ODBC/Unicode_WIN32.h | 57 + .../ODBC/include/Poco/Data/ODBC/Utility.h | 207 + base/poco/Data/ODBC/src/Binder.cpp | 541 + base/poco/Data/ODBC/src/ConnectionHandle.cpp | 57 + base/poco/Data/ODBC/src/Connector.cpp | 64 + base/poco/Data/ODBC/src/EnvironmentHandle.cpp | 54 + base/poco/Data/ODBC/src/Extractor.cpp | 1312 ++ base/poco/Data/ODBC/src/ODBCException.cpp | 30 + base/poco/Data/ODBC/src/ODBCMetaColumn.cpp | 168 + base/poco/Data/ODBC/src/ODBCStatementImpl.cpp | 458 + base/poco/Data/ODBC/src/Parameter.cpp | 53 + base/poco/Data/ODBC/src/Preparator.cpp | 209 + base/poco/Data/ODBC/src/SessionImpl.cpp | 418 + base/poco/Data/ODBC/src/TypeInfo.cpp | 265 + base/poco/Data/ODBC/src/Unicode.cpp | 22 + base/poco/Data/ODBC/src/Unicode_UNIXODBC.cpp | 775 + base/poco/Data/ODBC/src/Unicode_WIN32.cpp | 761 + base/poco/Data/ODBC/src/Utility.cpp | 159 + .../Data/include/Poco/Data/AbstractBinder.h | 373 + .../Data/include/Poco/Data/AbstractBinding.h | 144 + .../include/Poco/Data/AbstractExtraction.h | 278 + .../include/Poco/Data/AbstractExtractor.h | 351 + .../include/Poco/Data/AbstractPreparation.h | 71 + .../include/Poco/Data/AbstractPreparator.h | 388 + .../include/Poco/Data/AbstractSessionImpl.h | 315 + .../Data/include/Poco/Data/ArchiveStrategy.h | 222 + .../Data/include/Poco/Data/AutoTransaction.h | 35 + base/poco/Data/include/Poco/Data/Binding.h | 1496 ++ base/poco/Data/include/Poco/Data/Bulk.h | 96 + .../poco/Data/include/Poco/Data/BulkBinding.h | 148 + .../Data/include/Poco/Data/BulkExtraction.h | 276 + base/poco/Data/include/Poco/Data/Column.h | 490 + base/poco/Data/include/Poco/Data/Connector.h | 56 + base/poco/Data/include/Poco/Data/Constants.h | 36 + base/poco/Data/include/Poco/Data/Data.h | 62 + .../Data/include/Poco/Data/DataException.h | 50 + base/poco/Data/include/Poco/Data/Date.h | 225 + .../Data/include/Poco/Data/DynamicDateTime.h | 47 + base/poco/Data/include/Poco/Data/DynamicLOB.h | 45 + base/poco/Data/include/Poco/Data/Extraction.h | 869 ++ base/poco/Data/include/Poco/Data/LOB.h | 309 + base/poco/Data/include/Poco/Data/LOBStream.h | 150 + base/poco/Data/include/Poco/Data/Limit.h | 113 + base/poco/Data/include/Poco/Data/MetaColumn.h | 188 + .../include/Poco/Data/PooledSessionHolder.h | 99 + .../include/Poco/Data/PooledSessionImpl.h | 97 + base/poco/Data/include/Poco/Data/Position.h | 75 + .../poco/Data/include/Poco/Data/Preparation.h | 161 + base/poco/Data/include/Poco/Data/Range.h | 104 + base/poco/Data/include/Poco/Data/RecordSet.h | 653 + base/poco/Data/include/Poco/Data/Row.h | 312 + base/poco/Data/include/Poco/Data/RowFilter.h | 276 + .../Data/include/Poco/Data/RowFormatter.h | 231 + .../poco/Data/include/Poco/Data/RowIterator.h | 150 + base/poco/Data/include/Poco/Data/SQLChannel.h | 217 + base/poco/Data/include/Poco/Data/Session.h | 494 + .../Data/include/Poco/Data/SessionFactory.h | 99 + .../poco/Data/include/Poco/Data/SessionImpl.h | 221 + .../poco/Data/include/Poco/Data/SessionPool.h | 233 + .../include/Poco/Data/SessionPoolContainer.h | 114 + .../include/Poco/Data/SimpleRowFormatter.h | 122 + base/poco/Data/include/Poco/Data/Statement.h | 822 ++ .../Data/include/Poco/Data/StatementCreator.h | 73 + .../Data/include/Poco/Data/StatementImpl.h | 632 + base/poco/Data/include/Poco/Data/Time.h | 229 + .../poco/Data/include/Poco/Data/Transaction.h | 194 + .../poco/Data/include/Poco/Data/TypeHandler.h | 2090 +++ base/poco/Data/src/AbstractBinder.cpp | 527 + base/poco/Data/src/AbstractBinding.cpp | 45 + base/poco/Data/src/AbstractExtraction.cpp | 40 + base/poco/Data/src/AbstractExtractor.cpp | 437 + base/poco/Data/src/AbstractPreparation.cpp | 34 + base/poco/Data/src/AbstractPreparator.cpp | 438 + base/poco/Data/src/ArchiveStrategy.cpp | 146 + base/poco/Data/src/Bulk.cpp | 37 + base/poco/Data/src/Connector.cpp | 32 + base/poco/Data/src/DataException.cpp | 41 + base/poco/Data/src/Date.cpp | 142 + base/poco/Data/src/DynamicLOB.cpp | 74 + base/poco/Data/src/Limit.cpp | 35 + base/poco/Data/src/MetaColumn.cpp | 53 + base/poco/Data/src/PooledSessionHolder.cpp | 34 + base/poco/Data/src/PooledSessionImpl.cpp | 192 + base/poco/Data/src/Position.cpp | 32 + base/poco/Data/src/Range.cpp | 40 + base/poco/Data/src/RecordSet.cpp | 392 + base/poco/Data/src/Row.cpp | 369 + base/poco/Data/src/RowFilter.cpp | 225 + base/poco/Data/src/RowFormatter.cpp | 80 + base/poco/Data/src/RowIterator.cpp | 207 + base/poco/Data/src/SQLChannel.cpp | 296 + base/poco/Data/src/Session.cpp | 78 + base/poco/Data/src/SessionFactory.cpp | 93 + base/poco/Data/src/SessionImpl.cpp | 52 + base/poco/Data/src/SessionPool.cpp | 308 + base/poco/Data/src/SessionPoolContainer.cpp | 122 + base/poco/Data/src/SimpleRowFormatter.cpp | 113 + base/poco/Data/src/Statement.cpp | 296 + base/poco/Data/src/StatementCreator.cpp | 60 + base/poco/Data/src/StatementImpl.cpp | 476 + base/poco/Data/src/Time.cpp | 139 + base/poco/Data/src/Transaction.cpp | 124 + base/poco/Foundation/CMakeLists.txt | 211 + .../Foundation/include/Poco/ASCIIEncoding.h | 51 + .../Foundation/include/Poco/AbstractCache.h | 366 + .../include/Poco/AbstractDelegate.h | 110 + .../Foundation/include/Poco/AbstractEvent.h | 558 + .../include/Poco/AbstractObserver.h | 50 + .../include/Poco/AbstractPriorityDelegate.h | 63 + .../include/Poco/AbstractStrategy.h | 77 + .../include/Poco/AccessExpirationDecorator.h | 83 + .../include/Poco/AccessExpireCache.h | 63 + .../include/Poco/AccessExpireLRUCache.h | 62 + .../include/Poco/AccessExpireStrategy.h | 74 + .../include/Poco/ActiveDispatcher.h | 116 + .../Foundation/include/Poco/ActiveMethod.h | 218 + .../Foundation/include/Poco/ActiveResult.h | 495 + .../Foundation/include/Poco/ActiveRunnable.h | 231 + .../Foundation/include/Poco/ActiveStarter.h | 48 + base/poco/Foundation/include/Poco/Activity.h | 208 + base/poco/Foundation/include/Poco/Alignment.h | 246 + base/poco/Foundation/include/Poco/Any.h | 599 + .../Foundation/include/Poco/ArchiveStrategy.h | 173 + base/poco/Foundation/include/Poco/Array.h | 249 + base/poco/Foundation/include/Poco/Ascii.h | 220 + .../Foundation/include/Poco/AsyncChannel.h | 100 + .../Foundation/include/Poco/AtomicCounter.h | 417 + base/poco/Foundation/include/Poco/AutoPtr.h | 386 + .../Foundation/include/Poco/AutoReleasePool.h | 86 + .../Foundation/include/Poco/Base32Decoder.h | 104 + .../Foundation/include/Poco/Base32Encoder.h | 110 + .../Foundation/include/Poco/Base64Decoder.h | 108 + .../Foundation/include/Poco/Base64Encoder.h | 138 + .../poco/Foundation/include/Poco/BasicEvent.h | 60 + .../Foundation/include/Poco/BinaryReader.h | 265 + .../Foundation/include/Poco/BinaryWriter.h | 269 + base/poco/Foundation/include/Poco/Buffer.h | 329 + .../Foundation/include/Poco/BufferAllocator.h | 52 + .../Poco/BufferedBidirectionalStreamBuf.h | 181 + .../include/Poco/BufferedStreamBuf.h | 173 + base/poco/Foundation/include/Poco/Bugcheck.h | 206 + base/poco/Foundation/include/Poco/ByteOrder.h | 239 + base/poco/Foundation/include/Poco/Channel.h | 78 + base/poco/Foundation/include/Poco/Checksum.h | 108 + .../Foundation/include/Poco/ClassLibrary.h | 101 + .../Foundation/include/Poco/ClassLoader.h | 365 + base/poco/Foundation/include/Poco/Clock.h | 238 + .../include/Poco/CompressedLogFile.h | 60 + base/poco/Foundation/include/Poco/Condition.h | 137 + base/poco/Foundation/include/Poco/Config.h | 207 + .../Foundation/include/Poco/Configurable.h | 70 + .../Foundation/include/Poco/ConsoleChannel.h | 187 + .../Foundation/include/Poco/CountingStream.h | 237 + base/poco/Foundation/include/Poco/DateTime.h | 415 + .../Foundation/include/Poco/DateTimeFormat.h | 109 + .../include/Poco/DateTimeFormatter.h | 215 + .../Foundation/include/Poco/DateTimeParser.h | 116 + base/poco/Foundation/include/Poco/Debugger.h | 70 + .../Foundation/include/Poco/DefaultStrategy.h | 226 + .../Foundation/include/Poco/DeflatingStream.h | 201 + base/poco/Foundation/include/Poco/Delegate.h | 452 + .../Foundation/include/Poco/DigestEngine.h | 111 + .../Foundation/include/Poco/DigestStream.h | 100 + .../include/Poco/DirectoryIterator.h | 153 + .../include/Poco/DirectoryIteratorStrategy.h | 92 + .../include/Poco/DirectoryIterator_UNIX.h | 72 + .../include/Poco/DirectoryIterator_WIN32.h | 73 + .../include/Poco/DirectoryIterator_WIN32U.h | 73 + .../include/Poco/DirectoryWatcher.h | 232 + .../Foundation/include/Poco/Dynamic/Pair.h | 412 + .../Foundation/include/Poco/Dynamic/Struct.h | 567 + .../Foundation/include/Poco/Dynamic/Var.h | 2228 +++ .../include/Poco/Dynamic/VarHolder.h | 4186 ++++++ .../include/Poco/Dynamic/VarIterator.h | 150 + .../poco/Foundation/include/Poco/DynamicAny.h | 24 + .../include/Poco/DynamicAnyHolder.h | 24 + .../Foundation/include/Poco/DynamicFactory.h | 141 + .../Foundation/include/Poco/DynamicStruct.h | 24 + .../Foundation/include/Poco/Environment.h | 123 + .../include/Poco/Environment_UNIX.h | 56 + .../Foundation/include/Poco/Environment_VX.h | 56 + .../include/Poco/Environment_WIN32.h | 48 + .../include/Poco/Environment_WIN32U.h | 48 + .../include/Poco/Environment_WINCE.h | 59 + base/poco/Foundation/include/Poco/Error.h | 52 + .../Foundation/include/Poco/ErrorHandler.h | 120 + base/poco/Foundation/include/Poco/Event.h | 123 + base/poco/Foundation/include/Poco/EventArgs.h | 45 + .../Foundation/include/Poco/EventChannel.h | 54 + .../Foundation/include/Poco/EventLogChannel.h | 105 + .../Foundation/include/Poco/Event_POSIX.h | 77 + base/poco/Foundation/include/Poco/Event_VX.h | 49 + .../Foundation/include/Poco/Event_WIN32.h | 68 + base/poco/Foundation/include/Poco/Exception.h | 264 + .../include/Poco/ExpirationDecorator.h | 92 + base/poco/Foundation/include/Poco/Expire.h | 194 + .../Foundation/include/Poco/ExpireCache.h | 65 + .../Foundation/include/Poco/ExpireLRUCache.h | 62 + .../Foundation/include/Poco/ExpireStrategy.h | 132 + .../poco/Foundation/include/Poco/FIFOBuffer.h | 556 + .../include/Poco/FIFOBufferStream.h | 154 + base/poco/Foundation/include/Poco/FIFOEvent.h | 63 + .../Foundation/include/Poco/FIFOStrategy.h | 59 + .../Foundation/include/Poco/FPEnvironment.h | 207 + .../include/Poco/FPEnvironment_C99.h | 125 + .../include/Poco/FPEnvironment_DEC.h | 89 + .../include/Poco/FPEnvironment_DUMMY.h | 125 + .../include/Poco/FPEnvironment_QNX.h | 133 + .../include/Poco/FPEnvironment_SUN.h | 74 + .../include/Poco/FPEnvironment_WIN32.h | 150 + base/poco/Foundation/include/Poco/File.h | 315 + .../Foundation/include/Poco/FileChannel.h | 276 + .../poco/Foundation/include/Poco/FileStream.h | 176 + .../include/Poco/FileStreamFactory.h | 61 + .../include/Poco/FileStream_POSIX.h | 72 + .../include/Poco/FileStream_WIN32.h | 71 + base/poco/Foundation/include/Poco/File_UNIX.h | 86 + base/poco/Foundation/include/Poco/File_VX.h | 84 + .../poco/Foundation/include/Poco/File_WIN32.h | 87 + .../Foundation/include/Poco/File_WIN32U.h | 91 + .../poco/Foundation/include/Poco/File_WINCE.h | 90 + base/poco/Foundation/include/Poco/Format.h | 138 + base/poco/Foundation/include/Poco/Formatter.h | 74 + .../include/Poco/FormattingChannel.h | 96 + .../poco/Foundation/include/Poco/Foundation.h | 168 + .../include/Poco/FunctionDelegate.h | 438 + .../include/Poco/FunctionPriorityDelegate.h | 450 + base/poco/Foundation/include/Poco/Glob.h | 152 + .../poco/Foundation/include/Poco/HMACEngine.h | 146 + base/poco/Foundation/include/Poco/Hash.h | 105 + .../Foundation/include/Poco/HashFunction.h | 57 + base/poco/Foundation/include/Poco/HashMap.h | 228 + base/poco/Foundation/include/Poco/HashSet.h | 177 + .../Foundation/include/Poco/HashStatistic.h | 126 + base/poco/Foundation/include/Poco/HashTable.h | 369 + .../include/Poco/HexBinaryDecoder.h | 94 + .../include/Poco/HexBinaryEncoder.h | 117 + .../Foundation/include/Poco/InflatingStream.h | 211 + .../Foundation/include/Poco/Instantiator.h | 82 + .../poco/Foundation/include/Poco/JSONString.h | 93 + .../Foundation/include/Poco/KeyValueArgs.h | 73 + base/poco/Foundation/include/Poco/LRUCache.h | 56 + .../Foundation/include/Poco/LRUStrategy.h | 140 + .../Foundation/include/Poco/Latin1Encoding.h | 51 + .../Foundation/include/Poco/Latin2Encoding.h | 54 + .../Foundation/include/Poco/Latin9Encoding.h | 54 + .../include/Poco/LineEndingConverter.h | 181 + .../Foundation/include/Poco/LinearHashTable.h | 503 + base/poco/Foundation/include/Poco/ListMap.h | 246 + .../Foundation/include/Poco/LocalDateTime.h | 388 + base/poco/Foundation/include/Poco/LogFile.h | 104 + .../Foundation/include/Poco/LogFile_STD.h | 53 + .../Foundation/include/Poco/LogFile_WIN32.h | 55 + .../Foundation/include/Poco/LogFile_WIN32U.h | 55 + base/poco/Foundation/include/Poco/LogStream.h | 192 + base/poco/Foundation/include/Poco/Logger.h | 1447 ++ .../Foundation/include/Poco/LoggingFactory.h | 92 + .../Foundation/include/Poco/LoggingRegistry.h | 99 + base/poco/Foundation/include/Poco/MD4Engine.h | 95 + base/poco/Foundation/include/Poco/MD5Engine.h | 95 + base/poco/Foundation/include/Poco/Manifest.h | 180 + .../poco/Foundation/include/Poco/MemoryPool.h | 115 + .../Foundation/include/Poco/MemoryStream.h | 246 + base/poco/Foundation/include/Poco/Message.h | 270 + .../poco/Foundation/include/Poco/MetaObject.h | 201 + .../Foundation/include/Poco/MetaProgramming.h | 144 + base/poco/Foundation/include/Poco/Mutex.h | 270 + .../Foundation/include/Poco/Mutex_POSIX.h | 86 + base/poco/Foundation/include/Poco/Mutex_VX.h | 80 + .../Foundation/include/Poco/Mutex_WIN32.h | 85 + .../Foundation/include/Poco/Mutex_WINCE.h | 50 + base/poco/Foundation/include/Poco/NObserver.h | 126 + .../poco/Foundation/include/Poco/NamedEvent.h | 96 + .../include/Poco/NamedEvent_Android.h | 40 + .../Foundation/include/Poco/NamedEvent_UNIX.h | 53 + .../include/Poco/NamedEvent_WIN32.h | 45 + .../include/Poco/NamedEvent_WIN32U.h | 46 + .../poco/Foundation/include/Poco/NamedMutex.h | 110 + .../include/Poco/NamedMutex_Android.h | 41 + .../Foundation/include/Poco/NamedMutex_UNIX.h | 57 + .../include/Poco/NamedMutex_WIN32.h | 46 + .../include/Poco/NamedMutex_WIN32U.h | 47 + .../poco/Foundation/include/Poco/NamedTuple.h | 4462 ++++++ .../include/Poco/NestedDiagnosticContext.h | 181 + .../Foundation/include/Poco/Notification.h | 55 + .../include/Poco/NotificationCenter.h | 139 + .../include/Poco/NotificationQueue.h | 155 + .../include/Poco/NotificationStrategy.h | 112 + .../Foundation/include/Poco/NullChannel.h | 55 + .../poco/Foundation/include/Poco/NullStream.h | 90 + base/poco/Foundation/include/Poco/Nullable.h | 323 + .../Foundation/include/Poco/NumberFormatter.h | 907 ++ .../Foundation/include/Poco/NumberParser.h | 156 + .../Foundation/include/Poco/NumericString.h | 560 + .../poco/Foundation/include/Poco/ObjectPool.h | 332 + base/poco/Foundation/include/Poco/Observer.h | 124 + base/poco/Foundation/include/Poco/Optional.h | 161 + .../Foundation/include/Poco/PBKDF2Engine.h | 149 + base/poco/Foundation/include/Poco/Path.h | 501 + base/poco/Foundation/include/Poco/Path_UNIX.h | 48 + .../poco/Foundation/include/Poco/Path_WIN32.h | 49 + .../Foundation/include/Poco/Path_WIN32U.h | 54 + .../poco/Foundation/include/Poco/Path_WINCE.h | 54 + .../include/Poco/PatternFormatter.h | 148 + base/poco/Foundation/include/Poco/Pipe.h | 142 + base/poco/Foundation/include/Poco/PipeImpl.h | 37 + .../Foundation/include/Poco/PipeImpl_DUMMY.h | 49 + .../Foundation/include/Poco/PipeImpl_POSIX.h | 53 + .../Foundation/include/Poco/PipeImpl_WIN32.h | 54 + .../poco/Foundation/include/Poco/PipeStream.h | 120 + base/poco/Foundation/include/Poco/Platform.h | 284 + .../Foundation/include/Poco/Platform_POSIX.h | 53 + .../Foundation/include/Poco/Platform_VX.h | 29 + .../Foundation/include/Poco/Platform_WIN32.h | 216 + base/poco/Foundation/include/Poco/Poco.h | 24 + .../include/Poco/PriorityDelegate.h | 464 + .../Foundation/include/Poco/PriorityEvent.h | 61 + .../Foundation/include/Poco/PriorityExpire.h | 195 + .../include/Poco/PriorityNotificationQueue.h | 158 + .../include/Poco/PriorityStrategy.h | 230 + base/poco/Foundation/include/Poco/Process.h | 266 + .../Foundation/include/Poco/Process_UNIX.h | 86 + .../poco/Foundation/include/Poco/Process_VX.h | 78 + .../Foundation/include/Poco/Process_WIN32.h | 83 + .../Foundation/include/Poco/Process_WIN32U.h | 83 + .../Foundation/include/Poco/Process_WINCE.h | 83 + .../Foundation/include/Poco/PurgeStrategy.h | 107 + base/poco/Foundation/include/Poco/RWLock.h | 203 + .../Foundation/include/Poco/RWLock_Android.h | 97 + .../Foundation/include/Poco/RWLock_POSIX.h | 99 + base/poco/Foundation/include/Poco/RWLock_VX.h | 92 + .../Foundation/include/Poco/RWLock_WIN32.h | 57 + .../Foundation/include/Poco/RWLock_WINCE.h | 61 + base/poco/Foundation/include/Poco/Random.h | 165 + .../Foundation/include/Poco/RandomStream.h | 73 + .../include/Poco/RecursiveDirectoryIterator.h | 254 + .../Poco/RecursiveDirectoryIteratorImpl.h | 111 + .../include/Poco/RefCountedObject.h | 94 + .../include/Poco/RegularExpression.h | 247 + .../Foundation/include/Poco/RotateStrategy.h | 165 + base/poco/Foundation/include/Poco/Runnable.h | 45 + .../Foundation/include/Poco/RunnableAdapter.h | 78 + .../poco/Foundation/include/Poco/SHA1Engine.h | 81 + .../poco/Foundation/include/Poco/ScopedLock.h | 121 + .../Foundation/include/Poco/ScopedUnlock.h | 64 + base/poco/Foundation/include/Poco/Semaphore.h | 137 + .../Foundation/include/Poco/Semaphore_POSIX.h | 75 + .../Foundation/include/Poco/Semaphore_VX.h | 56 + .../Foundation/include/Poco/Semaphore_WIN32.h | 58 + .../Foundation/include/Poco/SharedLibrary.h | 144 + .../include/Poco/SharedLibrary_HPUX.h | 52 + .../include/Poco/SharedLibrary_UNIX.h | 57 + .../include/Poco/SharedLibrary_VX.h | 52 + .../include/Poco/SharedLibrary_WIN32.h | 51 + .../include/Poco/SharedLibrary_WIN32U.h | 51 + .../Foundation/include/Poco/SharedMemory.h | 113 + .../include/Poco/SharedMemory_DUMMY.h | 88 + .../include/Poco/SharedMemory_POSIX.h | 107 + .../include/Poco/SharedMemory_WIN32.h | 103 + base/poco/Foundation/include/Poco/SharedPtr.h | 448 + .../Foundation/include/Poco/SignalHandler.h | 129 + .../include/Poco/SimpleFileChannel.h | 150 + .../Foundation/include/Poco/SimpleHashTable.h | 398 + .../Foundation/include/Poco/SingletonHolder.h | 77 + .../include/Poco/SortedDirectoryIterator.h | 70 + .../Foundation/include/Poco/SplitterChannel.h | 77 + base/poco/Foundation/include/Poco/Stopwatch.h | 108 + .../include/Poco/StrategyCollection.h | 133 + .../Foundation/include/Poco/StreamChannel.h | 59 + .../Foundation/include/Poco/StreamConverter.h | 128 + .../Foundation/include/Poco/StreamCopier.h | 85 + .../Foundation/include/Poco/StreamTokenizer.h | 97 + .../poco/Foundation/include/Poco/StreamUtil.h | 92 + base/poco/Foundation/include/Poco/String.h | 706 + .../Foundation/include/Poco/StringTokenizer.h | 135 + .../include/Poco/SynchronizedObject.h | 131 + .../Foundation/include/Poco/SyslogChannel.h | 109 + base/poco/Foundation/include/Poco/Task.h | 190 + .../Foundation/include/Poco/TaskManager.h | 137 + .../include/Poco/TaskNotification.h | 170 + base/poco/Foundation/include/Poco/TeeStream.h | 132 + .../Foundation/include/Poco/TemporaryFile.h | 84 + .../include/Poco/TextBufferIterator.h | 145 + .../Foundation/include/Poco/TextConverter.h | 93 + .../Foundation/include/Poco/TextEncoding.h | 185 + .../Foundation/include/Poco/TextIterator.h | 141 + base/poco/Foundation/include/Poco/Thread.h | 367 + .../Foundation/include/Poco/ThreadLocal.h | 157 + .../poco/Foundation/include/Poco/ThreadPool.h | 208 + .../Foundation/include/Poco/ThreadTarget.h | 87 + .../Foundation/include/Poco/Thread_POSIX.h | 198 + base/poco/Foundation/include/Poco/Thread_VX.h | 169 + .../Foundation/include/Poco/Thread_WIN32.h | 183 + .../Foundation/include/Poco/Thread_WINCE.h | 179 + .../include/Poco/TimedNotificationQueue.h | 143 + base/poco/Foundation/include/Poco/Timer.h | 229 + base/poco/Foundation/include/Poco/Timespan.h | 303 + base/poco/Foundation/include/Poco/Timestamp.h | 277 + base/poco/Foundation/include/Poco/Timezone.h | 67 + base/poco/Foundation/include/Poco/Token.h | 172 + base/poco/Foundation/include/Poco/Tuple.h | 1747 +++ base/poco/Foundation/include/Poco/TypeList.h | 483 + base/poco/Foundation/include/Poco/Types.h | 201 + base/poco/Foundation/include/Poco/URI.h | 406 + .../include/Poco/URIStreamFactory.h | 97 + .../Foundation/include/Poco/URIStreamOpener.h | 135 + .../Foundation/include/Poco/UTF16Encoding.h | 85 + .../Foundation/include/Poco/UTF32Encoding.h | 82 + .../Foundation/include/Poco/UTF8Encoding.h | 61 + .../poco/Foundation/include/Poco/UTF8String.h | 83 + base/poco/Foundation/include/Poco/UTFString.h | 305 + base/poco/Foundation/include/Poco/UUID.h | 219 + .../Foundation/include/Poco/UUIDGenerator.h | 104 + base/poco/Foundation/include/Poco/UnWindows.h | 142 + .../include/Poco/UnbufferedStreamBuf.h | 179 + base/poco/Foundation/include/Poco/Unicode.h | 327 + .../include/Poco/UnicodeConverter.h | 141 + .../include/Poco/UniqueAccessExpireCache.h | 73 + .../include/Poco/UniqueAccessExpireLRUCache.h | 71 + .../include/Poco/UniqueAccessExpireStrategy.h | 156 + .../include/Poco/UniqueExpireCache.h | 72 + .../include/Poco/UniqueExpireLRUCache.h | 70 + .../include/Poco/UniqueExpireStrategy.h | 138 + base/poco/Foundation/include/Poco/ValidArgs.h | 74 + base/poco/Foundation/include/Poco/Version.h | 42 + base/poco/Foundation/include/Poco/Void.h | 73 + .../include/Poco/Windows1250Encoding.h | 52 + .../include/Poco/Windows1251Encoding.h | 52 + .../include/Poco/Windows1252Encoding.h | 51 + .../include/Poco/WindowsConsoleChannel.h | 183 + base/poco/Foundation/src/ASCIIEncoding.cpp | 116 + base/poco/Foundation/src/AbstractObserver.cpp | 42 + base/poco/Foundation/src/ActiveDispatcher.cpp | 116 + base/poco/Foundation/src/ArchiveStrategy.cpp | 212 + base/poco/Foundation/src/Ascii.cpp | 154 + base/poco/Foundation/src/AsyncChannel.cpp | 168 + base/poco/Foundation/src/AtomicCounter.cpp | 231 + base/poco/Foundation/src/Base32Decoder.cpp | 160 + base/poco/Foundation/src/Base32Encoder.cpp | 202 + base/poco/Foundation/src/Base64Decoder.cpp | 178 + base/poco/Foundation/src/Base64Encoder.cpp | 183 + base/poco/Foundation/src/BinaryReader.cpp | 287 + base/poco/Foundation/src/BinaryWriter.cpp | 351 + base/poco/Foundation/src/Bugcheck.cpp | 119 + base/poco/Foundation/src/ByteOrder.cpp | 15 + base/poco/Foundation/src/Channel.cpp | 53 + base/poco/Foundation/src/Checksum.cpp | 58 + base/poco/Foundation/src/Clock.cpp | 231 + .../poco/Foundation/src/CompressedLogFile.cpp | 112 + base/poco/Foundation/src/Condition.cpp | 79 + base/poco/Foundation/src/Configurable.cpp | 31 + base/poco/Foundation/src/ConsoleChannel.cpp | 281 + base/poco/Foundation/src/CountingStream.cpp | 199 + base/poco/Foundation/src/DateTime.cpp | 432 + base/poco/Foundation/src/DateTimeFormat.cpp | 61 + .../poco/Foundation/src/DateTimeFormatter.cpp | 155 + base/poco/Foundation/src/DateTimeParser.cpp | 393 + base/poco/Foundation/src/Debugger.cpp | 150 + base/poco/Foundation/src/DeflatingStream.cpp | 390 + base/poco/Foundation/src/DigestEngine.cpp | 99 + base/poco/Foundation/src/DigestStream.cpp | 150 + .../poco/Foundation/src/DirectoryIterator.cpp | 145 + .../src/DirectoryIteratorStrategy.cpp | 168 + .../Foundation/src/DirectoryIterator_UNIX.cpp | 64 + .../src/DirectoryIterator_WIN32.cpp | 66 + .../src/DirectoryIterator_WIN32U.cpp | 75 + base/poco/Foundation/src/DirectoryWatcher.cpp | 608 + base/poco/Foundation/src/Environment.cpp | 160 + base/poco/Foundation/src/Environment_UNIX.cpp | 332 + base/poco/Foundation/src/Environment_VX.cpp | 159 + .../poco/Foundation/src/Environment_WIN32.cpp | 221 + .../Foundation/src/Environment_WIN32U.cpp | 235 + .../poco/Foundation/src/Environment_WINCE.cpp | 243 + base/poco/Foundation/src/Error.cpp | 117 + base/poco/Foundation/src/ErrorHandler.cpp | 114 + base/poco/Foundation/src/Event.cpp | 40 + base/poco/Foundation/src/EventArgs.cpp | 33 + base/poco/Foundation/src/EventChannel.cpp | 37 + base/poco/Foundation/src/EventLogChannel.cpp | 319 + base/poco/Foundation/src/Event_POSIX.cpp | 170 + base/poco/Foundation/src/Event_VX.cpp | 79 + base/poco/Foundation/src/Event_WIN32.cpp | 61 + base/poco/Foundation/src/Exception.cpp | 185 + base/poco/Foundation/src/FIFOBufferStream.cpp | 186 + base/poco/Foundation/src/FPEnvironment.cpp | 83 + .../poco/Foundation/src/FPEnvironment_C99.cpp | 82 + .../poco/Foundation/src/FPEnvironment_DEC.cpp | 195 + .../Foundation/src/FPEnvironment_DUMMY.cpp | 79 + .../poco/Foundation/src/FPEnvironment_QNX.cpp | 82 + .../poco/Foundation/src/FPEnvironment_SUN.cpp | 139 + .../Foundation/src/FPEnvironment_WIN32.cpp | 76 + base/poco/Foundation/src/File.cpp | 394 + base/poco/Foundation/src/FileChannel.cpp | 570 + base/poco/Foundation/src/FileStream.cpp | 121 + .../poco/Foundation/src/FileStreamFactory.cpp | 64 + base/poco/Foundation/src/FileStream_POSIX.cpp | 170 + base/poco/Foundation/src/FileStream_WIN32.cpp | 208 + base/poco/Foundation/src/File_UNIX.cpp | 527 + base/poco/Foundation/src/File_VX.cpp | 408 + base/poco/Foundation/src/File_WIN32.cpp | 465 + base/poco/Foundation/src/File_WIN32U.cpp | 492 + base/poco/Foundation/src/File_WINCE.cpp | 441 + base/poco/Foundation/src/Format.cpp | 476 + base/poco/Foundation/src/Formatter.cpp | 44 + .../poco/Foundation/src/FormattingChannel.cpp | 126 + base/poco/Foundation/src/Glob.cpp | 285 + base/poco/Foundation/src/Hash.cpp | 34 + base/poco/Foundation/src/HashStatistic.cpp | 64 + base/poco/Foundation/src/HexBinaryDecoder.cpp | 94 + base/poco/Foundation/src/HexBinaryEncoder.cpp | 117 + base/poco/Foundation/src/InflatingStream.cpp | 360 + base/poco/Foundation/src/JSONString.cpp | 101 + base/poco/Foundation/src/Latin1Encoding.cpp | 119 + base/poco/Foundation/src/Latin2Encoding.cpp | 179 + base/poco/Foundation/src/Latin9Encoding.cpp | 130 + .../Foundation/src/LineEndingConverter.cpp | 175 + base/poco/Foundation/src/LocalDateTime.cpp | 317 + base/poco/Foundation/src/LogFile.cpp | 40 + base/poco/Foundation/src/LogFile_STD.cpp | 76 + base/poco/Foundation/src/LogFile_WIN32.cpp | 125 + base/poco/Foundation/src/LogFile_WIN32U.cpp | 130 + base/poco/Foundation/src/LogStream.cpp | 231 + base/poco/Foundation/src/Logger.cpp | 485 + base/poco/Foundation/src/LoggingFactory.cpp | 117 + base/poco/Foundation/src/LoggingRegistry.cpp | 117 + base/poco/Foundation/src/MD4Engine.cpp | 278 + base/poco/Foundation/src/MD5Engine.cpp | 309 + base/poco/Foundation/src/Manifest.cpp | 31 + base/poco/Foundation/src/MemoryPool.cpp | 105 + base/poco/Foundation/src/MemoryStream.cpp | 57 + base/poco/Foundation/src/Message.cpp | 289 + base/poco/Foundation/src/Mutex.cpp | 54 + base/poco/Foundation/src/Mutex_POSIX.cpp | 175 + base/poco/Foundation/src/Mutex_VX.cpp | 68 + base/poco/Foundation/src/Mutex_WIN32.cpp | 59 + base/poco/Foundation/src/Mutex_WINCE.cpp | 80 + base/poco/Foundation/src/NamedEvent.cpp | 43 + .../Foundation/src/NamedEvent_Android.cpp | 44 + base/poco/Foundation/src/NamedEvent_UNIX.cpp | 148 + base/poco/Foundation/src/NamedEvent_WIN32.cpp | 61 + .../poco/Foundation/src/NamedEvent_WIN32U.cpp | 63 + base/poco/Foundation/src/NamedMutex.cpp | 43 + .../Foundation/src/NamedMutex_Android.cpp | 50 + base/poco/Foundation/src/NamedMutex_UNIX.cpp | 170 + base/poco/Foundation/src/NamedMutex_WIN32.cpp | 73 + .../poco/Foundation/src/NamedMutex_WIN32U.cpp | 75 + .../src/NestedDiagnosticContext.cpp | 129 + base/poco/Foundation/src/Notification.cpp | 38 + .../Foundation/src/NotificationCenter.cpp | 109 + .../poco/Foundation/src/NotificationQueue.cpp | 211 + base/poco/Foundation/src/NullChannel.cpp | 41 + base/poco/Foundation/src/NullStream.cpp | 74 + base/poco/Foundation/src/NumberFormatter.cpp | 492 + base/poco/Foundation/src/NumberParser.cpp | 250 + base/poco/Foundation/src/NumericString.cpp | 316 + base/poco/Foundation/src/Path.cpp | 1064 ++ base/poco/Foundation/src/Path_UNIX.cpp | 265 + base/poco/Foundation/src/Path_WIN32.cpp | 203 + base/poco/Foundation/src/Path_WIN32U.cpp | 222 + base/poco/Foundation/src/Path_WINCE.cpp | 144 + base/poco/Foundation/src/PatternFormatter.cpp | 247 + base/poco/Foundation/src/Pipe.cpp | 70 + base/poco/Foundation/src/PipeImpl.cpp | 28 + base/poco/Foundation/src/PipeImpl_DUMMY.cpp | 65 + base/poco/Foundation/src/PipeImpl_POSIX.cpp | 111 + base/poco/Foundation/src/PipeImpl_WIN32.cpp | 97 + base/poco/Foundation/src/PipeStream.cpp | 127 + .../src/PriorityNotificationQueue.cpp | 195 + base/poco/Foundation/src/Process.cpp | 197 + base/poco/Foundation/src/Process_UNIX.cpp | 280 + base/poco/Foundation/src/Process_VX.cpp | 100 + base/poco/Foundation/src/Process_WIN32.cpp | 349 + base/poco/Foundation/src/Process_WIN32U.cpp | 371 + base/poco/Foundation/src/Process_WINCE.cpp | 223 + base/poco/Foundation/src/PurgeStrategy.cpp | 154 + base/poco/Foundation/src/RWLock.cpp | 46 + base/poco/Foundation/src/RWLock_Android.cpp | 39 + base/poco/Foundation/src/RWLock_POSIX.cpp | 34 + base/poco/Foundation/src/RWLock_VX.cpp | 48 + base/poco/Foundation/src/RWLock_WIN32.cpp | 207 + base/poco/Foundation/src/RWLock_WINCE.cpp | 174 + base/poco/Foundation/src/Random.cpp | 364 + base/poco/Foundation/src/RandomStream.cpp | 127 + base/poco/Foundation/src/RefCountedObject.cpp | 31 + .../poco/Foundation/src/RegularExpression.cpp | 284 + base/poco/Foundation/src/RotateStrategy.cpp | 113 + base/poco/Foundation/src/Runnable.cpp | 32 + base/poco/Foundation/src/SHA1Engine.cpp | 283 + base/poco/Foundation/src/Semaphore.cpp | 45 + base/poco/Foundation/src/Semaphore_POSIX.cpp | 170 + base/poco/Foundation/src/Semaphore_VX.cpp | 52 + base/poco/Foundation/src/Semaphore_WIN32.cpp | 65 + base/poco/Foundation/src/SharedLibrary.cpp | 115 + .../Foundation/src/SharedLibrary_HPUX.cpp | 100 + .../Foundation/src/SharedLibrary_UNIX.cpp | 136 + base/poco/Foundation/src/SharedLibrary_VX.cpp | 141 + .../Foundation/src/SharedLibrary_WIN32.cpp | 108 + .../Foundation/src/SharedLibrary_WIN32U.cpp | 121 + base/poco/Foundation/src/SharedMemory.cpp | 90 + .../Foundation/src/SharedMemory_DUMMY.cpp | 36 + .../Foundation/src/SharedMemory_POSIX.cpp | 134 + .../Foundation/src/SharedMemory_WIN32.cpp | 168 + base/poco/Foundation/src/SignalHandler.cpp | 111 + .../poco/Foundation/src/SimpleFileChannel.cpp | 227 + .../src/SortedDirectoryIterator.cpp | 122 + base/poco/Foundation/src/SplitterChannel.cpp | 113 + base/poco/Foundation/src/Stopwatch.cpp | 60 + base/poco/Foundation/src/StreamChannel.cpp | 40 + base/poco/Foundation/src/StreamConverter.cpp | 192 + base/poco/Foundation/src/StreamCopier.cpp | 150 + base/poco/Foundation/src/StreamTokenizer.cpp | 105 + base/poco/Foundation/src/String.cpp | 240 + base/poco/Foundation/src/StringTokenizer.cpp | 135 + .../Foundation/src/SynchronizedObject.cpp | 31 + base/poco/Foundation/src/SyslogChannel.cpp | 257 + base/poco/Foundation/src/Task.cpp | 146 + base/poco/Foundation/src/TaskManager.cpp | 156 + base/poco/Foundation/src/TaskNotification.cpp | 92 + base/poco/Foundation/src/TeeStream.cpp | 133 + base/poco/Foundation/src/TemporaryFile.cpp | 167 + .../Foundation/src/TextBufferIterator.cpp | 178 + base/poco/Foundation/src/TextConverter.cpp | 128 + base/poco/Foundation/src/TextEncoding.cpp | 208 + base/poco/Foundation/src/TextIterator.cpp | 175 + base/poco/Foundation/src/Thread.cpp | 212 + base/poco/Foundation/src/ThreadLocal.cpp | 85 + base/poco/Foundation/src/ThreadPool.cpp | 527 + base/poco/Foundation/src/ThreadTarget.cpp | 43 + base/poco/Foundation/src/Thread_POSIX.cpp | 412 + base/poco/Foundation/src/Thread_VX.cpp | 273 + base/poco/Foundation/src/Thread_WIN32.cpp | 223 + base/poco/Foundation/src/Thread_WINCE.cpp | 169 + .../Foundation/src/TimedNotificationQueue.cpp | 203 + base/poco/Foundation/src/Timer.cpp | 247 + base/poco/Foundation/src/Timespan.cpp | 145 + base/poco/Foundation/src/Timestamp.cpp | 313 + base/poco/Foundation/src/Timezone.cpp | 41 + base/poco/Foundation/src/Timezone_UNIX.cpp | 110 + base/poco/Foundation/src/Timezone_VX.cpp | 78 + base/poco/Foundation/src/Timezone_WIN32.cpp | 101 + base/poco/Foundation/src/Timezone_WINCE.cpp | 92 + base/poco/Foundation/src/Token.cpp | 166 + base/poco/Foundation/src/URI.cpp | 918 ++ base/poco/Foundation/src/URIStreamFactory.cpp | 58 + base/poco/Foundation/src/URIStreamOpener.cpp | 211 + base/poco/Foundation/src/UTF16Encoding.cpp | 273 + base/poco/Foundation/src/UTF32Encoding.cpp | 178 + base/poco/Foundation/src/UTF8Encoding.cpp | 246 + base/poco/Foundation/src/UTF8String.cpp | 377 + base/poco/Foundation/src/UUID.cpp | 370 + base/poco/Foundation/src/UUIDGenerator.cpp | 151 + base/poco/Foundation/src/Unicode.cpp | 56 + base/poco/Foundation/src/UnicodeConverter.cpp | 161 + base/poco/Foundation/src/Var.cpp | 636 + base/poco/Foundation/src/VarHolder.cpp | 92 + base/poco/Foundation/src/VarIterator.cpp | 167 + base/poco/Foundation/src/Void.cpp | 42 + .../Foundation/src/Windows1250Encoding.cpp | 196 + .../Foundation/src/Windows1251Encoding.cpp | 237 + .../Foundation/src/Windows1252Encoding.cpp | 151 + .../Foundation/src/WindowsConsoleChannel.cpp | 302 + base/poco/Foundation/src/adler32.c | 188 + base/poco/Foundation/src/bignum-dtoa.cc | 641 + base/poco/Foundation/src/bignum-dtoa.h | 84 + base/poco/Foundation/src/bignum.cc | 766 + base/poco/Foundation/src/bignum.h | 145 + base/poco/Foundation/src/cached-powers.cc | 176 + base/poco/Foundation/src/cached-powers.h | 64 + base/poco/Foundation/src/compress.c | 86 + base/poco/Foundation/src/crc32.c | 444 + base/poco/Foundation/src/crc32.h | 441 + base/poco/Foundation/src/deflate.c | 2163 +++ base/poco/Foundation/src/deflate.h | 349 + base/poco/Foundation/src/diy-fp.cc | 57 + base/poco/Foundation/src/diy-fp.h | 118 + base/poco/Foundation/src/double-conversion.cc | 911 ++ base/poco/Foundation/src/double-conversion.h | 536 + base/poco/Foundation/src/fast-dtoa.cc | 665 + base/poco/Foundation/src/fast-dtoa.h | 88 + base/poco/Foundation/src/fixed-dtoa.cc | 404 + base/poco/Foundation/src/fixed-dtoa.h | 56 + base/poco/Foundation/src/gzguts.h | 219 + base/poco/Foundation/src/ieee.h | 402 + base/poco/Foundation/src/infback.c | 640 + base/poco/Foundation/src/inffast.c | 323 + base/poco/Foundation/src/inffast.h | 11 + base/poco/Foundation/src/inffixed.h | 94 + base/poco/Foundation/src/inflate.c | 1561 ++ base/poco/Foundation/src/inflate.h | 125 + base/poco/Foundation/src/inftrees.c | 304 + base/poco/Foundation/src/inftrees.h | 62 + base/poco/Foundation/src/pcre.h | 677 + base/poco/Foundation/src/pcre_byte_order.c | 315 + base/poco/Foundation/src/pcre_chartables.c | 195 + base/poco/Foundation/src/pcre_compile.c | 9806 +++++++++++++ base/poco/Foundation/src/pcre_config.c | 187 + base/poco/Foundation/src/pcre_config.h | 413 + base/poco/Foundation/src/pcre_dfa_exec.c | 3675 +++++ base/poco/Foundation/src/pcre_exec.c | 7174 ++++++++++ base/poco/Foundation/src/pcre_fullinfo.c | 242 + base/poco/Foundation/src/pcre_get.c | 666 + base/poco/Foundation/src/pcre_globals.c | 83 + base/poco/Foundation/src/pcre_internal.h | 2807 ++++ base/poco/Foundation/src/pcre_jit_compile.c | 11912 ++++++++++++++++ base/poco/Foundation/src/pcre_maketables.c | 155 + base/poco/Foundation/src/pcre_newline.c | 207 + base/poco/Foundation/src/pcre_ord2utf8.c | 92 + base/poco/Foundation/src/pcre_refcount.c | 90 + base/poco/Foundation/src/pcre_string_utils.c | 207 + base/poco/Foundation/src/pcre_study.c | 1683 +++ base/poco/Foundation/src/pcre_tables.c | 723 + base/poco/Foundation/src/pcre_ucd.c | 3641 +++++ base/poco/Foundation/src/pcre_valid_utf8.c | 298 + base/poco/Foundation/src/pcre_version.c | 94 + base/poco/Foundation/src/pcre_xclass.c | 264 + base/poco/Foundation/src/pocomsg.mc | 87 + base/poco/Foundation/src/strtod.cc | 556 + base/poco/Foundation/src/strtod.h | 45 + base/poco/Foundation/src/trees.c | 1203 ++ base/poco/Foundation/src/trees.h | 128 + base/poco/Foundation/src/ucp.h | 224 + base/poco/Foundation/src/utils.h | 325 + base/poco/Foundation/src/zutil.c | 324 + base/poco/Foundation/src/zutil.h | 253 + base/poco/JSON/CMakeLists.txt | 18 + base/poco/JSON/include/Poco/JSON/Array.h | 566 + base/poco/JSON/include/Poco/JSON/Handler.h | 106 + base/poco/JSON/include/Poco/JSON/JSON.h | 62 + .../JSON/include/Poco/JSON/JSONException.h | 35 + base/poco/JSON/include/Poco/JSON/Object.h | 715 + .../JSON/include/Poco/JSON/ParseHandler.h | 169 + base/poco/JSON/include/Poco/JSON/Parser.h | 203 + base/poco/JSON/include/Poco/JSON/ParserImpl.h | 194 + .../JSON/include/Poco/JSON/PrintHandler.h | 139 + base/poco/JSON/include/Poco/JSON/Query.h | 125 + .../poco/JSON/include/Poco/JSON/Stringifier.h | 73 + base/poco/JSON/include/Poco/JSON/Template.h | 151 + .../JSON/include/Poco/JSON/TemplateCache.h | 105 + base/poco/JSON/src/Array.cpp | 268 + base/poco/JSON/src/Handler.cpp | 45 + base/poco/JSON/src/JSONException.cpp | 26 + base/poco/JSON/src/Object.cpp | 298 + base/poco/JSON/src/ParseHandler.cpp | 148 + base/poco/JSON/src/Parser.cpp | 49 + base/poco/JSON/src/ParserImpl.cpp | 241 + base/poco/JSON/src/PrintHandler.cpp | 215 + base/poco/JSON/src/Query.cpp | 172 + base/poco/JSON/src/Stringifier.cpp | 86 + base/poco/JSON/src/Template.cpp | 701 + base/poco/JSON/src/TemplateCache.cpp | 157 + base/poco/JSON/src/pdjson.c | 870 ++ base/poco/JSON/src/pdjson.h | 113 + base/poco/LICENSE | 32 + base/poco/MongoDB/CMakeLists.txt | 7 + .../poco/MongoDB/include/Poco/MongoDB/Array.h | 110 + .../MongoDB/include/Poco/MongoDB/BSONReader.h | 84 + .../MongoDB/include/Poco/MongoDB/BSONWriter.h | 74 + .../MongoDB/include/Poco/MongoDB/Binary.h | 155 + .../MongoDB/include/Poco/MongoDB/Connection.h | 164 + .../MongoDB/include/Poco/MongoDB/Cursor.h | 74 + .../MongoDB/include/Poco/MongoDB/Database.h | 162 + .../include/Poco/MongoDB/DeleteRequest.h | 113 + .../MongoDB/include/Poco/MongoDB/Document.h | 277 + .../MongoDB/include/Poco/MongoDB/Element.h | 403 + .../include/Poco/MongoDB/GetMoreRequest.h | 89 + .../include/Poco/MongoDB/InsertRequest.h | 97 + .../include/Poco/MongoDB/JavaScriptCode.h | 105 + .../include/Poco/MongoDB/KillCursorsRequest.h | 62 + .../MongoDB/include/Poco/MongoDB/Message.h | 73 + .../include/Poco/MongoDB/MessageHeader.h | 130 + .../MongoDB/include/Poco/MongoDB/MongoDB.h | 64 + .../MongoDB/include/Poco/MongoDB/ObjectId.h | 147 + .../Poco/MongoDB/PoolableConnectionFactory.h | 128 + .../include/Poco/MongoDB/QueryRequest.h | 187 + .../include/Poco/MongoDB/RegularExpression.h | 129 + .../MongoDB/include/Poco/MongoDB/ReplicaSet.h | 58 + .../include/Poco/MongoDB/RequestMessage.h | 51 + .../include/Poco/MongoDB/ResponseMessage.h | 108 + .../include/Poco/MongoDB/UpdateRequest.h | 114 + base/poco/MongoDB/src/Array.cpp | 75 + base/poco/MongoDB/src/Binary.cpp | 89 + base/poco/MongoDB/src/Connection.cpp | 235 + base/poco/MongoDB/src/Cursor.cpp | 77 + base/poco/MongoDB/src/Database.cpp | 430 + base/poco/MongoDB/src/DeleteRequest.cpp | 54 + base/poco/MongoDB/src/Document.cpp | 219 + base/poco/MongoDB/src/Element.cpp | 32 + base/poco/MongoDB/src/GetMoreRequest.cpp | 46 + base/poco/MongoDB/src/InsertRequest.cpp | 49 + base/poco/MongoDB/src/JavaScriptCode.cpp | 33 + base/poco/MongoDB/src/KillCursorsRequest.cpp | 44 + base/poco/MongoDB/src/Message.cpp | 33 + base/poco/MongoDB/src/MessageHeader.cpp | 63 + base/poco/MongoDB/src/ObjectId.cpp | 66 + base/poco/MongoDB/src/QueryRequest.cpp | 54 + base/poco/MongoDB/src/RegularExpression.cpp | 71 + base/poco/MongoDB/src/ReplicaSet.cpp | 89 + base/poco/MongoDB/src/RequestMessage.cpp | 51 + base/poco/MongoDB/src/ResponseMessage.cpp | 70 + base/poco/MongoDB/src/UpdateRequest.cpp | 47 + base/poco/Net/CMakeLists.txt | 25 + .../Poco/Net/AbstractHTTPRequestHandler.h | 136 + base/poco/Net/include/Poco/Net/DNS.h | 199 + .../Net/include/Poco/Net/DatagramSocket.h | 168 + .../Net/include/Poco/Net/DatagramSocketImpl.h | 55 + base/poco/Net/include/Poco/Net/DialogSocket.h | 211 + .../Net/include/Poco/Net/FTPClientSession.h | 403 + .../Net/include/Poco/Net/FTPStreamFactory.h | 120 + .../Net/include/Poco/Net/FilePartSource.h | 77 + base/poco/Net/include/Poco/Net/HTMLForm.h | 272 + .../Poco/Net/HTTPAuthenticationParams.h | 103 + .../include/Poco/Net/HTTPBasicCredentials.h | 123 + .../Net/include/Poco/Net/HTTPBasicStreamBuf.h | 36 + .../include/Poco/Net/HTTPBufferAllocator.h | 50 + .../Net/include/Poco/Net/HTTPChunkedStream.h | 108 + .../Net/include/Poco/Net/HTTPClientSession.h | 436 + base/poco/Net/include/Poco/Net/HTTPCookie.h | 272 + .../Net/include/Poco/Net/HTTPCredentials.h | 206 + .../include/Poco/Net/HTTPDigestCredentials.h | 186 + .../include/Poco/Net/HTTPFixedLengthStream.h | 110 + .../Net/include/Poco/Net/HTTPHeaderStream.h | 102 + base/poco/Net/include/Poco/Net/HTTPIOStream.h | 88 + base/poco/Net/include/Poco/Net/HTTPMessage.h | 185 + base/poco/Net/include/Poco/Net/HTTPRequest.h | 208 + .../Net/include/Poco/Net/HTTPRequestHandler.h | 67 + .../Poco/Net/HTTPRequestHandlerFactory.h | 78 + base/poco/Net/include/Poco/Net/HTTPResponse.h | 306 + base/poco/Net/include/Poco/Net/HTTPServer.h | 103 + .../include/Poco/Net/HTTPServerConnection.h | 65 + .../Poco/Net/HTTPServerConnectionFactory.h | 55 + .../Net/include/Poco/Net/HTTPServerParams.h | 154 + .../Net/include/Poco/Net/HTTPServerRequest.h | 80 + .../include/Poco/Net/HTTPServerRequestImpl.h | 142 + .../Net/include/Poco/Net/HTTPServerResponse.h | 127 + .../include/Poco/Net/HTTPServerResponseImpl.h | 151 + .../Net/include/Poco/Net/HTTPServerSession.h | 76 + base/poco/Net/include/Poco/Net/HTTPSession.h | 251 + .../Net/include/Poco/Net/HTTPSessionFactory.h | 155 + .../Poco/Net/HTTPSessionInstantiator.h | 117 + base/poco/Net/include/Poco/Net/HTTPStream.h | 103 + .../Net/include/Poco/Net/HTTPStreamFactory.h | 92 + base/poco/Net/include/Poco/Net/HostEntry.h | 115 + base/poco/Net/include/Poco/Net/ICMPClient.h | 97 + .../poco/Net/include/Poco/Net/ICMPEventArgs.h | 163 + base/poco/Net/include/Poco/Net/ICMPPacket.h | 89 + .../Net/include/Poco/Net/ICMPPacketImpl.h | 141 + base/poco/Net/include/Poco/Net/ICMPSocket.h | 91 + .../Net/include/Poco/Net/ICMPSocketImpl.h | 94 + .../Net/include/Poco/Net/ICMPv4PacketImpl.h | 173 + base/poco/Net/include/Poco/Net/IPAddress.h | 450 + .../poco/Net/include/Poco/Net/IPAddressImpl.h | 179 + base/poco/Net/include/Poco/Net/MailMessage.h | 314 + .../poco/Net/include/Poco/Net/MailRecipient.h | 120 + base/poco/Net/include/Poco/Net/MailStream.h | 146 + base/poco/Net/include/Poco/Net/MediaType.h | 169 + .../poco/Net/include/Poco/Net/MessageHeader.h | 169 + .../Net/include/Poco/Net/MulticastSocket.h | 135 + .../Net/include/Poco/Net/MultipartReader.h | 160 + .../Net/include/Poco/Net/MultipartWriter.h | 109 + base/poco/Net/include/Poco/Net/NTPClient.h | 64 + base/poco/Net/include/Poco/Net/NTPEventArgs.h | 84 + base/poco/Net/include/Poco/Net/NTPPacket.h | 207 + .../include/Poco/Net/NameValueCollection.h | 126 + base/poco/Net/include/Poco/Net/Net.h | 122 + base/poco/Net/include/Poco/Net/NetException.h | 60 + .../Net/include/Poco/Net/NetworkInterface.h | 352 + .../Net/include/Poco/Net/NullPartHandler.h | 47 + .../Net/include/Poco/Net/OAuth10Credentials.h | 274 + .../Net/include/Poco/Net/OAuth20Credentials.h | 132 + .../Net/include/Poco/Net/POP3ClientSession.h | 185 + .../include/Poco/Net/ParallelSocketAcceptor.h | 233 + .../include/Poco/Net/ParallelSocketReactor.h | 90 + base/poco/Net/include/Poco/Net/PartHandler.h | 68 + base/poco/Net/include/Poco/Net/PartSource.h | 110 + base/poco/Net/include/Poco/Net/PartStore.h | 106 + base/poco/Net/include/Poco/Net/PollSet.h | 86 + .../include/Poco/Net/QuotedPrintableDecoder.h | 86 + .../include/Poco/Net/QuotedPrintableEncoder.h | 85 + base/poco/Net/include/Poco/Net/RawSocket.h | 161 + .../poco/Net/include/Poco/Net/RawSocketImpl.h | 56 + .../include/Poco/Net/RemoteSyslogChannel.h | 158 + .../include/Poco/Net/RemoteSyslogListener.h | 121 + base/poco/Net/include/Poco/Net/SMTPChannel.h | 108 + .../Net/include/Poco/Net/SMTPClientSession.h | 234 + base/poco/Net/include/Poco/Net/ServerSocket.h | 230 + .../Net/include/Poco/Net/ServerSocketImpl.h | 45 + base/poco/Net/include/Poco/Net/Socket.h | 631 + .../Net/include/Poco/Net/SocketAcceptor.h | 196 + .../poco/Net/include/Poco/Net/SocketAddress.h | 301 + .../Net/include/Poco/Net/SocketAddressImpl.h | 258 + .../Net/include/Poco/Net/SocketConnector.h | 212 + base/poco/Net/include/Poco/Net/SocketDefs.h | 379 + base/poco/Net/include/Poco/Net/SocketImpl.h | 503 + .../Net/include/Poco/Net/SocketNotification.h | 153 + .../Net/include/Poco/Net/SocketNotifier.h | 110 + .../poco/Net/include/Poco/Net/SocketReactor.h | 235 + base/poco/Net/include/Poco/Net/SocketStream.h | 177 + base/poco/Net/include/Poco/Net/StreamSocket.h | 180 + .../Net/include/Poco/Net/StreamSocketImpl.h | 59 + .../Net/include/Poco/Net/StringPartSource.h | 71 + base/poco/Net/include/Poco/Net/TCPServer.h | 260 + .../include/Poco/Net/TCPServerConnection.h | 86 + .../Poco/Net/TCPServerConnectionFactory.h | 89 + .../include/Poco/Net/TCPServerDispatcher.h | 131 + .../Net/include/Poco/Net/TCPServerParams.h | 137 + base/poco/Net/include/Poco/Net/WebSocket.h | 297 + .../poco/Net/include/Poco/Net/WebSocketImpl.h | 142 + .../Net/src/AbstractHTTPRequestHandler.cpp | 109 + base/poco/Net/src/DNS.cpp | 687 + base/poco/Net/src/DatagramSocket.cpp | 114 + base/poco/Net/src/DatagramSocketImpl.cpp | 63 + base/poco/Net/src/DialogSocket.cpp | 295 + base/poco/Net/src/FTPClientSession.cpp | 599 + base/poco/Net/src/FTPStreamFactory.cpp | 243 + base/poco/Net/src/FilePartSource.cpp | 87 + base/poco/Net/src/HTMLForm.cpp | 468 + .../poco/Net/src/HTTPAuthenticationParams.cpp | 316 + base/poco/Net/src/HTTPBasicCredentials.cpp | 134 + base/poco/Net/src/HTTPBufferAllocator.cpp | 44 + base/poco/Net/src/HTTPChunkedStream.cpp | 258 + base/poco/Net/src/HTTPClientSession.cpp | 514 + base/poco/Net/src/HTTPCookie.cpp | 343 + base/poco/Net/src/HTTPCredentials.cpp | 208 + base/poco/Net/src/HTTPDigestCredentials.cpp | 313 + base/poco/Net/src/HTTPFixedLengthStream.cpp | 185 + base/poco/Net/src/HTTPHeaderStream.cpp | 178 + base/poco/Net/src/HTTPIOStream.cpp | 65 + base/poco/Net/src/HTTPMessage.cpp | 184 + base/poco/Net/src/HTTPRequest.cpp | 277 + base/poco/Net/src/HTTPRequestHandler.cpp | 32 + .../Net/src/HTTPRequestHandlerFactory.cpp | 32 + base/poco/Net/src/HTTPResponse.cpp | 390 + base/poco/Net/src/HTTPServer.cpp | 59 + base/poco/Net/src/HTTPServerConnection.cpp | 180 + .../Net/src/HTTPServerConnectionFactory.cpp | 43 + base/poco/Net/src/HTTPServerParams.cpp | 73 + base/poco/Net/src/HTTPServerRequest.cpp | 32 + base/poco/Net/src/HTTPServerRequestImpl.cpp | 88 + base/poco/Net/src/HTTPServerResponse.cpp | 32 + base/poco/Net/src/HTTPServerResponseImpl.cpp | 221 + base/poco/Net/src/HTTPServerSession.cpp | 70 + base/poco/Net/src/HTTPSession.cpp | 261 + base/poco/Net/src/HTTPSessionFactory.cpp | 146 + base/poco/Net/src/HTTPSessionInstantiator.cpp | 77 + base/poco/Net/src/HTTPStream.cpp | 173 + base/poco/Net/src/HTTPStreamFactory.cpp | 187 + base/poco/Net/src/HostEntry.cpp | 135 + base/poco/Net/src/ICMPClient.cpp | 150 + base/poco/Net/src/ICMPEventArgs.cpp | 168 + base/poco/Net/src/ICMPPacket.cpp | 110 + base/poco/Net/src/ICMPPacketImpl.cpp | 103 + base/poco/Net/src/ICMPSocket.cpp | 94 + base/poco/Net/src/ICMPSocketImpl.cpp | 97 + base/poco/Net/src/ICMPv4PacketImpl.cpp | 263 + base/poco/Net/src/IPAddress.cpp | 601 + base/poco/Net/src/IPAddressImpl.cpp | 813 ++ base/poco/Net/src/MailMessage.cpp | 695 + base/poco/Net/src/MailRecipient.cpp | 94 + base/poco/Net/src/MailStream.cpp | 221 + base/poco/Net/src/MediaType.cpp | 191 + base/poco/Net/src/MessageHeader.cpp | 394 + base/poco/Net/src/MulticastSocket.cpp | 289 + base/poco/Net/src/MultipartReader.cpp | 309 + base/poco/Net/src/MultipartWriter.cpp | 87 + base/poco/Net/src/NTPClient.cpp | 87 + base/poco/Net/src/NTPEventArgs.cpp | 71 + base/poco/Net/src/NTPPacket.cpp | 158 + base/poco/Net/src/NameValueCollection.cpp | 153 + base/poco/Net/src/Net.cpp | 76 + base/poco/Net/src/NetException.cpp | 54 + base/poco/Net/src/NetworkInterface.cpp | 1827 +++ base/poco/Net/src/NullPartHandler.cpp | 46 + base/poco/Net/src/OAuth10Credentials.cpp | 365 + base/poco/Net/src/OAuth20Credentials.cpp | 105 + base/poco/Net/src/POP3ClientSession.cpp | 280 + base/poco/Net/src/PartHandler.cpp | 32 + base/poco/Net/src/PartSource.cpp | 58 + base/poco/Net/src/PartStore.cpp | 87 + base/poco/Net/src/PollSet.cpp | 532 + base/poco/Net/src/QuotedPrintableDecoder.cpp | 102 + base/poco/Net/src/QuotedPrintableEncoder.cpp | 150 + base/poco/Net/src/RawSocket.cpp | 117 + base/poco/Net/src/RawSocketImpl.cpp | 69 + base/poco/Net/src/RemoteSyslogChannel.cpp | 356 + base/poco/Net/src/RemoteSyslogListener.cpp | 630 + base/poco/Net/src/SMTPChannel.cpp | 210 + base/poco/Net/src/SMTPClientSession.cpp | 466 + base/poco/Net/src/ServerSocket.cpp | 158 + base/poco/Net/src/ServerSocketImpl.cpp | 32 + base/poco/Net/src/Socket.cpp | 392 + base/poco/Net/src/SocketAddress.cpp | 435 + base/poco/Net/src/SocketAddressImpl.cpp | 184 + base/poco/Net/src/SocketImpl.cpp | 1090 ++ base/poco/Net/src/SocketNotification.cpp | 105 + base/poco/Net/src/SocketNotifier.cpp | 89 + base/poco/Net/src/SocketReactor.cpp | 297 + base/poco/Net/src/SocketStream.cpp | 156 + base/poco/Net/src/StreamSocket.cpp | 150 + base/poco/Net/src/StreamSocketImpl.cpp | 77 + base/poco/Net/src/StringPartSource.cpp | 67 + base/poco/Net/src/TCPServer.cpp | 243 + base/poco/Net/src/TCPServerConnection.cpp | 60 + .../Net/src/TCPServerConnectionFactory.cpp | 32 + base/poco/Net/src/TCPServerDispatcher.cpp | 240 + base/poco/Net/src/TCPServerParams.cpp | 64 + base/poco/Net/src/WebSocket.cpp | 263 + base/poco/Net/src/WebSocketImpl.cpp | 400 + base/poco/NetSSL_OpenSSL/CMakeLists.txt | 14 + .../Poco/Net/AcceptCertificateHandler.h | 52 + .../Poco/Net/CertificateHandlerFactory.h | 93 + .../Poco/Net/CertificateHandlerFactoryMgr.h | 64 + .../Poco/Net/ConsoleCertificateHandler.h | 50 + .../NetSSL_OpenSSL/include/Poco/Net/Context.h | 415 + .../include/Poco/Net/HTTPSClientSession.h | 165 + .../Poco/Net/HTTPSSessionInstantiator.h | 65 + .../include/Poco/Net/HTTPSStreamFactory.h | 83 + .../Poco/Net/InvalidCertificateHandler.h | 80 + .../include/Poco/Net/KeyConsoleHandler.h | 47 + .../include/Poco/Net/KeyFileHandler.h | 51 + .../NetSSL_OpenSSL/include/Poco/Net/NetSSL.h | 94 + .../include/Poco/Net/PrivateKeyFactory.h | 95 + .../include/Poco/Net/PrivateKeyFactoryMgr.h | 64 + .../Poco/Net/PrivateKeyPassphraseHandler.h | 84 + .../Poco/Net/RejectCertificateHandler.h | 48 + .../include/Poco/Net/SSLException.h | 39 + .../include/Poco/Net/SSLManager.h | 390 + .../Poco/Net/SecureSMTPClientSession.h | 96 + .../include/Poco/Net/SecureServerSocket.h | 136 + .../include/Poco/Net/SecureServerSocketImpl.h | 146 + .../include/Poco/Net/SecureSocketImpl.h | 268 + .../include/Poco/Net/SecureStreamSocket.h | 272 + .../include/Poco/Net/SecureStreamSocketImpl.h | 303 + .../NetSSL_OpenSSL/include/Poco/Net/Session.h | 79 + .../NetSSL_OpenSSL/include/Poco/Net/Utility.h | 52 + .../include/Poco/Net/VerificationErrorArgs.h | 108 + .../include/Poco/Net/X509Certificate.h | 102 + .../src/AcceptCertificateHandler.cpp | 38 + .../src/CertificateHandlerFactory.cpp | 44 + .../src/CertificateHandlerFactoryMgr.cpp | 69 + .../src/ConsoleCertificateHandler.cpp | 53 + base/poco/NetSSL_OpenSSL/src/Context.cpp | 733 + .../NetSSL_OpenSSL/src/HTTPSClientSession.cpp | 210 + .../src/HTTPSSessionInstantiator.cpp | 71 + .../NetSSL_OpenSSL/src/HTTPSStreamFactory.cpp | 194 + .../src/InvalidCertificateHandler.cpp | 52 + .../NetSSL_OpenSSL/src/KeyConsoleHandler.cpp | 40 + .../NetSSL_OpenSSL/src/KeyFileHandler.cpp | 61 + .../NetSSL_OpenSSL/src/PrivateKeyFactory.cpp | 44 + .../src/PrivateKeyFactoryMgr.cpp | 67 + .../src/PrivateKeyPassphraseHandler.cpp | 46 + .../src/RejectCertificateHandler.cpp | 38 + base/poco/NetSSL_OpenSSL/src/SSLException.cpp | 30 + base/poco/NetSSL_OpenSSL/src/SSLManager.cpp | 460 + .../src/SecureSMTPClientSession.cpp | 64 + .../NetSSL_OpenSSL/src/SecureServerSocket.cpp | 121 + .../src/SecureServerSocketImpl.cpp | 122 + .../NetSSL_OpenSSL/src/SecureSocketImpl.cpp | 633 + .../NetSSL_OpenSSL/src/SecureStreamSocket.cpp | 287 + .../src/SecureStreamSocketImpl.cpp | 241 + base/poco/NetSSL_OpenSSL/src/Session.cpp | 40 + base/poco/NetSSL_OpenSSL/src/Utility.cpp | 71 + .../src/VerificationErrorArgs.cpp | 37 + .../NetSSL_OpenSSL/src/X509Certificate.cpp | 175 + base/poco/README.md | 38 + base/poco/Redis/CMakeLists.txt | 8 + base/poco/Redis/include/Poco/Redis/Array.h | 322 + .../Redis/include/Poco/Redis/AsyncReader.h | 98 + base/poco/Redis/include/Poco/Redis/Client.h | 239 + base/poco/Redis/include/Poco/Redis/Command.h | 279 + base/poco/Redis/include/Poco/Redis/Error.h | 91 + .../poco/Redis/include/Poco/Redis/Exception.h | 36 + .../Poco/Redis/PoolableConnectionFactory.h | 114 + base/poco/Redis/include/Poco/Redis/Redis.h | 62 + .../Redis/include/Poco/Redis/RedisEventArgs.h | 99 + .../Redis/include/Poco/Redis/RedisStream.h | 114 + base/poco/Redis/include/Poco/Redis/Type.h | 293 + base/poco/Redis/src/Array.cpp | 67 + base/poco/Redis/src/AsyncReader.cpp | 61 + base/poco/Redis/src/Client.cpp | 213 + base/poco/Redis/src/Command.cpp | 738 + base/poco/Redis/src/Error.cpp | 39 + base/poco/Redis/src/Exception.cpp | 27 + base/poco/Redis/src/RedisEventArgs.cpp | 46 + base/poco/Redis/src/RedisStream.cpp | 133 + base/poco/Redis/src/Type.cpp | 62 + base/poco/Util/CMakeLists.txt | 7 + .../include/Poco/Util/AbstractConfiguration.h | 381 + .../poco/Util/include/Poco/Util/Application.h | 545 + .../include/Poco/Util/ConfigurationMapper.h | 94 + .../include/Poco/Util/ConfigurationView.h | 82 + .../Poco/Util/FilesystemConfiguration.h | 93 + .../Util/include/Poco/Util/HelpFormatter.h | 202 + .../include/Poco/Util/IniFileConfiguration.h | 99 + .../Util/include/Poco/Util/IntValidator.h | 56 + .../include/Poco/Util/JSONConfiguration.h | 150 + .../include/Poco/Util/LayeredConfiguration.h | 184 + .../include/Poco/Util/LoggingConfigurator.h | 139 + .../Util/include/Poco/Util/LoggingSubsystem.h | 52 + .../Util/include/Poco/Util/MapConfiguration.h | 64 + base/poco/Util/include/Poco/Util/Option.h | 332 + .../Util/include/Poco/Util/OptionCallback.h | 109 + .../Util/include/Poco/Util/OptionException.h | 44 + .../Util/include/Poco/Util/OptionProcessor.h | 137 + base/poco/Util/include/Poco/Util/OptionSet.h | 88 + .../Poco/Util/PropertyFileConfiguration.h | 99 + .../Util/include/Poco/Util/RegExpValidator.h | 54 + .../include/Poco/Util/ServerApplication.h | 287 + base/poco/Util/include/Poco/Util/Subsystem.h | 98 + .../include/Poco/Util/SystemConfiguration.h | 99 + base/poco/Util/include/Poco/Util/Timer.h | 169 + base/poco/Util/include/Poco/Util/TimerTask.h | 98 + .../Util/include/Poco/Util/TimerTaskAdapter.h | 69 + base/poco/Util/include/Poco/Util/Units.h | 1298 ++ base/poco/Util/include/Poco/Util/Util.h | 62 + base/poco/Util/include/Poco/Util/Validator.h | 57 + .../Poco/Util/WinRegistryConfiguration.h | 71 + .../Util/include/Poco/Util/WinRegistryKey.h | 198 + base/poco/Util/include/Poco/Util/WinService.h | 141 + .../Util/include/Poco/Util/XMLConfiguration.h | 209 + base/poco/Util/src/AbstractConfiguration.cpp | 536 + base/poco/Util/src/Application.cpp | 568 + base/poco/Util/src/ConfigurationMapper.cpp | 101 + base/poco/Util/src/ConfigurationView.cpp | 75 + .../poco/Util/src/FilesystemConfiguration.cpp | 127 + base/poco/Util/src/HelpFormatter.cpp | 296 + base/poco/Util/src/IniFileConfiguration.cpp | 190 + base/poco/Util/src/IntValidator.cpp | 54 + base/poco/Util/src/JSONConfiguration.cpp | 386 + base/poco/Util/src/LayeredConfiguration.cpp | 233 + base/poco/Util/src/LoggingConfigurator.cpp | 203 + base/poco/Util/src/LoggingSubsystem.cpp | 58 + base/poco/Util/src/MapConfiguration.cpp | 122 + base/poco/Util/src/Option.cpp | 292 + base/poco/Util/src/OptionCallback.cpp | 37 + base/poco/Util/src/OptionException.cpp | 35 + base/poco/Util/src/OptionProcessor.cpp | 159 + base/poco/Util/src/OptionSet.cpp | 122 + .../Util/src/PropertyFileConfiguration.cpp | 190 + base/poco/Util/src/RegExpValidator.cpp | 47 + base/poco/Util/src/ServerApplication.cpp | 749 + base/poco/Util/src/Subsystem.cpp | 44 + base/poco/Util/src/SystemConfiguration.cpp | 212 + base/poco/Util/src/Timer.cpp | 351 + base/poco/Util/src/TimerTask.cpp | 40 + base/poco/Util/src/Validator.cpp | 32 + base/poco/Util/src/XMLConfiguration.cpp | 479 + base/poco/XML/CMakeLists.txt | 19 + .../include/Poco/DOM/AbstractContainerNode.h | 78 + base/poco/XML/include/Poco/DOM/AbstractNode.h | 125 + base/poco/XML/include/Poco/DOM/Attr.h | 180 + base/poco/XML/include/Poco/DOM/AttrMap.h | 65 + base/poco/XML/include/Poco/DOM/AutoPtr.h | 35 + base/poco/XML/include/Poco/DOM/CDATASection.h | 84 + .../poco/XML/include/Poco/DOM/CharacterData.h | 123 + .../XML/include/Poco/DOM/ChildNodesList.h | 55 + base/poco/XML/include/Poco/DOM/Comment.h | 58 + base/poco/XML/include/Poco/DOM/DOMBuilder.h | 112 + base/poco/XML/include/Poco/DOM/DOMException.h | 112 + .../XML/include/Poco/DOM/DOMImplementation.h | 82 + base/poco/XML/include/Poco/DOM/DOMObject.h | 103 + base/poco/XML/include/Poco/DOM/DOMParser.h | 114 + .../poco/XML/include/Poco/DOM/DOMSerializer.h | 122 + base/poco/XML/include/Poco/DOM/DOMWriter.h | 127 + base/poco/XML/include/Poco/DOM/DTDMap.h | 67 + base/poco/XML/include/Poco/DOM/Document.h | 285 + .../poco/XML/include/Poco/DOM/DocumentEvent.h | 65 + .../XML/include/Poco/DOM/DocumentFragment.h | 84 + base/poco/XML/include/Poco/DOM/DocumentType.h | 125 + base/poco/XML/include/Poco/DOM/Element.h | 219 + .../include/Poco/DOM/ElementsByTagNameList.h | 86 + base/poco/XML/include/Poco/DOM/Entity.h | 126 + .../XML/include/Poco/DOM/EntityReference.h | 73 + base/poco/XML/include/Poco/DOM/Event.h | 209 + .../XML/include/Poco/DOM/EventDispatcher.h | 97 + .../XML/include/Poco/DOM/EventException.h | 81 + .../poco/XML/include/Poco/DOM/EventListener.h | 57 + base/poco/XML/include/Poco/DOM/EventTarget.h | 76 + .../poco/XML/include/Poco/DOM/MutationEvent.h | 143 + base/poco/XML/include/Poco/DOM/NamedNodeMap.h | 93 + base/poco/XML/include/Poco/DOM/Node.h | 285 + base/poco/XML/include/Poco/DOM/NodeAppender.h | 81 + base/poco/XML/include/Poco/DOM/NodeFilter.h | 146 + base/poco/XML/include/Poco/DOM/NodeIterator.h | 167 + base/poco/XML/include/Poco/DOM/NodeList.h | 61 + base/poco/XML/include/Poco/DOM/Notation.h | 93 + .../include/Poco/DOM/ProcessingInstruction.h | 99 + base/poco/XML/include/Poco/DOM/Text.h | 79 + base/poco/XML/include/Poco/DOM/TreeWalker.h | 211 + base/poco/XML/include/Poco/SAX/Attributes.h | 120 + .../XML/include/Poco/SAX/AttributesImpl.h | 307 + .../XML/include/Poco/SAX/ContentHandler.h | 240 + base/poco/XML/include/Poco/SAX/DTDHandler.h | 86 + base/poco/XML/include/Poco/SAX/DeclHandler.h | 91 + .../XML/include/Poco/SAX/DefaultHandler.h | 83 + .../XML/include/Poco/SAX/EntityResolver.h | 86 + .../XML/include/Poco/SAX/EntityResolverImpl.h | 78 + base/poco/XML/include/Poco/SAX/ErrorHandler.h | 92 + base/poco/XML/include/Poco/SAX/InputSource.h | 169 + .../XML/include/Poco/SAX/LexicalHandler.h | 125 + base/poco/XML/include/Poco/SAX/Locator.h | 103 + base/poco/XML/include/Poco/SAX/LocatorImpl.h | 88 + .../XML/include/Poco/SAX/NamespaceSupport.h | 195 + base/poco/XML/include/Poco/SAX/SAXException.h | 176 + base/poco/XML/include/Poco/SAX/SAXParser.h | 113 + .../XML/include/Poco/SAX/WhitespaceFilter.h | 81 + base/poco/XML/include/Poco/SAX/XMLFilter.h | 61 + .../poco/XML/include/Poco/SAX/XMLFilterImpl.h | 132 + base/poco/XML/include/Poco/SAX/XMLReader.h | 205 + base/poco/XML/include/Poco/XML/Content.h | 63 + base/poco/XML/include/Poco/XML/Name.h | 144 + base/poco/XML/include/Poco/XML/NamePool.h | 83 + .../XML/include/Poco/XML/NamespaceStrategy.h | 115 + base/poco/XML/include/Poco/XML/ParserEngine.h | 384 + base/poco/XML/include/Poco/XML/QName.h | 148 + base/poco/XML/include/Poco/XML/ValueTraits.h | 104 + base/poco/XML/include/Poco/XML/XML.h | 62 + base/poco/XML/include/Poco/XML/XMLException.h | 37 + base/poco/XML/include/Poco/XML/XMLStream.h | 71 + .../XML/include/Poco/XML/XMLStreamParser.h | 634 + .../Poco/XML/XMLStreamParserException.h | 58 + base/poco/XML/include/Poco/XML/XMLString.h | 87 + base/poco/XML/include/Poco/XML/XMLWriter.h | 375 + base/poco/XML/include/Poco/XML/expat.h | 1064 ++ .../XML/include/Poco/XML/expat_external.h | 165 + base/poco/XML/src/AbstractContainerNode.cpp | 580 + base/poco/XML/src/AbstractNode.cpp | 367 + base/poco/XML/src/Attr.cpp | 133 + base/poco/XML/src/AttrMap.cpp | 123 + base/poco/XML/src/Attributes.cpp | 27 + base/poco/XML/src/AttributesImpl.cpp | 299 + base/poco/XML/src/CDATASection.cpp | 74 + base/poco/XML/src/CharacterData.cpp | 152 + base/poco/XML/src/ChildNodesList.cpp | 70 + base/poco/XML/src/Comment.cpp | 60 + base/poco/XML/src/ContentHandler.cpp | 27 + base/poco/XML/src/DOMBuilder.cpp | 312 + base/poco/XML/src/DOMException.cpp | 107 + base/poco/XML/src/DOMImplementation.cpp | 86 + base/poco/XML/src/DOMObject.cpp | 32 + base/poco/XML/src/DOMParser.cpp | 157 + base/poco/XML/src/DOMSerializer.cpp | 323 + base/poco/XML/src/DOMWriter.cpp | 102 + base/poco/XML/src/DTDHandler.cpp | 27 + base/poco/XML/src/DTDMap.cpp | 117 + base/poco/XML/src/DeclHandler.cpp | 27 + base/poco/XML/src/DefaultHandler.cpp | 123 + base/poco/XML/src/Document.cpp | 326 + base/poco/XML/src/DocumentEvent.cpp | 27 + base/poco/XML/src/DocumentFragment.cpp | 70 + base/poco/XML/src/DocumentType.cpp | 84 + base/poco/XML/src/Element.cpp | 444 + base/poco/XML/src/ElementsByTagNameList.cpp | 151 + base/poco/XML/src/Entity.cpp | 68 + base/poco/XML/src/EntityReference.cpp | 59 + base/poco/XML/src/EntityResolver.cpp | 27 + base/poco/XML/src/EntityResolverImpl.cpp | 78 + base/poco/XML/src/ErrorHandler.cpp | 27 + base/poco/XML/src/Event.cpp | 102 + base/poco/XML/src/EventDispatcher.cpp | 146 + base/poco/XML/src/EventException.cpp | 65 + base/poco/XML/src/EventListener.cpp | 27 + base/poco/XML/src/EventTarget.cpp | 27 + base/poco/XML/src/InputSource.cpp | 80 + base/poco/XML/src/LexicalHandler.cpp | 27 + base/poco/XML/src/Locator.cpp | 27 + base/poco/XML/src/LocatorImpl.cpp | 104 + base/poco/XML/src/MutationEvent.cpp | 76 + base/poco/XML/src/Name.cpp | 188 + base/poco/XML/src/NamePool.cpp | 132 + base/poco/XML/src/NamedNodeMap.cpp | 27 + base/poco/XML/src/NamespaceStrategy.cpp | 195 + base/poco/XML/src/NamespaceSupport.cpp | 187 + base/poco/XML/src/Node.cpp | 27 + base/poco/XML/src/NodeAppender.cpp | 82 + base/poco/XML/src/NodeFilter.cpp | 27 + base/poco/XML/src/NodeIterator.cpp | 176 + base/poco/XML/src/NodeList.cpp | 27 + base/poco/XML/src/Notation.cpp | 63 + base/poco/XML/src/ParserEngine.cpp | 943 ++ base/poco/XML/src/ProcessingInstruction.cpp | 79 + base/poco/XML/src/QName.cpp | 114 + base/poco/XML/src/SAXException.cpp | 138 + base/poco/XML/src/SAXParser.cpp | 245 + base/poco/XML/src/Text.cpp | 80 + base/poco/XML/src/TreeWalker.cpp | 227 + base/poco/XML/src/ValueTraits.cpp | 39 + base/poco/XML/src/WhitespaceFilter.cpp | 212 + base/poco/XML/src/XMLException.cpp | 29 + base/poco/XML/src/XMLFilter.cpp | 27 + base/poco/XML/src/XMLFilterImpl.cpp | 313 + base/poco/XML/src/XMLReader.cpp | 37 + base/poco/XML/src/XMLStreamParser.cpp | 939 ++ .../poco/XML/src/XMLStreamParserException.cpp | 88 + base/poco/XML/src/XMLString.cpp | 64 + base/poco/XML/src/XMLWriter.cpp | 1068 ++ base/poco/XML/src/ascii.h | 123 + base/poco/XML/src/asciitab.h | 66 + base/poco/XML/src/expat_config.h | 39 + base/poco/XML/src/iasciitab.h | 67 + base/poco/XML/src/internal.h | 163 + base/poco/XML/src/latin1tab.h | 66 + base/poco/XML/src/nametab.h | 136 + base/poco/XML/src/siphash.h | 393 + base/poco/XML/src/utf8tab.h | 66 + base/poco/XML/src/xmlparse.cpp | 8402 +++++++++++ base/poco/XML/src/xmlrole.c | 1255 ++ base/poco/XML/src/xmlrole.h | 142 + base/poco/XML/src/xmltok.c | 1676 +++ base/poco/XML/src/xmltok.h | 319 + base/poco/XML/src/xmltok_impl.c | 1817 +++ base/poco/XML/src/xmltok_impl.h | 74 + base/poco/XML/src/xmltok_ns.c | 122 + base/poco/patches/AIX/xlC8.diff | 439 + base/readpassphrase/CMakeLists.txt | 4 + base/widechar_width/CMakeLists.txt | 4 + contrib/CMakeLists.txt | 1 - contrib/poco | 1 - contrib/poco-cmake/Crypto/CMakeLists.txt | 40 - contrib/poco-cmake/Data/CMakeLists.txt | 46 - contrib/poco-cmake/Data/ODBC/CMakeLists.txt | 39 - contrib/poco-cmake/Foundation/CMakeLists.txt | 236 - contrib/poco-cmake/JSON/CMakeLists.txt | 31 - contrib/poco-cmake/MongoDB/CMakeLists.txt | 29 - contrib/poco-cmake/Net/CMakeLists.txt | 128 - contrib/poco-cmake/Net/SSL/CMakeLists.txt | 42 - contrib/poco-cmake/Redis/CMakeLists.txt | 21 - contrib/poco-cmake/Util/CMakeLists.txt | 35 - contrib/poco-cmake/XML/CMakeLists.txt | 100 - docker/test/fasttest/run.sh | 1 - src/Functions/CMakeLists.txt | 4 - src/Functions/examples/CMakeLists.txt | 0 utils/check-style/check-duplicate-includes.sh | 2 +- utils/check-style/check-style | 2 +- utils/check-style/check-typos | 2 +- utils/check-style/check-whitespaces | 4 +- 1408 files changed, 318315 insertions(+), 769 deletions(-) create mode 100644 base/poco/CHANGELOG rename {contrib/poco-cmake => base/poco}/CMakeLists.txt (67%) create mode 100644 base/poco/CONTRIBUTORS create mode 100644 base/poco/Crypto/CMakeLists.txt create mode 100644 base/poco/Crypto/include/Poco/Crypto/Cipher.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/CipherFactory.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/CipherImpl.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/CipherKey.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/CipherKeyImpl.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/Crypto.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/CryptoException.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/CryptoStream.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/CryptoTransform.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/DigestEngine.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/ECDSADigestEngine.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/ECKey.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/ECKeyImpl.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/EVPPKey.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/KeyPair.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/KeyPairImpl.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/OpenSSLInitializer.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/PKCS12Container.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/RSACipherImpl.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/RSADigestEngine.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/RSAKey.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/RSAKeyImpl.h create mode 100644 base/poco/Crypto/include/Poco/Crypto/X509Certificate.h create mode 100644 base/poco/Crypto/src/Cipher.cpp create mode 100644 base/poco/Crypto/src/CipherFactory.cpp create mode 100644 base/poco/Crypto/src/CipherImpl.cpp create mode 100644 base/poco/Crypto/src/CipherKey.cpp create mode 100644 base/poco/Crypto/src/CipherKeyImpl.cpp create mode 100644 base/poco/Crypto/src/CryptoException.cpp create mode 100644 base/poco/Crypto/src/CryptoStream.cpp create mode 100644 base/poco/Crypto/src/CryptoTransform.cpp create mode 100644 base/poco/Crypto/src/DigestEngine.cpp create mode 100644 base/poco/Crypto/src/ECDSADigestEngine.cpp create mode 100644 base/poco/Crypto/src/ECKey.cpp create mode 100644 base/poco/Crypto/src/ECKeyImpl.cpp create mode 100644 base/poco/Crypto/src/EVPPKey.cpp create mode 100644 base/poco/Crypto/src/KeyPair.cpp create mode 100644 base/poco/Crypto/src/KeyPairImpl.cpp create mode 100644 base/poco/Crypto/src/OpenSSLInitializer.cpp create mode 100644 base/poco/Crypto/src/PKCS12Container.cpp create mode 100644 base/poco/Crypto/src/RSACipherImpl.cpp create mode 100644 base/poco/Crypto/src/RSADigestEngine.cpp create mode 100644 base/poco/Crypto/src/RSAKey.cpp create mode 100644 base/poco/Crypto/src/RSAKeyImpl.cpp create mode 100644 base/poco/Crypto/src/X509Certificate.cpp create mode 100644 base/poco/Data/CMakeLists.txt create mode 100644 base/poco/Data/ODBC/CMakeLists.txt create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Binder.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/ConnectionHandle.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Connector.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Diagnostics.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/EnvironmentHandle.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Error.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Extractor.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Handle.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBC.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBCException.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBCMetaColumn.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBCStatementImpl.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Parameter.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Preparator.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/SessionImpl.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/TypeInfo.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Unicode.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Unicode_UNIXODBC.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Unicode_WIN32.h create mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Utility.h create mode 100644 base/poco/Data/ODBC/src/Binder.cpp create mode 100644 base/poco/Data/ODBC/src/ConnectionHandle.cpp create mode 100644 base/poco/Data/ODBC/src/Connector.cpp create mode 100644 base/poco/Data/ODBC/src/EnvironmentHandle.cpp create mode 100644 base/poco/Data/ODBC/src/Extractor.cpp create mode 100644 base/poco/Data/ODBC/src/ODBCException.cpp create mode 100644 base/poco/Data/ODBC/src/ODBCMetaColumn.cpp create mode 100644 base/poco/Data/ODBC/src/ODBCStatementImpl.cpp create mode 100644 base/poco/Data/ODBC/src/Parameter.cpp create mode 100644 base/poco/Data/ODBC/src/Preparator.cpp create mode 100644 base/poco/Data/ODBC/src/SessionImpl.cpp create mode 100644 base/poco/Data/ODBC/src/TypeInfo.cpp create mode 100644 base/poco/Data/ODBC/src/Unicode.cpp create mode 100644 base/poco/Data/ODBC/src/Unicode_UNIXODBC.cpp create mode 100644 base/poco/Data/ODBC/src/Unicode_WIN32.cpp create mode 100644 base/poco/Data/ODBC/src/Utility.cpp create mode 100644 base/poco/Data/include/Poco/Data/AbstractBinder.h create mode 100644 base/poco/Data/include/Poco/Data/AbstractBinding.h create mode 100644 base/poco/Data/include/Poco/Data/AbstractExtraction.h create mode 100644 base/poco/Data/include/Poco/Data/AbstractExtractor.h create mode 100644 base/poco/Data/include/Poco/Data/AbstractPreparation.h create mode 100644 base/poco/Data/include/Poco/Data/AbstractPreparator.h create mode 100644 base/poco/Data/include/Poco/Data/AbstractSessionImpl.h create mode 100644 base/poco/Data/include/Poco/Data/ArchiveStrategy.h create mode 100644 base/poco/Data/include/Poco/Data/AutoTransaction.h create mode 100644 base/poco/Data/include/Poco/Data/Binding.h create mode 100644 base/poco/Data/include/Poco/Data/Bulk.h create mode 100644 base/poco/Data/include/Poco/Data/BulkBinding.h create mode 100644 base/poco/Data/include/Poco/Data/BulkExtraction.h create mode 100644 base/poco/Data/include/Poco/Data/Column.h create mode 100644 base/poco/Data/include/Poco/Data/Connector.h create mode 100644 base/poco/Data/include/Poco/Data/Constants.h create mode 100644 base/poco/Data/include/Poco/Data/Data.h create mode 100644 base/poco/Data/include/Poco/Data/DataException.h create mode 100644 base/poco/Data/include/Poco/Data/Date.h create mode 100644 base/poco/Data/include/Poco/Data/DynamicDateTime.h create mode 100644 base/poco/Data/include/Poco/Data/DynamicLOB.h create mode 100644 base/poco/Data/include/Poco/Data/Extraction.h create mode 100644 base/poco/Data/include/Poco/Data/LOB.h create mode 100644 base/poco/Data/include/Poco/Data/LOBStream.h create mode 100644 base/poco/Data/include/Poco/Data/Limit.h create mode 100644 base/poco/Data/include/Poco/Data/MetaColumn.h create mode 100644 base/poco/Data/include/Poco/Data/PooledSessionHolder.h create mode 100644 base/poco/Data/include/Poco/Data/PooledSessionImpl.h create mode 100644 base/poco/Data/include/Poco/Data/Position.h create mode 100644 base/poco/Data/include/Poco/Data/Preparation.h create mode 100644 base/poco/Data/include/Poco/Data/Range.h create mode 100644 base/poco/Data/include/Poco/Data/RecordSet.h create mode 100644 base/poco/Data/include/Poco/Data/Row.h create mode 100644 base/poco/Data/include/Poco/Data/RowFilter.h create mode 100644 base/poco/Data/include/Poco/Data/RowFormatter.h create mode 100644 base/poco/Data/include/Poco/Data/RowIterator.h create mode 100644 base/poco/Data/include/Poco/Data/SQLChannel.h create mode 100644 base/poco/Data/include/Poco/Data/Session.h create mode 100644 base/poco/Data/include/Poco/Data/SessionFactory.h create mode 100644 base/poco/Data/include/Poco/Data/SessionImpl.h create mode 100644 base/poco/Data/include/Poco/Data/SessionPool.h create mode 100644 base/poco/Data/include/Poco/Data/SessionPoolContainer.h create mode 100644 base/poco/Data/include/Poco/Data/SimpleRowFormatter.h create mode 100644 base/poco/Data/include/Poco/Data/Statement.h create mode 100644 base/poco/Data/include/Poco/Data/StatementCreator.h create mode 100644 base/poco/Data/include/Poco/Data/StatementImpl.h create mode 100644 base/poco/Data/include/Poco/Data/Time.h create mode 100644 base/poco/Data/include/Poco/Data/Transaction.h create mode 100644 base/poco/Data/include/Poco/Data/TypeHandler.h create mode 100644 base/poco/Data/src/AbstractBinder.cpp create mode 100644 base/poco/Data/src/AbstractBinding.cpp create mode 100644 base/poco/Data/src/AbstractExtraction.cpp create mode 100644 base/poco/Data/src/AbstractExtractor.cpp create mode 100644 base/poco/Data/src/AbstractPreparation.cpp create mode 100644 base/poco/Data/src/AbstractPreparator.cpp create mode 100644 base/poco/Data/src/ArchiveStrategy.cpp create mode 100644 base/poco/Data/src/Bulk.cpp create mode 100644 base/poco/Data/src/Connector.cpp create mode 100644 base/poco/Data/src/DataException.cpp create mode 100644 base/poco/Data/src/Date.cpp create mode 100644 base/poco/Data/src/DynamicLOB.cpp create mode 100644 base/poco/Data/src/Limit.cpp create mode 100644 base/poco/Data/src/MetaColumn.cpp create mode 100644 base/poco/Data/src/PooledSessionHolder.cpp create mode 100644 base/poco/Data/src/PooledSessionImpl.cpp create mode 100644 base/poco/Data/src/Position.cpp create mode 100644 base/poco/Data/src/Range.cpp create mode 100644 base/poco/Data/src/RecordSet.cpp create mode 100644 base/poco/Data/src/Row.cpp create mode 100644 base/poco/Data/src/RowFilter.cpp create mode 100644 base/poco/Data/src/RowFormatter.cpp create mode 100644 base/poco/Data/src/RowIterator.cpp create mode 100644 base/poco/Data/src/SQLChannel.cpp create mode 100644 base/poco/Data/src/Session.cpp create mode 100644 base/poco/Data/src/SessionFactory.cpp create mode 100644 base/poco/Data/src/SessionImpl.cpp create mode 100644 base/poco/Data/src/SessionPool.cpp create mode 100644 base/poco/Data/src/SessionPoolContainer.cpp create mode 100644 base/poco/Data/src/SimpleRowFormatter.cpp create mode 100644 base/poco/Data/src/Statement.cpp create mode 100644 base/poco/Data/src/StatementCreator.cpp create mode 100644 base/poco/Data/src/StatementImpl.cpp create mode 100644 base/poco/Data/src/Time.cpp create mode 100644 base/poco/Data/src/Transaction.cpp create mode 100644 base/poco/Foundation/CMakeLists.txt create mode 100644 base/poco/Foundation/include/Poco/ASCIIEncoding.h create mode 100644 base/poco/Foundation/include/Poco/AbstractCache.h create mode 100644 base/poco/Foundation/include/Poco/AbstractDelegate.h create mode 100644 base/poco/Foundation/include/Poco/AbstractEvent.h create mode 100644 base/poco/Foundation/include/Poco/AbstractObserver.h create mode 100644 base/poco/Foundation/include/Poco/AbstractPriorityDelegate.h create mode 100644 base/poco/Foundation/include/Poco/AbstractStrategy.h create mode 100644 base/poco/Foundation/include/Poco/AccessExpirationDecorator.h create mode 100644 base/poco/Foundation/include/Poco/AccessExpireCache.h create mode 100644 base/poco/Foundation/include/Poco/AccessExpireLRUCache.h create mode 100644 base/poco/Foundation/include/Poco/AccessExpireStrategy.h create mode 100644 base/poco/Foundation/include/Poco/ActiveDispatcher.h create mode 100644 base/poco/Foundation/include/Poco/ActiveMethod.h create mode 100644 base/poco/Foundation/include/Poco/ActiveResult.h create mode 100644 base/poco/Foundation/include/Poco/ActiveRunnable.h create mode 100644 base/poco/Foundation/include/Poco/ActiveStarter.h create mode 100644 base/poco/Foundation/include/Poco/Activity.h create mode 100644 base/poco/Foundation/include/Poco/Alignment.h create mode 100644 base/poco/Foundation/include/Poco/Any.h create mode 100644 base/poco/Foundation/include/Poco/ArchiveStrategy.h create mode 100644 base/poco/Foundation/include/Poco/Array.h create mode 100644 base/poco/Foundation/include/Poco/Ascii.h create mode 100644 base/poco/Foundation/include/Poco/AsyncChannel.h create mode 100644 base/poco/Foundation/include/Poco/AtomicCounter.h create mode 100644 base/poco/Foundation/include/Poco/AutoPtr.h create mode 100644 base/poco/Foundation/include/Poco/AutoReleasePool.h create mode 100644 base/poco/Foundation/include/Poco/Base32Decoder.h create mode 100644 base/poco/Foundation/include/Poco/Base32Encoder.h create mode 100644 base/poco/Foundation/include/Poco/Base64Decoder.h create mode 100644 base/poco/Foundation/include/Poco/Base64Encoder.h create mode 100644 base/poco/Foundation/include/Poco/BasicEvent.h create mode 100644 base/poco/Foundation/include/Poco/BinaryReader.h create mode 100644 base/poco/Foundation/include/Poco/BinaryWriter.h create mode 100644 base/poco/Foundation/include/Poco/Buffer.h create mode 100644 base/poco/Foundation/include/Poco/BufferAllocator.h create mode 100644 base/poco/Foundation/include/Poco/BufferedBidirectionalStreamBuf.h create mode 100644 base/poco/Foundation/include/Poco/BufferedStreamBuf.h create mode 100644 base/poco/Foundation/include/Poco/Bugcheck.h create mode 100644 base/poco/Foundation/include/Poco/ByteOrder.h create mode 100644 base/poco/Foundation/include/Poco/Channel.h create mode 100644 base/poco/Foundation/include/Poco/Checksum.h create mode 100644 base/poco/Foundation/include/Poco/ClassLibrary.h create mode 100644 base/poco/Foundation/include/Poco/ClassLoader.h create mode 100644 base/poco/Foundation/include/Poco/Clock.h create mode 100644 base/poco/Foundation/include/Poco/CompressedLogFile.h create mode 100644 base/poco/Foundation/include/Poco/Condition.h create mode 100644 base/poco/Foundation/include/Poco/Config.h create mode 100644 base/poco/Foundation/include/Poco/Configurable.h create mode 100644 base/poco/Foundation/include/Poco/ConsoleChannel.h create mode 100644 base/poco/Foundation/include/Poco/CountingStream.h create mode 100644 base/poco/Foundation/include/Poco/DateTime.h create mode 100644 base/poco/Foundation/include/Poco/DateTimeFormat.h create mode 100644 base/poco/Foundation/include/Poco/DateTimeFormatter.h create mode 100644 base/poco/Foundation/include/Poco/DateTimeParser.h create mode 100644 base/poco/Foundation/include/Poco/Debugger.h create mode 100644 base/poco/Foundation/include/Poco/DefaultStrategy.h create mode 100644 base/poco/Foundation/include/Poco/DeflatingStream.h create mode 100644 base/poco/Foundation/include/Poco/Delegate.h create mode 100644 base/poco/Foundation/include/Poco/DigestEngine.h create mode 100644 base/poco/Foundation/include/Poco/DigestStream.h create mode 100644 base/poco/Foundation/include/Poco/DirectoryIterator.h create mode 100644 base/poco/Foundation/include/Poco/DirectoryIteratorStrategy.h create mode 100644 base/poco/Foundation/include/Poco/DirectoryIterator_UNIX.h create mode 100644 base/poco/Foundation/include/Poco/DirectoryIterator_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/DirectoryIterator_WIN32U.h create mode 100644 base/poco/Foundation/include/Poco/DirectoryWatcher.h create mode 100644 base/poco/Foundation/include/Poco/Dynamic/Pair.h create mode 100644 base/poco/Foundation/include/Poco/Dynamic/Struct.h create mode 100644 base/poco/Foundation/include/Poco/Dynamic/Var.h create mode 100644 base/poco/Foundation/include/Poco/Dynamic/VarHolder.h create mode 100644 base/poco/Foundation/include/Poco/Dynamic/VarIterator.h create mode 100644 base/poco/Foundation/include/Poco/DynamicAny.h create mode 100644 base/poco/Foundation/include/Poco/DynamicAnyHolder.h create mode 100644 base/poco/Foundation/include/Poco/DynamicFactory.h create mode 100644 base/poco/Foundation/include/Poco/DynamicStruct.h create mode 100644 base/poco/Foundation/include/Poco/Environment.h create mode 100644 base/poco/Foundation/include/Poco/Environment_UNIX.h create mode 100644 base/poco/Foundation/include/Poco/Environment_VX.h create mode 100644 base/poco/Foundation/include/Poco/Environment_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/Environment_WIN32U.h create mode 100644 base/poco/Foundation/include/Poco/Environment_WINCE.h create mode 100644 base/poco/Foundation/include/Poco/Error.h create mode 100644 base/poco/Foundation/include/Poco/ErrorHandler.h create mode 100644 base/poco/Foundation/include/Poco/Event.h create mode 100644 base/poco/Foundation/include/Poco/EventArgs.h create mode 100644 base/poco/Foundation/include/Poco/EventChannel.h create mode 100644 base/poco/Foundation/include/Poco/EventLogChannel.h create mode 100644 base/poco/Foundation/include/Poco/Event_POSIX.h create mode 100644 base/poco/Foundation/include/Poco/Event_VX.h create mode 100644 base/poco/Foundation/include/Poco/Event_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/Exception.h create mode 100644 base/poco/Foundation/include/Poco/ExpirationDecorator.h create mode 100644 base/poco/Foundation/include/Poco/Expire.h create mode 100644 base/poco/Foundation/include/Poco/ExpireCache.h create mode 100644 base/poco/Foundation/include/Poco/ExpireLRUCache.h create mode 100644 base/poco/Foundation/include/Poco/ExpireStrategy.h create mode 100644 base/poco/Foundation/include/Poco/FIFOBuffer.h create mode 100644 base/poco/Foundation/include/Poco/FIFOBufferStream.h create mode 100644 base/poco/Foundation/include/Poco/FIFOEvent.h create mode 100644 base/poco/Foundation/include/Poco/FIFOStrategy.h create mode 100644 base/poco/Foundation/include/Poco/FPEnvironment.h create mode 100644 base/poco/Foundation/include/Poco/FPEnvironment_C99.h create mode 100644 base/poco/Foundation/include/Poco/FPEnvironment_DEC.h create mode 100644 base/poco/Foundation/include/Poco/FPEnvironment_DUMMY.h create mode 100644 base/poco/Foundation/include/Poco/FPEnvironment_QNX.h create mode 100644 base/poco/Foundation/include/Poco/FPEnvironment_SUN.h create mode 100644 base/poco/Foundation/include/Poco/FPEnvironment_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/File.h create mode 100644 base/poco/Foundation/include/Poco/FileChannel.h create mode 100644 base/poco/Foundation/include/Poco/FileStream.h create mode 100644 base/poco/Foundation/include/Poco/FileStreamFactory.h create mode 100644 base/poco/Foundation/include/Poco/FileStream_POSIX.h create mode 100644 base/poco/Foundation/include/Poco/FileStream_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/File_UNIX.h create mode 100644 base/poco/Foundation/include/Poco/File_VX.h create mode 100644 base/poco/Foundation/include/Poco/File_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/File_WIN32U.h create mode 100644 base/poco/Foundation/include/Poco/File_WINCE.h create mode 100644 base/poco/Foundation/include/Poco/Format.h create mode 100644 base/poco/Foundation/include/Poco/Formatter.h create mode 100644 base/poco/Foundation/include/Poco/FormattingChannel.h create mode 100644 base/poco/Foundation/include/Poco/Foundation.h create mode 100644 base/poco/Foundation/include/Poco/FunctionDelegate.h create mode 100644 base/poco/Foundation/include/Poco/FunctionPriorityDelegate.h create mode 100644 base/poco/Foundation/include/Poco/Glob.h create mode 100644 base/poco/Foundation/include/Poco/HMACEngine.h create mode 100644 base/poco/Foundation/include/Poco/Hash.h create mode 100644 base/poco/Foundation/include/Poco/HashFunction.h create mode 100644 base/poco/Foundation/include/Poco/HashMap.h create mode 100644 base/poco/Foundation/include/Poco/HashSet.h create mode 100644 base/poco/Foundation/include/Poco/HashStatistic.h create mode 100644 base/poco/Foundation/include/Poco/HashTable.h create mode 100644 base/poco/Foundation/include/Poco/HexBinaryDecoder.h create mode 100644 base/poco/Foundation/include/Poco/HexBinaryEncoder.h create mode 100644 base/poco/Foundation/include/Poco/InflatingStream.h create mode 100644 base/poco/Foundation/include/Poco/Instantiator.h create mode 100644 base/poco/Foundation/include/Poco/JSONString.h create mode 100644 base/poco/Foundation/include/Poco/KeyValueArgs.h create mode 100644 base/poco/Foundation/include/Poco/LRUCache.h create mode 100644 base/poco/Foundation/include/Poco/LRUStrategy.h create mode 100644 base/poco/Foundation/include/Poco/Latin1Encoding.h create mode 100644 base/poco/Foundation/include/Poco/Latin2Encoding.h create mode 100644 base/poco/Foundation/include/Poco/Latin9Encoding.h create mode 100644 base/poco/Foundation/include/Poco/LineEndingConverter.h create mode 100644 base/poco/Foundation/include/Poco/LinearHashTable.h create mode 100644 base/poco/Foundation/include/Poco/ListMap.h create mode 100644 base/poco/Foundation/include/Poco/LocalDateTime.h create mode 100644 base/poco/Foundation/include/Poco/LogFile.h create mode 100644 base/poco/Foundation/include/Poco/LogFile_STD.h create mode 100644 base/poco/Foundation/include/Poco/LogFile_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/LogFile_WIN32U.h create mode 100644 base/poco/Foundation/include/Poco/LogStream.h create mode 100644 base/poco/Foundation/include/Poco/Logger.h create mode 100644 base/poco/Foundation/include/Poco/LoggingFactory.h create mode 100644 base/poco/Foundation/include/Poco/LoggingRegistry.h create mode 100644 base/poco/Foundation/include/Poco/MD4Engine.h create mode 100644 base/poco/Foundation/include/Poco/MD5Engine.h create mode 100644 base/poco/Foundation/include/Poco/Manifest.h create mode 100644 base/poco/Foundation/include/Poco/MemoryPool.h create mode 100644 base/poco/Foundation/include/Poco/MemoryStream.h create mode 100644 base/poco/Foundation/include/Poco/Message.h create mode 100644 base/poco/Foundation/include/Poco/MetaObject.h create mode 100644 base/poco/Foundation/include/Poco/MetaProgramming.h create mode 100644 base/poco/Foundation/include/Poco/Mutex.h create mode 100644 base/poco/Foundation/include/Poco/Mutex_POSIX.h create mode 100644 base/poco/Foundation/include/Poco/Mutex_VX.h create mode 100644 base/poco/Foundation/include/Poco/Mutex_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/Mutex_WINCE.h create mode 100644 base/poco/Foundation/include/Poco/NObserver.h create mode 100644 base/poco/Foundation/include/Poco/NamedEvent.h create mode 100644 base/poco/Foundation/include/Poco/NamedEvent_Android.h create mode 100644 base/poco/Foundation/include/Poco/NamedEvent_UNIX.h create mode 100644 base/poco/Foundation/include/Poco/NamedEvent_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/NamedEvent_WIN32U.h create mode 100644 base/poco/Foundation/include/Poco/NamedMutex.h create mode 100644 base/poco/Foundation/include/Poco/NamedMutex_Android.h create mode 100644 base/poco/Foundation/include/Poco/NamedMutex_UNIX.h create mode 100644 base/poco/Foundation/include/Poco/NamedMutex_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/NamedMutex_WIN32U.h create mode 100644 base/poco/Foundation/include/Poco/NamedTuple.h create mode 100644 base/poco/Foundation/include/Poco/NestedDiagnosticContext.h create mode 100644 base/poco/Foundation/include/Poco/Notification.h create mode 100644 base/poco/Foundation/include/Poco/NotificationCenter.h create mode 100644 base/poco/Foundation/include/Poco/NotificationQueue.h create mode 100644 base/poco/Foundation/include/Poco/NotificationStrategy.h create mode 100644 base/poco/Foundation/include/Poco/NullChannel.h create mode 100644 base/poco/Foundation/include/Poco/NullStream.h create mode 100644 base/poco/Foundation/include/Poco/Nullable.h create mode 100644 base/poco/Foundation/include/Poco/NumberFormatter.h create mode 100644 base/poco/Foundation/include/Poco/NumberParser.h create mode 100644 base/poco/Foundation/include/Poco/NumericString.h create mode 100644 base/poco/Foundation/include/Poco/ObjectPool.h create mode 100644 base/poco/Foundation/include/Poco/Observer.h create mode 100644 base/poco/Foundation/include/Poco/Optional.h create mode 100644 base/poco/Foundation/include/Poco/PBKDF2Engine.h create mode 100644 base/poco/Foundation/include/Poco/Path.h create mode 100644 base/poco/Foundation/include/Poco/Path_UNIX.h create mode 100644 base/poco/Foundation/include/Poco/Path_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/Path_WIN32U.h create mode 100644 base/poco/Foundation/include/Poco/Path_WINCE.h create mode 100644 base/poco/Foundation/include/Poco/PatternFormatter.h create mode 100644 base/poco/Foundation/include/Poco/Pipe.h create mode 100644 base/poco/Foundation/include/Poco/PipeImpl.h create mode 100644 base/poco/Foundation/include/Poco/PipeImpl_DUMMY.h create mode 100644 base/poco/Foundation/include/Poco/PipeImpl_POSIX.h create mode 100644 base/poco/Foundation/include/Poco/PipeImpl_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/PipeStream.h create mode 100644 base/poco/Foundation/include/Poco/Platform.h create mode 100644 base/poco/Foundation/include/Poco/Platform_POSIX.h create mode 100644 base/poco/Foundation/include/Poco/Platform_VX.h create mode 100644 base/poco/Foundation/include/Poco/Platform_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/Poco.h create mode 100644 base/poco/Foundation/include/Poco/PriorityDelegate.h create mode 100644 base/poco/Foundation/include/Poco/PriorityEvent.h create mode 100644 base/poco/Foundation/include/Poco/PriorityExpire.h create mode 100644 base/poco/Foundation/include/Poco/PriorityNotificationQueue.h create mode 100644 base/poco/Foundation/include/Poco/PriorityStrategy.h create mode 100644 base/poco/Foundation/include/Poco/Process.h create mode 100644 base/poco/Foundation/include/Poco/Process_UNIX.h create mode 100644 base/poco/Foundation/include/Poco/Process_VX.h create mode 100644 base/poco/Foundation/include/Poco/Process_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/Process_WIN32U.h create mode 100644 base/poco/Foundation/include/Poco/Process_WINCE.h create mode 100644 base/poco/Foundation/include/Poco/PurgeStrategy.h create mode 100644 base/poco/Foundation/include/Poco/RWLock.h create mode 100644 base/poco/Foundation/include/Poco/RWLock_Android.h create mode 100644 base/poco/Foundation/include/Poco/RWLock_POSIX.h create mode 100644 base/poco/Foundation/include/Poco/RWLock_VX.h create mode 100644 base/poco/Foundation/include/Poco/RWLock_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/RWLock_WINCE.h create mode 100644 base/poco/Foundation/include/Poco/Random.h create mode 100644 base/poco/Foundation/include/Poco/RandomStream.h create mode 100644 base/poco/Foundation/include/Poco/RecursiveDirectoryIterator.h create mode 100644 base/poco/Foundation/include/Poco/RecursiveDirectoryIteratorImpl.h create mode 100644 base/poco/Foundation/include/Poco/RefCountedObject.h create mode 100644 base/poco/Foundation/include/Poco/RegularExpression.h create mode 100644 base/poco/Foundation/include/Poco/RotateStrategy.h create mode 100644 base/poco/Foundation/include/Poco/Runnable.h create mode 100644 base/poco/Foundation/include/Poco/RunnableAdapter.h create mode 100644 base/poco/Foundation/include/Poco/SHA1Engine.h create mode 100644 base/poco/Foundation/include/Poco/ScopedLock.h create mode 100644 base/poco/Foundation/include/Poco/ScopedUnlock.h create mode 100644 base/poco/Foundation/include/Poco/Semaphore.h create mode 100644 base/poco/Foundation/include/Poco/Semaphore_POSIX.h create mode 100644 base/poco/Foundation/include/Poco/Semaphore_VX.h create mode 100644 base/poco/Foundation/include/Poco/Semaphore_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/SharedLibrary.h create mode 100644 base/poco/Foundation/include/Poco/SharedLibrary_HPUX.h create mode 100644 base/poco/Foundation/include/Poco/SharedLibrary_UNIX.h create mode 100644 base/poco/Foundation/include/Poco/SharedLibrary_VX.h create mode 100644 base/poco/Foundation/include/Poco/SharedLibrary_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/SharedLibrary_WIN32U.h create mode 100644 base/poco/Foundation/include/Poco/SharedMemory.h create mode 100644 base/poco/Foundation/include/Poco/SharedMemory_DUMMY.h create mode 100644 base/poco/Foundation/include/Poco/SharedMemory_POSIX.h create mode 100644 base/poco/Foundation/include/Poco/SharedMemory_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/SharedPtr.h create mode 100644 base/poco/Foundation/include/Poco/SignalHandler.h create mode 100644 base/poco/Foundation/include/Poco/SimpleFileChannel.h create mode 100644 base/poco/Foundation/include/Poco/SimpleHashTable.h create mode 100644 base/poco/Foundation/include/Poco/SingletonHolder.h create mode 100644 base/poco/Foundation/include/Poco/SortedDirectoryIterator.h create mode 100644 base/poco/Foundation/include/Poco/SplitterChannel.h create mode 100644 base/poco/Foundation/include/Poco/Stopwatch.h create mode 100644 base/poco/Foundation/include/Poco/StrategyCollection.h create mode 100644 base/poco/Foundation/include/Poco/StreamChannel.h create mode 100644 base/poco/Foundation/include/Poco/StreamConverter.h create mode 100644 base/poco/Foundation/include/Poco/StreamCopier.h create mode 100644 base/poco/Foundation/include/Poco/StreamTokenizer.h create mode 100644 base/poco/Foundation/include/Poco/StreamUtil.h create mode 100644 base/poco/Foundation/include/Poco/String.h create mode 100644 base/poco/Foundation/include/Poco/StringTokenizer.h create mode 100644 base/poco/Foundation/include/Poco/SynchronizedObject.h create mode 100644 base/poco/Foundation/include/Poco/SyslogChannel.h create mode 100644 base/poco/Foundation/include/Poco/Task.h create mode 100644 base/poco/Foundation/include/Poco/TaskManager.h create mode 100644 base/poco/Foundation/include/Poco/TaskNotification.h create mode 100644 base/poco/Foundation/include/Poco/TeeStream.h create mode 100644 base/poco/Foundation/include/Poco/TemporaryFile.h create mode 100644 base/poco/Foundation/include/Poco/TextBufferIterator.h create mode 100644 base/poco/Foundation/include/Poco/TextConverter.h create mode 100644 base/poco/Foundation/include/Poco/TextEncoding.h create mode 100644 base/poco/Foundation/include/Poco/TextIterator.h create mode 100644 base/poco/Foundation/include/Poco/Thread.h create mode 100644 base/poco/Foundation/include/Poco/ThreadLocal.h create mode 100644 base/poco/Foundation/include/Poco/ThreadPool.h create mode 100644 base/poco/Foundation/include/Poco/ThreadTarget.h create mode 100644 base/poco/Foundation/include/Poco/Thread_POSIX.h create mode 100644 base/poco/Foundation/include/Poco/Thread_VX.h create mode 100644 base/poco/Foundation/include/Poco/Thread_WIN32.h create mode 100644 base/poco/Foundation/include/Poco/Thread_WINCE.h create mode 100644 base/poco/Foundation/include/Poco/TimedNotificationQueue.h create mode 100644 base/poco/Foundation/include/Poco/Timer.h create mode 100644 base/poco/Foundation/include/Poco/Timespan.h create mode 100644 base/poco/Foundation/include/Poco/Timestamp.h create mode 100644 base/poco/Foundation/include/Poco/Timezone.h create mode 100644 base/poco/Foundation/include/Poco/Token.h create mode 100644 base/poco/Foundation/include/Poco/Tuple.h create mode 100644 base/poco/Foundation/include/Poco/TypeList.h create mode 100644 base/poco/Foundation/include/Poco/Types.h create mode 100644 base/poco/Foundation/include/Poco/URI.h create mode 100644 base/poco/Foundation/include/Poco/URIStreamFactory.h create mode 100644 base/poco/Foundation/include/Poco/URIStreamOpener.h create mode 100644 base/poco/Foundation/include/Poco/UTF16Encoding.h create mode 100644 base/poco/Foundation/include/Poco/UTF32Encoding.h create mode 100644 base/poco/Foundation/include/Poco/UTF8Encoding.h create mode 100644 base/poco/Foundation/include/Poco/UTF8String.h create mode 100644 base/poco/Foundation/include/Poco/UTFString.h create mode 100644 base/poco/Foundation/include/Poco/UUID.h create mode 100644 base/poco/Foundation/include/Poco/UUIDGenerator.h create mode 100644 base/poco/Foundation/include/Poco/UnWindows.h create mode 100644 base/poco/Foundation/include/Poco/UnbufferedStreamBuf.h create mode 100644 base/poco/Foundation/include/Poco/Unicode.h create mode 100644 base/poco/Foundation/include/Poco/UnicodeConverter.h create mode 100644 base/poco/Foundation/include/Poco/UniqueAccessExpireCache.h create mode 100644 base/poco/Foundation/include/Poco/UniqueAccessExpireLRUCache.h create mode 100644 base/poco/Foundation/include/Poco/UniqueAccessExpireStrategy.h create mode 100644 base/poco/Foundation/include/Poco/UniqueExpireCache.h create mode 100644 base/poco/Foundation/include/Poco/UniqueExpireLRUCache.h create mode 100644 base/poco/Foundation/include/Poco/UniqueExpireStrategy.h create mode 100644 base/poco/Foundation/include/Poco/ValidArgs.h create mode 100644 base/poco/Foundation/include/Poco/Version.h create mode 100644 base/poco/Foundation/include/Poco/Void.h create mode 100644 base/poco/Foundation/include/Poco/Windows1250Encoding.h create mode 100644 base/poco/Foundation/include/Poco/Windows1251Encoding.h create mode 100644 base/poco/Foundation/include/Poco/Windows1252Encoding.h create mode 100644 base/poco/Foundation/include/Poco/WindowsConsoleChannel.h create mode 100644 base/poco/Foundation/src/ASCIIEncoding.cpp create mode 100644 base/poco/Foundation/src/AbstractObserver.cpp create mode 100644 base/poco/Foundation/src/ActiveDispatcher.cpp create mode 100644 base/poco/Foundation/src/ArchiveStrategy.cpp create mode 100644 base/poco/Foundation/src/Ascii.cpp create mode 100644 base/poco/Foundation/src/AsyncChannel.cpp create mode 100644 base/poco/Foundation/src/AtomicCounter.cpp create mode 100644 base/poco/Foundation/src/Base32Decoder.cpp create mode 100644 base/poco/Foundation/src/Base32Encoder.cpp create mode 100644 base/poco/Foundation/src/Base64Decoder.cpp create mode 100644 base/poco/Foundation/src/Base64Encoder.cpp create mode 100644 base/poco/Foundation/src/BinaryReader.cpp create mode 100644 base/poco/Foundation/src/BinaryWriter.cpp create mode 100644 base/poco/Foundation/src/Bugcheck.cpp create mode 100644 base/poco/Foundation/src/ByteOrder.cpp create mode 100644 base/poco/Foundation/src/Channel.cpp create mode 100644 base/poco/Foundation/src/Checksum.cpp create mode 100644 base/poco/Foundation/src/Clock.cpp create mode 100644 base/poco/Foundation/src/CompressedLogFile.cpp create mode 100644 base/poco/Foundation/src/Condition.cpp create mode 100644 base/poco/Foundation/src/Configurable.cpp create mode 100644 base/poco/Foundation/src/ConsoleChannel.cpp create mode 100644 base/poco/Foundation/src/CountingStream.cpp create mode 100644 base/poco/Foundation/src/DateTime.cpp create mode 100644 base/poco/Foundation/src/DateTimeFormat.cpp create mode 100644 base/poco/Foundation/src/DateTimeFormatter.cpp create mode 100644 base/poco/Foundation/src/DateTimeParser.cpp create mode 100644 base/poco/Foundation/src/Debugger.cpp create mode 100644 base/poco/Foundation/src/DeflatingStream.cpp create mode 100644 base/poco/Foundation/src/DigestEngine.cpp create mode 100644 base/poco/Foundation/src/DigestStream.cpp create mode 100644 base/poco/Foundation/src/DirectoryIterator.cpp create mode 100644 base/poco/Foundation/src/DirectoryIteratorStrategy.cpp create mode 100644 base/poco/Foundation/src/DirectoryIterator_UNIX.cpp create mode 100644 base/poco/Foundation/src/DirectoryIterator_WIN32.cpp create mode 100644 base/poco/Foundation/src/DirectoryIterator_WIN32U.cpp create mode 100644 base/poco/Foundation/src/DirectoryWatcher.cpp create mode 100644 base/poco/Foundation/src/Environment.cpp create mode 100644 base/poco/Foundation/src/Environment_UNIX.cpp create mode 100644 base/poco/Foundation/src/Environment_VX.cpp create mode 100644 base/poco/Foundation/src/Environment_WIN32.cpp create mode 100644 base/poco/Foundation/src/Environment_WIN32U.cpp create mode 100644 base/poco/Foundation/src/Environment_WINCE.cpp create mode 100644 base/poco/Foundation/src/Error.cpp create mode 100644 base/poco/Foundation/src/ErrorHandler.cpp create mode 100644 base/poco/Foundation/src/Event.cpp create mode 100644 base/poco/Foundation/src/EventArgs.cpp create mode 100644 base/poco/Foundation/src/EventChannel.cpp create mode 100644 base/poco/Foundation/src/EventLogChannel.cpp create mode 100644 base/poco/Foundation/src/Event_POSIX.cpp create mode 100644 base/poco/Foundation/src/Event_VX.cpp create mode 100644 base/poco/Foundation/src/Event_WIN32.cpp create mode 100644 base/poco/Foundation/src/Exception.cpp create mode 100644 base/poco/Foundation/src/FIFOBufferStream.cpp create mode 100644 base/poco/Foundation/src/FPEnvironment.cpp create mode 100644 base/poco/Foundation/src/FPEnvironment_C99.cpp create mode 100644 base/poco/Foundation/src/FPEnvironment_DEC.cpp create mode 100644 base/poco/Foundation/src/FPEnvironment_DUMMY.cpp create mode 100644 base/poco/Foundation/src/FPEnvironment_QNX.cpp create mode 100644 base/poco/Foundation/src/FPEnvironment_SUN.cpp create mode 100644 base/poco/Foundation/src/FPEnvironment_WIN32.cpp create mode 100644 base/poco/Foundation/src/File.cpp create mode 100644 base/poco/Foundation/src/FileChannel.cpp create mode 100644 base/poco/Foundation/src/FileStream.cpp create mode 100644 base/poco/Foundation/src/FileStreamFactory.cpp create mode 100644 base/poco/Foundation/src/FileStream_POSIX.cpp create mode 100644 base/poco/Foundation/src/FileStream_WIN32.cpp create mode 100644 base/poco/Foundation/src/File_UNIX.cpp create mode 100644 base/poco/Foundation/src/File_VX.cpp create mode 100644 base/poco/Foundation/src/File_WIN32.cpp create mode 100644 base/poco/Foundation/src/File_WIN32U.cpp create mode 100644 base/poco/Foundation/src/File_WINCE.cpp create mode 100644 base/poco/Foundation/src/Format.cpp create mode 100644 base/poco/Foundation/src/Formatter.cpp create mode 100644 base/poco/Foundation/src/FormattingChannel.cpp create mode 100644 base/poco/Foundation/src/Glob.cpp create mode 100644 base/poco/Foundation/src/Hash.cpp create mode 100644 base/poco/Foundation/src/HashStatistic.cpp create mode 100644 base/poco/Foundation/src/HexBinaryDecoder.cpp create mode 100644 base/poco/Foundation/src/HexBinaryEncoder.cpp create mode 100644 base/poco/Foundation/src/InflatingStream.cpp create mode 100644 base/poco/Foundation/src/JSONString.cpp create mode 100644 base/poco/Foundation/src/Latin1Encoding.cpp create mode 100644 base/poco/Foundation/src/Latin2Encoding.cpp create mode 100644 base/poco/Foundation/src/Latin9Encoding.cpp create mode 100644 base/poco/Foundation/src/LineEndingConverter.cpp create mode 100644 base/poco/Foundation/src/LocalDateTime.cpp create mode 100644 base/poco/Foundation/src/LogFile.cpp create mode 100644 base/poco/Foundation/src/LogFile_STD.cpp create mode 100644 base/poco/Foundation/src/LogFile_WIN32.cpp create mode 100644 base/poco/Foundation/src/LogFile_WIN32U.cpp create mode 100644 base/poco/Foundation/src/LogStream.cpp create mode 100644 base/poco/Foundation/src/Logger.cpp create mode 100644 base/poco/Foundation/src/LoggingFactory.cpp create mode 100644 base/poco/Foundation/src/LoggingRegistry.cpp create mode 100644 base/poco/Foundation/src/MD4Engine.cpp create mode 100644 base/poco/Foundation/src/MD5Engine.cpp create mode 100644 base/poco/Foundation/src/Manifest.cpp create mode 100644 base/poco/Foundation/src/MemoryPool.cpp create mode 100644 base/poco/Foundation/src/MemoryStream.cpp create mode 100644 base/poco/Foundation/src/Message.cpp create mode 100644 base/poco/Foundation/src/Mutex.cpp create mode 100644 base/poco/Foundation/src/Mutex_POSIX.cpp create mode 100644 base/poco/Foundation/src/Mutex_VX.cpp create mode 100644 base/poco/Foundation/src/Mutex_WIN32.cpp create mode 100644 base/poco/Foundation/src/Mutex_WINCE.cpp create mode 100644 base/poco/Foundation/src/NamedEvent.cpp create mode 100644 base/poco/Foundation/src/NamedEvent_Android.cpp create mode 100644 base/poco/Foundation/src/NamedEvent_UNIX.cpp create mode 100644 base/poco/Foundation/src/NamedEvent_WIN32.cpp create mode 100644 base/poco/Foundation/src/NamedEvent_WIN32U.cpp create mode 100644 base/poco/Foundation/src/NamedMutex.cpp create mode 100644 base/poco/Foundation/src/NamedMutex_Android.cpp create mode 100644 base/poco/Foundation/src/NamedMutex_UNIX.cpp create mode 100644 base/poco/Foundation/src/NamedMutex_WIN32.cpp create mode 100644 base/poco/Foundation/src/NamedMutex_WIN32U.cpp create mode 100644 base/poco/Foundation/src/NestedDiagnosticContext.cpp create mode 100644 base/poco/Foundation/src/Notification.cpp create mode 100644 base/poco/Foundation/src/NotificationCenter.cpp create mode 100644 base/poco/Foundation/src/NotificationQueue.cpp create mode 100644 base/poco/Foundation/src/NullChannel.cpp create mode 100644 base/poco/Foundation/src/NullStream.cpp create mode 100644 base/poco/Foundation/src/NumberFormatter.cpp create mode 100644 base/poco/Foundation/src/NumberParser.cpp create mode 100644 base/poco/Foundation/src/NumericString.cpp create mode 100644 base/poco/Foundation/src/Path.cpp create mode 100644 base/poco/Foundation/src/Path_UNIX.cpp create mode 100644 base/poco/Foundation/src/Path_WIN32.cpp create mode 100644 base/poco/Foundation/src/Path_WIN32U.cpp create mode 100644 base/poco/Foundation/src/Path_WINCE.cpp create mode 100644 base/poco/Foundation/src/PatternFormatter.cpp create mode 100644 base/poco/Foundation/src/Pipe.cpp create mode 100644 base/poco/Foundation/src/PipeImpl.cpp create mode 100644 base/poco/Foundation/src/PipeImpl_DUMMY.cpp create mode 100644 base/poco/Foundation/src/PipeImpl_POSIX.cpp create mode 100644 base/poco/Foundation/src/PipeImpl_WIN32.cpp create mode 100644 base/poco/Foundation/src/PipeStream.cpp create mode 100644 base/poco/Foundation/src/PriorityNotificationQueue.cpp create mode 100644 base/poco/Foundation/src/Process.cpp create mode 100644 base/poco/Foundation/src/Process_UNIX.cpp create mode 100644 base/poco/Foundation/src/Process_VX.cpp create mode 100644 base/poco/Foundation/src/Process_WIN32.cpp create mode 100644 base/poco/Foundation/src/Process_WIN32U.cpp create mode 100644 base/poco/Foundation/src/Process_WINCE.cpp create mode 100644 base/poco/Foundation/src/PurgeStrategy.cpp create mode 100644 base/poco/Foundation/src/RWLock.cpp create mode 100644 base/poco/Foundation/src/RWLock_Android.cpp create mode 100644 base/poco/Foundation/src/RWLock_POSIX.cpp create mode 100644 base/poco/Foundation/src/RWLock_VX.cpp create mode 100644 base/poco/Foundation/src/RWLock_WIN32.cpp create mode 100644 base/poco/Foundation/src/RWLock_WINCE.cpp create mode 100644 base/poco/Foundation/src/Random.cpp create mode 100644 base/poco/Foundation/src/RandomStream.cpp create mode 100644 base/poco/Foundation/src/RefCountedObject.cpp create mode 100644 base/poco/Foundation/src/RegularExpression.cpp create mode 100644 base/poco/Foundation/src/RotateStrategy.cpp create mode 100644 base/poco/Foundation/src/Runnable.cpp create mode 100644 base/poco/Foundation/src/SHA1Engine.cpp create mode 100644 base/poco/Foundation/src/Semaphore.cpp create mode 100644 base/poco/Foundation/src/Semaphore_POSIX.cpp create mode 100644 base/poco/Foundation/src/Semaphore_VX.cpp create mode 100644 base/poco/Foundation/src/Semaphore_WIN32.cpp create mode 100644 base/poco/Foundation/src/SharedLibrary.cpp create mode 100644 base/poco/Foundation/src/SharedLibrary_HPUX.cpp create mode 100644 base/poco/Foundation/src/SharedLibrary_UNIX.cpp create mode 100644 base/poco/Foundation/src/SharedLibrary_VX.cpp create mode 100644 base/poco/Foundation/src/SharedLibrary_WIN32.cpp create mode 100644 base/poco/Foundation/src/SharedLibrary_WIN32U.cpp create mode 100644 base/poco/Foundation/src/SharedMemory.cpp create mode 100644 base/poco/Foundation/src/SharedMemory_DUMMY.cpp create mode 100644 base/poco/Foundation/src/SharedMemory_POSIX.cpp create mode 100644 base/poco/Foundation/src/SharedMemory_WIN32.cpp create mode 100644 base/poco/Foundation/src/SignalHandler.cpp create mode 100644 base/poco/Foundation/src/SimpleFileChannel.cpp create mode 100644 base/poco/Foundation/src/SortedDirectoryIterator.cpp create mode 100644 base/poco/Foundation/src/SplitterChannel.cpp create mode 100644 base/poco/Foundation/src/Stopwatch.cpp create mode 100644 base/poco/Foundation/src/StreamChannel.cpp create mode 100644 base/poco/Foundation/src/StreamConverter.cpp create mode 100644 base/poco/Foundation/src/StreamCopier.cpp create mode 100644 base/poco/Foundation/src/StreamTokenizer.cpp create mode 100644 base/poco/Foundation/src/String.cpp create mode 100644 base/poco/Foundation/src/StringTokenizer.cpp create mode 100644 base/poco/Foundation/src/SynchronizedObject.cpp create mode 100644 base/poco/Foundation/src/SyslogChannel.cpp create mode 100644 base/poco/Foundation/src/Task.cpp create mode 100644 base/poco/Foundation/src/TaskManager.cpp create mode 100644 base/poco/Foundation/src/TaskNotification.cpp create mode 100644 base/poco/Foundation/src/TeeStream.cpp create mode 100644 base/poco/Foundation/src/TemporaryFile.cpp create mode 100644 base/poco/Foundation/src/TextBufferIterator.cpp create mode 100644 base/poco/Foundation/src/TextConverter.cpp create mode 100644 base/poco/Foundation/src/TextEncoding.cpp create mode 100644 base/poco/Foundation/src/TextIterator.cpp create mode 100644 base/poco/Foundation/src/Thread.cpp create mode 100644 base/poco/Foundation/src/ThreadLocal.cpp create mode 100644 base/poco/Foundation/src/ThreadPool.cpp create mode 100644 base/poco/Foundation/src/ThreadTarget.cpp create mode 100644 base/poco/Foundation/src/Thread_POSIX.cpp create mode 100644 base/poco/Foundation/src/Thread_VX.cpp create mode 100644 base/poco/Foundation/src/Thread_WIN32.cpp create mode 100644 base/poco/Foundation/src/Thread_WINCE.cpp create mode 100644 base/poco/Foundation/src/TimedNotificationQueue.cpp create mode 100644 base/poco/Foundation/src/Timer.cpp create mode 100644 base/poco/Foundation/src/Timespan.cpp create mode 100644 base/poco/Foundation/src/Timestamp.cpp create mode 100644 base/poco/Foundation/src/Timezone.cpp create mode 100644 base/poco/Foundation/src/Timezone_UNIX.cpp create mode 100644 base/poco/Foundation/src/Timezone_VX.cpp create mode 100644 base/poco/Foundation/src/Timezone_WIN32.cpp create mode 100644 base/poco/Foundation/src/Timezone_WINCE.cpp create mode 100644 base/poco/Foundation/src/Token.cpp create mode 100644 base/poco/Foundation/src/URI.cpp create mode 100644 base/poco/Foundation/src/URIStreamFactory.cpp create mode 100644 base/poco/Foundation/src/URIStreamOpener.cpp create mode 100644 base/poco/Foundation/src/UTF16Encoding.cpp create mode 100644 base/poco/Foundation/src/UTF32Encoding.cpp create mode 100644 base/poco/Foundation/src/UTF8Encoding.cpp create mode 100644 base/poco/Foundation/src/UTF8String.cpp create mode 100644 base/poco/Foundation/src/UUID.cpp create mode 100644 base/poco/Foundation/src/UUIDGenerator.cpp create mode 100644 base/poco/Foundation/src/Unicode.cpp create mode 100644 base/poco/Foundation/src/UnicodeConverter.cpp create mode 100644 base/poco/Foundation/src/Var.cpp create mode 100644 base/poco/Foundation/src/VarHolder.cpp create mode 100644 base/poco/Foundation/src/VarIterator.cpp create mode 100644 base/poco/Foundation/src/Void.cpp create mode 100644 base/poco/Foundation/src/Windows1250Encoding.cpp create mode 100644 base/poco/Foundation/src/Windows1251Encoding.cpp create mode 100644 base/poco/Foundation/src/Windows1252Encoding.cpp create mode 100644 base/poco/Foundation/src/WindowsConsoleChannel.cpp create mode 100644 base/poco/Foundation/src/adler32.c create mode 100644 base/poco/Foundation/src/bignum-dtoa.cc create mode 100644 base/poco/Foundation/src/bignum-dtoa.h create mode 100644 base/poco/Foundation/src/bignum.cc create mode 100644 base/poco/Foundation/src/bignum.h create mode 100644 base/poco/Foundation/src/cached-powers.cc create mode 100644 base/poco/Foundation/src/cached-powers.h create mode 100644 base/poco/Foundation/src/compress.c create mode 100644 base/poco/Foundation/src/crc32.c create mode 100644 base/poco/Foundation/src/crc32.h create mode 100644 base/poco/Foundation/src/deflate.c create mode 100644 base/poco/Foundation/src/deflate.h create mode 100644 base/poco/Foundation/src/diy-fp.cc create mode 100644 base/poco/Foundation/src/diy-fp.h create mode 100644 base/poco/Foundation/src/double-conversion.cc create mode 100644 base/poco/Foundation/src/double-conversion.h create mode 100644 base/poco/Foundation/src/fast-dtoa.cc create mode 100644 base/poco/Foundation/src/fast-dtoa.h create mode 100644 base/poco/Foundation/src/fixed-dtoa.cc create mode 100644 base/poco/Foundation/src/fixed-dtoa.h create mode 100644 base/poco/Foundation/src/gzguts.h create mode 100644 base/poco/Foundation/src/ieee.h create mode 100644 base/poco/Foundation/src/infback.c create mode 100644 base/poco/Foundation/src/inffast.c create mode 100644 base/poco/Foundation/src/inffast.h create mode 100644 base/poco/Foundation/src/inffixed.h create mode 100644 base/poco/Foundation/src/inflate.c create mode 100644 base/poco/Foundation/src/inflate.h create mode 100644 base/poco/Foundation/src/inftrees.c create mode 100644 base/poco/Foundation/src/inftrees.h create mode 100644 base/poco/Foundation/src/pcre.h create mode 100644 base/poco/Foundation/src/pcre_byte_order.c create mode 100644 base/poco/Foundation/src/pcre_chartables.c create mode 100644 base/poco/Foundation/src/pcre_compile.c create mode 100644 base/poco/Foundation/src/pcre_config.c create mode 100644 base/poco/Foundation/src/pcre_config.h create mode 100644 base/poco/Foundation/src/pcre_dfa_exec.c create mode 100644 base/poco/Foundation/src/pcre_exec.c create mode 100644 base/poco/Foundation/src/pcre_fullinfo.c create mode 100644 base/poco/Foundation/src/pcre_get.c create mode 100644 base/poco/Foundation/src/pcre_globals.c create mode 100644 base/poco/Foundation/src/pcre_internal.h create mode 100644 base/poco/Foundation/src/pcre_jit_compile.c create mode 100644 base/poco/Foundation/src/pcre_maketables.c create mode 100644 base/poco/Foundation/src/pcre_newline.c create mode 100644 base/poco/Foundation/src/pcre_ord2utf8.c create mode 100644 base/poco/Foundation/src/pcre_refcount.c create mode 100644 base/poco/Foundation/src/pcre_string_utils.c create mode 100644 base/poco/Foundation/src/pcre_study.c create mode 100644 base/poco/Foundation/src/pcre_tables.c create mode 100644 base/poco/Foundation/src/pcre_ucd.c create mode 100644 base/poco/Foundation/src/pcre_valid_utf8.c create mode 100644 base/poco/Foundation/src/pcre_version.c create mode 100644 base/poco/Foundation/src/pcre_xclass.c create mode 100644 base/poco/Foundation/src/pocomsg.mc create mode 100644 base/poco/Foundation/src/strtod.cc create mode 100644 base/poco/Foundation/src/strtod.h create mode 100644 base/poco/Foundation/src/trees.c create mode 100644 base/poco/Foundation/src/trees.h create mode 100644 base/poco/Foundation/src/ucp.h create mode 100644 base/poco/Foundation/src/utils.h create mode 100644 base/poco/Foundation/src/zutil.c create mode 100644 base/poco/Foundation/src/zutil.h create mode 100644 base/poco/JSON/CMakeLists.txt create mode 100644 base/poco/JSON/include/Poco/JSON/Array.h create mode 100644 base/poco/JSON/include/Poco/JSON/Handler.h create mode 100644 base/poco/JSON/include/Poco/JSON/JSON.h create mode 100644 base/poco/JSON/include/Poco/JSON/JSONException.h create mode 100644 base/poco/JSON/include/Poco/JSON/Object.h create mode 100644 base/poco/JSON/include/Poco/JSON/ParseHandler.h create mode 100644 base/poco/JSON/include/Poco/JSON/Parser.h create mode 100644 base/poco/JSON/include/Poco/JSON/ParserImpl.h create mode 100644 base/poco/JSON/include/Poco/JSON/PrintHandler.h create mode 100644 base/poco/JSON/include/Poco/JSON/Query.h create mode 100644 base/poco/JSON/include/Poco/JSON/Stringifier.h create mode 100644 base/poco/JSON/include/Poco/JSON/Template.h create mode 100644 base/poco/JSON/include/Poco/JSON/TemplateCache.h create mode 100644 base/poco/JSON/src/Array.cpp create mode 100644 base/poco/JSON/src/Handler.cpp create mode 100644 base/poco/JSON/src/JSONException.cpp create mode 100644 base/poco/JSON/src/Object.cpp create mode 100644 base/poco/JSON/src/ParseHandler.cpp create mode 100644 base/poco/JSON/src/Parser.cpp create mode 100644 base/poco/JSON/src/ParserImpl.cpp create mode 100644 base/poco/JSON/src/PrintHandler.cpp create mode 100644 base/poco/JSON/src/Query.cpp create mode 100644 base/poco/JSON/src/Stringifier.cpp create mode 100644 base/poco/JSON/src/Template.cpp create mode 100644 base/poco/JSON/src/TemplateCache.cpp create mode 100644 base/poco/JSON/src/pdjson.c create mode 100644 base/poco/JSON/src/pdjson.h create mode 100644 base/poco/LICENSE create mode 100644 base/poco/MongoDB/CMakeLists.txt create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Array.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/BSONReader.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/BSONWriter.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Binary.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Connection.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Cursor.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Database.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/DeleteRequest.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Document.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Element.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/GetMoreRequest.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/InsertRequest.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/JavaScriptCode.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/KillCursorsRequest.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Message.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/MessageHeader.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/MongoDB.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/ObjectId.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/PoolableConnectionFactory.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/QueryRequest.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/RegularExpression.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/ReplicaSet.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/RequestMessage.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/ResponseMessage.h create mode 100644 base/poco/MongoDB/include/Poco/MongoDB/UpdateRequest.h create mode 100644 base/poco/MongoDB/src/Array.cpp create mode 100644 base/poco/MongoDB/src/Binary.cpp create mode 100644 base/poco/MongoDB/src/Connection.cpp create mode 100644 base/poco/MongoDB/src/Cursor.cpp create mode 100644 base/poco/MongoDB/src/Database.cpp create mode 100644 base/poco/MongoDB/src/DeleteRequest.cpp create mode 100644 base/poco/MongoDB/src/Document.cpp create mode 100644 base/poco/MongoDB/src/Element.cpp create mode 100644 base/poco/MongoDB/src/GetMoreRequest.cpp create mode 100644 base/poco/MongoDB/src/InsertRequest.cpp create mode 100644 base/poco/MongoDB/src/JavaScriptCode.cpp create mode 100644 base/poco/MongoDB/src/KillCursorsRequest.cpp create mode 100644 base/poco/MongoDB/src/Message.cpp create mode 100644 base/poco/MongoDB/src/MessageHeader.cpp create mode 100644 base/poco/MongoDB/src/ObjectId.cpp create mode 100644 base/poco/MongoDB/src/QueryRequest.cpp create mode 100644 base/poco/MongoDB/src/RegularExpression.cpp create mode 100644 base/poco/MongoDB/src/ReplicaSet.cpp create mode 100644 base/poco/MongoDB/src/RequestMessage.cpp create mode 100644 base/poco/MongoDB/src/ResponseMessage.cpp create mode 100644 base/poco/MongoDB/src/UpdateRequest.cpp create mode 100644 base/poco/Net/CMakeLists.txt create mode 100644 base/poco/Net/include/Poco/Net/AbstractHTTPRequestHandler.h create mode 100644 base/poco/Net/include/Poco/Net/DNS.h create mode 100644 base/poco/Net/include/Poco/Net/DatagramSocket.h create mode 100644 base/poco/Net/include/Poco/Net/DatagramSocketImpl.h create mode 100644 base/poco/Net/include/Poco/Net/DialogSocket.h create mode 100644 base/poco/Net/include/Poco/Net/FTPClientSession.h create mode 100644 base/poco/Net/include/Poco/Net/FTPStreamFactory.h create mode 100644 base/poco/Net/include/Poco/Net/FilePartSource.h create mode 100644 base/poco/Net/include/Poco/Net/HTMLForm.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPAuthenticationParams.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPBasicCredentials.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPBasicStreamBuf.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPBufferAllocator.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPChunkedStream.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPClientSession.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPCookie.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPCredentials.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPDigestCredentials.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPFixedLengthStream.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPHeaderStream.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPIOStream.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPMessage.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPRequest.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPRequestHandler.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPRequestHandlerFactory.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPResponse.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPServer.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPServerConnection.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPServerConnectionFactory.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPServerParams.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPServerRequest.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPServerRequestImpl.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPServerResponse.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPServerResponseImpl.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPServerSession.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPSession.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPSessionFactory.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPSessionInstantiator.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPStream.h create mode 100644 base/poco/Net/include/Poco/Net/HTTPStreamFactory.h create mode 100644 base/poco/Net/include/Poco/Net/HostEntry.h create mode 100644 base/poco/Net/include/Poco/Net/ICMPClient.h create mode 100644 base/poco/Net/include/Poco/Net/ICMPEventArgs.h create mode 100644 base/poco/Net/include/Poco/Net/ICMPPacket.h create mode 100644 base/poco/Net/include/Poco/Net/ICMPPacketImpl.h create mode 100644 base/poco/Net/include/Poco/Net/ICMPSocket.h create mode 100644 base/poco/Net/include/Poco/Net/ICMPSocketImpl.h create mode 100644 base/poco/Net/include/Poco/Net/ICMPv4PacketImpl.h create mode 100644 base/poco/Net/include/Poco/Net/IPAddress.h create mode 100644 base/poco/Net/include/Poco/Net/IPAddressImpl.h create mode 100644 base/poco/Net/include/Poco/Net/MailMessage.h create mode 100644 base/poco/Net/include/Poco/Net/MailRecipient.h create mode 100644 base/poco/Net/include/Poco/Net/MailStream.h create mode 100644 base/poco/Net/include/Poco/Net/MediaType.h create mode 100644 base/poco/Net/include/Poco/Net/MessageHeader.h create mode 100644 base/poco/Net/include/Poco/Net/MulticastSocket.h create mode 100644 base/poco/Net/include/Poco/Net/MultipartReader.h create mode 100644 base/poco/Net/include/Poco/Net/MultipartWriter.h create mode 100644 base/poco/Net/include/Poco/Net/NTPClient.h create mode 100644 base/poco/Net/include/Poco/Net/NTPEventArgs.h create mode 100644 base/poco/Net/include/Poco/Net/NTPPacket.h create mode 100644 base/poco/Net/include/Poco/Net/NameValueCollection.h create mode 100644 base/poco/Net/include/Poco/Net/Net.h create mode 100644 base/poco/Net/include/Poco/Net/NetException.h create mode 100644 base/poco/Net/include/Poco/Net/NetworkInterface.h create mode 100644 base/poco/Net/include/Poco/Net/NullPartHandler.h create mode 100644 base/poco/Net/include/Poco/Net/OAuth10Credentials.h create mode 100644 base/poco/Net/include/Poco/Net/OAuth20Credentials.h create mode 100644 base/poco/Net/include/Poco/Net/POP3ClientSession.h create mode 100644 base/poco/Net/include/Poco/Net/ParallelSocketAcceptor.h create mode 100644 base/poco/Net/include/Poco/Net/ParallelSocketReactor.h create mode 100644 base/poco/Net/include/Poco/Net/PartHandler.h create mode 100644 base/poco/Net/include/Poco/Net/PartSource.h create mode 100644 base/poco/Net/include/Poco/Net/PartStore.h create mode 100644 base/poco/Net/include/Poco/Net/PollSet.h create mode 100644 base/poco/Net/include/Poco/Net/QuotedPrintableDecoder.h create mode 100644 base/poco/Net/include/Poco/Net/QuotedPrintableEncoder.h create mode 100644 base/poco/Net/include/Poco/Net/RawSocket.h create mode 100644 base/poco/Net/include/Poco/Net/RawSocketImpl.h create mode 100644 base/poco/Net/include/Poco/Net/RemoteSyslogChannel.h create mode 100644 base/poco/Net/include/Poco/Net/RemoteSyslogListener.h create mode 100644 base/poco/Net/include/Poco/Net/SMTPChannel.h create mode 100644 base/poco/Net/include/Poco/Net/SMTPClientSession.h create mode 100644 base/poco/Net/include/Poco/Net/ServerSocket.h create mode 100644 base/poco/Net/include/Poco/Net/ServerSocketImpl.h create mode 100644 base/poco/Net/include/Poco/Net/Socket.h create mode 100644 base/poco/Net/include/Poco/Net/SocketAcceptor.h create mode 100644 base/poco/Net/include/Poco/Net/SocketAddress.h create mode 100644 base/poco/Net/include/Poco/Net/SocketAddressImpl.h create mode 100644 base/poco/Net/include/Poco/Net/SocketConnector.h create mode 100644 base/poco/Net/include/Poco/Net/SocketDefs.h create mode 100644 base/poco/Net/include/Poco/Net/SocketImpl.h create mode 100644 base/poco/Net/include/Poco/Net/SocketNotification.h create mode 100644 base/poco/Net/include/Poco/Net/SocketNotifier.h create mode 100644 base/poco/Net/include/Poco/Net/SocketReactor.h create mode 100644 base/poco/Net/include/Poco/Net/SocketStream.h create mode 100644 base/poco/Net/include/Poco/Net/StreamSocket.h create mode 100644 base/poco/Net/include/Poco/Net/StreamSocketImpl.h create mode 100644 base/poco/Net/include/Poco/Net/StringPartSource.h create mode 100644 base/poco/Net/include/Poco/Net/TCPServer.h create mode 100644 base/poco/Net/include/Poco/Net/TCPServerConnection.h create mode 100644 base/poco/Net/include/Poco/Net/TCPServerConnectionFactory.h create mode 100644 base/poco/Net/include/Poco/Net/TCPServerDispatcher.h create mode 100644 base/poco/Net/include/Poco/Net/TCPServerParams.h create mode 100644 base/poco/Net/include/Poco/Net/WebSocket.h create mode 100644 base/poco/Net/include/Poco/Net/WebSocketImpl.h create mode 100644 base/poco/Net/src/AbstractHTTPRequestHandler.cpp create mode 100644 base/poco/Net/src/DNS.cpp create mode 100644 base/poco/Net/src/DatagramSocket.cpp create mode 100644 base/poco/Net/src/DatagramSocketImpl.cpp create mode 100644 base/poco/Net/src/DialogSocket.cpp create mode 100644 base/poco/Net/src/FTPClientSession.cpp create mode 100644 base/poco/Net/src/FTPStreamFactory.cpp create mode 100644 base/poco/Net/src/FilePartSource.cpp create mode 100644 base/poco/Net/src/HTMLForm.cpp create mode 100644 base/poco/Net/src/HTTPAuthenticationParams.cpp create mode 100644 base/poco/Net/src/HTTPBasicCredentials.cpp create mode 100644 base/poco/Net/src/HTTPBufferAllocator.cpp create mode 100644 base/poco/Net/src/HTTPChunkedStream.cpp create mode 100644 base/poco/Net/src/HTTPClientSession.cpp create mode 100644 base/poco/Net/src/HTTPCookie.cpp create mode 100644 base/poco/Net/src/HTTPCredentials.cpp create mode 100644 base/poco/Net/src/HTTPDigestCredentials.cpp create mode 100644 base/poco/Net/src/HTTPFixedLengthStream.cpp create mode 100644 base/poco/Net/src/HTTPHeaderStream.cpp create mode 100644 base/poco/Net/src/HTTPIOStream.cpp create mode 100644 base/poco/Net/src/HTTPMessage.cpp create mode 100644 base/poco/Net/src/HTTPRequest.cpp create mode 100644 base/poco/Net/src/HTTPRequestHandler.cpp create mode 100644 base/poco/Net/src/HTTPRequestHandlerFactory.cpp create mode 100644 base/poco/Net/src/HTTPResponse.cpp create mode 100644 base/poco/Net/src/HTTPServer.cpp create mode 100644 base/poco/Net/src/HTTPServerConnection.cpp create mode 100644 base/poco/Net/src/HTTPServerConnectionFactory.cpp create mode 100644 base/poco/Net/src/HTTPServerParams.cpp create mode 100644 base/poco/Net/src/HTTPServerRequest.cpp create mode 100644 base/poco/Net/src/HTTPServerRequestImpl.cpp create mode 100644 base/poco/Net/src/HTTPServerResponse.cpp create mode 100644 base/poco/Net/src/HTTPServerResponseImpl.cpp create mode 100644 base/poco/Net/src/HTTPServerSession.cpp create mode 100644 base/poco/Net/src/HTTPSession.cpp create mode 100644 base/poco/Net/src/HTTPSessionFactory.cpp create mode 100644 base/poco/Net/src/HTTPSessionInstantiator.cpp create mode 100644 base/poco/Net/src/HTTPStream.cpp create mode 100644 base/poco/Net/src/HTTPStreamFactory.cpp create mode 100644 base/poco/Net/src/HostEntry.cpp create mode 100644 base/poco/Net/src/ICMPClient.cpp create mode 100644 base/poco/Net/src/ICMPEventArgs.cpp create mode 100644 base/poco/Net/src/ICMPPacket.cpp create mode 100644 base/poco/Net/src/ICMPPacketImpl.cpp create mode 100644 base/poco/Net/src/ICMPSocket.cpp create mode 100644 base/poco/Net/src/ICMPSocketImpl.cpp create mode 100644 base/poco/Net/src/ICMPv4PacketImpl.cpp create mode 100644 base/poco/Net/src/IPAddress.cpp create mode 100644 base/poco/Net/src/IPAddressImpl.cpp create mode 100644 base/poco/Net/src/MailMessage.cpp create mode 100644 base/poco/Net/src/MailRecipient.cpp create mode 100644 base/poco/Net/src/MailStream.cpp create mode 100644 base/poco/Net/src/MediaType.cpp create mode 100644 base/poco/Net/src/MessageHeader.cpp create mode 100644 base/poco/Net/src/MulticastSocket.cpp create mode 100644 base/poco/Net/src/MultipartReader.cpp create mode 100644 base/poco/Net/src/MultipartWriter.cpp create mode 100644 base/poco/Net/src/NTPClient.cpp create mode 100644 base/poco/Net/src/NTPEventArgs.cpp create mode 100644 base/poco/Net/src/NTPPacket.cpp create mode 100644 base/poco/Net/src/NameValueCollection.cpp create mode 100644 base/poco/Net/src/Net.cpp create mode 100644 base/poco/Net/src/NetException.cpp create mode 100644 base/poco/Net/src/NetworkInterface.cpp create mode 100644 base/poco/Net/src/NullPartHandler.cpp create mode 100644 base/poco/Net/src/OAuth10Credentials.cpp create mode 100644 base/poco/Net/src/OAuth20Credentials.cpp create mode 100644 base/poco/Net/src/POP3ClientSession.cpp create mode 100644 base/poco/Net/src/PartHandler.cpp create mode 100644 base/poco/Net/src/PartSource.cpp create mode 100644 base/poco/Net/src/PartStore.cpp create mode 100644 base/poco/Net/src/PollSet.cpp create mode 100644 base/poco/Net/src/QuotedPrintableDecoder.cpp create mode 100644 base/poco/Net/src/QuotedPrintableEncoder.cpp create mode 100644 base/poco/Net/src/RawSocket.cpp create mode 100644 base/poco/Net/src/RawSocketImpl.cpp create mode 100644 base/poco/Net/src/RemoteSyslogChannel.cpp create mode 100644 base/poco/Net/src/RemoteSyslogListener.cpp create mode 100644 base/poco/Net/src/SMTPChannel.cpp create mode 100644 base/poco/Net/src/SMTPClientSession.cpp create mode 100644 base/poco/Net/src/ServerSocket.cpp create mode 100644 base/poco/Net/src/ServerSocketImpl.cpp create mode 100644 base/poco/Net/src/Socket.cpp create mode 100644 base/poco/Net/src/SocketAddress.cpp create mode 100644 base/poco/Net/src/SocketAddressImpl.cpp create mode 100644 base/poco/Net/src/SocketImpl.cpp create mode 100644 base/poco/Net/src/SocketNotification.cpp create mode 100644 base/poco/Net/src/SocketNotifier.cpp create mode 100644 base/poco/Net/src/SocketReactor.cpp create mode 100644 base/poco/Net/src/SocketStream.cpp create mode 100644 base/poco/Net/src/StreamSocket.cpp create mode 100644 base/poco/Net/src/StreamSocketImpl.cpp create mode 100644 base/poco/Net/src/StringPartSource.cpp create mode 100644 base/poco/Net/src/TCPServer.cpp create mode 100644 base/poco/Net/src/TCPServerConnection.cpp create mode 100644 base/poco/Net/src/TCPServerConnectionFactory.cpp create mode 100644 base/poco/Net/src/TCPServerDispatcher.cpp create mode 100644 base/poco/Net/src/TCPServerParams.cpp create mode 100644 base/poco/Net/src/WebSocket.cpp create mode 100644 base/poco/Net/src/WebSocketImpl.cpp create mode 100644 base/poco/NetSSL_OpenSSL/CMakeLists.txt create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/AcceptCertificateHandler.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/CertificateHandlerFactory.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/CertificateHandlerFactoryMgr.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/ConsoleCertificateHandler.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/Context.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/HTTPSClientSession.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/HTTPSSessionInstantiator.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/HTTPSStreamFactory.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/InvalidCertificateHandler.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/KeyConsoleHandler.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/KeyFileHandler.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/NetSSL.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/PrivateKeyFactory.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/PrivateKeyFactoryMgr.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/PrivateKeyPassphraseHandler.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/RejectCertificateHandler.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLException.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSMTPClientSession.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureServerSocket.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureServerSocketImpl.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocket.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/Session.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/Utility.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/VerificationErrorArgs.h create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/X509Certificate.h create mode 100644 base/poco/NetSSL_OpenSSL/src/AcceptCertificateHandler.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/CertificateHandlerFactory.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/CertificateHandlerFactoryMgr.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/ConsoleCertificateHandler.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/Context.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/HTTPSClientSession.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/HTTPSSessionInstantiator.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/HTTPSStreamFactory.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/InvalidCertificateHandler.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/KeyConsoleHandler.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/KeyFileHandler.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/PrivateKeyFactory.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/PrivateKeyFactoryMgr.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/PrivateKeyPassphraseHandler.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/RejectCertificateHandler.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/SSLException.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/SSLManager.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/SecureSMTPClientSession.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/SecureServerSocket.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/SecureServerSocketImpl.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/SecureStreamSocket.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/SecureStreamSocketImpl.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/Session.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/Utility.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/VerificationErrorArgs.cpp create mode 100644 base/poco/NetSSL_OpenSSL/src/X509Certificate.cpp create mode 100644 base/poco/README.md create mode 100644 base/poco/Redis/CMakeLists.txt create mode 100644 base/poco/Redis/include/Poco/Redis/Array.h create mode 100644 base/poco/Redis/include/Poco/Redis/AsyncReader.h create mode 100644 base/poco/Redis/include/Poco/Redis/Client.h create mode 100644 base/poco/Redis/include/Poco/Redis/Command.h create mode 100644 base/poco/Redis/include/Poco/Redis/Error.h create mode 100644 base/poco/Redis/include/Poco/Redis/Exception.h create mode 100644 base/poco/Redis/include/Poco/Redis/PoolableConnectionFactory.h create mode 100644 base/poco/Redis/include/Poco/Redis/Redis.h create mode 100644 base/poco/Redis/include/Poco/Redis/RedisEventArgs.h create mode 100644 base/poco/Redis/include/Poco/Redis/RedisStream.h create mode 100644 base/poco/Redis/include/Poco/Redis/Type.h create mode 100644 base/poco/Redis/src/Array.cpp create mode 100644 base/poco/Redis/src/AsyncReader.cpp create mode 100644 base/poco/Redis/src/Client.cpp create mode 100644 base/poco/Redis/src/Command.cpp create mode 100644 base/poco/Redis/src/Error.cpp create mode 100644 base/poco/Redis/src/Exception.cpp create mode 100644 base/poco/Redis/src/RedisEventArgs.cpp create mode 100644 base/poco/Redis/src/RedisStream.cpp create mode 100644 base/poco/Redis/src/Type.cpp create mode 100644 base/poco/Util/CMakeLists.txt create mode 100644 base/poco/Util/include/Poco/Util/AbstractConfiguration.h create mode 100644 base/poco/Util/include/Poco/Util/Application.h create mode 100644 base/poco/Util/include/Poco/Util/ConfigurationMapper.h create mode 100644 base/poco/Util/include/Poco/Util/ConfigurationView.h create mode 100644 base/poco/Util/include/Poco/Util/FilesystemConfiguration.h create mode 100644 base/poco/Util/include/Poco/Util/HelpFormatter.h create mode 100644 base/poco/Util/include/Poco/Util/IniFileConfiguration.h create mode 100644 base/poco/Util/include/Poco/Util/IntValidator.h create mode 100644 base/poco/Util/include/Poco/Util/JSONConfiguration.h create mode 100644 base/poco/Util/include/Poco/Util/LayeredConfiguration.h create mode 100644 base/poco/Util/include/Poco/Util/LoggingConfigurator.h create mode 100644 base/poco/Util/include/Poco/Util/LoggingSubsystem.h create mode 100644 base/poco/Util/include/Poco/Util/MapConfiguration.h create mode 100644 base/poco/Util/include/Poco/Util/Option.h create mode 100644 base/poco/Util/include/Poco/Util/OptionCallback.h create mode 100644 base/poco/Util/include/Poco/Util/OptionException.h create mode 100644 base/poco/Util/include/Poco/Util/OptionProcessor.h create mode 100644 base/poco/Util/include/Poco/Util/OptionSet.h create mode 100644 base/poco/Util/include/Poco/Util/PropertyFileConfiguration.h create mode 100644 base/poco/Util/include/Poco/Util/RegExpValidator.h create mode 100644 base/poco/Util/include/Poco/Util/ServerApplication.h create mode 100644 base/poco/Util/include/Poco/Util/Subsystem.h create mode 100644 base/poco/Util/include/Poco/Util/SystemConfiguration.h create mode 100644 base/poco/Util/include/Poco/Util/Timer.h create mode 100644 base/poco/Util/include/Poco/Util/TimerTask.h create mode 100644 base/poco/Util/include/Poco/Util/TimerTaskAdapter.h create mode 100644 base/poco/Util/include/Poco/Util/Units.h create mode 100644 base/poco/Util/include/Poco/Util/Util.h create mode 100644 base/poco/Util/include/Poco/Util/Validator.h create mode 100644 base/poco/Util/include/Poco/Util/WinRegistryConfiguration.h create mode 100644 base/poco/Util/include/Poco/Util/WinRegistryKey.h create mode 100644 base/poco/Util/include/Poco/Util/WinService.h create mode 100644 base/poco/Util/include/Poco/Util/XMLConfiguration.h create mode 100644 base/poco/Util/src/AbstractConfiguration.cpp create mode 100644 base/poco/Util/src/Application.cpp create mode 100644 base/poco/Util/src/ConfigurationMapper.cpp create mode 100644 base/poco/Util/src/ConfigurationView.cpp create mode 100644 base/poco/Util/src/FilesystemConfiguration.cpp create mode 100644 base/poco/Util/src/HelpFormatter.cpp create mode 100644 base/poco/Util/src/IniFileConfiguration.cpp create mode 100644 base/poco/Util/src/IntValidator.cpp create mode 100644 base/poco/Util/src/JSONConfiguration.cpp create mode 100644 base/poco/Util/src/LayeredConfiguration.cpp create mode 100644 base/poco/Util/src/LoggingConfigurator.cpp create mode 100644 base/poco/Util/src/LoggingSubsystem.cpp create mode 100644 base/poco/Util/src/MapConfiguration.cpp create mode 100644 base/poco/Util/src/Option.cpp create mode 100644 base/poco/Util/src/OptionCallback.cpp create mode 100644 base/poco/Util/src/OptionException.cpp create mode 100644 base/poco/Util/src/OptionProcessor.cpp create mode 100644 base/poco/Util/src/OptionSet.cpp create mode 100644 base/poco/Util/src/PropertyFileConfiguration.cpp create mode 100644 base/poco/Util/src/RegExpValidator.cpp create mode 100644 base/poco/Util/src/ServerApplication.cpp create mode 100644 base/poco/Util/src/Subsystem.cpp create mode 100644 base/poco/Util/src/SystemConfiguration.cpp create mode 100644 base/poco/Util/src/Timer.cpp create mode 100644 base/poco/Util/src/TimerTask.cpp create mode 100644 base/poco/Util/src/Validator.cpp create mode 100644 base/poco/Util/src/XMLConfiguration.cpp create mode 100644 base/poco/XML/CMakeLists.txt create mode 100644 base/poco/XML/include/Poco/DOM/AbstractContainerNode.h create mode 100644 base/poco/XML/include/Poco/DOM/AbstractNode.h create mode 100644 base/poco/XML/include/Poco/DOM/Attr.h create mode 100644 base/poco/XML/include/Poco/DOM/AttrMap.h create mode 100644 base/poco/XML/include/Poco/DOM/AutoPtr.h create mode 100644 base/poco/XML/include/Poco/DOM/CDATASection.h create mode 100644 base/poco/XML/include/Poco/DOM/CharacterData.h create mode 100644 base/poco/XML/include/Poco/DOM/ChildNodesList.h create mode 100644 base/poco/XML/include/Poco/DOM/Comment.h create mode 100644 base/poco/XML/include/Poco/DOM/DOMBuilder.h create mode 100644 base/poco/XML/include/Poco/DOM/DOMException.h create mode 100644 base/poco/XML/include/Poco/DOM/DOMImplementation.h create mode 100644 base/poco/XML/include/Poco/DOM/DOMObject.h create mode 100644 base/poco/XML/include/Poco/DOM/DOMParser.h create mode 100644 base/poco/XML/include/Poco/DOM/DOMSerializer.h create mode 100644 base/poco/XML/include/Poco/DOM/DOMWriter.h create mode 100644 base/poco/XML/include/Poco/DOM/DTDMap.h create mode 100644 base/poco/XML/include/Poco/DOM/Document.h create mode 100644 base/poco/XML/include/Poco/DOM/DocumentEvent.h create mode 100644 base/poco/XML/include/Poco/DOM/DocumentFragment.h create mode 100644 base/poco/XML/include/Poco/DOM/DocumentType.h create mode 100644 base/poco/XML/include/Poco/DOM/Element.h create mode 100644 base/poco/XML/include/Poco/DOM/ElementsByTagNameList.h create mode 100644 base/poco/XML/include/Poco/DOM/Entity.h create mode 100644 base/poco/XML/include/Poco/DOM/EntityReference.h create mode 100644 base/poco/XML/include/Poco/DOM/Event.h create mode 100644 base/poco/XML/include/Poco/DOM/EventDispatcher.h create mode 100644 base/poco/XML/include/Poco/DOM/EventException.h create mode 100644 base/poco/XML/include/Poco/DOM/EventListener.h create mode 100644 base/poco/XML/include/Poco/DOM/EventTarget.h create mode 100644 base/poco/XML/include/Poco/DOM/MutationEvent.h create mode 100644 base/poco/XML/include/Poco/DOM/NamedNodeMap.h create mode 100644 base/poco/XML/include/Poco/DOM/Node.h create mode 100644 base/poco/XML/include/Poco/DOM/NodeAppender.h create mode 100644 base/poco/XML/include/Poco/DOM/NodeFilter.h create mode 100644 base/poco/XML/include/Poco/DOM/NodeIterator.h create mode 100644 base/poco/XML/include/Poco/DOM/NodeList.h create mode 100644 base/poco/XML/include/Poco/DOM/Notation.h create mode 100644 base/poco/XML/include/Poco/DOM/ProcessingInstruction.h create mode 100644 base/poco/XML/include/Poco/DOM/Text.h create mode 100644 base/poco/XML/include/Poco/DOM/TreeWalker.h create mode 100644 base/poco/XML/include/Poco/SAX/Attributes.h create mode 100644 base/poco/XML/include/Poco/SAX/AttributesImpl.h create mode 100644 base/poco/XML/include/Poco/SAX/ContentHandler.h create mode 100644 base/poco/XML/include/Poco/SAX/DTDHandler.h create mode 100644 base/poco/XML/include/Poco/SAX/DeclHandler.h create mode 100644 base/poco/XML/include/Poco/SAX/DefaultHandler.h create mode 100644 base/poco/XML/include/Poco/SAX/EntityResolver.h create mode 100644 base/poco/XML/include/Poco/SAX/EntityResolverImpl.h create mode 100644 base/poco/XML/include/Poco/SAX/ErrorHandler.h create mode 100644 base/poco/XML/include/Poco/SAX/InputSource.h create mode 100644 base/poco/XML/include/Poco/SAX/LexicalHandler.h create mode 100644 base/poco/XML/include/Poco/SAX/Locator.h create mode 100644 base/poco/XML/include/Poco/SAX/LocatorImpl.h create mode 100644 base/poco/XML/include/Poco/SAX/NamespaceSupport.h create mode 100644 base/poco/XML/include/Poco/SAX/SAXException.h create mode 100644 base/poco/XML/include/Poco/SAX/SAXParser.h create mode 100644 base/poco/XML/include/Poco/SAX/WhitespaceFilter.h create mode 100644 base/poco/XML/include/Poco/SAX/XMLFilter.h create mode 100644 base/poco/XML/include/Poco/SAX/XMLFilterImpl.h create mode 100644 base/poco/XML/include/Poco/SAX/XMLReader.h create mode 100644 base/poco/XML/include/Poco/XML/Content.h create mode 100644 base/poco/XML/include/Poco/XML/Name.h create mode 100644 base/poco/XML/include/Poco/XML/NamePool.h create mode 100644 base/poco/XML/include/Poco/XML/NamespaceStrategy.h create mode 100644 base/poco/XML/include/Poco/XML/ParserEngine.h create mode 100644 base/poco/XML/include/Poco/XML/QName.h create mode 100644 base/poco/XML/include/Poco/XML/ValueTraits.h create mode 100644 base/poco/XML/include/Poco/XML/XML.h create mode 100644 base/poco/XML/include/Poco/XML/XMLException.h create mode 100644 base/poco/XML/include/Poco/XML/XMLStream.h create mode 100644 base/poco/XML/include/Poco/XML/XMLStreamParser.h create mode 100644 base/poco/XML/include/Poco/XML/XMLStreamParserException.h create mode 100644 base/poco/XML/include/Poco/XML/XMLString.h create mode 100644 base/poco/XML/include/Poco/XML/XMLWriter.h create mode 100644 base/poco/XML/include/Poco/XML/expat.h create mode 100644 base/poco/XML/include/Poco/XML/expat_external.h create mode 100644 base/poco/XML/src/AbstractContainerNode.cpp create mode 100644 base/poco/XML/src/AbstractNode.cpp create mode 100644 base/poco/XML/src/Attr.cpp create mode 100644 base/poco/XML/src/AttrMap.cpp create mode 100644 base/poco/XML/src/Attributes.cpp create mode 100644 base/poco/XML/src/AttributesImpl.cpp create mode 100644 base/poco/XML/src/CDATASection.cpp create mode 100644 base/poco/XML/src/CharacterData.cpp create mode 100644 base/poco/XML/src/ChildNodesList.cpp create mode 100644 base/poco/XML/src/Comment.cpp create mode 100644 base/poco/XML/src/ContentHandler.cpp create mode 100644 base/poco/XML/src/DOMBuilder.cpp create mode 100644 base/poco/XML/src/DOMException.cpp create mode 100644 base/poco/XML/src/DOMImplementation.cpp create mode 100644 base/poco/XML/src/DOMObject.cpp create mode 100644 base/poco/XML/src/DOMParser.cpp create mode 100644 base/poco/XML/src/DOMSerializer.cpp create mode 100644 base/poco/XML/src/DOMWriter.cpp create mode 100644 base/poco/XML/src/DTDHandler.cpp create mode 100644 base/poco/XML/src/DTDMap.cpp create mode 100644 base/poco/XML/src/DeclHandler.cpp create mode 100644 base/poco/XML/src/DefaultHandler.cpp create mode 100644 base/poco/XML/src/Document.cpp create mode 100644 base/poco/XML/src/DocumentEvent.cpp create mode 100644 base/poco/XML/src/DocumentFragment.cpp create mode 100644 base/poco/XML/src/DocumentType.cpp create mode 100644 base/poco/XML/src/Element.cpp create mode 100644 base/poco/XML/src/ElementsByTagNameList.cpp create mode 100644 base/poco/XML/src/Entity.cpp create mode 100644 base/poco/XML/src/EntityReference.cpp create mode 100644 base/poco/XML/src/EntityResolver.cpp create mode 100644 base/poco/XML/src/EntityResolverImpl.cpp create mode 100644 base/poco/XML/src/ErrorHandler.cpp create mode 100644 base/poco/XML/src/Event.cpp create mode 100644 base/poco/XML/src/EventDispatcher.cpp create mode 100644 base/poco/XML/src/EventException.cpp create mode 100644 base/poco/XML/src/EventListener.cpp create mode 100644 base/poco/XML/src/EventTarget.cpp create mode 100644 base/poco/XML/src/InputSource.cpp create mode 100644 base/poco/XML/src/LexicalHandler.cpp create mode 100644 base/poco/XML/src/Locator.cpp create mode 100644 base/poco/XML/src/LocatorImpl.cpp create mode 100644 base/poco/XML/src/MutationEvent.cpp create mode 100644 base/poco/XML/src/Name.cpp create mode 100644 base/poco/XML/src/NamePool.cpp create mode 100644 base/poco/XML/src/NamedNodeMap.cpp create mode 100644 base/poco/XML/src/NamespaceStrategy.cpp create mode 100644 base/poco/XML/src/NamespaceSupport.cpp create mode 100644 base/poco/XML/src/Node.cpp create mode 100644 base/poco/XML/src/NodeAppender.cpp create mode 100644 base/poco/XML/src/NodeFilter.cpp create mode 100644 base/poco/XML/src/NodeIterator.cpp create mode 100644 base/poco/XML/src/NodeList.cpp create mode 100644 base/poco/XML/src/Notation.cpp create mode 100644 base/poco/XML/src/ParserEngine.cpp create mode 100644 base/poco/XML/src/ProcessingInstruction.cpp create mode 100644 base/poco/XML/src/QName.cpp create mode 100644 base/poco/XML/src/SAXException.cpp create mode 100644 base/poco/XML/src/SAXParser.cpp create mode 100644 base/poco/XML/src/Text.cpp create mode 100644 base/poco/XML/src/TreeWalker.cpp create mode 100644 base/poco/XML/src/ValueTraits.cpp create mode 100644 base/poco/XML/src/WhitespaceFilter.cpp create mode 100644 base/poco/XML/src/XMLException.cpp create mode 100644 base/poco/XML/src/XMLFilter.cpp create mode 100644 base/poco/XML/src/XMLFilterImpl.cpp create mode 100644 base/poco/XML/src/XMLReader.cpp create mode 100644 base/poco/XML/src/XMLStreamParser.cpp create mode 100644 base/poco/XML/src/XMLStreamParserException.cpp create mode 100644 base/poco/XML/src/XMLString.cpp create mode 100644 base/poco/XML/src/XMLWriter.cpp create mode 100644 base/poco/XML/src/ascii.h create mode 100644 base/poco/XML/src/asciitab.h create mode 100644 base/poco/XML/src/expat_config.h create mode 100644 base/poco/XML/src/iasciitab.h create mode 100644 base/poco/XML/src/internal.h create mode 100644 base/poco/XML/src/latin1tab.h create mode 100644 base/poco/XML/src/nametab.h create mode 100644 base/poco/XML/src/siphash.h create mode 100644 base/poco/XML/src/utf8tab.h create mode 100644 base/poco/XML/src/xmlparse.cpp create mode 100644 base/poco/XML/src/xmlrole.c create mode 100644 base/poco/XML/src/xmlrole.h create mode 100644 base/poco/XML/src/xmltok.c create mode 100644 base/poco/XML/src/xmltok.h create mode 100644 base/poco/XML/src/xmltok_impl.c create mode 100644 base/poco/XML/src/xmltok_impl.h create mode 100644 base/poco/XML/src/xmltok_ns.c create mode 100644 base/poco/patches/AIX/xlC8.diff delete mode 160000 contrib/poco delete mode 100644 contrib/poco-cmake/Crypto/CMakeLists.txt delete mode 100644 contrib/poco-cmake/Data/CMakeLists.txt delete mode 100644 contrib/poco-cmake/Data/ODBC/CMakeLists.txt delete mode 100644 contrib/poco-cmake/Foundation/CMakeLists.txt delete mode 100644 contrib/poco-cmake/JSON/CMakeLists.txt delete mode 100644 contrib/poco-cmake/MongoDB/CMakeLists.txt delete mode 100644 contrib/poco-cmake/Net/CMakeLists.txt delete mode 100644 contrib/poco-cmake/Net/SSL/CMakeLists.txt delete mode 100644 contrib/poco-cmake/Redis/CMakeLists.txt delete mode 100644 contrib/poco-cmake/Util/CMakeLists.txt delete mode 100644 contrib/poco-cmake/XML/CMakeLists.txt delete mode 100644 src/Functions/examples/CMakeLists.txt diff --git a/.gitmodules b/.gitmodules index 13b1b2035be..0b88bd616fb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,3 @@ -[submodule "contrib/poco"] - path = contrib/poco - url = https://github.com/ClickHouse/poco - branch = clickhouse [submodule "contrib/zstd"] path = contrib/zstd url = https://github.com/facebook/zstd diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt index dcf8f650262..9c47d63e9a2 100644 --- a/base/CMakeLists.txt +++ b/base/CMakeLists.txt @@ -1,8 +1,5 @@ -if (USE_CLANG_TIDY) - set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") -endif () - add_subdirectory (base) add_subdirectory (pcg-random) +add_subdirectory (poco) add_subdirectory (widechar_width) add_subdirectory (readpassphrase) diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt index 5d9f1de8309..64785d575c5 100644 --- a/base/base/CMakeLists.txt +++ b/base/base/CMakeLists.txt @@ -1,3 +1,7 @@ +if (USE_CLANG_TIDY) + set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") +endif () + set (SRCS argsToConfig.cpp coverage.cpp diff --git a/base/pcg-random/CMakeLists.txt b/base/pcg-random/CMakeLists.txt index 88acabba6a7..005b8dfa61f 100644 --- a/base/pcg-random/CMakeLists.txt +++ b/base/pcg-random/CMakeLists.txt @@ -1,2 +1,6 @@ +if (USE_CLANG_TIDY) + set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") +endif () + add_library(pcg_random INTERFACE) target_include_directories(pcg_random INTERFACE .) diff --git a/base/poco/CHANGELOG b/base/poco/CHANGELOG new file mode 100644 index 00000000000..a10371b381c --- /dev/null +++ b/base/poco/CHANGELOG @@ -0,0 +1,2791 @@ +This is the changelog file for the POCO C++ Libraries. + +Release 1.9.3 (2019-08-20) +========================== + +- fixed GH #2603: Remove incorrect upper size limits for SSL certificates in NetSSL_Win +- fixed GH #2661: Poco::Zip::ZipArchive cannot load new tomcat.zip file (additional fix) +- fixed GH #2742: Support of vs150 & vs160 with the official Microsoft localization executable, + vswhere.exe, installed by MSVC starting from VS2017 +- Data/ODBC: make binding of std::string configurable (SQL_LONGVARCHAR - default or SQL_VARCHAR) + through a global setting (Poco::Data::ODBC::Connector::bindStringToLongVarChar()). +- added Poco::SharedLibrary::setSearchPath() (currently implemented on Windows only) +- Windows required minimum version is now Windows XP SP2 +- upgraded bundled SQLite to 3.29.0 +- CppParser now supports type aliases defined with using keyword. +- PageCompiler: added support for adding Content-Security-Policy and Cache-Control headers. + + +Release 1.9.2 (2019-07-02) +========================== + +- fixed GH #2736: Error using CMake gui - latest version +- fixed GH #2737: Bundles vulnerable copy of Expat — please upgrade to Expat 2.2.7 +- fixed GH #2738: Poco::AccessExpireStrategy::onGet() must not extend expiration time after expiration + + +Release 1.9.1 (2019-06-24) +========================== + +- Added support for building with different OpenSSL distributions on Windows. + See the POCO_EXTERNAL_OPENSSL macro defined in Foundation/include/Poco/Config.h + for options. +- Added Poco::Net::HTTPClientSession::flushRequest() +- Added Poco::Net::WebSocket::setMaxPayloadSize() and Poco::Net::WebSocket::getMaxPayloadSize() + to specify a maximum acceptable payload size for Poco::Net::WebSocket::receiveFrame(). +- Poco::Net::WebSocket: don't attempt to send empty credentials in response to 401 response. +- Redis: added support for additional commands (exists, expire, ping, multi, exec, discard) +- Redis: added Poco::Redis::Client::isConnected() +- Upgraded bundled PCRE to version 8.43 +- Upgraded bundled SQLite to version 3.28.0 +- Added project/solution files for Visual Studio 2019 +- Fixed Visual Studio project files (version information from DLLVersion.rc not included in DLLs) +- Include version resource in DLLs built with CMake +- Added HTTP*Credentials::empty() and HTTPCredentials::clear() +- fixed GH #2220: Encoding/DoubleByteEncoding.cpp fails to compile with VS2008 and _DEBUG +- fixed GH #2243: DLLVersion.rc is excluded from build, missing detail information in properties of *.dll +- fixed GH #2277: SQLite null pointer dereference occurs when exception is being thrown +- fixed GH #2313: PollSet behaves differently on windows +- fixed GH #2316: cmake can't find MySQL and ODBC libraries +- fixed GH #2336: Omit ContentLength in WebSocket accept response +- fixed GH #2358: Don't include for later OpenSSL +- fixed GH #2364: Stringify escapes every unicode symbol when object contain an array +- fixed GH #2380: Calling Poco::Net::X509Certificate::addChainCertificate() leads to double free. +- fixed GH #2492: Net::Socket::address() crash on Android +- fixed GH #2549: Fix keepAlive in http client session +- fixed GH #2565: HTMLForm: optional enforcement of Content-Length instead of Chunked Transfer-Encoding +- fixed GH #2570: DialogSocket: receiveStatusMessage() - line length limit applies to entire multi-line message +- fixed GH #2583: Crypto library does not build with OpenSSL 1.0.0 +- fixed GH #2655: MongoDB Binary element to string - bug +- fixed GH #2661: Poco::Zip::ZipArchive cannot load new tomcat.zip file +- fixed GH #2700: Invalid read of memory in Poco::Environment::set which may cause crashes. +- fixed GH #2712: File_WIN32.cpp(168): error C2065: "_upath":Undeclared identifier +- fixed GH #2723: Access violation when trying to decompress .zip file with unsupported compression method. + + +Release 1.9.0 (2018-03-07) +========================== + +- Added additional text encodings, available in the new PocoEncodings library (GH #2165) +- Added Punycode support for resolving Internationalized Domain Names to Poco::Net::DNS (GH #2164) +- Added XDG Base Directory Specification support in Poco::Path, Poco::Util::SystemConfiguration + and Poco::Util::Application (GH #1609, GH #561, GH #1609) +- Added support for GCM ciphers in Crypto library (GH #2129) +- Poco::Net::RemoteSyslogChannel and Poco::Net::RemoteSyslogListener now have basic + support for RFC 5424 structured data (GH #2173) +- Poco::File now has methods for obtaining partition space (GH #1545) +- Added Poco::Net::Context::addCertificateAuthority() (GH #2197) +- Added Poco::AutoPtr::reset() and Poco::SharedPtr::reset() to improve compatibility + with std::shared_ptr() (GH #2172) +- fixed GH #703: Poco::Data::SQLite data types (INTEGER is now mapped to Int64) +- fixed GH #1426: inttypes.h is available on sun solaris +- fixed GH #1912: Run ping with custom data size #1912 +- fixed GH #2177: Run ping with custom timeout instead of the default +- fixed GH #2058 and GH #2095: Synchronization issue/deadlock in Poco::Util::Timer at destruction +- fixed GH #2089: Allow hyphen in HTTP authentication scheme names +- fixed GH #2106: Undefined behavior in Delegate::equals() +- fixed GH #2108: POCO SQLite Data Connector hangs for 20 secs waiting for thread timeouts on Windows +- fixed GH #2142: JSON::Object preserveOrder keys not synced on assignment +- fixed GH #2199 and GH #2188: Illegal header in zip file exception/assertion failure during JAR file decompression +- fixed GH #2203: Use MAX_ADDRESS_LENGTH to determine buffer size + + +Release 1.8.1 (2018-01-09) +========================== + +- Added Poco::File::linkTo() +- fixed GH #2044: Poco::Net::NetworkInterface::list does not list inactive interfaces + even when explicitly being asked for it. +- fixed GH #2042: Android abstract namespace local socket address +- fixed GH #2038: Poco::Net::MultipartWriter::createBoundary() always returns the same string. +- fixed GH #2020: SQLite not handling parameter count mismatch correctly. +- fixed GH #2012: Data/SQLite: Exception messages contain duplicate text +- fixed GH #2005: Upgraded bundled PCRE to 8.41 +- fixed GH #2000: Fix building XMLStreamParser with unbundled expat +- fixed GH #1603: fix MinGW 4.8.2 Compilation +- fixed GH #1991: Support building poco 1.8.0 as cmake sub-project +- fixed GH #2080: Bugs in Poco::Net::Socket::select when POCO_HAVE_FD_POLL is defined + + +Release 1.8.0.1 (2017-11-11) +============================ + +- Reverted change for GH #1828; DeflatingStreamBuf::sync() no longer flushes + underlying stream as this causes corruption for some Zip files. +- PocoDoc: fix for handling compiler configuration for Gradle builds. + + +Release 1.8.0 (2017-11-10) +========================== + +- Poco::Base64Encoder: add support for base64url encoding (GH #1967) +- Add Poco::Net::PollSet class to Net library (GH #1763) +- The Net library now supports Unix Domain Sockets, where available. +- Added stream parser (Poco::XML::XMLStreamParser) to XML library (GH #1697) +- Added Poco::Net::TCPServerConnectionFilter and Poco::Net::TCPServer::setConnectionFilter() + to support connection filtering and IP blacklisting (GH #1485) +- Added Redis library (GH #1383) +- Added Zip64 support to Zip library (GH #1356) +- Upgraded bundled SQLite to 3.21.0 +- Removed OpenVMS support (GH #1988) +- fixed GH #271: NamedMutex_UNIX.cpp must remove semid +- fixed GH #739: Add WebSocket::receiveFrame() that appends to a Poco::Buffer +- fixed GH #749: NTP Packet impl not according to RFC958 +- fixed GH #896: Sample "TwitterClient" of NetSSL_OpenSSL can't be build +- fixed GH #1172: Poco::Data default storage should be std::vector +- fixed GH #1337: Poco::HTMLForm throws exception HTMLFormException("Form must be prepared") + even after form is prepared. +- fixed GH #1373: SessionImpl::close() does not check return code of close handle specific function +- fixed GH #1425: Workaround bug in SolarisStudio 12.4 on RVO-ed objects. +- fixed GH #1614: Problematic license for JSON component: the previously used JSON.org parser + has been replaced with pdjson +- fixed GH #1659: wrong field size calculation in ODBC code +- fixed GH #1683: Poco::Data ODBC impl doesn't bind to unsigned numeric types properly +- fixed GH #1705: MongoDB: support URI in Connection +- fixed GH #1708: "SocketReactor::addEventHandler" and "SocketReactor::removeEventHandler" + must protect the access to "NotifierPtr pNotifier" +- fixed GH #1729: getConnectionTimeout of SQLite DB wrapper returns wrong value + (in milliseconds, should be in seconds) +- fixed GH #1739: OpenSSLInitializer isn't threadsafe +- fixed GH #1750: double_conversion in NumericString is in conflict with Qt5 Core +- fixed GH #1804 and GH #1805: Integer Overflow or Wraparound +- fixed GH #1828: DeflatingStreamBuf::sync() should also flush underlying stream. +- fixed GH #1880: FTPClientSession::close() error +- fixed GH #1897: DateTime wrong binding/extraction for MySQL database +- fixed GH #1905: Compiling Foundation library with POCO_NO_FPENVIRONMENT in Config.h fails +- fixed GH #1906: Race condition in ThreadPool +- fixed GH #1913: Message Doesn't Support 64-bit Thread IDs +- fixed GH #1921: ICMPSocket does not check reply address +- fixed GH #1926: Exception when using SortedDirectoryIterator +- fixed GH #1934: File::setExecutable() on POSIX should set executable bit for group and + other if corresponding readable bit is set +- fixed GH #1950: Net Exception: Address family not supported with clang +- fixed GH #1964: Buffer<> swap miss ownMem + + +Release 1.7.9p2 (2017-11-06) +============================ + +- fixed GH #1628: Export Poco::Zip::ZipUtil class + + +Release 1.7.9p1 (2017-11-02) +============================ + +- fixed GH #1968: Zip Decompress Parent Path Injection + + +Release 1.7.9 (2017-09-11) +========================== + +- fixed GH #1813: xmlparse.cpp doesn't compile in WinCE (poco 1.7.8p3) +- fixed GH #1826: XPath query error +- fixed GH #1834: Visual Studio 2008 cannot find stdint.h +- fixed GH #1842: Upgrade bundled expat to 2.2.3 +- fixed GH #1843: Use random salt for Poco::XML::NamePool +- fixed GH #1865: AbstractEvent::hasDelegates() is not thread-safe +- improved/fixed QNX support +- Poco::Util::LayeredConfiguration: added support for labelling configurations and + finding them by their label +- upgraded bundled SQLite to 3.20.1 +- PageCompiler: support <%@ include file="" %> syntax for includes, in addition + to <%@ include page="" %> +- PageCompiler: optimize generated request handler code by removing useless + statements, e.g. writing empty strings. +- added POCO_DEPRECATED macro which will be used in the future to deprecate + classes and methods. +- Poco::NamedMutex and Poco::NamedEvent (System V Semaphores implementation): files are + now opened with O_RDONLY | O_CREAT instead of O_WRONLY | O_CREAT, allowing sharing + between different users. Furthermore, ftok() is called with 'p' as project ID + argument. + + +Release 1.7.8p3 (2017-06-22) +============================ + +- fixed GH #1760: Upgrade bundled expat to 2.2.1 which fixes some vulnerabilities: + http://seclists.org/oss-sec/2017/q2/499 + + +Release 1.7.8p2 (2017-04-18) +============================ + +- fixed GH #1655: CipherImpl memory leak with OpenSSL 1.1 + + +Release 1.7.8 (2017-02-21) +========================== + +- fixed GH #1212: Lost WebSocket Frames after Client Websocket Handshake is complete +- fixed GH #1260: URI encoding +- fixed GH #1501: Alpine 3.4 trouble with Foundation/src/Error.cpp +- fixed GH #1523: Long path names under Windows +- fixed GH #1536: Building with OS X 10.12 SDK and 10.7 deployment target without libc++ fails +- fixed GH #1537: Need to add multiple cflags parameters to configure +- fixed GH #1539: Allow overriding POCO_TARGET_OSARCH for iPhoneSimulator +- fixed GH #1546: Enable bitcode for iPhone build config +- fixed GH #1549: Latin2Encoding and 0xFF +- fixed GH #1551: Unable to use Poco on macOS 10.12 +- fixed GH #1552: IPv6 & operator throws an exception when scope = 0 +- fixed GH #1566: Poco/Zip issue with some CM_DEFLATE archives +- fixed GH #1567: Poco/ZIP issue with uncompressed archives +- fixed GH #1570: IPv6AddressImpl::toString() returns wrong output for IPv6 address "::" +- fixed GH #1571: ODBC Preparator memory leak +- fixed GH #1573: Poco::File::createDirectories() should not throw Poco::FileExistsException +- fixed GH #1580: Unable to unzip zip file created using non-seeking stream +- fixed GH #1581: Cannot find 'pcre.h' when using POCO_UNBUNDLED, a non-system PCRE, and CMake +- fixed GH #1588: Poco::Net::HTTPChunkedStreamBuf::readFromDevice(): restrict maximum + size of chunk length +- fixed GH #1589: Poco::Net::HTMLForm: restrict maximum field and value length +- fixed GH #1590: Poco::Net::DialogSocket: restrict maximum line length +- fixed GH #1591: Poco::Net::MultipartReader: restrict maximum boundary string length +- fixed GH #1597: adding empty file to zip leads to archive that can't be unzipped by windows +- fixed GH #1599: readFromDevice() in AutoDetectStream.cpp in Poco Zip cannot detect signature +- fixed GH #1534: Upgraded bundled zlib to 1.2.11 +- fixed GH #1558: Upgraded bundled SQLite to 3.16.2 +- fixed GH #1586: Upgraded bundled PCRE to 8.40 +- fixed GH #1538: Upgraded bundled double-conversion to 1.1.5 +- MongoDB: added support for authentication using "MONGODB-CR" and "SCRAM-SHA-1" + authentication schemes. +- MongoDB: additional documentation and fixes for style and consistency and minor + API improvements (e.g., Poco::MongoDB::Binary) + Note: some flag enumeration values have been renamed for better consistency + and readability; existing code using these will have to be updated. + + +Release 1.7.7 (2016-12-31) +========================== + +- fixed GH #865: FileChannel compress fails leaving empty .gz files +- fixed GH #990: Potential race condition in Poco::File on Windows +- fixed GH #1157: Fixing a bug in the NetSSL_Win module (Host name verification failed error) +- fixed GH #1351: Fix for android include pthread.h from /usr/include +- fixed GH #1436: ODBC Bug: Unicode text(NVARCHAT) read from DB is truncated to half +- fixed GH #1453: _clock_gettime Symbol not found on Mac 10.11 +- fixed GH #1460: POCO does not build with OpenSSL 1.1 +- fixed GH #1461: Poco::Data::SQLite::SQLiteStatementImpl::next() error +- fixed GH #1462: AbstractConfiguration::getUInt does not parse hex numbers +- fixed GH #1464: ODBCMetaColumn::init() always maps integer NUMERIC/DECIMAL to Int32 +- fixed GH #1465: Assertion violation in DateTime.cpp using ZipArchive +- fixed GH #1472: HTTP(S)StreamFactory should send a User-Agent header. +- fixed GH #1476: Fixed error with Poco::UTF8Encoding::isLegal() +- fixed GH #1484: ODBC: fix uninitialized variable +- fixed GH #1486: Support ODBC GUID data type as string +- fixed GH #1488: Poco::ObjectPool shrinks if returned object is not valid +- fixed GH #1515: Detection of closed websocket connection +- fixed GH #1521: bug in JSON ParseHandler.cpp (empty keys should be valid) +- fixed GH #1526: iOS app rejected, IPv6 not working +- fixed GH #1532: RecordSet and RowFilter: bad use of reference counter + + +Release 1.7.6 (2016-10-18) +========================== + +- fixed GH #1298: ZipFileInfo: Assertion violation when reading ods files +- fixed GH #1315: Redefine Poco assertions for static analysis +- fixed GH #1397: Fix issues reported by static source code analysis +- fixed GH #1403: Android compile with poco-1.7.5 no 'pthread_condattr_setclock' error +- fixed GH #1416: Assertion violation when unzipping +- fixed GH #1418: Poco::Delegate assignment operator fails to compile for some specializations +- fixed GH #1422: Can't build poco 1.7.4 or 1.7.5 on centos5 32 bit +- fixed GH #1429: exception thrown in MongoDB when using replicaset +- fixed GH #1431: Poco/FIFOBuffer.h copy issue +- fixed GH #1445: Use stable_sort to preserve order of IP addresses from DNS +- fixed GH #1456: better handle leap seconds in Poco::DateTime and Poco::LocalDateTime +- fixed GH #1458: Probably invalid epoll_create() usage inside Poco/Socket.cpp +- Poco::XML::NamePool: increased default size from 251 to 509. Default size can now + be changed by defining the POCO_XML_NAMEPOOL_DEFAULT_SIZE macro accordingly. +- Enchancements: Poco::XML::Document and Poco::XML::DOMParser have new constructors + taking a NamePool size. Poco::Util::XMLConfiguration::load() also has a new overload + for that purpose. +- Improved error handling in the Zip library (getting rid of some poco_assert macros + and did proper error handling instead). +- Added Poco::URISyntaxException (subclass of Poco::SyntaxException), which is now + thrown by Poco::URI. +- Improved error handling in Poco::URIStreamOpener::open(). +- Poco::Data::MySQL: Handle connection lost/server gone error when starting a transaction + and retry. +- XMLConfiguration default (and single-argument delimiter) constructor now loads an empty + XML document with "config" root element to make the configuration usable without an + additional call to load() or loadEmpty(). + + +Release 1.7.5 (2016-08-29) +========================== + +- fixed GH #1252: Unable to compile Poco::Data for Windows Compact Embedded 2013 +- fixed GH #1344: Poco::Event::wait(timeout) should use CLOCK_MONOTONIC on Linux +- fixed GH #1355: [JSON::Object] After copy-ctor, JSON::Object::_keys still points to + keys in map of copied object +- GH #1361: Shell expansion rules say that tilde must be replaced with $HOME before + calling getpwuid +- Poco::SingletonHolder: added reset() method +- prefer clock_getttime() over gettimeofday() if available +- Upgraded bundled SQLite to 3.14.1 + + +Release 1.7.4 (2016-07-20) +========================== + +- fixed GH #1300: Session constructor hangs +- fixed GH #1303: HTTPSClientSession::sendRequest() fails if server has wildcard cert +- fixed GH #1304: URI doesn't know "ws:/" or "wss://" schemes +- fixed GH #1307: Upgrade bundled expat to 2.2.0 +- fixed GH #1313: XML library compilation error +- fixed GH #1316: Empty SocketReactor never sleeps +- Upgraded bundled SQLite to 3.13.0 + + +Release 1.7.3 (2016-05-02) +========================== + +- fixed GH #993: Invalid zip format when opening a docx in word +- fixed GH #1235: Poco::Net::HTTPClientSession::sendRequest() should also handle HTTP_PATCH +- fixed GH #1236: Remove Poco::Data::Row::checkEmpty() as it prevents Row from being used + with all NULL rows +- fixed GH #1239: Poco::Zip::Compress with non-seekable stream fails for CM_STORE +- fixed GH #1242: Poco::Data::RowFormatter generate exception if the first column of first + row is null +- fixed GH #1253: ListMap does not maintain insertion order if key already exists +- Upgraded bundled SQLite to 3.12.2 + + +Release 1.7.2 (2016-03-21) +========================== + +- fixed GH #1197: Upgrade bundled expat to 2.1.1 + Expat 2.1.1 fixes a CVE: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-1283 +- fixed GH #1204: getdtablesize has been removed on Android 21 +- fixed GH #1203: Poco::Data::RecordSet should be reusable +- fixed GH #1198: Upgrade bundled SQLite to 3.12.1 + + +Release 1.7.1 (2016-03-14) +========================== + +- fixed GH #1187: Data/MySQL: Seeing frequent "MySQL server has gone away" errors +- fixed GH #1184: Attempting to connect via a proxy throws a DNS error "Host not found" +- fixed GH #1180: Possible deadlock when TaskManager::count() is called in onFinished +- NetSSL_OpenSSL: use TLS_*_method() instead of deprecated SSLv23_*_method() + if OpenSSL version is >= 1.1; initialize default/fallback client context to support + all TLS protocols, not just TLSv1 + + +Release 1.7.0 (2016-03-07) +========================== + +- POSSIBLE BREAKING CHANGE: removed automatic registration of Data connectors due to + issues with static initialization order. +- NetSSL_OpenSSL: added support for ECDH and DH ciphers; added support to disable + specific protocols (Poco::Net::Context::disableProtocols()); + new Poco::Net::Context constructor taking a Poco::Net::Context::Params structure that + allows specifying ECDH and DH parameters. +- Poco::Net::TCPServer: add additional try ... catch block around poll() to + gracefully deal with errors due to high system load (e.g., out of file descriptors). +- fixed GH #1171: Poco::Data::RecordSet: rowCount not reset after execute +- fixed GH #1167: CMake & POCO_UNBUNDLED: expat sources are compiled in libPocoXML +- fixed GH #1160: Poco::Net::NetException + "SSL Exception: error:1409F07F:SSL routines:ssl3_write_pending:bad write retry" +- fixed GH #1152: Wrong TaskProgressNotification description +- fixed GH #1141: Poco::StringTokenizer::TOK_TRIM changes behavior between 1.4 and 1.6 +- fixed GH #1137: Missing 'longint' type in SQLite +- fixed GH #1135: Different package on github and official web site +- fixed GH #1030: tvOS / WatchOS bitcode enabled for simulators +- fixed GH #1114: World-write permissions on files created by daemon +- fixed GH #1087: prevent line breaks in base64-encoded creds +- fixed GH #1026: Fixes for producing the poco-1.6.2 release on a Cygwin x86 platform +- fixed GH #1022: Abbreviation in setThreadName can happen even if thread name is not too long +- fixed GH #1002: ActiveDispatcher saves reference to event context after event was + performed until it gets new event +- fixed GH #973: overwrite existing files on windows when moving files +- fixed GH #969: Poco::File::renameTo() behaviour differs on windows and linux +- fixed GH #967: Missing data types in SQLite +- fixed GH #966: Possible crash when processing a corrupted Zip file +- fixed GH #958: Bug while reading X509Certificate subjectName +- fixed GH #937: Missing build_vs140.cmd +- fixed GH #933: Change in JSON::Object::set(key,value) behavior in 1.6.1 +- fixed GH #931: make strToInt() more strict in what it accepts +- fixed GH #921: `BasicUnbufferedStreamBuf` needs to be marked for import/export +- fixed GH #848: MailMessage::_encoding is not set when retrieving plain/text message +- fixed GH #767: Inconsistency in getPath & getPathAndQuery returns +- fixed GH #724: Poco 1.6.0 is not compiled with openssl 1.0.0 +- fixed GH #713: Improved support for producing Canonical XML in XMLWriter +- fixed GH #696: bug in parsing name of attachment poco c++ 1.6.0 +- fixed GH #335: Compress with nonseekable +- upgraded bundled SQLite to 3.11.0 +- added Poco::Crypto::X509Certificate::equals() to compare two certificates +- support for detecting Win8/Win10 in Poco::Environment +- Poco::Net::HTTPServerRequestImpl: fixed an issue with DELETE in persistent connections +- NetSSL: added Context::preferServerCiphers() +- NetSSL: added support for ECDH, new Context constructor +- NetSSL: add support for disabling certain protocols +- SMTPClientSession: added support for XOAUTH2 authentication +- Poco::Data::SessionPool: re-added customizeSession() method from 1.4.x releases +- improved SSLManager to automatically set-up a reasonable client Context if + none is configured +- add brew OpenSSL search paths to Darwin configs +- add HTTP/1.1 version to HTTPRequest for client WebSocket, as this is required for + most servers +- remove GCC_DIAG_OFF as this caused more issues than it solved +- respect POCO_NO_FORK_EXEC in ServerApplication (tvOS) +- tvOS and WatchOS support +- fix: need an implementation of available() for WebSocketImpl +- HTTPSessionInstantiator: respect global proxy config +- added constant for HTTP PATCH method to Poco::Net::HTTPRequest +- NumberParser::parseHex[64](): allow 0x/0X prefix + + +Release 1.6.1 (2015-08-03) +========================== + +- added project and solution files for Visual Studio 2015 +- upgraded bundled SQLite to 3.8.11.1 +- fixed GH #782: Poco::JSON::PrintHandler not working for nested arrays +- fixed GH #819: JSON Stringifier fails with preserve insert order +- fixed GH #878: UUID tryParse +- fixed GH #869: FIFOBuffer::read(T*, std::size_t) documentation inaccurate +- fixed GH #861: Var BadCastException +- fixed GH #779: BUG in 1.6.0 Zip code +- fixed GH #769: Poco::Var operator== throws exception +- fixed GH #766: Poco::JSON::PrintHandler not working for objects in array +- fixed GH #763: Unable to build static with NetSSL_OpenSSL for OS X +- fixed GH #750: BsonWriter::write missing size ? +- fixed GH #741: Timestamp anomaly in Poco::Logger on WindowsCE +- fixed GH #735: WEC2013 build fails due to missing Poco::Path methods. +- fixed GH #722: poco-1.6.0: Unicode Converter Test confuses string and char types +- fixed GH #719: StreamSocket::receiveBytes and FIFOBuffer issue in 1.6 +- fixed GH #706: POCO1.6 Sample EchoServer BUG +- fixed GH #646: Prevent possible data race in access to Timer::_periodicInerval +- DeflatingStream: do not flush underlying stream on sync() as these can cause + corrupted files in Zip archives + + +Release 1.6.0 (2014-12-22) +========================== + +- fixed GH #625: MongoDB ensureIndex double insert? +- fixed GH #622: Crypto: RSATest::testSign() should verify with public key only +- fixed GH #620: Data documentation sample code outdated +- fixed GH #618: OS X 10.10 defines PAGE_SIZE macro, conflicts with PAGE_SIZE in Thread_POSIX.cpp +- fixed GH #616: Visual Studio warning C4244 +- fixed GH #612: OpenSSLInitializer calls OPENSSL_config but not CONF_modules_free +- fixed GH #608: (Parallel)SocketAcceptor ctor/dtor call virtual functions +- fixed GH #607: Idle Reactor high CPU usage +- fixed GH #606: HTMLForm constructor read application/x-www-form-urlencoded UTF-8 request + body first parameter with BOM in name +- fixed GH #596: For OpenSSL 1.0.1, include openssl/crypto.h not openssl/fips.h +- fixed GH #592: Incorrect format string in Poco::Dynamic::Struct +- fixed GH #590: Poco::Data::SQlite doesn't support URI filenames +- fixed GH #564: URI::encode +- fixed GH #560: DateTime class calculates a wrong day +- fixed GH #549: Memory allocation is not safe between fork() and execve() +- fixed GH #500: SSLManager causes a crash +- fixed GH #490: 2 byte frame with payload length of 0 throws "Incomplete Frame Received" exception +- fixed GH #483: multiple cases for sqlite_busy +- fixed GH #482: Poco::JSON::Stringifier::stringify bad behaviour +- fixed GH #478: HTTPCredentials not according to HTTP spec +- fixed GH #471: vs2010 release builds have optimization disabled ? +- fixed GH #468: HTTPClientSession/HTTPResponse not forwarding exceptions +- fixed GH #438: Poco::File::setLastModified() doesn't work +- fixed GH #402: StreamSocket::receiveBytes(FIFOBuffer&) and sendBytes(FIFOBuffer&) are + not thread safe +- fixed GH #345: Linker warning LNK4221 in Foundation for SignalHandler.obj, String.obj + and ByteOrder.obj +- fixed GH #331: Poco::Zip does not support files with ".." in the name. +- fixed GH #318: Logger local time doesn't automatically account for DST +- fixed GH #294: Poco::Net::TCPServerParams::setMaxThreads(int count) will not accept count == 0. +- fixed GH #215: develop WinCE build broken +- fixed GH #63: Net::NameValueCollection::size() returns int +- Poco::Logger: formatting methods now support up to 10 arguments. +- added Poco::Timestamp::raw() +- Poco::DeflatingOutputStream and Poco::InflatingOutputStreams also flush underlying stream + on flush()/sync(). +- Poco::Util::Timer: prevent re-schedule of cancelled TimerTask +- enabled WinRegistryKey and WinRegistryConfiguration for WinCE +- Poco::BasicEvent improvements and preparations for future support of lambdas/std::function +- upgraded bundled sqlite to 3.8.7.2 +- Poco::Thread: added support for starting functors/lambdas +- Poco::Net::HTTPClientSession: added support for global proxy configuration +- added support for OAuth 1.0/2.0 via Poco::Net::OAuth10Credentials and + Poco::Net::OAuth20Credentials classes. +- Poco::Net::IPAddress: fixed IPv6 prefix handling issue on Windows +- added Poco::Timestamp::TIMEVAL_MIN and Poco::Timestamp::TIMEVAL_MAX +- added Poco::Clock::CLOCKVAL_MIN and Poco::Clock::CLOCKVAL_MAX +- added poco_assert_msg() and poco_assert_msg_dbg() macros +- Poco::Net::Context: fixed a memory leak if the CA file was not found while creating the + Context object (the underlying OpenSSL context would leak) +- Poco::URI: added new constructor to create URI from Path +- Various documentation and style fixes +- Removed support (project/solution files) for Visual Studio.NET 2003 and Visual Studio 2005. +- Improved CMake support + + +Release 1.5.4 (2014-10-14) +========================== + +- fixed GH #326: compile Net lib 1.5.2 without UTF8 support enabled +- fixed GH #518: NetworkInterface.cpp compile error w/ POCO_NO_WSTRING (1.5.3) +- Fixed MSVC 2010 warnings on large alignment +- make HTTPAuthenticationParams::parse() add value on end of string +- fixed GH #482: Poco::JSON::Stringifier::stringify bad behaviour +- fixed GH #508: Can't compile for arm64 architecture +- fixed GH #510: Incorrect RSAKey construction from istream +- fix SharedMemory for WinCE/WEC2013 +- Add NIOS2 double conversion detection, fixes compile errors +- added VS2013 project/solution files for Windows Embedded Compact 2013 +- added Process::isRunning() +- NetSSL: Fix typo in documentation +- NetSSL_OpenSSL: support for TLS 1.1 and 1.2 +- Zip: Added CM_AUTO, which automatically selects CM_STORE or CM_DEFLATE based + on file extension. Used to avoid double-compression of already compressed file + formats such as images. +- added %L modifier to PatternFormatter to switch to local time +- removed unnecessary explicit in some multi-arg constructors +- Allow SecureStreamSocket::attach() to be used in server connections +- added Var::isBoolean() and fixed JSON stringifier +- added poco_unexpected() macro invoking Bugcheck::unexpected() to deal + with unexpected exceptions in destructors +- fixed GH #538 prevent destructors from throwing exceptions +- improved HTTP server handling of errors while reading header +- fixed GH #545: use short for sign +- upgraded SQLite to 3.8.6 +- fixed GH #550 WebSocket fragmented message problem +- improved HTTPClientSession handling of network errors while sending the request +- updated bundled PCRE to 8.35.0 +- fixed GH #552: FIFOBuffer drain() problem +- fixed GH #402: StreamSocket::receiveBytes(FIFOBuffer&) and sendBytes(FIFOBuffer&) are + not thread safe +- HTTPCookie: fix documentation for max age +- added Timestamp::raw() and Clock::raw() +- Poco::Buffer properly handles zero-sized buffers +- GH #512: Poco:Data:ODBC:Binder.h causes a crash +- Added Crypto_Win and NetSSL_Win libraries which are re-implementations of existing + Crypto and NetSSL_OpenSSL libraries based on WinCrypt/Schannel. The new libraries + can be used as an almost drop-in replacement for the OpenSSL based libraries on + Windows and Windows Embedded Compact platforms. Only available from GitHub for now. + + +Release 1.5.3 (2014-06-30) +========================== + +- fixed GH# 316: Poco::DateTimeFormatter::append() gives wrong result for + Poco::LocalDateTime +- Poco::Data::MySQL: added SQLite thread cleanup handler +- Poco::Net::X509Certificate: improved and fixed domain name verification for + wildcard domains +- added Poco::Clock class, which uses a system-provided monotonic clock + (if available) and is thus not affected by system realtime clock changes. + Monotonic Clock is available on Windows, Linux, OS X and on POSIX platforms + supporting clock_gettime() and CLOCK_MONOTONIC. +- Poco::Timer, Poco::Stopwatch, Poco::TimedNotificationQueue and Poco::Util::Timer + have been changed to use Poco::Clock instead of Poco::Timestamp and are now + unaffected by system realtime clock changes. +- fixed GH# 350: Memory leak in Data/ODBC with BLOB +- Correctly set MySQL time_type for Poco::Data::Date. +- fixed GH #352: Removed redundant #includes and fixed spelling mistakes. +- fixed setting of MYSQL_BIND is_unsigned value. +- fixed GH #360: CMakeLists foundation: add Clock.cpp in the list of source files +- Add extern "C" around on HPUX platform. +- added runtests.sh +- fixed CPPUNIT_IGNORE parsing +- fixed Glob from start path, for platforms not alowing transverse from root (Android) +- added NTPClient (Rangel Reale) +- added PowerShell build script +- added SmartOS build support +- fix warnings in headers +- XMLWriter: removed unnecessary apostrophe escaping (&apos) +- MongoDB: use Int32 for messageLength +- fixed GH #380: SecureSocket+DialogSocket crashes with SIGSEGV when timeout occours +- Improve RSADigestEngine, using Poco::Crypto::DigestEngine to calculate hash before signing +- added Poco::PBKDF2Engine +- Fixed GH #380: SecureSocket+DialogSocket crashes with SIGSEGV when timeout occours +- added support for a 'Priority' attribute on cookies. +- GH #386: fixed bug in MailMessage without content-transfer-encoding header +- GH #384: ew hash algorithms support for RSADigestEngine +- fixed Clock overflow bug on Windows +- Poco::ByteOrder now uses intrinsics, if available +- CMake: added /bigobj option for msvc +- Fix typo to restore Net/TestSuite_x64_vs120 build +- correct path for CONFIGURE_FILE in CMakeLists.txt +- Building Poco 1.5.2 for Synology RS812+ (Intel Atom) (honor POCO_NO_INOTIFY) +- added WEC2013 support to buildwin.cmd and buildwin.ps1 +- HTMLForm: in URL encoding, percent-encode more characters +- Fixed #include conflict with other libraries +- Poco::Net::X509Certificate::verify() no longer uses DNS reverse lookups to validate host names +- cert hostname validation is case insensitive and stricter for wildcard certificates +- TCPServer: do not reduce the capacity of the default ThreadPool +- added POCO_LOG_DEBUG flag +- Zip: fixed a crash caused by an I/O error +- added runtest script for windows +- added SQlite Full Text Search support +- added Thread::trySleep() and Thread::wakeUp() +- fixed GH #410: Bug in JSON::Object.stringify() in 1.5.2 +- fixed GH #362: Defect in Var::parseString when there is no space between value and newline +- fixed GH #314: JSON parsing bug +- added GH #313: MetaColumn additions for Data::ODBC and Data::SQLite +- fixed GH #346: Make Poco::Data::Date and Poco::Data::Time compare functions const. +- fixed GH #341: Compiling poco-1.5.2 for Cygwin +- fixed GH #305: There are bugs in Buffer.h +- fixed GH #321: trivial build fixes (BB QNX build) +- fixed GH #440: MongoDB ObjectId string formatting +- added SevenZip library (Guenter Obiltschnig) +- fixed GH #442: Use correct prefix length field of Windows IP_ADAPTER_PREFIX structure +- improved GH #328: NetworkInterface on Windows XP +- fixed GH #154 Add support for MYSQL_TYPE_NEWDECIMAL to Poco::Data::MySQL +- fixed GH #290: Unicode support +- fixed GH #318: Logger local time doesn't automatically account for DST +- fixed GH #363: DateTimeParser tryParse/parse +- added HTMLForm Content-Length calculation (Rangel Reale) +- Make TemporaryFile append a slash to tempDir +- fixed GH #319 android build with cmake +- added hasDelegates() method to AbstractEvent +- fixed GH #230: Poco::Timer problem +- fixed GH #317: Poco::Zip does not support newer Zip file versions. +- fixed GH #176: Poco::JSON::Stringifier UTF encoding +- fixed GH #458: Broadcast address and subnet mask for IEEE802.11 network interface +- fixed GH #456: poco: library install dirs per RUNTIME/LIBRARY/ARCHIVE + + +Release 1.5.2 (2013-09-16) +========================== + +- added MongoDB library +- fixed GH #57: poco-1.5.1: Doesn't compile for Android +- added VoidEvent (Arturo Castro) +- fixed GH #80: NumberFormatter::append broken +- fixed GH #93: ParallelSocketAcceptor virtual functions +- optional small object optimization for IPAddress, SocketAddress, Any and Dynamic::Var +- SQLite events (insert, update, delete, commit, rollback) handlers +- merged GH #91: Improve SQLite multi-threaded use (Rangel Reale) +- merged GH #86: Invalid pointers to vector internals (Adrian Imboden) +- automatic library initialization macros +- fixed GH #110: WebSocket accept() fails when Connection header contains multiple tokens +- fixed GH #71: WebSocket and broken Timeouts (POCO_BROKEN_TIMEOUTS) +- fixed a warning in Poco/Crypto/OpenSSLInitializer.h +- fixed GH #109: Bug in Poco::Net::SMTPClientSession::loginUsingPlain +- added clang libc++ build configurations for Darwin and iPhone (Andrea Bigagli) +- fixed GH #116: Wrong timezone parsing in DateTimeParse (Matej Knopp) +- fixed GH #118: JSON::Object::stringify endless loop +- added Recursive and SortedDirectoryIterator (Marian Krivos) +- added ListMap (map-like container with preserving insertion order) +- MailMessage: attachments saving support and consistent read/write +- fixed GH #124: Possible buffer overrun in Foundation/EventLogChannel +- fixed GH #119: JSON::Object holds values in ordered map +- added JSON::PrintHandler +- renamed JSON::DefaultHandler to ParseHandler (breaking change!) +- fixed GH #127: Eliminate -Wshadow warnings +- fixed GH #79: Poco::Thread leak on Linux +- fixed GH #61: static_md build configs for Crypto and NetSSL +- fixed GH #130: prefer sysconf over sysctlbyname +- fixed GH #131: no timezone global var on OpenBSD +- fixed GH #102: Some subprojects don't have x64 solutions for VS 2010 +- added GH #75: Poco::Uri addQueryParameter method +- Poco::Environment::osDisplayName() now recognizes Windows 8/Server 2012 +- fixed GH #140: Poco::Runnable threading cleanup issue +- simplified default TCP/HTTPServer construction +- fixed GH #141: Application::run() documentation/implementation discrepancy +- changed RowFormatter to SharedPtr in Data::RecordSet interface (breaking change!) +- fixed GH #144: Poco::Dynamic emits invalid JSON +- removed naked pointers from Data interfaces +- fixed GH #82: name conflict in Data::Keywords::bind +- fixed GH #157: MySQL: cannot bind to 'long' data type on Windows/Visual C++ +- fixed GH #158: MySQL: MYSQL_BIND 'is_unsigned' member is not set +- fixed GH #160: MultipartReader ignores first part, if preamble is missing +- fixed GH #156: Possible buffer overrun in Foundation/EventLogChannel +- XML: fixed an issue with parsing a memory buffer > 2 GB +- upgraded to expat 2.1.0 +- Data/ODBC: added support for setting query timeout (via setProperty + of "queryTimeout"). Timeout is int, given in seconds. +- fixed a potential endless loop in SecureStreamSocketImpl::sendBytes() + and also removed unnecessary code. +- fixed GH #159: Crash in openssl CRYPTO_thread_id() after library libPocoCrypto.so + has been unloaded. +- fixed GH #155: MailOutputStream mangles consecutive newline sequences +- fixed GH #139: FileChannel::PROP_FLUSH is invalid (contains a tab character) +- fixed GH #173: HTTPClientSession::proxyConnect forces DNS lookup of host names +- fixed GH #194: MessageNotification constructor is inefficient. +- fixed GH #189: Poco::NumberParser::tryParse() documentation bug +- fixed GH #172: IPv6 Host field is stripped of Brackets in HTTPClientSession +- fixed GH #188: Net: SocketAddress operator < unusable for std::map key +- fixed GH #128: DOMWriter incorrectly adds SYSTEM keyword to DTD if PUBLIC is + already specified +- fixed GH #65: Poco::format() misorders sign and padding specifiers +- upgraded bundled SQLite to 3.7.17 +- replaced JSON parser with Poco::Web::JSON parser (from sandbox) +- added JSON conversion to Dynamic Struct and Array +- added VarIterator +- modified behavior of empty Var (empty == empty) +- added Alignment.h header for C++03 alignment needs +- added Data/WebNotifier (DB, WebSocket) example +- fixed GH #209: Poco::NumberFormatter double length +- fixed GH #204: Upgrade zlib to 1.2.8 +- fixed GH #198: The "application.configDir" property is not always created. +- fixed GH #185: Poco::NumberFormatter::format(double value, int precision) + ignore precision == 0 +- fixed GH #138: FreeBSD JSON tests fail +- fixed GH #99: JSON::Query an JSON::Object +- limited allowed types for JSON::Query to Object, Array, Object::Ptr, + Array::Ptr and empty +- fixed GH #175: HTMLForm does not read URL parameters on POST or PUT +- added GH #187: MySQL: allow access to the underlying connection handle +- added GH #186: MySQL: support for MYSQL_SECURE_AUTH +- fixed GH #174: MySQL: 4GB allocated when reading any largetext or largeblob field +- fixed a potential memory leak in Poco::Net::HTTPClientSession if it is misused + (e.g., sendRequest() is sent two times in a row without an intermediate call to + receiveResponse(), or by calling receiveResponse() two times in a row without + an intermediate call to sendRequest()) - GH #217 +- removed a few unnecessary protected accessor methods from Poco::Net::HTTPClientSession + that would provide inappropriate access to internal state +- merged GH #210: Don't call CloseHandle() twice on Windows; Ability to select the + threadpool that will be used to start an Activity(Patrice Tarabbia) +- fixed GH #212: JSONConfiguration was missing from the vs90 project(Patrice Tarabbia) +- fixed GH #220: add qualifiers for FPEnvironment in C99 (Lucas Clemente) +- fixed GH #222: HTTPCookie doesn't support expiry times in the past (Karl Reid) +- fixed GH #224: building 1.5.1 on Windows for x64 +- fixed GH# 233: ServerSocket::bind6(Poco::UInt16 port, bool reuseAddress, bool ipV6Only) does not work +- fixed GH# 231: Compatibility issue with Poco::Net::NetworkInterface +- fixed GH# 236: Bug in RecursiveDirectoryIterator +- added ColorConsoleChannel and WindowsColorConsoleChannel classes supporting + colorizing log messages +- fixed GH# 259: Poco::EventLogChannel fails to find 64bit Poco Foundation dll +- fixed GH# 254: UTF8::icompare unexpected behavior +- Poco::UUID::tryParse() also accepts UUIDs without hyphens. Also updated documentation + (links to specifications). +- added GH# 268: Method to get JSON object value using Poco::Nullable +- fixed GH# 267: JSON 'find' not returning empty result if object is expected but another value is found +- Added support for ARM64 architecture and iPhone 5s 64-bit builds + (POCO_TARGET_OSARCH=arm64). + + +Release 1.5.1 (2013-01-11) +========================== + +- using double-conversion library for floating-point numeric/string conversions +- added Poco::istring (case-insensitive string) and Poco::isubstr +- added SQLite sys.dual (in-memory system table) +- applied SF Patch #120: The ExpireLRUCache does not compile with a tuple as key on Visual Studio 2010 +- fixed SF Bug #599: JSON::Array and JSON::Object size() member can implicitly lose precision +- fixed SF Bug #602: iterating database table rows not correct if no data in table +- fixed SF Bug #603: count() is missing in HashMap +- fixed GH #23: JSON::Object::stringify throw BadCastException +- fixed GH #16: NetworkInterface::firstAddress() should not throw on unconfigured interfaces +- Android compile/build support (by Rangel Reale) +- TypeHandler::prepare() now takes const-reference +- fixed GH #27: Poco::URI::decode() doesn't properly handle '+' +- fixed GH #31: JSON implementation bug +- fixed SF #597: Configure script ignores cflags +- fixed SF #593: Poco 1.5.0 on FreeBSD: cannot find -ldl +- added SF #542: SocketAddress() needs port-only constructor +- fixed SF #215: Wrong return type in SocketConnector.h +- applied SF Patch #97: fix c++0x / clang++ bugs +- fixed GH32/SF596: Poco::JSON: Parsing long integer (int64) value fails. +- added Net ifconfig sample (contributed by Philip Prindeville) +- merged GH #34: add algorithm header (Roger Meier/Philip Prindeville) +- fixed GH #26: Cannot compile on gcc +- merged SF #111: FTP Client logging (Marian Krivos) +- fixed GH #30: Poco::Path::home() throws when called from Windows Service +- fixed GH #22: MySQL connection string lowercased +- added MySQL support for Date/Time +- upgraded SQLite to version 3.7.15.1 (2012-12-19) +- improved SQLite execute() return (affected rows) value and added tests +- added SQLite::Utility::isThreadSafe() function +- added SQLite::Utility::setThreadMode(int mode) function +- fixed GH #36: 'distclean' requires 3 traversals of project tree +- fixed GH #41: Buffer::resize crash +- fixed GH #42: Linux unbundled builds don't link +- fixed GH #44: Problems with win x64 build +- fixed GH #46: 1.5.1 build fails on OS X when using libc++ +- fixed GH #48: Need getArgs() accessor to Util::Application to retrieve start-up arguments +- fixed GH #49: NetworkInterface::list doesn't return MAC addresses +- fixed GH #51: Android should use isfinite, isinf, isnan and signbit from the std namespace +- fixed GH #53: JSON unicode fixes and running tests on invalid unicode JSON +- added ParallelAcceptor and ParallelReactor classes +- added EOF and error to FIFOBuffer + + +Release 1.5.0 (2012-10-14) +========================== + +- added JSON library +- added Util::JSONConfiguration +- added FIFOBuffer and FIFOBufferStream +- fixed SF# 3522906: Unregistering handlers from SocketReactor +- fixed SF# 3522084: AbstractConfiguration does not support 64-bit integers +- HTTPServer::stopAll(): close the socket instead of just shutting it down, as the latter won't wake up a select() on Windows +- added SMTPLogger +- added cmake support +- fixed SF#3538778: NetworkInterface enumeration uses deprecated API +- fixed SF#3538779: IPAddress lacks useful constructors: from prefix mask, native SOCKADDR +- fixed SF#3538780: SocketAddress needs operator < function +- fixed SF#3538775: Issues building on Fedora/Centos, etc. for AMD64 +- fixed SF#3538786: Use size_t for describing data-blocks in DigestEngine +- added IPAddress bitwise operators (&,|,^,~) +- added IPAddress BinaryReader/Writer << and >> operators +- modified IPAddress to force IPv6 to lowercase (RFC 5952) +- fixed SF#3538785: SMTPClientSession::sendMessage() should take recipient list +- added IPAddress::prefixLength() +- UTF portability improvements +- fixed SF#3556186: Linux shouldn't use in Net/SocketDefs.h +- added IPAddress RFC 4291 compatible site-local prefix support +- fixed SF#3012166: IPv6 patch +- added SF#3558085: Add formatter to MACAddress object +- fixed SF#3552774: Don't hide default target in subordinate makefile +- fixed SF#3534307: Building IPv6 for Linux by default +- fixed SF#3516844: poco missing symbols with external >=lipcre-8.13 +- added SF#3544720: AbstractConfigurator to support 64bit values +- fixed SF#3522081: WinRegistryConfiguration unable to read REG_QWORD values +- fixed SF#3563626: For Win32 set Up/Running flags on NetworkInterface +- fixed SF#3560807: Deprecate setPeerAddress() as this is now done in getifaddrs +- fixed SF#3560776: Fix byte-ordering issues with INADDR_* literals +- fixed SF#3563627: Set IP address on multicast socket from socket family +- fixed SF#3563999: Size BinaryWriter based on buffer's capacity(), not size() +- fixed SF#102 Fix building Poco on Debian GNU/kFreeBSD +- fixed SF#321 Binding DatTime or Timestamp +- fixed SF#307 Detect the SQL driver type at run time +- added VS 2012 Projects/Solutions +- enhanced and accelerated numeric parsing for integers and floats +- fixed SF#590 Segfault on FreeBSD when stack size not rounded +- added warn function and warnmsg macro in CppUnit +- fixed SF# 3558012 Compilation fails when building with -ansi or -std=c++0x +- fixed SF# 3563517 Get rid of loss-of-precision warnings on x64 MacOS +- fixed SF#3562244: Portability fix for AF_LINK +- fixed SF #3562400: DatagramSocketImpl comment is incorrect + + +Release 1.4.7p1 (2014-11-25) +============================ + +- Fixed Visual C++ 2010-2013 project files. Release builds now have optimization enabled. +- Poco::URI: added constructor to create URI from Path. +- fixed GH #618: OS X 10.10 defines PAGE_SIZE macro, conflicts with PAGE_SIZE in Thread_POSIX.cpp +- Poco::Net::HTTPClientSession: added support for global proxy configuration +- fixed GH #331: Poco::Zip does not support files with .. in the name. +- fixed a memory leak in Poco::Net::Context constructor when it fails to load the certificate + or private key files. +- upgraded bundled SQLite to 3.8.7.2 +- fixed GH #229: added missing value() function +- fixed GH #69: MySQL empty text/blob + + +Release 1.4.7 (2014-10-06) +========================== + +- fixed GH #398: PropertyFileConfiguration: input != output +- fixed GH #368: Build failure of Poco 1.4.6p2 on FreeBSD 9.2 +- fixed GH #318: Logger local time doesn't automatically account for DST +- fixed GH #317: Poco::Zip does not support newer Zip file versions. +- fixed GH #454: Fix: handle unhandled exceptions +- fixed GH #463: XML does not compile with XML_UNICODE_WCHAR_T +- fixed GH #282: Using Thread in a global can cause crash on Windows +- fixed GH #424: Poco::Timer deadlock +- fixed GH #465: Fix result enum type XML_Error -> XML_Status +- fixed GH #510: Incorrect RSAKey construction from istream +- fixed GH #332: POCO::ConsoleChannnel::initColors() assigns no color to + PRIO_TRACE and wrong color to PRIO_FATAL +- fixed GH #550: WebSocket fragmented message problem +- Poco::Data::MySQL: added SQLite thread cleanup handler +- Poco::Net::X509Certificate: improved and fixed domain name verification for + wildcard domains +- fixed a crash in Foundation testsuite with Visual C++ 2012 +- improved and fixed domain name verification for wildcard domains in + Poco::Net::X509Certificate +- updated TwitterClient sample to use new 1.1 API and OAuth +- added Poco::Clock class, which uses a system-provided monotonic clock + (if available) and is thus not affected by system realtime clock changes. + Monotonic Clock is available on Windows, Linux, OS X and on POSIX platforms + supporting clock_gettime() and CLOCK_MONOTONIC. +- Poco::Timer, Poco::Stopwatch, Poco::TimedNotificationQueue and Poco::Util::Timer + have been changed to use Poco::Clock instead of Poco::Timestamp and are now + unaffected by system realtime clock changes. +- added Poco::PBKDF2Engine class template +- Poco::Net::HTTPCookie: added support for Priority attribute (backport from develop) +- fixed makedepend.* scripts to work in paths containing '.o*' + (contributed by Per-Erik Bjorkstad, Hakan Bengtsen) +- Upgraded bundled SQLite to 3.8.6 +- Support for Windows Embedded Compact 2013 (Visual Studio 2012) +- Project and solution files for Visual Studio 2013 +- Changes for C++11 compatibility. +- fixed an issue with receiving empty web socket frames (such as ping) +- improved error handling in secure socket classes +- Poco::ByteOrder now uses intrinsics if available +- added new text encoding classes: Latin2Encoding, Windows1250Encoding, Windows1251Encoding +- Zip: Added CM_AUTO, which automatically selects CM_STORE or CM_DEFLATE based on file extension. + Used to avoid double-compression of already compressed file formats such as images. + + +Release 1.4.6p4 (2014-04-18) +============================ + +- no longer use reverse DNS lookups for cert hostname validation +- cert hostname validation is case insensitive and more strict +- HTMLForm: in URL encoding, percent-encode more special characters +- fixed thread priority issues on POSIX platforms with non-standard scheduling policy +- XMLWriter no longer escapes apostrophe character +- fixed GH #316: Poco::DateTimeFormatter::append() gives wrong result for Poco::LocalDateTime +- fixed GH #305 (memcpy in Poco::Buffer uses wrong size if type != char) +- Zip: fixed a crash caused by an I/O error (e.g., full disk) while creating a Zip archive + + +Release 1.4.6p3 (2014-04-02) +============================ + +- Fixed a potential security vulnerability in client-side X509 + certificate verification. + + +Release 1.4.6p2 (2013-09-16) +============================ + +- fixed GH #156: Possible buffer overrun in Foundation/EventLogChannel +- XML: fixed an issue with parsing a memory buffer > 2 GB +- upgraded to expat 2.1.0 +- Data/ODBC: added support for setting query timeout (via setProperty + of "queryTimeout"). Timeout is int, given in seconds. +- fixed a potential endless loop in SecureStreamSocketImpl::sendBytes() + and also removed unnecessary code. +- fixed GH #159: Crash in openssl CRYPTO_thread_id() after library libPocoCrypto.so + has been unloaded. +- fixed GH #155: MailOutputStream mangles consecutive newline sequences +- fixed GH# 139: FileChannel::PROP_FLUSH is invalid (contains a tab character) +- fixed GH# 173: HTTPClientSession::proxyConnect forces DNS lookup of host names +- fixed GH# 194: MessageNotification constructor is inefficient. +- fixed GH# 189: Poco::NumberParser::tryParse() documentation bug +- fixed GH# 172: IPv6 Host field is stripped of Brackets in HTTPClientSession +- fixed GH# 188: Net: SocketAddress operator < unusable for std::map key +- fixed GH# 128: DOMWriter incorrectly adds SYSTEM keyword to DTD if PUBLIC is + already specified +- fixed GH# 65: Poco::format() misorders sign and padding specifiers +- upgraded bundled SQLite to 3.7.17 +- upgraded bundled zlib to 1.2.8 +- fixed a potential memory leak in Poco::Net::HTTPClientSession if it is misused + (e.g., sendRequest() is sent two times in a row without an intermediate call to + receiveResponse(), or by calling receiveResponse() two times in a row without + an intermediate call to sendRequest()) - GH #217 +- removed a few unnecessary protected accessor methods from Poco::Net::HTTPClientSession + that would provide inappropriate access to internal state +- fixed GH# 223 (Poco::Net::HTTPCookie does not support expiry times in the past) +- fixed GH# 233: ServerSocket::bind6(Poco::UInt16 port, bool reuseAddress, bool ipV6Only) + does not work +- added ColorConsoleChannel and WindowsColorConsoleChannel classes supporting + colorizing log messages +- fixed GH# 259: Poco::EventLogChannel fails to find 64bit Poco Foundation dll +- fixed GH# 254: UTF8::icompare unexpected behavior +- Poco::UUID::tryParse() also accepts UUIDs without hyphens. Also updated documentation + (links to specifications). +- Added support for ARM64 architecture and iPhone 5s 64-bit builds + (POCO_TARGET_OSARCH=arm64). + + +Release 1.4.6p1 (2013-03-06) +============================ + +- fixed GH# 71: WebSocket and broken Timeouts (POCO_BROKEN_TIMEOUTS) +- fixed an ambiguity error with VC++ 2010 in Data/MySQL testsuite +- Poco::Net::NetworkInterface now provides the interface index even for IPv4 +- added DNS::reload() as a wrapper for res_init(). +- On Linux, Poco::Environment::nodeId() first always tries to obtain the + MAC address of eth0, before looking for other interfaces. +- Poco::Net::HTTPSession now always resets the buffer in connect() to clear + any leftover data from a (failed) previous session +- fixed copysign namespace issue in FPEnvironment_DUMMY.h +- fixed a warning in Poco/Crypto/OpenSSLInitializer.h +- added a build configuration for BeagleBoard/Angstrom +- fixed GH# 109: Bug in Poco::Net::SMTPClientSession::loginUsingPlain) +- fixed compile errors with clang -std=c++11 +- fixed GH# 116: Wrong timezone parsing in DateTimeParse (fix by Matej Knopp) +- updated bundled SQLite to 3.7.15.2 + + +Release 1.4.6 (2013-01-10) +========================== + +- changed FPEnvironment_DUMMY.h to include instead of +- updated bundled SQLite to 3.7.15.1 +- fixed GH# 30: Poco::Path::home() throws +- fixed SF Patch# 120 The ExpireLRUCache does not compile with a tuple as key on VS2010 +- fixed SF# 603 count() is missing in HashMap +- Crypto and NetSSL_OpenSSL project files now use OpenSSL *MD.lib library files for + static_md builds. Previously, the DLL import libs were used. +- Poco::Environment::osDisplayName() now recognizes Windows 8/Server 2012 + + +Release 1.4.5 (2012-11-19) +========================== + +- added Visual Studio 2012 project files +- buildwin.cmd now support building with msbuild for VS2010 and 2012. +- added Poco::Optional class +- fixed SF# 3558012 Compilation fails when building with -ansi or -std=c++0x +- fixed SF# 3563517 Get rid of loss-of-precision warnings on x64 MacOS +- fixed SF# 3562244: Portability fix for AF_LINK +- fixed SF# 3562400: DatagramSocketImpl comment +- fixed SF# 594: Websocket fails with small masked payloads +- fixed SF# 588: Missing POCO_ARCH and POCO_ARCH_LITTLE_ENDIAN define for WinCE on SH4 +- fixed SF# 581: Out-of-bound array access in Unicode::properties() function. +- fixed SF# 590: Segfault on FreeBSD when stack size not rounded +- fixed SF# 586: Poco::DateTimeParser and ISO8601 issues when seconds fraction has more than 6 digits +- Poco::Net::HTTPSSessionInstantiator::registerInstantiator() now optionally accepts a + Poco::Net::Context object. +- added Poco::XML::XMLWriter::depth() member function. +- added Poco::XML::XMLWriter::uniquePrefix() and Poco::XML::XMLWriter::isNamespaceMapped(). +- Poco::FileChannel now supports a new rotateOnOpen property (true/false) which can be used + to force rotation of the log file when it's opened. +- fixed a bug in Poco::XML::XMLWriter::emptyElement(): need to pop namespace context +- OS X builds now use Clang as default compiler +- Updated SQLite to 3.7.14.1 +- POCO_SERVER_MAIN macro now has a try ... catch block for Poco::Exception and writes + the displayText to stderr. +- Poco/Platform.h now defines POCO_LOCAL_STATIC_INIT_IS_THREADSAFE macro if the compiler + generates thread-safe static local initialization code. + + +Release 1.4.4 (2012-09-03) +========================== + +- ZipStream now builds correctly in unbundled build. +- added proxy digest authentication support to Net library +- integrated MySQL BLOB fixes from Franky Braem. +- use standard OpenSSL import libraries (libeay32.lib, ssleay32.lib) for Crypto and + NetSSL_OpenSSL Visual Studio project files. +- fixed a potential buffer corruption issue in Poco::Net::SecureStreamSocket if lazy + handshake is enabled and the first attempt to complete the handshake fails +- Poco::DateTimeParser::tryParse() without format specifier now correctly parses ISO8601 + date/times with fractional seconds. +- Poco::Process::launch() now has additional overloads allowing to specify an initial + directory and/or environment. +- Poco::Net::FTPClientSession: timeout was not applied to data connection, only to + control connection. +- Fixed potential IPv6 issue with socket constructors if IPv6 SocketAddress is given + (contributed by ??????? ????????? ). +- Added an additional (optional) parameter to Poco::Thread::setOSPriority() allowing to + specify a scheduling policy. Currently this is only used on POSIX platforms and allows + specifying SCHED_OTHER (default), SCHED_FIFO or SCHED_RR, as well as other + platform-specific policy values. +- Added Poco::Crypto::DigestEngine class providing a Poco::DigestEngine interface to + the digest algorithms provided by OpenSSL. +- Fixed some potential compiler warnings in Crypto library +- In some cases, when an SSL exception was unexpectedly closed, a generic Poco::IOException + was thrown. This was fixed to throw a SSLConnectionUnexpectedlyClosedException instead. +- Added Poco::ObjectPool class template. +- Poco::Net::HTTPServer has a new stopAll() method allowing stopping/aborting of all + currently active client connections. +- The HTTP server framework now actively prevents sending a message body in the + response to a HEAD request, or in case of a 204 No Content or 304 Not Modified + response status. +- fixed a DOM parser performance bug (patch by Peter Klotz) +- fixed SF# 3559325: Util Windows broken in non-Unicode +- updated iOS build configuration to use xcode-select for finding toolchain +- Poco::Net::SecureSocketImpl::shutdown() now also shuts down the underlying socket. +- fixed SF# 3552597: Crypto des-ecb error +- fixed SF# 3550553: SecureSocketImpl::connect hangs +- fixed SF# 3543047: Poco::Timer bug for long startInterval/periodic interval +- fixed SF# 3539695: Thread attributes should be destroyed using the pthread_attr_destroy() +- fixed SF# 3532311: Not able to set socket option on ServerSocket before bind + Added Poco::Net::Socket::init(int af) which can be used to explicitely + initialize the underlying socket before calling bind(), connect(), etc. +- fixed SF# 3521347: Typo in UnWindows.h undef +- fixed SF# 3519474: WinRegistryConfiguration bug + Also added tests and fixed another potential issue with an empty root path passed to the constructor. +- fixed SF# 3516827: wrong return value of WinRegistryKey::exists() +- fixed SF# 3515284: RSA publickey format(X.509 SubjectPublicKeyInfo) +- fixed SF# 3503267: VxWorks OS prio is not set in standard constructor +- fixed SF# 3500438: HTTPResponse failure when reason is empty +- fixed SF# 3495656: numberformater, numberparser error in mingw +- fixed SF# 3496493: Reference counting broken in TaskManager postNotification +- fixed SF# 3483174: LogFile flushing behavior on Windows + Flushing is now configurable for FileChannel and SimpleFileChannel + using the "flush" property (true or false). +- fixed SF# 3479561: Subsequent IPs on a NIC is not enumerated +- fixed SF# 3478665: Permission checks in Poco::File not correct for root +- fixed SF# 3475050: Threading bug in initializeNetwork() on Windows +- fixed SF# 3552680: websocket small frames bug and proposed fix +- fixed a WebSocket interop issue with Firefox +- added Poco::Net::MessageHeader::hasToken() +- Poco::AtomicCounter now uses GCC 4.3 builtin atomics on more platforms +- fixed SF# 3555938: NetSSL: socket closed twice +- socket exceptions now include OS error code +- fixed SF# 3556975: Need to fix Shared Memory for memory map +- Poco::Net::SecureSocketImpl::close() now catches exceptions thrown by its call to shutdown(). +- fixed SF# 3535990: POCO_HAVE_IPv6 without POCO_WIN32_UTF8 conflict +- fixed SF# 3559665: Poco::InflatingInputStream may not always inflate completely +- added Poco::DirectoryWatcher class +- fixed SF# 3561464: Poco::File::isDevice() can throw due to sharing violation +- Poco::Zip::Compress::addRecursive() has a second variant that allows to specify the compression method. +- Upgraded internal SQLite to 3.7.14 + + +Release 1.4.3p1 (2012-01-23) +============================ + +- fixed SF# 3476926: RegDeleteKeyEx not available on Windows XP 32-bit + + +Release 1.4.3 (2012-01-16) +========================== + +- fixed a compilation error with Data/MySQL on QNX. +- fixed Util project files for WinCE (removed sources not compileable on CE) +- removed MD2 license text from Ackowledgements document +- fixed iPhone build config for Xcode 4.2 (compiler name changed to llvm-g++) +- Poco::Util::XMLConfiguration: delimiter char (default '.') is now configurable. + This allows for working with XML documents having element names with '.' in them. +- Poco::Util::OptionProcessor: Required option arguments can now be specified as + separate command line arguments, as in "--option value" in addition to the + "--option=value" format. +- Poco::Util::HelpFormatter: improved option help formatting if indentation has + been set explicitely. +- added Mail sample to NetSSL_OpenSSL, showing use of Poco::Net::SecureSMTPClientSession. +- added additional read() overloads to Poco::Net::HTMLForm. +- fixed SF# 3440769: Poco::Net::HTTPResponse doesn't like Amazon EC2 cookies. +- added support for requiring TLSv1 to Poco::Net::Context. +- added an additional constructor to Poco::Net::HTTPBasicCredentials, allowing + the object to be created from a string containing a base64-encoded, colon-separated + username and password. +- Poco::Zip::ZipStreamBuf: fixed a crash if CM_STORE was used. +- Added setContentLength64() and getContentLength64() to Poco::Net::HTTPMessage. +- added Poco::Environment::osDisplayName(). +- fixed SF# 3463096: WinService leaves dangling handles (open() now does not reopen the + service handle if it's already open) +- fixed SF# 3426537: WinRegistryConfiguration can't read virtualized keys +- added Poco::Buffer::resize() +- fixed SF# 3441822: thread safety issue in HTTPClientSession: + always use getaddrinfo() instead of gethostbyname() on all platforms supporting it +- added version resource to POCO DLLs +- fixed SF# 3440599: Dir Path in Quotes in PATH cause PathTest::testFind to fail. +- fixed SF# 3406030: Glob::collect problem +- added Poco::Util::AbstractConfiguration::enableEvents() +- Poco::AtomicCounter now uses GCC builtins with GCC 4.1 or newer + (contributed by Alexey Milovidov) +- made Poco::Logger::formatDump() public as it may be useful for others as well + (SF# 3453446) +- Poco::Net::DialogSocket now has a proper copy constructor (SF# 3414602) +- Poco::Net::MessageHeader and Poco::Net::HTMLForm now limit the maximum number of + fields parsed from a message to prevent certain kinds of denial-of-service + attacks. The field limit can be changed with the new method setFieldLimit(). + The default limit is 100. +- Poco::NumberFormatter, Poco::NumberParser and Poco::format() now always use the + classic ("C") locale to format and parse floating-point numbers. +- added Poco::StreamCopier::copyStream64(), Poco::StreamCopier::copyStreamUnbuffered64() + and Poco::StreamCopier::copyToString64(). These functions use a 64-bit integer + to count the number of bytes copied. +- upgraded internal zlib to 1.2.5 +- upgraded internal sqlite to 3.7.9 +- XML: integrated bugfix for Expat bug# 2958794 (memory leak in poolGrow) +- Added support for HTTP Digest authentication (based on a contribution by + Anton V. Yabchinskiy (arn at bestmx dot ru)). For information on how + to use this, see the Poco::Net::HTTPCredentials, Poco::Net::HTTPDigestCredentials + and Poco::Net::HTTPAuthenticationParams classes. +- Poco::Net::HTTPStreamFactory and Poco::Net::HTTPSStreamFactory now support Basic + and Digest authentication. Username and password must be provided in the URI. +- added Poco::Net::WebSocket, supporting the WebSocket protocol as described in RFC 6455 +- NetSSL_OpenSSL: added client-side support for Server Name Indication. + Poco::Net::SecureSocketImpl::connectSSL() now calls SSL_set_tlsext_host_name() + if its available (OpenSSL 9.8.6f and later). +- added Poco::Net::HTTPClientSession::proxyConnect() (factored out from + Poco::Net::HTTPSClientSession::connect()) +- added Poco::Process::kill(const Poco::ProcessHandle&) which is preferable to + kill(pid) on Windows, as process IDs on Windows may be reused. +- fixed SF# 3471463: Compiler warnings with -Wformat +- Poco::Util::Application::run() now catches and logs exceptions thrown in initialize() +- Fixed a WinCE-specific bug in Poco::Util::ServerApplication where uninitialize() would + be called twice. +- fixed SF# 3471957: WinRegistryKey::deleteKey() unable to delete alt views +- Added additional constructor to Poco::ScopedLock and Poco::ScopedLockWithUnlock + accepting a timeout as second argument. +- Added Poco::Logger::parseLevel() +- Poco::format(): an argument that does not match the format + specifier no longer results in a BadCastException. The string [ERRFMT] is + written to the result string instead. +- PageCompiler: added createSession page attribute. + + +Release 1.4.2p1 (2011-09-24) +============================ + +- On Linux, the RTLD_DEEPBIND option is no longer passed to dlopen(). + This change was introduced in 1.4.2 to solve a specific problem one customer + was having. Unfortunately, it leads to problems with RTTI. +- It's now possible to pass flags (SHLIB_GLOBAL, SHLIB_LOCAL) to + Poco::SharedLibrary::load() (and the constructor implicitly calling load()), + controlling the mode flags (RTLD_GLOBAL, RTLD_LOCAL) passed to dlopen(). + On platforms not using dlopen(), these flags are ignored. +- fixed SF# 3400267: Path_WIN32.cpp bug + + +Release 1.4.2 (2011-08-28) +========================== + +- added Poco::DateTimeFormat::ISO8601_FRAC_FORMAT +- added new Poco::DateTimeFormatter and Poco::DateTimeParser format specifier: + %s for seconds with optional fractions of a second +- fixed a problem with ioctl() on BSD platforms (including OS X) where the + second argument to ioctl() is unsigned long instead of int, causing bad + things on a OS X 64-bit kernel. +- fixed a potential endless loop when enumerating IPv6 network addresses + (reported by Laurent Carcagno) +- new compile-time config option on Windows to set thread names in + debugger. Enable with -DPOCO_WIN32_DEBUGGER_THREAD_NAMES. Available + only in debug builds. +- Cipher can now create Base64 and HexBinary encoded output without linefeeds + (suitable for use in cookies, etc.) +- added Poco::Path::popFrontDirectory() +- improved VxWorks support +- IPv6 fixes: added proper scope id handling in IPAddress, SocketAddress + and related classes. +- Added Poco::Net::ServerSocket::bind6() which allows control over the + IPPROTO_IPV6/IPV6_V6ONLY socket option. +- Removed Poco::MD2Engine class due to licensing issues (the + license for the MD2 code from RSA only allows non-commercial + use). Note that the MD4 and MD5 code from RSA does not have + this issue. +- fixed a Net HTTP client testsuite issue where some tests might + have failed due to prematurely aborted connections by + the HTTPTestServer. +- Poco::Net::SocketAddress: when there is more than one address + returned by a DNS lookup for a name, IPv4 addresses will be + preferred to IPv6 ones. +- NetworkInterface::list() now also returns IPv4 interfaces on Windows when + built with -DPOCO_HAVE_IPv6 +- XMLWriter: fixed a bug with attribute namespaces (no namespace prefix + written if attribute namespace is the same as element namespace) +- fixed SF# 3378588: Mismatched new[]/delete (in RSAEncryptImpl and RSADecryptImpl) +- fixed SF# 3212954 (OpenSSLInitializer::uninitialize() crash) and + SF# 3196862 (Static OpenSSLInitializer instance causes Windows deadlocks) by + removing the static Poco::Crypto::OpenSSLInitializer instance. Automatic OpenSSL + initialization is now done through Poco::Crypto::Cipher, Poco::Crypto::CipherKey, + Poco::Crypto::X509Certificate, Poco::Net::Context classes; however, it is still + recommended to call Poco::Crypto::initializeCrypto() and + Poco::Crypto::uninitializeCrypto() early at application startup, and late at + shutdown respectively (or Poco::Net::initializeSSL()/Poco::Net::uninitializeSSL() + if the NetSSL library is used) to avoid multiple full OpenSSL init/uninit cycles + during application runtime. +- Poco::Logger now also support a symbolic log level "none" + (for use with setLevel()) that disables logging completely + for that Logger (equivalent to setLevel(0)). +- Added experimental Android support, using the existing gmake-based + build system. +- fixed SF# 3288584: DateTimeFormatter link error +- fixed SF# 3187117: Typo in InflatingInputStream doc +- fixed SF# 3309731: _WIN32_WCE comparison should be with 0x600 not 600 +- fixed SF# 3393026: RegularExpression.h identical enum value +- fixed SF# 3274222: AtomicCounter's postfix operators aren't atomic on Windows +- fixed SF# 3317177: Handle leak on windows +- fixed SF# 3181882: Poco::URI::getPathEtc() double-encodes query +- fixed SF# 3379935: ThreadPool Start Bug +- fixed SF# 3354451: Poco::Format::parsePrec never sets the precision to zero +- fixed SF# 3387258: _MAX_PATH used but unknown in Path_WIN32 +- fixed a problem in RSAKeyImpl where direct access to the RSA in a EVP_PKEY + would no longer work in recent OpenSSL versions. Using EVP_PKEY_get1_RSA() + fixes the issue. +- added Poco::Crypto::EncryptingInputStream, Poco::Crypto::EncryptingOutputStream, + Poco::Crypto::DecryptingInputStream and Poco::Crypto::DecryptingOutputStream. +- fixed SF# 3148126: HTTPSClientSession destructor (!) throws an IOException +- fixed SF# 3178098: Add constructor to Poco::TemporaryFile to specify directory +- fixed SF# 3175310: Absolute path when device +- fixed SF# 3301207: Guided tour example contradicts apidoc (API doc was wrong) +- Poco::Net::HTTPMessage::setContentLength() and Poco::Net::HTTPMessage::getContentLength() now + use std::streamsize instead of int. This enables 64-bit Content-Length support at least + on 64-bit platforms. +- fixed SF# 3177530: TemporaryFile::tempName() + glob bug on xp +- fixed SF# 3177372: FileChannel documentation inconsistency +- added %E format specifier to Poco::PattermFormatter (epoch time in seconds + since midnight, January 1 1970) +- On Windows, Poco::Util::ServerApplication now supports a /description command + line argument for specifying a service description (together with /registerService) +- added Poco::Util::WinService::setDescription() and + Poco::Util::WinService::getDescription() +- fixed SF# 3155477: Incorrect URI path handling +- fixed SF# 3309736: Extended Exception macros to set default exception code + new macro is named POCO_DECLARE_EXCEPTION_CODE +- added getter functions for modulus and exponents to Poco::Crypto::RSAKey. +- added Poco::Net::SocketAddress::operator == () and + Poco::Net::SocketAddress::operator != () +- fixed SF# 3182746: IPAddress.cpp IPv6 bug on big-endian +- fixed SF# 3196961: Unix daemon fails to loadConfiguration() if started from cwd +- fixed SF# 3393700: NotificationCenter may call a removed observer and crash. +- Reworked implementation of the events framework (Poco::BasicEvent and friends). + The framework is now completely multithreading save (even in the case that + an event subscriber object unsubscribes and is deleted while an event is + being dispatched). Also, the restriction that any object can only register + one delegate for each event has been removed. For most cases, dispatching + events should be faster, as dispatching an event now needs less dynamic memory + allocations. +- fixed SF# 3178109: getNodeByPath() changes: + getNodeByPath() and getNodeByPathNS() have been moved to Poco::XML::Node. + Furthermore, when invoked on a Poco::XML::Document, the behavior has changed + so that the document element is now included when traversing the path (previously, + traversal would start at the document element, now it starts at the document). + The path expression can now start with a double-slash, which results in a recursive + search for the path's first element in the DOM tree. +- fixed SF# 3382935: String data being truncated using ODBC, and + SF# 2921813: Wrong implementation of the ODBC string binding + + +Release 1.4.1p1 (2011-02-08) +============================ + +- Poco::Mutex is now a recursive mutex again on Linux + (this was caused by an unfortunate feature test for + PTHREAD_MUTEX_RECURSIVE which did not work on Linux + as PTHREAD_MUTEX_RECURSIVE is an enum value and not + a macro) +- Poco::Net::SecureSocketImpl::abort() now only shuts + down the underlying socket connection and does not free + the SSL object, due to multithreading issues. + + +Release 1.4.1 (2011-01-29) +========================== + +- fixed SF# 3150223: Poco::BinaryReader cannot read std::vector correctly +- fixed SF# 3146326: SharedMemory issue +- made Poco::Net::HTTPSession::abort() virtual +- added Poco::Net::SecureStreamSocket::abort() to immediately close + a SSL/TLS connection without performing an orderly SSL/TLS shutdown. +- fixed SF# 3148126: HTTPSClientSession destructor (!) throws an IOException. + Added try/catch block to Poco::Net::SecureSocketImpl destructor. +- added additional constructor to Poco::Net::HTTPSClientSession, taking + both a socket and a session object. +- Poco::Net::HTTPSession::abort() now also can be used with a + Poco::Net::HTTPSClientSession. +- fixed SF# 3148045: make clean and distclean issues +- changed Data library names on Unix/Linux platforms to + match the names on Windows (PocoSQLite -> PocoDataSQLite, + PocoMySQL -> PocoDataMySQL, PocoODBC -> PocoDataODBC) +- added additional options to configure script +- added additional documentation to Poco::Net::HTTPClientSession +- Poco::Net::HTTPClientSession::receiveResponse() closes the connection + if an exception is thrown while reading the response header. + This ensures that a new connection will be set up for the next request + if persistent connections are used. +- improved Poco::Net::MultipartDecoder performance by reading directly from streambuf +- improved performance of Poco::Base64Encoder, Poco::Base64Decoder, + Poco::HexBinaryEncoder and Poco::HexBinaryDecoder by working directly with the + given stream's streambuf. +- improved performance of MessageHeader::read() by reading directly from streambuf + instead of istream. +- it is now possible to specify additional MIME part header fields + for a MIME part through the Poco::Net::PartSource class. +- upgraded SQLite to release 3.7.4 +- added experimental VxWorks support for VxWorks 5.5.1/Tornado 2.2 and + newer. Please see the VxWorks Platform Notes in the reference documentation + for more information. Currently, the VxWorks is untested; full support + will be available in release 1.4.2. +- fixed SF# 3165918: Poco::DynamicAny fails to convert from string to float +- fixed SF# 3165910: Poco::Net::MessageHeader does not accept HTTP conforming header +- made Poco::Task::cancel() virtual so that tasks can implement custom + cancellation behavior. +- added optional argument to Poco::Util::WinRegistryKey constructor + to specify additional flags (in addition to KEY_READ and KEY_WRITE) + for the samDesired argument of RegOpenKeyEx() or RegCreateKeyEx(). +- improved Poco::BasicEvent::notify() performance by avoiding an unnecessary heap + allocation. +- added additional well-known port numbers to Poco::URI: rtsp, sip, sips, xmpp. +- added Poco::Net::MediaType::matchesRange() +- improved invalid socket handling: a Poco::Net::InvalidSocketException is + now thrown instead of an assertion when an operation is attempted on a closed or + otherwise uninitialized socket. + + +Release 1.4.0 (2010-12-14) +========================== + +- SSLManager: documentation fixes, code cleanup +- SSLManager: renamed PrivateKeyPassPhrase event to PrivateKeyPassphraseRequired +- added HTTPServerRequestImpl::socket() to get access to the underlying socket +- added Socket::secure() to find out whether a given socket supports SSL/TLS +- added SecureStreamSocket::havePeerCertificate() +- NetSSL: added support for turning off extended certificate validation (hostname matching) +- fixed SF# 2941228: ICMPClient::ping() issues on Mac OS X +- fixed SF# 2941231: ICMPEventArgs out of bounds array access +- added PageCompiler sample +- added missing newline at end of xmlparse.c +- Poco::Glob can now be used with an empty pattern which will match nothing (patch from Kim Graesman) +- added support for HTTP proxy authentication (Basic authentication only) +- fixed SF# 2958959: XMLWriter must encode CR, LF and TAB in attribute values as character entities. +- HTMLForm now supports PUT requests as well (see ) +- fixed SF# #2970521: FileOutputStream and file permissions. + (also fixed in File class) +- removed an unused (and wrong) default parameter from EventImpl constructor for WIN32. +- added full support for session caching to NetSSL_OpenSSL +- fixed SF# 2984454: Poco::Util::Timer::scheduleAtFixedRate() works incorrectly +- fixed a bug in Poco::Util::Timer that could lead to high CPU load if + the system clock is moved forward. +- added system.nodeId to SystemConfiguration +- added a note to Poco::Util::ServerApplication documentation regarding + creation of threads +- added Poco::Net::IPAddress::broadcast() and Poco::Net::IPAddress::wildcard() to + create broadcast (255.255.255.255) and wildcard (0.0.0.0) addresses. +- fixed SF# 2916154: Poco::Net::IPAddress::isLoopback() only works for 127.0.0.1. +- added build configuration for iPhone Simulator +- GNU Make based build system provides new variables: POCO_HOST_BINDIR, POCO_HOST_BINPATH, + POCO_HOST_LIBDIR, POCO_HOST_LIBPATH and POCO_TARGET_* equivalents. +- Application::initialize() and Application::uninitialize() will now be called from within run(). + This solves various issues with uninitialize() not being called, or being called inappropriately + from the Application destructor. + Please note that this change will break applications that use the Application class, + but only call init() and not run(). +- added /startup option to specify startup mode for Windows services (automatic or manual) +- fixed SF# 2967354: SecureSocketImpl shutdown/close problem +- fixed SF# 3006340: LinearHashTable grows even if key already exists +- fixed a particularly nasty Windows error handling issue that manifested itself on WinCE: + WSAGetLastError() would be called after a std::string was created. The string creation could result + in a heap operation which called a Windows API to allocate memory. This would reset the + GetLastError() error code. Since WSAGetLastError() is just an alias for GetLastError(), the actual + error code from the socket operation would be lost. +- upgraded SQLite to 3.7.3 +- added --header-prefix option to PageCompiler +- fixed SF# 3003875: SQLite data binding is broken +- fixed SF# 2993988: Issue with multiple calls to open()/close() on File*Stream +- fixed SF# 2990256: HTMLForm and file uploads +- fixed SF# 2969227: DateTimeParser bug +- fixed SF# 2966698: Socket connect with timeout issue +- fixed SF# 2981041: Bind NULL to a query (patch supplied) +- fixed SF# 2961419: UTF8Encoding::convert() doesn't work properly in DEBUG mode +- fixed SF# 2957068: Timeout value not picked up by proxy in HTTPSClientSession +- fixed NetSSL_OpenSSL test runner for Poco::Util::Application class changes +- Poco::AbstractEvent, Poco::AbstractCache and related classes now accept a Mutex class as additional template argument. + Poco::NullMutex can be used if no synchronization is desired. +- Added Poco::AbstractEvent::empty() to check whether an event has registered delegates. +- Poco::URI now correctly handles IPv6 addresses. +- Added Poco::Nullable class template. +- Added Poco::NullMutex, a no-op mutex to be used as template argument for template classes + taking a mutex policy argument. +- Poco::XML::XMLWriter: fixed a namespace handling issue that occured with startPrefixMapping() and endPrefixMapping() +- Poco::Net::Context now allows for loading certificates and private keys from Poco::Crypto::X509Certificate objects + and Poco::Crypto::RSAKey objects. +- Poco::Crypto::RSAKey no longer uses temporary files for stream operations. Memory buffers are used instead. +- fixed SF# 2957865: added Poco::UUID::tryParse() +- All Zip classes now use Poco::File[Input|Output]Stream instead of std::[i|o]fstream. + UTF-8 filenames will now be handled correctly on Windows. +- fixed SF# 2902029: zlib flush support (Z_SYNC_FLUSH) +- added Poco::TextBufferIterator class +- fixed SF# 2977249: Use epoll instead select under Linux + Socket::select() and Socket::poll() will use epoll under Linux if the Net library is compiled + with -DPOCO_HAVE_FD_EPOLL. This is the default for the Linux build configuration (but not for + the various build configurations targeting embedded Linux platforms). +- fixed SF# 2941664: Memory leak in DeflatingStream with zero-length streams (also fixed some other potential, + but unlikely, memory leaks) +- fixed SF# 2946457: added RejectCertificateHandler +- fixed SF# 2946621: Poco::Path bug with POCO_WIN32_UTF8 +- fixed SF# 2929805: Environment::nodeId() does not work if no eth0 device exists +- Environment::nodeId() no longer throws if no hardware ethernet address can be determined. + It returns an all-zero address instead. +- Added additional classification functions to Poco::Unicode class; made classification functions inline. +- added Ascii class for ASCII character classification. + Methods of the Ascii class are now used instead of the + standard library functions (std::isspace(), etc.) due to + possible inconsistent results or assertions when the + standard library functions are used with character codes + outside the ASCII range. +- Poco::Net::MailMessage: fixed a bug in StringPartHandler that resulted in incorrect handling of non-ASCII data if + char is signed. +- Improved Poco::Net::SMTPClientSession compatibility with various mail servers when using AUTH_LOGIN authentication. +- Added CRAM-SHA1 support to Poco::Net::SMTPClientSession +- Poco::Net::SMTPClientSession now also supports login with AUTH PLAIN. +- Added Poco::Net::SecureSMTPClientSession class, supporting STARTTLS for secure SMTP connections. +- fixed an issue with SharedMemory on POSIX systems, where a shared memory region would be deleted + despite the server flag set to true (see http://pocoproject.org/forum/viewtopic.php?f=12&t=3494). +- PageCompiler: added a new page context directive, to allow passing custom context objects to the + request handler. +- fixed StreamSocketImpl::sendBytes() for non-blocking sockets +- added Poco::Net::DialogSocket::receiveRawBytes(), which should be used instead of receiveBytes() due to internal + buffering by DialogSocket. +- DOMParser: FEATURE_WHITESPACE has been renamed to FEATURE_FILTER_WHITESPACE (which now matches the underlying URI) + and is now handled correctly (previously we did the exact reverse thing) +- added Poco::Util::AbstractConfiguration::remove() to remove a configuration property; added removeRaw() implementations + to all implementations (contributions by Daniel Hobi and Alexey Shults). +- fixed NetSSL_OpenSSL compilation error on Windows with OpenSSL 1.0 +- Added optional FIPS mode support to NetSSL_OpenSSL (contributed by Lior Okman). + If OpenSSL has been configured and built with FIPS support, then FIPS support can + be enabled by calling Poco::Crypto::OpenSSLInitializer::enableFIPSMode(true); or + by setting the fips property in the OpenSSL configuration to true (see Poco::Net::SSLManager + for details). +- fixed SF# 3031530: Ping and possible no timeout +- added Poco::Net::SocketReactor::onBusy(), called whenever at least one notification will + be dispatched. +- fixed SF# 3034863: Compiler warning in net/IPAddress.h with poco 1.3.2 +- added support for CRAM-SHA1 authentication to SMTPClientSession +- Poco::format(): arguments can now be addressed by their index, e.g. %[2]d +- Poco::Util::Timer::cancel() now accepts an optional boolean argument. + If true is passed, cancel() waits until the task queue has been purged. + Otherwise, it returns immediately and works asynchronously, as before. +- Poco::Net::HTTPServerResponse::redirect() now accepts an optional additional + argument to specify the HTTP status code for the redirection. +- fixed a warning (BinaryReader.cpp) and error (ThreadLocal.cpp) in Foundation when compiling with Visual Studio 2010 +- fixed a wrong exception in Poco::Net::POP3ClientSession +- Poco::Net::FTPClientSession and Poco::Net::SMTPClientSession now set the error code in exceptions they throw +- fixed a potential race condition with terminating a Windows service based on Poco::Util::ServerApplication +- fixed a bug in global build configuration file: explicitly setting POCO_CONFIG did not work on Solaris platforms, + as it was always overridden by the automatically determined configuration. +- Added support for MinGW cross builds on Linux. +- Changed location of statically linked build products in the gmake-based build system. + Statically linked executables are now in bin/$(OSNAME)/$(OSARCH)/static and no longer + have the _s suffix +- The POCO_VERSION macro now is in its own header file, "Poco/Version.h". It is no longer + available through "Poco/Foundation.h". +- added Poco::Net::HTTPCookie::escape() and Poco::Net::HTTPCookie::unescape(). +- fixed SF# 3021173: Thread (POSIX) returns uninitialised value for OS priority +- fixed SF# 3040870: ThreadPool has no function to get assigned name +- fixed SF# 3044303: Can't use own config file on Solaris & OSARCH_64BITS ignored +- fixed SF# 2943896: AsyncChannel::log blocks +- fixed a bug in Poco::Util::WinRegistryKey::getInt(): + The size variable passed to RegQueryValueExW() should be initialized to the size + of the output buffer. +- Added rudimentary support for compiling with Clang 2.0 (Xcode 4) on Mac OS X. +- New build configurations for Mac OS X: Darwin32 and Darwin64 for explicit + 32-bit and 64-bit builds. Note that the default Darwin build configuration + will build 64-bit on Snow Leopard and 32-bit on Leopard, but will always place + build products in Darwin/i386. The new Darwin32 and Darwin64 configurations + will use the correct directories. +- fixed SF# 3051598: Bug in URL encoding +- Poco::ThreadPool::stopAll() (and thus also the destructor) will now wait for each + pooled thread to terminate before returning. This fixes an issue with creating + and orderly shutting down a thread pool in a plugin. Previously, a pooled thread + in a thread pool created by a dynamically loaded library might still be running + when the plugin's shared library was unloaded, resulting in Bad Things happening. + This can now no longer happen. As a downside, a pooled thread that fails to + finish will block stopAll() and the destructor forever. +- NetSSL_OpenSSL: for a SecureStreamSocket, available() now returns the number of bytes that + are pending in the SSL buffer (SSL_pending()), not the actual socket buffer. +- Added Poco::Net::HTTPClientSession::secure() to check for a secure connection. +- Poco::Net::HTTPRequest::setHost() now does not include the port number in the Host header + if it's either 80 or 443. +- log messages can now optionally include source file path and line number +- Poco::PatternFormatter can format source file path and line number (%U, %u) +- logging macros (poco_information(), etc.) now use __LINE__ and __FILE__ +- new logging macros that incorporate Poco::format(): poco_information_f1(logger, format, arg) with up to 4 arguments +- added Poco::Net::HTTPSession::attachSessionData() and Poco::Net::HTTPSession::sessionData() + to attach arbitrary data to a HTTP session. +- added additional constructors to zlib stream classes that allow passing + a windowBits parameter to the underlying zlib library. +- fixed a potential error handling issue in Poco::Net::SecureSocketImpl. +- fixed SF# 3110272: RSACipherImpl bug. +- fixed SF# 3081677: ConfigurationView's getRaw not retrieving xml attributes. +- added basic support for Canonical XML and better pretty-printing support to Poco::XML::XMLWriter. +- Poco::Util::AbstractConfiguration now supports events fired when changing or + removing properties. +- XML: added support for finding DOM nodes by XPath-like + expressions. Only a very minimal subset of XPath is supported. + See Poco::XML::Element::getNodeByPath(), Poco::XML::Element::getNodeByPathNS() + and the same methods in Poco::XML::Document. +- Poco::Timer: If the callback takes longer to execute than the + timer interval, the callback function will not be called until the next + proper interval. The number of skipped invocations since the last + invocation will be recorded and can be obtained by the callback + by calling skipped(). +- Poco::BinaryReader and Poco::BinaryWriter now support reading and + writing std::vectors of the supported basic types. Also, strings + can now be written in a different encoding (a Poco::TextEncoding + can be optionally passed to the constructor). +- Poco::UUID::nil() and Poco::UUID::isNil() have been renamed to + Poco::UUID::null() and Poco::UUID::isNull(), respectively, to avoid + issues with Objective-C++ projects on Mac OS X and iOS where nil is + a system-provided macro. +- Crypto bugfixes: RSACipherImpl now pads every block of data, not just the + last (or last two). +- Improved Crypto testsuite by adding new tests +- Added new Visual Studio project configurations: debug_static_mt and release_static_mt + (linking with static runtime libraries). The existing configurations debug_static + and release_static have been renamed to debug_static_md and release_static_md, respectively. + The suffixes of the static libraries have also changed. The static_md configurations + now build libraries with suffixes md[d], while the libraries built by the static_mt + configurations have mt[d] suffixes. +- Added Visual Studio project files for 64-bit builds. +- Added Visual Studio 2010 project files. +- Removed the use of local static objects in various methods due to + their construction not being threadsafe (and thus leading to + potential race conditions) on Windows/Visual C++. +- Fixed some warning on 64-bit Windows builds. +- The name of the Data connector libraries have changed. They are now + named PocoDataMySQL, PocoDataODBC and PocoDataSQLite. +- fixed SF# 3125498: Linux NetworkInterface::list() doesn't return IPv6 IPs +- fixed SF# 3125457: IPv6 IPAddress tests are wrong +- Added initialization functions for the NetSSL_OpenSSL and Crypto libraries. + These should be called instead of relying on automatic initialization, + implemented with static initializer objects, as this won't work with + statically linked executables (where the linker won't include the + static initializer object). + The functions are Poco::Crypto::initializeCrypto(), Poco::Crypto::uninitializeCrypto(), + Poco::Net::initializeSSL() and Poco::Net::uninitializeSSL(). + Applications using Crypto and/or NetSSL should call these methods appropriately at + program startup and shutdown. + Note: In release 1.3.6, similar functions have been added to the Net library. + + +Release 1.3.6p2 (2010-01-15) +============================ + +- fixed an issue in the Windows implementation Poco::RWLock, where + tryReadLock() sometimes would return false even if no writers + were using the lock (fix contributed by Bjrn Carlsson) +- added Poco::Environment::libraryVersion(). +- fixed SF# 2919461: Context ignores parameter cypherList +- removed an unused enum from RSACipherImpl.cpp (Crypto) +- integrated a new expat patch for CVE-2009-3560. +- fixed SF# 2926458: SSL Context Problem. The Poco::Net::Context + class now makes sure that OpenSSL is properly initialized. +- updated iPhone build configuration (contributed by Martin York) +- fixed SF# 1815124 (reopened): XML Compile failed on VS7.1 with + XML_UNICODE_WCHAR_T +- fixed SF# 2932647: FTPClientSession::getWorkingDirectory() returns a bad result + + +Release 1.3.6p1 (2009-12-21) +============================ + +- added support for using external zlib, pcre, expat and sqlite3 instead of + bundled ones (-DPOCO_UNBUNDLED, configure --unbundled) +- fixed SF# 2911407: Add sh4 support +- fixed SF# 2912746: RSAKey::EXP_LARGE doesn't work +- fixed SF# 2904119: abstractstrategy uses std::set but do not includes it +- fixed SF# 2909946: localtime NULL pointer +- fixed SF# 2914986: potential expat DoS security issues (CVE-2009-3560 and CVE-2009-3720) +- fixed SF# 2916305: SSL Manager crashes +- fixed SF# 2903676: Tuple TypeHander does not handle composites. + + +Release 1.3.6 (2009-11-24) +========================== + +- added Environment::processorCount() +- added POCO_VERSION macro to Poco/Foundation.h +- fixed SF# 2807527: Poco::Timer bug for long startInterval/periodic interval +- fixed a bug similar to SF# 2807527 in Poco::Util::Timer. +- fixed SF# 2795395: Constructor doesn't treat the params "key" and "iv" +- fixed SF# 2804457: DateTime::checkLimit looks wrong +- fixed SF# 2804546: DateTimeParser requires explicit RFC1123 format +- added ReleaseArrayPolicy to Poco::SharedPtr +- upgraded to SQLite 3.6.20 +- fixed SF# 2782709: Missing semicolons in "Logger.h" convenience +- fixed SF# 2526407: DefaultStrategy.h ++it instead of it++ in a loop +- fixed SF# 2502235: Poco STLPort patch +- fixed SF# 2186643: Data::Statement::reset() not implemented in 1.3.3 +- fixed SF# 2164227: Allow File opened read only by FileInputSteam to be writable +- fixed SF# 2791934: use of char_traits::copy in BufferedStreamBuf::underflow +- fixed SF# 2807750: Support additional SQL types in SQLite +- fixed documentation bugs in Timed/PriorityNotificationQueue +- fixed SF# 2828401: Deadlock in SocketReactor/NotificationCenter (also fixes patch# 1956490) + NotificationCenter now uses a std::vector internally instead of a std::list, and the mutex is + no longer held while notifications are sent to observers. +- fixed SF# 2835206: File_WIN32 not checking aganist INVALID_HANDLE_VALUE +- fixed SF# 2841812: Posix ThreadImpl::sleepImpl throws exceptions on EINTR +- fixed SF# 2839579: simple DoS for SSL TCPServer, HTTPS server + No SSL handshake is performed during accept() - the handshake is delayed until + sendBytes(), receiveBytes() or completeHandshake() is called for the first time. + This also allows for better handshake and certificate validation when using + nonblocking connections. +- fixed SF# 2836049: Possible handle leak in FileStream + If sync() fails, close() now simply set's the stream's bad bit. + In any case, close() closes the file handle/descriptor. +- fixed SF# 2814451: NetSSL: receiveBytes crashes if socket is closed +- added a workaround for Vista service network initialization issue + (an Windows service using the Net library running under Vista will + crash in the call to WSAStartup() done in NetworkInitializer). + Workaround is to call WSAStartup() in the application's main(). + Automatic call to WSAStartup() in the Net library can now be disabled + by compiling Net with -DPOCO_NET_NO_AUTOMATIC_WSASTARTUP. Also + the new Poco::Net::initializeNetwork() and Poco::Net::uninitializeNetwork() + functions can be used to call WSAStartup() and WSACleanup(), respectively, + in a platform-independent way (on platforms other than Windows, these + functions will simply do nothing). +- added VCexpress build script support (contributed by Jolyon Wright) +- fixed SF# 2851052: Poco::DirectoryIterator copy constructor is broken +- fixed SF# 2851197: IPAddress ctor throw keyword missing +- added Poco::ProtocolException +- PageCompiler improvements: new tags, support for buffered output, etc. +- better error reporting in Data MySQL connector (patch #2881270 by Jan "HanzZ" Kaluza) +- fixed SF# 1892462: FTPClient:Choose explicitely between EPSV and PASV +- fixed SF# 2806365: Option for PageCompiler to write output to different dir +- fixed a documentation bug (wrong sample code) in Process::launch() documentation +- added --header-output-dir option to PageCompiler +- fixed SF# 2849144: Zip::Decompress notifications error +- SAXParser has a new feature: "http://www.appinf.com/features/enable-partial-reads". + See ParserEngine::setEnablePartialReads() for a description of what this does. +- fixed SF# 2876179: MySQL Signed/Unsigned value bug +- fixed SF# 2877970: possible bug in timer task +- fixed SF# 2874104: wrong parsing empty http headers +- fixed SF# 2860694: Incorrect return code from SecureStreamSocketImpl::sendBytes +- fixed SF# 2849750: Possible bug with XMLWriter? +- added MailMessage::encodeWord() to support RFC 2047 word encoded + mail header fields when sending out mail containing non-ASCII + characters. +- fixed SF# 2890975: SMTPClientSession bug with 7BIT encoding +- fixed an issue with retrieving the value of socket options on Windows 7. + Before obtaining the value of a socket, we now initialize the variable receiving the + socket option value to zero. +- fixed SF# 2836141: Documentation errors +- fixed SF# 2864232: Socket::select() does not detect closed sockets on windows +- fixed SF# 2812143: Socket::select() should check socket descriptors... +- fixed SF# 2801750: NetworkInterface forName returns wrong subnetMask +- fixed SF# 2816315: Problem with POSIX Thread::sleepImpl +- fixed SF# 2795646: IPv6 address parsing bug +- fixed #0000092: ServerApplication::waitForTerminationRequest(), SIGINT and GDB. + Poco::Util::ServerApplication::waitForTerminationRequest() no longer registers a + signal handler for SIGINT if the environment variable POCO_ENABLE_DEBUGGER + is defined. +- fixed SF# 2896070: Poco::Net::Context with non-ASCII paths +- added Unicode Surrogate support to Poco::UTF16Encoding. + See Poco::TextEncoding::queryConvert() and Poco::TextEncoding::sequenceLength() + for how this is implemented. Contributed by Philippe Cuvillier. +- fixed SF# 2897650: [branch 1.3.6] Net.SocketAddress won't compile for CYGWIN +- fixed SF# 2896161: Building on Windows fails when basedir has space in it +- fixed SF# 2864380: Memory leak when using secure sockets +- NetSSL_OpenSSL: the SSL/TLS session cache is now disabled by default and + can be enabled per Context using Poco::Net::Context::enableSessionCache(). +- fixed SF# 2899039: Wrong DST handling in LocalDateTime +- added RWLock::ScopedReadLock and RWLock::ScopedWriteLock (contributed by Marc Chevrier) +- added Thread::TID type, as well as Thread::tid() and Thread::currentTid() to obtain the native + thread handle/ID +- added Zip file comment support +- On Windows, Poco::SharedLibrary::load() now uses LoadLibraryEx instead of LoadLibrary + and uses the LOAD_WITH_ALTERED_SEARCH_PATH if an absolute path is specified. This will + add the directory containing the library to the search path for DLLs that the + loaded library depends upon. +- Mac OS X build settings now match those used by default Xcode projects, making linking the + POCO libs to Xcode projects easier +- Replaced use of std::valarray in Poco::Net::ICMPEventArgs with std::vector due to issues with + std::valarray together with STDCXX debug mode on OS X + + +Release 1.3.5 (2009-05-11) +========================== + +- fixed SF# 2779410: Poco::Data::ODBC::HandleException impovement +- fixed wrong exception text for Poco::UnhandledException +- Fixed a problem with SSL shutdown that causes clients (web browsers) + to hang when the server attempts to perform a clean SSL shutdown. We now call + SSL_shutdown() once, even if the shutdown is not complete after the first call. +- added Poco::Crypto::X509Certificate::save() +- fixed a bug in Poco::Zip::Decompress that results in wrong paths for extracted files +- fixed a bug in Poco::Zip::ZipManipulator where the Zip file was opened in text format + on Windows. +- added Poco::Crypto::X509Certificate::issuedBy() to verify certificate chain. +- fixed 0000089: Thread::sleep() on Linux is extremely inaccurate +- added methods to extract the contents of specific fields from the + subject and issuer distinguished names of a certificate. + + +Release 1.3.4 (2009-04-21) +========================== + +- fixed SF# 2611804: PropertyFileConfiguration continuation lines +- fixed SF# 2529788: ServerApplication::beDaemon() broken +- fixed SF# 2445467: Bug in Thread_WIN32.cpp +- Improved performance of HTTP Server by removing some + string copy operations +- fixed SF# 2310735: HTTPServer: Keep-Alive only works with send() +- fixed appinf.com IP address in Net testsuite +- fixed RFC-00188: NumberFormatter and float/double numbers +- added --pidfile option to ServerApplication on Unix +- fixed SF# 2499504: Bug in Win32_Thread when using from dll (fixed also for POSIX threads) +- fixed SF# 2465794: HTTPServerRequestImpl memory leak +- fixed SF# 2583934: Zip: No Unix permissions set +- the NetSSL_OpenSSL library has been heavily refactored +- added NumberFormatter::append*() and DateTimeFormatter::append() functions +- use NumberFormatter::append() and DateTimeFormatter::append() instead of format() where + it makes sense to gain some performance +- added system.dateTime and system.pid to Poco::Util::SystemConfiguration +- added %F format specifier (fractional seconds/microseconds) to DateTimeFormatter, + DateTimeParser and PatternFormatter. +- fixed SF# 2630476: Thread_POSIX::setStackSize() failure with g++ 4.3 +- fixed SF# 2679279: Handling of -- option broken +- added compile options to reduce memory footprint of statically linked applications + by excluding various classes from automatically being linked. + See the POCO_NO_* macros in Poco/Config.h. +- fixed SF# 2644940: on Windows the COMPUTER-NAME and the HOSTNAME can be different +- added DNS::hostName() function +- added build configuration for iPhone (using Apple's SDK) +- basic support for AIX 5.x/xlC 8 +- fixed a bug resulting in a badly formatted exception message with IOException + thrown due to a socket-related error +- fixed SF# 2644718: NetworkInterface name conflict in MinGW +- added a missing #include to CryptoTransform.h +- fixed SF# 2635377: HTTPServer::HTTPServer should take AutoPtr +- replaced plain pointers with smart pointers in some interfaces +- upgraded to sqlite 3.6.13 +- improved Data::SQLite error reporting +- Poco::Glob now works with UTF-8 encoded strings and supports case-insensitive comparison. + This also fixes SF# 1944831: Glob::glob on windows should be case insensitve +- added Twitter client sample to Net library +- Fixed SF# 2513643: Seg fault in Poco::UTF8::toLower on 64-bit Linux +- Poco::Data::SessionPool: the janitor can be disabled by specifying a zero idle time. +- added Poco::Data::SessionPool::customizeSession() +- added support for different SQLite transaction modes (DEFERRED, IMMEDIATE, EXCLUSIVE) +- fixed a few wrong #if POCO_HAVE_IPv6 in the Net library +- added support for creating an initialized, but unconnected StreamSocket. +- added File::isDevice() +- added family() member function to SocketAddress, +- Data::SQLite: added support for automatic retries if the database is locked +- XMLConfiguration is now writable +- fixed an IPv6 implementation for Windows bug in HostEntry +- Timer class improvement: interval between callback is no longer influenced by the + time needed to execute the callback. +- added PriorityNotificationQueue and TimedNotificationQueue classes to Foundation. + These are variants of the NotificationQueue class that support priority and + timestamp-tagged notifications. +- added Poco::Util::Timer class. This implements a timer that can schedule different + tasks at different times, using only one thread. +- the signatures of Poco::NotificationQueue and Poco::NotificationCenter member functions + have been changed to accept a Poco::Notification::Ptr instead of Poco::Notification* + to improve exception safety. This change should be transparent and fully backwards + compatible. The signature of the methods returning a Poco::Notification* have not been + changed for backwards compatibility. It is recommended, that any Notification* obtained + should be immediately assigned to a Notification::Ptr. +- SQLite::SessionImpl::isTransaction() now uses sqlite3_get_autocommit() to find out + about the transaction state. +- refactored Crypto library to make it independent from NetSSL_OpenSSL. +- added support for RSA-MD5 digital signatures to Crypto library. +- removed SSLInitializer from NetSSL library (now moved to Crypto library) +- added build configs for static libraries to Crypto library +- OpenSSL now depends on Crypto library (which makes more sense than + vice versa, as it was before). Poco::Net::X509Certificate is now + a subclass of Poco::Crypto::X509Certificate (adding the verify() + member function) and the Poco::Net::SSLInitializer class was + moved to Poco::Crypto::OpenSSLInitializer. +- added build configs for static libraries to Zip +- added batch mode to CppUnit::WinTestRunner. + WinTestRunnerApp supports a batch mode, which runs the + test using the standard text-based TestRunner from CppUnit. + To enable batch mode, start the application with the "/b" + or "/B" command line argument. Optionally, a path to a file + where the test output will be written to may be given: + "/b:" or "/B:". + When run in batch mode, the exit code of the application + will denote test success (0) or failure (1). +- testsuites now also work for static builds on Windows +- The IPv6 support for Windows now basically works (Net library compiled with POCO_HAVE_IPv6) +- fixed a potential error when shutting down openssl in a statically linked application +- added static build configs to Data library +- added Poco::AtomicCounter class, which uses OS-specific APIs for atomic (thread-safe) + manipulation of counter values. +- Poco::RefCountedObject and Poco::SharedPtr now use Poco::AtomicCounter for + reference counting +- fixed SF# 2765569: LoadConfiguration failing from current directory + + +Release 1.3.3p1 (2008-10-09) +============================ + +- Fixed SF# 2153031: 1.3.3 Crypto won't compile on 64-bit Linux +- Fixed a warning in MySQL connector +- Updated README +- The global Makefile in the Zip archive is no longer broken + + +Release 1.3.3 (2008-10-07) +========================== + +- Threads now have optional user-settable stack size (if the OS supports that feature) +- Events now support simplified delegate syntax based on delegate function template. + See Poco::AbstractEvent documentation for new syntax. +- Cache supports new access expire strategy. +- Upgraded to SQLite 3.6.2 +- Upgraded to PCRE 7.8 +- added HttpOnly support to Poco::Net::HTTPCookie +- NetworkInterface now has displayName() member (useful only on Windows) +- Poco::Util::WinRegistryKey now has a read-only mode +- Poco::Util::WinRegistryKey::deleteKey() can now recursively delete registry keys +- Poco::File::created() now returns 0 if the creation date/time is not known, as + it's the case on most Unix platforms (including Linux). + On FreeBSD and Mac OS X, it returns the real creation time. +- Time interval based log file rotation (Poco::FileChannel) now works + correctly. Since there's no reliable and portable way to find out the creation + date of a file (Windows has the tunneling "feature", most Unixes don't provide + the creation date), the creation/rotation date of the log file is written into + the log file as the first line. +- added Environment::nodeId() for obtaining the Ethernet address of the system + (this is now also used by UUIDGenerator - the corresponding code from UUIDGenerator + was moved into Environment) +- added a release policy argument to SharedPtr template +- Socket::select() will no longer throw an InvalidArgumentException + on Windows when called with no sockets at all. If all three socket + sets are empty, Socket::select() will return 0 immediately. +- SocketReactor::run() now catches exceptions and reports them via + the ErrorHandler. +- SocketReactor has a new IdleNotification, which will be posted when + the SocketReactor has no sockets to handle. +- added referenceCount() method to Poco::SharedPtr. +- POCO now builds with GCC 4.3 (but there are some stupid warnings: + "suggest parentheses around && within ||". +- Solution and project files for Visual Studio 2008 are included +- The Zip library is now officially part of the standard POCO release. +- The Crypto library (based on OpenSSL) has been added. The original code + was kindly contributed by Ferdinand Beyer. +- A Data Connector to MySQL, contributed by Sergey Kholodilov, is now part + of the POCO release. +- fixed SF# 1859738: AsyncChannel stall +- fixed SF# 1815124: XML Compile failed on VS7.1 with XML_UNICODE_WCHAR_T +- fixed SF# 1867340: Net and NetSSL additional dependency not set - ws2_32.lib +- fixed SF# 1871946: no exception thrown on error +- fixed SF# 1881113: LinearHashTable does not conform to stl iterators +- fixed SF# 1899808: HTMLForm.load() should call clear() first +- fixed SF# 2030074: Cookie problem with .NET server +- fixed SF# 2009707: small bug in Net/ICMPPacketImpl.cpp +- fixed SF# 1988579: Intel Warning: invalid multibyte character sequence +- fixed SF# 2007486: Please clarify license for Data/samples/* +- fixed SF# 1985180: Poco::Net::DNS multithreading issue +- fixed SF# 1968106: DigestOutputStream losing data +- fixed SF# 1980478: FileChannel loses messages with "archive"="timestamp" +- fixed SF# 1906481: mingw build WC_NO_BEST_FIT_CHARS is not defined +- fixed SF# 1916763: Bug in Activity? +- fixed SF# 1956300: HTTPServerConnection hanging +- fixed SF# 1963214: Typo in documentation for NumberParser::parseFloat +- fixed SF# 1981865: Cygwin Makefile lacks ThreadTarget.cpp +- fixed SF# 1981130: pointless comparison of unsigned integer with zero +- fixed SF# 1943728: POCO_APP_MAIN namespace issue +- fixed SF# 1981139: initial value of reference to non-const must be an lvalue +- fixed SF# 1995073: setupRegistry is broken if POCO_WIN32_UTF8 enabled +- fixed SF# 1981125: std::swap_ranges overloading resolution failed +- fixed SF# 2019857: Memory leak in Data::ODBC Extractor +- fixed SF# 1916761: Bug in Stopwatch? +- fixed SF# 1951443: NetworkInterface::list BSD/QNX no netmask and broadcast addr +- fixed SF# 1935310: Unhandled characters in Windows1252Encoding +- fixed SF# 1948361: a little bug for win32 +- fixed SF# 1896482: tryReadLock intermittent error +- workaround for SF# 1959059: Poco::SignalHandler deadlock + the SignalHandler can now be disabled globally by adding a + #define POCO_NO_SIGNAL_HANDLER to Poco/Config.h +- fixed SF# 2012050: Configuration key created on read access +- fixed SF# 1895483: PCRE - possible buffer overflow +- fixed SF# 2062835: Logfile _creationDate is wrong +- fixed SF# 2118943: out_of_bound access in Poco::Data::BLOB:rawContent +- fixed SF# 2121732: Prevent InvalidArgumentException in SocketReactor +- fixed SF# 1891132: Poco::Data::StatementImpl::executeWithLimit is not correct +- fixed SF# 1951604: POCO refuses to compile with g++ 4.3.0 +- fixed SF# 1954327: CYGWIN's pthread does not define PTHREAD_STACK_MIN +- fixed SF# 2124636: Discrepancy between FileWIN32(U)::handleLastError +- fixed SF# 1558300: MinGW/MSYS Builds +- fixed SF# 2123266: Memory leak under QNX6 with dinkum library +- fixed SF# 2140411: ScopedUnlock documentation bug +- fixed SF# 2036460: UUID regression tests are failing on Linux with g++ 4.3.1 +- fixed SF# 2150438: Tuple TypeHandler position increment size is wrong + + +Release 1.3.2 (2008-02-04) +========================== + +Foundation, XML, Net, Util: +- added POCO_NO_SHAREDMEMORY to Config.h +- POCO_NO_WSTRING now really disables all wide string related calls +- added template specialization for string hashfunction (performance) +- XML parser performance improvements (SAX parser is now up to 40 % faster +- added parseMemoryNP() to XMLReader and friends +- URIStreamOpener improvement: redirect logic is now in URIStreamOpener. + this enables support for redirects from http to https. +- added support for temporary redirects and useproxy return code +- added getBlocking() to Socket +- added File::isHidden() +- better WIN64 support (AMD64 and IA64 platforms are recognized) +- added support for timed lock operations to [Fast]Mutex +- SharedLibrary: dlopen() is called with RTLD_GLOBAL instead of RTLD_LOCAL + (see http://gcc.gnu.org/faq.html#dso) +- Poco::Timer threads can now run with a specified priority +- added testcase for SF# 1774351 +- fixed SF# 1784772: Message::swap omits _tid mem +- fixed SF# 1790894: IPAddress(addr,family) doesn't fail on invalid address +- fixed SF# 1804395: Constructor argument name wrong +- fixed SF# 1806807: XMLWriter::characters should ignore empty strings +- fixed SF# 1806994: property application.runAsService set too late +- fixed SF# 1828908: HTMLForm does not encode '+' +- fixed SF# 1831871: Windows configuration file line endings not correct. +- fixed SF# 1845545: TCP server hangs on shutdown +- fixed SF# 1846734: Option::validator() does not behave according to doc +- fixed SF# 1856567: Assertion in DateTimeParser::tryParse() +- fixed SF# 1864832: HTTP server sendFile() uses incorrect date +- HTTPServerResponseImpl now always sets the Date header automatically + in the constructor. +- fixed SF# 1787667: DateTimeFormatter and time related classes + (also SF# 1800031: The wrong behavior of time related classes) +- fixed SF# 1829700: TaskManager::_taskList contains tasks that never started +- fixed SF# 1834127: Anonymous enums in Tuple.h result in invalid C++ +- fixed SF# 1834130: RunnableAdapter::operator= not returning a value +- fixed SF# 1873924: Add exception code to NetException +- fixed SF# 1873929: SMTPClientSession support for name in sender field +- logging performance improvements (PatternFormatter) +- fixed SF# 1883871: TypeList operator < fails for tuples with duplicate values +- CYGWIN build works again (most things work but Foundation testsuite still fails) +- new build configuration for Digi Embedded Linux (ARM9, uclibc) +- new build configuration for PowerPC Linux + +Data: +- fixed SF# 1724388: ODBC Diagnostics +- fixed SF# 1804797: ODBC Statement multiple execution fails +- fixed SF# 1803435: SessionPool onJanitorTimer called too often? +- fixed SF# 1851997: Undefined Behavior in ODBC::Preparation +- updated SQlite to 3.5.5 + + +Release 1.3.1 (2007-08-08) +========================== + +Foundation, XML, Net, Util: +- DynamicAny fixes for char conversions +- fixed SF# 1733362: Strange timeout handling in SocketImpl::poll and Socket::select +- fixed SF patch# 1728912: crash in POCO on Solaris +- fixed SF# 1732138: Bug in WinRegistryConfiguration::getString +- fixed SF# 1730790: Reference counting breaks NetworkInterface::list() +- fixed SF# 1720733: Poco::SignalHandler bug +- fixed SF# 1718724: Poco::StreamCopier::copyStream loops forever +- fixed SF# 1718437: HashMap bug +- changed LinearHashTable iterator implementation. less templates -> good thing. +- fixed SF# 1733964: DynamicAny compile error +- UUIDGenerator: fixed infinite loop with non ethernet interfaces +- updated expat to 2.0.1 +- fixed SF# 1730566: HTTP server throws exception +- Glob supports symbolic links (additional flag to control behavior) +- fixed a problem with non blocking connect in NetSSL_OpenSSL + (see http://www.appinf.com/poco/wiki/tiki-view_forum_thread.php?comments_parentId=441&topics_threshold=0&topics_offset=29&topics_sort_mode=commentDate_desc&topics_find=&forumId=6) +- fixed a problem with SSL renegotiation in NetSSL_OpenSSL (thanks to Sanjay Chouksey for the fix) +- fixed SF# 1714753: NetSSL_OpenSSL: HTTPS connections fail with wildcard certs +- HTTPClientSession: set Host header only if it's not already set (proposed by EHL) +- NetworkInterface (Windows): Loopback interface now has correct netmask; + interfaces that do not have an IP address assigned are no longer reported. +- Fixes for VC++ W4 warnings from EHL +- SharedMemory: first constructor has an additional "server" parameter + Setting to true does not unlink the shared memory region when the SharedMemory object is destroyed. (Alessandro Oliveira Ungaro) +- fixed SF# 1768231: MemoryPool constructor + +Data: +- fixed SF# 1739989: Data::RecordSet::operator = () (in 1.3 branch) +- fixed SF# 1747525: SQLite, Transactions and Session Pooling (in 1.3 branch) +- upgraded to SQLite 3.4.1 + + +Release 1.3.0 (2007-05-07) +========================== + +- added HashMap, HashSet classes +- the HashFunction class template has been changed in an incompatible + way. The member function formerly named hash() is now the function + call operator. If you have defined your own HashFunction classes, + you have to update your code. Sorry for the inconvenience. +- added Poco::Tuple +- added AbstractCache::getAllKeys(), improved performance of the get operation +- fixed AbstractCache::size() to do cache replacement before returning the size +- added additional match() method to RegularExpression and documented the fact that the simple + match() method internally sets RE_ANCHORED and RE_NOTEMPTY. +- added ExpirationDecorator template. Decorates data types so that they can be used with UniqueExpireCaches +- added operator ! to AutoPtr and SharedPtr +- Buffer uses std::size_t instead of int +- Exception::what() now returns exception name instead of message +- added poco_ndc_dbg() macro (same as poco_ndc(), but only enabled in debug builds) +- added Environment::get(name, defaultValue); +- Foundation.h now includes Config.h at the very beginning. +- added replace() and replaceInPlace() to Poco/String.h +- added AutoPtr::assign() and SharedPtr::assign() +- added operator () to AbstractEvent +- gcc Makefiles now strip release builds +- Void now has a == and != operator +- Base64Encoder and HexBinaryEncoder now support an unlimited line length + (no newlines written), by specifying a line length of 0 +- NumberParser now has stricter syntax requirements: garbage following a number leads to a SyntaxException + (Thanks to phireis@gmail.com for the suggestion) +- fixed SF# 1676830: Don't use -rpath in libraries +- fixed SF# 1670279: AbstractConfiguration::unckeckedExpand crash +- fixed a warning in Hashtable +- HTTPClientSession now uses a keepAliveTimeout for better persistent connection handling +- added DateTime::makeUTC() and DateTime::makeLocal() +- added another constructor to LocalDateTime +- POCO_WIN32_UTF8 is ignored on non-Windows platforms +- fixed a timeout bug (with NetSSL) in HTTPSession +- AsyncChannel is automatically opened with first log() +- minor fix to NotificationQueue sample (reported by Laszlo Keresztfalvi) +- added File::canExecute() and File::setExecutable() +- added SharedMemory class to Foundation +- added FileStream, FileInputStream, FileOutputStream to Foundation +- added NodeAppender class to XML for faster DOM tree creation +- HTTPServerRequest and HTTPServerResponse are now abstract base classes, + actual functionality has moved into HTTPServerRequestImpl and + HTTPServerResponseImpl. This allows us to plug other HTTP servers + into POCO. +- added DynamicAny class to Foundation +- replaced std::fstream with Poco::FileStream across POCO. +- added Poco::Checksum class to Foundation. +- fixed SF# 1700811: conflict in threadpool +- bugfix: File::moveTo() does not work if the target is a directory +- File::copyTo() and File::moveTo() now copy/move directories recursively +- refactored NetworkInterface (now using pimpl idiom); + added broadcast address and netmask support +- fixed SF# 1688982: POP3ClientSession fails when retrieving mails with attachment +- fixed SF# 1655104: Enhance Poco::TextEncoding functionality +- added Poco::Condition class, implementing a POSIX-style condition variable +- fixed a bug in File::create() for Windows +- added poco_static_assert (imported from boost) +- added Thread::join(timeout) and Thread::tryJoin() +- ClassLoader support for named manifests (see ClassLibrary.h - POCO_EXPORT_NAMED_MANIFEST) +- POCO_WIN32_UTF8: UNICODE #define is no longer required (and no longer + automatically defined in POCO_WIN32_UTF8 is defined) +- PCRE: upgraded to PCRE version 7.1 +- fixed SF# 1682162: Suggestion on thread priority +- fixed SF# 1613460: MSVC/STLPort warnings +- fixed SF# 1709358: Format double percent std::String bug +- added WindowsConsoleChannel class to Foundation +- added AutoPtr::unsafeCast<>() and SharedPtr::unsafeCast<>() +- fixed SF# 1708552: Failed to build on arm and powerpc +- fixed SF$ 1708529: Failed to build using GCC 4.3: missing #includes +- fixed SF# 1710053: LogStream proposal +- fixed a bug involving empty root directories in Windows DirectoryIterator implementation + (see http://www.appinf.com/poco/wiki/tiki-view_forum_thread.php?comments_parentId=343&forumId=6) +- robustness improvements to ActiveMethod - removed the opportunity for memory leaks in + case something goes while invoking the method +- made C library usage more C++-like - use C++ headers (e.g. ) instead of + C ones (). Also, use C library functions in std namespace. +- added Unicode and UTF8String for improved Unicode support. + The Unicode class can be used to obtain the Unicode properties of a character. + The UTF8 class provides case insensitive comparison and case conversion + for UTF-8 encoded strings. +- added UnWindows.h header file, replaced all #include with #include "Poco/UnWindows.h". + See the Poco/UnWindows.h header file for a rationale and explanations. +- fixed SF# 1713820: StreamSocketImpl::sendBytes sends too many bytes +- File::copyTo(): on Windows, the copy now always has the read-only flag reset, to be consistent + with other platforms. +- With Microsoft Visual C++, the necessary POCO libraries are now implicitly linked when + the corresponding header files are included (#pragma comment(lib, "PocoXYZ.lib") is used). + To disable this, compile POCO with the preprocessor symbol POCO_NO_AUTOMATIC_LIBS #define'd + (see Poco/Foundation.h and Poco/Config.h). +- The Visual Studio project files for the POCO libraries now include configurations + for building static libraries. + + +Release 1.2.9 (2007-02-26) +========================== + +- fixed a formatting problem in Util::HelpFormatter +- HTTPClientSession::sendRequest() now attempts to send the complete request in one network packet. +- improved network performance of ChunkedOutputStream: chunk size and chunk data + are sent in one network packet if possible +- fixed SF# 1655035: Wrong expires field calculation in HTTPCookie + (thanks to Sergey N. Yatskevich for this and other fixes) +- fixed SF# 1655049: Fix discrepancy of a code to the description +- fixed SF# 1655170: Poco::Timezone::standardName() problem on WIN32 +- fixed SF# 1629095: POCO_WIN32_UTF8 problem + There is a new function Path::transcode() that can be used to convert a path (or any other string) + from UTF-8 to the current Windows code page. This string can the be passed as a filename + to an fstream or fopen(). This function only does the conversion on Windows, + and only, if POCO_WIN32_UTF8 is defined. Otherwise, it simply returns the unmodified argument. +- fixed SF# 1659607: Probably a bug in Poco::Net::DialogSocket +- HTTPServer network performance improvement: responses that fit into a single network packet + sent with HTTPServerResponse::sendFile() or the new HTTPServerResponse::sendBuffer() are + sent in only one packet. +- added HTTPServerResponse::sendBuffer() +- HTTPServer now sends a Bad Request response if it fails to parse the HTTP request header. +- HTTPServer now sends an Internal Server Error response if the request handler throws an + exception prior to sending a response.- enabled TCP_NODELAY per default on TCPServer/HTTPServer +- fixed a bug in HTTP persistent connection handling + (server does not send Connection: close when it reaches connection maximum) +- HTMLForm - POST submission of URL encoded form no longer uses chunked transfer encoding + (thus improving interoperability with certain web servers) +- integrated Environment.cpp from Main (missing get(var, default)) +- added missing AutoPtr include to Util/Application + (and using Poco::AutoPtr is no longer necessary for POCO_APP_MAIN macro) +- fixed SF# 1635420: Per Regents of the University of Calfornia letter, + remove advertising from BSD licensed parts +- fixed SF# 1633133: MultipartWriter writes superluous CR-LF at beginning + + +Release 1.2.8 (2007-01-04) +========================== + +- fixed SF# 1613906: Util/Application.h and GCC 3.3 +- fixed a byte order issue (failed test) in IPv6 address formatting +- fixed SF# 1626640: Poco::Net::SocketReactor bug +- fixed client side chunked transfer encoding handling +- fixed client side persistent connection handling +- fixed SF# 1623536: HTTP Server Chunked Transfer Encoding Bug +- improved HTTP server exception text +- fixed SF# 1616294: KeepAlive HTTPServerSession patch +- fixed SF# 1616296: Trivial Poco::TaskCustomNotification patch +- fixed SF# 1619282: PurgeStrategy bug fix +- fixed SF# 1620855: Format problem + there is a new format specifier %z for std::size_t, as well as a new + flag ? for %d, %i, %o, %x meaning any signed or unsigned integer + + +Release 1.2.7 (2006-12-07) +========================== + +- Poco::File: fixed root directory handling +- fixed UUIDGenerator documentation +- clarified Application::setUnixOptions() documentation +- fixes for issue [SOAPLite Transport 0000023]: SOAP Transport Listener should be able to use existing HTTPServer instance +- fixing mantis issues 13, 14, 15, 16, 17, 18, 19, 21 +- fixed SF# 1597022: Signed/unsigned warning in StringTokenizer::operator[] +- fixed SF# 1598601: Message::op= leaks +- fixed SF# 1605960: PatternFormatter crashes on custom property +- fixed SF# 1605950: Memory leak in Logger sample code +- fixed SF# 1591635: Copy Paste Error in sample code +- fixed SF# 1591512: SMTPClientSession response stream +- fixed SF #1592776: LayeredConfiguration: getRaw should enumerate in reverse order +- SF Patch # 1599848 ] VS 2005 Util build fails +- Logger::dump() now uses std::size_t instead of int for buffer size +- LayeredConfiguration now supports a priority value for each configuration. + Also, it's possible to specify for each configuration added whether it + should be writeable. +- ServerApplication: cd to root directory only if running as a daemon +- added Message::swap() +- improvements to build system: + global Makefile has correct dependencies for samples + on Windows, samples build after libraries are ready + configure supports --no-wstring and --no-fpenvironment flags + build system supports POCO_FLAGS environment variable for compiler flags +- RemoteGen: fixed error handling for write protected files (SystemException) + fixing integral constant overflow messages with large cache expiration, m_ support for type serializers, + case-insensitive comparison added + + +Release 1.2.6 (2006-11-19) +========================== + +- added additional match() method to RegularExpression and documented the fact that the simple + match() method internally sets RE_ANCHORED and RE_NOTEMPTY. +- added ExpirationDecorator template. Decorates data types so that they can be used with UniqueExpireCaches +- added operator ! to AutoPtr and SharedPtr +- Buffer uses std::size_t instead of int +- added poco_ndc_dbg() macro (same as poco_ndc(), but only enabled in debug builds) +- Foundation.h now includes Config.h at the very beginning. +- added AutoPtr::assign() and SharedPtr::assign() +- added operator () to AbstractEvent +- gcc Makefiles now strip release builds +- documentation improvements + + +Release 1.2.5 (2006-10-23) +========================== + +- Improved LoggingConfigurator: channel creation and configuration is now a two-step process. + This means that the previous problems with PropertyFileConfiguration and IniFileConfiguration when referencing other channels are solved. +- improved options handling: better handling of (non) ambiguities. + If both an option named "help" and one named "helper" is specified, this no longer causes ambiguity errors. +- added check for duplicate option definition +- ThreadPool bugfix: fixed a crash that occurs on Linux multiprocessor machines + (caused by an thread unsafe string assignment corrupting the heap...) + (SF# 1575315) +- improved ThreadPool performance +- XML now compiles with -DXML_UNICODE_WCHAR_T (SF# 1575174) +- fixed SF# 1572757: HTML forms can have more than one key/value pair with the same name +- got rid of the dynamic casts in Events, Events/Cache: simpler/faster Delegate < operator, + prevents some rare dynamic casts error from occuring when using StrategyCollection with Caches +- improvements to Logger and LoggingConfigurator: + * added Logger::unsafeGet() + * added Logger::setProperty(loggerName, propertyName, value) + * LoggingConfigurator now correctly (re)configures existing Loggers + (prior to this change, if a Logger named "a.b.c" existed before + the LoggingConfigurator started its work, and the LoggingConfigurator + configured a Logger named "a.b", then "a.b.c" would not inherit + the new configuration). +- improvements to SplitterChannel and EventLogChannel configuration +- improved LoggingRegistry exception messages +- MessageHeader::read() is more liberal with malformed message headers. + This fixes problems with certain network cameras sending malformed HTTP headers. + + +Release 1.2.4 (2006-10-02) +========================== + +- some code beautifying and improvements to comments +- DOMParser now automatically sets FEATURE_NAMESPACE_PREFIXES +- fixed SF #1567051: DOMBuilder/DOMParser/NamespaceStrategy bug +- fixed SF #1567364: POCO_APP_MAIN +- added Document::getElementById() (two-argument) and getElementByIdNS() +- added another test for DOMParser +- added AutoPtr::isNull() (to be consistent with SharedPtr) +- this release again compiles on PA-RISC HP-UX systems with aCC +- added CMAKE support files contributed by Andrew J. P. Maclean + + +Release 1.2.3 (2006-09-14) +========================== + +- configure script now checks if (auto)selected configuration is supported +- fixed SF #1552904: NamedEvent bug? +- fixed SF #1552787: POCO not handling EINTR +- fixed SF #1552846: Random::~Random uses scalar delete +- fixed SF #1552987: TLSSlot should explicitly default-construct _value +- IPAddress no longer accepts an empty address string +- split up Observer.h into AbstractObserver.h and Observer.h +- added NObserver class template which supports an AutoPtr + argument for the notification callback +- changed EchoServer sample to use NObserver +- some Windows-specific files were missing in the tarballs + + +Release 1.2.2 (2006-09-01) +========================== + +- fixed SF # 1549973: NotificationCenter::hasObservers() returns wrong result +- fixed a memory leak in EchoServer sample +- fixed SocketReactor TimeoutNotification bug (SF #1549365, SocketNotifier::addObserver() incorrect behavior) +- fixed SF# 1549513: MultipartReader does not work with Unix-style linefeeds +- MailMessage and HTMLForm: processing of multipart messages will no longer fail if a PartHandler does not read all data from the part stream. +- added additional test case (Unix-style line ends) to MultipartReaderTest + + +Release 1.2.1 (2006-08-29) +========================== + +- fixed Config.h header (no more #undefs) + +Release 1.2.0 (2006-08-29) +========================== + +- DateTime fixes: Julian Day is no longer stored internally. + Times (hours, minutes, seconds, ...) are now always taken from an utcValue (if available) and not from the Julian day. + The Julian day is only used for calculating year, month and day (except when the Julian day is the only thing we have) + This helps us get rid of rounding errors that the Julian Day arithmetic introduced.- on Windows, UUIDGenerator no longer uses Netbios, but GetAdaptersInfo instead +- The main Makefile now has correct dependencies +- updated poco-doc.pl with latest version by Caleb Epstein +- fixed SF #1542722: InflatingInputStream: buffer error +- improved Windows UTF-8 support +- added Logger::names() +- added configure script and make install target +- XMLWriter bugfix: pretty-print bug with characters() and rawCharacters() +- improvements to build system: support builds outside of source tree +- added header doc conversion tool contributed by Caleb Epstein +- fixed SF #1542618 (build/config/Linux patch) +- bugfix: BinaryReader/BinaryWriter BOM is now 16 bits, as documented +- fixed SF #1542247 (Compiler warning from OptionCallback) +- fixed SF #1542253 (ServerApplication::handleOption doesn't call Application::handleOption) +- added Application::stopOptionsProcessing() +- updated samples +- Util::Application command line handling now supports: + * argument validation (Option::validator(); see Validator, IntValidator, RegExpValidator) + * binding of argument values to config properties (Option::binding()) + * callbacks for arguments (Option::callback()) + * checking of required parameters +- changed header file locations: + Foundation headers are now in Poco (#include "Poco/Foundation.h") + XML headers are now in Poco/XML, Poco/SAX and Poco/DOM (#include "Poco/XML/XML.h") + Util headers are now in Poco/Util (#include "Poco/Util/Util.h") + etc. + Unfortunately, this change will break existing code. However, fixing the code is + a matter of a few global search/replace operations and can be done quickly. + On the plus side, POCO is now a much better citizen when used with other + libraries. +- changed namespaces: + Foundation is now Poco + XML is now Poco::XML + Util is now Poco::Util + Net is now Poco::Net +- removed namespace macros +- fixed some warnings reported by gcc -Wall -Wextra +- fixed AutoPtr and LayeredConfiguration documentation +- improved StreamSocket::receiveBytes() doc +- added Pipe and PipeStream classes +- added support for I/O redirection (pipes) to Process::launch() +- added LogStream class (ostream interface to Logger) +- improved Makefiles (no more double-building if clean all is specified) +- added CppUnit and DateTime testsuite contributions by Andrew Marlow +- improved Cygwin and minimal MinGW support +- FileChannel: gzip compression if archived files now runs in a background thread (SF #1537481) +- POCO now compiles with large (64-bit) file support on Linux (SF #1536634) +- added format() function, which provides typesafe sprintf-like functionality (SF #1327621) +- added File::isLink() +- bugfix: dangling symbolic links in a directory no longer cause recursive remove to fail with file not found error +- added Void class (useful as argument to ActiveMethod) +- ActiveResult now supports exceptions +- bugfix: Timezone::utcOffset() and Timezone::dst() returned wrong values on Unix platforms (SF #1535428) +- added ActiveDispatcher class +- added ActiveStarter class, which is a policy used by ActiveMethod for starting methods +- ActiveRunnable moved to its own header file +- ThreadPool: added startWithPriority(), which allows for running threads with a different priority +- added error handling to dir sample +- added additional test case to HTTPServer test suite- HTMLForm: should now work with request methods other than POST and GET (all non-POST requests are treated the same as GET) +- clarified HTMLForm documentation +- HTMLForm bugfix: uploaded files no longer end up in value; PartHandler is called instead +- NameValueCollection: added get(name, defaultValue) +- added HTTPFormServer sample +- added Foundation::HashTable and SimpleHashTable +- added Net::HTTPSessionFactory +- improvements to AutoPtr and SharedPtr +- improvements to namespaces handling in XMLWriter +- Foundation Cache: fixed add implementation to match the docu: a 2nd add will now simply overwrite existing entries +- added DateTime::isValid() +- added Exception::rethrow() (virtual, must be overridden by all subclasses) +- Timer can now use a user-supplied ThreadPool +- added rethrow() to exception classes +- Net: made some constructors explicit +- Net: added SocketAddress constructor to HTTPClientSession +- Net: added HTTPSession::networkException() to check for exceptions swallowed by stream classes +- Net: added single string argument constructor to SocketAddress. +- Net: improved HTTPClientSession error handling (no more "Invalid HTTP version string" exceptions when the server prematurely closes the connection due to too much load) +- Net: improved HTTPSession error handling. Exceptions while sending and receiving data are stored for later retrieval and no longer get lost since streambufs swallow them. +- Net: added HTTPLoadTest sample +- fixed a bug when opening logfiles on Unix platforms causing an existing logfile to be truncated +- bugfix: log file purge intervals given in months did not work, due to a stupid typo +- added RawSocket and ICMP classes +- UUID: fixed a doc formatting bug +- NetworkInterface::list() now includes loopback interface on Windows (SF #1460309) +- made Exception::message() and Exception::nested() inline +- added Net::UnsupportedRedirectException +- HTTPStreamFactory throws an UnsupportedRedirectException if it encounters a redirect to https +- HTTP: fixed bad 100 Continue handling in client and server code +- added CONTRIBUTORS file + + +Release 1.1.2 (2006-07-07) +========================== + +- Changed license to Boost license +- DBlite and NetSSL have been removed from the Boost-licensed release. + Please contact Applied Informatics (info@appinf.com) if you're interested in them. + + +Release 1.1.1 (2006-04-03) +========================== + +- NetSSL_OpenSSL now supports separate certificate verification + settings for client and server. +- fixed SF #1460309 (enumerating network interfaces failed on 64bit Linux) +- TCPServer no longer crashes if accept() fails + + +Release 1.1.0 (2006-03-23) +========================== + +- events no longer require awkward += new syntax +- source code and documentation cleanups +- basic support for new compilers and platforms + + +Release 1.1b2 (2006-03-04) +========================== + +- made NetSSL threadsafe (added locking callbacks for OpenSSL) +- improved OpenSSL initialization (random generator seeding) +- various changes to improve compatibility with various platforms + + +Release 1.1b1 (2006-03-03) +========================== + +- New Events package in Foundation. The package supports C#-style event handling +- New Cache package in Foundation: a templates-based caching framework +- added Any class to Foundation +- added DBLite library +- fixed a memory leak with layered configurations in the application +- made POCO_DLL the default (unless POCO_STATIC is #defined) + It is no longer necessary to specify POCO_DLL in projects that use Poco + (SourceForge Patch #1408231 and Feature Request #1407575). +- added Buffer template class to Foundation +- added the UnicodeConverter utility class. This is mainly used for Windows Unicode support and probably of little use for anything else. +- added Path::resolve() +- added Windows Unicode support. This calls the Unicode variant of the Windows API functions. + For this to work, all strings must be UTF-8 encoded and POCO_WIN32_UTF8 must be defined in all compilation units. +- added StreamCopier::copyToString() +- added URIStreamOpener::unregisterStreamFactory() and new variants of URIStreamOpener::open() that also work with filesystem paths. + This fixes SourceForge Bug #1409064 and Feature Request #1409062. +- added NodeIterator::currentNodeNP() to XML library +- added some sanity checks to UTF8Encoding::convert() +- added NetSSL - SSL support for Net library, based on OpenSSL +- console output of processes launched with Process::launch() is now visible + + +Release 1.0.0 (2006-01-19) +========================== + +- removed unnecessary console output from ProcessTest +- documentation fixes + + +Release 1.0b2 (2006-01-16) +========================== + +- added ProcessHandle class +- Process::launch() now returns a ProcessHandle instead of a process ID. + This fixes a potential problem on Windows with Process::wait() when + the process terminates before wait() is called. +- added SplitterChannel::close() +- added Logger::destroy() +- added POP3ClientSession::deleteMessage() +- added test for Process::launch() +- documentation fixes + + +Release 1.0b1 (2006-01-09) +========================== + +- improved recognition of Windows paths in Path::parseGuess() +- added setCurrentLineNumber()/getCurrentLineNumber() to CountingStreamBuf +- improvememts to StreamTokenizer and Token; fixed documentation +- added a workaround for some strange istream behaviour with VS 2005 and FTPClientSessionTest +- improved exception/error reporting in cppunit +- added POP3ClientSession +- added Process::launch() and Process::wait() +- added Mail sample +- added MailStream and SMTPClientSession classes +- renamed some methods in DialogSocket to make them more general +- NullPartHandler has moved out of HTMLForm.cpp into a separate file +- Base64Encoder now always writes \r\n line ends +- MessageHeader::quote has an optional addition arg controlling the treatment of whitespace +- bugfix: MultipartReader had a problem with empty lines (\r\n sequences) in a part +- added MailMessage and MailRecipient classes +- added text encoding support for Windows-1252 codepage + + +Release 1.0a1 (2006-01-03) [internal] +===================================== + +- mediaType is used consistently to refer to a MIME media type (some occurences of contentType and mimeType have been replaced) +- moved MediaType::quote() to MessageHeader and made it public +- added MultipartWriter::stream() +- Renamed AttachmentSource to PartSource and AttachmentHandler to PartHandler +- SIGPIPE is always blocked in main thread on Unix systems +- added EchoServer sample +- fixed a bug in SocketImpl::setBlocking() - did exactly the opposite (value to ioctl was wrong) +- fixed a memory leak in NotificationQueue sample +- added comparison operators to Socket so that Sockets can be used as keys in maps +- added Socket::setBlocking() +- added StreamSocket::connectNB() (non-blocking connect) +- added Observer::accepts() +- added SocketReactor, SocketConnector and SocketAcceptor classes to support event-based socket programming +- NamespacePrefixesStrategy now uses expat's XML_SetReturnNSTriplet(). + The previously used separate namespace handling code has been removed. + This improves performance if NamespacePrefixesStrategy is used (both the n + amespaces and namespace-prefixes SAX2 features are used) +- upgraded expat to 2.0 pre-release (2005-12-27) snapshot +- added TeeInputStream and TeeOutputStream classes +- added download sample for URIStreamOpener +- renamed registerOpener() to registerFactory() in HTTPStreamFactory and FTPStreamFactory +- added LineEndingConverter streams +- added FTPClientSession +- code and documentation clean-up +- added DialogSocket class +- reorganized HTTP test suites +- added FTPClientSession and FTPStreamFactory +- added DialogSocket class + + +Release 0.96.1 (2005-12-28) +=========================== + +- fixed a memory leak caused by a bug in Microsoft's stream implementation (see the comment in Foundation/StreamUtil.h for an explanation) +- added samples for Net library +- added uptime() and startTime() to Util::Application +- added DateTimeFormatter::format() for Timespan +- added ErrorHandler class and better exception handling for threads +- added poco_debugger() and poco_debugger_msg() macros +- added project and solution files for Visual Studio 2005 (due to some bugs/leaks in Microsofts standard library - see + http://lab.msdn.microsoft.com/productfeedback/viewfeedback.aspx?feedbackid=e08bd793-3fef-40ff-adda-ed313e0eafcc + we do not recommend using this for production purposes) +- fixed two problems with out-of-range string iterator in Path (the testsuite triggered an assertion in VC++ 8.0) +- fixed mac line endings in a few files +- added a workaround to the class loader that fixes strange behavior with VC++ 8.0. There seems to be a problem with typeid() not returning a valid typeinfo under certain circumstances. +- added buffer allocator argument to buffered stream buffer templates +- added buffer pools to HTTP to reduce memory fragmentation and to improve performance +- added Net to Windows build.cmd script +- added swap() to various classes that already support assignment +- added a null pointer check in DOMWriter::writeNode() +- fixed documentation in BinaryWriter.h and BinaryReader.h +- added explicit support for network byte order to BinaryReader and BinaryWriter +- added basic support for FreeBSD (needs more testing) +- BinaryReader: renamed readRawData() to readRaw() to be consistent with BinaryWriter::writeRaw() +- added support for uppercase output to HexBinaryEncoder. +- added MediaType class +- added QuotedPrintableEncoder and QuotedPrintableDecoder classes +- renamed ObjectFactory to Instantiator. This should prevent the confusion caused by DynamicFactory and ObjectFactory. Sorry for the inconvenience if you are already using this. +- AttachmentSource::filename() now returns const string& +- added StringAttachmentSource +- replaced old-style C casts with C++ casts in NetworkInterface.cpp +- MutexImpl (WIN32): replaced InitializeCriticalSection with InitializeCriticalSectionAndSpinCount, which should increase performance on multiprocessor or multicore systems when many locks are used. +- fixed a problem with STLport 5.0 when compiling StreamTokenizer +- HTTPStreamOpener now also works with no-path URIs (like http://www.appinf.com) +- fixed wrong delete usage (plain delete instead of delete [] was used in a few cases) +- fixed a handle leak in WinTestRunner + + +Release 0.95.4 (2005-11-07) +=========================== + +- fixed #1348006 and #1348005 + + +Release 0.95.3 (2005-10-28) [internal] +====================================== + +- updated build scripts (patch #1339015) +- added support for AMD64 platforms (patch #1339015) +- MultipartWriter creates its own boundary if an empty string is passed in as boundary +- made MultipartWriter::createBoundary() public +- fixed wrong documentation for DateTimeFormat::HTTP_FORMAT +- added support for HTTP Basic authentication +- added support for HTTP Cookies +- added support for HTML forms + + +Release 0.95.2 (2005-10-22) [internal] +====================================== + +- fixed a potential problems with streams when close in destructor fails (added try..catch block around close in destructors) +- added HTTPServer & friends +- added hasIdleThreads() method to NotificationQueue +- added TCPServer and friend +- added support for HTTP proxies to HTTPClientSession and HTTPStreamOpener +- fixed documentation bugs (Mutex.h, ClassLoader.h) + + +Relesae 0.95.1 (2005-10-15) [internal] +====================================== + +- Tasks can now throw custom notifications (contributed by Alex Fabijanic) +- renamed URIFileStreamFactory to FileStreamFactory +- added a few methods to URI (setPathEtc(), getPathEtc(), getPathAndQuery()) +- added new exception classes +- fixed some documentation +- added basic checks when reading a MessageHeader from a stream +- added HTTP classes (testsuite still incomplete) +- added MessageHeader, NameValueCollection, MultipartReader and MultipartWriter classes +- added Timespan::useconds() +- added ClassLoader::isLibraryLoaded() +- Socket classes use Timespan::useconds() to fill struct timeval +- added DatagramSocket, MulticastSocket and NetworkInterface classes +- added socket classes and related basic stuff +- added additonal constructor/assign to Timespan- added BasicBufferedBidirectionalStreamBuf +- fixed a potential MT issue in Base64Decoder +- code beautifying in [Un]BufferedStreamBuf +- more improvements to ClassLoader +- code cleanup and naming convention fixes (changed all *Imp classes to *Impl for consistency) + + +Release 0.94.1 (2005-09-30) [internal] +====================================== + +- added MetaSingleton (based on a contribution by Alex Fabijanic) +- added ClassLoader::create() +- added ClassLoader::instance() +- code clean-ups in FileChannel and related classes +- added SimpleFileChannel +- RotateAtTimeStrategy: + ::getNextRollover() rewritten (buggy) +- DateTime + microseconds assert corrected + asserts in computeGregorian() (except for year - see comment in computeGregorian()) + milliseconds calculation modified in computeGregorian() + microseconds assigned in computeGregorian() + normalize() and checkLimit() private functions to correct cases of overflow for milli/microseconds +- LocalDateTime: added timestamp() method +- FileChannel: + added "times" property (used to determine whether to use UTC or local time with RotateAtTimeStrategy) + ::setProperty() modified (whenever "times" property is set, methods setRotation and setArchive are + reinvoked to reflect the change) +- FileChannel: added support for archived file compression and archived file purging +- FileChannel tests modified +- FileChannel: put LogFile, RotateStrategy and ArchiveStrategy into their own files +- Message: added thread id field +- PatternFormatter: added %I specifier for thread id +- ThreadPool: PooledThread can be assigned a name +- TaskManager: task name is reflected in thread name +- fixed LocalDateTime::operator - (const Timespan&) [#0000004] +- upon startup all loggers' channels are set to a console channel +- improved search for application configuration files (see loadConfiguration()). +- added Glob class (fixes #1249700) +- upgraded to zlib 1.2.3 (fixes #1261712) +- added Logger::dump() +- fixed a wrong condition in Logger::log(const Message&) +- Path::find() now also works with relative paths in addition to plain file names +- added Path(const Path&, const Path&) constructor +- added SharedPtr template +- added Path::tryParse() +- SAXParser::parse()/EntityResolverImpl now works for both URIs and local filesystem paths (fixes #1254812) + + +Release 0.93.1 (2005-08-01) +=========================== + +This release contains various new features, improvements and bugfixes: +- bugfix: UUIDGenerator throws an exception if no connected ethernet adapter can + be found (and thus no MAC address can be obtained) +- added UUIDGenerator::createOne() method +- added error handling to UUID sample application +- added relational (==, !=, <, <=, >, >=) and arithmetic operators (+, -, +=, -=) to DateTime +- added LocalDateTime class +- added support for LocalDateTime to DateTimeParser and DateTimeFormatter +- added enqueueUrgentNotification() to NotificationQueue +- added support for timezone specifiers (%z, %Z) to PatternFormatter +- added [] operator and count() to StringTokenizer +- added elapsed() and isElapsed() to Timestamp +- added tzd() to Timezone +- added WinRegistryKey and WinService classes (Windows only) +- added index operator and count() to StringTokenizer +- added day/time-based log rotation (thanks to Alex Fabijanic), minor improvements to DateTimeParser +- support for Mac OS X 10.4/gcc 4.0.0 +- added NamedMutex and NamedEvent +- added Process::kill() +- added NoPermissionException +- added Task and TaskManager classes +- added ServerApplication class +- bugfix: EventLogChannel - _logFile was not properly initialized in one constructor +- bugfix: File::createDirectories did not work for hierarchies deeper than three +- added Util::FilesystemConfiguration +- documented logging policy: log() must open channel if it hasn't been opened yet +- FileChannel::log() opens channel if necessary +- the application reference passed to initialize() and reinitialize() is no longer const +- improved application logging initialization +- fixed a problem with configuration view and property placeholders +- fixed Util build configuration for Visual Studio +- improved application samples +- fixed documentation for Semaphore class + + +Release 0.92.1 (2005-05-09) +=========================== + +This release introduces the Util library that provides support for +configuration file parsing (different file formats), command line +argument processing, logging configuration and a framework for +command line/server applications. +There have also been various changes to the Foundation library: +- a new RefCountedObject class that acts as a base class for + various classes that use reference counting +- some missing members have been added to the AutoPtr template +- various improvements and bugfixes to the Logging framework, as well as + new LoggingFactory and LoggingRegistry classses, and a NullChannel class +- the SignalHandler class (Unix platforms only) +- ObjectFactory and DynamicFactory template classes +- the Path::find method for searching a file in a list of directories +- various new Exception classes + + +Release 0.91.4 (2005-04-11) +=========================== + +This is mainly a maintenance release that adds support for QNX Neutrino +and OpenVMS. There are also minor bugfixes and improvements. + +The Unix build system has been modified to work on QNX Neutrino. +The OpenVMS build system has been fixed and works now. +Some missing #include's have been added for QNX Neutrino. +Foundation/String.h: icompare now supports comparison with const char*; +the classic C version of isspace() has been used in a few places instead of the +C++ version, this has been fixed. +Foundation/Exception.h: IllegalStateException added. + + +Release 0.91.3 (2005-03-19) +=========================== + +This is a maintenance release that adds support for Solaris/Sun Forte C++. +No new features have been added. + +An implementation of FPEnvironment for Solaris has been included. +All stream classes have been modified to work around an initialization +problem that surfaced with Sun's C++ compiler when using STLport. +Source-code compatibility with the previous release is not affected. Various +minor changes, mostly adding missing #include's for Solaris. + + +Release 0.91.2 (2005-02-27) +=========================== + +Minor improvements to the Unix build system. No actual changes in the +libraries. + + +Release 0.91.1 (2005-02-21) +=========================== + +This is the first public release of the C++ Portable Components. +The release does not contain all features planned for the later 1.0 release +(the NET library is missing, for example), but is already quite usable. +Please refer to the README file for more information and instructions for +building the libraries. diff --git a/contrib/poco-cmake/CMakeLists.txt b/base/poco/CMakeLists.txt similarity index 67% rename from contrib/poco-cmake/CMakeLists.txt rename to base/poco/CMakeLists.txt index d173f35b9bf..0432ce196bf 100644 --- a/contrib/poco-cmake/CMakeLists.txt +++ b/base/poco/CMakeLists.txt @@ -1,4 +1,5 @@ -set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/poco") +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w") add_subdirectory (Crypto) add_subdirectory (Data) @@ -7,7 +8,7 @@ add_subdirectory (Foundation) add_subdirectory (JSON) add_subdirectory (MongoDB) add_subdirectory (Net) -add_subdirectory (Net/SSL) +add_subdirectory (NetSSL_OpenSSL) add_subdirectory (Redis) add_subdirectory (Util) add_subdirectory (XML) diff --git a/base/poco/CONTRIBUTORS b/base/poco/CONTRIBUTORS new file mode 100644 index 00000000000..7a74a9cc00c --- /dev/null +++ b/base/poco/CONTRIBUTORS @@ -0,0 +1,52 @@ +Guenter Obiltschnig +Alex Fabijanic +Peter Schojer +Ferdinand Beyer +Krzysztof Burghardt +Claus Dabringer +Caleb Epstein +Eran Hammer-Lahav +Chris Johnson +Sergey Kholodilov +Ryan Kraay +Larry Lewis +Andrew J. P. Maclean +Andrew Marlow +Paschal Mushubi +Jiang Shan +David Shawley +Sergey Skorokhodov +Tom Tan +Sergey N. Yatskevich +Marc Chevrier +Philippe Cuvillier +Marian Krivos +Franky Braem +Philip Prindeville +Anton Yabchinskiy +Rangel Reale +Fabrizio Duhem +Patrick White +Mike Naquin +Roger Meier +Mathaus Mendel +Arturo Castro +Adrian Imboden +Matej Knopp +Patrice Tarabbia +Lucas Clemente +Karl Reid +Pascal Bach +Cristian Thiago Moecke +Sergei Nikulov +Aaron Kaluszka +Iyed Bennour +Scott Davis +Kristin Cowalcijk +Yuval Kashtan +Christopher Baker +Scott Davis +Jeff Adams +Martin Osborne +Björn Schramke +Francis Andre diff --git a/base/poco/Crypto/CMakeLists.txt b/base/poco/Crypto/CMakeLists.txt new file mode 100644 index 00000000000..3753c5eed7b --- /dev/null +++ b/base/poco/Crypto/CMakeLists.txt @@ -0,0 +1,17 @@ +if (ENABLE_SSL) + file (GLOB SRCS src/*.cpp) + + add_library (_poco_crypto ${SRCS}) + add_library (Poco::Crypto ALIAS _poco_crypto) + + target_compile_options (_poco_crypto PRIVATE -Wno-newline-eof) + target_include_directories (_poco_crypto SYSTEM PUBLIC "include") + target_link_libraries (_poco_crypto PUBLIC Poco::Foundation OpenSSL::SSL OpenSSL::Crypto) + + message (STATUS "Using Poco::Crypto") +else () + add_library (_poco_crypto INTERFACE) + add_library (Poco::Crypto ALIAS _poco_crypto) + + message (STATUS "Not using Poco::Crypto") +endif () diff --git a/base/poco/Crypto/include/Poco/Crypto/Cipher.h b/base/poco/Crypto/include/Poco/Crypto/Cipher.h new file mode 100644 index 00000000000..fbe0c30ae8c --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/Cipher.h @@ -0,0 +1,138 @@ +// +// Cipher.h +// +// Library: Crypto +// Package: Cipher +// Module: Cipher +// +// Definition of the Cipher class. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_Cipher_INCLUDED +#define Crypto_Cipher_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/RefCountedObject.h" +#include "Poco/AutoPtr.h" +#include +#include +#include + + +namespace Poco { +namespace Crypto { + + +class CryptoTransform; + + +class Crypto_API Cipher: public Poco::RefCountedObject + /// Represents the abstract base class from which all implementations of + /// symmetric/asymmetric encryption algorithms must inherit. Use the CipherFactory + /// class to obtain an instance of this class: + /// + /// CipherFactory& factory = CipherFactory::defaultFactory(); + /// // Creates a 256-bit AES cipher + /// Cipher* pCipher = factory.createCipher(CipherKey("aes-256")); + /// Cipher* pRSACipher = factory.createCipher(RSAKey(RSAKey::KL_1024, RSAKey::EXP_SMALL)); + /// + /// Check the different Key constructors on how to initialize/create + /// a key. The above example auto-generates random keys. + /// + /// Note that you won't be able to decrypt data encrypted with a random key + /// once the Cipher is destroyed unless you persist the generated key and IV. + /// An example usage for random keys is to encrypt data saved in a temporary + /// file. + /// + /// Once your key is set up, you can use the Cipher object to encrypt or + /// decrypt strings or, in conjunction with a CryptoInputStream or a + /// CryptoOutputStream, to encrypt streams of data. + /// + /// Since encrypted strings will contain arbitrary binary data that will cause + /// problems in applications that are not binary-safe (eg., when sending + /// encrypted data in e-mails), the encryptString() and decryptString() can + /// encode (or decode, respectively) encrypted data using a "transport encoding". + /// Supported encodings are Base64 and BinHex. + /// + /// The following example encrypts and decrypts a string utilizing Base64 + /// encoding: + /// + /// std::string plainText = "This is my secret information"; + /// std::string encrypted = pCipher->encryptString(plainText, Cipher::ENC_BASE64); + /// std::string decrypted = pCipher->decryptString(encrypted, Cipher::ENC_BASE64); + /// + /// In order to encrypt a stream of data (eg. to encrypt files), you can use + /// a CryptoStream: + /// + /// // Create an output stream that will encrypt all data going through it + /// // and write pass it to the underlying file stream. + /// Poco::FileOutputStream sink("encrypted.dat"); + /// CryptoOutputStream encryptor(sink, pCipher->createEncryptor()); + /// + /// Poco::FileInputStream source("source.txt"); + /// Poco::StreamCopier::copyStream(source, encryptor); + /// + /// // Always close output streams to flush all internal buffers + /// encryptor.close(); + /// sink.close(); +{ +public: + typedef Poco::AutoPtr Ptr; + typedef std::vector ByteVec; + + enum Encoding + /// Transport encoding to use for encryptString() and decryptString(). + { + ENC_NONE = 0x00, /// Plain binary output + ENC_BASE64 = 0x01, /// Base64-encoded output + ENC_BINHEX = 0x02, /// BinHex-encoded output + ENC_BASE64_NO_LF = 0x81, /// Base64-encoded output, no linefeeds + ENC_BINHEX_NO_LF = 0x82 /// BinHex-encoded output, no linefeeds + + }; + + virtual ~Cipher(); + /// Destroys the Cipher. + + virtual const std::string& name() const = 0; + /// Returns the name of the Cipher. + + virtual CryptoTransform* createEncryptor() = 0; + /// Creates an encryptor object to be used with a CryptoStream. + + virtual CryptoTransform* createDecryptor() = 0; + /// Creates a decryptor object to be used with a CryptoStream. + + virtual std::string encryptString(const std::string& str, Encoding encoding = ENC_NONE); + /// Directly encrypt a string and encode it using the given encoding. + + virtual std::string decryptString(const std::string& str, Encoding encoding = ENC_NONE); + /// Directly decrypt a string that is encoded with the given encoding. + + virtual void encrypt(std::istream& source, std::ostream& sink, Encoding encoding = ENC_NONE); + /// Directly encrypts an input stream and encodes it using the given encoding. + + virtual void decrypt(std::istream& source, std::ostream& sink, Encoding encoding = ENC_NONE); + /// Directly decrypt an input stream that is encoded with the given encoding. + +protected: + Cipher(); + /// Creates a new Cipher object. + +private: + Cipher(const Cipher&); + Cipher& operator = (const Cipher&); +}; + + +} } // namespace Poco::Crypto + + +#endif // Crypto_Cipher_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/CipherFactory.h b/base/poco/Crypto/include/Poco/Crypto/CipherFactory.h new file mode 100644 index 00000000000..36aa964a1e6 --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/CipherFactory.h @@ -0,0 +1,75 @@ +// +// CipherFactory.h +// +// Library: Crypto +// Package: Cipher +// Module: CipherFactory +// +// Definition of the CipherFactory class. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_CipherFactory_INCLUDED +#define Crypto_CipherFactory_INCLUDED + + +#include "Poco/Crypto/Crypto.h" + + +namespace Poco { +namespace Crypto { + + +class Cipher; +class CipherKey; +class RSAKey; + + +class Crypto_API CipherFactory + /// A factory for Cipher objects. See the Cipher class for examples on how to + /// use the CipherFactory. +{ +public: + CipherFactory(); + /// Creates a new CipherFactory object. + + virtual ~CipherFactory(); + /// Destroys the CipherFactory. + + Cipher* createCipher(const CipherKey& key); + /// Creates a Cipher object for the given Cipher name. Valid cipher + /// names depend on the OpenSSL version the library is linked with; + /// see the output of + /// + /// openssl enc --help + /// + /// for a list of supported block and stream ciphers. + /// + /// Common examples are: + /// + /// * AES: "aes-128", "aes-256" + /// * DES: "des", "des3" + /// * Blowfish: "bf" + + Cipher* createCipher(const RSAKey& key, RSAPaddingMode paddingMode = RSA_PADDING_PKCS1); + /// Creates a RSACipher using the given RSA key and padding mode + /// for public key encryption/private key decryption. + + static CipherFactory& defaultFactory(); + /// Returns the default CipherFactory. + +private: + CipherFactory(const CipherFactory&); + CipherFactory& operator = (const CipherFactory&); +}; + + +} } // namespace Poco::Crypto + + +#endif // Crypto_CipherFactory_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/CipherImpl.h b/base/poco/Crypto/include/Poco/Crypto/CipherImpl.h new file mode 100644 index 00000000000..d6e8e0e79b2 --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/CipherImpl.h @@ -0,0 +1,69 @@ +// +// CipherImpl.h +// +// Library: Crypto +// Package: Cipher +// Module: CipherImpl +// +// Definition of the CipherImpl class. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_CipherImpl_INCLUDED +#define Crypto_CipherImpl_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/Cipher.h" +#include "Poco/Crypto/CipherKey.h" +#include "Poco/Crypto/OpenSSLInitializer.h" +#include + + +namespace Poco { +namespace Crypto { + + +class CipherImpl: public Cipher + /// An implementation of the Cipher class for OpenSSL's crypto library. +{ +public: + CipherImpl(const CipherKey& key); + /// Creates a new CipherImpl object for the given CipherKey. + + virtual ~CipherImpl(); + /// Destroys the CipherImpl. + + const std::string& name() const; + /// Returns the name of the cipher. + + CryptoTransform* createEncryptor(); + /// Creates an encryptor object. + + CryptoTransform* createDecryptor(); + /// Creates a decryptor object. + +private: + CipherKey _key; + OpenSSLInitializer _openSSLInitializer; +}; + + +// +// Inlines +// +inline const std::string& CipherImpl::name() const +{ + return _key.name(); +} + + +} } // namespace Poco::Crypto + + +#endif // Crypto_CipherImpl_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/CipherKey.h b/base/poco/Crypto/include/Poco/Crypto/CipherKey.h new file mode 100644 index 00000000000..b102cc2310b --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/CipherKey.h @@ -0,0 +1,201 @@ +// +// CipherKey.h +// +// Library: Crypto +// Package: Cipher +// Module: CipherKey +// +// Definition of the CipherKey class. +// +// Copyright (c) 2007, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_CipherKey_INCLUDED +#define Crypto_CipherKey_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/CipherKeyImpl.h" + + +namespace Poco { +namespace Crypto { + + +class Crypto_API CipherKey + /// CipherKey stores the key information for decryption/encryption of data. + /// To create a random key, using the following code: + /// + /// CipherKey key("aes-256"); + /// + /// Note that you won't be able to decrypt data encrypted with a random key + /// once the Cipher is destroyed unless you persist the generated key and IV. + /// An example usage for random keys is to encrypt data saved in a temporary + /// file. + /// + /// To create a key using a human-readable password + /// string, use the following code. We create a AES Cipher and + /// use a salt value to make the key more robust: + /// + /// std::string password = "secret"; + /// std::string salt("asdff8723lasdf(**923412"); + /// CipherKey key("aes-256", password, salt); + /// + /// You may also control the digest and the number of iterations used to generate the key + /// by specifying the specific values. Here we create a key with the same data as before, + /// except that we use 100 iterations instead of DEFAULT_ITERATION_COUNT, and sha1 instead of + /// the default md5: + /// + /// std::string password = "secret"; + /// std::string salt("asdff8723lasdf(**923412"); + /// std::string digest ("sha1"); + /// CipherKey key("aes-256", password, salt, 100, digest); + /// +{ +public: + typedef CipherKeyImpl::Mode Mode; + typedef CipherKeyImpl::ByteVec ByteVec; + + enum + { + DEFAULT_ITERATION_COUNT = 2000 + /// Default iteration count to use with + /// generateKey(). RSA security recommends + /// an iteration count of at least 1000. + }; + + CipherKey(const std::string& name, + const std::string& passphrase, + const std::string& salt = "", + int iterationCount = DEFAULT_ITERATION_COUNT, + const std::string& digest = "md5"); + /// Creates a new CipherKeyImpl object using the given + /// cipher name, passphrase, salt value, iteration count and digest. + + CipherKey(const std::string& name, + const ByteVec& key, + const ByteVec& iv); + /// Creates a new CipherKeyImpl object using the given cipher + /// name, key and initialization vector (IV). + /// + /// The size of the IV must match the cipher's expected + /// IV size (see ivSize()), except for GCM mode, which allows + /// a custom IV size. + + CipherKey(const std::string& name); + /// Creates a new CipherKeyImpl object. Autoinitializes key and + /// initialization vector. + + ~CipherKey(); + /// Destroys the CipherKeyImpl. + + const std::string& name() const; + /// Returns the name of the Cipher. + + int keySize() const; + /// Returns the key size of the Cipher. + + int blockSize() const; + /// Returns the block size of the Cipher. + + int ivSize() const; + /// Returns the IV size of the Cipher. + + Mode mode() const; + /// Returns the Cipher's mode of operation. + + const ByteVec& getKey() const; + /// Returns the key for the Cipher. + + void setKey(const ByteVec& key); + /// Sets the key for the Cipher. + + const ByteVec& getIV() const; + /// Returns the initialization vector (IV) for the Cipher. + + void setIV(const ByteVec& iv); + /// Sets the initialization vector (IV) for the Cipher. + /// + /// The size of the vector must match the cipher's expected + /// IV size (see ivSize()), except for GCM mode, which allows + /// a custom IV size. + + CipherKeyImpl::Ptr impl(); + /// Returns the impl object + +private: + CipherKeyImpl::Ptr _pImpl; +}; + + +// +// inlines +// +inline const std::string& CipherKey::name() const +{ + return _pImpl->name(); +} + + +inline int CipherKey::keySize() const +{ + return _pImpl->keySize(); +} + + +inline int CipherKey::blockSize() const +{ + return _pImpl->blockSize(); +} + + +inline int CipherKey::ivSize() const +{ + return _pImpl->ivSize(); +} + + +inline CipherKey::Mode CipherKey::mode() const +{ + return _pImpl->mode(); +} + + +inline const CipherKey::ByteVec& CipherKey::getKey() const +{ + return _pImpl->getKey(); +} + + +inline void CipherKey::setKey(const CipherKey::ByteVec& key) +{ + _pImpl->setKey(key); +} + + +inline const CipherKey::ByteVec& CipherKey::getIV() const +{ + return _pImpl->getIV(); +} + + +inline void CipherKey::setIV(const CipherKey::ByteVec& iv) +{ + _pImpl->setIV(iv); +} + + +inline CipherKeyImpl::Ptr CipherKey::impl() +{ + return _pImpl; +} + + +} } // namespace Poco::Crypto + + +#endif // Crypto_CipherKey_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/CipherKeyImpl.h b/base/poco/Crypto/include/Poco/Crypto/CipherKeyImpl.h new file mode 100644 index 00000000000..f7807aad9f8 --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/CipherKeyImpl.h @@ -0,0 +1,168 @@ +// +// CipherKeyImpl.h +// +// Library: Crypto +// Package: Cipher +// Module: CipherKeyImpl +// +// Definition of the CipherKeyImpl class. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_CipherKeyImpl_INCLUDED +#define Crypto_CipherKeyImpl_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/OpenSSLInitializer.h" +#include "Poco/RefCountedObject.h" +#include "Poco/AutoPtr.h" +#include + + +struct evp_cipher_st; +typedef struct evp_cipher_st EVP_CIPHER; + + +namespace Poco { +namespace Crypto { + + +class CipherKeyImpl: public RefCountedObject + /// An implementation of the CipherKey class for OpenSSL's crypto library. +{ +public: + typedef std::vector ByteVec; + typedef Poco::AutoPtr Ptr; + + enum Mode + /// Cipher mode of operation. This mode determines how multiple blocks + /// are connected; this is essential to improve security. + { + MODE_STREAM_CIPHER, /// Stream cipher + MODE_ECB, /// Electronic codebook (plain concatenation) + MODE_CBC, /// Cipher block chaining (default) + MODE_CFB, /// Cipher feedback + MODE_OFB, /// Output feedback + MODE_CTR, /// Counter mode + MODE_GCM, /// Galois/Counter mode + MODE_CCM /// Counter with CBC-MAC + }; + + CipherKeyImpl(const std::string& name, + const std::string& passphrase, + const std::string& salt, + int iterationCount, + const std::string& digest); + /// Creates a new CipherKeyImpl object, using + /// the given cipher name, passphrase, salt value + /// and iteration count. + + CipherKeyImpl(const std::string& name, + const ByteVec& key, + const ByteVec& iv); + /// Creates a new CipherKeyImpl object, using the + /// given cipher name, key and initialization vector. + + CipherKeyImpl(const std::string& name); + /// Creates a new CipherKeyImpl object. Autoinitializes key + /// and initialization vector. + + virtual ~CipherKeyImpl(); + /// Destroys the CipherKeyImpl. + + const std::string& name() const; + /// Returns the name of the Cipher. + + int keySize() const; + /// Returns the key size of the Cipher. + + int blockSize() const; + /// Returns the block size of the Cipher. + + int ivSize() const; + /// Returns the IV size of the Cipher. + + Mode mode() const; + /// Returns the Cipher's mode of operation. + + const ByteVec& getKey() const; + /// Returns the key for the Cipher. + + void setKey(const ByteVec& key); + /// Sets the key for the Cipher. + + const ByteVec& getIV() const; + /// Returns the initialization vector (IV) for the Cipher. + + void setIV(const ByteVec& iv); + /// Sets the initialization vector (IV) for the Cipher. + + const EVP_CIPHER* cipher(); + /// Returns the cipher object + +private: + void generateKey(const std::string& passphrase, + const std::string& salt, + int iterationCount); + /// Generates key and IV from a password and optional salt string. + + void generateKey(); + /// Generates key and IV from random data. + + void getRandomBytes(ByteVec& vec, std::size_t count); + /// Stores random bytes in vec. + +private: + const EVP_CIPHER* _pCipher; + const EVP_MD* _pDigest; + std::string _name; + ByteVec _key; + ByteVec _iv; + OpenSSLInitializer _openSSLInitializer; +}; + + +// +// Inlines +// +inline const std::string& CipherKeyImpl::name() const +{ + return _name; +} + + +inline const CipherKeyImpl::ByteVec& CipherKeyImpl::getKey() const +{ + return _key; +} + + +inline void CipherKeyImpl::setKey(const ByteVec& key) +{ + poco_assert(key.size() == static_cast(keySize())); + _key = key; +} + + +inline const CipherKeyImpl::ByteVec& CipherKeyImpl::getIV() const +{ + return _iv; +} + + +inline const EVP_CIPHER* CipherKeyImpl::cipher() +{ + return _pCipher; +} + + +} } // namespace Poco::Crypto + + +#endif // Crypto_CipherKeyImpl_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/Crypto.h b/base/poco/Crypto/include/Poco/Crypto/Crypto.h new file mode 100644 index 00000000000..d6b3ede65cb --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/Crypto.h @@ -0,0 +1,195 @@ +// +// Crypto.h +// +// Library: Crypto +// Package: CryptoCore +// Module: Crypto +// +// Basic definitions for the Poco Crypto library. +// This file must be the first file included by every other Crypto +// header file. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_Crypto_INCLUDED +#define Crypto_Crypto_INCLUDED + + +#define POCO_EXTERNAL_OPENSSL_DEFAULT 1 +#define POCO_EXTERNAL_OPENSSL_SLPRO 2 + + +#include "Poco/Foundation.h" +#include + + +#ifndef OPENSSL_VERSION_PREREQ + #if defined(OPENSSL_VERSION_MAJOR) && defined(OPENSSL_VERSION_MINOR) + #define OPENSSL_VERSION_PREREQ(maj, min) \ + ((OPENSSL_VERSION_MAJOR << 16) + OPENSSL_VERSION_MINOR >= ((maj) << 16) + (min)) + #else + #define OPENSSL_VERSION_PREREQ(maj, min) \ + (OPENSSL_VERSION_NUMBER >= (((maj) << 28) | ((min) << 20))) + #endif +#endif + + +enum RSAPaddingMode + /// The padding mode used for RSA public key encryption. +{ + RSA_PADDING_PKCS1, + /// PKCS #1 v1.5 padding. This currently is the most widely used mode. + + RSA_PADDING_PKCS1_OAEP, + /// EME-OAEP as defined in PKCS #1 v2.0 with SHA-1, MGF1 and an empty + /// encoding parameter. This mode is recommended for all new applications. + + RSA_PADDING_SSLV23, + /// PKCS #1 v1.5 padding with an SSL-specific modification that denotes + /// that the server is SSL3 capable. + + RSA_PADDING_NONE + /// Raw RSA encryption. This mode should only be used to implement cryptographically + /// sound padding modes in the application code. Encrypting user data directly with RSA + /// is insecure. +}; + + +// +// The following block is the standard way of creating macros which make exporting +// from a DLL simpler. All files within this DLL are compiled with the Crypto_EXPORTS +// symbol defined on the command line. this symbol should not be defined on any project +// that uses this DLL. This way any other project whose source files include this file see +// Crypto_API functions as being imported from a DLL, whereas this DLL sees symbols +// defined with this macro as being exported. +// +#if defined(_WIN32) + #if defined(POCO_DLL) + #if defined(Crypto_EXPORTS) + #define Crypto_API __declspec(dllexport) + #else + #define Crypto_API __declspec(dllimport) + #endif + #endif +#endif + + +#if !defined(Crypto_API) + #if !defined(POCO_NO_GCC_API_ATTRIBUTE) && defined (__GNUC__) && (__GNUC__ >= 4) + #define Crypto_API __attribute__ ((visibility ("default"))) + #else + #define Crypto_API + #endif +#endif + + +// +// Automatically link Crypto and OpenSSL libraries. +// +#if defined(_MSC_VER) + #if !defined(POCO_NO_AUTOMATIC_LIBS) + #if defined(POCO_INTERNAL_OPENSSL_MSVC_VER) + #if defined(POCO_EXTERNAL_OPENSSL) + #pragma message("External OpenSSL defined but internal headers used - possible mismatch!") + #endif // POCO_EXTERNAL_OPENSSL + #if !defined(_DEBUG) + #define POCO_DEBUG_SUFFIX "" + #if !defined (_DLL) + #define POCO_STATIC_SUFFIX "mt" + #else // _DLL + #define POCO_STATIC_SUFFIX "" + #endif + #else // _DEBUG + #define POCO_DEBUG_SUFFIX "d" + #if !defined (_DLL) + #define POCO_STATIC_SUFFIX "mt" + #else // _DLL + #define POCO_STATIC_SUFFIX "" + #endif + #endif + #pragma comment(lib, "libcrypto" POCO_STATIC_SUFFIX POCO_DEBUG_SUFFIX ".lib") + #pragma comment(lib, "libssl" POCO_STATIC_SUFFIX POCO_DEBUG_SUFFIX ".lib") + #if !defined(_WIN64) && !defined (_DLL) && \ + (POCO_INTERNAL_OPENSSL_MSVC_VER == 120) && \ + (POCO_MSVC_VERSION < POCO_INTERNAL_OPENSSL_MSVC_VER) + #pragma comment(lib, "libPreVS2013CRT" POCO_STATIC_SUFFIX POCO_DEBUG_SUFFIX ".lib") + #endif + #if !defined (_DLL) && (POCO_MSVS_VERSION >= 2015) + #pragma comment(lib, "legacy_stdio_definitions.lib") + #pragma comment(lib, "legacy_stdio_wide_specifiers.lib") + #endif + #elif defined(POCO_EXTERNAL_OPENSSL) + #if POCO_EXTERNAL_OPENSSL == POCO_EXTERNAL_OPENSSL_SLPRO + #if defined(POCO_DLL) + #if OPENSSL_VERSION_PREREQ(1,1) + #pragma comment(lib, "libcrypto.lib") + #pragma comment(lib, "libssl.lib") + #else + #pragma comment(lib, "libeay32.lib") + #pragma comment(lib, "ssleay32.lib") + #endif + #else + #if OPENSSL_VERSION_PREREQ(1,1) + #if defined(_WIN64) + #pragma comment(lib, "libcrypto64" POCO_LIB_SUFFIX) + #pragma comment(lib, "libssl64" POCO_LIB_SUFFIX) + #else + #pragma comment(lib, "libcrypto32" POCO_LIB_SUFFIX) + #pragma comment(lib, "libssl32" POCO_LIB_SUFFIX) + #endif + #else + #pragma comment(lib, "libeay32" POCO_LIB_SUFFIX) + #pragma comment(lib, "ssleay32" POCO_LIB_SUFFIX) + #endif + #endif + #elif POCO_EXTERNAL_OPENSSL == POCO_EXTERNAL_OPENSSL_DEFAULT + #if OPENSSL_VERSION_PREREQ(1,1) + #pragma comment(lib, "libcrypto.lib") + #pragma comment(lib, "libssl.lib") + #else + #pragma comment(lib, "libeay32.lib") + #pragma comment(lib, "ssleay32.lib") + #endif + #endif + #endif // POCO_INTERNAL_OPENSSL_MSVC_VER + #if !defined(Crypto_EXPORTS) + #pragma comment(lib, "PocoCrypto" POCO_LIB_SUFFIX) + #endif + #endif // POCO_NO_AUTOMATIC_LIBS +#endif + + +namespace Poco { +namespace Crypto { + + +void Crypto_API initializeCrypto(); + /// Initialize the Crypto library, as well as the underlying OpenSSL + /// libraries, by calling OpenSSLInitializer::initialize(). + /// + /// Should be called before using any class from the Crypto library. + /// The Crypto library will be initialized automatically, through + /// OpenSSLInitializer instances held by various Crypto classes + /// (Cipher, CipherKey, RSAKey, X509Certificate). + /// However, it is recommended to call initializeCrypto() + /// in any case at application startup. + /// + /// Can be called multiple times; however, for every call to + /// initializeCrypto(), a matching call to uninitializeCrypto() + /// must be performed. + + +void Crypto_API uninitializeCrypto(); + /// Uninitializes the Crypto library by calling + /// OpenSSLInitializer::uninitialize(). + + +} } // namespace Poco::Crypto + + +#endif // Crypto_Crypto_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/CryptoException.h b/base/poco/Crypto/include/Poco/Crypto/CryptoException.h new file mode 100644 index 00000000000..34c15111e6a --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/CryptoException.h @@ -0,0 +1,56 @@ +// +// CryptoException.h +// +// +// Library: Crypto +// Package: Crypto +// Module: CryptoException +// +// Definition of the CryptoException class. +// +// Copyright (c) 2012, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_CryptoException_INCLUDED +#define Crypto_CryptoException_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Exception.h" + + +namespace Poco { +namespace Crypto { + + +POCO_DECLARE_EXCEPTION(Crypto_API, CryptoException, Poco::Exception) + + +class Crypto_API OpenSSLException : public CryptoException +{ +public: + OpenSSLException(int code = 0); + OpenSSLException(const std::string& msg, int code = 0); + OpenSSLException(const std::string& msg, const std::string& arg, int code = 0); + OpenSSLException(const std::string& msg, const Poco::Exception& exc, int code = 0); + OpenSSLException(const OpenSSLException& exc); + ~OpenSSLException() throw(); + OpenSSLException& operator = (const OpenSSLException& exc); + const char* name() const throw(); + const char* className() const throw(); + Poco::Exception* clone() const; + void rethrow() const; + +private: + void setExtMessage(); +}; + + +} } // namespace Poco::Crypto + + +#endif // Crypto_CryptoException_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/CryptoStream.h b/base/poco/Crypto/include/Poco/Crypto/CryptoStream.h new file mode 100644 index 00000000000..25a99a4ae58 --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/CryptoStream.h @@ -0,0 +1,192 @@ +// +// CryptoStream.h +// +// Library: Crypto +// Package: Cipher +// Module: CryptoStream +// +// Definition of the CryptoStreamBuf, CryptoInputStream and CryptoOutputStream +// classes. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_CryptoStream_INCLUDED +#define Crypto_CryptoStream_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/BufferedStreamBuf.h" +#include "Poco/Buffer.h" +#include + + +namespace Poco { +namespace Crypto { + + +class CryptoTransform; +class Cipher; + + +class Crypto_API CryptoStreamBuf: public Poco::BufferedStreamBuf + /// This stream buffer performs cryptographic transformation on the data + /// going through it. +{ +public: + CryptoStreamBuf(std::istream& istr, CryptoTransform* pTransform, std::streamsize bufferSize = 8192); + CryptoStreamBuf(std::ostream& ostr, CryptoTransform* pTransform, std::streamsize bufferSize = 8192); + + virtual ~CryptoStreamBuf(); + + void close(); + /// Flushes all buffers and finishes the encryption. + +protected: + int readFromDevice(char* buffer, std::streamsize length); + int writeToDevice(const char* buffer, std::streamsize length); + +private: + CryptoTransform* _pTransform; + std::istream* _pIstr; + std::ostream* _pOstr; + bool _eof; + + Poco::Buffer _buffer; + + CryptoStreamBuf(const CryptoStreamBuf&); + CryptoStreamBuf& operator = (const CryptoStreamBuf&); +}; + + +class Crypto_API CryptoIOS: public virtual std::ios + /// The base class for CryptoInputStream and CryptoOutputStream. + /// + /// This class is needed to ensure correct initialization order of the + /// stream buffer and base classes. +{ +public: + CryptoIOS(std::istream& istr, CryptoTransform* pTransform, std::streamsize bufferSize = 8192); + CryptoIOS(std::ostream& ostr, CryptoTransform* pTransform, std::streamsize bufferSize = 8192); + ~CryptoIOS(); + CryptoStreamBuf* rdbuf(); + +protected: + CryptoStreamBuf _buf; +}; + + +class Crypto_API CryptoInputStream: public CryptoIOS, public std::istream + /// This stream transforms all data passing through it using the given + /// CryptoTransform. + /// + /// Use a CryptoTransform object provided by Cipher::createEncrytor() or + /// Cipher::createDecryptor() to create an encrypting or decrypting stream, + /// respectively. +{ +public: + CryptoInputStream(std::istream& istr, CryptoTransform* pTransform, std::streamsize bufferSize = 8192); + /// Create a new CryptoInputStream object. The CryptoInputStream takes the + /// ownership of the given CryptoTransform object. + + CryptoInputStream(std::istream& istr, Cipher& cipher, std::streamsize bufferSize = 8192); + /// Create a new encrypting CryptoInputStream object using the given cipher. + + ~CryptoInputStream(); + /// Destroys the CryptoInputStream. +}; + + +class Crypto_API CryptoOutputStream: public CryptoIOS, public std::ostream + /// This stream transforms all data passing through it using the given + /// CryptoTransform. + /// + /// Use a CryptoTransform object provided by Cipher::createEncrytor() or + /// Cipher::createDecryptor() to create an encrypting or decrypting stream, + /// respectively. + /// + /// After all data has been passed through the stream, close() must be called + /// to ensure completion of cryptographic transformation. +{ +public: + CryptoOutputStream(std::ostream& ostr, CryptoTransform* pTransform, std::streamsize bufferSize = 8192); + /// Create a new CryptoOutputStream object. The CryptoOutputStream takes the + /// ownership of the given CryptoTransform object. + + CryptoOutputStream(std::ostream& ostr, Cipher& cipher, std::streamsize bufferSize = 8192); + /// Create a new decrypting CryptoOutputStream object using the given cipher. + + ~CryptoOutputStream(); + /// Destroys the CryptoOutputStream. + + void close(); + /// Flushes all buffers and finishes the encryption. +}; + + +class Crypto_API DecryptingInputStream: public CryptoIOS, public std::istream + /// This stream decrypts all data passing through it using the given + /// Cipher. +{ +public: + DecryptingInputStream(std::istream& istr, Cipher& cipher, std::streamsize bufferSize = 8192); + /// Create a new DecryptingInputStream object using the given cipher. + + ~DecryptingInputStream(); + /// Destroys the DecryptingInputStream. +}; + + +class Crypto_API DecryptingOutputStream: public CryptoIOS, public std::ostream + /// This stream decrypts all data passing through it using the given + /// Cipher. +{ +public: + DecryptingOutputStream(std::ostream& ostr, Cipher& cipher, std::streamsize bufferSize = 8192); + /// Create a new DecryptingOutputStream object using the given cipher. + + ~DecryptingOutputStream(); + /// Destroys the DecryptingOutputStream. + + void close(); + /// Flushes all buffers and finishes the decryption. +}; + + +class Crypto_API EncryptingInputStream: public CryptoIOS, public std::istream + /// This stream encrypts all data passing through it using the given + /// Cipher. +{ +public: + EncryptingInputStream(std::istream& istr, Cipher& cipher, std::streamsize bufferSize = 8192); + /// Create a new EncryptingInputStream object using the given cipher. + + ~EncryptingInputStream(); + /// Destroys the EncryptingInputStream. +}; + + +class Crypto_API EncryptingOutputStream: public CryptoIOS, public std::ostream + /// This stream encrypts all data passing through it using the given + /// Cipher. +{ +public: + EncryptingOutputStream(std::ostream& ostr, Cipher& cipher, std::streamsize bufferSize = 8192); + /// Create a new EncryptingOutputStream object using the given cipher. + + ~EncryptingOutputStream(); + /// Destroys the EncryptingOutputStream. + + void close(); + /// Flushes all buffers and finishes the encryption. +}; + + +} } // namespace Poco::Crypto + + +#endif // Crypto_CryptoStream_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/CryptoTransform.h b/base/poco/Crypto/include/Poco/Crypto/CryptoTransform.h new file mode 100644 index 00000000000..9fa3806c653 --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/CryptoTransform.h @@ -0,0 +1,87 @@ +// +// CryptoTransform.h +// +// Library: Crypto +// Package: Cipher +// Module: CryptoTransform +// +// Definition of the CryptoTransform class. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_CryptoTransform_INCLUDED +#define Crypto_CryptoTransform_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include + + +namespace Poco { +namespace Crypto { + + +class Crypto_API CryptoTransform + /// This interface represents the basic operations for cryptographic + /// transformations to be used with a CryptoInputStream or a + /// CryptoOutputStream. + /// + /// Implementations of this class are returned by the Cipher class to + /// perform encryption or decryption of data. +{ +public: + CryptoTransform(); + /// Creates a new CryptoTransform object. + + virtual ~CryptoTransform(); + /// Destroys the CryptoTransform. + + virtual std::size_t blockSize() const = 0; + /// Returns the block size for this CryptoTransform. + + virtual int setPadding(int padding); + /// Enables or disables padding. By default encryption operations are padded using standard block + /// padding and the padding is checked and removed when decrypting. If the padding parameter is zero then + /// no padding is performed, the total amount of data encrypted or decrypted must then be a multiple of + /// the block size or an error will occur. + + virtual std::string getTag(std::size_t tagSize = 16) = 0; + /// Returns the GCM tag after encrypting using GCM mode. + /// + /// Must be called after finalize(). + + virtual void setTag(const std::string& tag) = 0; + /// Sets the GCM tag for authenticated decryption using GCM mode. + /// + /// Must be set before finalize() is called, otherwise + /// decryption will fail. + + virtual std::streamsize transform( + const unsigned char* input, + std::streamsize inputLength, + unsigned char* output, + std::streamsize outputLength) = 0; + /// Transforms a chunk of data. The inputLength is arbitrary and does not + /// need to be a multiple of the block size. The output buffer has a maximum + /// capacity of the given outputLength that must be at least + /// inputLength + blockSize() - 1 + /// Returns the number of bytes written to the output buffer. + + virtual std::streamsize finalize(unsigned char* output, std::streamsize length) = 0; + /// Finalizes the transformation. The output buffer must contain enough + /// space for at least two blocks, ie. + /// length >= 2*blockSize() + /// must be true. Returns the number of bytes written to the output + /// buffer. +}; + + +} } // namespace Poco::Crypto + + +#endif // Crypto_CryptoTransform_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/DigestEngine.h b/base/poco/Crypto/include/Poco/Crypto/DigestEngine.h new file mode 100644 index 00000000000..1c30e769d5f --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/DigestEngine.h @@ -0,0 +1,80 @@ +// +// DigestEngine.h +// +// Library: Crypto +// Package: Digest +// Module: DigestEngine +// +// Definition of the DigestEngine class. +// +// Copyright (c) 2012, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_DigestEngine_INCLUDED +#define Crypto_DigestEngine_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/OpenSSLInitializer.h" +#include "Poco/DigestEngine.h" +#include + + +namespace Poco { +namespace Crypto { + + +class Crypto_API DigestEngine: public Poco::DigestEngine + /// This class implements a Poco::DigestEngine for all + /// digest algorithms supported by OpenSSL. +{ +public: + DigestEngine(const std::string& name); + /// Creates a DigestEngine using the digest with the given name + /// (e.g., "MD5", "SHA1", "SHA256", "SHA512", etc.). + /// See the OpenSSL documentation for a list of supported digest algorithms. + /// + /// Throws a Poco::NotFoundException if no algorithm with the given name exists. + + ~DigestEngine(); + /// Destroys the DigestEngine. + + const std::string& algorithm() const; + /// Returns the name of the digest algorithm. + + int nid() const; + /// Returns the NID (OpenSSL object identifier) of the digest algorithm. + + // DigestEngine + std::size_t digestLength() const; + void reset(); + const Poco::DigestEngine::Digest& digest(); + +protected: + void updateImpl(const void* data, std::size_t length); + +private: + std::string _name; + EVP_MD_CTX* _pContext; + Poco::DigestEngine::Digest _digest; + OpenSSLInitializer _openSSLInitializer; +}; + + +// +// inlines +// +inline const std::string& DigestEngine::algorithm() const +{ + return _name; +} + + +} } // namespace Poco::Crypto + + +#endif // Crypto_DigestEngine_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/ECDSADigestEngine.h b/base/poco/Crypto/include/Poco/Crypto/ECDSADigestEngine.h new file mode 100644 index 00000000000..ed6fab442f6 --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/ECDSADigestEngine.h @@ -0,0 +1,101 @@ +// +// ECDSADigestEngine.h +// +// +// Library: Crypto +// Package: ECDSA +// Module: ECDSADigestEngine +// +// Definition of the ECDSADigestEngine class. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_ECDSADigestEngine_INCLUDED +#define Crypto_ECDSADigestEngine_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/ECKey.h" +#include "Poco/DigestEngine.h" +#include "Poco/Crypto/DigestEngine.h" +#include +#include + + +namespace Poco { +namespace Crypto { + + +class Crypto_API ECDSADigestEngine: public Poco::DigestEngine + /// This class implements a Poco::DigestEngine that can be + /// used to compute a secure digital signature. + /// + /// First another Poco::Crypto::DigestEngine is created and + /// used to compute a cryptographic hash of the data to be + /// signed. Then, the hash value is encrypted, using + /// the ECDSA private key. + /// + /// To verify a signature, pass it to the verify() + /// member function. It will decrypt the signature + /// using the ECDSA public key and compare the resulting + /// hash with the actual hash of the data. +{ +public: + + ECDSADigestEngine(const ECKey& key, const std::string &name); + /// Creates the ECDSADigestEngine with the given ECDSA key, + /// using the hash algorithm with the given name + /// (e.g., "SHA1", "SHA256", "SHA512", etc.). + /// See the OpenSSL documentation for a list of supported digest algorithms. + /// + /// Throws a Poco::NotFoundException if no algorithm with the given name exists. + + ~ECDSADigestEngine(); + /// Destroys the ECDSADigestEngine. + + std::size_t digestLength() const; + /// Returns the length of the digest in bytes. + + void reset(); + /// Resets the engine so that a new + /// digest can be computed. + + const DigestEngine::Digest& digest(); + /// Finishes the computation of the digest + /// (the first time it's called) and + /// returns the message digest. + /// + /// Can be called multiple times. + + const DigestEngine::Digest& signature(); + /// Signs the digest using the ECDSADSA algorithm + /// and the private key (the first time it's + /// called) and returns the result. + /// + /// Can be called multiple times. + + bool verify(const DigestEngine::Digest& signature); + /// Verifies the data against the signature. + /// + /// Returns true if the signature can be verified, false otherwise. + +protected: + void updateImpl(const void* data, std::size_t length); + +private: + ECKey _key; + Poco::Crypto::DigestEngine _engine; + Poco::DigestEngine::Digest _digest; + Poco::DigestEngine::Digest _signature; +}; + + +} } // namespace Poco::Crypto + + +#endif // Crypto_ECDSADigestEngine_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/ECKey.h b/base/poco/Crypto/include/Poco/Crypto/ECKey.h new file mode 100644 index 00000000000..14f2ac0a189 --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/ECKey.h @@ -0,0 +1,136 @@ +// +// ECKey.h +// +// +// Library: Crypto +// Package: EC +// Module: ECKey +// +// Definition of the ECKey class. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_ECKey_INCLUDED +#define Crypto_ECKey_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/KeyPair.h" +#include "Poco/Crypto/ECKeyImpl.h" + + +namespace Poco { +namespace Crypto { + + +class X509Certificate; +class PKCS12Container; + + +class Crypto_API ECKey : public KeyPair + /// This class stores an EC key pair, consisting + /// of private and public key. Storage of the private + /// key is optional. + /// + /// If a private key is available, the ECKey can be + /// used for decrypting data (encrypted with the public key) + /// or computing secure digital signatures. +{ +public: + ECKey(const EVPPKey& key); + /// Constructs ECKeyImpl by extracting the EC key. + + ECKey(const X509Certificate& cert); + /// Extracts the EC public key from the given certificate. + + ECKey(const PKCS12Container& cert); + /// Extracts the EC private key from the given certificate. + + ECKey(const std::string& eccGroup); + /// Creates the ECKey. Creates a new public/private keypair using the given parameters. + /// Can be used to sign data and verify signatures. + + ECKey(const std::string& publicKeyFile, const std::string& privateKeyFile, const std::string& privateKeyPassphrase = ""); + /// Creates the ECKey, by reading public and private key from the given files and + /// using the given passphrase for the private key. + /// + /// Cannot be used for signing or decryption unless a private key is available. + /// + /// If a private key is specified, you don't need to specify a public key file. + /// OpenSSL will auto-create the public key from the private key. + + ECKey(std::istream* pPublicKeyStream, std::istream* pPrivateKeyStream = 0, const std::string& privateKeyPassphrase = ""); + /// Creates the ECKey, by reading public and private key from the given streams and + /// using the given passphrase for the private key. + /// + /// Cannot be used for signing or decryption unless a private key is available. + /// + /// If a private key is specified, you don't need to specify a public key file. + /// OpenSSL will auto-create the public key from the private key. + + ~ECKey(); + /// Destroys the ECKey. + + ECKeyImpl::Ptr impl() const; + /// Returns the impl object. + + static std::string getCurveName(int nid = -1); + /// Returns elliptical curve name corresponding to + /// the given nid; if nid is not found, returns + /// empty string. + /// + /// If nid is -1, returns first curve name. + /// + /// If no curves are found, returns empty string; + + static int getCurveNID(std::string& name); + /// Returns the NID of the specified curve. + /// + /// If name is empty, returns the first curve NID + /// and updates the name accordingly. + + static bool hasCurve(const std::string& name); + /// Returns true if the named curve is found, + /// false otherwise. + +private: + ECKeyImpl::Ptr _pImpl; +}; + + +// +// inlines +// +inline ECKeyImpl::Ptr ECKey::impl() const +{ + return _pImpl; +} + + +inline std::string ECKey::getCurveName(int nid) +{ + return ECKeyImpl::getCurveName(nid); +} + + +inline int ECKey::getCurveNID(std::string& name) +{ + return ECKeyImpl::getCurveNID(name); +} + + +inline bool ECKey::hasCurve(const std::string& name) +{ + return ECKeyImpl::hasCurve(name); +} + + +} } // namespace Poco::Crypto + + +#endif // Crypto_ECKey_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/ECKeyImpl.h b/base/poco/Crypto/include/Poco/Crypto/ECKeyImpl.h new file mode 100644 index 00000000000..840764304d1 --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/ECKeyImpl.h @@ -0,0 +1,174 @@ +// +// ECKeyImpl.h +// +// +// Library: Crypto +// Package: EC +// Module: ECKeyImpl +// +// Definition of the ECKeyImpl class. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_ECKeyImplImpl_INCLUDED +#define Crypto_ECKeyImplImpl_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/EVPPKey.h" +#include "Poco/Crypto/KeyPairImpl.h" +#include "Poco/Crypto/OpenSSLInitializer.h" +#include "Poco/RefCountedObject.h" +#include "Poco/AutoPtr.h" +#include +#include +#include +#include +#include + + +namespace Poco { +namespace Crypto { + + +class X509Certificate; +class PKCS12Container; + + +class ECKeyImpl: public KeyPairImpl + /// Elliptic Curve key clas implementation. +{ +public: + typedef Poco::AutoPtr Ptr; + typedef std::vector ByteVec; + + ECKeyImpl(const EVPPKey& key); + /// Constructs ECKeyImpl by extracting the EC key. + + ECKeyImpl(const X509Certificate& cert); + /// Constructs ECKeyImpl by extracting the EC public key from the given certificate. + + ECKeyImpl(const PKCS12Container& cert); + /// Constructs ECKeyImpl by extracting the EC private key from the given certificate. + + ECKeyImpl(int eccGroup); + /// Creates the ECKey of the specified group. Creates a new public/private keypair using the given parameters. + /// Can be used to sign data and verify signatures. + + ECKeyImpl(const std::string& publicKeyFile, const std::string& privateKeyFile, const std::string& privateKeyPassphrase); + /// Creates the ECKey, by reading public and private key from the given files and + /// using the given passphrase for the private key. Can only by used for signing if + /// a private key is available. + + ECKeyImpl(std::istream* pPublicKeyStream, std::istream* pPrivateKeyStream, const std::string& privateKeyPassphrase); + /// Creates the ECKey. Can only by used for signing if pPrivKey + /// is not null. If a private key file is specified, you don't need to + /// specify a public key file. OpenSSL will auto-create it from the private key. + + ~ECKeyImpl(); + /// Destroys the ECKeyImpl. + + EC_KEY* getECKey(); + /// Returns the OpenSSL EC key. + + const EC_KEY* getECKey() const; + /// Returns the OpenSSL EC key. + + int size() const; + /// Returns the EC key length in bits. + + int groupId() const; + /// Returns the EC key group integer Id. + + std::string groupName() const; + /// Returns the EC key group name. + + void save(const std::string& publicKeyFile, + const std::string& privateKeyFile = "", + const std::string& privateKeyPassphrase = "") const; + /// Exports the public and private keys to the given files. + /// + /// If an empty filename is specified, the corresponding key + /// is not exported. + + void save(std::ostream* pPublicKeyStream, + std::ostream* pPrivateKeyStream = 0, + const std::string& privateKeyPassphrase = "") const; + /// Exports the public and private key to the given streams. + /// + /// If a null pointer is passed for a stream, the corresponding + /// key is not exported. + + static std::string getCurveName(int nid = -1); + /// Returns elliptical curve name corresponding to + /// the given nid; if nid is not found, returns + /// empty string. + /// + /// If nid is -1, returns first curve name. + /// + /// If no curves are found, returns empty string; + + static int getCurveNID(std::string& name); + /// Returns the NID of the specified curve. + /// + /// If name is empty, returns the first curve NID + /// and updates the name accordingly. + + static bool hasCurve(const std::string& name); + /// Returns true if the named curve is found, + /// false otherwise. + +private: + void checkEC(const std::string& method, const std::string& func) const; + void freeEC(); + + EC_KEY* _pEC; +}; + + +// +// inlines +// +inline EC_KEY* ECKeyImpl::getECKey() +{ + return _pEC; +} + + +inline const EC_KEY* ECKeyImpl::getECKey() const +{ + return _pEC; +} + + +inline std::string ECKeyImpl::groupName() const +{ + return OBJ_nid2sn(groupId()); +} + + +inline void ECKeyImpl::save(const std::string& publicKeyFile, + const std::string& privateKeyFile, + const std::string& privateKeyPassphrase) const +{ + EVPPKey(_pEC).save(publicKeyFile, privateKeyFile, privateKeyPassphrase); +} + + +inline void ECKeyImpl::save(std::ostream* pPublicKeyStream, + std::ostream* pPrivateKeyStream, + const std::string& privateKeyPassphrase) const +{ + EVPPKey(_pEC).save(pPublicKeyStream, pPrivateKeyStream, privateKeyPassphrase); +} + + +} } // namespace Poco::Crypto + + +#endif // Crypto_ECKeyImplImpl_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h b/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h new file mode 100644 index 00000000000..fbcdad5b19c --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h @@ -0,0 +1,354 @@ +// +// EVPPKey.h +// +// +// Library: Crypto +// Package: CryptoCore +// Module: EVPPKey +// +// Definition of the EVPPKey class. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_EVPPKeyImpl_INCLUDED +#define Crypto_EVPPKeyImpl_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/CryptoException.h" +#include "Poco/StreamCopier.h" +#include +#include +#include +#include +#include +#include + + +namespace Poco { +namespace Crypto { + + +class ECKey; +class RSAKey; + + +class Crypto_API EVPPKey + /// Utility class for conversion of native keys to EVP. + /// Currently, only RSA and EC keys are supported. +{ +public: + explicit EVPPKey(const std::string& ecCurveName); + /// Constructs EVPPKey from ECC curve name. + /// + /// Only EC keys can be wrapped by an EVPPKey + /// created using this constructor. + + explicit EVPPKey(const char* ecCurveName); + /// Constructs EVPPKey from ECC curve name. + /// + /// Only EC keys can be wrapped by an EVPPKey + /// created using this constructor. + + explicit EVPPKey(EVP_PKEY* pEVPPKey); + /// Constructs EVPPKey from EVP_PKEY pointer. + /// The content behind the supplied pointer is internally duplicated. + + template + explicit EVPPKey(K* pKey): _pEVPPKey(EVP_PKEY_new()) + /// Constructs EVPPKey from a "native" OpenSSL (RSA or EC_KEY), + /// or a Poco wrapper (RSAKey, ECKey) key pointer. + { + if (!_pEVPPKey) throw OpenSSLException(); + setKey(pKey); + } + + EVPPKey(const std::string& publicKeyFile, const std::string& privateKeyFile, const std::string& privateKeyPassphrase = ""); + /// Creates the EVPPKey, by reading public and private key from the given files and + /// using the given passphrase for the private key. Can only by used for signing if + /// a private key is available. + + EVPPKey(std::istream* pPublicKeyStream, std::istream* pPrivateKeyStream, const std::string& privateKeyPassphrase = ""); + /// Creates the EVPPKey. Can only by used for signing if pPrivKey + /// is not null. If a private key file is specified, you don't need to + /// specify a public key file. OpenSSL will auto-create it from the private key. + + EVPPKey(const EVPPKey& other); + /// Copy constructor. + + EVPPKey& operator=(const EVPPKey& other); + /// Assignment operator. + +#ifdef POCO_ENABLE_CPP11 + + EVPPKey(EVPPKey&& other); + /// Move constructor. + + EVPPKey& operator=(EVPPKey&& other); + /// Assignment move operator. + +#endif // POCO_ENABLE_CPP11 + + ~EVPPKey(); + /// Destroys the EVPPKey. + + bool operator == (const EVPPKey& other) const; + /// Comparison operator. + /// Returns true if public key components and parameters + /// of the other key are equal to this key. + /// + /// Works as expected when one key contains only public key, + /// while the other one contains private (thus also public) key. + + bool operator != (const EVPPKey& other) const; + /// Comparison operator. + /// Returns true if public key components and parameters + /// of the other key are different from this key. + /// + /// Works as expected when one key contains only public key, + /// while the other one contains private (thus also public) key. + + void save(const std::string& publicKeyFile, const std::string& privateKeyFile = "", const std::string& privateKeyPassphrase = "") const; + /// Exports the public and/or private keys to the given files. + /// + /// If an empty filename is specified, the corresponding key + /// is not exported. + + void save(std::ostream* pPublicKeyStream, std::ostream* pPrivateKeyStream = 0, const std::string& privateKeyPassphrase = "") const; + /// Exports the public and/or private key to the given streams. + /// + /// If a null pointer is passed for a stream, the corresponding + /// key is not exported. + + int type() const; + /// Retuns the EVPPKey type NID. + + bool isSupported(int type) const; + /// Returns true if OpenSSL type is supported + + operator const EVP_PKEY*() const; + /// Returns const pointer to the OpenSSL EVP_PKEY structure. + + operator EVP_PKEY*(); + /// Returns pointer to the OpenSSL EVP_PKEY structure. + + static EVP_PKEY* duplicate(const EVP_PKEY* pFromKey, EVP_PKEY** pToKey); + /// Duplicates pFromKey into *pToKey and returns + // the pointer to duplicated EVP_PKEY. + +private: + EVPPKey(); + + static int type(const EVP_PKEY* pEVPPKey); + void newECKey(const char* group); + void duplicate(EVP_PKEY* pEVPPKey); + + void setKey(ECKey* pKey); + void setKey(RSAKey* pKey); + void setKey(EC_KEY* pKey); + void setKey(RSA* pKey); + static int passCB(char* buf, int size, int, void* pass); + + typedef EVP_PKEY* (*PEM_read_FILE_Key_fn)(FILE*, EVP_PKEY**, pem_password_cb*, void*); + typedef EVP_PKEY* (*PEM_read_BIO_Key_fn)(BIO*, EVP_PKEY**, pem_password_cb*, void*); + typedef void* (*EVP_PKEY_get_Key_fn)(EVP_PKEY*); + + // The following load*() functions are used by both native and EVP_PKEY type key + // loading from BIO/FILE. + // When used for EVP key loading, getFunc is null (ie. native key is not extracted + // from the loaded EVP_PKEY). + template + static bool loadKey(K** ppKey, + PEM_read_FILE_Key_fn readFunc, + F getFunc, + const std::string& keyFile, + const std::string& pass = "") + { + poco_assert_dbg (((typeid(K*) == typeid(RSA*) || typeid(K*) == typeid(EC_KEY*)) && getFunc) || + ((typeid(K*) == typeid(EVP_PKEY*)) && !getFunc)); + poco_check_ptr (ppKey); + poco_assert_dbg (!*ppKey); + + FILE* pFile = 0; + if (!keyFile.empty()) + { + if (!getFunc) *ppKey = (K*)EVP_PKEY_new(); + EVP_PKEY* pKey = getFunc ? EVP_PKEY_new() : (EVP_PKEY*)*ppKey; + if (pKey) + { + pFile = fopen(keyFile.c_str(), "r"); + if (pFile) + { + pem_password_cb* pCB = pass.empty() ? (pem_password_cb*)0 : &passCB; + void* pPassword = pass.empty() ? (void*)0 : (void*)pass.c_str(); + if (readFunc(pFile, &pKey, pCB, pPassword)) + { + fclose(pFile); pFile = 0; + if(getFunc) + { + *ppKey = (K*)getFunc(pKey); + EVP_PKEY_free(pKey); + } + else + { + poco_assert_dbg (typeid(K*) == typeid(EVP_PKEY*)); + *ppKey = (K*)pKey; + } + if(!*ppKey) goto error; + return true; + } + goto error; + } + else + { + if (getFunc) EVP_PKEY_free(pKey); + throw IOException("ECKeyImpl, cannot open file", keyFile); + } + } + else goto error; + } + return false; + + error: + if (pFile) fclose(pFile); + throw OpenSSLException("EVPKey::loadKey(string)"); + } + + template + static bool loadKey(K** ppKey, + PEM_read_BIO_Key_fn readFunc, + F getFunc, + std::istream* pIstr, + const std::string& pass = "") + { + poco_assert_dbg (((typeid(K*) == typeid(RSA*) || typeid(K*) == typeid(EC_KEY*)) && getFunc) || + ((typeid(K*) == typeid(EVP_PKEY*)) && !getFunc)); + poco_check_ptr(ppKey); + poco_assert_dbg(!*ppKey); + + BIO* pBIO = 0; + if (pIstr) + { + std::ostringstream ostr; + Poco::StreamCopier::copyStream(*pIstr, ostr); + std::string key = ostr.str(); + pBIO = BIO_new_mem_buf(const_cast(key.data()), static_cast(key.size())); + if (pBIO) + { + if (!getFunc) *ppKey = (K*)EVP_PKEY_new(); + EVP_PKEY* pKey = getFunc ? EVP_PKEY_new() : (EVP_PKEY*)*ppKey; + if (pKey) + { + pem_password_cb* pCB = pass.empty() ? (pem_password_cb*)0 : &passCB; + void* pPassword = pass.empty() ? (void*)0 : (void*)pass.c_str(); + if (readFunc(pBIO, &pKey, pCB, pPassword)) + { + BIO_free(pBIO); pBIO = 0; + if (getFunc) + { + *ppKey = (K*)getFunc(pKey); + EVP_PKEY_free(pKey); + } + else + { + poco_assert_dbg (typeid(K*) == typeid(EVP_PKEY*)); + *ppKey = (K*)pKey; + } + if (!*ppKey) goto error; + return true; + } + if (getFunc) EVP_PKEY_free(pKey); + goto error; + } + else goto error; + } + else goto error; + } + return false; + + error: + if (pBIO) BIO_free(pBIO); + throw OpenSSLException("EVPKey::loadKey(stream)"); + } + + EVP_PKEY* _pEVPPKey; + + friend class ECKeyImpl; + friend class RSAKeyImpl; +}; + + +// +// inlines +// + + +inline bool EVPPKey::operator == (const EVPPKey& other) const +{ + poco_check_ptr (other._pEVPPKey); + poco_check_ptr (_pEVPPKey); + return (1 == EVP_PKEY_cmp(_pEVPPKey, other._pEVPPKey)); +} + + +inline bool EVPPKey::operator != (const EVPPKey& other) const +{ + return !(other == *this); +} + + +inline int EVPPKey::type(const EVP_PKEY* pEVPPKey) +{ + if (!pEVPPKey) return NID_undef; + + return EVP_PKEY_type(EVP_PKEY_id(pEVPPKey)); +} + + +inline int EVPPKey::type() const +{ + return type(_pEVPPKey); +} + + +inline bool EVPPKey::isSupported(int type) const +{ + return type == EVP_PKEY_EC || type == EVP_PKEY_RSA; +} + + +inline EVPPKey::operator const EVP_PKEY*() const +{ + return _pEVPPKey; +} + + +inline EVPPKey::operator EVP_PKEY*() +{ + return _pEVPPKey; +} + + +inline void EVPPKey::setKey(EC_KEY* pKey) +{ + if (!EVP_PKEY_set1_EC_KEY(_pEVPPKey, pKey)) + throw OpenSSLException(); +} + + +inline void EVPPKey::setKey(RSA* pKey) +{ + if (!EVP_PKEY_set1_RSA(_pEVPPKey, pKey)) + throw OpenSSLException(); +} + + +} } // namespace Poco::Crypto + + +#endif // Crypto_EVPPKeyImpl_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/KeyPair.h b/base/poco/Crypto/include/Poco/Crypto/KeyPair.h new file mode 100644 index 00000000000..b9a705f8f1b --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/KeyPair.h @@ -0,0 +1,133 @@ +// +// KeyPair.h +// +// +// Library: Crypto +// Package: CryptoCore +// Module: KeyPair +// +// Definition of the KeyPair class. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_KeyPair_INCLUDED +#define Crypto_KeyPair_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/KeyPairImpl.h" + + +namespace Poco { +namespace Crypto { + + +class X509Certificate; + + +class Crypto_API KeyPair + /// This is a parent class for classes storing a key pair, consisting + /// of private and public key. Storage of the private key is optional. + /// + /// If a private key is available, the KeyPair can be + /// used for decrypting data (encrypted with the public key) + /// or computing secure digital signatures. +{ +public: + enum Type + { + KT_RSA = KeyPairImpl::KT_RSA_IMPL, + KT_EC = KeyPairImpl::KT_EC_IMPL + }; + + explicit KeyPair(KeyPairImpl::Ptr pKeyPairImpl = 0); + /// Extracts the RSA public key from the given certificate. + + virtual ~KeyPair(); + /// Destroys the KeyPair. + + virtual int size() const; + /// Returns the RSA modulus size. + + virtual void save(const std::string& publicKeyPairFile, + const std::string& privateKeyPairFile = "", + const std::string& privateKeyPairPassphrase = "") const; + /// Exports the public and private keys to the given files. + /// + /// If an empty filename is specified, the corresponding key + /// is not exported. + + virtual void save(std::ostream* pPublicKeyPairStream, + std::ostream* pPrivateKeyPairStream = 0, + const std::string& privateKeyPairPassphrase = "") const; + /// Exports the public and private key to the given streams. + /// + /// If a null pointer is passed for a stream, the corresponding + /// key is not exported. + + KeyPairImpl::Ptr impl() const; + /// Returns the impl object. + + const std::string& name() const; + /// Returns key pair name + + Type type() const; + /// Returns key pair type + +private: + KeyPairImpl::Ptr _pImpl; +}; + + +// +// inlines +// + +inline int KeyPair::size() const +{ + return _pImpl->size(); +} + + +inline void KeyPair::save(const std::string& publicKeyFile, + const std::string& privateKeyFile, + const std::string& privateKeyPassphrase) const +{ + _pImpl->save(publicKeyFile, privateKeyFile, privateKeyPassphrase); +} + + +inline void KeyPair::save(std::ostream* pPublicKeyStream, + std::ostream* pPrivateKeyStream, + const std::string& privateKeyPassphrase) const +{ + _pImpl->save(pPublicKeyStream, pPrivateKeyStream, privateKeyPassphrase); +} + + +inline const std::string& KeyPair::name() const +{ + return _pImpl->name(); +} + +inline KeyPairImpl::Ptr KeyPair::impl() const +{ + return _pImpl; +} + + +inline KeyPair::Type KeyPair::type() const +{ + return (KeyPair::Type)impl()->type(); +} + + +} } // namespace Poco::Crypto + + +#endif // Crypto_KeyPair_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/KeyPairImpl.h b/base/poco/Crypto/include/Poco/Crypto/KeyPairImpl.h new file mode 100644 index 00000000000..e6320df044b --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/KeyPairImpl.h @@ -0,0 +1,107 @@ +// +// KeyPairImpl.h +// +// +// Library: Crypto +// Package: CryptoCore +// Module: KeyPairImpl +// +// Definition of the KeyPairImpl class. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_KeyPairImplImpl_INCLUDED +#define Crypto_KeyPairImplImpl_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/OpenSSLInitializer.h" +#include "Poco/RefCountedObject.h" +#include "Poco/AutoPtr.h" +#include +#include + + +namespace Poco { +namespace Crypto { + + +class KeyPairImpl: public Poco::RefCountedObject + /// Class KeyPairImpl +{ +public: + enum Type + { + KT_RSA_IMPL = 0, + KT_EC_IMPL + }; + + typedef Poco::AutoPtr Ptr; + typedef std::vector ByteVec; + + KeyPairImpl(const std::string& name, Type type); + /// Create KeyPairImpl with specified type and name. + + virtual ~KeyPairImpl(); + /// Destroys the KeyPairImpl. + + virtual int size() const = 0; + /// Returns the key size. + + virtual void save(const std::string& publicKeyFile, + const std::string& privateKeyFile = "", + const std::string& privateKeyPassphrase = "") const = 0; + /// Exports the public and private keys to the given files. + /// + /// If an empty filename is specified, the corresponding key + /// is not exported. + + virtual void save(std::ostream* pPublicKeyStream, + std::ostream* pPrivateKeyStream = 0, + const std::string& privateKeyPassphrase = "") const = 0; + /// Exports the public and private key to the given streams. + /// + /// If a null pointer is passed for a stream, the corresponding + /// key is not exported. + + const std::string& name() const; + /// Returns key pair name + + Type type() const; + /// Returns key pair type + +private: + KeyPairImpl(); + + std::string _name; + Type _type; + OpenSSLInitializer _openSSLInitializer; +}; + + +// +// inlines +// + + +inline const std::string& KeyPairImpl::name() const +{ + return _name; +} + + +inline KeyPairImpl::Type KeyPairImpl::type() const +{ + return _type; +} + + +} } // namespace Poco::Crypto + + +#endif // Crypto_KeyPairImplImpl_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/OpenSSLInitializer.h b/base/poco/Crypto/include/Poco/Crypto/OpenSSLInitializer.h new file mode 100644 index 00000000000..42c97ae465d --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/OpenSSLInitializer.h @@ -0,0 +1,115 @@ +// +// OpenSSLInitializer.h +// +// Library: Crypto +// Package: CryptoCore +// Module: OpenSSLInitializer +// +// Definition of the OpenSSLInitializer class. +// +// Copyright (c) 2006-2009, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_OpenSSLInitializer_INCLUDED +#define Crypto_OpenSSLInitializer_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Mutex.h" +#include "Poco/AtomicCounter.h" +#include + +#if defined(OPENSSL_FIPS) && OPENSSL_VERSION_NUMBER < 0x010001000L +#include +#endif + + +extern "C" +{ + struct CRYPTO_dynlock_value + { + Poco::FastMutex _mutex; + }; +} + + +namespace Poco { +namespace Crypto { + + +class Crypto_API OpenSSLInitializer + /// Initalizes the OpenSSL library. + /// + /// The class ensures the earliest initialization and the + /// latest shutdown of the OpenSSL library. +{ +public: + OpenSSLInitializer(); + /// Automatically initialize OpenSSL on startup. + + ~OpenSSLInitializer(); + /// Automatically shut down OpenSSL on exit. + + static void initialize(); + /// Initializes the OpenSSL machinery. + + static void uninitialize(); + /// Shuts down the OpenSSL machinery. + + static bool isFIPSEnabled(); + // Returns true if FIPS mode is enabled, false otherwise. + + static void enableFIPSMode(bool enabled); + // Enable or disable FIPS mode. If FIPS is not available, this method doesn't do anything. + +protected: + enum + { + SEEDSIZE = 256 + }; + + // OpenSSL multithreading support + static void lock(int mode, int n, const char* file, int line); + static unsigned long id(); + static struct CRYPTO_dynlock_value* dynlockCreate(const char* file, int line); + static void dynlock(int mode, struct CRYPTO_dynlock_value* lock, const char* file, int line); + static void dynlockDestroy(struct CRYPTO_dynlock_value* lock, const char* file, int line); + +private: + static Poco::FastMutex* _mutexes; + static Poco::AtomicCounter _rc; +}; + + +// +// inlines +// +inline bool OpenSSLInitializer::isFIPSEnabled() +{ +#ifdef OPENSSL_FIPS + return FIPS_mode() ? true : false; +#else + return false; +#endif +} + +#ifdef OPENSSL_FIPS +inline void OpenSSLInitializer::enableFIPSMode(bool enabled) +{ + FIPS_mode_set(enabled); +} +#else +inline void OpenSSLInitializer::enableFIPSMode(bool /*enabled*/) +{ +} +#endif + + +} } // namespace Poco::Crypto + + +#endif // Crypto_OpenSSLInitializer_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/PKCS12Container.h b/base/poco/Crypto/include/Poco/Crypto/PKCS12Container.h new file mode 100644 index 00000000000..63cc224d8cc --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/PKCS12Container.h @@ -0,0 +1,159 @@ +// +// PKCS12Container.h +// +// Library: Crypto +// Package: Certificate +// Module: PKCS12Container +// +// Definition of the PKCS12Container class. +// +// Copyright (c) 2006-2009, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_PKCS12Container_INCLUDED +#define Crypto_PKCS12Container_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/OpenSSLInitializer.h" +#include "Poco/Crypto/X509Certificate.h" +#include "Poco/Crypto/EVPPKey.h" +#include "Poco/Path.h" +#include +#include +#include + + +namespace Poco { +namespace Crypto { + + +class Crypto_API PKCS12Container + /// This class implements PKCS#12 container functionality. +{ +public: + typedef X509Certificate::List CAList; + typedef std::vector CANameList; + + explicit PKCS12Container(std::istream& istr, const std::string& password = ""); + /// Creates the PKCS12Container object from a stream. + + explicit PKCS12Container(const std::string& path, const std::string& password = ""); + /// Creates the PKCS12Container object from a file. + + PKCS12Container(const PKCS12Container& cont); + /// Copy constructor. + + PKCS12Container& operator = (const PKCS12Container& cont); + /// Assignment operator. + +#ifdef POCO_ENABLE_CPP11 + + PKCS12Container(PKCS12Container&& cont); + /// Move constructor. + + PKCS12Container& operator = (PKCS12Container&& cont); + /// Move assignment operator. + +#endif // POCO_ENABLE_CPP11 + + ~PKCS12Container(); + /// Destroys the PKCS12Container. + + bool hasKey() const; + /// Returns true if container contains the key. + + EVPPKey getKey() const; + /// Return key as openssl EVP_PKEY wrapper object. + + bool hasX509Certificate() const; + /// Returns true if container has X509 certificate. + + const X509Certificate& getX509Certificate() const; + /// Returns the X509 certificate. + /// Throws NotFoundException if there is no certificate. + + const CAList& getCACerts() const; + /// Returns the list of CA certificates in this container. + + const std::string& getFriendlyName() const; + /// Returns the friendly name of the certificate bag. + + const CANameList& getFriendlyNamesCA() const; + /// Returns a list of CA certificates friendly names. + +private: + void load(PKCS12* pPKCS12, const std::string& password = ""); + std::string extractFriendlyName(X509* pCert); + +#ifdef POCO_ENABLE_CPP11 + typedef std::unique_ptr CertPtr; +#else + typedef std::auto_ptr CertPtr; +#endif // #ifdef POCO_ENABLE_CPP11 + + OpenSSLInitializer _openSSLInitializer; + EVP_PKEY* _pKey; + CertPtr _pX509Cert; + CAList _caCertList; + CANameList _caCertNames; + std::string _pkcsFriendlyName; +}; + + +// +// inlines +// + +inline bool PKCS12Container::hasX509Certificate() const +{ + return _pX509Cert.get() != 0; +} + + +inline const X509Certificate& PKCS12Container::getX509Certificate() const +{ + if (!hasX509Certificate()) + throw NotFoundException("PKCS12Container X509 certificate"); + return *_pX509Cert; +} + + +inline const std::string& PKCS12Container::getFriendlyName() const +{ + return _pkcsFriendlyName; +} + + +inline const PKCS12Container::CAList& PKCS12Container::getCACerts() const +{ + return _caCertList; +} + + +inline const PKCS12Container::CANameList& PKCS12Container::getFriendlyNamesCA() const +{ + return _caCertNames; +} + + +inline bool PKCS12Container::hasKey() const +{ + return _pKey != 0; +} + + +inline EVPPKey PKCS12Container::getKey() const +{ + return EVPPKey(_pKey); +} + + +} } // namespace Poco::Crypto + + +#endif // Crypto_PKCS12Container_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/RSACipherImpl.h b/base/poco/Crypto/include/Poco/Crypto/RSACipherImpl.h new file mode 100644 index 00000000000..2ebc38e3b55 --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/RSACipherImpl.h @@ -0,0 +1,77 @@ +// +// RSACipherImpl.h +// +// Library: Crypto +// Package: RSA +// Module: RSACipherImpl +// +// Definition of the RSACipherImpl class. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_RSACipherImpl_INCLUDED +#define Crypto_RSACipherImpl_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/Cipher.h" +#include "Poco/Crypto/RSAKey.h" +#include "Poco/Crypto/OpenSSLInitializer.h" +#include + + +namespace Poco { +namespace Crypto { + + +class RSACipherImpl: public Cipher + /// An implementation of the Cipher class for + /// asymmetric (public-private key) encryption + /// based on the the RSA algorithm in OpenSSL's + /// crypto library. + /// + /// Encryption is using the public key, decryption + /// requires the private key. +{ +public: + RSACipherImpl(const RSAKey& key, RSAPaddingMode paddingMode); + /// Creates a new RSACipherImpl object for the given RSAKey + /// and using the given padding mode. + + virtual ~RSACipherImpl(); + /// Destroys the RSACipherImpl. + + const std::string& name() const; + /// Returns the name of the Cipher. + + CryptoTransform* createEncryptor(); + /// Creates an encryptor object. + + CryptoTransform* createDecryptor(); + /// Creates a decryptor object. + +private: + RSAKey _key; + RSAPaddingMode _paddingMode; + OpenSSLInitializer _openSSLInitializer; +}; + + +// +// Inlines +// +inline const std::string& RSACipherImpl::name() const +{ + return _key.name(); +} + + +} } // namespace Poco::Crypto + + +#endif // Crypto_RSACipherImpl_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/RSADigestEngine.h b/base/poco/Crypto/include/Poco/Crypto/RSADigestEngine.h new file mode 100644 index 00000000000..7c4d3860508 --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/RSADigestEngine.h @@ -0,0 +1,111 @@ +// +// RSADigestEngine.h +// +// Library: Crypto +// Package: RSA +// Module: RSADigestEngine +// +// Definition of the RSADigestEngine class. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_RSADigestEngine_INCLUDED +#define Crypto_RSADigestEngine_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/RSAKey.h" +#include "Poco/DigestEngine.h" +#include "Poco/Crypto/DigestEngine.h" +#include +#include + + +namespace Poco { +namespace Crypto { + + +class Crypto_API RSADigestEngine: public Poco::DigestEngine + /// This class implements a Poco::DigestEngine that can be + /// used to compute a secure digital signature. + /// + /// First another Poco::Crypto::DigestEngine is created and + /// used to compute a cryptographic hash of the data to be + /// signed. Then, the hash value is encrypted, using + /// the RSA private key. + /// + /// To verify a signature, pass it to the verify() + /// member function. It will decrypt the signature + /// using the RSA public key and compare the resulting + /// hash with the actual hash of the data. +{ +public: + enum DigestType + { + DIGEST_MD5, + DIGEST_SHA1 + }; + + //@ deprecated + RSADigestEngine(const RSAKey& key, DigestType digestType = DIGEST_SHA1); + /// Creates the RSADigestEngine with the given RSA key, + /// using the MD5 or SHA-1 hash algorithm. + /// Kept for backward compatibility + + RSADigestEngine(const RSAKey& key, const std::string &name); + /// Creates the RSADigestEngine with the given RSA key, + /// using the hash algorithm with the given name + /// (e.g., "MD5", "SHA1", "SHA256", "SHA512", etc.). + /// See the OpenSSL documentation for a list of supported digest algorithms. + /// + /// Throws a Poco::NotFoundException if no algorithm with the given name exists. + + ~RSADigestEngine(); + /// Destroys the RSADigestEngine. + + std::size_t digestLength() const; + /// Returns the length of the digest in bytes. + + void reset(); + /// Resets the engine so that a new + /// digest can be computed. + + const DigestEngine::Digest& digest(); + /// Finishes the computation of the digest + /// (the first time it's called) and + /// returns the message digest. + /// + /// Can be called multiple times. + + const DigestEngine::Digest& signature(); + /// Signs the digest using the RSA algorithm + /// and the private key (the first time it's + /// called) and returns the result. + /// + /// Can be called multiple times. + + bool verify(const DigestEngine::Digest& signature); + /// Verifies the data against the signature. + /// + /// Returns true if the signature can be verified, false otherwise. + +protected: + void updateImpl(const void* data, std::size_t length); + +private: + RSAKey _key; + Poco::Crypto::DigestEngine _engine; + Poco::DigestEngine::Digest _digest; + Poco::DigestEngine::Digest _signature; +}; + + +} } // namespace Poco::Crypto + + +#endif // Crypto_RSADigestEngine_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/RSAKey.h b/base/poco/Crypto/include/Poco/Crypto/RSAKey.h new file mode 100644 index 00000000000..ad9163ed42f --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/RSAKey.h @@ -0,0 +1,125 @@ +// +// RSAKey.h +// +// Library: Crypto +// Package: RSA +// Module: RSAKey +// +// Definition of the RSAKey class. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_RSAKey_INCLUDED +#define Crypto_RSAKey_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/KeyPair.h" +#include "Poco/Crypto/RSAKeyImpl.h" + + +namespace Poco { +namespace Crypto { + + +class X509Certificate; +class PKCS12Container; + + +class Crypto_API RSAKey : public KeyPair + /// This class stores an RSA key pair, consisting + /// of private and public key. Storage of the private + /// key is optional. + /// + /// If a private key is available, the RSAKey can be + /// used for decrypting data (encrypted with the public key) + /// or computing secure digital signatures. +{ +public: + enum KeyLength + { + KL_512 = 512, + KL_1024 = 1024, + KL_2048 = 2048, + KL_4096 = 4096 + }; + + enum Exponent + { + EXP_SMALL = 0, + EXP_LARGE + }; + + RSAKey(const EVPPKey& key); + /// Constructs ECKeyImpl by extracting the EC key. + + RSAKey(const X509Certificate& cert); + /// Extracts the RSA public key from the given certificate. + + RSAKey(const PKCS12Container& cert); + /// Extracts the RSA private key from the given certificate. + + RSAKey(KeyLength keyLength, Exponent exp); + /// Creates the RSAKey. Creates a new public/private keypair using the given parameters. + /// Can be used to sign data and verify signatures. + + RSAKey(const std::string& publicKeyFile, + const std::string& privateKeyFile = "", + const std::string& privateKeyPassphrase = ""); + /// Creates the RSAKey, by reading public and private key from the given files and + /// using the given passphrase for the private key. + /// + /// Cannot be used for signing or decryption unless a private key is available. + /// + /// If a private key is specified, you don't need to specify a public key file. + /// OpenSSL will auto-create the public key from the private key. + + RSAKey(std::istream* pPublicKeyStream, + std::istream* pPrivateKeyStream = 0, + const std::string& privateKeyPassphrase = ""); + /// Creates the RSAKey, by reading public and private key from the given streams and + /// using the given passphrase for the private key. + /// + /// Cannot be used for signing or decryption unless a private key is available. + /// + /// If a private key is specified, you don't need to specify a public key file. + /// OpenSSL will auto-create the public key from the private key. + + ~RSAKey(); + /// Destroys the RSAKey. + + RSAKeyImpl::ByteVec modulus() const; + /// Returns the RSA modulus. + + RSAKeyImpl::ByteVec encryptionExponent() const; + /// Returns the RSA encryption exponent. + + RSAKeyImpl::ByteVec decryptionExponent() const; + /// Returns the RSA decryption exponent. + + RSAKeyImpl::Ptr impl() const; + /// Returns the impl object. + +private: + RSAKeyImpl::Ptr _pImpl; +}; + + +// +// inlines +// +inline RSAKeyImpl::Ptr RSAKey::impl() const +{ + return _pImpl; +} + + +} } // namespace Poco::Crypto + + +#endif // Crypto_RSAKey_INCLUDED \ No newline at end of file diff --git a/base/poco/Crypto/include/Poco/Crypto/RSAKeyImpl.h b/base/poco/Crypto/include/Poco/Crypto/RSAKeyImpl.h new file mode 100644 index 00000000000..035881636b2 --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/RSAKeyImpl.h @@ -0,0 +1,141 @@ +// +// RSAKeyImpl.h +// +// Library: Crypto +// Package: RSA +// Module: RSAKeyImpl +// +// Definition of the RSAKeyImpl class. +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_RSAKeyImplImpl_INCLUDED +#define Crypto_RSAKeyImplImpl_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/EVPPKey.h" +#include "Poco/Crypto/KeyPairImpl.h" +#include "Poco/Crypto/OpenSSLInitializer.h" +#include "Poco/RefCountedObject.h" +#include "Poco/AutoPtr.h" +#include +#include +#include + + +struct bignum_st; +struct rsa_st; +typedef struct bignum_st BIGNUM; +typedef struct rsa_st RSA; + + +namespace Poco { +namespace Crypto { + + +class X509Certificate; +class PKCS12Container; + + +class RSAKeyImpl: public KeyPairImpl + /// class RSAKeyImpl +{ +public: + typedef Poco::AutoPtr Ptr; + typedef std::vector ByteVec; + + RSAKeyImpl(const EVPPKey& key); + /// Constructs ECKeyImpl by extracting the EC key. + + RSAKeyImpl(const X509Certificate& cert); + /// Extracts the RSA public key from the given certificate. + + RSAKeyImpl(const PKCS12Container& cert); + /// Extracts the EC private key from the given certificate. + + RSAKeyImpl(int keyLength, unsigned long exponent); + /// Creates the RSAKey. Creates a new public/private keypair using the given parameters. + /// Can be used to sign data and verify signatures. + + RSAKeyImpl(const std::string& publicKeyFile, const std::string& privateKeyFile, const std::string& privateKeyPassphrase); + /// Creates the RSAKey, by reading public and private key from the given files and + /// using the given passphrase for the private key. Can only by used for signing if + /// a private key is available. + + RSAKeyImpl(std::istream* pPublicKeyStream, std::istream* pPrivateKeyStream, const std::string& privateKeyPassphrase); + /// Creates the RSAKey. Can only by used for signing if pPrivKey + /// is not null. If a private key file is specified, you don't need to + /// specify a public key file. OpenSSL will auto-create it from the private key. + + ~RSAKeyImpl(); + /// Destroys the RSAKeyImpl. + + RSA* getRSA(); + /// Returns the OpenSSL RSA object. + + const RSA* getRSA() const; + /// Returns the OpenSSL RSA object. + + int size() const; + /// Returns the RSA modulus size. + + ByteVec modulus() const; + /// Returns the RSA modulus. + + ByteVec encryptionExponent() const; + /// Returns the RSA encryption exponent. + + ByteVec decryptionExponent() const; + /// Returns the RSA decryption exponent. + + void save(const std::string& publicKeyFile, + const std::string& privateKeyFile = "", + const std::string& privateKeyPassphrase = "") const; + /// Exports the public and private keys to the given files. + /// + /// If an empty filename is specified, the corresponding key + /// is not exported. + + void save(std::ostream* pPublicKeyStream, + std::ostream* pPrivateKeyStream = 0, + const std::string& privateKeyPassphrase = "") const; + /// Exports the public and private key to the given streams. + /// + /// If a null pointer is passed for a stream, the corresponding + /// key is not exported. + +private: + RSAKeyImpl(); + + void freeRSA(); + static ByteVec convertToByteVec(const BIGNUM* bn); + + RSA* _pRSA; +}; + + +// +// inlines +// +inline RSA* RSAKeyImpl::getRSA() +{ + return _pRSA; +} + + +inline const RSA* RSAKeyImpl::getRSA() const +{ + return _pRSA; +} + + +} } // namespace Poco::Crypto + + +#endif // Crypto_RSAKeyImplImpl_INCLUDED \ No newline at end of file diff --git a/base/poco/Crypto/include/Poco/Crypto/X509Certificate.h b/base/poco/Crypto/include/Poco/Crypto/X509Certificate.h new file mode 100644 index 00000000000..ccdab1ce37e --- /dev/null +++ b/base/poco/Crypto/include/Poco/Crypto/X509Certificate.h @@ -0,0 +1,245 @@ +// +// X509Certificate.h +// +// Library: Crypto +// Package: Certificate +// Module: X509Certificate +// +// Definition of the X509Certificate class. +// +// Copyright (c) 2006-2009, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Crypto_X509Certificate_INCLUDED +#define Crypto_X509Certificate_INCLUDED + + +#include "Poco/Crypto/Crypto.h" +#include "Poco/Crypto/OpenSSLInitializer.h" +#include "Poco/DateTime.h" +#include "Poco/SharedPtr.h" +#include +#include +#include +#include + + +namespace Poco { +namespace Crypto { + + +class Crypto_API X509Certificate + /// This class represents a X509 Certificate. +{ +public: + typedef std::vector List; + + enum NID + /// Name identifier for extracting information from + /// a certificate subject's or issuer's distinguished name. + { + NID_COMMON_NAME = 13, + NID_COUNTRY = 14, + NID_LOCALITY_NAME = 15, + NID_STATE_OR_PROVINCE = 16, + NID_ORGANIZATION_NAME = 17, + NID_ORGANIZATION_UNIT_NAME = 18, + NID_PKCS9_EMAIL_ADDRESS = 48, + NID_SERIAL_NUMBER = 105 + }; + + explicit X509Certificate(std::istream& istr); + /// Creates the X509Certificate object by reading + /// a certificate in PEM format from a stream. + + explicit X509Certificate(const std::string& path); + /// Creates the X509Certificate object by reading + /// a certificate in PEM format from a file. + + explicit X509Certificate(X509* pCert); + /// Creates the X509Certificate from an existing + /// OpenSSL certificate. Ownership is taken of + /// the certificate. + + X509Certificate(X509* pCert, bool shared); + /// Creates the X509Certificate from an existing + /// OpenSSL certificate. Ownership is taken of + /// the certificate. If shared is true, the + /// certificate's reference count is incremented. + + X509Certificate(const X509Certificate& cert); + /// Creates the certificate by copying another one. + + X509Certificate& operator = (const X509Certificate& cert); + /// Assigns a certificate. + + void swap(X509Certificate& cert); + /// Exchanges the certificate with another one. + + ~X509Certificate(); + /// Destroys the X509Certificate. + + long version() const; + /// Returns the version of the certificate. + + const std::string& serialNumber() const; + /// Returns the certificate serial number as a + /// string in decimal encoding. + + const std::string& issuerName() const; + /// Returns the certificate issuer's distinguished name. + + std::string issuerName(NID nid) const; + /// Extracts the information specified by the given + /// NID (name identifier) from the certificate issuer's + /// distinguished name. + + const std::string& subjectName() const; + /// Returns the certificate subject's distinguished name. + + std::string subjectName(NID nid) const; + /// Extracts the information specified by the given + /// NID (name identifier) from the certificate subject's + /// distinguished name. + + std::string commonName() const; + /// Returns the common name stored in the certificate + /// subject's distinguished name. + + void extractNames(std::string& commonName, std::set& domainNames) const; + /// Extracts the common name and the alias domain names from the + /// certificate. + + Poco::DateTime validFrom() const; + /// Returns the date and time the certificate is valid from. + + Poco::DateTime expiresOn() const; + /// Returns the date and time the certificate expires. + + void save(std::ostream& stream) const; + /// Writes the certificate to the given stream. + /// The certificate is written in PEM format. + + void save(const std::string& path) const; + /// Writes the certificate to the file given by path. + /// The certificate is written in PEM format. + + bool issuedBy(const X509Certificate& issuerCertificate) const; + /// Checks whether the certificate has been issued by + /// the issuer given by issuerCertificate. This can be + /// used to validate a certificate chain. + /// + /// Verifies if the certificate has been signed with the + /// issuer's private key, using the public key from the issuer + /// certificate. + /// + /// Returns true if verification against the issuer certificate + /// was successful, false otherwise. + + bool equals(const X509Certificate& otherCertificate) const; + /// Checks whether the certificate is equal to + /// the other certificate, by comparing the hashes + /// of both certificates. + /// + /// Returns true if both certificates are identical, + /// otherwise false. + + const X509* certificate() const; + /// Returns the underlying OpenSSL certificate. + + X509* dup() const; + /// Duplicates and returns the underlying OpenSSL certificate. Note that + /// the caller assumes responsibility for the lifecycle of the created + /// certificate. + + std::string signatureAlgorithm() const; + /// Returns the certificate signature algorithm long name. + + void print(std::ostream& out) const; + /// Prints the certificate information to ostream. + + static List readPEM(const std::string& pemFileName); + /// Reads and returns a list of certificates from + /// the specified PEM file. + + static void writePEM(const std::string& pemFileName, const List& list); + /// Writes the list of certificates to the specified PEM file. + +protected: + void load(std::istream& stream); + /// Loads the certificate from the given stream. The + /// certificate must be in PEM format. + + void load(const std::string& path); + /// Loads the certificate from the given file. The + /// certificate must be in PEM format. + + void init(); + /// Extracts issuer and subject name from the certificate. + +private: + enum + { + NAME_BUFFER_SIZE = 256 + }; + + std::string _issuerName; + std::string _subjectName; + std::string _serialNumber; + X509* _pCert; + OpenSSLInitializer _openSSLInitializer; +}; + + +// +// inlines +// + + +inline long X509Certificate::version() const +{ + // This is defined by standards (X.509 et al) to be + // one less than the certificate version. + // So, eg. a version 3 certificate will return 2. + return X509_get_version(_pCert) + 1; +} + + +inline const std::string& X509Certificate::serialNumber() const +{ + return _serialNumber; +} + + +inline const std::string& X509Certificate::issuerName() const +{ + return _issuerName; +} + + +inline const std::string& X509Certificate::subjectName() const +{ + return _subjectName; +} + + +inline const X509* X509Certificate::certificate() const +{ + return _pCert; +} + + +inline X509* X509Certificate::dup() const +{ + return X509_dup(_pCert); +} + + +} } // namespace Poco::Crypto + + +#endif // Crypto_X509Certificate_INCLUDED diff --git a/base/poco/Crypto/src/Cipher.cpp b/base/poco/Crypto/src/Cipher.cpp new file mode 100644 index 00000000000..8b4d57964f5 --- /dev/null +++ b/base/poco/Crypto/src/Cipher.cpp @@ -0,0 +1,140 @@ +// +// Cipher.cpp +// +// Library: Crypto +// Package: Cipher +// Module: Cipher +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/Cipher.h" +#include "Poco/Crypto/CryptoStream.h" +#include "Poco/Crypto/CryptoTransform.h" +#include "Poco/Base64Encoder.h" +#include "Poco/Base64Decoder.h" +#include "Poco/HexBinaryEncoder.h" +#include "Poco/HexBinaryDecoder.h" +#include "Poco/StreamCopier.h" +#include "Poco/Exception.h" +#include +#include + + +namespace Poco { +namespace Crypto { + + +Cipher::Cipher() +{ +} + + +Cipher::~Cipher() +{ +} + + +std::string Cipher::encryptString(const std::string& str, Encoding encoding) +{ + std::istringstream source(str); + std::ostringstream sink; + + encrypt(source, sink, encoding); + + return sink.str(); +} + + +std::string Cipher::decryptString(const std::string& str, Encoding encoding) +{ + std::istringstream source(str); + std::ostringstream sink; + + decrypt(source, sink, encoding); + return sink.str(); +} + + +void Cipher::encrypt(std::istream& source, std::ostream& sink, Encoding encoding) +{ + CryptoInputStream encryptor(source, createEncryptor()); + + switch (encoding) + { + case ENC_NONE: + StreamCopier::copyStream(encryptor, sink); + break; + + case ENC_BASE64: + case ENC_BASE64_NO_LF: + { + Poco::Base64Encoder encoder(sink); + if (encoding == ENC_BASE64_NO_LF) + { + encoder.rdbuf()->setLineLength(0); + } + StreamCopier::copyStream(encryptor, encoder); + encoder.close(); + } + break; + + case ENC_BINHEX: + case ENC_BINHEX_NO_LF: + { + Poco::HexBinaryEncoder encoder(sink); + if (encoding == ENC_BINHEX_NO_LF) + { + encoder.rdbuf()->setLineLength(0); + } + StreamCopier::copyStream(encryptor, encoder); + encoder.close(); + } + break; + + default: + throw Poco::InvalidArgumentException("Invalid argument", "encoding"); + } +} + + +void Cipher::decrypt(std::istream& source, std::ostream& sink, Encoding encoding) +{ + CryptoOutputStream decryptor(sink, createDecryptor()); + + switch (encoding) + { + case ENC_NONE: + StreamCopier::copyStream(source, decryptor); + decryptor.close(); + break; + + case ENC_BASE64: + case ENC_BASE64_NO_LF: + { + Poco::Base64Decoder decoder(source); + StreamCopier::copyStream(decoder, decryptor); + decryptor.close(); + } + break; + + case ENC_BINHEX: + case ENC_BINHEX_NO_LF: + { + Poco::HexBinaryDecoder decoder(source); + StreamCopier::copyStream(decoder, decryptor); + decryptor.close(); + } + break; + + default: + throw Poco::InvalidArgumentException("Invalid argument", "encoding"); + } +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/CipherFactory.cpp b/base/poco/Crypto/src/CipherFactory.cpp new file mode 100644 index 00000000000..28ba3748643 --- /dev/null +++ b/base/poco/Crypto/src/CipherFactory.cpp @@ -0,0 +1,65 @@ +// +// CipherFactory.cpp +// +// Library: Crypto +// Package: Cipher +// Module: CipherFactory +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/CipherFactory.h" +#include "Poco/Crypto/Cipher.h" +#include "Poco/Crypto/CipherKey.h" +#include "Poco/Crypto/RSAKey.h" +#include "Poco/Crypto/CipherImpl.h" +#include "Poco/Crypto/RSACipherImpl.h" +#include "Poco/Exception.h" +#include "Poco/SingletonHolder.h" +#include +#include + + +namespace Poco { +namespace Crypto { + + +CipherFactory::CipherFactory() +{ +} + + +CipherFactory::~CipherFactory() +{ +} + + +namespace +{ + static Poco::SingletonHolder holder; +} + + +CipherFactory& CipherFactory::defaultFactory() +{ + return *holder.get(); +} + + +Cipher* CipherFactory::createCipher(const CipherKey& key) +{ + return new CipherImpl(key); +} + + +Cipher* CipherFactory::createCipher(const RSAKey& key, RSAPaddingMode paddingMode) +{ + return new RSACipherImpl(key, paddingMode); +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/CipherImpl.cpp b/base/poco/Crypto/src/CipherImpl.cpp new file mode 100644 index 00000000000..ae574775957 --- /dev/null +++ b/base/poco/Crypto/src/CipherImpl.cpp @@ -0,0 +1,272 @@ +// +// CipherImpl.cpp +// +// Library: Crypto +// Package: Cipher +// Module: CipherImpl +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/CipherImpl.h" +#include "Poco/Crypto/CryptoTransform.h" +#include "Poco/Exception.h" +#include "Poco/Buffer.h" +#include + + +namespace Poco { +namespace Crypto { + + +namespace +{ + void throwError() + { + unsigned long err; + std::string msg; + + while ((err = ERR_get_error())) + { + if (!msg.empty()) + msg.append("; "); + msg.append(ERR_error_string(err, 0)); + } + + throw Poco::IOException(msg); + } + + + class CryptoTransformImpl: public CryptoTransform + { + public: + typedef Cipher::ByteVec ByteVec; + + enum Direction + { + DIR_ENCRYPT, + DIR_DECRYPT + }; + + CryptoTransformImpl( + const EVP_CIPHER* pCipher, + const ByteVec& key, + const ByteVec& iv, + Direction dir); + + ~CryptoTransformImpl(); + + std::size_t blockSize() const; + int setPadding(int padding); + std::string getTag(std::size_t tagSize); + void setTag(const std::string& tag); + + std::streamsize transform( + const unsigned char* input, + std::streamsize inputLength, + unsigned char* output, + std::streamsize outputLength); + + std::streamsize finalize( + unsigned char* output, + std::streamsize length); + + private: + const EVP_CIPHER* _pCipher; +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + EVP_CIPHER_CTX* _pContext; +#else + EVP_CIPHER_CTX _context; +#endif + ByteVec _key; + ByteVec _iv; + }; + + + CryptoTransformImpl::CryptoTransformImpl( + const EVP_CIPHER* pCipher, + const ByteVec& key, + const ByteVec& iv, + Direction dir): + _pCipher(pCipher), + _key(key), + _iv(iv) + { +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + _pContext = EVP_CIPHER_CTX_new(); + EVP_CipherInit( + _pContext, + _pCipher, + &_key[0], + _iv.empty() ? 0 : &_iv[0], + (dir == DIR_ENCRYPT) ? 1 : 0); +#else + EVP_CipherInit( + &_context, + _pCipher, + &_key[0], + _iv.empty() ? 0 : &_iv[0], + (dir == DIR_ENCRYPT) ? 1 : 0); +#endif + +#if OPENSSL_VERSION_NUMBER >= 0x10001000L + if (_iv.size() != EVP_CIPHER_iv_length(_pCipher) && EVP_CIPHER_mode(_pCipher) == EVP_CIPH_GCM_MODE) + { +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + int rc = EVP_CIPHER_CTX_ctrl(_pContext, EVP_CTRL_GCM_SET_IVLEN, _iv.size(), NULL); +#else + int rc = EVP_CIPHER_CTX_ctrl(&_context, EVP_CTRL_GCM_SET_IVLEN, _iv.size(), NULL); +#endif + if (rc == 0) throwError(); + } +#endif + } + + + CryptoTransformImpl::~CryptoTransformImpl() + { +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + EVP_CIPHER_CTX_cleanup(_pContext); + EVP_CIPHER_CTX_free(_pContext); +#else + EVP_CIPHER_CTX_cleanup(&_context); +#endif + } + + + std::size_t CryptoTransformImpl::blockSize() const + { +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + return EVP_CIPHER_CTX_block_size(_pContext); +#else + return EVP_CIPHER_CTX_block_size(&_context); +#endif + } + + + int CryptoTransformImpl::setPadding(int padding) + { +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + return EVP_CIPHER_CTX_block_size(_pContext); +#else + return EVP_CIPHER_CTX_set_padding(&_context, padding); +#endif + } + + + std::string CryptoTransformImpl::getTag(std::size_t tagSize) + { + std::string tag; +#if OPENSSL_VERSION_NUMBER >= 0x10001000L + Poco::Buffer buffer(tagSize); +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + int rc = EVP_CIPHER_CTX_ctrl(_pContext, EVP_CTRL_GCM_GET_TAG, tagSize, buffer.begin()); +#else + int rc = EVP_CIPHER_CTX_ctrl(&_context, EVP_CTRL_GCM_GET_TAG, tagSize, buffer.begin()); +#endif + if (rc == 0) throwError(); + tag.assign(buffer.begin(), tagSize); +#endif + return tag; + } + + + void CryptoTransformImpl::setTag(const std::string& tag) + { +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + int rc = EVP_CIPHER_CTX_ctrl(_pContext, EVP_CTRL_GCM_SET_TAG, tag.size(), const_cast(tag.data())); +#elif OPENSSL_VERSION_NUMBER >= 0x10001000L + int rc = EVP_CIPHER_CTX_ctrl(&_context, EVP_CTRL_GCM_SET_TAG, tag.size(), const_cast(tag.data())); +#else + int rc = 0; +#endif + if (rc == 0) throwError(); + } + + + std::streamsize CryptoTransformImpl::transform( + const unsigned char* input, + std::streamsize inputLength, + unsigned char* output, + std::streamsize outputLength) + { + poco_assert (outputLength >= (inputLength + blockSize() - 1)); + + int outLen = static_cast(outputLength); +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + int rc = EVP_CipherUpdate( + _pContext, + output, + &outLen, + input, + static_cast(inputLength)); +#else + int rc = EVP_CipherUpdate( + &_context, + output, + &outLen, + input, + static_cast(inputLength)); +#endif + if (rc == 0) + throwError(); + + return static_cast(outLen); + } + + + std::streamsize CryptoTransformImpl::finalize( + unsigned char* output, + std::streamsize length) + { + poco_assert (length >= blockSize()); + + int len = static_cast(length); + + // Use the '_ex' version that does not perform implicit cleanup since we + // will call EVP_CIPHER_CTX_cleanup() from the dtor as there is no + // guarantee that finalize() will be called if an error occurred. +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + int rc = EVP_CipherFinal_ex(_pContext, output, &len); +#else + int rc = EVP_CipherFinal_ex(&_context, output, &len); +#endif + + if (rc == 0) + throwError(); + + return static_cast(len); + } +} + + +CipherImpl::CipherImpl(const CipherKey& key): + _key(key) +{ +} + + +CipherImpl::~CipherImpl() +{ +} + + +CryptoTransform* CipherImpl::createEncryptor() +{ + CipherKeyImpl::Ptr p = _key.impl(); + return new CryptoTransformImpl(p->cipher(), p->getKey(), p->getIV(), CryptoTransformImpl::DIR_ENCRYPT); +} + + +CryptoTransform* CipherImpl::createDecryptor() +{ + CipherKeyImpl::Ptr p = _key.impl(); + return new CryptoTransformImpl(p->cipher(), p->getKey(), p->getIV(), CryptoTransformImpl::DIR_DECRYPT); +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/CipherKey.cpp b/base/poco/Crypto/src/CipherKey.cpp new file mode 100644 index 00000000000..275edd1c883 --- /dev/null +++ b/base/poco/Crypto/src/CipherKey.cpp @@ -0,0 +1,49 @@ +// +// CipherKey.cpp +// +// Library: Crypto +// Package: Cipher +// Module: CipherKey +// +// Copyright (c) 2007, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/CipherKey.h" + + +namespace Poco { +namespace Crypto { + + +CipherKey::CipherKey(const std::string& name, + const std::string& passphrase, + const std::string& salt, + int iterationCount, + const std::string &digest): + _pImpl(new CipherKeyImpl(name, passphrase, salt, iterationCount, digest)) +{ +} + + +CipherKey::CipherKey(const std::string& name, const ByteVec& key, const ByteVec& iv): + _pImpl(new CipherKeyImpl(name, key, iv)) +{ +} + + +CipherKey::CipherKey(const std::string& name): + _pImpl(new CipherKeyImpl(name)) +{ +} + + +CipherKey::~CipherKey() +{ +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/CipherKeyImpl.cpp b/base/poco/Crypto/src/CipherKeyImpl.cpp new file mode 100644 index 00000000000..d5ebc5e3b17 --- /dev/null +++ b/base/poco/Crypto/src/CipherKeyImpl.cpp @@ -0,0 +1,222 @@ +// +// CipherKeyImpl.cpp +// +// Library: Crypto +// Package: Cipher +// Module: CipherKeyImpl +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/CipherKeyImpl.h" +#include "Poco/Crypto/CryptoTransform.h" +#include "Poco/Crypto/CipherFactory.h" +#include "Poco/Exception.h" +#include "Poco/RandomStream.h" +#include +#include + + +namespace Poco { +namespace Crypto { + + +CipherKeyImpl::CipherKeyImpl(const std::string& name, + const std::string& passphrase, + const std::string& salt, + int iterationCount, + const std::string& digest): + _pCipher(0), + _pDigest(0), + _name(name), + _key(), + _iv() +{ + // dummy access to Cipherfactory so that the EVP lib is initilaized + CipherFactory::defaultFactory(); + _pCipher = EVP_get_cipherbyname(name.c_str()); + + if (!_pCipher) + throw Poco::NotFoundException("Cipher " + name + " was not found"); + + _pDigest = EVP_get_digestbyname(digest.c_str()); + + if (!_pDigest) + throw Poco::NotFoundException("Digest " + name + " was not found"); + + _key = ByteVec(keySize()); + _iv = ByteVec(ivSize()); + generateKey(passphrase, salt, iterationCount); +} + + +CipherKeyImpl::CipherKeyImpl(const std::string& name, + const ByteVec& key, + const ByteVec& iv): + _pCipher(0), + _pDigest(0), + _name(name), + _key(key), + _iv(iv) +{ + // dummy access to Cipherfactory so that the EVP lib is initialized + CipherFactory::defaultFactory(); + _pCipher = EVP_get_cipherbyname(name.c_str()); + + if (!_pCipher) + throw Poco::NotFoundException("Cipher " + name + " was not found"); +} + + +CipherKeyImpl::CipherKeyImpl(const std::string& name): + _pCipher(0), + _pDigest(0), + _name(name), + _key(), + _iv() +{ + // dummy access to Cipherfactory so that the EVP lib is initilaized + CipherFactory::defaultFactory(); + _pCipher = EVP_get_cipherbyname(name.c_str()); + + if (!_pCipher) + throw Poco::NotFoundException("Cipher " + name + " was not found"); + _key = ByteVec(keySize()); + _iv = ByteVec(ivSize()); + generateKey(); +} + + +CipherKeyImpl::~CipherKeyImpl() +{ +} + + +CipherKeyImpl::Mode CipherKeyImpl::mode() const +{ + switch (EVP_CIPHER_mode(_pCipher)) + { + case EVP_CIPH_STREAM_CIPHER: + return MODE_STREAM_CIPHER; + + case EVP_CIPH_ECB_MODE: + return MODE_ECB; + + case EVP_CIPH_CBC_MODE: + return MODE_CBC; + + case EVP_CIPH_CFB_MODE: + return MODE_CFB; + + case EVP_CIPH_OFB_MODE: + return MODE_OFB; + +#if OPENSSL_VERSION_NUMBER >= 0x10001000L + case EVP_CIPH_CTR_MODE: + return MODE_CTR; + + case EVP_CIPH_GCM_MODE: + return MODE_GCM; + +#endif + } + throw Poco::IllegalStateException("Unexpected value of EVP_CIPHER_mode()"); +} + + +void CipherKeyImpl::generateKey() +{ + ByteVec vec; + + getRandomBytes(vec, keySize()); + setKey(vec); + + getRandomBytes(vec, ivSize()); + setIV(vec); +} + + +void CipherKeyImpl::getRandomBytes(ByteVec& vec, std::size_t count) +{ + Poco::RandomInputStream random; + + vec.clear(); + vec.reserve(count); + + for (int i = 0; i < count; ++i) + vec.push_back(static_cast(random.get())); +} + + +void CipherKeyImpl::generateKey( + const std::string& password, + const std::string& salt, + int iterationCount) +{ + unsigned char keyBytes[EVP_MAX_KEY_LENGTH]; + unsigned char ivBytes[EVP_MAX_IV_LENGTH]; + + // OpenSSL documentation specifies that the salt must be an 8-byte array. + unsigned char saltBytes[8]; + + if (!salt.empty()) + { + int len = static_cast(salt.size()); + // Create the salt array from the salt string + for (int i = 0; i < 8; ++i) + saltBytes[i] = salt.at(i % len); + for (int i = 8; i < len; ++i) + saltBytes[i % 8] ^= salt.at(i); + } + + // Now create the key and IV, using the MD5 digest algorithm. + int keySize = EVP_BytesToKey( + _pCipher, + _pDigest ? _pDigest : EVP_md5(), + (salt.empty() ? 0 : saltBytes), + reinterpret_cast(password.data()), + static_cast(password.size()), + iterationCount, + keyBytes, + ivBytes); + + // Copy the buffers to our member byte vectors. + _key.assign(keyBytes, keyBytes + keySize); + + if (ivSize() == 0) + _iv.clear(); + else + _iv.assign(ivBytes, ivBytes + ivSize()); +} + + +int CipherKeyImpl::keySize() const +{ + return EVP_CIPHER_key_length(_pCipher); +} + + +int CipherKeyImpl::blockSize() const +{ + return EVP_CIPHER_block_size(_pCipher); +} + + +int CipherKeyImpl::ivSize() const +{ + return EVP_CIPHER_iv_length(_pCipher); +} + + +void CipherKeyImpl::setIV(const ByteVec& iv) +{ + poco_assert(mode() == MODE_GCM || iv.size() == static_cast(ivSize())); + _iv = iv; +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/CryptoException.cpp b/base/poco/Crypto/src/CryptoException.cpp new file mode 100644 index 00000000000..a4450af9bf0 --- /dev/null +++ b/base/poco/Crypto/src/CryptoException.cpp @@ -0,0 +1,108 @@ +// +// CryptoException.cpp +// +// +// Library: Crypto +// Package: Crypto +// Module: CryptoException +// +// Copyright (c) 2012, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/CryptoException.h" +#include "Poco/NumberFormatter.h" +#include +#include + + +namespace Poco { +namespace Crypto { + + +POCO_IMPLEMENT_EXCEPTION(CryptoException, Exception, "Crypto Exception") + + +OpenSSLException::OpenSSLException(int otherCode): CryptoException(otherCode) +{ + setExtMessage(); +} + + +OpenSSLException::OpenSSLException(const std::string& msg, int otherCode): CryptoException(msg, otherCode) +{ + setExtMessage(); +} + + +OpenSSLException::OpenSSLException(const std::string& msg, const std::string& arg, int otherCode): CryptoException(msg, arg, otherCode) +{ + setExtMessage(); +} + + +OpenSSLException::OpenSSLException(const std::string& msg, const Poco::Exception& exc, int otherCode): CryptoException(msg, exc, otherCode) +{ + setExtMessage(); +} + + +OpenSSLException::OpenSSLException(const OpenSSLException& exc): CryptoException(exc) +{ + setExtMessage(); +} + + +OpenSSLException::~OpenSSLException() throw() +{ +} + + +OpenSSLException& OpenSSLException::operator = (const OpenSSLException& exc) +{ + CryptoException::operator = (exc); + return *this; +} + + +const char* OpenSSLException::name() const throw() +{ + return "OpenSSLException"; +} + + +const char* OpenSSLException::className() const throw() +{ + return typeid(*this).name(); +} + + +Poco::Exception* OpenSSLException::clone() const +{ + return new OpenSSLException(*this); +} + + +void OpenSSLException::setExtMessage() +{ + Poco::UInt64 e = static_cast(ERR_get_error()); + char buf[128] = { 0 }; + char* pErr = ERR_error_string(static_cast(e), buf); + std::string err; + if (pErr) err = pErr; + else err = NumberFormatter::format(e); + + extendedMessage(err); +} + + +void OpenSSLException::rethrow() const +{ + throw *this; +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/CryptoStream.cpp b/base/poco/Crypto/src/CryptoStream.cpp new file mode 100644 index 00000000000..6b29b7f0775 --- /dev/null +++ b/base/poco/Crypto/src/CryptoStream.cpp @@ -0,0 +1,355 @@ +// +// CryptoStream.cpp +// +// Library: Crypto +// Package: Cipher +// Module: CryptoStream +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/CryptoStream.h" +#include "Poco/Crypto/CryptoTransform.h" +#include "Poco/Crypto/Cipher.h" +#include "Poco/Exception.h" +#include + + +#undef min +#undef max + + +namespace Poco { +namespace Crypto { + + +// +// CryptoStreamBuf +// + + +CryptoStreamBuf::CryptoStreamBuf(std::istream& istr, CryptoTransform* pTransform, std::streamsize bufferSize): + Poco::BufferedStreamBuf(bufferSize, std::ios::in), + _pTransform(pTransform), + _pIstr(&istr), + _pOstr(0), + _eof(false), + _buffer(static_cast(bufferSize)) +{ + poco_check_ptr (pTransform); + poco_assert (bufferSize > 2 * pTransform->blockSize()); +} + + +CryptoStreamBuf::CryptoStreamBuf(std::ostream& ostr, CryptoTransform* pTransform, std::streamsize bufferSize): + Poco::BufferedStreamBuf(bufferSize, std::ios::out), + _pTransform(pTransform), + _pIstr(0), + _pOstr(&ostr), + _eof(false), + _buffer(static_cast(bufferSize)) +{ + poco_check_ptr (pTransform); + poco_assert (bufferSize > 2 * pTransform->blockSize()); +} + + +CryptoStreamBuf::~CryptoStreamBuf() +{ + try + { + close(); + } + catch (...) + { + } + delete _pTransform; +} + + +void CryptoStreamBuf::close() +{ + sync(); + + if (_pIstr) + { + _pIstr = 0; + } + else if (_pOstr) + { + // Close can be called multiple times. By zeroing the pointer we make + // sure that we call finalize() only once, even if an exception is + // thrown. + std::ostream* pOstr = _pOstr; + _pOstr = 0; + + // Finalize transformation. + std::streamsize n = _pTransform->finalize(_buffer.begin(), static_cast(_buffer.size())); + + if (n > 0) + { + pOstr->write(reinterpret_cast(_buffer.begin()), n); + if (!pOstr->good()) + throw Poco::IOException("Output stream failure"); + } + } +} + + +int CryptoStreamBuf::readFromDevice(char* buffer, std::streamsize length) +{ + if (!_pIstr) + return 0; + + int count = 0; + + while (!_eof) + { + int m = (static_cast(length) - count)/2 - static_cast(_pTransform->blockSize()); + + // Make sure we can read at least one more block. Explicitely check + // for m < 0 since blockSize() returns an unsigned int and the + // comparison might give false results for m < 0. + if (m <= 0) + break; + + int n = 0; + + if (_pIstr->good()) + { + _pIstr->read(reinterpret_cast(_buffer.begin()), m); + n = static_cast(_pIstr->gcount()); + } + + if (n == 0) + { + _eof = true; + + // No more data, finalize transformation + count += static_cast(_pTransform->finalize( + reinterpret_cast(buffer + count), + static_cast(length) - count)); + } + else + { + // Transform next chunk of data + count += static_cast(_pTransform->transform( + _buffer.begin(), + n, + reinterpret_cast(buffer + count), + static_cast(length) - count)); + } + } + + return count; +} + + +int CryptoStreamBuf::writeToDevice(const char* buffer, std::streamsize length) +{ + if (!_pOstr) + return 0; + + std::size_t maxChunkSize = _buffer.size()/2; + std::size_t count = 0; + + while (count < length) + { + // Truncate chunk size so that the maximum output fits into _buffer. + std::size_t n = static_cast(length) - count; + if (n > maxChunkSize) + n = maxChunkSize; + + // Transform next chunk of data + std::streamsize k = _pTransform->transform( + reinterpret_cast(buffer + count), + static_cast(n), + _buffer.begin(), + static_cast(_buffer.size())); + + // Attention: (n != k) might be true. In count, we have to track how + // many bytes from buffer have been consumed, not how many bytes have + // been written to _pOstr! + count += n; + + if (k > 0) + { + _pOstr->write(reinterpret_cast(_buffer.begin()), k); + if (!_pOstr->good()) + throw Poco::IOException("Output stream failure"); + } + } + + return static_cast(count); +} + + +// +// CryptoIOS +// + + +CryptoIOS::CryptoIOS(std::istream& istr, CryptoTransform* pTransform, std::streamsize bufferSize): + _buf(istr, pTransform, bufferSize) +{ + poco_ios_init(&_buf); +} + + +CryptoIOS::CryptoIOS(std::ostream& ostr, CryptoTransform* pTransform, std::streamsize bufferSize): + _buf(ostr, pTransform, bufferSize) +{ + poco_ios_init(&_buf); +} + + +CryptoIOS::~CryptoIOS() +{ +} + + +CryptoStreamBuf* CryptoIOS::rdbuf() +{ + return &_buf; +} + + +// +// CryptoInputStream +// + + +CryptoInputStream::CryptoInputStream(std::istream& istr, CryptoTransform* pTransform, std::streamsize bufferSize): + CryptoIOS(istr, pTransform, bufferSize), + std::istream(&_buf) +{ +} + + +CryptoInputStream::CryptoInputStream(std::istream& istr, Cipher& cipher, std::streamsize bufferSize): + CryptoIOS(istr, cipher.createEncryptor(), bufferSize), + std::istream(&_buf) +{ +} + + +CryptoInputStream::~CryptoInputStream() +{ +} + + +// +// CryptoOutputStream +// + + +CryptoOutputStream::CryptoOutputStream(std::ostream& ostr, CryptoTransform* pTransform, std::streamsize bufferSize): + CryptoIOS(ostr, pTransform, bufferSize), + std::ostream(&_buf) +{ +} + + +CryptoOutputStream::CryptoOutputStream(std::ostream& ostr, Cipher& cipher, std::streamsize bufferSize): + CryptoIOS(ostr, cipher.createDecryptor(), bufferSize), + std::ostream(&_buf) +{ +} + + +CryptoOutputStream::~CryptoOutputStream() +{ +} + + +void CryptoOutputStream::close() +{ + _buf.close(); +} + + +// +// EncryptingInputStream +// + + +EncryptingInputStream::EncryptingInputStream(std::istream& istr, Cipher& cipher, std::streamsize bufferSize): + CryptoIOS(istr, cipher.createEncryptor(), bufferSize), + std::istream(&_buf) +{ +} + + +EncryptingInputStream::~EncryptingInputStream() +{ +} + + +// +// EncryptingOuputStream +// + + +EncryptingOutputStream::EncryptingOutputStream(std::ostream& ostr, Cipher& cipher, std::streamsize bufferSize): + CryptoIOS(ostr, cipher.createEncryptor(), bufferSize), + std::ostream(&_buf) +{ +} + + +EncryptingOutputStream::~EncryptingOutputStream() +{ +} + + +void EncryptingOutputStream::close() +{ + _buf.close(); +} + + +// +// DecryptingInputStream +// + + +DecryptingInputStream::DecryptingInputStream(std::istream& istr, Cipher& cipher, std::streamsize bufferSize): + CryptoIOS(istr, cipher.createDecryptor(), bufferSize), + std::istream(&_buf) +{ +} + + +DecryptingInputStream::~DecryptingInputStream() +{ +} + + +// +// DecryptingOuputStream +// + + +DecryptingOutputStream::DecryptingOutputStream(std::ostream& ostr, Cipher& cipher, std::streamsize bufferSize): + CryptoIOS(ostr, cipher.createDecryptor(), bufferSize), + std::ostream(&_buf) +{ +} + + +DecryptingOutputStream::~DecryptingOutputStream() +{ +} + + +void DecryptingOutputStream::close() +{ + _buf.close(); +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/CryptoTransform.cpp b/base/poco/Crypto/src/CryptoTransform.cpp new file mode 100644 index 00000000000..e4e58fd6e64 --- /dev/null +++ b/base/poco/Crypto/src/CryptoTransform.cpp @@ -0,0 +1,38 @@ +// +// CryptoTransform.cpp +// +// Library: Crypto +// Package: Cipher +// Module: CryptoTransform +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/CryptoTransform.h" + + +namespace Poco { +namespace Crypto { + + +CryptoTransform::CryptoTransform() +{ +} + + +CryptoTransform::~CryptoTransform() +{ +} + + +int CryptoTransform::setPadding(int padding) +{ + return 1; +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/DigestEngine.cpp b/base/poco/Crypto/src/DigestEngine.cpp new file mode 100644 index 00000000000..a09879fe732 --- /dev/null +++ b/base/poco/Crypto/src/DigestEngine.cpp @@ -0,0 +1,80 @@ +// +// DigestEngine.cpp +// +// Library: Crypto +// Package: Digest +// Module: DigestEngine +// +// Copyright (c) 2012, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/DigestEngine.h" +#include "Poco/Exception.h" + + +namespace Poco { +namespace Crypto { + + +DigestEngine::DigestEngine(const std::string& name): + _name(name), + _pContext(EVP_MD_CTX_create()) +{ + const EVP_MD* md = EVP_get_digestbyname(_name.c_str()); + if (!md) throw Poco::NotFoundException(_name); + EVP_DigestInit_ex(_pContext, md, NULL); +} + + +DigestEngine::~DigestEngine() +{ + EVP_MD_CTX_destroy(_pContext); +} + +int DigestEngine::nid() const +{ + return EVP_MD_type(EVP_MD_CTX_md(_pContext)); +} + +std::size_t DigestEngine::digestLength() const +{ + return EVP_MD_CTX_size(_pContext); +} + + +void DigestEngine::reset() +{ +#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) + EVP_MD_CTX_free(_pContext); + _pContext = EVP_MD_CTX_create(); +#else + EVP_MD_CTX_cleanup(_pContext); +#endif + const EVP_MD* md = EVP_get_digestbyname(_name.c_str()); + if (!md) throw Poco::NotFoundException(_name); + EVP_DigestInit_ex(_pContext, md, NULL); +} + + +const Poco::DigestEngine::Digest& DigestEngine::digest() +{ + _digest.clear(); + unsigned len = EVP_MD_CTX_size(_pContext); + _digest.resize(len); + EVP_DigestFinal_ex(_pContext, &_digest[0], &len); + reset(); + return _digest; +} + + +void DigestEngine::updateImpl(const void* data, std::size_t length) +{ + EVP_DigestUpdate(_pContext, data, length); +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/ECDSADigestEngine.cpp b/base/poco/Crypto/src/ECDSADigestEngine.cpp new file mode 100644 index 00000000000..59512770cd1 --- /dev/null +++ b/base/poco/Crypto/src/ECDSADigestEngine.cpp @@ -0,0 +1,100 @@ +// +// ECDSADigestEngine.cpp +// +// +// Library: Crypto +// Package: ECDSA +// Module: ECDSADigestEngine +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/ECDSADigestEngine.h" +#include + + +namespace Poco { +namespace Crypto { + + +ECDSADigestEngine::ECDSADigestEngine(const ECKey& key, const std::string &name): + _key(key), + _engine(name) +{ +} + + +ECDSADigestEngine::~ECDSADigestEngine() +{ +} + + +std::size_t ECDSADigestEngine::digestLength() const +{ + return _engine.digestLength(); +} + + +void ECDSADigestEngine::reset() +{ + _engine.reset(); + _digest.clear(); + _signature.clear(); +} + + +const DigestEngine::Digest& ECDSADigestEngine::digest() +{ + if (_digest.empty()) + { + _digest = _engine.digest(); + } + return _digest; +} + + +const DigestEngine::Digest& ECDSADigestEngine::signature() +{ + if (_signature.empty()) + { + digest(); + _signature.resize(_key.size()); + unsigned sigLen = static_cast(_signature.size()); + if (!ECDSA_sign(0, &_digest[0], static_cast(_digest.size()), + &_signature[0], &sigLen, _key.impl()->getECKey())) + { + throw OpenSSLException(); + } + if (sigLen < _signature.size()) _signature.resize(sigLen); + } + return _signature; +} + + +bool ECDSADigestEngine::verify(const DigestEngine::Digest& sig) +{ + digest(); + EC_KEY* pKey = _key.impl()->getECKey(); + if (pKey) + { + int ret = ECDSA_verify(0, &_digest[0], static_cast(_digest.size()), + &sig[0], static_cast(sig.size()), + pKey); + if (1 == ret) return true; + else if (0 == ret) return false; + } + throw OpenSSLException(); +} + + +void ECDSADigestEngine::updateImpl(const void* data, std::size_t length) +{ + _engine.update(data, length); +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/ECKey.cpp b/base/poco/Crypto/src/ECKey.cpp new file mode 100644 index 00000000000..56b8663776e --- /dev/null +++ b/base/poco/Crypto/src/ECKey.cpp @@ -0,0 +1,75 @@ +// +// ECKey.cpp +// +// +// Library: Crypto +// Package: EC +// Module: ECKey +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/ECKey.h" +#include + + +namespace Poco { +namespace Crypto { + + +ECKey::ECKey(const EVPPKey& key): + KeyPair(new ECKeyImpl(key)), + _pImpl(KeyPair::impl().cast()) +{ +} + + +ECKey::ECKey(const X509Certificate& cert): + KeyPair(new ECKeyImpl(cert)), + _pImpl(KeyPair::impl().cast()) +{ +} + + +ECKey::ECKey(const PKCS12Container& cont): + KeyPair(new ECKeyImpl(cont)), + _pImpl(KeyPair::impl().cast()) +{ +} + + +ECKey::ECKey(const std::string& eccGroup): + KeyPair(new ECKeyImpl(OBJ_txt2nid(eccGroup.c_str()))), + _pImpl(KeyPair::impl().cast()) +{ +} + + +ECKey::ECKey(const std::string& publicKeyFile, + const std::string& privateKeyFile, + const std::string& privateKeyPassphrase): + KeyPair(new ECKeyImpl(publicKeyFile, privateKeyFile, privateKeyPassphrase)), + _pImpl(KeyPair::impl().cast()) +{ +} + + +ECKey::ECKey(std::istream* pPublicKeyStream, + std::istream* pPrivateKeyStream, + const std::string& privateKeyPassphrase): + KeyPair(new ECKeyImpl(pPublicKeyStream, pPrivateKeyStream, privateKeyPassphrase)), + _pImpl(KeyPair::impl().cast()) +{ +} + + +ECKey::~ECKey() +{ +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/ECKeyImpl.cpp b/base/poco/Crypto/src/ECKeyImpl.cpp new file mode 100644 index 00000000000..99ffae76067 --- /dev/null +++ b/base/poco/Crypto/src/ECKeyImpl.cpp @@ -0,0 +1,258 @@ +// +// ECKeyImpl.cpp +// +// +// Library: Crypto +// Package: EC +// Module: ECKeyImpl +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/ECKeyImpl.h" +#include "Poco/Crypto/X509Certificate.h" +#include "Poco/Crypto/PKCS12Container.h" +#include "Poco/FileStream.h" +#include "Poco/Format.h" +#include "Poco/StreamCopier.h" +#include +#include +#if OPENSSL_VERSION_NUMBER >= 0x00908000L +#include +#endif + + +namespace Poco { +namespace Crypto { + + +ECKeyImpl::ECKeyImpl(const EVPPKey& key): + KeyPairImpl("ec", KT_EC_IMPL), + _pEC(EVP_PKEY_get1_EC_KEY(const_cast((const EVP_PKEY*)key))) +{ + checkEC("ECKeyImpl(const EVPPKey&)", "EVP_PKEY_get1_EC_KEY()"); +} + + +ECKeyImpl::ECKeyImpl(const X509Certificate& cert): + KeyPairImpl("ec", KT_EC_IMPL), + _pEC(0) +{ + const X509* pCert = cert.certificate(); + if (pCert) + { + EVP_PKEY* pKey = X509_get_pubkey(const_cast(pCert)); + if (pKey) + { + _pEC = EVP_PKEY_get1_EC_KEY(pKey); + EVP_PKEY_free(pKey); + checkEC("ECKeyImpl(const const X509Certificate&)", "EVP_PKEY_get1_EC_KEY()"); + return; + } + } + throw OpenSSLException("ECKeyImpl(const X509Certificate&)"); +} + + +ECKeyImpl::ECKeyImpl(const PKCS12Container& cont): + KeyPairImpl("ec", KT_EC_IMPL), + _pEC(EVP_PKEY_get1_EC_KEY(cont.getKey())) +{ + checkEC("ECKeyImpl(const PKCS12Container&)", "EVP_PKEY_get1_EC_KEY()"); +} + + +ECKeyImpl::ECKeyImpl(int curve): + KeyPairImpl("ec", KT_EC_IMPL), + _pEC(EC_KEY_new_by_curve_name(curve)) +{ + poco_check_ptr(_pEC); + EC_KEY_set_asn1_flag(_pEC, OPENSSL_EC_NAMED_CURVE); + if (!(EC_KEY_generate_key(_pEC))) + throw OpenSSLException("ECKeyImpl(int curve): EC_KEY_generate_key()"); + checkEC("ECKeyImpl(int curve)", "EC_KEY_generate_key()"); +} + + +ECKeyImpl::ECKeyImpl(const std::string& publicKeyFile, + const std::string& privateKeyFile, + const std::string& privateKeyPassphrase): KeyPairImpl("ec", KT_EC_IMPL), _pEC(0) +{ + if (EVPPKey::loadKey(&_pEC, PEM_read_PrivateKey, EVP_PKEY_get1_EC_KEY, privateKeyFile, privateKeyPassphrase)) + { + checkEC(Poco::format("ECKeyImpl(%s, %s, %s)", + publicKeyFile, privateKeyFile, privateKeyPassphrase.empty() ? privateKeyPassphrase : std::string("***")), + "PEM_read_PrivateKey() or EVP_PKEY_get1_EC_KEY()"); + return; // private key is enough + } + + // no private key, this must be public key only, otherwise throw + if (!EVPPKey::loadKey(&_pEC, PEM_read_PUBKEY, EVP_PKEY_get1_EC_KEY, publicKeyFile)) + { + throw OpenSSLException("ECKeyImpl(const string&, const string&, const string&"); + } + checkEC(Poco::format("ECKeyImpl(%s, %s, %s)", + publicKeyFile, privateKeyFile, privateKeyPassphrase.empty() ? privateKeyPassphrase : std::string("***")), + "PEM_read_PUBKEY() or EVP_PKEY_get1_EC_KEY()"); +} + + +ECKeyImpl::ECKeyImpl(std::istream* pPublicKeyStream, + std::istream* pPrivateKeyStream, + const std::string& privateKeyPassphrase): KeyPairImpl("ec", KT_EC_IMPL), _pEC(0) +{ + if (EVPPKey::loadKey(&_pEC, PEM_read_bio_PrivateKey, EVP_PKEY_get1_EC_KEY, pPrivateKeyStream, privateKeyPassphrase)) + { + checkEC(Poco::format("ECKeyImpl(stream, stream, %s)", + privateKeyPassphrase.empty() ? privateKeyPassphrase : std::string("***")), + "PEM_read_bio_PrivateKey() or EVP_PKEY_get1_EC_KEY()"); + return; // private key is enough + } + + // no private key, this must be public key only, otherwise throw + if (!EVPPKey::loadKey(&_pEC, PEM_read_bio_PUBKEY, EVP_PKEY_get1_EC_KEY, pPublicKeyStream)) + { + throw OpenSSLException("ECKeyImpl(istream*, istream*, const string&"); + } + checkEC(Poco::format("ECKeyImpl(stream, stream, %s)", + privateKeyPassphrase.empty() ? privateKeyPassphrase : std::string("***")), + "PEM_read_bio_PUBKEY() or EVP_PKEY_get1_EC_KEY()"); +} + + +ECKeyImpl::~ECKeyImpl() +{ + freeEC(); +} + + +void ECKeyImpl::checkEC(const std::string& method, const std::string& func) const +{ + if (!_pEC) throw OpenSSLException(Poco::format("%s: %s", method, func)); + if (!EC_KEY_check_key(_pEC)) + throw OpenSSLException(Poco::format("%s: EC_KEY_check_key()", method)); +} + + +void ECKeyImpl::freeEC() +{ + if (_pEC) + { + EC_KEY_free(_pEC); + _pEC = 0; + } +} + + +int ECKeyImpl::size() const +{ + int sz = -1; + EVP_PKEY* pKey = EVP_PKEY_new(); + if (pKey && EVP_PKEY_set1_EC_KEY(pKey, _pEC)) + { + sz = EVP_PKEY_bits(pKey); + EVP_PKEY_free(pKey); + return sz; + } + throw OpenSSLException("ECKeyImpl::size()"); +} + + +int ECKeyImpl::groupId() const +{ + if (_pEC) + { + const EC_GROUP* ecGroup = EC_KEY_get0_group(_pEC); + if (ecGroup) + { + return EC_GROUP_get_curve_name(ecGroup); + } + else + { + throw OpenSSLException("ECKeyImpl::groupName()"); + } + } + throw NullPointerException("ECKeyImpl::groupName() => _pEC"); +} + + +std::string ECKeyImpl::getCurveName(int nid) +{ + std::string curveName; + size_t len = EC_get_builtin_curves(NULL, 0); + EC_builtin_curve* pCurves = + (EC_builtin_curve*) OPENSSL_malloc(sizeof(EC_builtin_curve) * len); + if (!pCurves) return curveName; + + if (!EC_get_builtin_curves(pCurves, len)) + { + OPENSSL_free(pCurves); + return curveName; + } + + if (-1 == nid) nid = pCurves[0].nid; + const int bufLen = 128; + char buf[bufLen]; + std::memset(buf, 0, bufLen); + OBJ_obj2txt(buf, bufLen, OBJ_nid2obj(nid), 0); + curveName = buf; + OPENSSL_free(pCurves); + return curveName; +} + + +int ECKeyImpl::getCurveNID(std::string& name) +{ + std::string curveName; + size_t len = EC_get_builtin_curves(NULL, 0); + EC_builtin_curve* pCurves = + (EC_builtin_curve*)OPENSSL_malloc(static_cast(sizeof(EC_builtin_curve) * len)); + if (!pCurves) return -1; + + if (!EC_get_builtin_curves(pCurves, len)) + { + OPENSSL_free(pCurves); + return -1; + } + + int nid = -1; + const int bufLen = 128; + char buf[bufLen]; + if (name.empty()) + { + std::memset(buf, 0, bufLen); + OBJ_obj2txt(buf, bufLen, OBJ_nid2obj(nid), 0); + name = buf; + nid = pCurves[0].nid; + } + else + { + for (int i = 0; i < len; ++i) + { + std::memset(buf, 0, bufLen); + OBJ_obj2txt(buf, bufLen, OBJ_nid2obj(pCurves[i].nid), 0); + if (strncmp(name.c_str(), buf, name.size() > bufLen ? bufLen : name.size()) == 0) + { + nid = pCurves[i].nid; + break; + } + } + } + + OPENSSL_free(pCurves); + return nid; +} + + +bool ECKeyImpl::hasCurve(const std::string& name) +{ + std::string tmp(name); + return (-1 != getCurveNID(tmp)); +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/EVPPKey.cpp b/base/poco/Crypto/src/EVPPKey.cpp new file mode 100644 index 00000000000..ffa0a78c070 --- /dev/null +++ b/base/poco/Crypto/src/EVPPKey.cpp @@ -0,0 +1,321 @@ +// +// EVPPKey.cpp +// +// +// Library: Crypto +// Package: CryptoCore +// Module: EVPPKey +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/EVPPKey.h" +#include "Poco/Crypto/ECKey.h" +#include "Poco/Crypto/RSAKey.h" +#include "Poco/NumberFormatter.h" + + +namespace Poco { +namespace Crypto { + + +EVPPKey::EVPPKey(const std::string& ecCurveName): _pEVPPKey(0) +{ + newECKey(ecCurveName.c_str()); + poco_check_ptr(_pEVPPKey); +} + + +EVPPKey::EVPPKey(const char* ecCurveName): _pEVPPKey(0) +{ + newECKey(ecCurveName); + poco_check_ptr(_pEVPPKey); +} + + +EVPPKey::EVPPKey(EVP_PKEY* pEVPPKey): _pEVPPKey(0) +{ + duplicate(pEVPPKey, &_pEVPPKey); + poco_check_ptr(_pEVPPKey); +} + + +EVPPKey::EVPPKey(const std::string& publicKeyFile, + const std::string& privateKeyFile, + const std::string& privateKeyPassphrase): _pEVPPKey(0) +{ + if (loadKey(&_pEVPPKey, PEM_read_PrivateKey, (EVP_PKEY_get_Key_fn)0, privateKeyFile, privateKeyPassphrase)) + { + poco_check_ptr(_pEVPPKey); + return; // private key is enough + } + + // no private key, this must be public key only, otherwise throw + if (!loadKey(&_pEVPPKey, PEM_read_PUBKEY, (EVP_PKEY_get_Key_fn)0, publicKeyFile)) + { + throw OpenSSLException("ECKeyImpl(const string&, const string&, const string&"); + } + poco_check_ptr(_pEVPPKey); +} + + +EVPPKey::EVPPKey(std::istream* pPublicKeyStream, + std::istream* pPrivateKeyStream, + const std::string& privateKeyPassphrase): _pEVPPKey(0) +{ + if (loadKey(&_pEVPPKey, PEM_read_bio_PrivateKey, (EVP_PKEY_get_Key_fn)0, pPrivateKeyStream, privateKeyPassphrase)) + { + poco_check_ptr(_pEVPPKey); + return; // private key is enough + } + + // no private key, this must be public key only, otherwise throw + if (!loadKey(&_pEVPPKey, PEM_read_bio_PUBKEY, (EVP_PKEY_get_Key_fn)0, pPublicKeyStream)) + { + throw OpenSSLException("ECKeyImpl(istream*, istream*, const string&"); + } + poco_check_ptr(_pEVPPKey); +} + + +EVPPKey::EVPPKey(const EVPPKey& other) +{ + duplicate(other._pEVPPKey, &_pEVPPKey); + poco_check_ptr(_pEVPPKey); +} + + +EVPPKey& EVPPKey::operator=(const EVPPKey& other) +{ + duplicate(other._pEVPPKey, &_pEVPPKey); + poco_check_ptr(_pEVPPKey); + return *this; +} + + +#ifdef POCO_ENABLE_CPP11 + +EVPPKey::EVPPKey(EVPPKey&& other): _pEVPPKey(other._pEVPPKey) +{ + other._pEVPPKey = nullptr; + poco_check_ptr(_pEVPPKey); +} + + +EVPPKey& EVPPKey::operator=(EVPPKey&& other) +{ + _pEVPPKey = other._pEVPPKey; + other._pEVPPKey = nullptr; + poco_check_ptr(_pEVPPKey); + return *this; +} + +#endif // POCO_ENABLE_CPP11 + +EVPPKey::~EVPPKey() +{ + if (_pEVPPKey) EVP_PKEY_free(_pEVPPKey); +} + + +void EVPPKey::save(const std::string& publicKeyFile, const std::string& privateKeyFile, const std::string& privateKeyPassphrase) const +{ + if (!publicKeyFile.empty() && (publicKeyFile != privateKeyFile)) + { + BIO* bio = BIO_new(BIO_s_file()); + if (!bio) throw Poco::IOException("Cannot create BIO for writing public key file", publicKeyFile); + try + { + if (BIO_write_filename(bio, const_cast(publicKeyFile.c_str()))) + { + if (!PEM_write_bio_PUBKEY(bio, _pEVPPKey)) + { + throw Poco::WriteFileException("Failed to write public key to file", publicKeyFile); + } + } + else throw Poco::CreateFileException("Cannot create public key file"); + } + catch (...) + { + BIO_free(bio); + throw; + } + BIO_free(bio); + } + + if (!privateKeyFile.empty()) + { + BIO* bio = BIO_new(BIO_s_file()); + if (!bio) throw Poco::IOException("Cannot create BIO for writing private key file", privateKeyFile); + try + { + if (BIO_write_filename(bio, const_cast(privateKeyFile.c_str()))) + { + int rc = 0; + if (privateKeyPassphrase.empty()) + { + rc = PEM_write_bio_PrivateKey(bio, _pEVPPKey, 0, 0, 0, 0, 0); + } + else + { + rc = PEM_write_bio_PrivateKey(bio, _pEVPPKey, EVP_des_ede3_cbc(), + reinterpret_cast(const_cast(privateKeyPassphrase.c_str())), + static_cast(privateKeyPassphrase.length()), 0, 0); + } + if (!rc) + throw Poco::FileException("Failed to write private key to file", privateKeyFile); + } + else throw Poco::CreateFileException("Cannot create private key file", privateKeyFile); + } + catch (...) + { + BIO_free(bio); + throw; + } + BIO_free(bio); + } +} + + +void EVPPKey::save(std::ostream* pPublicKeyStream, std::ostream* pPrivateKeyStream, const std::string& privateKeyPassphrase) const +{ + if (pPublicKeyStream && (pPublicKeyStream != pPrivateKeyStream)) + { + BIO* bio = BIO_new(BIO_s_mem()); + if (!bio) throw Poco::IOException("Cannot create BIO for writing public key"); + if (!PEM_write_bio_PUBKEY(bio, _pEVPPKey)) + { + BIO_free(bio); + throw Poco::WriteFileException("Failed to write public key to stream"); + } + char* pData; + long size = BIO_get_mem_data(bio, &pData); + pPublicKeyStream->write(pData, static_cast(size)); + BIO_free(bio); + } + + if (pPrivateKeyStream) + { + BIO* bio = BIO_new(BIO_s_mem()); + if (!bio) throw Poco::IOException("Cannot create BIO for writing public key"); + int rc = 0; + if (privateKeyPassphrase.empty()) + rc = PEM_write_bio_PrivateKey(bio, _pEVPPKey, 0, 0, 0, 0, 0); + else + rc = PEM_write_bio_PrivateKey(bio, _pEVPPKey, EVP_des_ede3_cbc(), + reinterpret_cast(const_cast(privateKeyPassphrase.c_str())), + static_cast(privateKeyPassphrase.length()), 0, 0); + if (!rc) + { + BIO_free(bio); + throw Poco::FileException("Failed to write private key to stream"); + } + char* pData; + long size = BIO_get_mem_data(bio, &pData); + pPrivateKeyStream->write(pData, static_cast(size)); + BIO_free(bio); + } +} + + +EVP_PKEY* EVPPKey::duplicate(const EVP_PKEY* pFromKey, EVP_PKEY** pToKey) +{ + if (!pFromKey) throw NullPointerException("EVPPKey::duplicate(): " + "provided key pointer is null."); + + *pToKey = EVP_PKEY_new(); + if (!*pToKey) throw NullPointerException("EVPPKey::duplicate(): " + "EVP_PKEY_new() returned null."); + + int keyType = type(pFromKey); + switch (keyType) + { + case EVP_PKEY_RSA: + { + RSA* pRSA = EVP_PKEY_get1_RSA(const_cast(pFromKey)); + if (pRSA) + { + EVP_PKEY_set1_RSA(*pToKey, pRSA); + RSA_free(pRSA); + } + else throw OpenSSLException("EVPPKey::duplicate(): EVP_PKEY_get1_RSA()"); + break; + } + case EVP_PKEY_EC: + { + EC_KEY* pEC = EVP_PKEY_get1_EC_KEY(const_cast(pFromKey)); + if (pEC) + { + EVP_PKEY_set1_EC_KEY(*pToKey, pEC); + EC_KEY_free(pEC); + int cmp = EVP_PKEY_cmp_parameters(*pToKey, pFromKey); + if (cmp < 0) + throw OpenSSLException("EVPPKey::duplicate(): EVP_PKEY_cmp_parameters()"); + if (0 == cmp) + { + if(!EVP_PKEY_copy_parameters(*pToKey, pFromKey)) + throw OpenSSLException("EVPPKey::duplicate(): EVP_PKEY_copy_parameters()"); + } + } + else throw OpenSSLException(); + break; + } + default: + throw NotImplementedException("EVPPKey:duplicate(); Key type: " + + NumberFormatter::format(keyType)); + } + + return *pToKey; +} + + +void EVPPKey::newECKey(const char* ecCurveName) +{ + int curveID = OBJ_txt2nid(ecCurveName); + EC_KEY* pEC = EC_KEY_new_by_curve_name(curveID); + if (!pEC) goto err; + if (!EC_KEY_generate_key(pEC)) goto err; + _pEVPPKey = EVP_PKEY_new(); + if (!_pEVPPKey) goto err; + if (!EVP_PKEY_set1_EC_KEY(_pEVPPKey, pEC)) goto err; + EC_KEY_free(pEC); + return; +err: + throw OpenSSLException("EVPPKey:newECKey()"); +} + + +void EVPPKey::setKey(ECKey* pKey) +{ + poco_check_ptr(pKey); + poco_check_ptr(pKey->impl()); + setKey(pKey->impl()->getECKey()); +} + + +void EVPPKey::setKey(RSAKey* pKey) +{ + poco_check_ptr(pKey); + poco_check_ptr(pKey->impl()); + setKey(pKey->impl()->getRSA()); +} + + +int EVPPKey::passCB(char* buf, int size, int, void* pass) +{ + if (pass) + { + int len = (int)std::strlen((char*)pass); + if(len > size) len = size; + std::memcpy(buf, pass, len); + return len; + } + return 0; +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/KeyPair.cpp b/base/poco/Crypto/src/KeyPair.cpp new file mode 100644 index 00000000000..1c650806b37 --- /dev/null +++ b/base/poco/Crypto/src/KeyPair.cpp @@ -0,0 +1,34 @@ +// +// KeyPair.cpp +// +// +// Library: Crypto +// Package: CryptoCore +// Module: KeyPair +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/KeyPair.h" +#include + + +namespace Poco { +namespace Crypto { + + +KeyPair::KeyPair(KeyPairImpl::Ptr pKeyPairImpl): _pImpl(pKeyPairImpl) +{ +} + + +KeyPair::~KeyPair() +{ +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/KeyPairImpl.cpp b/base/poco/Crypto/src/KeyPairImpl.cpp new file mode 100644 index 00000000000..c782f04270d --- /dev/null +++ b/base/poco/Crypto/src/KeyPairImpl.cpp @@ -0,0 +1,35 @@ +// +// KeyPairImpl.cpp +// +// +// Library: Crypto +// Package: CryptoCore +// Module: KeyPairImpl +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/KeyPairImpl.h" + + +namespace Poco { +namespace Crypto { + + +KeyPairImpl::KeyPairImpl(const std::string& name, Type type): + _name(name), + _type(type) +{ +} + + +KeyPairImpl::~KeyPairImpl() +{ +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/OpenSSLInitializer.cpp b/base/poco/Crypto/src/OpenSSLInitializer.cpp new file mode 100644 index 00000000000..a3bc1d01431 --- /dev/null +++ b/base/poco/Crypto/src/OpenSSLInitializer.cpp @@ -0,0 +1,190 @@ +// +// OpenSSLInitializer.cpp +// +// Library: Crypto +// Package: CryptoCore +// Module: OpenSSLInitializer +// +// Copyright (c) 2006-2009, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/OpenSSLInitializer.h" +#include "Poco/RandomStream.h" +#include "Poco/Thread.h" +#include +#include +#include +#include +#if OPENSSL_VERSION_NUMBER >= 0x0907000L +#include +#endif +#if defined(POCO_OS_FAMILY_WINDOWS) + #define POCO_STR_HELPER(x) #x + #define POCO_STR(x) POCO_STR_HELPER(x) + #if defined POCO_INTERNAL_OPENSSL_MSVC_VER + #define POCO_INTERNAL_OPENSSL_BUILD \ + " (POCO internal build, MSVC version " \ + POCO_STR(POCO_INTERNAL_OPENSSL_MSVC_VER) ")" + #else + #define POCO_INTERNAL_OPENSSL_BUILD "" + #endif + #pragma message (OPENSSL_VERSION_TEXT POCO_INTERNAL_OPENSSL_BUILD) +#endif + + +using Poco::RandomInputStream; +using Poco::Thread; + + +#if defined(_MSC_VER) && !defined(_DLL) && defined(POCO_INTERNAL_OPENSSL_MSVC_VER) + + #if (POCO_MSVS_VERSION >= 2015) + FILE _iob[] = { *stdin, *stdout, *stderr }; + extern "C" FILE * __cdecl __iob_func(void) { return _iob; } + #endif // (POCO_MSVS_VERSION >= 2015) + + #if (POCO_MSVS_VERSION < 2012) + extern "C" __declspec(noreturn) void __cdecl __report_rangecheckfailure(void) { ::ExitProcess(1); } + #endif // (POCO_MSVS_VERSION < 2012) + +#endif // _MSC_VER && _MT && !POCO_EXTERNAL_OPENSSL && (POCO_MSVS_VERSION < 2013) + + +namespace Poco { +namespace Crypto { + + +Poco::FastMutex* OpenSSLInitializer::_mutexes(0); +Poco::AtomicCounter OpenSSLInitializer::_rc; + + +OpenSSLInitializer::OpenSSLInitializer() +{ + initialize(); +} + + +OpenSSLInitializer::~OpenSSLInitializer() +{ + try + { + uninitialize(); + } + catch (...) + { + poco_unexpected(); + } +} + + +void OpenSSLInitializer::initialize() +{ + if (++_rc == 1) + { +#if OPENSSL_VERSION_NUMBER >= 0x0907000L + OPENSSL_config(NULL); +#endif + SSL_library_init(); + SSL_load_error_strings(); + OpenSSL_add_all_algorithms(); + + char seed[SEEDSIZE]; + RandomInputStream rnd; + rnd.read(seed, sizeof(seed)); + RAND_seed(seed, SEEDSIZE); + + int nMutexes = CRYPTO_num_locks(); + _mutexes = new Poco::FastMutex[nMutexes]; + CRYPTO_set_locking_callback(&OpenSSLInitializer::lock); +#ifndef POCO_OS_FAMILY_WINDOWS +// Not needed on Windows (see SF #110: random unhandled exceptions when linking with ssl). +// https://sourceforge.net/p/poco/bugs/110/ +// +// From http://www.openssl.org/docs/crypto/threads.html : +// "If the application does not register such a callback using CRYPTO_THREADID_set_callback(), +// then a default implementation is used - on Windows and BeOS this uses the system's +// default thread identifying APIs" + CRYPTO_set_id_callback(&OpenSSLInitializer::id); +#endif + CRYPTO_set_dynlock_create_callback(&OpenSSLInitializer::dynlockCreate); + CRYPTO_set_dynlock_lock_callback(&OpenSSLInitializer::dynlock); + CRYPTO_set_dynlock_destroy_callback(&OpenSSLInitializer::dynlockDestroy); + } +} + + +void OpenSSLInitializer::uninitialize() +{ + if (--_rc == 0) + { + EVP_cleanup(); + ERR_free_strings(); + CRYPTO_set_locking_callback(0); +#ifndef POCO_OS_FAMILY_WINDOWS + CRYPTO_set_id_callback(0); +#endif + delete [] _mutexes; + + CONF_modules_free(); + } +} + + +void OpenSSLInitializer::lock(int mode, int n, const char* file, int line) +{ + if (mode & CRYPTO_LOCK) + _mutexes[n].lock(); + else + _mutexes[n].unlock(); +} + + +unsigned long OpenSSLInitializer::id() +{ + // Note: we use an old-style C cast here because + // neither static_cast<> nor reinterpret_cast<> + // work uniformly across all platforms. + return (unsigned long) Poco::Thread::currentTid(); +} + + +struct CRYPTO_dynlock_value* OpenSSLInitializer::dynlockCreate(const char* file, int line) +{ + return new CRYPTO_dynlock_value; +} + + +void OpenSSLInitializer::dynlock(int mode, struct CRYPTO_dynlock_value* lock, const char* file, int line) +{ + poco_check_ptr (lock); + + if (mode & CRYPTO_LOCK) + lock->_mutex.lock(); + else + lock->_mutex.unlock(); +} + + +void OpenSSLInitializer::dynlockDestroy(struct CRYPTO_dynlock_value* lock, const char* file, int line) +{ + delete lock; +} + + +void initializeCrypto() +{ + OpenSSLInitializer::initialize(); +} + + +void uninitializeCrypto() +{ + OpenSSLInitializer::uninitialize(); +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/PKCS12Container.cpp b/base/poco/Crypto/src/PKCS12Container.cpp new file mode 100644 index 00000000000..def89bf0898 --- /dev/null +++ b/base/poco/Crypto/src/PKCS12Container.cpp @@ -0,0 +1,191 @@ +// +// PKCS12Container.cpp +// +// +// Library: Crypto +// Package: Certificate +// Module: PKCS12Container +// +// Copyright (c) 2006-2009, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/PKCS12Container.h" +#include "Poco/NumberFormatter.h" +#include "Poco/StreamCopier.h" +#include +#include + + +namespace Poco { +namespace Crypto { + + +PKCS12Container::PKCS12Container(std::istream& istr, const std::string& password): _pKey(0) +{ + std::ostringstream ostr; + Poco::StreamCopier::copyStream(istr, ostr); + const std::string& cont = ostr.str(); + + BIO *pBIO = BIO_new_mem_buf(const_cast(cont.data()), static_cast(cont.size())); + if (pBIO) + { + PKCS12* pPKCS12 = 0; + d2i_PKCS12_bio(pBIO, &pPKCS12); + BIO_free(pBIO); + if (!pPKCS12) throw OpenSSLException("PKCS12Container(istream&, const string&)"); + load(pPKCS12, password); + } + else + { + throw Poco::NullPointerException("PKCS12Container(istream&, const string&)"); + } +} + + +PKCS12Container::PKCS12Container(const std::string& path, const std::string& password): _pKey(0) +{ + FILE* pFile = fopen(path.c_str(), "rb"); + if (pFile) + { + PKCS12* pPKCS12 = d2i_PKCS12_fp(pFile, NULL); + fclose (pFile); + if (!pPKCS12) throw OpenSSLException("PKCS12Container(const string&, const string&)"); + load(pPKCS12, password); + } + else + { + throw Poco::OpenFileException("PKCS12Container: " + path); + } +} + + +PKCS12Container::PKCS12Container(const PKCS12Container& other): + _pKey(EVPPKey::duplicate(other._pKey, &_pKey)), + _pX509Cert(new X509Certificate(*other._pX509Cert)), + _caCertList(other._caCertList), + _caCertNames(other._caCertNames), + _pkcsFriendlyName(other._pkcsFriendlyName) +{ +} + + +PKCS12Container& PKCS12Container::operator = (const PKCS12Container& other) +{ + if (&other != this) + { + if (_pKey) EVP_PKEY_free(_pKey); + _pKey = EVPPKey::duplicate(other._pKey, &_pKey); + _pX509Cert.reset(new X509Certificate(*other._pX509Cert)); + _caCertList = other._caCertList; + _caCertNames = other._caCertNames; + _pkcsFriendlyName = other._pkcsFriendlyName; + } + return *this; +} + + +#ifdef POCO_ENABLE_CPP11 + + +PKCS12Container::PKCS12Container(PKCS12Container&& other): + _pKey(other._pKey), + _pX509Cert(std::move(other._pX509Cert)), + _caCertList(std::move(other._caCertList)), + _caCertNames(std::move(other._caCertNames)), + _pkcsFriendlyName(std::move(other._pkcsFriendlyName)) +{ + other._pKey = 0; +} + + +PKCS12Container& PKCS12Container::operator = (PKCS12Container&& other) +{ + if (&other != this) + { + if (_pKey) EVP_PKEY_free(_pKey); + _pKey = other._pKey; other._pKey = 0; + _pX509Cert = std::move(other._pX509Cert); + _caCertList = std::move(other._caCertList); + _caCertNames = std::move(other._caCertNames); + _pkcsFriendlyName = std::move(other._pkcsFriendlyName); + } + return *this; +} + + +#endif // POCO_ENABLE_CPP11 + + +PKCS12Container::~PKCS12Container() +{ + if (_pKey) EVP_PKEY_free(_pKey); +} + + +std::string PKCS12Container::extractFriendlyName(X509* pCert) +{ + std::string friendlyName; + if(!pCert) throw NullPointerException("PKCS12Container::extractFriendlyName(X509)"); + + // This is how PKCS12_add_cert() sets friendlyName (via PKCS12_add_friendlyname()) + int namelen = 0; + char *name = (char *)X509_alias_get0(pCert, &namelen); + + friendlyName = std::string(name, namelen); + return friendlyName; +} + + +void PKCS12Container::load(PKCS12* pPKCS12, const std::string& password) +{ + if (pPKCS12) + { + X509* pCert = 0; + STACK_OF(X509)* pCA = 0; + if (PKCS12_parse(pPKCS12, password.c_str(), &_pKey, &pCert, &pCA)) + { + if (pCert) + { + _pX509Cert.reset(new X509Certificate(pCert, true)); + _pkcsFriendlyName = extractFriendlyName(pCert); + } + else _pX509Cert.reset(); + + _caCertList.clear(); + _caCertNames.clear(); + if (pCA) + { + int certCount = sk_X509_num(pCA); + for (int i = 0; i < certCount; ++i) + { + X509* pX509 = sk_X509_value(pCA, i); + if (pX509) + { + _caCertList.push_back(X509Certificate(pX509, true)); + _caCertNames.push_back(extractFriendlyName(pX509)); + } + else throw OpenSSLException("PKCS12Container::load()"); + } + } + } + else + { + throw OpenSSLException(); + } + PKCS12_free(pPKCS12); + sk_X509_pop_free(pCA, X509_free); + if (pCert) X509_free(pCert); + poco_assert_dbg (_caCertList.size() == _caCertNames.size()); + } + else + { + throw NullPointerException("PKCS12Container: struct PKCS12"); + } +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/RSACipherImpl.cpp b/base/poco/Crypto/src/RSACipherImpl.cpp new file mode 100644 index 00000000000..4f9ddcf32ad --- /dev/null +++ b/base/poco/Crypto/src/RSACipherImpl.cpp @@ -0,0 +1,342 @@ +// +// RSACipherImpl.cpp +// +// Library: Crypto +// Package: RSA +// Module: RSACipherImpl +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/RSACipherImpl.h" +#include "Poco/Crypto/CryptoTransform.h" +#include "Poco/Exception.h" +#include +#include +#include + + +namespace Poco { +namespace Crypto { + + +namespace +{ + void throwError() + { + unsigned long err; + std::string msg; + + while ((err = ERR_get_error())) + { + if (!msg.empty()) + msg.append("; "); + msg.append(ERR_error_string(err, 0)); + } + + throw Poco::IOException(msg); + } + + + int mapPaddingMode(RSAPaddingMode paddingMode) + { + switch (paddingMode) + { + case RSA_PADDING_PKCS1: + return RSA_PKCS1_PADDING; + case RSA_PADDING_PKCS1_OAEP: + return RSA_PKCS1_OAEP_PADDING; + case RSA_PADDING_NONE: + return RSA_NO_PADDING; + default: + poco_bugcheck(); + return RSA_NO_PADDING; + } + } + + + class RSAEncryptImpl: public CryptoTransform + { + public: + RSAEncryptImpl(const RSA* pRSA, RSAPaddingMode paddingMode); + ~RSAEncryptImpl(); + + std::size_t blockSize() const; + std::size_t maxDataSize() const; + std::string getTag(std::size_t); + void setTag(const std::string&); + + std::streamsize transform( + const unsigned char* input, + std::streamsize inputLength, + unsigned char* output, + std::streamsize outputLength); + + std::streamsize finalize(unsigned char* output, std::streamsize length); + + private: + const RSA* _pRSA; + RSAPaddingMode _paddingMode; + std::streamsize _pos; + unsigned char* _pBuf; + }; + + + RSAEncryptImpl::RSAEncryptImpl(const RSA* pRSA, RSAPaddingMode paddingMode): + _pRSA(pRSA), + _paddingMode(paddingMode), + _pos(0), + _pBuf(0) + { + _pBuf = new unsigned char[blockSize()]; + } + + + RSAEncryptImpl::~RSAEncryptImpl() + { + delete [] _pBuf; + } + + + std::size_t RSAEncryptImpl::blockSize() const + { + return RSA_size(_pRSA); + } + + + std::size_t RSAEncryptImpl::maxDataSize() const + { + std::size_t size = blockSize(); + switch (_paddingMode) + { + case RSA_PADDING_PKCS1: + case RSA_PADDING_SSLV23: + size -= 11; + break; + case RSA_PADDING_PKCS1_OAEP: + size -= 41; + break; + default: + break; + } + return size; + } + + + std::string RSAEncryptImpl::getTag(std::size_t) + { + return std::string(); + } + + + void RSAEncryptImpl::setTag(const std::string&) + { + } + + + std::streamsize RSAEncryptImpl::transform( + const unsigned char* input, + std::streamsize inputLength, + unsigned char* output, + std::streamsize outputLength) + { + // always fill up the buffer before writing! + std::streamsize maxSize = static_cast(maxDataSize()); + std::streamsize rsaSize = static_cast(blockSize()); + poco_assert_dbg(_pos <= maxSize); + poco_assert (outputLength >= rsaSize); + int rc = 0; + while (inputLength > 0) + { + // check how many data bytes we are missing to get the buffer full + poco_assert_dbg (maxSize >= _pos); + std::streamsize missing = maxSize - _pos; + if (missing == 0) + { + poco_assert (outputLength >= rsaSize); + int n = RSA_public_encrypt(static_cast(maxSize), _pBuf, output, const_cast(_pRSA), mapPaddingMode(_paddingMode)); + if (n == -1) + throwError(); + rc += n; + output += n; + outputLength -= n; + _pos = 0; + + } + else + { + if (missing > inputLength) + missing = inputLength; + + std::memcpy(_pBuf + _pos, input, static_cast(missing)); + input += missing; + _pos += missing; + inputLength -= missing; + } + } + return rc; + } + + + std::streamsize RSAEncryptImpl::finalize(unsigned char* output, std::streamsize length) + { + poco_assert (length >= blockSize()); + poco_assert (_pos <= maxDataSize()); + int rc = 0; + if (_pos > 0) + { + rc = RSA_public_encrypt(static_cast(_pos), _pBuf, output, const_cast(_pRSA), mapPaddingMode(_paddingMode)); + if (rc == -1) throwError(); + } + return rc; + } + + + class RSADecryptImpl: public CryptoTransform + { + public: + RSADecryptImpl(const RSA* pRSA, RSAPaddingMode paddingMode); + ~RSADecryptImpl(); + + std::size_t blockSize() const; + std::string getTag(std::size_t); + void setTag(const std::string&); + + std::streamsize transform( + const unsigned char* input, + std::streamsize inputLength, + unsigned char* output, + std::streamsize outputLength); + + std::streamsize finalize( + unsigned char* output, + std::streamsize length); + + private: + const RSA* _pRSA; + RSAPaddingMode _paddingMode; + std::streamsize _pos; + unsigned char* _pBuf; + }; + + + RSADecryptImpl::RSADecryptImpl(const RSA* pRSA, RSAPaddingMode paddingMode): + _pRSA(pRSA), + _paddingMode(paddingMode), + _pos(0), + _pBuf(0) + { + _pBuf = new unsigned char[blockSize()]; + } + + + RSADecryptImpl::~RSADecryptImpl() + { + delete [] _pBuf; + } + + + std::size_t RSADecryptImpl::blockSize() const + { + return RSA_size(_pRSA); + } + + + std::string RSADecryptImpl::getTag(std::size_t) + { + return std::string(); + } + + + void RSADecryptImpl::setTag(const std::string&) + { + } + + + std::streamsize RSADecryptImpl::transform( + const unsigned char* input, + std::streamsize inputLength, + unsigned char* output, + std::streamsize outputLength) + { + + // always fill up the buffer before decrypting! + std::streamsize rsaSize = static_cast(blockSize()); + poco_assert_dbg(_pos <= rsaSize); + poco_assert (outputLength >= rsaSize); + int rc = 0; + while (inputLength > 0) + { + // check how many data bytes we are missing to get the buffer full + poco_assert_dbg (rsaSize >= _pos); + std::streamsize missing = rsaSize - _pos; + if (missing == 0) + { + int tmp = RSA_private_decrypt(static_cast(rsaSize), _pBuf, output, const_cast(_pRSA), mapPaddingMode(_paddingMode)); + if (tmp == -1) + throwError(); + rc += tmp; + output += tmp; + outputLength -= tmp; + _pos = 0; + + } + else + { + if (missing > inputLength) + missing = inputLength; + + std::memcpy(_pBuf + _pos, input, static_cast(missing)); + input += missing; + _pos += missing; + inputLength -= missing; + } + } + return rc; + } + + + std::streamsize RSADecryptImpl::finalize(unsigned char* output, std::streamsize length) + { + poco_assert (length >= blockSize()); + int rc = 0; + if (_pos > 0) + { + rc = RSA_private_decrypt(static_cast(_pos), _pBuf, output, const_cast(_pRSA), mapPaddingMode(_paddingMode)); + if (rc == -1) + throwError(); + } + return rc; + } +} + + +RSACipherImpl::RSACipherImpl(const RSAKey& key, RSAPaddingMode paddingMode): + _key(key), + _paddingMode(paddingMode) +{ +} + + +RSACipherImpl::~RSACipherImpl() +{ +} + + +CryptoTransform* RSACipherImpl::createEncryptor() +{ + return new RSAEncryptImpl(_key.impl()->getRSA(), _paddingMode); +} + + +CryptoTransform* RSACipherImpl::createDecryptor() +{ + return new RSADecryptImpl(_key.impl()->getRSA(), _paddingMode); +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/RSADigestEngine.cpp b/base/poco/Crypto/src/RSADigestEngine.cpp new file mode 100644 index 00000000000..948aa25ac6c --- /dev/null +++ b/base/poco/Crypto/src/RSADigestEngine.cpp @@ -0,0 +1,96 @@ +// +// RSADigestEngine.cpp +// +// Library: Crypto +// Package: RSA +// Module: RSADigestEngine +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/RSADigestEngine.h" +#include + + +namespace Poco { +namespace Crypto { + + +RSADigestEngine::RSADigestEngine(const RSAKey& key, DigestType digestType): + _key(key), + _engine(digestType == DIGEST_MD5 ? "MD5" : "SHA1") +{ +} + +RSADigestEngine::RSADigestEngine(const RSAKey& key, const std::string &name): + _key(key), + _engine(name) +{ +} + + +RSADigestEngine::~RSADigestEngine() +{ +} + + +std::size_t RSADigestEngine::digestLength() const +{ + return _engine.digestLength(); +} + + +void RSADigestEngine::reset() +{ + _engine.reset(); + _digest.clear(); + _signature.clear(); +} + + +const DigestEngine::Digest& RSADigestEngine::digest() +{ + if (_digest.empty()) + { + _digest = _engine.digest(); + } + return _digest; +} + + +const DigestEngine::Digest& RSADigestEngine::signature() +{ + if (_signature.empty()) + { + digest(); + _signature.resize(_key.size()); + unsigned sigLen = static_cast(_signature.size()); + RSA_sign(_engine.nid(), &_digest[0], static_cast(_digest.size()), &_signature[0], &sigLen, _key.impl()->getRSA()); + // truncate _sig to sigLen + if (sigLen < _signature.size()) + _signature.resize(sigLen); + } + return _signature; +} + + +bool RSADigestEngine::verify(const DigestEngine::Digest& sig) +{ + digest(); + DigestEngine::Digest sigCpy = sig; // copy becausse RSA_verify can modify sigCpy + int ret = RSA_verify(_engine.nid(), &_digest[0], static_cast(_digest.size()), &sigCpy[0], static_cast(sigCpy.size()), _key.impl()->getRSA()); + return ret != 0; +} + + +void RSADigestEngine::updateImpl(const void* data, std::size_t length) +{ + _engine.update(data, length); +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/RSAKey.cpp b/base/poco/Crypto/src/RSAKey.cpp new file mode 100644 index 00000000000..b81a0281bf5 --- /dev/null +++ b/base/poco/Crypto/src/RSAKey.cpp @@ -0,0 +1,87 @@ +// +// RSAKey.cpp +// +// Library: Crypto +// Package: RSA +// Module: RSAKey +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/RSAKey.h" +#include + + +namespace Poco { +namespace Crypto { + + +RSAKey::RSAKey(const EVPPKey& key): + KeyPair(new RSAKeyImpl(key)), + _pImpl(KeyPair::impl().cast()) +{ +} + + +RSAKey::RSAKey(const X509Certificate& cert): + KeyPair(new RSAKeyImpl(cert)), + _pImpl(KeyPair::impl().cast()) +{ +} + + +RSAKey::RSAKey(const PKCS12Container& cont): + KeyPair(new RSAKeyImpl(cont)), + _pImpl(KeyPair::impl().cast()) +{ +} + + +RSAKey::RSAKey(KeyLength keyLength, Exponent exp): + KeyPair(new RSAKeyImpl(keyLength, (exp == EXP_LARGE) ? RSA_F4 : RSA_3)), + _pImpl(KeyPair::impl().cast()) +{ +} + + +RSAKey::RSAKey(const std::string& publicKeyFile, const std::string& privateKeyFile, const std::string& privateKeyPassphrase): + KeyPair(new RSAKeyImpl(publicKeyFile, privateKeyFile, privateKeyPassphrase)), + _pImpl(KeyPair::impl().cast()) +{ +} + + +RSAKey::RSAKey(std::istream* pPublicKeyStream, std::istream* pPrivateKeyStream, const std::string& privateKeyPassphrase): + KeyPair(new RSAKeyImpl(pPublicKeyStream, pPrivateKeyStream, privateKeyPassphrase)), + _pImpl(KeyPair::impl().cast()) +{ +} + + +RSAKey::~RSAKey() +{ +} + +RSAKeyImpl::ByteVec RSAKey::modulus() const +{ + return _pImpl->modulus(); +} + + +RSAKeyImpl::ByteVec RSAKey::encryptionExponent() const +{ + return _pImpl->encryptionExponent(); +} + + +RSAKeyImpl::ByteVec RSAKey::decryptionExponent() const +{ + return _pImpl->decryptionExponent(); +} + + +} } // namespace Poco::Crypto \ No newline at end of file diff --git a/base/poco/Crypto/src/RSAKeyImpl.cpp b/base/poco/Crypto/src/RSAKeyImpl.cpp new file mode 100644 index 00000000000..86089b828b1 --- /dev/null +++ b/base/poco/Crypto/src/RSAKeyImpl.cpp @@ -0,0 +1,386 @@ +// +// RSAKeyImpl.cpp +// +// Library: Crypto +// Package: RSA +// Module: RSAKeyImpl +// +// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/RSAKeyImpl.h" +#include "Poco/Crypto/X509Certificate.h" +#include "Poco/Crypto/PKCS12Container.h" +#include "Poco/FileStream.h" +#include "Poco/StreamCopier.h" +#include +#include +#include +#include +#if OPENSSL_VERSION_NUMBER >= 0x00908000L +#include +#endif + + +namespace Poco { +namespace Crypto { + + +RSAKeyImpl::RSAKeyImpl(const EVPPKey& key): + KeyPairImpl("rsa", KT_RSA_IMPL), + _pRSA(EVP_PKEY_get1_RSA(const_cast((const EVP_PKEY*)key))) +{ + if (!_pRSA) throw OpenSSLException(); +} + + +RSAKeyImpl::RSAKeyImpl(const X509Certificate& cert): + KeyPairImpl("rsa", KT_RSA_IMPL), + _pRSA(0) +{ + const X509* pCert = cert.certificate(); + EVP_PKEY* pKey = X509_get_pubkey(const_cast(pCert)); + if (pKey) + { + _pRSA = EVP_PKEY_get1_RSA(pKey); + EVP_PKEY_free(pKey); + } + else + throw OpenSSLException("RSAKeyImpl(const X509Certificate&)"); +} + + +RSAKeyImpl::RSAKeyImpl(const PKCS12Container& cont): + KeyPairImpl("rsa", KT_RSA_IMPL), + _pRSA(0) +{ + EVPPKey key = cont.getKey(); + _pRSA = EVP_PKEY_get1_RSA(key); +} + + +RSAKeyImpl::RSAKeyImpl(int keyLength, unsigned long exponent): KeyPairImpl("rsa", KT_RSA_IMPL), + _pRSA(0) +{ +#if OPENSSL_VERSION_NUMBER >= 0x00908000L + _pRSA = RSA_new(); + int ret = 0; + BIGNUM* bn = 0; + try + { + bn = BN_new(); + BN_set_word(bn, exponent); + ret = RSA_generate_key_ex(_pRSA, keyLength, bn, 0); + BN_free(bn); + } + catch (...) + { + BN_free(bn); + throw; + } + if (!ret) throw Poco::InvalidArgumentException("Failed to create RSA context"); +#else + _pRSA = RSA_generate_key(keyLength, exponent, 0, 0); + if (!_pRSA) throw Poco::InvalidArgumentException("Failed to create RSA context"); +#endif +} + + +RSAKeyImpl::RSAKeyImpl(const std::string& publicKeyFile, + const std::string& privateKeyFile, + const std::string& privateKeyPassphrase): KeyPairImpl("rsa", KT_RSA_IMPL), + _pRSA(0) +{ + poco_assert_dbg(_pRSA == 0); + + _pRSA = RSA_new(); + if (!publicKeyFile.empty()) + { + BIO* bio = BIO_new(BIO_s_file()); + if (!bio) throw Poco::IOException("Cannot create BIO for reading public key", publicKeyFile); + int rc = BIO_read_filename(bio, publicKeyFile.c_str()); + if (rc) + { + RSA* pubKey = PEM_read_bio_RSAPublicKey(bio, &_pRSA, 0, 0); + if (!pubKey) + { + int rc = BIO_reset(bio); + // BIO_reset() normally returns 1 for success and 0 or -1 for failure. + // File BIOs are an exception, they return 0 for success and -1 for failure. + if (rc != 0) throw Poco::FileException("Failed to load public key", publicKeyFile); + pubKey = PEM_read_bio_RSA_PUBKEY(bio, &_pRSA, 0, 0); + } + BIO_free(bio); + if (!pubKey) + { + freeRSA(); + throw Poco::FileException("Failed to load public key", publicKeyFile); + } + } + else + { + freeRSA(); + throw Poco::FileNotFoundException("Public key file", publicKeyFile); + } + } + + if (!privateKeyFile.empty()) + { + BIO* bio = BIO_new(BIO_s_file()); + if (!bio) throw Poco::IOException("Cannot create BIO for reading private key", privateKeyFile); + int rc = BIO_read_filename(bio, privateKeyFile.c_str()); + if (rc) + { + RSA* privKey = 0; + if (privateKeyPassphrase.empty()) + privKey = PEM_read_bio_RSAPrivateKey(bio, &_pRSA, 0, 0); + else + privKey = PEM_read_bio_RSAPrivateKey(bio, &_pRSA, 0, const_cast(privateKeyPassphrase.c_str())); + BIO_free(bio); + if (!privKey) + { + freeRSA(); + throw Poco::FileException("Failed to load private key", privateKeyFile); + } + } + else + { + freeRSA(); + throw Poco::FileNotFoundException("Private key file", privateKeyFile); + } + } +} + + +RSAKeyImpl::RSAKeyImpl(std::istream* pPublicKeyStream, + std::istream* pPrivateKeyStream, + const std::string& privateKeyPassphrase): KeyPairImpl("rsa", KT_RSA_IMPL), + _pRSA(0) +{ + poco_assert_dbg(_pRSA == 0); + + _pRSA = RSA_new(); + if (pPublicKeyStream) + { + std::string publicKeyData; + Poco::StreamCopier::copyToString(*pPublicKeyStream, publicKeyData); + BIO* bio = BIO_new_mem_buf(const_cast(publicKeyData.data()), static_cast(publicKeyData.size())); + if (!bio) throw Poco::IOException("Cannot create BIO for reading public key"); + RSA* publicKey = PEM_read_bio_RSAPublicKey(bio, &_pRSA, 0, 0); + if (!publicKey) + { + int rc = BIO_reset(bio); + // BIO_reset() normally returns 1 for success and 0 or -1 for failure. + // File BIOs are an exception, they return 0 for success and -1 for failure. + if (rc != 1) throw Poco::FileException("Failed to load public key"); + publicKey = PEM_read_bio_RSA_PUBKEY(bio, &_pRSA, 0, 0); + } + BIO_free(bio); + if (!publicKey) + { + freeRSA(); + throw Poco::FileException("Failed to load public key"); + } + } + + if (pPrivateKeyStream) + { + std::string privateKeyData; + Poco::StreamCopier::copyToString(*pPrivateKeyStream, privateKeyData); + BIO* bio = BIO_new_mem_buf(const_cast(privateKeyData.data()), static_cast(privateKeyData.size())); + if (!bio) throw Poco::IOException("Cannot create BIO for reading private key"); + RSA* privateKey = 0; + if (privateKeyPassphrase.empty()) + privateKey = PEM_read_bio_RSAPrivateKey(bio, &_pRSA, 0, 0); + else + privateKey = PEM_read_bio_RSAPrivateKey(bio, &_pRSA, 0, const_cast(privateKeyPassphrase.c_str())); + BIO_free(bio); + if (!privateKey) + { + freeRSA(); + throw Poco::FileException("Failed to load private key"); + } + } +} + + +RSAKeyImpl::~RSAKeyImpl() +{ + freeRSA(); +} + + +void RSAKeyImpl::freeRSA() +{ + if (_pRSA) RSA_free(_pRSA); + _pRSA = 0; +} + + +int RSAKeyImpl::size() const +{ + return RSA_size(_pRSA); +} + + +RSAKeyImpl::ByteVec RSAKeyImpl::modulus() const +{ +#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) + const BIGNUM* n = 0; + const BIGNUM* e = 0; + const BIGNUM* d = 0; + RSA_get0_key(_pRSA, &n, &e, &d); + return convertToByteVec(n); +#else + return convertToByteVec(_pRSA->n); +#endif +} + + +RSAKeyImpl::ByteVec RSAKeyImpl::encryptionExponent() const +{ +#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) + const BIGNUM* n = 0; + const BIGNUM* e = 0; + const BIGNUM* d = 0; + RSA_get0_key(_pRSA, &n, &e, &d); + return convertToByteVec(e); +#else + return convertToByteVec(_pRSA->e); +#endif +} + + +RSAKeyImpl::ByteVec RSAKeyImpl::decryptionExponent() const +{ +#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) + const BIGNUM* n = 0; + const BIGNUM* e = 0; + const BIGNUM* d = 0; + RSA_get0_key(_pRSA, &n, &e, &d); + return convertToByteVec(d); +#else + return convertToByteVec(_pRSA->d); +#endif +} + + +void RSAKeyImpl::save(const std::string& publicKeyFile, + const std::string& privateKeyFile, + const std::string& privateKeyPassphrase) const +{ + if (!publicKeyFile.empty()) + { + BIO* bio = BIO_new(BIO_s_file()); + if (!bio) throw Poco::IOException("Cannot create BIO for writing public key file", publicKeyFile); + try + { + if (BIO_write_filename(bio, const_cast(publicKeyFile.c_str()))) + { + if (!PEM_write_bio_RSAPublicKey(bio, _pRSA)) + throw Poco::WriteFileException("Failed to write public key to file", publicKeyFile); + } + else throw Poco::CreateFileException("Cannot create public key file"); + } + catch (...) + { + BIO_free(bio); + throw; + } + BIO_free(bio); + } + + if (!privateKeyFile.empty()) + { + BIO* bio = BIO_new(BIO_s_file()); + if (!bio) throw Poco::IOException("Cannot create BIO for writing private key file", privateKeyFile); + try + { + if (BIO_write_filename(bio, const_cast(privateKeyFile.c_str()))) + { + int rc = 0; + if (privateKeyPassphrase.empty()) + rc = PEM_write_bio_RSAPrivateKey(bio, _pRSA, 0, 0, 0, 0, 0); + else + rc = PEM_write_bio_RSAPrivateKey(bio, _pRSA, EVP_des_ede3_cbc(), + reinterpret_cast(const_cast(privateKeyPassphrase.c_str())), + static_cast(privateKeyPassphrase.length()), 0, 0); + if (!rc) throw Poco::FileException("Failed to write private key to file", privateKeyFile); + } + else throw Poco::CreateFileException("Cannot create private key file", privateKeyFile); + } + catch (...) + { + BIO_free(bio); + throw; + } + BIO_free(bio); + } +} + + +void RSAKeyImpl::save(std::ostream* pPublicKeyStream, + std::ostream* pPrivateKeyStream, + const std::string& privateKeyPassphrase) const +{ + if (pPublicKeyStream) + { + BIO* bio = BIO_new(BIO_s_mem()); + if (!bio) throw Poco::IOException("Cannot create BIO for writing public key"); + if (!PEM_write_bio_RSAPublicKey(bio, _pRSA)) + { + BIO_free(bio); + throw Poco::WriteFileException("Failed to write public key to stream"); + } + char* pData; + long size = BIO_get_mem_data(bio, &pData); + pPublicKeyStream->write(pData, static_cast(size)); + BIO_free(bio); + } + + if (pPrivateKeyStream) + { + BIO* bio = BIO_new(BIO_s_mem()); + if (!bio) throw Poco::IOException("Cannot create BIO for writing public key"); + int rc = 0; + if (privateKeyPassphrase.empty()) + rc = PEM_write_bio_RSAPrivateKey(bio, _pRSA, 0, 0, 0, 0, 0); + else + rc = PEM_write_bio_RSAPrivateKey(bio, _pRSA, EVP_des_ede3_cbc(), + reinterpret_cast(const_cast(privateKeyPassphrase.c_str())), + static_cast(privateKeyPassphrase.length()), 0, 0); + if (!rc) + { + BIO_free(bio); + throw Poco::FileException("Failed to write private key to stream"); + } + char* pData; + long size = BIO_get_mem_data(bio, &pData); + pPrivateKeyStream->write(pData, static_cast(size)); + BIO_free(bio); + } +} + + +RSAKeyImpl::ByteVec RSAKeyImpl::convertToByteVec(const BIGNUM* bn) +{ + int numBytes = BN_num_bytes(bn); + ByteVec byteVector(numBytes); + + ByteVec::value_type* buffer = new ByteVec::value_type[numBytes]; + BN_bn2bin(bn, buffer); + + for (int i = 0; i < numBytes; ++i) + byteVector[i] = buffer[i]; + + delete [] buffer; + + return byteVector; +} + + +} } // namespace Poco::Crypto \ No newline at end of file diff --git a/base/poco/Crypto/src/X509Certificate.cpp b/base/poco/Crypto/src/X509Certificate.cpp new file mode 100644 index 00000000000..a32e465bb0a --- /dev/null +++ b/base/poco/Crypto/src/X509Certificate.cpp @@ -0,0 +1,386 @@ +// +// X509Certificate.cpp +// +// Library: Crypto +// Package: Certificate +// Module: X509Certificate +// +// Copyright (c) 2006-2009, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Crypto/X509Certificate.h" +#include "Poco/Crypto/CryptoException.h" +#include "Poco/StreamCopier.h" +#include "Poco/String.h" +#include "Poco/DateTimeParser.h" +#include "Poco/Format.h" +#include +#include +#ifdef _WIN32 +// fix for WIN32 header conflict +#undef X509_NAME +#endif +#include +#include +#include + + +namespace Poco { +namespace Crypto { + + +X509Certificate::X509Certificate(std::istream& istr): + _pCert(0) +{ + load(istr); +} + + +X509Certificate::X509Certificate(const std::string& path): + _pCert(0) +{ + load(path); +} + + +X509Certificate::X509Certificate(X509* pCert): + _pCert(pCert) +{ + poco_check_ptr(_pCert); + + init(); +} + + +X509Certificate::X509Certificate(X509* pCert, bool shared): + _pCert(pCert) +{ + poco_check_ptr(_pCert); + + if (shared) + { +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + X509_up_ref(_pCert); +#else + _pCert->references++; +#endif + } + + init(); +} + + +X509Certificate::X509Certificate(const X509Certificate& cert): + _issuerName(cert._issuerName), + _subjectName(cert._subjectName), + _serialNumber(cert._serialNumber), + _pCert(cert._pCert) +{ + _pCert = X509_dup(_pCert); +} + + +X509Certificate& X509Certificate::operator = (const X509Certificate& cert) +{ + X509Certificate tmp(cert); + swap(tmp); + return *this; +} + + +void X509Certificate::swap(X509Certificate& cert) +{ + using std::swap; + swap(cert._issuerName, _issuerName); + swap(cert._subjectName, _subjectName); + swap(cert._serialNumber, _serialNumber); + swap(cert._pCert, _pCert); +} + + +X509Certificate::~X509Certificate() +{ + X509_free(_pCert); +} + + +void X509Certificate::load(std::istream& istr) +{ + poco_assert (!_pCert); + + std::stringstream certStream; + Poco::StreamCopier::copyStream(istr, certStream); + std::string cert = certStream.str(); + + BIO *pBIO = BIO_new_mem_buf(const_cast(cert.data()), static_cast(cert.size())); + if (!pBIO) throw Poco::IOException("Cannot create BIO for reading certificate"); + _pCert = PEM_read_bio_X509(pBIO, 0, 0, 0); + BIO_free(pBIO); + + if (!_pCert) throw Poco::IOException("Failed to load certificate from stream"); + + init(); +} + + +void X509Certificate::load(const std::string& path) +{ + poco_assert (!_pCert); + + BIO *pBIO = BIO_new(BIO_s_file()); + if (!pBIO) throw Poco::IOException("Cannot create BIO for reading certificate file", path); + if (!BIO_read_filename(pBIO, path.c_str())) + { + BIO_free(pBIO); + throw Poco::OpenFileException("Cannot open certificate file for reading", path); + } + + _pCert = PEM_read_bio_X509(pBIO, 0, 0, 0); + BIO_free(pBIO); + + if (!_pCert) throw Poco::ReadFileException("Faild to load certificate from", path); + + init(); +} + + +void X509Certificate::save(std::ostream& stream) const +{ + BIO *pBIO = BIO_new(BIO_s_mem()); + if (!pBIO) throw Poco::IOException("Cannot create BIO for writing certificate"); + try + { + if (!PEM_write_bio_X509(pBIO, _pCert)) + throw Poco::IOException("Failed to write certificate to stream"); + + char *pData; + long size; + size = BIO_get_mem_data(pBIO, &pData); + stream.write(pData, size); + } + catch (...) + { + BIO_free(pBIO); + throw; + } + BIO_free(pBIO); +} + + +void X509Certificate::save(const std::string& path) const +{ + BIO *pBIO = BIO_new(BIO_s_file()); + if (!pBIO) throw Poco::IOException("Cannot create BIO for reading certificate file", path); + if (!BIO_write_filename(pBIO, const_cast(path.c_str()))) + { + BIO_free(pBIO); + throw Poco::CreateFileException("Cannot create certificate file", path); + } + try + { + if (!PEM_write_bio_X509(pBIO, _pCert)) + throw Poco::WriteFileException("Failed to write certificate to file", path); + } + catch (...) + { + BIO_free(pBIO); + throw; + } + BIO_free(pBIO); +} + + +void X509Certificate::init() +{ + char buffer[NAME_BUFFER_SIZE]; + X509_NAME_oneline(X509_get_issuer_name(_pCert), buffer, sizeof(buffer)); + _issuerName = buffer; + X509_NAME_oneline(X509_get_subject_name(_pCert), buffer, sizeof(buffer)); + _subjectName = buffer; + BIGNUM* pBN = ASN1_INTEGER_to_BN(X509_get_serialNumber(const_cast(_pCert)), 0); + if (pBN) + { + char* pSN = BN_bn2hex(pBN); + if (pSN) + { + _serialNumber = pSN; + OPENSSL_free(pSN); + } + BN_free(pBN); + } +} + + +std::string X509Certificate::commonName() const +{ + return subjectName(NID_COMMON_NAME); +} + + +std::string X509Certificate::issuerName(NID nid) const +{ + if (X509_NAME* issuer = X509_get_issuer_name(_pCert)) + { + char buffer[NAME_BUFFER_SIZE]; + if (X509_NAME_get_text_by_NID(issuer, nid, buffer, sizeof(buffer)) >= 0) + return std::string(buffer); + } + return std::string(); +} + + +std::string X509Certificate::subjectName(NID nid) const +{ + if (X509_NAME* subj = X509_get_subject_name(_pCert)) + { + char buffer[NAME_BUFFER_SIZE]; + if (X509_NAME_get_text_by_NID(subj, nid, buffer, sizeof(buffer)) >= 0) + return std::string(buffer); + } + return std::string(); +} + + +void X509Certificate::extractNames(std::string& cmnName, std::set& domainNames) const +{ + domainNames.clear(); + if (STACK_OF(GENERAL_NAME)* names = static_cast(X509_get_ext_d2i(_pCert, NID_subject_alt_name, 0, 0))) + { + for (int i = 0; i < sk_GENERAL_NAME_num(names); ++i) + { + const GENERAL_NAME* name = sk_GENERAL_NAME_value(names, i); + if (name->type == GEN_DNS) + { + const char* data = reinterpret_cast(ASN1_STRING_data(name->d.ia5)); + std::size_t len = ASN1_STRING_length(name->d.ia5); + domainNames.insert(std::string(data, len)); + } + } + GENERAL_NAMES_free(names); + } + + cmnName = commonName(); + if (!cmnName.empty() && domainNames.empty()) + { + domainNames.insert(cmnName); + } +} + + +Poco::DateTime X509Certificate::validFrom() const +{ + ASN1_TIME* certTime = X509_get_notBefore(_pCert); + std::string dateTime(reinterpret_cast(certTime->data)); + int tzd; + return DateTimeParser::parse("%y%m%d%H%M%S", dateTime, tzd); +} + + +Poco::DateTime X509Certificate::expiresOn() const +{ + ASN1_TIME* certTime = X509_get_notAfter(_pCert); + std::string dateTime(reinterpret_cast(certTime->data)); + int tzd; + return DateTimeParser::parse("%y%m%d%H%M%S", dateTime, tzd); +} + + +bool X509Certificate::issuedBy(const X509Certificate& issuerCertificate) const +{ + X509* pCert = const_cast(_pCert); + X509* pIssuerCert = const_cast(issuerCertificate.certificate()); + EVP_PKEY* pIssuerPublicKey = X509_get_pubkey(pIssuerCert); + if (!pIssuerPublicKey) throw Poco::InvalidArgumentException("Issuer certificate has no public key"); + int rc = X509_verify(pCert, pIssuerPublicKey); + EVP_PKEY_free(pIssuerPublicKey); + return rc == 1; +} + + +bool X509Certificate::equals(const X509Certificate& otherCertificate) const +{ + X509* pCert = const_cast(_pCert); + X509* pOtherCert = const_cast(otherCertificate.certificate()); + return X509_cmp(pCert, pOtherCert) == 0; +} + + +std::string X509Certificate::signatureAlgorithm() const +{ + int sigNID = NID_undef; + +#if (OPENSSL_VERSION_NUMBER >= 0x1010000fL) && !defined(LIBRESSL_VERSION_NUMBER) + sigNID = X509_get_signature_nid(_pCert); +#else + poco_check_ptr(_pCert->sig_alg); + sigNID = OBJ_obj2nid(_pCert->sig_alg->algorithm); +#endif + + if (sigNID != NID_undef) + { + const char* pAlgName = OBJ_nid2ln(sigNID); + if (pAlgName) return std::string(pAlgName); + else throw OpenSSLException(Poco::format("X509Certificate::" + "signatureAlgorithm(): OBJ_nid2ln(%d)", sigNID)); + } + else + throw NotFoundException("X509Certificate::signatureAlgorithm()"); + + return ""; +} + + +X509Certificate::List X509Certificate::readPEM(const std::string& pemFileName) +{ + List caCertList; + BIO* pBIO = BIO_new_file(pemFileName.c_str(), "r"); + if (pBIO == NULL) throw OpenFileException("X509Certificate::readPEM()"); + X509* x = PEM_read_bio_X509(pBIO, NULL, 0, NULL); + if (!x) throw OpenSSLException(Poco::format("X509Certificate::readPEM(%s)", pemFileName)); + while(x) + { + caCertList.push_back(X509Certificate(x)); + x = PEM_read_bio_X509(pBIO, NULL, 0, NULL); + } + BIO_free(pBIO); + return caCertList; +} + + +void X509Certificate::writePEM(const std::string& pemFileName, const List& list) +{ + BIO* pBIO = BIO_new_file(pemFileName.c_str(), "a"); + if (pBIO == NULL) throw OpenFileException("X509Certificate::writePEM()"); + List::const_iterator it = list.begin(); + List::const_iterator end = list.end(); + for (; it != end; ++it) + { + if (!PEM_write_bio_X509(pBIO, const_cast(it->certificate()))) + { + throw OpenSSLException("X509Certificate::writePEM()"); + } + } + BIO_free(pBIO); +} + + +void X509Certificate::print(std::ostream& out) const +{ + out << "subjectName: " << subjectName() << std::endl; + out << "issuerName: " << issuerName() << std::endl; + out << "commonName: " << commonName() << std::endl; + out << "country: " << subjectName(X509Certificate::NID_COUNTRY) << std::endl; + out << "localityName: " << subjectName(X509Certificate::NID_LOCALITY_NAME) << std::endl; + out << "stateOrProvince: " << subjectName(X509Certificate::NID_STATE_OR_PROVINCE) << std::endl; + out << "organizationName: " << subjectName(X509Certificate::NID_ORGANIZATION_NAME) << std::endl; + out << "organizationUnitName: " << subjectName(X509Certificate::NID_ORGANIZATION_UNIT_NAME) << std::endl; + out << "emailAddress: " << subjectName(X509Certificate::NID_PKCS9_EMAIL_ADDRESS) << std::endl; + out << "serialNumber: " << subjectName(X509Certificate::NID_SERIAL_NUMBER) << std::endl; +} + + +} } // namespace Poco::Crypto diff --git a/base/poco/Data/CMakeLists.txt b/base/poco/Data/CMakeLists.txt new file mode 100644 index 00000000000..142d3592b69 --- /dev/null +++ b/base/poco/Data/CMakeLists.txt @@ -0,0 +1,7 @@ +file (GLOB SRCS src/*.cpp) + +add_library (_poco_data ${SRCS}) +add_library (Poco::Data ALIAS _poco_data) + +target_include_directories (_poco_data SYSTEM PUBLIC "include") +target_link_libraries (_poco_data PUBLIC Poco::Foundation) diff --git a/base/poco/Data/ODBC/CMakeLists.txt b/base/poco/Data/ODBC/CMakeLists.txt new file mode 100644 index 00000000000..4a49f915b67 --- /dev/null +++ b/base/poco/Data/ODBC/CMakeLists.txt @@ -0,0 +1,39 @@ +if (ENABLE_ODBC) + if (NOT TARGET ch_contrib::unixodbc) + message(FATAL_ERROR "Configuration error: unixodbc is not a target") + endif() + + set (SRCS + src/Binder.cpp + src/ConnectionHandle.cpp + src/Connector.cpp + src/EnvironmentHandle.cpp + src/Extractor.cpp + src/ODBCException.cpp + src/ODBCMetaColumn.cpp + src/ODBCStatementImpl.cpp + src/Parameter.cpp + src/Preparator.cpp + src/SessionImpl.cpp + src/TypeInfo.cpp + src/Unicode.cpp + src/Utility.cpp + ) + + add_library (_poco_data_odbc ${SRCS}) + add_library (Poco::Data::ODBC ALIAS _poco_data_odbc) + + target_compile_options (_poco_data_odbc PRIVATE -Wno-unused-variable) + target_include_directories (_poco_data_odbc SYSTEM PUBLIC "include") + target_link_libraries (_poco_data_odbc PUBLIC Poco::Data ch_contrib::unixodbc) + + message (STATUS "Using Poco::Data::ODBC") +else () + add_library (_poco_data_odbc INTERFACE) + add_library (Poco::Data::ODBC ALIAS _poco_data_odbc) + if (TARGET ch_contrib::unixodbc) + target_link_libraries (_poco_data_odbc INTERFACE ch_contrib::unixodbc) + endif() + + message (STATUS "Not using Poco::Data::ODBC") +endif () diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Binder.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Binder.h new file mode 100644 index 00000000000..fd3241fbf8c --- /dev/null +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Binder.h @@ -0,0 +1,1516 @@ +// +// Binder.h +// +// Library: Data/ODBC +// Package: ODBC +// Module: Binder +// +// Definition of the Binder class. +// +// Copyright (c) 2006, Applied Informatics Software Engineering GmbH. +// and Contributors. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef Data_ODBC_Binder_INCLUDED +#define Data_ODBC_Binder_INCLUDED + + +#include "Poco/Data/ODBC/ODBC.h" +#include "Poco/Data/AbstractBinder.h" +#include "Poco/Data/LOB.h" +#include "Poco/Data/ODBC/Handle.h" +#include "Poco/Data/ODBC/Parameter.h" +#include "Poco/Data/ODBC/ODBCMetaColumn.h" +#include "Poco/Data/ODBC/Utility.h" +#include "Poco/Data/ODBC/TypeInfo.h" +#include "Poco/Exception.h" +#include +#include +#include +#include +#ifdef POCO_OS_FAMILY_WINDOWS +#include +#endif +#include + + +namespace Poco { + + +class DateTime; + + +namespace Data { + + +class Date; +class Time; + + +namespace ODBC { + + +class ODBC_API Binder: public Poco::Data::AbstractBinder + /// Binds placeholders in the sql query to the provided values. Performs data types mapping. +{ +public: + typedef AbstractBinder::Direction Direction; + typedef std::map ParamMap; + + static const size_t DEFAULT_PARAM_SIZE = 1024; + + enum ParameterBinding + { + PB_IMMEDIATE, + PB_AT_EXEC + }; + + Binder(const StatementHandle& rStmt, + std::size_t maxFieldSize, + ParameterBinding dataBinding = PB_IMMEDIATE, + TypeInfo* pDataTypes = 0); + /// Creates the Binder. + + ~Binder(); + /// Destroys the Binder. + + void bind(std::size_t pos, const Poco::Int8& val, Direction dir); + /// Binds an Int8. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds an Int8 vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds an Int8 deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds an Int8 list. + + void bind(std::size_t pos, const Poco::UInt8& val, Direction dir); + /// Binds an UInt8. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds an UInt8 vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds an UInt8 deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds an UInt8 list. + + void bind(std::size_t pos, const Poco::Int16& val, Direction dir); + /// Binds an Int16. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds an Int16 vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds an Int16 deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds an Int16 list. + + void bind(std::size_t pos, const Poco::UInt16& val, Direction dir); + /// Binds an UInt16. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds an UInt16 vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds an UInt16 deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds an UInt16 list. + + void bind(std::size_t pos, const Poco::Int32& val, Direction dir); + /// Binds an Int32. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds an Int32 vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds an Int32 deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds an Int32 list. + + void bind(std::size_t pos, const Poco::UInt32& val, Direction dir); + /// Binds an UInt32. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds an UInt32 vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds an UInt32 deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds an UInt32 list. + + void bind(std::size_t pos, const Poco::Int64& val, Direction dir); + /// Binds an Int64. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds an Int64 vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds an Int64 deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds an Int64 list. + + void bind(std::size_t pos, const Poco::UInt64& val, Direction dir); + /// Binds an UInt64. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds an UInt64 vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds an UInt64 deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds an UInt64 list. + +#ifndef POCO_LONG_IS_64_BIT + void bind(std::size_t pos, const long& val, Direction dir); + /// Binds a long. + + void bind(std::size_t pos, const unsigned long& val, Direction dir); + /// Binds an unsigned long. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds a long vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds a long deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds a long list. +#endif + + void bind(std::size_t pos, const bool& val, Direction dir); + /// Binds a boolean. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds a boolean vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds a boolean deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds a boolean list. + + void bind(std::size_t pos, const float& val, Direction dir); + /// Binds a float. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds a float vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds a float deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds a float list. + + void bind(std::size_t pos, const double& val, Direction dir); + /// Binds a double. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds a double vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds a double deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds a double list. + + void bind(std::size_t pos, const char& val, Direction dir); + /// Binds a single character. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds a character vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds a character deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds a character list. + + void bind(std::size_t pos, const std::string& val, Direction dir); + /// Binds a string. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds a string vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds a string deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds a string list. + + void bind(std::size_t pos, const UTF16String& val, Direction dir); + /// Binds a string. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds a string vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds a string deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds a string list. + + void bind(std::size_t pos, const BLOB& val, Direction dir); + /// Binds a BLOB. In-bound only. + + void bind(std::size_t pos, const CLOB& val, Direction dir); + /// Binds a CLOB. In-bound only. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds a BLOB vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds a BLOB deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds a BLOB list. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds a CLOB vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds a CLOB deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds a CLOB list. + + void bind(std::size_t pos, const Date& val, Direction dir); + /// Binds a Date. + + void bind(std::size_t pos, const std::vector& val, Direction dir); + /// Binds a Date vector. + + void bind(std::size_t pos, const std::deque& val, Direction dir); + /// Binds a Date deque. + + void bind(std::size_t pos, const std::list& val, Direction dir); + /// Binds a Date list. + + void bind(std::size_t pos, const Time& val, Direction dir); + /// Binds a Time. + + void bind(std::size_t pos, const std::vector