From b264b853a5242de28d31b31fb0b092fa3e3670e2 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Mon, 21 Nov 2016 16:21:35 +0300 Subject: [PATCH 1/5] Added accurate numbers comparison. [#CLICKHOUSE-194] --- .../include/DB/Functions/AccurateComparison.h | 302 ++++++++++++++++++ .../DB/Functions/FunctionsComparison.h | 33 +- 2 files changed, 324 insertions(+), 11 deletions(-) create mode 100644 dbms/include/DB/Functions/AccurateComparison.h diff --git a/dbms/include/DB/Functions/AccurateComparison.h b/dbms/include/DB/Functions/AccurateComparison.h new file mode 100644 index 00000000000..c4bc1d9438a --- /dev/null +++ b/dbms/include/DB/Functions/AccurateComparison.h @@ -0,0 +1,302 @@ +#include + +namespace accurate +{ + +/** Cases: + 1) int vs uint + a) sizeof(int) <= sizeof(uint). Accurate comparison with MAX_INT tresholds + b) sizeof(int) > sizeof(uint). Casting to int + 2) integral_type vs floating_type + a) sizeof(integral_type) <= 4. Comparison via casting arguments to Float64 + b) sizeof(integral_type) == 8. Accurate comparison. Consider 3 sets of intervals: + 1) interval between adjacent floats less or equal 1 + 2) interval between adjacent floats greater then 2 + 3) float is outside [MIN_INT64; MAX_INT64] + 3) Safe conversion + a) int vs any int + b) uint vs any uint + c) float vs any float +*/ + +/// Are params IntXX and UIntYY ? +template +using is_any_int_vs_uint = std::integral_constant::value && std::is_integral::value && + std::is_signed::value && std::is_unsigned::value>; + + +/// Are params IntXX and UIntYY and sizeof(IntXX) >= sizeof(UIntYY) (in such case will use accurate compare) +template +using is_le_int_vs_uint_t = std::integral_constant::value && (sizeof(TInt) <= sizeof(TUInt))>; + +template +using bool_if_le_int_vs_uint_t = std::enable_if_t::value, bool>; + +template +bool_if_le_int_vs_uint_t greaterOpTmpl(TInt a, TUInt b) +{ + return (b > static_cast(std::numeric_limits::max()) || a < 0) ? false : static_cast(a) > b; +} + +template +bool_if_le_int_vs_uint_t greaterOpTmpl(TUInt a, TInt b) +{ + return (a > static_cast(std::numeric_limits::max()) || b < 0) ? true : a > static_cast(b); +} + +template +bool_if_le_int_vs_uint_t equalsOpTmpl(TInt a, TUInt b) +{ + return (a < 0 || b > static_cast(std::numeric_limits::max())) ? false : static_cast(a) == b; +} + +template +bool_if_le_int_vs_uint_t equalsOpTmpl(TUInt a, TInt b) +{ + return (b < 0 || a > static_cast(std::numeric_limits::max())) ? false : a == static_cast(b); +} + + +/// Is pair of floats or pair of ints or pair of uints +template +using is_safe_convervsion = std::integral_constant::value && std::is_floating_point::value) + || (std::is_integral::value && std::is_integral::value && !(std::is_signed::value ^ std::is_signed::value))>; +template +using bool_if_safe_convervsion = std::enable_if_t::value, bool>; + +template +bool_if_safe_convervsion greaterOpTmpl(A a, B b) +{ + return a > b; +} + +template +bool_if_safe_convervsion equalsOpTmpl(A a, B b) +{ + return a == b; +} + + +/// Are params IntXX and UIntYY and sizeof(IntXX) > sizeof(UIntYY) (in such case will cast UIntYY to IntXX and compare) +template +using is_gt_int_vs_uint = std::integral_constant::value && (sizeof(TInt) > sizeof(TUInt))>; + +template +using bool_if_gt_int_vs_uint = std::enable_if_t::value, bool>; + +template +bool_if_gt_int_vs_uint greaterOpTmpl(TInt a, TUInt b) +{ + return static_cast(a) > static_cast(b); +} + +template +bool_if_gt_int_vs_uint greaterOpTmpl(TUInt a, TInt b) +{ + return static_cast(a) > static_cast(b); +} + +template +bool_if_gt_int_vs_uint equalsOpTmpl(TInt a, TUInt b) +{ + return static_cast(a) == static_cast(b); +} + +template +bool_if_gt_int_vs_uint equalsOpTmpl(TUInt a, TInt b) +{ + return static_cast(a) == static_cast(b); +} + + +template +using bool_if_double_can_be_used = std::enable_if_t< + std::is_integral::value && (sizeof(TAInt) <= 4) && std::is_floating_point::value, + bool>; + +template +bool_if_double_can_be_used greaterOpTmpl(TAInt a, TAFloat b) +{ + return static_cast(a) > static_cast(b); +} + +template +bool_if_double_can_be_used greaterOpTmpl(TAFloat a, TAInt b) +{ + return static_cast(a) > static_cast(b); +} + +template +bool_if_double_can_be_used equalsOpTmpl(TAInt a, TAFloat b) +{ + return static_cast(a) == static_cast(b); +} + +template +bool_if_double_can_be_used equalsOpTmpl(TAFloat a, TAInt b) +{ + return static_cast(a) == static_cast(b); +} + + +template +inline bool greaterOp(A a, B b) +{ + return greaterOpTmpl(a, b); +} + +// See hint at https://github.com/JuliaLang/julia/issues/257 +constexpr DB::Int64 MAX_INT64_WITH_EXACT_FLOAT64_REPR = 9007199254740992LL; // 2^53 + +template<> +inline bool greaterOp(DB::Float64 f, DB::Int64 i) +{ + if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR) + return f > static_cast(i); + + return (f >= static_cast(std::numeric_limits::max())) + || (f > static_cast(std::numeric_limits::min()) && static_cast(f) > i); +} + +template<> +inline bool greaterOp(DB::Int64 i, DB::Float64 f) +{ + if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR) + return f < static_cast(i); + + return (f <= static_cast(std::numeric_limits::min())) + || (f < static_cast(std::numeric_limits::max()) && i > static_cast(f)); +} + +template<> +inline bool greaterOp(DB::Float64 f, DB::UInt64 u) +{ + if (u <= static_cast(MAX_INT64_WITH_EXACT_FLOAT64_REPR)) + return f > static_cast(u); + + return (f >= static_cast(std::numeric_limits::max())) + || (f >= 0 && static_cast(f) > u); +} + +template<> +inline bool greaterOp(DB::UInt64 u, DB::Float64 f) +{ + if (u <= static_cast(MAX_INT64_WITH_EXACT_FLOAT64_REPR)) + return static_cast(u) > f; + + return (f < 0) + || (f < static_cast(std::numeric_limits::max()) && u > static_cast(f)); +} + + +template<> +inline bool greaterOp(DB::Float32 f, DB::Int64 i) +{ + return greaterOp(static_cast(f), i); +} + +template<> +inline bool greaterOp(DB::Int64 i, DB::Float32 f) +{ + return greaterOp(i, static_cast(f)); +} + +template<> +inline bool greaterOp(DB::Float32 f, DB::UInt64 u) +{ + return greaterOp(static_cast(f), u); +} + +template<> +inline bool greaterOp(DB::UInt64 u, DB::Float32 f) +{ + return greaterOp(u, static_cast(f)); +} + + +template +inline bool equalsOp(A a, B b) +{ + return equalsOpTmpl(a, b); +} + +template<> +inline bool equalsOp(DB::Float64 f, DB::UInt64 u) +{ + return static_cast(f) == u && f == static_cast(u); +} + +template<> +inline bool equalsOp(DB::UInt64 u, DB::Float64 f) +{ + return u == static_cast(f) && static_cast(u) == f; +} + +template<> +inline bool equalsOp(DB::Float64 f, DB::Int64 u) +{ + return static_cast(f) == u && f == static_cast(u); +} + +template<> +inline bool equalsOp(DB::Int64 u, DB::Float64 f) +{ + return u == static_cast(f) && static_cast(u) == f; +} + +template<> +inline bool equalsOp(DB::Float32 f, DB::UInt64 u) +{ + return static_cast(f) == u && f == static_cast(u); +} + +template<> +inline bool equalsOp(DB::UInt64 u, DB::Float32 f) +{ + return u == static_cast(f) && static_cast(u) == f; +} + +template<> +inline bool equalsOp(DB::Float32 f, DB::Int64 u) +{ + return static_cast(f) == u && f == static_cast(u); +} + +template<> +inline bool equalsOp(DB::Int64 u, DB::Float32 f) +{ + return u == static_cast(f) && static_cast(u) == f; +} + + + +template +inline bool notEqualsOp(A a, B b) +{ + return !equalsOp(a, b); +} + + +template +inline bool lessOp(A a, B b) +{ + return greaterOp(b, a); +} + + +template +inline bool lessOrEqualsOp(A a, B b) +{ + return !greaterOp(a, b); +} + + +template +inline bool greaterOrEqualsOp(A a, B b) +{ + return !greaterOp(b, a); +} + + +} diff --git a/dbms/include/DB/Functions/FunctionsComparison.h b/dbms/include/DB/Functions/FunctionsComparison.h index 5ff6381418c..d2928f9786a 100644 --- a/dbms/include/DB/Functions/FunctionsComparison.h +++ b/dbms/include/DB/Functions/FunctionsComparison.h @@ -16,10 +16,14 @@ #include #include +#include #include #include +#include +#include + namespace DB { @@ -40,20 +44,27 @@ namespace DB * TODO Массивы. */ +template struct EqualsOp { static UInt8 apply(A a, B b) { return accurate::equalsOp(a, b); } }; +template struct NotEqualsOp { static UInt8 apply(A a, B b) { return accurate::notEqualsOp(a, b); } }; +template struct LessOp { static UInt8 apply(A a, B b) { return accurate::lessOp(a, b); } }; +template struct GreaterOp { static UInt8 apply(A a, B b) { return accurate::greaterOp(a, b); } }; +template struct LessOrEqualsOp { static UInt8 apply(A a, B b) { return accurate::lessOrEqualsOp(a, b); } }; +template struct GreaterOrEqualsOp { static UInt8 apply(A a, B b) { return accurate::greaterOrEqualsOp(a, b); } }; + /** Игнорируем warning о сравнении signed и unsigned. * (Результат может быть некорректным.) */ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wsign-compare" - -template struct EqualsOp { static UInt8 apply(A a, B b) { return a == b; } }; -template struct NotEqualsOp { static UInt8 apply(A a, B b) { return a != b; } }; -template struct LessOp { static UInt8 apply(A a, B b) { return a < b; } }; -template struct GreaterOp { static UInt8 apply(A a, B b) { return a > b; } }; -template struct LessOrEqualsOp { static UInt8 apply(A a, B b) { return a <= b; } }; -template struct GreaterOrEqualsOp { static UInt8 apply(A a, B b) { return a >= b; } }; - -#pragma GCC diagnostic pop +// #pragma GCC diagnostic push +// #pragma GCC diagnostic ignored "-Wsign-compare" +// +// template struct EqualsOp { static UInt8 apply(A a, B b) { return a == b; } }; +// template struct NotEqualsOp { static UInt8 apply(A a, B b) { return a != b; } }; +// template struct LessOp { static UInt8 apply(A a, B b) { return a < b; } }; +// template struct GreaterOp { static UInt8 apply(A a, B b) { return a > b; } }; +// template struct LessOrEqualsOp { static UInt8 apply(A a, B b) { return a <= b; } }; +// template struct GreaterOrEqualsOp { static UInt8 apply(A a, B b) { return a >= b; } }; +// +// #pragma GCC diagnostic pop From 7d86246b4613ace06af1db7b74e1f86aabce2ae6 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Wed, 11 Jan 2017 16:38:36 +0300 Subject: [PATCH 2/5] Add safe_convervsion case optimization. Fixed MIN_INT64 comparison. Fixed NaN comparisons between floats. [#CLICKHOUSE-194] --- .../include/DB/Functions/AccurateComparison.h | 124 ++++++++++++------ 1 file changed, 85 insertions(+), 39 deletions(-) diff --git a/dbms/include/DB/Functions/AccurateComparison.h b/dbms/include/DB/Functions/AccurateComparison.h index c4bc1d9438a..48e94f0186b 100644 --- a/dbms/include/DB/Functions/AccurateComparison.h +++ b/dbms/include/DB/Functions/AccurateComparison.h @@ -1,32 +1,57 @@ #include +/** Preceptually-correct number comparisons. + * Example: Int8(-1) != UInt8(255) +*/ + namespace accurate { /** Cases: - 1) int vs uint + 1) Safe conversion (in case of default C++ operators) + a) int vs any int + b) uint vs any uint + c) float vs any float + 2) int vs uint a) sizeof(int) <= sizeof(uint). Accurate comparison with MAX_INT tresholds b) sizeof(int) > sizeof(uint). Casting to int - 2) integral_type vs floating_type + 3) integral_type vs floating_type a) sizeof(integral_type) <= 4. Comparison via casting arguments to Float64 b) sizeof(integral_type) == 8. Accurate comparison. Consider 3 sets of intervals: 1) interval between adjacent floats less or equal 1 2) interval between adjacent floats greater then 2 3) float is outside [MIN_INT64; MAX_INT64] - 3) Safe conversion - a) int vs any int - b) uint vs any uint - c) float vs any float */ -/// Are params IntXX and UIntYY ? +// Case 1. Is pair of floats or pair of ints or pair of uints +template +using is_safe_convervsion = std::integral_constant::value && std::is_floating_point::value) + || (std::is_integral::value && std::is_integral::value && !(std::is_signed::value ^ std::is_signed::value))>; +template +using bool_if_safe_convervsion = std::enable_if_t::value, bool>; +template +using bool_if_not_safe_convervsion = std::enable_if_t::value, bool>; + +// template +// bool_if_safe_convervsion greaterOpTmpl(A a, B b) +// { +// return a > b; +// } +// +// template +// bool_if_safe_convervsion equalsOpTmpl(A a, B b) +// { +// return a == b; +// } + +/// Case 2. Are params IntXX and UIntYY ? template using is_any_int_vs_uint = std::integral_constant::value && std::is_integral::value && std::is_signed::value && std::is_unsigned::value>; -/// Are params IntXX and UIntYY and sizeof(IntXX) >= sizeof(UIntYY) (in such case will use accurate compare) +// Case 2a. Are params IntXX and UIntYY and sizeof(IntXX) >= sizeof(UIntYY) (in such case will use accurate compare) template using is_le_int_vs_uint_t = std::integral_constant::value && (sizeof(TInt) <= sizeof(TUInt))>; @@ -58,27 +83,7 @@ bool_if_le_int_vs_uint_t equalsOpTmpl(TUInt a, TInt b) } -/// Is pair of floats or pair of ints or pair of uints -template -using is_safe_convervsion = std::integral_constant::value && std::is_floating_point::value) - || (std::is_integral::value && std::is_integral::value && !(std::is_signed::value ^ std::is_signed::value))>; -template -using bool_if_safe_convervsion = std::enable_if_t::value, bool>; - -template -bool_if_safe_convervsion greaterOpTmpl(A a, B b) -{ - return a > b; -} - -template -bool_if_safe_convervsion equalsOpTmpl(A a, B b) -{ - return a == b; -} - - -/// Are params IntXX and UIntYY and sizeof(IntXX) > sizeof(UIntYY) (in such case will cast UIntYY to IntXX and compare) +// Case 2b. Are params IntXX and UIntYY and sizeof(IntXX) > sizeof(UIntYY) (in such case will cast UIntYY to IntXX and compare) template using is_gt_int_vs_uint = std::integral_constant::value && (sizeof(TInt) > sizeof(TUInt))>; @@ -110,6 +115,7 @@ bool_if_gt_int_vs_uint equalsOpTmpl(TUInt a, TInt b) } +// Case 3a. Comparison via conversion to double. template using bool_if_double_can_be_used = std::enable_if_t< std::is_integral::value && (sizeof(TAInt) <= 4) && std::is_floating_point::value, @@ -140,13 +146,24 @@ bool_if_double_can_be_used equalsOpTmpl(TAFloat a, TAInt b) } +/* Final realiztions */ + + template -inline bool greaterOp(A a, B b) +inline bool_if_not_safe_convervsion greaterOp(A a, B b) { return greaterOpTmpl(a, b); } +template +inline bool_if_safe_convervsion greaterOp(A a, B b) +{ + return a > b; +} + +// Case 3b. 64-bit integers vs floats comparison. // See hint at https://github.com/JuliaLang/julia/issues/257 + constexpr DB::Int64 MAX_INT64_WITH_EXACT_FLOAT64_REPR = 9007199254740992LL; // 2^53 template<> @@ -155,7 +172,7 @@ inline bool greaterOp(DB::Float64 f, DB::Int64 i) if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR) return f > static_cast(i); - return (f >= static_cast(std::numeric_limits::max())) + return (f >= static_cast(std::numeric_limits::max())) // rhs is 2**63 (not 2^63 - 1) || (f > static_cast(std::numeric_limits::min()) && static_cast(f) > i); } @@ -165,7 +182,7 @@ inline bool greaterOp(DB::Int64 i, DB::Float64 f) if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR) return f < static_cast(i); - return (f <= static_cast(std::numeric_limits::min())) + return (f < static_cast(std::numeric_limits::min())) || (f < static_cast(std::numeric_limits::max()) && i > static_cast(f)); } @@ -189,7 +206,7 @@ inline bool greaterOp(DB::UInt64 u, DB::Float64 f) || (f < static_cast(std::numeric_limits::max()) && u > static_cast(f)); } - +// Case 3b for float32 template<> inline bool greaterOp(DB::Float32 f, DB::Int64 i) { @@ -216,11 +233,17 @@ inline bool greaterOp(DB::UInt64 u, DB::Float32 f) template -inline bool equalsOp(A a, B b) +inline bool_if_not_safe_convervsion equalsOp(A a, B b) { return equalsOpTmpl(a, b); } +template +inline bool_if_safe_convervsion equalsOp(A a, B b) +{ + return a == b; +} + template<> inline bool equalsOp(DB::Float64 f, DB::UInt64 u) { @@ -270,33 +293,56 @@ inline bool equalsOp(DB::Int64 u, DB::Float32 f) } - template -inline bool notEqualsOp(A a, B b) +inline bool_if_not_safe_convervsion notEqualsOp(A a, B b) { return !equalsOp(a, b); } +template +inline bool_if_safe_convervsion notEqualsOp(A a, B b) +{ + return a != b; +} + template -inline bool lessOp(A a, B b) +inline bool_if_not_safe_convervsion lessOp(A a, B b) { return greaterOp(b, a); } +template +inline bool_if_safe_convervsion lessOp(A a, B b) +{ + return a < b; +} + template -inline bool lessOrEqualsOp(A a, B b) +inline bool_if_not_safe_convervsion lessOrEqualsOp(A a, B b) { return !greaterOp(a, b); } +template +inline bool_if_safe_convervsion lessOrEqualsOp(A a, B b) +{ + return a <= b; +} + template -inline bool greaterOrEqualsOp(A a, B b) +inline bool_if_not_safe_convervsion greaterOrEqualsOp(A a, B b) { return !greaterOp(b, a); } +template +inline bool_if_safe_convervsion greaterOrEqualsOp(A a, B b) +{ + return a >= b; +} + } From 3e53e3268ad46d65c3c9bfae30c6b6eac1e04d6f Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Wed, 11 Jan 2017 18:28:41 +0300 Subject: [PATCH 3/5] Add test for accurate numbers comparisons. [#CLICKHOUSE-194] --- .../DB/Functions/FunctionsComparison.h | 18 +-- .../accurate_comparisons.sh | 25 +++ ...00411_accurate_number_comparison.reference | 1 + .../00411_accurate_number_comparison.sh | 143 ++++++++++++++++++ 4 files changed, 178 insertions(+), 9 deletions(-) create mode 100755 dbms/tests/perf_drafts/accurate_comparisons/accurate_comparisons.sh create mode 100644 dbms/tests/queries/0_stateless/00411_accurate_number_comparison.reference create mode 100755 dbms/tests/queries/0_stateless/00411_accurate_number_comparison.sh diff --git a/dbms/include/DB/Functions/FunctionsComparison.h b/dbms/include/DB/Functions/FunctionsComparison.h index d2928f9786a..6e56b5b9db0 100644 --- a/dbms/include/DB/Functions/FunctionsComparison.h +++ b/dbms/include/DB/Functions/FunctionsComparison.h @@ -44,16 +44,16 @@ namespace DB * TODO Массивы. */ -template struct EqualsOp { static UInt8 apply(A a, B b) { return accurate::equalsOp(a, b); } }; -template struct NotEqualsOp { static UInt8 apply(A a, B b) { return accurate::notEqualsOp(a, b); } }; -template struct LessOp { static UInt8 apply(A a, B b) { return accurate::lessOp(a, b); } }; -template struct GreaterOp { static UInt8 apply(A a, B b) { return accurate::greaterOp(a, b); } }; -template struct LessOrEqualsOp { static UInt8 apply(A a, B b) { return accurate::lessOrEqualsOp(a, b); } }; -template struct GreaterOrEqualsOp { static UInt8 apply(A a, B b) { return accurate::greaterOrEqualsOp(a, b); } }; +template struct EqualsOp { static UInt8 apply(A a, B b) { return accurate::equalsOp(a, b); } }; +template struct NotEqualsOp { static UInt8 apply(A a, B b) { return accurate::notEqualsOp(a, b); } }; +template struct LessOp { static UInt8 apply(A a, B b) { return accurate::lessOp(a, b); } }; +template struct GreaterOp { static UInt8 apply(A a, B b) { return accurate::greaterOp(a, b); } }; +template struct LessOrEqualsOp { static UInt8 apply(A a, B b) { return accurate::lessOrEqualsOp(a, b); } }; +template struct GreaterOrEqualsOp { static UInt8 apply(A a, B b) { return accurate::greaterOrEqualsOp(a, b); } }; -/** Игнорируем warning о сравнении signed и unsigned. - * (Результат может быть некорректным.) - */ +// /** Игнорируем warning о сравнении signed и unsigned. +// * (Результат может быть некорректным.) +// */ // #pragma GCC diagnostic push // #pragma GCC diagnostic ignored "-Wsign-compare" // diff --git a/dbms/tests/perf_drafts/accurate_comparisons/accurate_comparisons.sh b/dbms/tests/perf_drafts/accurate_comparisons/accurate_comparisons.sh new file mode 100755 index 00000000000..6600fe43afa --- /dev/null +++ b/dbms/tests/perf_drafts/accurate_comparisons/accurate_comparisons.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e + +clickhouse-client -q "DROP TABLE IF EXISTS test.comparisons" +clickhouse-client -q "CREATE TABLE test.comparisons (i64 Int64, u64 UInt64, f64 Float64) ENGINE = Memory" +clickhouse-client -q "INSERT INTO test.comparisons SELECT toInt64(rand64()) + number AS i64, number AS u64, reinterpretAsFloat64(reinterpretAsString(rand64())) AS f64 FROM system.numbers LIMIT 90000000" + +function test_cmp { + echo -n "$1 : " + echo -n $(clickhouse-client --max_threads=1 --time -q "SELECT sum(ignore($i)) FROM test.comparisons" 1>/dev/null) +} + +test_cmp "u64 = i64" +test_cmp "u64 >= i64" + +test_cmp "i64 > -1 " +test_cmp "i64 = 0 " +test_cmp "u64 != 0 " + +test_cmp "i64 = f64" +test_cmp "i64 < f64" + +test_cmp "f64 >= 0 " + +clickhouse-client -q "DROP TABLE IF EXISTS test.comparisons" \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/00411_accurate_number_comparison.reference b/dbms/tests/queries/0_stateless/00411_accurate_number_comparison.reference new file mode 100644 index 00000000000..53cdf1e9393 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00411_accurate_number_comparison.reference @@ -0,0 +1 @@ +PASSED diff --git a/dbms/tests/queries/0_stateless/00411_accurate_number_comparison.sh b/dbms/tests/queries/0_stateless/00411_accurate_number_comparison.sh new file mode 100755 index 00000000000..59d5ebcdd7e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00411_accurate_number_comparison.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env python +from __future__ import print_function +import os, itertools, urllib + +def get_ch_answer(query): + return urllib.urlopen('http://127.0.0.1:8123', data=query).read() + +def check_answers(query, answer): + ch_answer = get_ch_answer(query) + if ch_answer.strip() != answer.strip(): + print("FAIL on query:", query) + print("Expected answer:", answer) + print("Fetched answer :", ch_answer) + exit(-1) + +def get_values(): + values = [0, 1, -1] + for bits in [8, 16, 32, 64]: + values += [2**bits, 2**bits - 1] + values += [2**(bits-1) - 1, 2**(bits-1), 2**(bits-1) + 1] + values += [-2**(bits-1) - 1, -2**(bits-1), -2**(bits-1) + 1] + return values + +def is_valid_integer(x): + return -2**63 <= x and x <= 2**64-1 + + +TEST_WITH_CASTING=True +GENERATE_TEST_FILES=False + +TYPES = { + "UInt8" : { "bits" : 8, "sign" : False, "float" : False }, + "Int8" : { "bits" : 8, "sign" : True, "float" : False }, + + "UInt16": { "bits" : 16, "sign" : False, "float" : False }, + "Int16" : { "bits" : 16, "sign" : True, "float" : False }, + + "UInt32": { "bits" : 32, "sign" : False, "float" : False }, + "Int32" : { "bits" : 32, "sign" : True, "float" : False }, + + "UInt64": { "bits" : 64, "sign" : False, "float" : False }, + "Int64" : { "bits" : 64, "sign" : True, "float" : False } + + #"Float32" : { "bits" : 32, "sign" : True, "float" : True }, + #"Float64" : { "bits" : 64, "sign" : True, "float" : True } +} + + +def inside_range(value, type_name): + bits = TYPES[type_name]["bits"] + signed = TYPES[type_name]["sign"] + is_float = TYPES[type_name]["float"] + + if is_float: + return True + + if signed: + return -2**(bits-1) <= value and value <= 2**(bits-1) - 1 + else: + return 0 <= value and value <= 2**bits - 1 + + +def test_operators(v1, v2, v1_passed, v2_passed): + query_str = "{v1} = {v2}, {v1} != {v2}, {v1} < {v2}, {v1} <= {v2}, {v1} > {v2}, {v1} >= {v2},\t".format(v1=v1_passed, v2=v2_passed) + query_str += "{v1} = {v2}, {v1} != {v2}, {v1} < {v2}, {v1} <= {v2}, {v1} > {v2}, {v1} >= {v2} ".format(v1=v2_passed, v2=v1_passed) + + answers = [v1 == v2, v1 != v2, v1 < v2, v1 <= v2, v1 > v2, v1 >= v2] + answers += [v2 == v1, v2 != v1, v2 < v1, v2 <= v1, v2 > v1, v2 >= v1] + + answers_str = "\t".join([str(int(x)) for x in answers]) + + return (query_str, answers_str) + + +VALUES = [x for x in get_values() if is_valid_integer(x)] + +def test_pair(v1, v2): + query = "SELECT {}, {}, ".format(v1, v2) + answers = "{}\t{}\t".format(v1, v2) + + q, a = test_operators(v1, v2, str(v1), str(v2)) + query += q + answers += a + + if TEST_WITH_CASTING: + for t1 in TYPES.iterkeys(): + if inside_range(v1, t1): + for t2 in TYPES.iterkeys(): + if inside_range(v2, t2): + q, a = test_operators(v1, v2, 'to{}({})'.format(t1, v1), 'to{}({})'.format(t2, v2)) + query += ', ' + q + answers += "\t" + a + + check_answers(query, answers) + return query, answers + + +VALUES_INT = [0, -1, 1, 2**64-1, 2**63, -2**63, 2**63-1, 2**51, 2**52, 2**53-1, 2**53, 2**53+1, 2**53+2, -2**53+1, -2**53, -2**53-1, -2**53-2, 2*52, -2**52] +VALUES_FLOAT = [float(x) for x in VALUES_INT + [-0.5, 0.5, -1.5, 1.5, 2**53, 2**51 - 0.5, 2**51 + 0.5, 2**60, -2**60, -2**63 - 10000, 2**63 + 10000]] + +def test_float_pair(i, f): + f_str = ("%.9f" % f) + query = "SELECT '{}', '{}', ".format(i, f_str) + answers = "{}\t{}\t".format(i, f_str) + + q, a = test_operators(i, f, i, f_str) + query += q + answers += a + + if TEST_WITH_CASTING: + for t1 in TYPES.iterkeys(): + if inside_range(i, t1): + q, a = test_operators(i, f, 'to{}({})'.format(t1, i), f_str) + query += ', ' + q + answers += "\t" + a + + check_answers(query, answers) + return query, answers + + +def main(): + if GENERATE_TEST_FILES: + base_name = '00411_accurate_number_comparison' + sql_file = open(base_name + '.sql', 'wt') + ref_file = open(base_name + '.reference', 'wt') + + for (v1, v2) in itertools.combinations(VALUES, 2): + q, a = test_pair(v1, v2) + if GENERATE_TEST_FILES: + sql_file.write(q + ";\n") + ref_file.write(a + "\n") + + for (i, f) in itertools.product(VALUES_INT, VALUES_FLOAT): + q, a = test_float_pair(i, f) + if GENERATE_TEST_FILES: + sql_file.write(q + ";\n") + ref_file.write(a + "\n") + + print("PASSED") + + +if __name__ == "__main__": + main() From 9dd3bc3b60eebcac971d49fa62c9b87c094e1543 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Wed, 11 Jan 2017 18:31:54 +0300 Subject: [PATCH 4/5] Removed old code. [#CLICKHOUSE-194] --- dbms/include/DB/Functions/AccurateComparison.h | 13 +------------ dbms/include/DB/Functions/FunctionsComparison.h | 16 ---------------- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/dbms/include/DB/Functions/AccurateComparison.h b/dbms/include/DB/Functions/AccurateComparison.h index 48e94f0186b..4871a667f1d 100644 --- a/dbms/include/DB/Functions/AccurateComparison.h +++ b/dbms/include/DB/Functions/AccurateComparison.h @@ -32,17 +32,6 @@ using bool_if_safe_convervsion = std::enable_if_t::val template using bool_if_not_safe_convervsion = std::enable_if_t::value, bool>; -// template -// bool_if_safe_convervsion greaterOpTmpl(A a, B b) -// { -// return a > b; -// } -// -// template -// bool_if_safe_convervsion equalsOpTmpl(A a, B b) -// { -// return a == b; -// } /// Case 2. Are params IntXX and UIntYY ? template @@ -162,7 +151,7 @@ inline bool_if_safe_convervsion greaterOp(A a, B b) } // Case 3b. 64-bit integers vs floats comparison. -// See hint at https://github.com/JuliaLang/julia/issues/257 +// See hint at https://github.com/JuliaLang/julia/issues/257 (but it doesn't work properly for -2**63) constexpr DB::Int64 MAX_INT64_WITH_EXACT_FLOAT64_REPR = 9007199254740992LL; // 2^53 diff --git a/dbms/include/DB/Functions/FunctionsComparison.h b/dbms/include/DB/Functions/FunctionsComparison.h index 6e56b5b9db0..9cc3310b2f0 100644 --- a/dbms/include/DB/Functions/FunctionsComparison.h +++ b/dbms/include/DB/Functions/FunctionsComparison.h @@ -51,22 +51,6 @@ template struct GreaterOp { static UInt8 apply(A a, template struct LessOrEqualsOp { static UInt8 apply(A a, B b) { return accurate::lessOrEqualsOp(a, b); } }; template struct GreaterOrEqualsOp { static UInt8 apply(A a, B b) { return accurate::greaterOrEqualsOp(a, b); } }; -// /** Игнорируем warning о сравнении signed и unsigned. -// * (Результат может быть некорректным.) -// */ -// #pragma GCC diagnostic push -// #pragma GCC diagnostic ignored "-Wsign-compare" -// -// template struct EqualsOp { static UInt8 apply(A a, B b) { return a == b; } }; -// template struct NotEqualsOp { static UInt8 apply(A a, B b) { return a != b; } }; -// template struct LessOp { static UInt8 apply(A a, B b) { return a < b; } }; -// template struct GreaterOp { static UInt8 apply(A a, B b) { return a > b; } }; -// template struct LessOrEqualsOp { static UInt8 apply(A a, B b) { return a <= b; } }; -// template struct GreaterOrEqualsOp { static UInt8 apply(A a, B b) { return a >= b; } }; -// -// #pragma GCC diagnostic pop - - template struct NumComparisonImpl From 0807fb6d86117da9a06b4394486cc261dc74ca7d Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Wed, 11 Jan 2017 19:29:27 +0300 Subject: [PATCH 5/5] Interpret .py files in test dir as tests. [#CLICKHOUSE-194] More precise performance test. --- dbms/tests/clickhouse-test | 2 +- .../accurate_comparisons/accurate_comparisons.sh | 6 ++++-- ...er_comparison.sh => 00411_accurate_number_comparison.py} | 0 3 files changed, 5 insertions(+), 3 deletions(-) rename dbms/tests/queries/0_stateless/{00411_accurate_number_comparison.sh => 00411_accurate_number_comparison.py} (100%) diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test index 024961d2e59..d45410c889a 100755 --- a/dbms/tests/clickhouse-test +++ b/dbms/tests/clickhouse-test @@ -88,7 +88,7 @@ def main(args): break case_file = os.path.join(suite_dir, case) - if os.path.isfile(case_file) and (case.endswith('.sh') or case.endswith('.sql')): + if os.path.isfile(case_file) and (case.endswith('.sh') or case.endswith('.py') or case.endswith('.sql')): (name, ext) = os.path.splitext(case) report_testcase = et.Element("testcase", attrib = {"name": name}) diff --git a/dbms/tests/perf_drafts/accurate_comparisons/accurate_comparisons.sh b/dbms/tests/perf_drafts/accurate_comparisons/accurate_comparisons.sh index 6600fe43afa..b4c1b4930bd 100755 --- a/dbms/tests/perf_drafts/accurate_comparisons/accurate_comparisons.sh +++ b/dbms/tests/perf_drafts/accurate_comparisons/accurate_comparisons.sh @@ -7,7 +7,9 @@ clickhouse-client -q "INSERT INTO test.comparisons SELECT toInt64(rand64()) + nu function test_cmp { echo -n "$1 : " - echo -n $(clickhouse-client --max_threads=1 --time -q "SELECT sum(ignore($i)) FROM test.comparisons" 1>/dev/null) + echo "SELECT count() FROM test.comparisons WHERE ($1)" | clickhouse-benchmark --max_threads=1 -i 20 -d 0 --json test.json 1>&2 2>/dev/null + python2 -c "import json; print json.load(open('test.json'))['query_time_percentiles']['0']" + rm test.json } test_cmp "u64 = i64" @@ -22,4 +24,4 @@ test_cmp "i64 < f64" test_cmp "f64 >= 0 " -clickhouse-client -q "DROP TABLE IF EXISTS test.comparisons" \ No newline at end of file +clickhouse-client -q "DROP TABLE IF EXISTS test.comparisons" diff --git a/dbms/tests/queries/0_stateless/00411_accurate_number_comparison.sh b/dbms/tests/queries/0_stateless/00411_accurate_number_comparison.py similarity index 100% rename from dbms/tests/queries/0_stateless/00411_accurate_number_comparison.sh rename to dbms/tests/queries/0_stateless/00411_accurate_number_comparison.py