From 16404c8e7b65734922d5475a5cf5bbe08928f456 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Tue, 28 Aug 2018 12:23:29 +0800 Subject: [PATCH] ISSUES-2973 support nested json struct for visitParamExtractRaw --- dbms/src/Functions/FunctionsVisitParam.h | 70 ++++++------------- .../visit_param_extract_raw.xml | 31 ++++++++ ..._functions_for_working_with_json.reference | 13 ++++ .../00539_functions_for_working_with_json.sql | 15 ++++ .../00539_visit_param_bad_values.reference | 4 -- .../00539_visit_param_bad_values.sql | 4 -- 6 files changed, 82 insertions(+), 55 deletions(-) create mode 100644 dbms/tests/performance/functions_json/visit_param_extract_raw.xml create mode 100644 dbms/tests/queries/0_stateless/00539_functions_for_working_with_json.reference create mode 100644 dbms/tests/queries/0_stateless/00539_functions_for_working_with_json.sql delete mode 100644 dbms/tests/queries/0_stateless/00539_visit_param_bad_values.reference delete mode 100644 dbms/tests/queries/0_stateless/00539_visit_param_bad_values.sql diff --git a/dbms/src/Functions/FunctionsVisitParam.h b/dbms/src/Functions/FunctionsVisitParam.h index 2ebe185d9a7..8567f057a62 100644 --- a/dbms/src/Functions/FunctionsVisitParam.h +++ b/dbms/src/Functions/FunctionsVisitParam.h @@ -87,61 +87,37 @@ struct ExtractBool struct ExtractRaw { + inline static void skipAfterQuotationIfNeed(const UInt8 ** pos, const UInt8 * end, ColumnString::Chars_t & res_data) + { + if (*pos + 1 < end && *(*pos + 1) == '"') + { + res_data.push_back(**pos); + ++(*pos); + } + } + static void extract(const UInt8 * pos, const UInt8 * end, ColumnString::Chars_t & res_data) { - if (pos == end) - return; + std::vector expect_end; - UInt8 open_char = *pos; - UInt8 close_char = 0; - switch (open_char) + for (; pos != end; ++pos) { - case '[': - close_char = ']'; - break; - case '{': - close_char = '}'; - break; - case '"': - close_char = '"'; - break; - } - - if (close_char != 0) - { - size_t balance = 1; - char last_char = 0; - - res_data.push_back(*pos); - - ++pos; - for (; pos != end && balance > 0; ++pos) + if (!expect_end.empty() && *pos == expect_end.back()) + expect_end.pop_back(); + else { - res_data.push_back(*pos); - - if (open_char == '"' && *pos == '"') + switch(*pos) { - if (last_char != '\\') - break; + case '[' : expect_end.push_back(']'); break; + case '{' : expect_end.push_back('}'); break; + case '"' : expect_end.push_back('"'); break; + case '\\' : skipAfterQuotationIfNeed(&pos, end, res_data); break; + default : + if (expect_end.empty() && (*pos == ',' || *pos == '}')) + return; } - else - { - if (*pos == open_char) - ++balance; - if (*pos == close_char) - --balance; - } - - if (last_char == '\\') - last_char = 0; - else - last_char = *pos; } - } - else - { - for (; pos != end && *pos != ',' && *pos != '}'; ++pos) - res_data.push_back(*pos); + res_data.push_back(*pos); } } }; diff --git a/dbms/tests/performance/functions_json/visit_param_extract_raw.xml b/dbms/tests/performance/functions_json/visit_param_extract_raw.xml new file mode 100644 index 00000000000..c697373a72c --- /dev/null +++ b/dbms/tests/performance/functions_json/visit_param_extract_raw.xml @@ -0,0 +1,31 @@ + + visit_param_extract_raw + once + + + + 10000 + 1000 + + + + + + + + + + + + + param + + '{"myparam":"test_string"}' + '{"myparam":{"nested_1":"test_string","nested_2":"test_2"}}' + '{"myparam":{"nested_1":"test_string","nested_2":"test_2","nested_1":"test_string","nested_2":"test_2","nested_1":"test_string","nested_2":"test_2"}}' + + + + + SELECT count() FROM system.numbers WHERE NOT ignore(visitParamExtractRaw({param}, 'myparam')) + diff --git a/dbms/tests/queries/0_stateless/00539_functions_for_working_with_json.reference b/dbms/tests/queries/0_stateless/00539_functions_for_working_with_json.reference new file mode 100644 index 00000000000..1b9bc7d15fe --- /dev/null +++ b/dbms/tests/queries/0_stateless/00539_functions_for_working_with_json.reference @@ -0,0 +1,13 @@ +-1 +0 +0 +-1 +1 +test_string +test"string +"test_string" + "test_string" + "test\\"string" + "test\\"string" + [1,2,3] + {"nested" : [1,2,3]} diff --git a/dbms/tests/queries/0_stateless/00539_functions_for_working_with_json.sql b/dbms/tests/queries/0_stateless/00539_functions_for_working_with_json.sql new file mode 100644 index 00000000000..14d4fa3faf2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00539_functions_for_working_with_json.sql @@ -0,0 +1,15 @@ +-- VisitParam with basic type +SELECT visitParamExtractInt('{"myparam":-1}', 'myparam'); +SELECT visitParamExtractUInt('{"myparam":-1}', 'myparam'); +SELECT visitParamExtractFloat('{"myparam":null}', 'myparam'); +SELECT visitParamExtractFloat('{"myparam":-1}', 'myparam'); +SELECT visitParamExtractBool('{"myparam":true}', 'myparam'); +SELECT visitParamExtractString('{"myparam":"test_string"}', 'myparam'); +SELECT visitParamExtractString('{"myparam":"test\\"string"}', 'myparam'); +-- VisitParam with complex type +SELECT visitParamExtractRaw('{"myparam":"test_string"}', 'myparam'); +SELECT visitParamExtractRaw('{"myparam": "test_string"}', 'myparam'); +SELECT visitParamExtractRaw('{"myparam": "test\\"string"}', 'myparam'); +SELECT visitParamExtractRaw('{"myparam": "test\\"string", "other":123}', 'myparam'); +SELECT visitParamExtractRaw('{"myparam": [1,2,3], "other":123}', 'myparam'); +SELECT visitParamExtractRaw('{"myparam": {"nested" : [1,2,3]}, "other":123}', 'myparam'); diff --git a/dbms/tests/queries/0_stateless/00539_visit_param_bad_values.reference b/dbms/tests/queries/0_stateless/00539_visit_param_bad_values.reference deleted file mode 100644 index 9dde4b3b6d5..00000000000 --- a/dbms/tests/queries/0_stateless/00539_visit_param_bad_values.reference +++ /dev/null @@ -1,4 +0,0 @@ --1 -0 -0 --1 diff --git a/dbms/tests/queries/0_stateless/00539_visit_param_bad_values.sql b/dbms/tests/queries/0_stateless/00539_visit_param_bad_values.sql deleted file mode 100644 index 17cd44c6e56..00000000000 --- a/dbms/tests/queries/0_stateless/00539_visit_param_bad_values.sql +++ /dev/null @@ -1,4 +0,0 @@ -select visitParamExtractInt('{"myparam":-1}', 'myparam') AS res; -select visitParamExtractUInt('{"myparam":-1}', 'myparam') AS res; -select visitParamExtractFloat('{"myparam":null}', 'myparam') AS res; -select visitParamExtractFloat('{"myparam":-1}', 'myparam') AS res;