From 65c6605c2ce422dc24505e66783ea2ed7c500466 Mon Sep 17 00:00:00 2001 From: Vitaly Orlov Date: Tue, 12 Oct 2021 18:00:25 +0300 Subject: [PATCH 1/4] Add JSONExtractKeys function --- src/Functions/FunctionsJSON.cpp | 1 + src/Functions/FunctionsJSON.h | 35 +++++++++++++++++++ tests/fuzz/all.dict | 1 + tests/fuzz/dictionaries/functions.dict | 1 + .../00918_json_functions.reference | 5 +++ .../0_stateless/00918_json_functions.sql | 6 ++++ 6 files changed, 49 insertions(+) diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index 47f485e9d6c..7fec45d1f72 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -73,6 +73,7 @@ void registerFunctionsJSON(FunctionFactory & factory) factory.registerFunction>(); factory.registerFunction>(); factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index b468f39af80..7407a3ec611 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -433,6 +433,7 @@ struct NameJSONExtractKeysAndValues { static constexpr auto name{"JSONExtractKey struct NameJSONExtractRaw { static constexpr auto name{"JSONExtractRaw"}; }; struct NameJSONExtractArrayRaw { static constexpr auto name{"JSONExtractArrayRaw"}; }; struct NameJSONExtractKeysAndValuesRaw { static constexpr auto name{"JSONExtractKeysAndValuesRaw"}; }; +struct NameJSONExtractKeys { static constexpr auto name{"JSONExtractKeys"}; }; template @@ -1353,4 +1354,38 @@ public: } }; +template +class JSONExtractKeysImpl +{ +public: + using Element = typename JSONParser::Element; + + static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) + { + return std::make_unique(); + } + + static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } + + bool insertResultToColumn(IColumn & dest, const Element & element, const std::string_view &) + { + if (!element.isObject()) + return false; + + auto object = element.getObject(); + + auto & col_arr = assert_cast(dest); + auto & col_tuple = assert_cast(col_arr.getData()); + auto & col_key = assert_cast(col_tuple.getColumn(0)); + + for (auto [key, value] : object) + { + col_key.insertData(key.data(), key.size()); + } + + col_arr.getOffsets().push_back(col_arr.getOffsets().back() + object.size()); + return true; + } +}; + } diff --git a/tests/fuzz/all.dict b/tests/fuzz/all.dict index 73c86a13714..4a9afc348cf 100644 --- a/tests/fuzz/all.dict +++ b/tests/fuzz/all.dict @@ -650,6 +650,7 @@ "JSONExtractInt" "JSONExtractKeysAndValues" "JSONExtractKeysAndValuesRaw" +"JSONExtractKeys" "JSONExtractRaw" "JSONExtractString" "JSONExtractUInt" diff --git a/tests/fuzz/dictionaries/functions.dict b/tests/fuzz/dictionaries/functions.dict index ba9daa72199..fb35375f284 100644 --- a/tests/fuzz/dictionaries/functions.dict +++ b/tests/fuzz/dictionaries/functions.dict @@ -31,6 +31,7 @@ "toJSONString" "JSON_VALUE" "JSONExtractKeysAndValuesRaw" +"JSONExtractKeys" "JSONExtractString" "JSONType" "JSONKey" diff --git a/tests/queries/0_stateless/00918_json_functions.reference b/tests/queries/0_stateless/00918_json_functions.reference index 462449222b3..4c8977114e6 100644 --- a/tests/queries/0_stateless/00918_json_functions.reference +++ b/tests/queries/0_stateless/00918_json_functions.reference @@ -101,6 +101,11 @@ hello [('a','"hello"'),('b','[-100,200,300]')] [('a','"hello"'),('b','[-100,200,300]'),('c','{"d":[121,144]}')] [('d','[121,144]')] +--JSONExtractKeys-- +['a','b'] +[] +[] +['d'] --const/non-const mixed-- a b diff --git a/tests/queries/0_stateless/00918_json_functions.sql b/tests/queries/0_stateless/00918_json_functions.sql index 6504b735371..bce84df08bf 100644 --- a/tests/queries/0_stateless/00918_json_functions.sql +++ b/tests/queries/0_stateless/00918_json_functions.sql @@ -231,6 +231,12 @@ SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300]}'); SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300], "c":{"d":[121,144]}}'); SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300], "c":{"d":[121,144]}}', 'c'); +SELECT '--JSONExtractKeys--'; +SELECT JSONExtractKeys('{"a": "hello", "b": [-100, 200.0, 300]}'); +SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b'); +SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'a'); +SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300], "c":{"d":[121,144]}}', 'c'); + SELECT '--const/non-const mixed--'; SELECT JSONExtractString('["a", "b", "c", "d", "e"]', idx) FROM (SELECT arrayJoin([1,2,3,4,5]) AS idx); SELECT JSONExtractString(json, 's') FROM (SELECT arrayJoin(['{"s":"u"}', '{"s":"v"}']) AS json); From 4d2b34a7f0d346710172d71fc168cd0eb1edac17 Mon Sep 17 00:00:00 2001 From: Vitaly Orlov Date: Wed, 13 Oct 2021 00:08:00 +0300 Subject: [PATCH 2/4] Review fixes --- src/Functions/FunctionsJSON.h | 2 +- tests/queries/0_stateless/00918_json_functions.sql | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 7407a3ec611..4f91440105d 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -1378,7 +1378,7 @@ public: auto & col_tuple = assert_cast(col_arr.getData()); auto & col_key = assert_cast(col_tuple.getColumn(0)); - for (auto [key, value] : object) + for (const auto & [key, value] : object) { col_key.insertData(key.data(), key.size()); } diff --git a/tests/queries/0_stateless/00918_json_functions.sql b/tests/queries/0_stateless/00918_json_functions.sql index bce84df08bf..87682587c8e 100644 --- a/tests/queries/0_stateless/00918_json_functions.sql +++ b/tests/queries/0_stateless/00918_json_functions.sql @@ -233,9 +233,9 @@ SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300], "c": SELECT '--JSONExtractKeys--'; SELECT JSONExtractKeys('{"a": "hello", "b": [-100, 200.0, 300]}'); -SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b'); -SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'a'); -SELECT JSONExtractKeysAndValuesRaw('{"a": "hello", "b": [-100, 200.0, 300], "c":{"d":[121,144]}}', 'c'); +SELECT JSONExtractKeys('{"a": "hello", "b": [-100, 200.0, 300]}', 'b'); +SELECT JSONExtractKeys('{"a": "hello", "b": [-100, 200.0, 300]}', 'a'); +SELECT JSONExtractKeys('{"a": "hello", "b": [-100, 200.0, 300], "c":{"d":[121,144]}}', 'c'); SELECT '--const/non-const mixed--'; SELECT JSONExtractString('["a", "b", "c", "d", "e"]', idx) FROM (SELECT arrayJoin([1,2,3,4,5]) AS idx); From 1623bfa1a27a8175f964197194dd9d3d9bf7c84c Mon Sep 17 00:00:00 2001 From: Vitaly Orlov Date: Wed, 13 Oct 2021 07:33:53 +0300 Subject: [PATCH 3/4] review fixes --- src/Functions/FunctionsJSON.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 4f91440105d..4fd4e6b2d00 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -1362,7 +1362,7 @@ public: static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) { - return std::make_unique(); + return std::make_unique(std::make_shared()); } static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } @@ -1374,16 +1374,15 @@ public: auto object = element.getObject(); - auto & col_arr = assert_cast(dest); - auto & col_tuple = assert_cast(col_arr.getData()); - auto & col_key = assert_cast(col_tuple.getColumn(0)); + ColumnArray & col_res = assert_cast(dest); + auto & col_key = assert_cast(col_res.getData()); for (const auto & [key, value] : object) { col_key.insertData(key.data(), key.size()); } - col_arr.getOffsets().push_back(col_arr.getOffsets().back() + object.size()); + col_res.getOffsets().push_back(col_res.getOffsets().back() + object.size()); return true; } }; From 573076588de1231f1cbc53f4718b6791027cefe8 Mon Sep 17 00:00:00 2001 From: Vitaly Orlov Date: Wed, 13 Oct 2021 16:04:08 +0300 Subject: [PATCH 4/4] Fix reference --- .../queries/0_stateless/00918_json_functions.reference | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/00918_json_functions.reference b/tests/queries/0_stateless/00918_json_functions.reference index 4c8977114e6..8e6fc3914e0 100644 --- a/tests/queries/0_stateless/00918_json_functions.reference +++ b/tests/queries/0_stateless/00918_json_functions.reference @@ -101,11 +101,6 @@ hello [('a','"hello"'),('b','[-100,200,300]')] [('a','"hello"'),('b','[-100,200,300]'),('c','{"d":[121,144]}')] [('d','[121,144]')] ---JSONExtractKeys-- -['a','b'] -[] -[] -['d'] --const/non-const mixed-- a b @@ -208,6 +203,11 @@ Friday [('a','"hello"'),('b','[-100,200,300]')] [('a','"hello"'),('b','[-100,200,300]'),('c','{"d":[121,144]}')] [('d','[121,144]')] +--JSONExtractKeys-- +['a','b'] +[] +[] +['d'] --const/non-const mixed-- a b