Merge pull request #47494 from KevinyhZou/enable_return_null_element_not_exist

Enable return null and complext type for JSON_VALUE function

docs/en/operations/settings/settings.md

@@ -4071,3 +4071,44 @@ SELECT sum(number) FROM numbers(10000000000) SETTINGS partial_result_on_first_ca
 Possible values: `true`, `false`
 
 Default value: `false`
+## function_json_value_return_type_allow_nullable
+
+Control whether allow to return `NULL` when value is not exist for JSON_VALUE function.
+
+```sql
+SELECT JSON_VALUE('{"hello":"world"}', '$.b') settings function_json_value_return_type_allow_nullable=true;
+
+┌─JSON_VALUE('{"hello":"world"}', '$.b')─┐
+│ ᴺᵁᴸᴸ                                   │
+└────────────────────────────────────────┘
+
+1 row in set. Elapsed: 0.001 sec.
+```
+
+Possible values:
+
+-   true — Allow.
+-   false — Disallow.
+
+Default value: `false`.
+
+## function_json_value_return_type_allow_complex
+
+Control whether allow to return complex type (such as: struct, array, map) for json_value function.
+
+```sql
+SELECT JSON_VALUE('{"hello":{"world":"!"}}', '$.hello') settings function_json_value_return_type_allow_complex=true
+
+┌─JSON_VALUE('{"hello":{"world":"!"}}', '$.hello')─┐
+│ {"world":"!"}                                    │
+└──────────────────────────────────────────────────┘
+
+1 row in set. Elapsed: 0.001 sec.
+```
+
+Possible values:
+
+-   true — Allow.
+-   false — Disallow.
+
+Default value: `false`.

docs/en/sql-reference/functions/json-functions.md

@@ -401,7 +401,7 @@ Before version 21.11 the order of arguments was wrong, i.e. JSON_QUERY(path, jso
 
 Parses a JSON and extract a value as JSON scalar.
 
-If the value does not exist, an empty string will be returned.
+If the value does not exist, an empty string will be returned by default, and by SET `function_return_type_allow_nullable` = `true`, `NULL` will be returned. If the value is complex type (such as: struct, array, map), an empty string will be returned by default, and by SET `function_json_value_return_type_allow_complex` = `true`, the complex value will be returned.
 
 Example:
 
@@ -410,6 +410,8 @@ SELECT JSON_VALUE('{"hello":"world"}', '$.hello');
 SELECT JSON_VALUE('{"array":[[0, 1, 2, 3, 4, 5], [0, -1, -2, -3, -4, -5]]}', '$.array[*][0 to 2, 4]');
 SELECT JSON_VALUE('{"hello":2}', '$.hello');
 SELECT toTypeName(JSON_VALUE('{"hello":2}', '$.hello'));
+select JSON_VALUE('{"hello":"world"}', '$.b') settings function_return_type_allow_nullable=true;
+select JSON_VALUE('{"hello":{"world":"!"}}', '$.hello') settings function_json_value_return_type_allow_complex=true;
 ```
 
 Result:

src/Core/Settings.h

@@ -724,6 +724,8 @@ class IColumn;
     M(Bool, force_aggregation_in_order, false, "Force use of aggregation in order on remote nodes during distributed aggregation. PLEASE, NEVER CHANGE THIS SETTING VALUE MANUALLY!", IMPORTANT) \
     M(UInt64, http_max_request_param_data_size, 10_MiB, "Limit on size of request data used as a query parameter in predefined HTTP requests.", 0) \
     M(Bool, allow_experimental_undrop_table_query, false, "Allow to use undrop query to restore dropped table in a limited time", 0) \
+    M(Bool, function_json_value_return_type_allow_nullable, false, "Allow function to return nullable type.", 0) \
+    M(Bool, function_json_value_return_type_allow_complex, false, "Allow function to return complex type, such as: struct, array, map.", 0) \
     // End of COMMON_SETTINGS
     // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS.
 
@@ -947,7 +949,6 @@ class IColumn;
     \
     M(Bool, dictionary_use_async_executor, false, "Execute a pipeline for reading from a dictionary with several threads. It's supported only by DIRECT dictionary with CLICKHOUSE source.", 0) \
 
-
 // End of FORMAT_FACTORY_SETTINGS
 // Please add settings non-related to formats into the COMMON_SETTINGS above.
 

src/Functions/FunctionSQLJSON.h

@@ -3,9 +3,11 @@
 #include <sstream>
 #include <type_traits>
 #include <Columns/ColumnConst.h>
+#include <Columns/ColumnNullable.h>
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnsNumber.h>
 #include <Core/Settings.h>
+#include <DataTypes/DataTypeNullable.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <Common/JSONParsers/DummyJSONParser.h>
@@ -40,7 +42,7 @@ public:
     class Executor
     {
     public:
-        static ColumnPtr run(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, uint32_t parse_depth)
+        static ColumnPtr run(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, uint32_t parse_depth, const ContextPtr & context)
         {
             MutableColumnPtr to{result_type->createColumn()};
             to->reserve(input_rows_count);
@@ -115,7 +117,6 @@ public:
 
             /// Parse JSON for every row
             Impl<JSONParser> impl;
-
             for (const auto i : collections::range(0, input_rows_count))
             {
                 std::string_view json{
@@ -125,7 +126,7 @@ public:
                 bool added_to_column = false;
                 if (document_ok)
                 {
-                    added_to_column = impl.insertResultToColumn(*to, document, res);
+                    added_to_column = impl.insertResultToColumn(*to, document, res, context);
                 }
                 if (!added_to_column)
                 {
@@ -154,7 +155,7 @@ public:
 
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
-        return Impl<DummyJSONParser>::getReturnType(Name::name, arguments);
+        return Impl<DummyJSONParser>::getReturnType(Name::name, arguments, getContext());
     }
 
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
@@ -167,9 +168,9 @@ public:
         unsigned parse_depth = static_cast<unsigned>(getContext()->getSettingsRef().max_parser_depth);
 #if USE_SIMDJSON
         if (getContext()->getSettingsRef().allow_simdjson)
-            return FunctionSQLJSONHelpers::Executor<Name, Impl, SimdJSONParser>::run(arguments, result_type, input_rows_count, parse_depth);
+            return FunctionSQLJSONHelpers::Executor<Name, Impl, SimdJSONParser>::run(arguments, result_type, input_rows_count, parse_depth, getContext());
 #endif
-        return FunctionSQLJSONHelpers::Executor<Name, Impl, DummyJSONParser>::run(arguments, result_type, input_rows_count, parse_depth);
+        return FunctionSQLJSONHelpers::Executor<Name, Impl, DummyJSONParser>::run(arguments, result_type, input_rows_count, parse_depth, getContext());
     }
 };
 
@@ -194,11 +195,11 @@ class JSONExistsImpl
 public:
     using Element = typename JSONParser::Element;
 
-    static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) { return std::make_shared<DataTypeUInt8>(); }
+    static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &, const ContextPtr &) { return std::make_shared<DataTypeUInt8>(); }
 
     static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
 
-    static bool insertResultToColumn(IColumn & dest, const Element & root, ASTPtr & query_ptr)
+    static bool insertResultToColumn(IColumn & dest, const Element & root, ASTPtr & query_ptr, const ContextPtr &)
     {
         GeneratorJSONPath<JSONParser> generator_json_path(query_ptr);
         Element current_element = root;
@@ -233,11 +234,22 @@ class JSONValueImpl
 public:
     using Element = typename JSONParser::Element;
 
-    static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) { return std::make_shared<DataTypeString>(); }
+    static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &, const ContextPtr & context)
+    {
+        if (context->getSettingsRef().function_json_value_return_type_allow_nullable)
+        {
+            DataTypePtr string_type = std::make_shared<DataTypeString>();
+            return std::make_shared<DataTypeNullable>(string_type);
+        }
+        else
+        {
+            return std::make_shared<DataTypeString>();
+        }
+    }
 
     static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
 
-    static bool insertResultToColumn(IColumn & dest, const Element & root, ASTPtr & query_ptr)
+    static bool insertResultToColumn(IColumn & dest, const Element & root, ASTPtr & query_ptr, const ContextPtr & context)
     {
         GeneratorJSONPath<JSONParser> generator_json_path(query_ptr);
         Element current_element = root;
@@ -247,7 +259,11 @@ public:
         {
             if (status == VisitorStatus::Ok)
             {
-                if (!(current_element.isArray() || current_element.isObject()))
+                if (context->getSettingsRef().function_json_value_return_type_allow_complex)
+                {
+                    break;
+                }
+                else if (!(current_element.isArray() || current_element.isObject()))
                 {
                     break;
                 }
@@ -267,9 +283,19 @@ public:
         std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
         out << current_element.getElement();
         auto output_str = out.str();
-        ColumnString & col_str = assert_cast<ColumnString &>(dest);
-        ColumnString::Chars & data = col_str.getChars();
-        ColumnString::Offsets & offsets = col_str.getOffsets();
+        ColumnString * col_str;
+        if (isColumnNullable(dest))
+        {
+            ColumnNullable & col_null = assert_cast<ColumnNullable &>(dest);
+            col_null.getNullMapData().push_back(0);
+            col_str = assert_cast<ColumnString *>(&col_null.getNestedColumn());
+        }
+        else
+        {
+            col_str = assert_cast<ColumnString *>(&dest);
+        }
+        ColumnString::Chars & data = col_str->getChars();
+        ColumnString::Offsets & offsets = col_str->getOffsets();
 
         if (current_element.isString())
         {
@@ -280,7 +306,7 @@ public:
         }
         else
         {
-            col_str.insertData(output_str.data(), output_str.size());
+            col_str->insertData(output_str.data(), output_str.size());
         }
         return true;
     }
@@ -296,11 +322,11 @@ class JSONQueryImpl
 public:
     using Element = typename JSONParser::Element;
 
-    static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) { return std::make_shared<DataTypeString>(); }
+    static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &, const ContextPtr &) { return std::make_shared<DataTypeString>(); }
 
     static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; }
 
-    static bool insertResultToColumn(IColumn & dest, const Element & root, ASTPtr & query_ptr)
+    static bool insertResultToColumn(IColumn & dest, const Element & root, ASTPtr & query_ptr, const ContextPtr &)
     {
         GeneratorJSONPath<JSONParser> generator_json_path(query_ptr);
         Element current_element = root;

tests/queries/0_stateless/01889_sql_json_functions.reference

@@ -31,6 +31,12 @@ select JSON_VALUE('{"a":"\\n\\u0000"}', '$.a');
 \n\0
 select JSON_VALUE('{"a":"\\u263a"}', '$.a');
 ☺
+select JSON_VALUE('{"hello":"world"}', '$.b') settings function_json_value_return_type_allow_nullable=true;
+\N
+select JSON_VALUE('{"hello":{"world":"!"}}', '$.hello') settings function_json_value_return_type_allow_complex=true;
+{"world":"!"}
+SELECT JSON_VALUE('{"hello":["world","world2"]}', '$.hello') settings function_json_value_return_type_allow_complex=true;
+["world","world2"]
 SELECT '--JSON_QUERY--';
 --JSON_QUERY--
 SELECT JSON_QUERY('{"hello":1}', '$');

tests/queries/0_stateless/01889_sql_json_functions.sql

@@ -17,6 +17,9 @@ SELECT JSON_VALUE('{"hello":"\\uD83C\\uDF3A \\uD83C\\uDF38 \\uD83C\\uDF37 Hello,
 SELECT JSON_VALUE('{"a":"Hello \\"World\\" \\\\"}', '$.a');
 select JSON_VALUE('{"a":"\\n\\u0000"}', '$.a');
 select JSON_VALUE('{"a":"\\u263a"}', '$.a');
+select JSON_VALUE('{"hello":"world"}', '$.b') settings function_json_value_return_type_allow_nullable=true;
+select JSON_VALUE('{"hello":{"world":"!"}}', '$.hello') settings function_json_value_return_type_allow_complex=true;
+SELECT JSON_VALUE('{"hello":["world","world2"]}', '$.hello') settings function_json_value_return_type_allow_complex=true;
 
 SELECT '--JSON_QUERY--';
 SELECT JSON_QUERY('{"hello":1}', '$');