From 2d6d05386a6d403f7ea720168252e53545d2b180 Mon Sep 17 00:00:00 2001 From: "d.v.semenov" Date: Wed, 24 Mar 2021 22:47:28 +0300 Subject: [PATCH 01/42] Just Works Just works (remastered) First steps First steps fixed First steps first fails Research first steps Tokenizer created Sprint to the moon Rename Rename 2.0 Rename 3.0 Work in progress Update Oops Oops x2 Try this Now surely works Maybe now? Now? Cmake first try Restore to previous Cmake second try Make this work Correct mistakes Third try cmake Exclude simd Better Try Add std::cerr More std::cerr More and more std::cerr Maybe fix? A B C D E F G H I J K L M N O P AA AB AC AD AE AF AAA AAB AAC AAD AAF AAE AAF AAG AAH AAI AAJ AAK AAAA AAAB AAAC AAAD AAAE AAAF AAAG AAAH AAAAA AAAAB First try v2 First try v2.1 First try v2.2 First try v2.3 First try v2.4 First try v2.5 First try v2.6 First try v2.7 First try v2.8 First try v2.9 First try v2.10 First try v2.11 First try v2.12 First try v2.13 First try v2.14 First try v2.15 First try v2.16 First try v2.16 First try v2.17 First try v2.18 First try v2.19 First try v2.20 First try v2.21 First try v2.22 First try v2.23 First try v2.24 First try v2.25 First try v2.26 First try v2.27 First try v2.28 First try v2.29 First try v2.30 First try v2.31 First try v2.32 First try v2.33 First try v2.34 First try v2.35 First try v2.36 First try v2.37 Second try v2.00 Second try v2.01 Second try v2.02 Second try v2.03 Second try v2.04 Second try v2.05 Second try v2.06 Second try v2.07 Second try v2.08 Second try v2.09 Second try v2.10 Second try v2.11 Second try v2.12 Second try v2.13 Second try v2.14 Second try v2.15 Second try v2.16 Second try v2.17 Cleanup Link SQLJSON only in simdjson build Fix? Fix?1.1 Fix Revert "Fix" This reverts commit 9df7aa977c880ec130062bceece7e215190b4837. Revert "Fix?1.1" This reverts commit 37429ecc9003fd73c106344186e39ff6603dde6c. Revert "Fix?" This reverts commit c1236fb8f4b5a799a5564aecf81136301f226e33. Revert "Link SQLJSON only in simdjson build" This reverts commit 8795cd8b143f3cfd312ddbf1b98e10d0d6fcaf51. Revert "Cleanup" This reverts commit e100dbc545f54421276be2e5d44f99f52fe1d87c. Third try v2.0 Third try v2.1 Third try v2.2 Third try v2.3 Third try v2.4 Third try v2.5 Third try v2.6 Third try v2.7 Third try v2.8 Third try v2.9 Third try v2.10 Third try v2.11 Third try v2.12 Third try v2.13 Third try v2.14 Third try v2.15 Pre-intermediate touches v1.0 Pre-intermediate touches v1.1 Pre-intermediate touches v1.2 Pre-intermediate touches v1.3 Last changes --- src/Functions/CMakeLists.txt | 2 + src/Functions/FunctionSQLJSON.cpp | 19 ++ src/Functions/FunctionSQLJSON.h | 277 ++++++++++++++++++ src/Functions/FunctionsJSON.h | 2 - src/Functions/JSONPath/ASTs/ASTJSONPath.h | 26 ++ .../JSONPath/ASTs/ASTJSONPathMemberAccess.h | 25 ++ .../JSONPath/ASTs/ASTJSONPathQuery.h | 23 ++ src/Functions/JSONPath/ASTs/CMakeLists.txt | 8 + src/Functions/JSONPath/CMakeLists.txt | 8 + .../JSONPath/Generators/CMakeLists.txt | 8 + .../JSONPath/Generators/GeneratorJSONPath.h | 98 +++++++ .../JSONPath/Generators/IGenerator.h | 30 ++ .../JSONPath/Generators/IGenerator_fwd.h | 16 + src/Functions/JSONPath/Generators/IVisitor.h | 40 +++ .../Generators/VisitorJSONPathMemberAccess.h | 39 +++ .../JSONPath/Generators/VisitorStatus.h | 11 + src/Functions/JSONPath/Parsers/CMakeLists.txt | 8 + .../JSONPath/Parsers/ParserJSONPath.cpp | 34 +++ .../JSONPath/Parsers/ParserJSONPath.h | 22 ++ .../Parsers/ParserJSONPathMemberAccess.cpp | 42 +++ .../Parsers/ParserJSONPathMemberAccess.h | 12 + .../JSONPath/Parsers/ParserJSONPathQuery.cpp | 39 +++ .../JSONPath/Parsers/ParserJSONPathQuery.h | 17 ++ src/Functions/RapidJSONParser.h | 3 +- src/Functions/SimdJSONParser.h | 64 +++- src/Functions/registerFunctions.cpp | 2 + src/Parsers/Lexer.cpp | 3 + src/Parsers/Lexer.h | 1 + 28 files changed, 860 insertions(+), 19 deletions(-) create mode 100644 src/Functions/FunctionSQLJSON.cpp create mode 100644 src/Functions/FunctionSQLJSON.h create mode 100644 src/Functions/JSONPath/ASTs/ASTJSONPath.h create mode 100644 src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h create mode 100644 src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h create mode 100644 src/Functions/JSONPath/ASTs/CMakeLists.txt create mode 100644 src/Functions/JSONPath/CMakeLists.txt create mode 100644 src/Functions/JSONPath/Generators/CMakeLists.txt create mode 100644 src/Functions/JSONPath/Generators/GeneratorJSONPath.h create mode 100644 src/Functions/JSONPath/Generators/IGenerator.h create mode 100644 src/Functions/JSONPath/Generators/IGenerator_fwd.h create mode 100644 src/Functions/JSONPath/Generators/IVisitor.h create mode 100644 src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h create mode 100644 src/Functions/JSONPath/Generators/VisitorStatus.h create mode 100644 src/Functions/JSONPath/Parsers/CMakeLists.txt create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPath.cpp create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPath.h create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 1c3beb2e47d..24add0f4f0a 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -114,6 +114,8 @@ target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_url) add_subdirectory(array) target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_array) +add_subdirectory(JSONPath) + if (USE_STATS) target_link_libraries(clickhouse_functions PRIVATE stats) endif() diff --git a/src/Functions/FunctionSQLJSON.cpp b/src/Functions/FunctionSQLJSON.cpp new file mode 100644 index 00000000000..ddcca12835f --- /dev/null +++ b/src/Functions/FunctionSQLJSON.cpp @@ -0,0 +1,19 @@ +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + + +void registerFunctionsSQLJSON(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); +} + +} diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h new file mode 100644 index 00000000000..24749099e57 --- /dev/null +++ b/src/Functions/FunctionSQLJSON.h @@ -0,0 +1,277 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +//#include +#include + +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; +} + +class FunctionSQLJSONHelpers +{ +public: + template typename Impl, class JSONParser> + class Executor + { + public: + static ColumnPtr run(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) + { + MutableColumnPtr to{result_type->createColumn()}; + to->reserve(input_rows_count); + + if (arguments.size() < 2) + { + throw Exception{"JSONPath functions require at least 2 arguments", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION}; + } + + /// Check 1 argument: must be of type String (JSONPath) + const auto & first_column = arguments[0]; + if (!isString(first_column.type)) + { + throw Exception{ + "JSONPath functions require 1 argument to be JSONPath of type string, illegal type: " + first_column.type->getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + } + + /// Check 2 argument: must be of type String (JSON) + const auto & second_column = arguments[1]; + if (!isString(second_column.type)) + { + throw Exception{ + "JSONPath functions require 2 argument to be JSON of string, illegal type: " + second_column.type->getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + } + + /// If argument is successfully cast to (ColumnConst *) then it is quoted string + /// Example: + /// SomeFunction('some string argument') + /// + /// Otherwise it is a column + /// Example: + /// SomeFunction(database.table.column) + + /// Check 1 argument: must be const String (JSONPath) + const ColumnPtr & arg_jsonpath = first_column.column; + const auto * arg_jsonpath_const = typeid_cast(arg_jsonpath.get()); + if (!arg_jsonpath_const) + { + throw Exception{"JSONPath argument must be of type const String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + } + /// Retrieve data from 1 argument + const auto * arg_jsonpath_string = typeid_cast(arg_jsonpath_const->getDataColumnPtr().get()); + if (!arg_jsonpath_string) + { + throw Exception{"Illegal column " + arg_jsonpath->getName(), ErrorCodes::ILLEGAL_COLUMN}; + } + + /// Check 2 argument: must be const or non-const String (JSON) + const ColumnPtr & arg_json = second_column.column; + const auto * col_json_const = typeid_cast(arg_json.get()); + const auto * col_json_string + = typeid_cast(col_json_const ? col_json_const->getDataColumnPtr().get() : arg_json.get()); + + /// Get data and offsets for 1 argument (JSONPath) + const ColumnString::Chars & chars_path = arg_jsonpath_string->getChars(); + const ColumnString::Offsets & offsets_path = arg_jsonpath_string->getOffsets(); + + /// Get data and offsets for 1 argument (JSON) + const char * query_begin = reinterpret_cast(&chars_path[0]); + const char * query_end = query_begin + offsets_path[0] - 1; + + /// Tokenize query + Tokens tokens(query_begin, query_end); + /// Max depth 0 indicates that depth is not limited + IParser::Pos token_iterator(tokens, 0); + + /// Parse query and create AST tree + Expected expected; + ASTPtr res; + ParserJSONPath parser; + const bool parse_res = parser.parse(token_iterator, res, expected); + if (!parse_res) + { + throw Exception{"Unable to parse JSONPath", ErrorCodes::BAD_ARGUMENTS}; + } + + /// Get data and offsets for 1 argument (JSON) + const ColumnString::Chars & chars_json = col_json_string->getChars(); + const ColumnString::Offsets & offsets_json = col_json_string->getOffsets(); + + JSONParser json_parser; + using Element = typename JSONParser::Element; + Element document; + bool document_ok = false; + + /// Parse JSON for every row + Impl impl; + for (const auto i : ext::range(0, input_rows_count)) + { + std::string_view json{ + reinterpret_cast(&chars_json[offsets_json[i - 1]]), offsets_json[i] - offsets_json[i - 1] - 1}; + document_ok = json_parser.parse(json, document); + + bool added_to_column = false; + if (document_ok) + { + added_to_column = impl.insertResultToColumn(*to, document, res); + } + if (!added_to_column) + { + to->insertDefault(); + } + } + return to; + } + }; + +private: +}; + +template typename Impl> +class FunctionSQLJSON : public IFunction +{ +public: + static FunctionPtr create(const Context & context_) { return std::make_shared(context_); } + FunctionSQLJSON(const Context & context_) : context(context_) { } + + static constexpr auto name = Name::name; + String getName() const override { return Name::name; } + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + return Impl::getReturnType(Name::name, arguments); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + { + /// Choose JSONParser. + /// 1. Lexer(path) -> Tokens + /// 2. Create ASTPtr + /// 3. Parser(Tokens, ASTPtr) -> complete AST + /// 4. Execute functions, call interpreter for each json (in function) +#if USE_SIMDJSON + if (context.getSettingsRef().allow_simdjson) + return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); +#endif + +#if USE_RAPIDJSON + throw Exception{"RapidJSON is not supported :(", ErrorCodes::BAD_ARGUMENTS}; +#else + return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); +#endif + } + +private: + const Context & context; +}; + +struct NameSQLJSONTest +{ + static constexpr auto name{"SQLJSONTest"}; +}; + +struct NameSQLJSONMemberAccess +{ + static constexpr auto name{"SQLJSONMemberAccess"}; +}; + +/** + * Function to test logic before function calling, will be removed in final PR + * @tparam JSONParser parser + */ +template +class SQLJSONTestImpl +{ +public: + using Element = typename JSONParser::Element; + + static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) { return std::make_shared(); } + + static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } + + static bool insertResultToColumn(IColumn & dest, const Element &, ASTPtr &) + { + String str = "I am working:-)"; + ColumnString & col_str = assert_cast(dest); + col_str.insertData(str.data(), str.size()); + return true; + } +}; + +/** + * Function to test jsonpath member access, will be removed in final PR + * @tparam JSONParser parser + */ +template +class SQLJSONMemberAccessImpl +{ +public: + using Element = typename JSONParser::Element; + + static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) { return std::make_shared(); } + + static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } + + static bool insertResultToColumn(IColumn & dest, const Element & root, ASTPtr & query_ptr) + { + GeneratorJSONPath generator_json_path(query_ptr); + Element current_element = root; + VisitorStatus status; + while ((status = generator_json_path.getNextItem(current_element)) == VisitorStatus::Ok) + { + /// No-op + } + if (status == VisitorStatus::Error) + { + return false; + } + ColumnString & col_str = assert_cast(dest); + std::stringstream ostr; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + ostr << current_element.getElement(); + auto output_str = ostr.str(); + col_str.insertData(output_str.data(), output_str.size()); + return true; + } +}; + +} diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index f066bb1029a..581cc2015aa 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -80,8 +80,6 @@ public: const ColumnString::Chars & chars = col_json_string->getChars(); const ColumnString::Offsets & offsets = col_json_string->getOffsets(); - size_t num_index_arguments = Impl::getNumberOfIndexArguments(arguments); - std::vector moves = prepareMoves(Name::name, arguments, 1, num_index_arguments); /// Preallocate memory in parser if necessary. JSONParser parser; diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPath.h b/src/Functions/JSONPath/ASTs/ASTJSONPath.h new file mode 100644 index 00000000000..cd73cd14257 --- /dev/null +++ b/src/Functions/JSONPath/ASTs/ASTJSONPath.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +namespace DB +{ +class ASTJSONPath : public IAST +{ +public: + String getID(char) const override + { + std::cerr << "in ASTJSONPath: getID\n"; + return "ASTJSONPath"; + } + + ASTPtr clone() const override + { + std::cerr << "in " << "ASTJSONPath" << ": clone\n"; + return std::make_shared(*this); + } + + ASTJSONPathQuery * jsonpath_query; +}; + +} diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h b/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h new file mode 100644 index 00000000000..663859f566f --- /dev/null +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +namespace DB +{ +class ASTJSONPathMemberAccess : public IAST +{ +public: + String getID(char) const override + { + return "ASTJSONPathMemberAccess"; + } + + ASTPtr clone() const override + { + return std::make_shared(*this); + } + +public: + /// Member name to lookup in json document (in path: $.some_key.another_key. ...) + String member_name; +}; + +} diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h b/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h new file mode 100644 index 00000000000..6b952c2519d --- /dev/null +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h @@ -0,0 +1,23 @@ +#pragma once + +#include + +namespace DB +{ +class ASTJSONPathQuery : public IAST +{ +public: + String getID(char) const override + { + std::cerr << "in ASTJSONPathQuery: getID\n"; + return "ASTJSONPathQuery"; + } + + ASTPtr clone() const override + { + std::cerr << "in " << getID(' ') << ": clone\n"; + return std::make_shared(*this); + } +}; + +} diff --git a/src/Functions/JSONPath/ASTs/CMakeLists.txt b/src/Functions/JSONPath/ASTs/CMakeLists.txt new file mode 100644 index 00000000000..c671dbbc001 --- /dev/null +++ b/src/Functions/JSONPath/ASTs/CMakeLists.txt @@ -0,0 +1,8 @@ +include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") +add_headers_and_sources(clickhouse_functions_jsonpath_asts .) +add_library(clickhouse_functions_jsonpath_asts ${clickhouse_functions_jsonpath_asts_sources} ${clickhouse_functions_jsonpath_asts_headers}) +target_link_libraries(clickhouse_functions_jsonpath_asts PRIVATE dbms) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + target_compile_options(clickhouse_functions_jsonpath_asts PRIVATE "-g0") +endif() \ No newline at end of file diff --git a/src/Functions/JSONPath/CMakeLists.txt b/src/Functions/JSONPath/CMakeLists.txt new file mode 100644 index 00000000000..8a46909f555 --- /dev/null +++ b/src/Functions/JSONPath/CMakeLists.txt @@ -0,0 +1,8 @@ +add_subdirectory(ASTs) +target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_asts) + +add_subdirectory(Generators) +target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_generators) + +add_subdirectory(Parsers) +target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_parsers) \ No newline at end of file diff --git a/src/Functions/JSONPath/Generators/CMakeLists.txt b/src/Functions/JSONPath/Generators/CMakeLists.txt new file mode 100644 index 00000000000..0d1a289e8b4 --- /dev/null +++ b/src/Functions/JSONPath/Generators/CMakeLists.txt @@ -0,0 +1,8 @@ +include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") +add_headers_and_sources(clickhouse_functions_jsonpath_generators .) +add_library(clickhouse_functions_jsonpath_generators ${clickhouse_functions_jsonpath_generators_sources} ${clickhouse_functions_jsonpath_generators_headers}) +target_link_libraries(clickhouse_functions_jsonpath_generators PRIVATE dbms) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + target_compile_options(clickhouse_functions_jsonpath_generators PRIVATE "-g0") +endif() \ No newline at end of file diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h new file mode 100644 index 00000000000..dd4354a4613 --- /dev/null +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -0,0 +1,98 @@ +#include +#include +#include + +#include + +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +template +class GeneratorJSONPath : public IGenerator +{ +public: + GeneratorJSONPath(ASTPtr query_ptr_) + { + query_ptr = query_ptr_; + const auto * path = query_ptr->as(); + if (!path) { + throw Exception("Invalid path", ErrorCodes::LOGICAL_ERROR); + } + const auto * query = path->jsonpath_query; + if (!path || !query) + { + throw Exception("Something went terribly wrong", ErrorCodes::LOGICAL_ERROR); + } + + for (auto child_ast : query->children) + { + if (child_ast->getID() == "ASTJSONPathMemberAccess") + { + auto member_access_generator = std::make_shared>(child_ast); + if (member_access_generator) { + visitors.push_back(member_access_generator); + } else { + throw Exception("member_access_generator could not be nullptr", ErrorCodes::LOGICAL_ERROR); + } + } + } + } + + const char * getName() const override { return "GeneratorJSONPath"; } + + /** + * The only generator which is called from JSONPath functions. + * @param element root of JSON document + * @return is the generator exhausted + */ + VisitorStatus getNextItem(typename JSONParser::Element & element) override + { + if (visitors[current_visitor]->isExhausted()) { + if (!backtrace()) { + return VisitorStatus::Exhausted; + } + } + + /// Apply all non-exhausted visitors + for (int i = 0; i < current_visitor; ++i) { + VisitorStatus status = visitors[i]->apply(element); + /// on fail return immediately + if (status == VisitorStatus::Error) { + return status; + } + } + + /// Visit newly initialized (for the first time or through reinitialize) visitors + for (size_t i = current_visitor; i < visitors.size(); ++i) { + VisitorStatus status = visitors[i]->visit(element); + current_visitor = i; + /// on fail return immediately + if (status == VisitorStatus::Error) { + return status; + } + } + return VisitorStatus::Ok; + } + +private: + bool backtrace() { + while (current_visitor >= 0 && visitors[current_visitor]->isExhausted()) { + visitors[current_visitor]->reinitialize(); + current_visitor--; + } + return current_visitor >= 0; + } + + int current_visitor = 0; + ASTPtr query_ptr; + VisitorList visitors; +}; + +} // namespace DB diff --git a/src/Functions/JSONPath/Generators/IGenerator.h b/src/Functions/JSONPath/Generators/IGenerator.h new file mode 100644 index 00000000000..31d9e167f24 --- /dev/null +++ b/src/Functions/JSONPath/Generators/IGenerator.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +template +class IGenerator +{ +public: + IGenerator() = default; + + virtual const char * getName() const = 0; + + /** + * Used to yield next element in JSONPath query. Does so by recursively calling getNextItem + * on its children Generators one by one. + * + * @param element to be extracted into + * @return true if generator is not exhausted + */ + virtual VisitorStatus getNextItem(typename JSONParser::Element & element) = 0; + + virtual ~IGenerator() = default; +}; + +} // namespace DB diff --git a/src/Functions/JSONPath/Generators/IGenerator_fwd.h b/src/Functions/JSONPath/Generators/IGenerator_fwd.h new file mode 100644 index 00000000000..27c3976b95b --- /dev/null +++ b/src/Functions/JSONPath/Generators/IGenerator_fwd.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace DB { + +template +class IGenerator; + +template +using IVisitorPtr = std::shared_ptr>; + +template +using VisitorList = std::vector>; + +} // namespace DB diff --git a/src/Functions/JSONPath/Generators/IVisitor.h b/src/Functions/JSONPath/Generators/IVisitor.h new file mode 100644 index 00000000000..fdd254478a5 --- /dev/null +++ b/src/Functions/JSONPath/Generators/IVisitor.h @@ -0,0 +1,40 @@ +#pragma once + +#include + +namespace DB { + +template +class IVisitor { +public: + /** + * Applies this visitor to document and mutates its state + * @param element simdjson element + */ + virtual VisitorStatus visit(typename JSONParser::Element & element) = 0; + + /** + * Applies this visitor to document, but does not mutate state + * @param element simdjson element + */ + virtual VisitorStatus apply(typename JSONParser::Element & element) const = 0; + + /** + * Restores visitor's initial state for later use + */ + virtual void reinitialize() = 0; + + bool isExhausted() { + return is_exhausted; + } + + void setExhausted(bool exhausted) { + is_exhausted = exhausted; + } + + virtual ~IVisitor() = default; +private: + bool is_exhausted = false; +}; + +} // namespace DB diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h new file mode 100644 index 00000000000..50b814eeaeb --- /dev/null +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h @@ -0,0 +1,39 @@ +#include +#include +#include + +namespace DB +{ +template +class VisitorJSONPathMemberAccess : public IVisitor +{ +public: + VisitorJSONPathMemberAccess(ASTPtr member_access_ptr_) : member_access_ptr(member_access_ptr_) { } + + VisitorStatus apply(typename JSONParser::Element & element) const override { + const auto * member_access = member_access_ptr->as(); + typename JSONParser::Element result; + bool result_ok = element.getObject().find(std::string_view(member_access->member_name), result); + if (result_ok) + { + element = result; + return VisitorStatus::Ok; + } + return VisitorStatus::Error; + } + + VisitorStatus visit(typename JSONParser::Element & element) override + { + this->setExhausted(true); + return apply(element); + } + + void reinitialize() override { + this->setExhausted(false); + } + +private: + ASTPtr member_access_ptr; +}; + +} // namespace DB diff --git a/src/Functions/JSONPath/Generators/VisitorStatus.h b/src/Functions/JSONPath/Generators/VisitorStatus.h new file mode 100644 index 00000000000..51d795efbf7 --- /dev/null +++ b/src/Functions/JSONPath/Generators/VisitorStatus.h @@ -0,0 +1,11 @@ +#pragma once + +namespace DB { + +enum VisitorStatus { + Ok, + Exhausted, + Error +}; + +} diff --git a/src/Functions/JSONPath/Parsers/CMakeLists.txt b/src/Functions/JSONPath/Parsers/CMakeLists.txt new file mode 100644 index 00000000000..f2f94298576 --- /dev/null +++ b/src/Functions/JSONPath/Parsers/CMakeLists.txt @@ -0,0 +1,8 @@ +include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") +add_headers_and_sources(clickhouse_functions_jsonpath_parsers .) +add_library(clickhouse_functions_jsonpath_parsers ${clickhouse_functions_jsonpath_parsers_sources} ${clickhouse_functions_jsonpath_parsers_headers}) +target_link_libraries(clickhouse_functions_jsonpath_parsers PRIVATE dbms) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + target_compile_options(clickhouse_functions_jsonpath_parsers PRIVATE "-g0") +endif() \ No newline at end of file diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp new file mode 100644 index 00000000000..bf62f44fade --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp @@ -0,0 +1,34 @@ +#include + +#include + +#include + +#include + +namespace DB +{ + +/** + * Entry parser for JSONPath + */ +bool ParserJSONPath::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + auto ast_jsonpath = std::make_shared(); + ParserJSONPathQuery parser_jsonpath_query; + + /// Push back dot AST and brackets AST to query->children + ASTPtr query; + + bool res = parser_jsonpath_query.parse(pos, query, expected); + + if (res) { + /// Set ASTJSONPathQuery of ASTJSONPath + ast_jsonpath->set(ast_jsonpath->jsonpath_query, query); + } + + node = ast_jsonpath; + return res; +} + +} // namespace DB diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPath.h b/src/Functions/JSONPath/Parsers/ParserJSONPath.h new file mode 100644 index 00000000000..5defc76b515 --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPath.h @@ -0,0 +1,22 @@ +#pragma once + +#include + + +namespace DB +{ + +/** + * Entry parser for JSONPath + */ +class ParserJSONPath : public IParserBase +{ +private: + const char * getName() const override { return "ParserJSONPath"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +public: + explicit ParserJSONPath() = default; +}; + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp new file mode 100644 index 00000000000..10ae128616b --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp @@ -0,0 +1,42 @@ +#include +#include + +#include +#include +#include + +namespace DB +{ + +/** + * + * @param pos token iterator + * @param node node of ASTJSONPathMemberAccess + * @param expected stuff for logging + * @return was parse successful + */ +bool ParserJSONPathMemberAccess::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + auto member_access = std::make_shared(); + node = member_access; + if (pos->type != TokenType::Dot) { + return false; + } + ++pos; + + if (pos->type != TokenType::BareWord) { + return false; + } + ParserIdentifier name_p; + ASTPtr member_name; + if (!name_p.parse(pos, member_name, expected)) { + return false; + } + + if (!tryGetIdentifierNameInto(member_name, member_access->member_name)) { + return false; + } + return true; +} + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h new file mode 100644 index 00000000000..49fda6f1ac8 --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h @@ -0,0 +1,12 @@ +#include + +namespace DB +{ +class ParserJSONPathMemberAccess : public IParserBase +{ + const char * getName() const override {return "ParserJSONPathMemberAccess";} + + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp new file mode 100644 index 00000000000..c0831780fc4 --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp @@ -0,0 +1,39 @@ +#include + +#include + +#include + +namespace DB + +{ +/** + * + * @param pos token iterator + * @param query node of ASTJSONPathQuery + * @param expected stuff for logging + * @return was parse successful + */ +bool ParserJSONPathQuery::parseImpl(Pos & pos, ASTPtr & query, Expected & expected) +{ + query = std::make_shared(); + ParserJSONPathMemberAccess parser_jsonpath_member_access; + + if (pos->type != TokenType::DollarSign) { + return false; + } + ++pos; + + bool res = false; + ASTPtr member_access; + while (parser_jsonpath_member_access.parse(pos, member_access, expected)) + { + query->children.push_back(member_access); + member_access = nullptr; + res = true; + } + /// true in case of at least one success + return res; +} + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h new file mode 100644 index 00000000000..cffec125c70 --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h @@ -0,0 +1,17 @@ +#pragma once + +#include + + +namespace DB +{ +class ParserJSONPathQuery : public IParserBase +{ +protected: + const char * getName() const override { return "ParserJSONPathQuery"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +private: + /// backlog: strict or lax mode +}; +} diff --git a/src/Functions/RapidJSONParser.h b/src/Functions/RapidJSONParser.h index 992480d64f7..5604a8c9fe0 100644 --- a/src/Functions/RapidJSONParser.h +++ b/src/Functions/RapidJSONParser.h @@ -12,7 +12,6 @@ namespace DB { - /// This class can be used as an argument for the template class FunctionJSON. /// It provides ability to parse JSONs using rapidjson library. struct RapidJSONParser @@ -45,6 +44,8 @@ struct RapidJSONParser Array getArray() const; Object getObject() const; + ALWAYS_INLINE rapidjson::Value * getDom() const { return nullptr; } + private: const rapidjson::Value * ptr = nullptr; }; diff --git a/src/Functions/SimdJSONParser.h b/src/Functions/SimdJSONParser.h index a9adfa27e2c..2dd952d920f 100644 --- a/src/Functions/SimdJSONParser.h +++ b/src/Functions/SimdJSONParser.h @@ -5,10 +5,10 @@ #endif #if USE_SIMDJSON -# include +# include # include # include -# include +# include namespace DB @@ -30,8 +30,8 @@ struct SimdJSONParser class Element { public: - ALWAYS_INLINE Element() {} - ALWAYS_INLINE Element(const simdjson::dom::element & element_) : element(element_) {} + ALWAYS_INLINE Element() { } + ALWAYS_INLINE Element(const simdjson::dom::element & element_) : element(element_) { } ALWAYS_INLINE bool isInt64() const { return element.type() == simdjson::dom::element_type::INT64; } ALWAYS_INLINE bool isUInt64() const { return element.type() == simdjson::dom::element_type::UINT64; } @@ -50,6 +50,8 @@ struct SimdJSONParser ALWAYS_INLINE Array getArray() const; ALWAYS_INLINE Object getObject() const; + ALWAYS_INLINE simdjson::dom::element getElement() const { return element; } + private: simdjson::dom::element element; }; @@ -61,21 +63,35 @@ struct SimdJSONParser class Iterator { public: - ALWAYS_INLINE Iterator(const simdjson::dom::array::iterator & it_) : it(it_) {} + ALWAYS_INLINE Iterator(const simdjson::dom::array::iterator & it_) : it(it_) { } ALWAYS_INLINE Element operator*() const { return *it; } - ALWAYS_INLINE Iterator & operator++() { ++it; return *this; } - ALWAYS_INLINE Iterator operator++(int) { auto res = *this; ++it; return res; } + ALWAYS_INLINE Iterator & operator++() + { + ++it; + return *this; + } + ALWAYS_INLINE Iterator operator++(int) + { + auto res = *this; + ++it; + return res; + } ALWAYS_INLINE friend bool operator!=(const Iterator & left, const Iterator & right) { return left.it != right.it; } ALWAYS_INLINE friend bool operator==(const Iterator & left, const Iterator & right) { return !(left != right); } + private: simdjson::dom::array::iterator it; }; - ALWAYS_INLINE Array(const simdjson::dom::array & array_) : array(array_) {} + ALWAYS_INLINE Array(const simdjson::dom::array & array_) : array(array_) { } ALWAYS_INLINE Iterator begin() const { return array.begin(); } ALWAYS_INLINE Iterator end() const { return array.end(); } ALWAYS_INLINE size_t size() const { return array.size(); } - ALWAYS_INLINE Element operator[](size_t index) const { assert(index < size()); return array.at(index).first; } + ALWAYS_INLINE Element operator[](size_t index) const + { + assert(index < size()); + return array.at(index).first; + } private: simdjson::dom::array array; @@ -90,17 +106,31 @@ struct SimdJSONParser class Iterator { public: - ALWAYS_INLINE Iterator(const simdjson::dom::object::iterator & it_) : it(it_) {} - ALWAYS_INLINE KeyValuePair operator*() const { const auto & res = *it; return {res.key, res.value}; } - ALWAYS_INLINE Iterator & operator++() { ++it; return *this; } - ALWAYS_INLINE Iterator operator++(int) { auto res = *this; ++it; return res; } + ALWAYS_INLINE Iterator(const simdjson::dom::object::iterator & it_) : it(it_) { } + ALWAYS_INLINE KeyValuePair operator*() const + { + const auto & res = *it; + return {res.key, res.value}; + } + ALWAYS_INLINE Iterator & operator++() + { + ++it; + return *this; + } + ALWAYS_INLINE Iterator operator++(int) + { + auto res = *this; + ++it; + return res; + } ALWAYS_INLINE friend bool operator!=(const Iterator & left, const Iterator & right) { return left.it != right.it; } ALWAYS_INLINE friend bool operator==(const Iterator & left, const Iterator & right) { return !(left != right); } + private: simdjson::dom::object::iterator it; }; - ALWAYS_INLINE Object(const simdjson::dom::object & object_) : object(object_) {} + ALWAYS_INLINE Object(const simdjson::dom::object & object_) : object(object_) { } ALWAYS_INLINE Iterator begin() const { return object.begin(); } ALWAYS_INLINE Iterator end() const { return object.end(); } ALWAYS_INLINE size_t size() const { return object.size(); } @@ -126,6 +156,8 @@ struct SimdJSONParser return {res.key, res.value}; } + ALWAYS_INLINE simdjson::dom::object getDom() const { return object; } + private: simdjson::dom::object object; }; @@ -145,8 +177,8 @@ struct SimdJSONParser void reserve(size_t max_size) { if (parser.allocate(max_size) != simdjson::error_code::SUCCESS) - throw Exception{"Couldn't allocate " + std::to_string(max_size) + " bytes when parsing JSON", - ErrorCodes::CANNOT_ALLOCATE_MEMORY}; + throw Exception{ + "Couldn't allocate " + std::to_string(max_size) + " bytes when parsing JSON", ErrorCodes::CANNOT_ALLOCATE_MEMORY}; } private: diff --git a/src/Functions/registerFunctions.cpp b/src/Functions/registerFunctions.cpp index d827cc40a86..116a2ae0324 100644 --- a/src/Functions/registerFunctions.cpp +++ b/src/Functions/registerFunctions.cpp @@ -40,6 +40,7 @@ void registerFunctionsGeo(FunctionFactory &); void registerFunctionsIntrospection(FunctionFactory &); void registerFunctionsNull(FunctionFactory &); void registerFunctionsJSON(FunctionFactory &); +void registerFunctionsSQLJSON(FunctionFactory &); void registerFunctionsConsistentHashing(FunctionFactory & factory); void registerFunctionsUnixTimestamp64(FunctionFactory & factory); void registerFunctionBitHammingDistance(FunctionFactory & factory); @@ -97,6 +98,7 @@ void registerFunctions() registerFunctionsGeo(factory); registerFunctionsNull(factory); registerFunctionsJSON(factory); + registerFunctionsSQLJSON(factory); registerFunctionsIntrospection(factory); registerFunctionsConsistentHashing(factory); registerFunctionsUnixTimestamp64(factory); diff --git a/src/Parsers/Lexer.cpp b/src/Parsers/Lexer.cpp index ffa8250a3f3..33cecab208f 100644 --- a/src/Parsers/Lexer.cpp +++ b/src/Parsers/Lexer.cpp @@ -240,6 +240,9 @@ Token Lexer::nextTokenImpl() case '*': ++pos; return Token(TokenType::Asterisk, token_begin, pos); + case '$': + ++pos; + return Token(TokenType::DollarSign, token_begin, pos); case '/': /// division (/) or start of comment (//, /*) { ++pos; diff --git a/src/Parsers/Lexer.h b/src/Parsers/Lexer.h index dc1c9824b6b..e9885d8df83 100644 --- a/src/Parsers/Lexer.h +++ b/src/Parsers/Lexer.h @@ -33,6 +33,7 @@ namespace DB \ M(Asterisk) /** Could be used as multiplication operator or on it's own: "SELECT *" */ \ \ + M(DollarSign) \ M(Plus) \ M(Minus) \ M(Slash) \ From 8199b45e95b169e86adea0ab806b464c120c941f Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 15 May 2021 13:10:19 +0300 Subject: [PATCH 02/42] Fix style Fifth try v2.0 Fifth try v2.1 Fifth try v2.2 Fifth try v2.3 Fifth try v2.4 Fifth try v2.5 Fifth try v2.6 Fifth try v2.7 Fifth try v2.8 Fifth try v2.9 Fifth try v2.10 Fifth try v2.11 Fifth try v2.12 Fifth try v2.13 Fifth try v2.14 Fifth try v2.15 Fifth try v2.16 Fifth try v2.17 Fifth try v2.18 Fifth try v2.19 Fifth try v2.20 Fifth try v2.21 Fifth try v2.22 Fifth try v2.23 Fifth try v2.24 Fifth try v2.25 Fifth try v2.26 Fifth try v2.27 Fifth try v2.28 Add ranges Add ranges try v1.1 Add ranges try v1.2 Add ranges try v1.3 Add ranges try v1.4 Add ranges try v1.5 Add ranges try v1.6 Add ranges try v1.7 Add ranges try v1.8 Add ranges try v1.9 Add ranges try v1.10 Add ranges try v1.11 Add ranges try v1.12 Add ranges try v1.13 Add ranges try v1.14 Add ranges try v1.15 Add ranges try v1.16 Add ranges try v1.17 Add ranges try v1.18 Add ranges try v1.19 Add ranges try v1.20 Add ranges try v1.21 Add ranges try v1.22 Add ranges try v1.23 Add ranges try v1.24 Add ranges try v1.25 Add ranges try v1.26 Add ranges try v1.27 Add ranges try v1.28 Add ranges try v1.29 Add ranges try v1.30 Add ranges try v1.31 Add ranges try v1.32 Add ranges try v1.33 Add ranges try v1.34 Add ranges try v1.35 Add ranges try v1.36 Add ranges try v1.37 Add ranges try v1.38 Add ranges try v1.39 Add ranges try v1.40 Add ranges try v1.41 Add ranges try v1.42 Add ranges try v1.43 Add ranges try v1.44 Add ranges try v1.45 Add ranges try v1.46 Add ranges try v1.47 Leftover comment Try wildcard Try wildcard v1.1 Try wildcard v1.2 Try wildcard v1.3 New functions New functions 1.1 New functions 1.2 New functions 1.3 New functions 1.4 New functions 1.5 New functions 1.6 New functions 1.7 New functions 1.8 New functions 1.9 New functions 1.10 New functions 1.11 New functions 1.12 New functions 1.13 New functions 1.14 New functions 1.15 New functions 1.16 Final steps Final steps v1.1 Final steps v1.2 --- src/Functions/DummyJSONParser.h | 2 + src/Functions/FunctionSQLJSON.cpp | 6 +- src/Functions/FunctionSQLJSON.h | 145 +++++++++++++----- src/Functions/FunctionsJSON.h | 2 + src/Functions/JSONPath/ASTs/ASTJSONPath.h | 2 - .../JSONPath/ASTs/ASTJSONPathQuery.h | 2 - .../JSONPath/ASTs/ASTJSONPathRange.h | 30 ++++ src/Functions/JSONPath/ASTs/CMakeLists.txt | 2 +- src/Functions/JSONPath/CMakeLists.txt | 2 +- .../JSONPath/Generators/CMakeLists.txt | 2 +- .../JSONPath/Generators/GeneratorJSONPath.h | 65 ++++---- .../JSONPath/Generators/IGenerator.h | 3 +- src/Functions/JSONPath/Generators/IVisitor.h | 5 + .../Generators/VisitorJSONPathMemberAccess.h | 27 +++- .../Generators/VisitorJSONPathRange.h | 93 +++++++++++ .../JSONPath/Generators/VisitorStatus.h | 3 +- src/Functions/JSONPath/Parsers/CMakeLists.txt | 2 +- .../JSONPath/Parsers/ParserJSONPath.cpp | 3 - .../JSONPath/Parsers/ParserJSONPathQuery.cpp | 20 ++- .../JSONPath/Parsers/ParserJSONPathRange.cpp | 98 ++++++++++++ .../JSONPath/Parsers/ParserJSONPathRange.h | 19 +++ src/Functions/RapidJSONParser.h | 3 +- src/Functions/SimdJSONParser.h | 62 ++------ 23 files changed, 445 insertions(+), 153 deletions(-) create mode 100644 src/Functions/JSONPath/ASTs/ASTJSONPathRange.h create mode 100644 src/Functions/JSONPath/Generators/VisitorJSONPathRange.h create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathRange.h diff --git a/src/Functions/DummyJSONParser.h b/src/Functions/DummyJSONParser.h index a71c90e4a19..3010347e4c4 100644 --- a/src/Functions/DummyJSONParser.h +++ b/src/Functions/DummyJSONParser.h @@ -39,6 +39,8 @@ struct DummyJSONParser std::string_view getString() const { return {}; } Array getArray() const { return {}; } Object getObject() const { return {}; } + + Element getElement() {return {}; } }; /// References an array in a JSON document. diff --git a/src/Functions/FunctionSQLJSON.cpp b/src/Functions/FunctionSQLJSON.cpp index ddcca12835f..7d558dd0950 100644 --- a/src/Functions/FunctionSQLJSON.cpp +++ b/src/Functions/FunctionSQLJSON.cpp @@ -9,11 +9,11 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } - void registerFunctionsSQLJSON(FunctionFactory & factory) { - factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 24749099e57..1fc6986256d 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -1,16 +1,12 @@ #pragma once #include -#include -#include -#include -#include #include -#include -#include -#include +#include +#include #include #include +#include #include #include #include @@ -21,12 +17,7 @@ #include #include #include -#include -#include -#include -#include #include -//#include #include #if !defined(ARCADIA_BUILD) @@ -37,11 +28,11 @@ namespace DB { namespace ErrorCodes { - extern const int ILLEGAL_COLUMN; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; - extern const int BAD_ARGUMENTS; +extern const int ILLEGAL_COLUMN; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; +extern const int BAD_ARGUMENTS; } class FunctionSQLJSONHelpers @@ -141,6 +132,7 @@ public: /// Parse JSON for every row Impl impl; + for (const auto i : ext::range(0, input_rows_count)) { std::string_view json{ @@ -205,45 +197,58 @@ private: const Context & context; }; -struct NameSQLJSONTest +struct NameJSONExists { - static constexpr auto name{"SQLJSONTest"}; + static constexpr auto name{"JSON_EXISTS"}; }; -struct NameSQLJSONMemberAccess +struct NameJSONValue { - static constexpr auto name{"SQLJSONMemberAccess"}; + static constexpr auto name{"JSON_VALUE"}; +}; + +struct NameJSONQuery +{ + static constexpr auto name{"JSON_QUERY"}; }; -/** - * Function to test logic before function calling, will be removed in final PR - * @tparam JSONParser parser - */ template -class SQLJSONTestImpl +class JSONExistsImpl { public: using Element = typename JSONParser::Element; - static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) { return std::make_shared(); } + static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) { return std::make_shared(); } static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } - static bool insertResultToColumn(IColumn & dest, const Element &, ASTPtr &) + static bool insertResultToColumn(IColumn & dest, const Element & root, ASTPtr & query_ptr) { - String str = "I am working:-)"; - ColumnString & col_str = assert_cast(dest); - col_str.insertData(str.data(), str.size()); + GeneratorJSONPath generator_json_path(query_ptr); + Element current_element = root; + VisitorStatus status; + while ((status = generator_json_path.getNextItem(current_element)) != VisitorStatus::Exhausted) + { + if (status == VisitorStatus::Ok) { + break; + } + current_element = root; + } + + /// insert result, status can be either Ok (if we found the item) + /// or Exhausted (if we never found the item) + ColumnUInt8 & col_bool = assert_cast(dest); + if (status == VisitorStatus::Ok) { + col_bool.insert(0); + } else { + col_bool.insert(1); + } return true; } }; -/** - * Function to test jsonpath member access, will be removed in final PR - * @tparam JSONParser parser - */ template -class SQLJSONMemberAccessImpl +class JSONValueImpl { public: using Element = typename JSONParser::Element; @@ -257,18 +262,74 @@ public: GeneratorJSONPath generator_json_path(query_ptr); Element current_element = root; VisitorStatus status; - while ((status = generator_json_path.getNextItem(current_element)) == VisitorStatus::Ok) + Element res; + while ((status = generator_json_path.getNextItem(current_element)) != VisitorStatus::Exhausted) { - /// No-op + if (status == VisitorStatus::Ok) { + if (!(current_element.isArray() || current_element.isObject())) { + break; + } + } else if (status == VisitorStatus::Error) { + /// ON ERROR + } + current_element = root; } - if (status == VisitorStatus::Error) + + if (status == VisitorStatus::Exhausted) { + return false; + } + + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + out << current_element.getElement(); + auto output_str = out.str(); + ColumnString & col_str = assert_cast(dest); + col_str.insertData(output_str.data(), output_str.size()); + return true; + } +}; + +/** + * Function to test jsonpath member access, will be removed in final PR + * @tparam JSONParser parser + */ +template +class JSONQueryImpl +{ +public: + using Element = typename JSONParser::Element; + + static DataTypePtr getReturnType(const char *, const ColumnsWithTypeAndName &) { return std::make_shared(); } + + static size_t getNumberOfIndexArguments(const ColumnsWithTypeAndName & arguments) { return arguments.size() - 1; } + + static bool insertResultToColumn(IColumn & dest, const Element & root, ASTPtr & query_ptr) + { + GeneratorJSONPath generator_json_path(query_ptr); + Element current_element = root; + VisitorStatus status; + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + /// Create json array of results: [res1, res2, ...] + out << "["; + bool success = false; + while ((status = generator_json_path.getNextItem(current_element)) != VisitorStatus::Exhausted) { + if (status == VisitorStatus::Ok) { + if (success) { + out << ", "; + } + success = true; + out << current_element.getElement(); + } else if (status == VisitorStatus::Error) { + /// ON ERROR + } + current_element = root; + } + out << "]"; + if (!success) { return false; } ColumnString & col_str = assert_cast(dest); - std::stringstream ostr; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - ostr << current_element.getElement(); - auto output_str = ostr.str(); + auto output_str = out.str(); col_str.insertData(output_str.data(), output_str.size()); return true; } diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 581cc2015aa..f066bb1029a 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -80,6 +80,8 @@ public: const ColumnString::Chars & chars = col_json_string->getChars(); const ColumnString::Offsets & offsets = col_json_string->getOffsets(); + size_t num_index_arguments = Impl::getNumberOfIndexArguments(arguments); + std::vector moves = prepareMoves(Name::name, arguments, 1, num_index_arguments); /// Preallocate memory in parser if necessary. JSONParser parser; diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPath.h b/src/Functions/JSONPath/ASTs/ASTJSONPath.h index cd73cd14257..092fe16bd9e 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPath.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPath.h @@ -10,13 +10,11 @@ class ASTJSONPath : public IAST public: String getID(char) const override { - std::cerr << "in ASTJSONPath: getID\n"; return "ASTJSONPath"; } ASTPtr clone() const override { - std::cerr << "in " << "ASTJSONPath" << ": clone\n"; return std::make_shared(*this); } diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h b/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h index 6b952c2519d..8da8a7baafb 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h @@ -9,13 +9,11 @@ class ASTJSONPathQuery : public IAST public: String getID(char) const override { - std::cerr << "in ASTJSONPathQuery: getID\n"; return "ASTJSONPathQuery"; } ASTPtr clone() const override { - std::cerr << "in " << getID(' ') << ": clone\n"; return std::make_shared(*this); } }; diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h b/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h new file mode 100644 index 00000000000..21af3cff363 --- /dev/null +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +namespace DB +{ + +class ASTJSONPathRange : public IAST +{ +public: + String getID(char) const override + { + return "ASTJSONPathRange"; + } + + ASTPtr clone() const override + { + return std::make_shared(*this); + } + +public: + /// Ranges to lookup in json array ($[0, 1, 2, 4 to 9]) + /// Range is represented as + /// Single index is represented as + std::vector> ranges; + bool is_star = false; +}; + +} diff --git a/src/Functions/JSONPath/ASTs/CMakeLists.txt b/src/Functions/JSONPath/ASTs/CMakeLists.txt index c671dbbc001..ef56e3b0072 100644 --- a/src/Functions/JSONPath/ASTs/CMakeLists.txt +++ b/src/Functions/JSONPath/ASTs/CMakeLists.txt @@ -5,4 +5,4 @@ target_link_libraries(clickhouse_functions_jsonpath_asts PRIVATE dbms) if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) target_compile_options(clickhouse_functions_jsonpath_asts PRIVATE "-g0") -endif() \ No newline at end of file +endif() diff --git a/src/Functions/JSONPath/CMakeLists.txt b/src/Functions/JSONPath/CMakeLists.txt index 8a46909f555..8e65f7c8c6d 100644 --- a/src/Functions/JSONPath/CMakeLists.txt +++ b/src/Functions/JSONPath/CMakeLists.txt @@ -5,4 +5,4 @@ add_subdirectory(Generators) target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_generators) add_subdirectory(Parsers) -target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_parsers) \ No newline at end of file +target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_parsers) diff --git a/src/Functions/JSONPath/Generators/CMakeLists.txt b/src/Functions/JSONPath/Generators/CMakeLists.txt index 0d1a289e8b4..76a116132fd 100644 --- a/src/Functions/JSONPath/Generators/CMakeLists.txt +++ b/src/Functions/JSONPath/Generators/CMakeLists.txt @@ -5,4 +5,4 @@ target_link_libraries(clickhouse_functions_jsonpath_generators PRIVATE dbms) if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) target_compile_options(clickhouse_functions_jsonpath_generators PRIVATE "-g0") -endif() \ No newline at end of file +endif() diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h index dd4354a4613..68ea5a2a3c5 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -1,11 +1,10 @@ #include #include +#include #include #include -#include - namespace DB { @@ -26,20 +25,23 @@ public: throw Exception("Invalid path", ErrorCodes::LOGICAL_ERROR); } const auto * query = path->jsonpath_query; - if (!path || !query) - { - throw Exception("Something went terribly wrong", ErrorCodes::LOGICAL_ERROR); - } for (auto child_ast : query->children) { if (child_ast->getID() == "ASTJSONPathMemberAccess") { - auto member_access_generator = std::make_shared>(child_ast); - if (member_access_generator) { - visitors.push_back(member_access_generator); + auto member_access_visitor = std::make_shared>(child_ast); + if (member_access_visitor) { + visitors.push_back(member_access_visitor); } else { - throw Exception("member_access_generator could not be nullptr", ErrorCodes::LOGICAL_ERROR); + throw Exception("member_access_visitor could not be nullptr", ErrorCodes::LOGICAL_ERROR); + } + } else if (child_ast->getID() == "ASTJSONPathRange") { + auto range_visitor = std::make_shared>(child_ast); + if (range_visitor) { + visitors.push_back(range_visitor); + } else { + throw Exception("range_visitor could not be nullptr", ErrorCodes::LOGICAL_ERROR); } } } @@ -54,39 +56,42 @@ public: */ VisitorStatus getNextItem(typename JSONParser::Element & element) override { - if (visitors[current_visitor]->isExhausted()) { - if (!backtrace()) { + while (true) { + auto root = element; + if (current_visitor < 0) { return VisitorStatus::Exhausted; } - } - /// Apply all non-exhausted visitors - for (int i = 0; i < current_visitor; ++i) { - VisitorStatus status = visitors[i]->apply(element); - /// on fail return immediately - if (status == VisitorStatus::Error) { + for (int i = 0; i < current_visitor; ++i) { + visitors[i]->apply(root); + } + + VisitorStatus status = VisitorStatus::Error; + for (size_t i = current_visitor; i < visitors.size(); ++i) { + status = visitors[i]->visit(root); + current_visitor = i; + if (status == VisitorStatus::Error || status == VisitorStatus::Ignore) { + break; + } + } + updateVisitorsForNextRun(); + + if (status != VisitorStatus::Ignore) { + element = root; return status; } } - - /// Visit newly initialized (for the first time or through reinitialize) visitors - for (size_t i = current_visitor; i < visitors.size(); ++i) { - VisitorStatus status = visitors[i]->visit(element); - current_visitor = i; - /// on fail return immediately - if (status == VisitorStatus::Error) { - return status; - } - } - return VisitorStatus::Ok; } private: - bool backtrace() { + bool updateVisitorsForNextRun() { while (current_visitor >= 0 && visitors[current_visitor]->isExhausted()) { visitors[current_visitor]->reinitialize(); current_visitor--; } + if (current_visitor >= 0) { + visitors[current_visitor]->updateState(); + } return current_visitor >= 0; } diff --git a/src/Functions/JSONPath/Generators/IGenerator.h b/src/Functions/JSONPath/Generators/IGenerator.h index 31d9e167f24..18c0ac7da67 100644 --- a/src/Functions/JSONPath/Generators/IGenerator.h +++ b/src/Functions/JSONPath/Generators/IGenerator.h @@ -16,8 +16,7 @@ public: virtual const char * getName() const = 0; /** - * Used to yield next element in JSONPath query. Does so by recursively calling getNextItem - * on its children Generators one by one. + * Used to yield next non-ignored element describes by JSONPath query. * * @param element to be extracted into * @return true if generator is not exhausted diff --git a/src/Functions/JSONPath/Generators/IVisitor.h b/src/Functions/JSONPath/Generators/IVisitor.h index fdd254478a5..78c1efe64fc 100644 --- a/src/Functions/JSONPath/Generators/IVisitor.h +++ b/src/Functions/JSONPath/Generators/IVisitor.h @@ -7,6 +7,9 @@ namespace DB { template class IVisitor { public: + + virtual const char * getName() const = 0; + /** * Applies this visitor to document and mutates its state * @param element simdjson element @@ -24,6 +27,8 @@ public: */ virtual void reinitialize() = 0; + virtual void updateState() = 0; + bool isExhausted() { return is_exhausted; } diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h index 50b814eeaeb..cad36e40e4d 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h @@ -10,28 +10,39 @@ class VisitorJSONPathMemberAccess : public IVisitor public: VisitorJSONPathMemberAccess(ASTPtr member_access_ptr_) : member_access_ptr(member_access_ptr_) { } + const char * getName() const override { return "VisitorJSONPathMemberAccess"; } + VisitorStatus apply(typename JSONParser::Element & element) const override { const auto * member_access = member_access_ptr->as(); typename JSONParser::Element result; - bool result_ok = element.getObject().find(std::string_view(member_access->member_name), result); - if (result_ok) - { - element = result; - return VisitorStatus::Ok; - } - return VisitorStatus::Error; + element.getObject().find(std::string_view(member_access->member_name), result); + element = result; + return VisitorStatus::Ok; } VisitorStatus visit(typename JSONParser::Element & element) override { + if (!element.isObject()) { + this->setExhausted(true); + return VisitorStatus::Error; + } + const auto * member_access = member_access_ptr->as(); + typename JSONParser::Element result; + if (!element.getObject().find(std::string_view(member_access->member_name), result)) { + this->setExhausted(true); + return VisitorStatus::Error; + } + apply(element); this->setExhausted(true); - return apply(element); + return VisitorStatus::Ok; } void reinitialize() override { this->setExhausted(false); } + void updateState() override {} + private: ASTPtr member_access_ptr; }; diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h new file mode 100644 index 00000000000..0858d9b70da --- /dev/null +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h @@ -0,0 +1,93 @@ +#include +#include +#include + +namespace DB { + +template +class VisitorJSONPathRange : public IVisitor +{ +public: + VisitorJSONPathRange(ASTPtr range_ptr_) : range_ptr(range_ptr_) { + const auto * range = range_ptr->as(); + current_range = 0; + if (range->is_star) { + current_index = 0; + } else { + current_index = range->ranges[current_range].first; + } + } + + const char * getName() const override { return "VisitorJSONPathRange"; } + + VisitorStatus apply(typename JSONParser::Element & element) const override { + typename JSONParser::Element result; + typename JSONParser::Array array = element.getArray(); + if (current_index >= array.size()) { + return VisitorStatus::Error; + } + result = array[current_index]; + element = result; + return VisitorStatus::Ok; + } + + VisitorStatus visit(typename JSONParser::Element & element) override + { + if (!element.isArray()) { + this->setExhausted(true); + return VisitorStatus::Error; + } + + const auto * range = range_ptr->as(); + VisitorStatus status; + if (current_index < element.getArray().size()) { + apply(element); + status = VisitorStatus::Ok; + } else if (!range->is_star) { + status = VisitorStatus::Ignore; + } else { + status = VisitorStatus::Ignore; + this->setExhausted(true); + } + + if (!range->is_star) { + if (current_index + 1 == range->ranges[current_range].second) { + if (current_range + 1 == range->ranges.size()) { + this->setExhausted(true); + } + } + } + + return status; + } + + void reinitialize() override { + const auto * range = range_ptr->as(); + current_range = 0; + if (range->is_star) { + current_index = 0; + } else { + current_index = range->ranges[current_range].first; + } + this->setExhausted(false); + } + + void updateState() override { + const auto * range = range_ptr->as(); + current_index++; + if (range->is_star) { + return; + } + if (current_index == range->ranges[current_range].second) { + current_range++; + current_index = range->ranges[current_range].first; + } + } + +private: + ASTPtr range_ptr; + size_t current_range; + UInt32 current_index; +}; + +} // namespace diff --git a/src/Functions/JSONPath/Generators/VisitorStatus.h b/src/Functions/JSONPath/Generators/VisitorStatus.h index 51d795efbf7..17b424a3bf6 100644 --- a/src/Functions/JSONPath/Generators/VisitorStatus.h +++ b/src/Functions/JSONPath/Generators/VisitorStatus.h @@ -5,7 +5,8 @@ namespace DB { enum VisitorStatus { Ok, Exhausted, - Error + Error, + Ignore }; } diff --git a/src/Functions/JSONPath/Parsers/CMakeLists.txt b/src/Functions/JSONPath/Parsers/CMakeLists.txt index f2f94298576..ecabe5cc13b 100644 --- a/src/Functions/JSONPath/Parsers/CMakeLists.txt +++ b/src/Functions/JSONPath/Parsers/CMakeLists.txt @@ -5,4 +5,4 @@ target_link_libraries(clickhouse_functions_jsonpath_parsers PRIVATE dbms) if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) target_compile_options(clickhouse_functions_jsonpath_parsers PRIVATE "-g0") -endif() \ No newline at end of file +endif() diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp index bf62f44fade..b65de621f9a 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp @@ -1,9 +1,6 @@ #include - #include - #include - #include namespace DB diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp index c0831780fc4..10cf6f2915c 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp @@ -1,7 +1,6 @@ #include - #include - +#include #include namespace DB @@ -18,6 +17,7 @@ bool ParserJSONPathQuery::parseImpl(Pos & pos, ASTPtr & query, Expected & expect { query = std::make_shared(); ParserJSONPathMemberAccess parser_jsonpath_member_access; + ParserJSONPathRange parser_jsonpath_range; if (pos->type != TokenType::DollarSign) { return false; @@ -25,15 +25,19 @@ bool ParserJSONPathQuery::parseImpl(Pos & pos, ASTPtr & query, Expected & expect ++pos; bool res = false; - ASTPtr member_access; - while (parser_jsonpath_member_access.parse(pos, member_access, expected)) + ASTPtr subquery; + while (parser_jsonpath_member_access.parse(pos, subquery, expected) || + parser_jsonpath_range.parse(pos, subquery, expected)) { - query->children.push_back(member_access); - member_access = nullptr; + if (subquery) + { + query->children.push_back(subquery); + subquery = nullptr; + } res = true; } - /// true in case of at least one success - return res; + /// if we had at least one success and no fails + return res && pos->type == TokenType::EndOfStream; } } diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp new file mode 100644 index 00000000000..4f4a87f15ce --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp @@ -0,0 +1,98 @@ +#include +#include +#include + +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; +} +/** + * + * @param pos token iterator + * @param node node of ASTJSONPathQuery + * @param expected stuff for logging + * @return was parse successful + */ +bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + auto range = std::make_shared(); + node = range; + + if (pos->type != TokenType::OpeningSquareBracket) { + return false; + } + ++pos; + + while (pos->type != TokenType::ClosingSquareBracket) { + if (pos->type != TokenType::Number && pos->type != TokenType::Asterisk) + { + return false; + } + if (pos->type == TokenType::Asterisk) { + if (range->is_star) { + throw Exception{"Multiple asterisks in square array range are not allowed", ErrorCodes::BAD_ARGUMENTS}; + } + range->is_star = true; + ++pos; + continue; + } + + std::pair range_indices; + ParserNumber number_p; + ASTPtr number_ptr; + if (!number_p.parse(pos, number_ptr, expected)) + { + return false; + } + range_indices.first = number_ptr->as()->value.get(); + + if (pos->type == TokenType::Comma || pos->type == TokenType::ClosingSquareBracket) { + /// Single index case + range_indices.second = range_indices.first + 1; + } else if (pos->type == TokenType::BareWord) { + /// Range case + ParserIdentifier name_p; + ASTPtr word; + if (!name_p.parse(pos, word, expected)) { + return false; + } + String to_identifier; + if (!tryGetIdentifierNameInto(word, to_identifier) || to_identifier != "to") { + return false; + } + if (!number_p.parse(pos, number_ptr, expected)) + { + return false; + } + range_indices.second = number_ptr->as()->value.get(); + } else { + return false; + } + + if (range_indices.first >= range_indices.second) { + throw Exception{ErrorCodes::BAD_ARGUMENTS, "Start of range must be greater than end of range, however {} >= {}", + range_indices.first, range_indices.second}; + } + + range->ranges.push_back(std::move(range_indices)); + if (pos->type != TokenType::ClosingSquareBracket) { + ++pos; + } + } + ++pos; + + /// We cant have both ranges and star present, so parse was successful <=> exactly 1 of these conditions is true + return !range->ranges.empty() != range->is_star; +} + +} // namespace DB diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.h b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.h new file mode 100644 index 00000000000..95708e5e7b8 --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.h @@ -0,0 +1,19 @@ +#pragma once + +#include + + +namespace DB +{ + +class ParserJSONPathRange : public IParserBase +{ +private: + const char * getName() const override { return "ParserJSONPathRange"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +public: + explicit ParserJSONPathRange() = default; +}; + +} diff --git a/src/Functions/RapidJSONParser.h b/src/Functions/RapidJSONParser.h index 5604a8c9fe0..992480d64f7 100644 --- a/src/Functions/RapidJSONParser.h +++ b/src/Functions/RapidJSONParser.h @@ -12,6 +12,7 @@ namespace DB { + /// This class can be used as an argument for the template class FunctionJSON. /// It provides ability to parse JSONs using rapidjson library. struct RapidJSONParser @@ -44,8 +45,6 @@ struct RapidJSONParser Array getArray() const; Object getObject() const; - ALWAYS_INLINE rapidjson::Value * getDom() const { return nullptr; } - private: const rapidjson::Value * ptr = nullptr; }; diff --git a/src/Functions/SimdJSONParser.h b/src/Functions/SimdJSONParser.h index 2dd952d920f..b9df8b142e3 100644 --- a/src/Functions/SimdJSONParser.h +++ b/src/Functions/SimdJSONParser.h @@ -5,10 +5,10 @@ #endif #if USE_SIMDJSON -# include +# include # include # include -# include +# include namespace DB @@ -30,8 +30,8 @@ struct SimdJSONParser class Element { public: - ALWAYS_INLINE Element() { } - ALWAYS_INLINE Element(const simdjson::dom::element & element_) : element(element_) { } + ALWAYS_INLINE Element() {} + ALWAYS_INLINE Element(const simdjson::dom::element & element_) : element(element_) {} ALWAYS_INLINE bool isInt64() const { return element.type() == simdjson::dom::element_type::INT64; } ALWAYS_INLINE bool isUInt64() const { return element.type() == simdjson::dom::element_type::UINT64; } @@ -63,35 +63,21 @@ struct SimdJSONParser class Iterator { public: - ALWAYS_INLINE Iterator(const simdjson::dom::array::iterator & it_) : it(it_) { } + ALWAYS_INLINE Iterator(const simdjson::dom::array::iterator & it_) : it(it_) {} ALWAYS_INLINE Element operator*() const { return *it; } - ALWAYS_INLINE Iterator & operator++() - { - ++it; - return *this; - } - ALWAYS_INLINE Iterator operator++(int) - { - auto res = *this; - ++it; - return res; - } + ALWAYS_INLINE Iterator & operator++() { ++it; return *this; } + ALWAYS_INLINE Iterator operator++(int) { auto res = *this; ++it; return res; } ALWAYS_INLINE friend bool operator!=(const Iterator & left, const Iterator & right) { return left.it != right.it; } ALWAYS_INLINE friend bool operator==(const Iterator & left, const Iterator & right) { return !(left != right); } - private: simdjson::dom::array::iterator it; }; - ALWAYS_INLINE Array(const simdjson::dom::array & array_) : array(array_) { } + ALWAYS_INLINE Array(const simdjson::dom::array & array_) : array(array_) {} ALWAYS_INLINE Iterator begin() const { return array.begin(); } ALWAYS_INLINE Iterator end() const { return array.end(); } ALWAYS_INLINE size_t size() const { return array.size(); } - ALWAYS_INLINE Element operator[](size_t index) const - { - assert(index < size()); - return array.at(index).first; - } + ALWAYS_INLINE Element operator[](size_t index) const { assert(index < size()); return array.at(index).first; } private: simdjson::dom::array array; @@ -106,31 +92,17 @@ struct SimdJSONParser class Iterator { public: - ALWAYS_INLINE Iterator(const simdjson::dom::object::iterator & it_) : it(it_) { } - ALWAYS_INLINE KeyValuePair operator*() const - { - const auto & res = *it; - return {res.key, res.value}; - } - ALWAYS_INLINE Iterator & operator++() - { - ++it; - return *this; - } - ALWAYS_INLINE Iterator operator++(int) - { - auto res = *this; - ++it; - return res; - } + ALWAYS_INLINE Iterator(const simdjson::dom::object::iterator & it_) : it(it_) {} + ALWAYS_INLINE KeyValuePair operator*() const { const auto & res = *it; return {res.key, res.value}; } + ALWAYS_INLINE Iterator & operator++() { ++it; return *this; } + ALWAYS_INLINE Iterator operator++(int) { auto res = *this; ++it; return res; } ALWAYS_INLINE friend bool operator!=(const Iterator & left, const Iterator & right) { return left.it != right.it; } ALWAYS_INLINE friend bool operator==(const Iterator & left, const Iterator & right) { return !(left != right); } - private: simdjson::dom::object::iterator it; }; - ALWAYS_INLINE Object(const simdjson::dom::object & object_) : object(object_) { } + ALWAYS_INLINE Object(const simdjson::dom::object & object_) : object(object_) {} ALWAYS_INLINE Iterator begin() const { return object.begin(); } ALWAYS_INLINE Iterator end() const { return object.end(); } ALWAYS_INLINE size_t size() const { return object.size(); } @@ -156,8 +128,6 @@ struct SimdJSONParser return {res.key, res.value}; } - ALWAYS_INLINE simdjson::dom::object getDom() const { return object; } - private: simdjson::dom::object object; }; @@ -177,8 +147,8 @@ struct SimdJSONParser void reserve(size_t max_size) { if (parser.allocate(max_size) != simdjson::error_code::SUCCESS) - throw Exception{ - "Couldn't allocate " + std::to_string(max_size) + " bytes when parsing JSON", ErrorCodes::CANNOT_ALLOCATE_MEMORY}; + throw Exception{"Couldn't allocate " + std::to_string(max_size) + " bytes when parsing JSON", + ErrorCodes::CANNOT_ALLOCATE_MEMORY}; } private: From b9b28c3b7f2d8f0a4f01d8125c8405a9508ff324 Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 29 May 2021 15:34:39 +0300 Subject: [PATCH 03/42] Fix include --- src/Functions/FunctionSQLJSON.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 1fc6986256d..b3f9db87ac9 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include From 2035c0cd6aa0338ecabb81fa8d5c3c5d260a62ea Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 29 May 2021 16:15:58 +0300 Subject: [PATCH 04/42] Change to IFunction --- src/Functions/FunctionsJSON.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index f066bb1029a..1032ab15ff9 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include From 819ad748a4016b35ad682fae09599f1e6f8282b1 Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 29 May 2021 16:53:55 +0300 Subject: [PATCH 05/42] Fix IFunction --- src/Functions/FunctionSQLJSON.h | 2 +- src/Functions/FunctionsJSON.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index b3f9db87ac9..cee7390d67d 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -8,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 1032ab15ff9..f066bb1029a 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include From b00c3d8f5a911bd6ef3af9a1af50ae0794041734 Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Thu, 3 Jun 2021 20:47:53 +0300 Subject: [PATCH 06/42] Fix build (probably) --- src/Functions/DummyJSONParser.h | 3 ++- src/Functions/FunctionSQLJSON.h | 15 ++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/Functions/DummyJSONParser.h b/src/Functions/DummyJSONParser.h index 3010347e4c4..21b4ebef150 100644 --- a/src/Functions/DummyJSONParser.h +++ b/src/Functions/DummyJSONParser.h @@ -40,7 +40,8 @@ struct DummyJSONParser Array getArray() const { return {}; } Object getObject() const { return {}; } - Element getElement() {return {}; } + Element getElement() { return {}; } + std::ostream & operator<<(std::ostream & os) { return os; } }; /// References an array in a JSON document. diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index cee7390d67d..cf33cf804ce 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -157,11 +157,11 @@ private: }; template typename Impl> -class FunctionSQLJSON : public IFunction +class FunctionSQLJSON : public IFunction, WithConstContext { public: - static FunctionPtr create(const Context & context_) { return std::make_shared(context_); } - FunctionSQLJSON(const Context & context_) : context(context_) { } + static FunctionPtr create(ContextConstPtr context_) { return std::make_shared(context_); } + FunctionSQLJSON(ContextConstPtr context_) : WithConstContext(context_) {} static constexpr auto name = Name::name; String getName() const override { return Name::name; } @@ -182,7 +182,7 @@ public: /// 3. Parser(Tokens, ASTPtr) -> complete AST /// 4. Execute functions, call interpreter for each json (in function) #if USE_SIMDJSON - if (context.getSettingsRef().allow_simdjson) + if (getContext()->getSettingsRef().allow_simdjson) return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); #endif @@ -192,9 +192,6 @@ public: return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); #endif } - -private: - const Context & context; }; struct NameJSONExists @@ -239,9 +236,9 @@ public: /// or Exhausted (if we never found the item) ColumnUInt8 & col_bool = assert_cast(dest); if (status == VisitorStatus::Ok) { - col_bool.insert(0); - } else { col_bool.insert(1); + } else { + col_bool.insert(0); } return true; } From 04daa9b5ed79c358dee587afe009bf8229fe79ef Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Fri, 4 Jun 2021 00:17:51 +0300 Subject: [PATCH 07/42] Second try to fix build --- src/Functions/DummyJSONParser.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Functions/DummyJSONParser.h b/src/Functions/DummyJSONParser.h index 21b4ebef150..e74ab57ded1 100644 --- a/src/Functions/DummyJSONParser.h +++ b/src/Functions/DummyJSONParser.h @@ -41,7 +41,7 @@ struct DummyJSONParser Object getObject() const { return {}; } Element getElement() { return {}; } - std::ostream & operator<<(std::ostream & os) { return os; } + std::ostream & operator<<(std::ostream & os) const { return os; } }; /// References an array in a JSON document. @@ -100,4 +100,9 @@ struct DummyJSONParser #endif }; +ALWAYS_INLINE std::ostream& operator<<(std::ostream& out, DummyJSONParser::Element) +{ + return out; +} + } From bef8b4c2c6b205d639b69571e039fe82f5e9b713 Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Fri, 4 Jun 2021 00:56:06 +0300 Subject: [PATCH 08/42] Add 'inline' to DummyParser's << operator; don't throw exceptioin on USE_RAPIDJSON --- src/Functions/DummyJSONParser.h | 3 +-- src/Functions/FunctionSQLJSON.h | 6 ------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Functions/DummyJSONParser.h b/src/Functions/DummyJSONParser.h index e74ab57ded1..01fdab1abb6 100644 --- a/src/Functions/DummyJSONParser.h +++ b/src/Functions/DummyJSONParser.h @@ -41,7 +41,6 @@ struct DummyJSONParser Object getObject() const { return {}; } Element getElement() { return {}; } - std::ostream & operator<<(std::ostream & os) const { return os; } }; /// References an array in a JSON document. @@ -100,7 +99,7 @@ struct DummyJSONParser #endif }; -ALWAYS_INLINE std::ostream& operator<<(std::ostream& out, DummyJSONParser::Element) +inline ALWAYS_INLINE std::ostream& operator<<(std::ostream& out, DummyJSONParser::Element) { return out; } diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index cf33cf804ce..2d4f51e63c4 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -152,8 +152,6 @@ public: return to; } }; - -private: }; template typename Impl> @@ -184,10 +182,6 @@ public: #if USE_SIMDJSON if (getContext()->getSettingsRef().allow_simdjson) return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); -#endif - -#if USE_RAPIDJSON - throw Exception{"RapidJSON is not supported :(", ErrorCodes::BAD_ARGUMENTS}; #else return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); #endif From 14687eccf7fb29efd8cf92b1354f6bca58c075c6 Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 5 Jun 2021 00:04:53 +0300 Subject: [PATCH 09/42] Fix dollar token for lexer --- src/Functions/ya.make | 5 +++++ src/Parsers/Lexer.cpp | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 2a541369ff4..76fa893791e 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -42,6 +42,7 @@ SRCS( FunctionFile.cpp FunctionHelpers.cpp FunctionJoinGet.cpp + FunctionSQLJSON.cpp FunctionsAES.cpp FunctionsCoding.cpp FunctionsConversion.cpp @@ -74,6 +75,10 @@ SRCS( GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp GeoHash.cpp IFunction.cpp + JSONPath/Parsers/ParserJSONPath.cpp + JSONPath/Parsers/ParserJSONPathMemberAccess.cpp + JSONPath/Parsers/ParserJSONPathQuery.cpp + JSONPath/Parsers/ParserJSONPathRange.cpp TargetSpecific.cpp URL/URLHierarchy.cpp URL/URLPathHierarchy.cpp diff --git a/src/Parsers/Lexer.cpp b/src/Parsers/Lexer.cpp index c3b3cf98a2e..3a6e7a26700 100644 --- a/src/Parsers/Lexer.cpp +++ b/src/Parsers/Lexer.cpp @@ -240,9 +240,6 @@ Token Lexer::nextTokenImpl() case '*': ++pos; return Token(TokenType::Asterisk, token_begin, pos); - case '$': - ++pos; - return Token(TokenType::DollarSign, token_begin, pos); case '/': /// division (/) or start of comment (//, /*) { ++pos; @@ -341,6 +338,9 @@ Token Lexer::nextTokenImpl() } default: + if (*pos == '$' && pos + 1 < end && !isWordCharASCII(pos[1])) { + return Token(TokenType::DollarSign, token_begin, ++pos); + } if (isWordCharASCII(*pos) || *pos == '$') { ++pos; From cdd13b5ab42a53e65fea32235943749f73a5ef8d Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 5 Jun 2021 01:31:55 +0300 Subject: [PATCH 10/42] Style --- src/Functions/FunctionSQLJSON.h | 54 ++++++++++------ src/Functions/JSONPath/ASTs/ASTJSONPath.h | 12 +--- .../JSONPath/ASTs/ASTJSONPathMemberAccess.h | 10 +-- .../JSONPath/ASTs/ASTJSONPathQuery.h | 10 +-- .../JSONPath/ASTs/ASTJSONPathRange.h | 13 +--- .../JSONPath/Generators/GeneratorJSONPath.h | 2 +- .../JSONPath/Generators/IGenerator.h | 2 +- .../JSONPath/Generators/IGenerator_fwd.h | 6 +- src/Functions/JSONPath/Generators/IVisitor.h | 19 +++--- .../Generators/VisitorJSONPathMemberAccess.h | 17 ++--- .../Generators/VisitorJSONPathRange.h | 62 +++++++++++++------ .../JSONPath/Generators/VisitorStatus.h | 7 ++- .../JSONPath/Parsers/ParserJSONPath.cpp | 10 +-- .../JSONPath/Parsers/ParserJSONPath.h | 1 - .../Parsers/ParserJSONPathMemberAccess.h | 2 +- .../JSONPath/Parsers/ParserJSONPathQuery.cpp | 13 ++-- .../JSONPath/Parsers/ParserJSONPathRange.cpp | 44 ++++++++----- .../JSONPath/Parsers/ParserJSONPathRange.h | 1 - 18 files changed, 155 insertions(+), 130 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 2d4f51e63c4..5f478cebad8 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -1,14 +1,15 @@ #pragma once -#include +#include #include -#include #include +#include #include #include #include #include #include +#include #include #include #include @@ -18,7 +19,6 @@ #include #include #include -#include #if !defined(ARCADIA_BUILD) # include "config_functions.h" @@ -28,11 +28,11 @@ namespace DB { namespace ErrorCodes { -extern const int ILLEGAL_COLUMN; -extern const int ILLEGAL_TYPE_OF_ARGUMENT; -extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; -extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; } class FunctionSQLJSONHelpers @@ -159,7 +159,7 @@ class FunctionSQLJSON : public IFunction, WithConstContext { public: static FunctionPtr create(ContextConstPtr context_) { return std::make_shared(context_); } - FunctionSQLJSON(ContextConstPtr context_) : WithConstContext(context_) {} + FunctionSQLJSON(ContextConstPtr context_) : WithConstContext(context_) { } static constexpr auto name = Name::name; String getName() const override { return Name::name; } @@ -220,7 +220,8 @@ public: VisitorStatus status; while ((status = generator_json_path.getNextItem(current_element)) != VisitorStatus::Exhausted) { - if (status == VisitorStatus::Ok) { + if (status == VisitorStatus::Ok) + { break; } current_element = root; @@ -229,9 +230,12 @@ public: /// insert result, status can be either Ok (if we found the item) /// or Exhausted (if we never found the item) ColumnUInt8 & col_bool = assert_cast(dest); - if (status == VisitorStatus::Ok) { + if (status == VisitorStatus::Ok) + { col_bool.insert(1); - } else { + } + else + { col_bool.insert(0); } return true; @@ -256,17 +260,22 @@ public: Element res; while ((status = generator_json_path.getNextItem(current_element)) != VisitorStatus::Exhausted) { - if (status == VisitorStatus::Ok) { - if (!(current_element.isArray() || current_element.isObject())) { + if (status == VisitorStatus::Ok) + { + if (!(current_element.isArray() || current_element.isObject())) + { break; } - } else if (status == VisitorStatus::Error) { + } + else if (status == VisitorStatus::Error) + { /// ON ERROR } current_element = root; } - if (status == VisitorStatus::Exhausted) { + if (status == VisitorStatus::Exhausted) + { return false; } @@ -304,19 +313,24 @@ public: bool success = false; while ((status = generator_json_path.getNextItem(current_element)) != VisitorStatus::Exhausted) { - if (status == VisitorStatus::Ok) { - if (success) { + if (status == VisitorStatus::Ok) + { + if (success) + { out << ", "; } success = true; out << current_element.getElement(); - } else if (status == VisitorStatus::Error) { + } + else if (status == VisitorStatus::Error) + { /// ON ERROR } current_element = root; } out << "]"; - if (!success) { + if (!success) + { return false; } ColumnString & col_str = assert_cast(dest); diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPath.h b/src/Functions/JSONPath/ASTs/ASTJSONPath.h index 092fe16bd9e..dfc117db846 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPath.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPath.h @@ -1,22 +1,16 @@ #pragma once -#include #include +#include namespace DB { class ASTJSONPath : public IAST { public: - String getID(char) const override - { - return "ASTJSONPath"; - } + String getID(char) const override { return "ASTJSONPath"; } - ASTPtr clone() const override - { - return std::make_shared(*this); - } + ASTPtr clone() const override { return std::make_shared(*this); } ASTJSONPathQuery * jsonpath_query; }; diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h b/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h index 663859f566f..2c9482b665e 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathMemberAccess.h @@ -7,15 +7,9 @@ namespace DB class ASTJSONPathMemberAccess : public IAST { public: - String getID(char) const override - { - return "ASTJSONPathMemberAccess"; - } + String getID(char) const override { return "ASTJSONPathMemberAccess"; } - ASTPtr clone() const override - { - return std::make_shared(*this); - } + ASTPtr clone() const override { return std::make_shared(*this); } public: /// Member name to lookup in json document (in path: $.some_key.another_key. ...) diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h b/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h index 8da8a7baafb..ed2992777b2 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathQuery.h @@ -7,15 +7,9 @@ namespace DB class ASTJSONPathQuery : public IAST { public: - String getID(char) const override - { - return "ASTJSONPathQuery"; - } + String getID(char) const override { return "ASTJSONPathQuery"; } - ASTPtr clone() const override - { - return std::make_shared(*this); - } + ASTPtr clone() const override { return std::make_shared(*this); } }; } diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h b/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h index 21af3cff363..8a963d7fc6b 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h @@ -1,23 +1,16 @@ #pragma once -#include #include +#include namespace DB { - class ASTJSONPathRange : public IAST { public: - String getID(char) const override - { - return "ASTJSONPathRange"; - } + String getID(char) const override { return "ASTJSONPathRange"; } - ASTPtr clone() const override - { - return std::make_shared(*this); - } + ASTPtr clone() const override { return std::make_shared(*this); } public: /// Ranges to lookup in json array ($[0, 1, 2, 4 to 9]) diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h index 68ea5a2a3c5..6eea19cb516 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -100,4 +100,4 @@ private: VisitorList visitors; }; -} // namespace DB +} diff --git a/src/Functions/JSONPath/Generators/IGenerator.h b/src/Functions/JSONPath/Generators/IGenerator.h index 18c0ac7da67..d2cef9fe27b 100644 --- a/src/Functions/JSONPath/Generators/IGenerator.h +++ b/src/Functions/JSONPath/Generators/IGenerator.h @@ -26,4 +26,4 @@ public: virtual ~IGenerator() = default; }; -} // namespace DB +} diff --git a/src/Functions/JSONPath/Generators/IGenerator_fwd.h b/src/Functions/JSONPath/Generators/IGenerator_fwd.h index 27c3976b95b..57ed04d0f6f 100644 --- a/src/Functions/JSONPath/Generators/IGenerator_fwd.h +++ b/src/Functions/JSONPath/Generators/IGenerator_fwd.h @@ -2,8 +2,8 @@ #include -namespace DB { - +namespace DB +{ template class IGenerator; @@ -13,4 +13,4 @@ using IVisitorPtr = std::shared_ptr>; template using VisitorList = std::vector>; -} // namespace DB +} diff --git a/src/Functions/JSONPath/Generators/IVisitor.h b/src/Functions/JSONPath/Generators/IVisitor.h index 78c1efe64fc..d9917087cb0 100644 --- a/src/Functions/JSONPath/Generators/IVisitor.h +++ b/src/Functions/JSONPath/Generators/IVisitor.h @@ -2,12 +2,12 @@ #include -namespace DB { - +namespace DB +{ template -class IVisitor { +class IVisitor +{ public: - virtual const char * getName() const = 0; /** @@ -29,17 +29,14 @@ public: virtual void updateState() = 0; - bool isExhausted() { - return is_exhausted; - } + bool isExhausted() { return is_exhausted; } - void setExhausted(bool exhausted) { - is_exhausted = exhausted; - } + void setExhausted(bool exhausted) { is_exhausted = exhausted; } virtual ~IVisitor() = default; + private: bool is_exhausted = false; }; -} // namespace DB +} diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h index cad36e40e4d..10ee2a0c5d6 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h @@ -12,7 +12,8 @@ public: const char * getName() const override { return "VisitorJSONPathMemberAccess"; } - VisitorStatus apply(typename JSONParser::Element & element) const override { + VisitorStatus apply(typename JSONParser::Element & element) const override + { const auto * member_access = member_access_ptr->as(); typename JSONParser::Element result; element.getObject().find(std::string_view(member_access->member_name), result); @@ -22,13 +23,15 @@ public: VisitorStatus visit(typename JSONParser::Element & element) override { - if (!element.isObject()) { + if (!element.isObject()) + { this->setExhausted(true); return VisitorStatus::Error; } const auto * member_access = member_access_ptr->as(); typename JSONParser::Element result; - if (!element.getObject().find(std::string_view(member_access->member_name), result)) { + if (!element.getObject().find(std::string_view(member_access->member_name), result)) + { this->setExhausted(true); return VisitorStatus::Error; } @@ -37,14 +40,12 @@ public: return VisitorStatus::Ok; } - void reinitialize() override { - this->setExhausted(false); - } + void reinitialize() override { this->setExhausted(false); } - void updateState() override {} + void updateState() override { } private: ASTPtr member_access_ptr; }; -} // namespace DB +} diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h index 0858d9b70da..d3313331593 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h @@ -2,28 +2,34 @@ #include #include -namespace DB { - +namespace DB +{ template class VisitorJSONPathRange : public IVisitor { public: - VisitorJSONPathRange(ASTPtr range_ptr_) : range_ptr(range_ptr_) { + VisitorJSONPathRange(ASTPtr range_ptr_) : range_ptr(range_ptr_) + { const auto * range = range_ptr->as(); current_range = 0; - if (range->is_star) { + if (range->is_star) + { current_index = 0; - } else { + } + else + { current_index = range->ranges[current_range].first; } } const char * getName() const override { return "VisitorJSONPathRange"; } - VisitorStatus apply(typename JSONParser::Element & element) const override { + VisitorStatus apply(typename JSONParser::Element & element) const override + { typename JSONParser::Element result; typename JSONParser::Array array = element.getArray(); - if (current_index >= array.size()) { + if (current_index >= array.size()) + { return VisitorStatus::Error; } result = array[current_index]; @@ -33,26 +39,35 @@ public: VisitorStatus visit(typename JSONParser::Element & element) override { - if (!element.isArray()) { + if (!element.isArray()) + { this->setExhausted(true); return VisitorStatus::Error; } const auto * range = range_ptr->as(); VisitorStatus status; - if (current_index < element.getArray().size()) { + if (current_index < element.getArray().size()) + { apply(element); status = VisitorStatus::Ok; - } else if (!range->is_star) { + } + else if (!range->is_star) + { status = VisitorStatus::Ignore; - } else { + } + else + { status = VisitorStatus::Ignore; this->setExhausted(true); } - if (!range->is_star) { - if (current_index + 1 == range->ranges[current_range].second) { - if (current_range + 1 == range->ranges.size()) { + if (!range->is_star) + { + if (current_index + 1 == range->ranges[current_range].second) + { + if (current_range + 1 == range->ranges.size()) + { this->setExhausted(true); } } @@ -61,24 +76,31 @@ public: return status; } - void reinitialize() override { + void reinitialize() override + { const auto * range = range_ptr->as(); current_range = 0; - if (range->is_star) { + if (range->is_star) + { current_index = 0; - } else { + } + else + { current_index = range->ranges[current_range].first; } this->setExhausted(false); } - void updateState() override { + void updateState() override + { const auto * range = range_ptr->as(); current_index++; - if (range->is_star) { + if (range->is_star) + { return; } - if (current_index == range->ranges[current_range].second) { + if (current_index == range->ranges[current_range].second) + { current_range++; current_index = range->ranges[current_range].first; } diff --git a/src/Functions/JSONPath/Generators/VisitorStatus.h b/src/Functions/JSONPath/Generators/VisitorStatus.h index 17b424a3bf6..96b2ea72f18 100644 --- a/src/Functions/JSONPath/Generators/VisitorStatus.h +++ b/src/Functions/JSONPath/Generators/VisitorStatus.h @@ -1,8 +1,9 @@ #pragma once -namespace DB { - -enum VisitorStatus { +namespace DB +{ +enum VisitorStatus +{ Ok, Exhausted, Error, diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp index b65de621f9a..003e97af38b 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPath.cpp @@ -1,11 +1,10 @@ +#include +#include #include #include -#include -#include namespace DB { - /** * Entry parser for JSONPath */ @@ -19,7 +18,8 @@ bool ParserJSONPath::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) bool res = parser_jsonpath_query.parse(pos, query, expected); - if (res) { + if (res) + { /// Set ASTJSONPathQuery of ASTJSONPath ast_jsonpath->set(ast_jsonpath->jsonpath_query, query); } @@ -28,4 +28,4 @@ bool ParserJSONPath::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) return res; } -} // namespace DB +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPath.h b/src/Functions/JSONPath/Parsers/ParserJSONPath.h index 5defc76b515..7d2c2ad642c 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPath.h +++ b/src/Functions/JSONPath/Parsers/ParserJSONPath.h @@ -5,7 +5,6 @@ namespace DB { - /** * Entry parser for JSONPath */ diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h index 49fda6f1ac8..000f20c8551 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h @@ -4,7 +4,7 @@ namespace DB { class ParserJSONPathMemberAccess : public IParserBase { - const char * getName() const override {return "ParserJSONPathMemberAccess";} + const char * getName() const override { return "ParserJSONPathMemberAccess"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp index 10cf6f2915c..0f171c0a82c 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp @@ -1,7 +1,7 @@ -#include -#include -#include #include +#include +#include +#include namespace DB @@ -19,15 +19,16 @@ bool ParserJSONPathQuery::parseImpl(Pos & pos, ASTPtr & query, Expected & expect ParserJSONPathMemberAccess parser_jsonpath_member_access; ParserJSONPathRange parser_jsonpath_range; - if (pos->type != TokenType::DollarSign) { + if (pos->type != TokenType::DollarSign) + { return false; } ++pos; bool res = false; ASTPtr subquery; - while (parser_jsonpath_member_access.parse(pos, subquery, expected) || - parser_jsonpath_range.parse(pos, subquery, expected)) + while (parser_jsonpath_member_access.parse(pos, subquery, expected) + || parser_jsonpath_range.parse(pos, subquery, expected)) { if (subquery) { diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp index 4f4a87f15ce..4027ec67ecb 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp @@ -28,18 +28,22 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte auto range = std::make_shared(); node = range; - if (pos->type != TokenType::OpeningSquareBracket) { + if (pos->type != TokenType::OpeningSquareBracket) + { return false; } ++pos; - while (pos->type != TokenType::ClosingSquareBracket) { + while (pos->type != TokenType::ClosingSquareBracket) + { if (pos->type != TokenType::Number && pos->type != TokenType::Asterisk) { return false; } - if (pos->type == TokenType::Asterisk) { - if (range->is_star) { + if (pos->type == TokenType::Asterisk) + { + if (range->is_star) + { throw Exception{"Multiple asterisks in square array range are not allowed", ErrorCodes::BAD_ARGUMENTS}; } range->is_star = true; @@ -56,18 +60,23 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte } range_indices.first = number_ptr->as()->value.get(); - if (pos->type == TokenType::Comma || pos->type == TokenType::ClosingSquareBracket) { + if (pos->type == TokenType::Comma || pos->type == TokenType::ClosingSquareBracket) + { /// Single index case range_indices.second = range_indices.first + 1; - } else if (pos->type == TokenType::BareWord) { + } + else if (pos->type == TokenType::BareWord) + { /// Range case ParserIdentifier name_p; ASTPtr word; - if (!name_p.parse(pos, word, expected)) { + if (!name_p.parse(pos, word, expected)) + { return false; } String to_identifier; - if (!tryGetIdentifierNameInto(word, to_identifier) || to_identifier != "to") { + if (!tryGetIdentifierNameInto(word, to_identifier) || to_identifier != "to") + { return false; } if (!number_p.parse(pos, number_ptr, expected)) @@ -75,17 +84,24 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte return false; } range_indices.second = number_ptr->as()->value.get(); - } else { + } + else + { return false; } - if (range_indices.first >= range_indices.second) { - throw Exception{ErrorCodes::BAD_ARGUMENTS, "Start of range must be greater than end of range, however {} >= {}", - range_indices.first, range_indices.second}; + if (range_indices.first >= range_indices.second) + { + throw Exception{ + ErrorCodes::BAD_ARGUMENTS, + "Start of range must be greater than end of range, however {} >= {}", + range_indices.first, + range_indices.second}; } range->ranges.push_back(std::move(range_indices)); - if (pos->type != TokenType::ClosingSquareBracket) { + if (pos->type != TokenType::ClosingSquareBracket) + { ++pos; } } @@ -95,4 +111,4 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte return !range->ranges.empty() != range->is_star; } -} // namespace DB +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.h b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.h index 95708e5e7b8..94db29577ab 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.h +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.h @@ -5,7 +5,6 @@ namespace DB { - class ParserJSONPathRange : public IParserBase { private: From 3f469fe0cde1f843286e7aaaa1db62da29098a9c Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 5 Jun 2021 15:13:46 +0300 Subject: [PATCH 11/42] Style --- src/Functions/FunctionSQLJSON.h | 8 +-- .../JSONPath/Generators/GeneratorJSONPath.h | 51 ++++++++++--------- .../Parsers/ParserJSONPathMemberAccess.cpp | 5 +- .../JSONPath/Parsers/ParserJSONPathRange.cpp | 9 ++-- src/Parsers/Lexer.cpp | 3 +- 5 files changed, 41 insertions(+), 35 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 5f478cebad8..67380a11235 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -56,18 +56,18 @@ public: const auto & first_column = arguments[0]; if (!isString(first_column.type)) { - throw Exception{ + throw Exception( "JSONPath functions require 1 argument to be JSONPath of type string, illegal type: " + first_column.type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } /// Check 2 argument: must be of type String (JSON) const auto & second_column = arguments[1]; if (!isString(second_column.type)) { - throw Exception{ + throw Exception( "JSONPath functions require 2 argument to be JSON of string, illegal type: " + second_column.type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } /// If argument is successfully cast to (ColumnConst *) then it is quoted string diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h index 6eea19cb516..450a4f31dae 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -21,28 +21,22 @@ public: { query_ptr = query_ptr_; const auto * path = query_ptr->as(); - if (!path) { + if (!path) + { throw Exception("Invalid path", ErrorCodes::LOGICAL_ERROR); } const auto * query = path->jsonpath_query; for (auto child_ast : query->children) { - if (child_ast->getID() == "ASTJSONPathMemberAccess") + if (typeid_cast(child_ast.get())) { - auto member_access_visitor = std::make_shared>(child_ast); - if (member_access_visitor) { - visitors.push_back(member_access_visitor); - } else { - throw Exception("member_access_visitor could not be nullptr", ErrorCodes::LOGICAL_ERROR); - } - } else if (child_ast->getID() == "ASTJSONPathRange") { - auto range_visitor = std::make_shared>(child_ast); - if (range_visitor) { - visitors.push_back(range_visitor); - } else { - throw Exception("range_visitor could not be nullptr", ErrorCodes::LOGICAL_ERROR); - } + visitors.push_back(std::make_shared>(child_ast)); + } + else if (child_ast->getID() == "ASTJSONPathRange") + { + + visitors.push_back(std::make_shared>(child_ast)); } } } @@ -56,27 +50,33 @@ public: */ VisitorStatus getNextItem(typename JSONParser::Element & element) override { - while (true) { + while (true) + { auto root = element; - if (current_visitor < 0) { + if (current_visitor < 0) + { return VisitorStatus::Exhausted; } - for (int i = 0; i < current_visitor; ++i) { + for (int i = 0; i < current_visitor; ++i) + { visitors[i]->apply(root); } VisitorStatus status = VisitorStatus::Error; - for (size_t i = current_visitor; i < visitors.size(); ++i) { + for (size_t i = current_visitor; i < visitors.size(); ++i) + { status = visitors[i]->visit(root); current_visitor = i; - if (status == VisitorStatus::Error || status == VisitorStatus::Ignore) { + if (status == VisitorStatus::Error || status == VisitorStatus::Ignore) + { break; } } updateVisitorsForNextRun(); - if (status != VisitorStatus::Ignore) { + if (status != VisitorStatus::Ignore) + { element = root; return status; } @@ -84,12 +84,15 @@ public: } private: - bool updateVisitorsForNextRun() { - while (current_visitor >= 0 && visitors[current_visitor]->isExhausted()) { + bool updateVisitorsForNextRun() + { + while (current_visitor >= 0 && visitors[current_visitor]->isExhausted()) + { visitors[current_visitor]->reinitialize(); current_visitor--; } - if (current_visitor >= 0) { + if (current_visitor >= 0) + { visitors[current_visitor]->updateState(); } return current_visitor >= 0; diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp index 10ae128616b..0acb800dc1b 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp @@ -17,8 +17,6 @@ namespace DB */ bool ParserJSONPathMemberAccess::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - auto member_access = std::make_shared(); - node = member_access; if (pos->type != TokenType::Dot) { return false; } @@ -27,12 +25,15 @@ bool ParserJSONPathMemberAccess::parseImpl(Pos & pos, ASTPtr & node, Expected & if (pos->type != TokenType::BareWord) { return false; } + ParserIdentifier name_p; ASTPtr member_name; if (!name_p.parse(pos, member_name, expected)) { return false; } + auto member_access = std::make_shared(); + node = member_access; if (!tryGetIdentifierNameInto(member_name, member_access->member_name)) { return false; } diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp index 4027ec67ecb..25c5c76dd40 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp @@ -25,8 +25,6 @@ namespace ErrorCodes */ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - auto range = std::make_shared(); - node = range; if (pos->type != TokenType::OpeningSquareBracket) { @@ -34,6 +32,9 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte } ++pos; + auto range = std::make_shared(); + node = range; + while (pos->type != TokenType::ClosingSquareBracket) { if (pos->type != TokenType::Number && pos->type != TokenType::Asterisk) @@ -107,8 +108,8 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte } ++pos; - /// We cant have both ranges and star present, so parse was successful <=> exactly 1 of these conditions is true - return !range->ranges.empty() != range->is_star; + /// We can't have both ranges and star present, so parse was successful <=> exactly 1 of these conditions is true + return !range->ranges.empty() ^ range->is_star; } } diff --git a/src/Parsers/Lexer.cpp b/src/Parsers/Lexer.cpp index 3a6e7a26700..4de72ebc2fd 100644 --- a/src/Parsers/Lexer.cpp +++ b/src/Parsers/Lexer.cpp @@ -338,7 +338,8 @@ Token Lexer::nextTokenImpl() } default: - if (*pos == '$' && pos + 1 < end && !isWordCharASCII(pos[1])) { + if (*pos == '$' && pos + 1 < end && !isWordCharASCII(pos[1])) + { return Token(TokenType::DollarSign, token_begin, ++pos); } if (isWordCharASCII(*pos) || *pos == '$') From 582cc3daa91ecccf8c2337ac5e1c306897c839ea Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sat, 5 Jun 2021 15:27:37 +0300 Subject: [PATCH 12/42] Make 1st arg always be const, fix style --- src/Functions/FunctionSQLJSON.cpp | 4 ---- src/Functions/FunctionSQLJSON.h | 24 +++++++++---------- .../JSONPath/Generators/GeneratorJSONPath.h | 2 ++ .../Generators/VisitorJSONPathRange.h | 4 +++- .../Parsers/ParserJSONPathMemberAccess.cpp | 23 +++++++++++------- .../JSONPath/Parsers/ParserJSONPathRange.cpp | 6 +---- 6 files changed, 32 insertions(+), 31 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.cpp b/src/Functions/FunctionSQLJSON.cpp index 7d558dd0950..a316d9de7ab 100644 --- a/src/Functions/FunctionSQLJSON.cpp +++ b/src/Functions/FunctionSQLJSON.cpp @@ -4,10 +4,6 @@ namespace DB { -namespace ErrorCodes -{ -extern const int ILLEGAL_TYPE_OF_ARGUMENT; -} void registerFunctionsSQLJSON(FunctionFactory & factory) { diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 67380a11235..b13b03cb089 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -30,7 +30,6 @@ namespace ErrorCodes { extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; extern const int BAD_ARGUMENTS; } @@ -52,17 +51,24 @@ public: throw Exception{"JSONPath functions require at least 2 arguments", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION}; } - /// Check 1 argument: must be of type String (JSONPath) const auto & first_column = arguments[0]; + + /// Check 1 argument: must be of type String (JSONPath) if (!isString(first_column.type)) { throw Exception( "JSONPath functions require 1 argument to be JSONPath of type string, illegal type: " + first_column.type->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } + /// Check 1 argument: must be const (JSONPath) + if (!isColumnConst(*first_column.column)) + { + throw Exception("1 argument (JSONPath) must be const", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + const auto & second_column = arguments[1]; /// Check 2 argument: must be of type String (JSON) - const auto & second_column = arguments[1]; if (!isString(second_column.type)) { throw Exception( @@ -78,21 +84,15 @@ public: /// Example: /// SomeFunction(database.table.column) - /// Check 1 argument: must be const String (JSONPath) const ColumnPtr & arg_jsonpath = first_column.column; const auto * arg_jsonpath_const = typeid_cast(arg_jsonpath.get()); - if (!arg_jsonpath_const) - { - throw Exception{"JSONPath argument must be of type const String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; - } - /// Retrieve data from 1 argument const auto * arg_jsonpath_string = typeid_cast(arg_jsonpath_const->getDataColumnPtr().get()); + if (!arg_jsonpath_string) { throw Exception{"Illegal column " + arg_jsonpath->getName(), ErrorCodes::ILLEGAL_COLUMN}; } - /// Check 2 argument: must be const or non-const String (JSON) const ColumnPtr & arg_json = second_column.column; const auto * col_json_const = typeid_cast(arg_json.get()); const auto * col_json_string @@ -166,6 +166,7 @@ public: bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { @@ -182,9 +183,8 @@ public: #if USE_SIMDJSON if (getContext()->getSettingsRef().allow_simdjson) return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); -#else - return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); #endif + return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); } }; diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h index 450a4f31dae..39f385fafc2 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -1,3 +1,5 @@ +#pragma once + #include #include #include diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h index d3313331593..601c5ea80b9 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h @@ -1,3 +1,5 @@ +#pragma once + #include #include #include @@ -112,4 +114,4 @@ private: UInt32 current_index; }; -} // namespace +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp index 0acb800dc1b..841e3fc4adc 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp @@ -1,13 +1,14 @@ -#include -#include +#pragma once + +#include +#include -#include -#include #include +#include +#include namespace DB { - /** * * @param pos token iterator @@ -17,24 +18,28 @@ namespace DB */ bool ParserJSONPathMemberAccess::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - if (pos->type != TokenType::Dot) { + if (pos->type != TokenType::Dot) + { return false; } ++pos; - if (pos->type != TokenType::BareWord) { + if (pos->type != TokenType::BareWord) + { return false; } ParserIdentifier name_p; ASTPtr member_name; - if (!name_p.parse(pos, member_name, expected)) { + if (!name_p.parse(pos, member_name, expected)) + { return false; } auto member_access = std::make_shared(); node = member_access; - if (!tryGetIdentifierNameInto(member_name, member_access->member_name)) { + if (!tryGetIdentifierNameInto(member_name, member_access->member_name)) + { return false; } return true; diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp index 25c5c76dd40..f0d1af47b15 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp @@ -10,10 +10,6 @@ namespace DB { namespace ErrorCodes { - extern const int ILLEGAL_COLUMN; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; extern const int BAD_ARGUMENTS; } /** @@ -45,7 +41,7 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte { if (range->is_star) { - throw Exception{"Multiple asterisks in square array range are not allowed", ErrorCodes::BAD_ARGUMENTS}; + throw Exception("Multiple asterisks in square array range are not allowed", ErrorCodes::BAD_ARGUMENTS); } range->is_star = true; ++pos; From dfba5a479b5eaa1e0f972be7977b463ee1347549 Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sun, 6 Jun 2021 14:50:10 +0300 Subject: [PATCH 13/42] Remove pragma once --- src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp index 841e3fc4adc..85b43217867 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp @@ -1,5 +1,3 @@ -#pragma once - #include #include From bad7d56aaa69feb74982159b343483e233a61629 Mon Sep 17 00:00:00 2001 From: Konstantin Rudenskii Date: Sun, 6 Jun 2021 16:00:46 +0300 Subject: [PATCH 14/42] =?UTF-8?q?Style=20again=20=F0=9F=98=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../JSONPath/Generators/VisitorJSONPathMemberAccess.h | 2 ++ src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h | 2 ++ src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h index 10ee2a0c5d6..fd83c478227 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h @@ -1,3 +1,5 @@ +#pragma once + #include #include #include diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h index 000f20c8551..b28bf37d5ef 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.h @@ -1,3 +1,5 @@ +#pragma once + #include namespace DB diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp index f0d1af47b15..3da0d508b27 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp @@ -89,11 +89,11 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte if (range_indices.first >= range_indices.second) { - throw Exception{ + throw Exception( ErrorCodes::BAD_ARGUMENTS, "Start of range must be greater than end of range, however {} >= {}", range_indices.first, - range_indices.second}; + range_indices.second); } range->ranges.push_back(std::move(range_indices)); From ba37ebb1335a36bd5b708870de9c02e10e29e500 Mon Sep 17 00:00:00 2001 From: "d.v.semenov" Date: Mon, 7 Jun 2021 23:54:44 +0300 Subject: [PATCH 15/42] Added functional tests --- .../01889_sql_json_functions.reference | 33 +++++++++++++++++ .../0_stateless/01889_sql_json_functions.sql | 35 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 tests/queries/0_stateless/01889_sql_json_functions.reference create mode 100644 tests/queries/0_stateless/01889_sql_json_functions.sql diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference new file mode 100644 index 00000000000..b8dc5d8e416 --- /dev/null +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -0,0 +1,33 @@ +--JSON_VALUE-- +1 +1.2 +true +"world" +null + + + + +--JSON_QUERY-- +[1] +[1.2] +[true] +["world"] +[null] +[["world","world2"]] +[{"world":"!"}] + + +--JSON_EXISTS-- +1 +1 +0 +1 +0 +0 +1 +1 +0 +1 +0 +1 \ No newline at end of file diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql new file mode 100644 index 00000000000..6d67b73f305 --- /dev/null +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -0,0 +1,35 @@ +SELECT '--JSON_VALUE--'; +SELECT JSON_VALUE('$.hello', '{"hello":1}'); +SELECT JSON_VALUE('$.hello', '{"hello":1.2}'); +SELECT JSON_VALUE('$.hello', '{"hello":true}'); +SELECT JSON_VALUE('$.hello', '{"hello":"world"}'); +SELECT JSON_VALUE('$.hello', '{"hello":null}'); +SELECT JSON_VALUE('$.hello', '{"hello":["world","world2"]}'); +SELECT JSON_VALUE('$.hello', '{"hello":{"world":"!"}}'); +SELECT JSON_VALUE('$.hello', '{hello:world}'); -- invalid json => default value (empty string) +SELECT JSON_VALUE('$.hello', ''); + +SELECT '--JSON_QUERY--'; +SELECT JSON_QUERY('$.hello', '{"hello":1}'); +SELECT JSON_QUERY('$.hello', '{"hello":1.2}'); +SELECT JSON_QUERY('$.hello', '{"hello":true}'); +SELECT JSON_QUERY('$.hello', '{"hello":"world"}'); +SELECT JSON_QUERY('$.hello', '{"hello":null}'); +SELECT JSON_QUERY('$.hello', '{"hello":["world","world2"]}'); +SELECT JSON_QUERY('$.hello', '{"hello":{"world":"!"}}'); +SELECT JSON_QUERY('$.hello', '{hello:{"world":"!"}}}'); -- invalid json => default value (empty string) +SELECT JSON_QUERY('$.hello', ''); + +SELECT '--JSON_EXISTS--'; +SELECT JSON_EXISTS('$.hello', '{"hello":1}'); +SELECT JSON_EXISTS('$.world', '{"hello":1,"world":2}'); +SELECT JSON_EXISTS('$.world', '{"hello":{"world":1}}'); +SELECT JSON_EXISTS('$.hello.world', '{"hello":{"world":1}}'); +SELECT JSON_EXISTS('$.hello', '{hello:world}'); -- invalid json => default value (zero integer) +SELECT JSON_EXISTS('$.hello', ''); +SELECT JSON_EXISTS('$.hello[*]', '{"hello":["world"]}'); +SELECT JSON_EXISTS('$.hello[0]', '{"hello":["world"]}'); +SELECT JSON_EXISTS('$.hello[1]', '{"hello":["world"]}'); +SELECT JSON_EXISTS('$.a[*].b', '{"a":[{"b":1},{"c":2}]}'); +SELECT JSON_EXISTS('$.a[*].f', '{"a":[{"b":1},{"c":2}]}'); +SELECT JSON_EXISTS('$.a[*][0].h', '{"a":[[{"b":1}, {"g":1}],[{"h":1},{"y":1}]]}'); \ No newline at end of file From 00ce6f69dfc431442e106da62aae7c581bed324d Mon Sep 17 00:00:00 2001 From: "d.v.semenov" Date: Wed, 9 Jun 2021 12:46:17 +0300 Subject: [PATCH 16/42] Added newline --- tests/queries/0_stateless/01889_sql_json_functions.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index b8dc5d8e416..7457aca18ed 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -30,4 +30,4 @@ null 0 1 0 -1 \ No newline at end of file +1 From ff3857fbe73d354bf446d85466b127462212e5b3 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Thu, 17 Jun 2021 12:24:10 +0300 Subject: [PATCH 17/42] Add root parsing --- src/Functions/JSONPath/ASTs/ASTJSONPathRoot.h | 15 ++++++++ .../JSONPath/Generators/GeneratorJSONPath.h | 21 ++++++++--- src/Functions/JSONPath/Generators/IVisitor.h | 4 +++ .../JSONPath/Generators/VisitorJSONPathRoot.h | 35 +++++++++++++++++++ .../JSONPath/Parsers/ParserJSONPathQuery.cpp | 28 +++++++-------- .../JSONPath/Parsers/ParserJSONPathRoot.cpp | 28 +++++++++++++++ .../JSONPath/Parsers/ParserJSONPathRoot.h | 18 ++++++++++ src/Parsers/Lexer.cpp | 3 +- .../01889_sql_json_functions.reference | 5 +++ .../0_stateless/01889_sql_json_functions.sql | 5 +++ 10 files changed, 143 insertions(+), 19 deletions(-) create mode 100644 src/Functions/JSONPath/ASTs/ASTJSONPathRoot.h create mode 100644 src/Functions/JSONPath/Generators/VisitorJSONPathRoot.h create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathRoot.h diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathRoot.h b/src/Functions/JSONPath/ASTs/ASTJSONPathRoot.h new file mode 100644 index 00000000000..1c6469c5b75 --- /dev/null +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathRoot.h @@ -0,0 +1,15 @@ +#pragma once + +#include + +namespace DB +{ +class ASTJSONPathRoot : public IAST +{ +public: + String getID(char) const override { return "ASTJSONPathRoot"; } + + ASTPtr clone() const override { return std::make_shared(*this); } +}; + +} diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h index 39f385fafc2..2583ef8c921 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -19,6 +20,10 @@ template class GeneratorJSONPath : public IGenerator { public: + /** + * Traverses children ASTs of ASTJSONPathQuery and creates a vector of corresponding visitors + * @param query_ptr_ pointer to ASTJSONPathQuery + */ GeneratorJSONPath(ASTPtr query_ptr_) { query_ptr = query_ptr_; @@ -31,13 +36,15 @@ public: for (auto child_ast : query->children) { - if (typeid_cast(child_ast.get())) + if (typeid_cast(child_ast.get())) { + visitors.push_back(std::make_shared>(child_ast)); + } + else if (typeid_cast(child_ast.get())) { visitors.push_back(std::make_shared>(child_ast)); } - else if (child_ast->getID() == "ASTJSONPathRange") + else if (typeid_cast(child_ast.get())) { - visitors.push_back(std::make_shared>(child_ast)); } } @@ -46,7 +53,13 @@ public: const char * getName() const override { return "GeneratorJSONPath"; } /** - * The only generator which is called from JSONPath functions. + * This method exposes API of traversing all paths, described by JSONPath, + * to SQLJSON Functions. + * Expected usage is to iteratively call this method from inside the function + * and to execute custom logic with received element or handle an error. + * On each such call getNextItem will yield next item into element argument + * and modify its internal state to prepare for next call. + * * @param element root of JSON document * @return is the generator exhausted */ diff --git a/src/Functions/JSONPath/Generators/IVisitor.h b/src/Functions/JSONPath/Generators/IVisitor.h index d9917087cb0..1461b842829 100644 --- a/src/Functions/JSONPath/Generators/IVisitor.h +++ b/src/Functions/JSONPath/Generators/IVisitor.h @@ -36,6 +36,10 @@ public: virtual ~IVisitor() = default; private: + /** + * This variable is for detecting whether a visitor's next visit will be able + * to yield a new item. + */ bool is_exhausted = false; }; diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathRoot.h b/src/Functions/JSONPath/Generators/VisitorJSONPathRoot.h new file mode 100644 index 00000000000..d8b88ce0255 --- /dev/null +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathRoot.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ +template +class VisitorJSONPathRoot : public IVisitor +{ +public: + VisitorJSONPathRoot(ASTPtr) { } + + const char * getName() const override { return "VisitorJSONPathRoot"; } + + VisitorStatus apply(typename JSONParser::Element & /*element*/) const override + { + /// No-op on document, since we are already passed document's root + return VisitorStatus::Ok; + } + + VisitorStatus visit(typename JSONParser::Element & element) override + { + apply(element); + this->setExhausted(true); + return VisitorStatus::Ok; + } + + void reinitialize() override { this->setExhausted(false); } + + void updateState() override { } +}; + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp index 0f171c0a82c..0ab09733890 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp @@ -1,6 +1,7 @@ #include -#include #include +#include +#include #include namespace DB @@ -18,27 +19,26 @@ bool ParserJSONPathQuery::parseImpl(Pos & pos, ASTPtr & query, Expected & expect query = std::make_shared(); ParserJSONPathMemberAccess parser_jsonpath_member_access; ParserJSONPathRange parser_jsonpath_range; + ParserJSONPathRoot parser_jsonpath_root; - if (pos->type != TokenType::DollarSign) - { + ASTPtr path_root; + if (!parser_jsonpath_root.parse(pos, path_root, expected)) { return false; } - ++pos; + query->children.push_back(path_root); - bool res = false; - ASTPtr subquery; - while (parser_jsonpath_member_access.parse(pos, subquery, expected) - || parser_jsonpath_range.parse(pos, subquery, expected)) + ASTPtr accessor; + while (parser_jsonpath_member_access.parse(pos, accessor, expected) + || parser_jsonpath_range.parse(pos, accessor, expected)) { - if (subquery) + if (accessor) { - query->children.push_back(subquery); - subquery = nullptr; + query->children.push_back(accessor); + accessor = nullptr; } - res = true; } - /// if we had at least one success and no fails - return res && pos->type == TokenType::EndOfStream; + /// parsing was successful if we reached the end of query by this point + return pos->type == TokenType::EndOfStream; } } diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp new file mode 100644 index 00000000000..a67d284e40c --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp @@ -0,0 +1,28 @@ +#include +#include + +#include + +namespace DB +{ +/** + * + * @param pos token iterator + * @param node node of ASTJSONPathRoot + * @param expected stuff for logging + * @return was parse successful + */ +bool ParserJSONPathRoot::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + if (pos->type != TokenType::DollarSign) + { + expected.add(pos, "dollar sign (start of jsonpath)"); + return false; + } + auto path_root = std::make_shared(); + node = path_root; + ++pos; + return true; +} + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.h b/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.h new file mode 100644 index 00000000000..59fed28d63e --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.h @@ -0,0 +1,18 @@ +#pragma once + +#include + + +namespace DB +{ +class ParserJSONPathRoot : public IParserBase +{ +private: + const char * getName() const override { return "ParserJSONPathRoot"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +public: + explicit ParserJSONPathRoot() = default; +}; + +} diff --git a/src/Parsers/Lexer.cpp b/src/Parsers/Lexer.cpp index 4de72ebc2fd..be956ee705a 100644 --- a/src/Parsers/Lexer.cpp +++ b/src/Parsers/Lexer.cpp @@ -338,8 +338,9 @@ Token Lexer::nextTokenImpl() } default: - if (*pos == '$' && pos + 1 < end && !isWordCharASCII(pos[1])) + if (*pos == '$' && ((pos + 1 < end && !isWordCharASCII(pos[1])) || pos + 1 == end)) { + /// Capture standalone dollar sign return Token(TokenType::DollarSign, token_begin, ++pos); } if (isWordCharASCII(*pos) || *pos == '$') diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index 7457aca18ed..e38058ffc50 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -1,4 +1,5 @@ --JSON_VALUE-- + 1 1.2 true @@ -9,6 +10,7 @@ null --JSON_QUERY-- +[{"hello":1}] [1] [1.2] [true] @@ -20,6 +22,9 @@ null --JSON_EXISTS-- 1 +0 +1 +1 1 0 1 diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 6d67b73f305..a1749b3be24 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -1,4 +1,5 @@ SELECT '--JSON_VALUE--'; +SELECT JSON_VALUE('$', '{"hello":1}'); -- root is a complex object => default value (empty string) SELECT JSON_VALUE('$.hello', '{"hello":1}'); SELECT JSON_VALUE('$.hello', '{"hello":1.2}'); SELECT JSON_VALUE('$.hello', '{"hello":true}'); @@ -10,6 +11,7 @@ SELECT JSON_VALUE('$.hello', '{hello:world}'); -- invalid json => default value SELECT JSON_VALUE('$.hello', ''); SELECT '--JSON_QUERY--'; +SELECT JSON_QUERY('$', '{"hello":1}'); SELECT JSON_QUERY('$.hello', '{"hello":1}'); SELECT JSON_QUERY('$.hello', '{"hello":1.2}'); SELECT JSON_QUERY('$.hello', '{"hello":true}'); @@ -21,6 +23,9 @@ SELECT JSON_QUERY('$.hello', '{hello:{"world":"!"}}}'); -- invalid json => defau SELECT JSON_QUERY('$.hello', ''); SELECT '--JSON_EXISTS--'; +SELECT JSON_EXISTS('$', '{"hello":1}'); +SELECT JSON_EXISTS('$', ''); +SELECT JSON_EXISTS('$', '{}'); SELECT JSON_EXISTS('$.hello', '{"hello":1}'); SELECT JSON_EXISTS('$.world', '{"hello":1,"world":2}'); SELECT JSON_EXISTS('$.world', '{"hello":{"world":1}}'); From f44d6792b07733c39980f4b0ff633610ccd4e3fa Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Thu, 17 Jun 2021 14:48:05 +0300 Subject: [PATCH 18/42] Fix ext::range --- src/Functions/FunctionSQLJSON.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index b13b03cb089..b605645499e 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -18,7 +18,7 @@ #include #include #include -#include +#include #if !defined(ARCADIA_BUILD) # include "config_functions.h" @@ -133,7 +133,7 @@ public: /// Parse JSON for every row Impl impl; - for (const auto i : ext::range(0, input_rows_count)) + for (const auto i : collections::range(0, input_rows_count)) { std::string_view json{ reinterpret_cast(&chars_json[offsets_json[i - 1]]), offsets_json[i] - offsets_json[i - 1] - 1}; From e2765991b04d3ef312c9e0e3370d1b23ad49cf52 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Fri, 18 Jun 2021 01:37:59 +0300 Subject: [PATCH 19/42] Separate star from ranges for better code --- src/Functions/JSONPath/ASTs/ASTJSONPathStar.h | 15 +++++ .../JSONPath/Generators/GeneratorJSONPath.h | 8 ++- .../Generators/VisitorJSONPathMemberAccess.h | 11 ++-- .../Generators/VisitorJSONPathRange.h | 57 +++------------- .../JSONPath/Generators/VisitorJSONPathStar.h | 66 +++++++++++++++++++ .../JSONPath/Parsers/ParserJSONPathQuery.cpp | 8 ++- .../JSONPath/Parsers/ParserJSONPathRange.cpp | 28 ++------ .../JSONPath/Parsers/ParserJSONPathStar.cpp | 31 +++++++++ .../JSONPath/Parsers/ParserJSONPathStar.h | 18 +++++ 9 files changed, 163 insertions(+), 79 deletions(-) create mode 100644 src/Functions/JSONPath/ASTs/ASTJSONPathStar.h create mode 100644 src/Functions/JSONPath/Generators/VisitorJSONPathStar.h create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp create mode 100644 src/Functions/JSONPath/Parsers/ParserJSONPathStar.h diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathStar.h b/src/Functions/JSONPath/ASTs/ASTJSONPathStar.h new file mode 100644 index 00000000000..2aada47c459 --- /dev/null +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathStar.h @@ -0,0 +1,15 @@ +#pragma once + +#include + +namespace DB +{ +class ASTJSONPathStar : public IAST +{ +public: + String getID(char) const override { return "ASTJSONPathStar"; } + + ASTPtr clone() const override { return std::make_shared(*this); } +}; + +} diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h index 2583ef8c921..071a7ac3089 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -36,7 +37,8 @@ public: for (auto child_ast : query->children) { - if (typeid_cast(child_ast.get())) { + if (typeid_cast(child_ast.get())) + { visitors.push_back(std::make_shared>(child_ast)); } else if (typeid_cast(child_ast.get())) @@ -47,6 +49,10 @@ public: { visitors.push_back(std::make_shared>(child_ast)); } + else if (typeid_cast(child_ast.get())) + { + visitors.push_back(std::make_shared>(child_ast)); + } } } diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h index fd83c478227..b0c601458b6 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h @@ -10,15 +10,15 @@ template class VisitorJSONPathMemberAccess : public IVisitor { public: - VisitorJSONPathMemberAccess(ASTPtr member_access_ptr_) : member_access_ptr(member_access_ptr_) { } + VisitorJSONPathMemberAccess(ASTPtr member_access_ptr_) + : member_access_ptr(member_access_ptr_->as()) { } const char * getName() const override { return "VisitorJSONPathMemberAccess"; } VisitorStatus apply(typename JSONParser::Element & element) const override { - const auto * member_access = member_access_ptr->as(); typename JSONParser::Element result; - element.getObject().find(std::string_view(member_access->member_name), result); + element.getObject().find(std::string_view(member_access_ptr->member_name), result); element = result; return VisitorStatus::Ok; } @@ -30,9 +30,8 @@ public: this->setExhausted(true); return VisitorStatus::Error; } - const auto * member_access = member_access_ptr->as(); typename JSONParser::Element result; - if (!element.getObject().find(std::string_view(member_access->member_name), result)) + if (!element.getObject().find(std::string_view(member_access_ptr->member_name), result)) { this->setExhausted(true); return VisitorStatus::Error; @@ -47,7 +46,7 @@ public: void updateState() override { } private: - ASTPtr member_access_ptr; + ASTJSONPathMemberAccess * member_access_ptr; }; } diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h index 601c5ea80b9..57e208271d0 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h @@ -10,18 +10,10 @@ template class VisitorJSONPathRange : public IVisitor { public: - VisitorJSONPathRange(ASTPtr range_ptr_) : range_ptr(range_ptr_) + VisitorJSONPathRange(ASTPtr range_ptr_) : range_ptr(range_ptr_->as()) { - const auto * range = range_ptr->as(); current_range = 0; - if (range->is_star) - { - current_index = 0; - } - else - { - current_index = range->ranges[current_range].first; - } + current_index = range_ptr->ranges[current_range].first; } const char * getName() const override { return "VisitorJSONPathRange"; } @@ -30,12 +22,7 @@ public: { typename JSONParser::Element result; typename JSONParser::Array array = element.getArray(); - if (current_index >= array.size()) - { - return VisitorStatus::Error; - } - result = array[current_index]; - element = result; + element = array[current_index]; return VisitorStatus::Ok; } @@ -47,32 +34,21 @@ public: return VisitorStatus::Error; } - const auto * range = range_ptr->as(); VisitorStatus status; if (current_index < element.getArray().size()) { apply(element); status = VisitorStatus::Ok; } - else if (!range->is_star) - { - status = VisitorStatus::Ignore; - } else { status = VisitorStatus::Ignore; - this->setExhausted(true); } - if (!range->is_star) + if (current_index + 1 == range_ptr->ranges[current_range].second + && current_range + 1 == range_ptr->ranges.size()) { - if (current_index + 1 == range->ranges[current_range].second) - { - if (current_range + 1 == range->ranges.size()) - { - this->setExhausted(true); - } - } + this->setExhausted(true); } return status; @@ -80,36 +56,23 @@ public: void reinitialize() override { - const auto * range = range_ptr->as(); current_range = 0; - if (range->is_star) - { - current_index = 0; - } - else - { - current_index = range->ranges[current_range].first; - } + current_index = range_ptr->ranges[current_range].first; this->setExhausted(false); } void updateState() override { - const auto * range = range_ptr->as(); current_index++; - if (range->is_star) - { - return; - } - if (current_index == range->ranges[current_range].second) + if (current_index == range_ptr->ranges[current_range].second) { current_range++; - current_index = range->ranges[current_range].first; + current_index = range_ptr->ranges[current_range].first; } } private: - ASTPtr range_ptr; + ASTJSONPathRange * range_ptr; size_t current_range; UInt32 current_index; }; diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathStar.h b/src/Functions/JSONPath/Generators/VisitorJSONPathStar.h new file mode 100644 index 00000000000..bc840597f2a --- /dev/null +++ b/src/Functions/JSONPath/Generators/VisitorJSONPathStar.h @@ -0,0 +1,66 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ +template +class VisitorJSONPathStar : public IVisitor +{ +public: + VisitorJSONPathStar(ASTPtr) + { + current_index = 0; + } + + const char * getName() const override { return "VisitorJSONPathStar"; } + + VisitorStatus apply(typename JSONParser::Element & element) const override + { + typename JSONParser::Element result; + typename JSONParser::Array array = element.getArray(); + element = array[current_index]; + return VisitorStatus::Ok; + } + + VisitorStatus visit(typename JSONParser::Element & element) override + { + if (!element.isArray()) + { + this->setExhausted(true); + return VisitorStatus::Error; + } + + VisitorStatus status; + if (current_index < element.getArray().size()) + { + apply(element); + status = VisitorStatus::Ok; + } + else + { + status = VisitorStatus::Ignore; + this->setExhausted(true); + } + + return status; + } + + void reinitialize() override + { + current_index = 0; + this->setExhausted(false); + } + + void updateState() override + { + current_index++; + } + +private: + UInt32 current_index; +}; + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp index 0ab09733890..c18b2ad9b31 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB @@ -19,17 +20,20 @@ bool ParserJSONPathQuery::parseImpl(Pos & pos, ASTPtr & query, Expected & expect query = std::make_shared(); ParserJSONPathMemberAccess parser_jsonpath_member_access; ParserJSONPathRange parser_jsonpath_range; + ParserJSONPathStar parser_jsonpath_star; ParserJSONPathRoot parser_jsonpath_root; ASTPtr path_root; - if (!parser_jsonpath_root.parse(pos, path_root, expected)) { + if (!parser_jsonpath_root.parse(pos, path_root, expected)) + { return false; } query->children.push_back(path_root); ASTPtr accessor; while (parser_jsonpath_member_access.parse(pos, accessor, expected) - || parser_jsonpath_range.parse(pos, accessor, expected)) + || parser_jsonpath_range.parse(pos, accessor, expected) + || parser_jsonpath_star.parse(pos, accessor, expected)) { if (accessor) { diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp index 3da0d508b27..f8496cd67d0 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -31,26 +32,16 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte auto range = std::make_shared(); node = range; + ParserNumber number_p; + ASTPtr number_ptr; while (pos->type != TokenType::ClosingSquareBracket) { - if (pos->type != TokenType::Number && pos->type != TokenType::Asterisk) + if (pos->type != TokenType::Number) { return false; } - if (pos->type == TokenType::Asterisk) - { - if (range->is_star) - { - throw Exception("Multiple asterisks in square array range are not allowed", ErrorCodes::BAD_ARGUMENTS); - } - range->is_star = true; - ++pos; - continue; - } std::pair range_indices; - ParserNumber number_p; - ASTPtr number_ptr; if (!number_p.parse(pos, number_ptr, expected)) { return false; @@ -64,16 +55,7 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte } else if (pos->type == TokenType::BareWord) { - /// Range case - ParserIdentifier name_p; - ASTPtr word; - if (!name_p.parse(pos, word, expected)) - { - return false; - } - String to_identifier; - if (!tryGetIdentifierNameInto(word, to_identifier) || to_identifier != "to") - { + if (!ParserKeyword("TO").ignore(pos, expected)) { return false; } if (!number_p.parse(pos, number_ptr, expected)) diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp new file mode 100644 index 00000000000..c0d2b376794 --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp @@ -0,0 +1,31 @@ +#include + +#include + +namespace DB +{ +bool ParserJSONPathStar::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + + if (pos->type != TokenType::OpeningSquareBracket) + { + return false; + } + ++pos; + if (pos->type != TokenType::Asterisk) { + return false; + } + ++pos; + if (pos->type != TokenType::ClosingSquareBracket) { + expected.add(pos, "Closing square bracket"); + return false; + } + ++pos; + + auto star = std::make_shared(); + node = star; + + return true; +} + +} diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathStar.h b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.h new file mode 100644 index 00000000000..543823357de --- /dev/null +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.h @@ -0,0 +1,18 @@ +#pragma once + +#include + + +namespace DB +{ +class ParserJSONPathStar : public IParserBase +{ +private: + const char * getName() const override { return "ParserJSONPathStar"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; + +public: + explicit ParserJSONPathStar() = default; +}; + +} From 1863a9beb044104e7dd7bab0f97508f46a41a3bb Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Fri, 18 Jun 2021 02:15:15 +0300 Subject: [PATCH 20/42] Change stringstream to WriteBuffer --- src/Functions/FunctionSQLJSON.h | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index b605645499e..9bfb4291ba8 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -279,11 +281,11 @@ public: return false; } - std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + String result; + WriteBufferFromString out(result); out << current_element.getElement(); - auto output_str = out.str(); ColumnString & col_str = assert_cast(dest); - col_str.insertData(output_str.data(), output_str.size()); + col_str.insertData(result.data(), result.size()); return true; } }; @@ -307,7 +309,9 @@ public: GeneratorJSONPath generator_json_path(query_ptr); Element current_element = root; VisitorStatus status; - std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + String result; + WriteBufferFromString out(result); + /// Create json array of results: [res1, res2, ...] out << "["; bool success = false; @@ -334,8 +338,7 @@ public: return false; } ColumnString & col_str = assert_cast(dest); - auto output_str = out.str(); - col_str.insertData(output_str.data(), output_str.size()); + col_str.insertData(reinterpret_cast(result.data()), result.size()); return true; } }; From a5d3600f202d70aaf62231fbf231325270c7f881 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Fri, 18 Jun 2021 10:07:53 +0300 Subject: [PATCH 21/42] Fix compile errors with WriteBuffer --- src/Functions/DummyJSONParser.h | 6 ++++-- src/Functions/FunctionSQLJSON.h | 4 ++-- src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp | 3 ++- src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp | 3 +-- src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp | 4 +--- src/Functions/SimdJSONParser.h | 6 +++++- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/Functions/DummyJSONParser.h b/src/Functions/DummyJSONParser.h index 01fdab1abb6..128ee88e0ca 100644 --- a/src/Functions/DummyJSONParser.h +++ b/src/Functions/DummyJSONParser.h @@ -2,6 +2,8 @@ #include #include +#include +#include namespace DB { @@ -40,7 +42,7 @@ struct DummyJSONParser Array getArray() const { return {}; } Object getObject() const { return {}; } - Element getElement() { return {}; } + ALWAYS_INLINE Element getUnderlyingElement() const { return {}; } }; /// References an array in a JSON document. @@ -99,7 +101,7 @@ struct DummyJSONParser #endif }; -inline ALWAYS_INLINE std::ostream& operator<<(std::ostream& out, DummyJSONParser::Element) +inline ALWAYS_INLINE WriteBufferFromString& operator<<(WriteBufferFromString& out, const DB::DummyJSONParser::Element &) { return out; } diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 9bfb4291ba8..3ff1b575bfc 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -283,7 +283,7 @@ public: String result; WriteBufferFromString out(result); - out << current_element.getElement(); + out << current_element; ColumnString & col_str = assert_cast(dest); col_str.insertData(result.data(), result.size()); return true; @@ -324,7 +324,7 @@ public: out << ", "; } success = true; - out << current_element.getElement(); + out << current_element; } else if (status == VisitorStatus::Error) { diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp index f8496cd67d0..bc153b9d747 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRange.cpp @@ -55,7 +55,8 @@ bool ParserJSONPathRange::parseImpl(Pos & pos, ASTPtr & node, Expected & expecte } else if (pos->type == TokenType::BareWord) { - if (!ParserKeyword("TO").ignore(pos, expected)) { + if (!ParserKeyword("TO").ignore(pos, expected)) + { return false; } if (!number_p.parse(pos, number_ptr, expected)) diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp index a67d284e40c..86cf793fb52 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathRoot.cpp @@ -19,8 +19,7 @@ bool ParserJSONPathRoot::parseImpl(Pos & pos, ASTPtr & node, Expected & expected expected.add(pos, "dollar sign (start of jsonpath)"); return false; } - auto path_root = std::make_shared(); - node = path_root; + node = std::make_shared(); ++pos; return true; } diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp index c0d2b376794..97ab9ffec36 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp @@ -6,7 +6,6 @@ namespace DB { bool ParserJSONPathStar::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - if (pos->type != TokenType::OpeningSquareBracket) { return false; @@ -22,8 +21,7 @@ bool ParserJSONPathStar::parseImpl(Pos & pos, ASTPtr & node, Expected & expected } ++pos; - auto star = std::make_shared(); - node = star; + node = std::make_shared(); return true; } diff --git a/src/Functions/SimdJSONParser.h b/src/Functions/SimdJSONParser.h index c5793088baf..a176f2c5961 100644 --- a/src/Functions/SimdJSONParser.h +++ b/src/Functions/SimdJSONParser.h @@ -50,7 +50,7 @@ struct SimdJSONParser ALWAYS_INLINE Array getArray() const; ALWAYS_INLINE Object getObject() const; - ALWAYS_INLINE simdjson::dom::element getElement() const { return element; } + ALWAYS_INLINE simdjson::dom::element getUnderlyingElement() const { return element; } private: simdjson::dom::element element; @@ -165,6 +165,10 @@ inline ALWAYS_INLINE SimdJSONParser::Object SimdJSONParser::Element::getObject() return element.get_object().value_unsafe(); } +inline ALWAYS_INLINE WriteBuffer& operator<<(WriteBuffer& out, const DB::SimdJSONParser::Element & element) { + return out << element.getUnderlyingElement(); +} + } #endif From ec7ec63a40605b7ba4afdd1ffb436d352b07a0d8 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Wed, 23 Jun 2021 13:22:38 +0300 Subject: [PATCH 22/42] Fix style --- src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp | 6 ++++-- src/Functions/SimdJSONParser.h | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp index 97ab9ffec36..1338a2064f1 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathStar.cpp @@ -11,11 +11,13 @@ bool ParserJSONPathStar::parseImpl(Pos & pos, ASTPtr & node, Expected & expected return false; } ++pos; - if (pos->type != TokenType::Asterisk) { + if (pos->type != TokenType::Asterisk) + { return false; } ++pos; - if (pos->type != TokenType::ClosingSquareBracket) { + if (pos->type != TokenType::ClosingSquareBracket) + { expected.add(pos, "Closing square bracket"); return false; } diff --git a/src/Functions/SimdJSONParser.h b/src/Functions/SimdJSONParser.h index a176f2c5961..c11fca3272c 100644 --- a/src/Functions/SimdJSONParser.h +++ b/src/Functions/SimdJSONParser.h @@ -165,7 +165,8 @@ inline ALWAYS_INLINE SimdJSONParser::Object SimdJSONParser::Element::getObject() return element.get_object().value_unsafe(); } -inline ALWAYS_INLINE WriteBuffer& operator<<(WriteBuffer& out, const DB::SimdJSONParser::Element & element) { +inline ALWAYS_INLINE WriteBuffer& operator<<(WriteBuffer& out, const DB::SimdJSONParser::Element & element) +{ return out << element.getUnderlyingElement(); } From abe7e4195ebce6a6a54d18f3e67d3b5712c2e602 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Wed, 23 Jun 2021 15:02:47 +0300 Subject: [PATCH 23/42] . --- src/Functions/FunctionSQLJSON.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 3ff1b575bfc..1e9b25ee508 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -283,7 +283,7 @@ public: String result; WriteBufferFromString out(result); - out << current_element; + out << current_element.getUnderlyingElement(); ColumnString & col_str = assert_cast(dest); col_str.insertData(result.data(), result.size()); return true; @@ -324,7 +324,7 @@ public: out << ", "; } success = true; - out << current_element; + out << current_element.getUnderlyingElement(); } else if (status == VisitorStatus::Error) { From f8b1a6d185ddfedfd0d8b54bbc2c1d6eaebe38a9 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Fri, 25 Jun 2021 18:33:31 +0300 Subject: [PATCH 24/42] =?UTF-8?q?WriteBuffer=20is=20canceled=20?= =?UTF-8?q?=F0=9F=98=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Functions/DummyJSONParser.h | 6 ++---- src/Functions/FunctionSQLJSON.h | 27 ++++++++++++--------------- src/Functions/SimdJSONParser.h | 7 +------ 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/src/Functions/DummyJSONParser.h b/src/Functions/DummyJSONParser.h index 128ee88e0ca..01fdab1abb6 100644 --- a/src/Functions/DummyJSONParser.h +++ b/src/Functions/DummyJSONParser.h @@ -2,8 +2,6 @@ #include #include -#include -#include namespace DB { @@ -42,7 +40,7 @@ struct DummyJSONParser Array getArray() const { return {}; } Object getObject() const { return {}; } - ALWAYS_INLINE Element getUnderlyingElement() const { return {}; } + Element getElement() { return {}; } }; /// References an array in a JSON document. @@ -101,7 +99,7 @@ struct DummyJSONParser #endif }; -inline ALWAYS_INLINE WriteBufferFromString& operator<<(WriteBufferFromString& out, const DB::DummyJSONParser::Element &) +inline ALWAYS_INLINE std::ostream& operator<<(std::ostream& out, DummyJSONParser::Element) { return out; } diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 1e9b25ee508..dc31ee2d3ff 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -8,8 +8,6 @@ #include #include #include -#include -#include #include #include #include @@ -30,10 +28,10 @@ namespace DB { namespace ErrorCodes { - extern const int ILLEGAL_COLUMN; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; - extern const int BAD_ARGUMENTS; +extern const int ILLEGAL_COLUMN; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; +extern const int BAD_ARGUMENTS; } class FunctionSQLJSONHelpers @@ -281,11 +279,11 @@ public: return false; } - String result; - WriteBufferFromString out(result); - out << current_element.getUnderlyingElement(); + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + out << current_element.getElement(); + auto output_str = out.str(); ColumnString & col_str = assert_cast(dest); - col_str.insertData(result.data(), result.size()); + col_str.insertData(output_str.data(), output_str.size()); return true; } }; @@ -309,9 +307,7 @@ public: GeneratorJSONPath generator_json_path(query_ptr); Element current_element = root; VisitorStatus status; - String result; - WriteBufferFromString out(result); - + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM /// Create json array of results: [res1, res2, ...] out << "["; bool success = false; @@ -324,7 +320,7 @@ public: out << ", "; } success = true; - out << current_element.getUnderlyingElement(); + out << current_element.getElement(); } else if (status == VisitorStatus::Error) { @@ -338,7 +334,8 @@ public: return false; } ColumnString & col_str = assert_cast(dest); - col_str.insertData(reinterpret_cast(result.data()), result.size()); + auto output_str = out.str(); + col_str.insertData(output_str.data(), output_str.size()); return true; } }; diff --git a/src/Functions/SimdJSONParser.h b/src/Functions/SimdJSONParser.h index c11fca3272c..c5793088baf 100644 --- a/src/Functions/SimdJSONParser.h +++ b/src/Functions/SimdJSONParser.h @@ -50,7 +50,7 @@ struct SimdJSONParser ALWAYS_INLINE Array getArray() const; ALWAYS_INLINE Object getObject() const; - ALWAYS_INLINE simdjson::dom::element getUnderlyingElement() const { return element; } + ALWAYS_INLINE simdjson::dom::element getElement() const { return element; } private: simdjson::dom::element element; @@ -165,11 +165,6 @@ inline ALWAYS_INLINE SimdJSONParser::Object SimdJSONParser::Element::getObject() return element.get_object().value_unsafe(); } -inline ALWAYS_INLINE WriteBuffer& operator<<(WriteBuffer& out, const DB::SimdJSONParser::Element & element) -{ - return out << element.getUnderlyingElement(); -} - } #endif From 6981eb64ac9e641070c167fff598a4172e75948c Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Fri, 25 Jun 2021 19:24:22 +0300 Subject: [PATCH 25/42] Fixes --- src/Functions/FunctionSQLJSON.h | 30 +++++++------------ .../JSONPath/ASTs/ASTJSONPathRange.h | 4 +-- .../JSONPath/Generators/GeneratorJSONPath.h | 9 +++--- 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index dc31ee2d3ff..9e13d447b7d 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -41,7 +41,7 @@ public: class Executor { public: - static ColumnPtr run(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) + static ColumnPtr run(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, uint32_t parse_depth) { MutableColumnPtr to{result_type->createColumn()}; to->reserve(input_rows_count); @@ -76,23 +76,10 @@ public: ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } - /// If argument is successfully cast to (ColumnConst *) then it is quoted string - /// Example: - /// SomeFunction('some string argument') - /// - /// Otherwise it is a column - /// Example: - /// SomeFunction(database.table.column) - const ColumnPtr & arg_jsonpath = first_column.column; const auto * arg_jsonpath_const = typeid_cast(arg_jsonpath.get()); const auto * arg_jsonpath_string = typeid_cast(arg_jsonpath_const->getDataColumnPtr().get()); - if (!arg_jsonpath_string) - { - throw Exception{"Illegal column " + arg_jsonpath->getName(), ErrorCodes::ILLEGAL_COLUMN}; - } - const ColumnPtr & arg_json = second_column.column; const auto * col_json_const = typeid_cast(arg_json.get()); const auto * col_json_string @@ -102,14 +89,14 @@ public: const ColumnString::Chars & chars_path = arg_jsonpath_string->getChars(); const ColumnString::Offsets & offsets_path = arg_jsonpath_string->getOffsets(); - /// Get data and offsets for 1 argument (JSON) + /// Prepare to parse 1 argument (JSONPath) const char * query_begin = reinterpret_cast(&chars_path[0]); const char * query_end = query_begin + offsets_path[0] - 1; /// Tokenize query Tokens tokens(query_begin, query_end); /// Max depth 0 indicates that depth is not limited - IParser::Pos token_iterator(tokens, 0); + IParser::Pos token_iterator(tokens, parse_depth); /// Parse query and create AST tree Expected expected; @@ -121,7 +108,7 @@ public: throw Exception{"Unable to parse JSONPath", ErrorCodes::BAD_ARGUMENTS}; } - /// Get data and offsets for 1 argument (JSON) + /// Get data and offsets for 2 argument (JSON) const ColumnString::Chars & chars_json = col_json_string->getChars(); const ColumnString::Offsets & offsets_json = col_json_string->getOffsets(); @@ -179,12 +166,13 @@ public: /// 1. Lexer(path) -> Tokens /// 2. Create ASTPtr /// 3. Parser(Tokens, ASTPtr) -> complete AST - /// 4. Execute functions, call interpreter for each json (in function) + /// 4. Execute functions: call getNextItem on generator and handle each item + uint32_t parse_depth = getContext()->getSettingsRef().max_parser_depth; #if USE_SIMDJSON if (getContext()->getSettingsRef().allow_simdjson) - return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); + return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count, parse_depth); #endif - return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count); + return FunctionSQLJSONHelpers::Executor::run(arguments, result_type, input_rows_count, parse_depth); } }; @@ -325,6 +313,8 @@ public: else if (status == VisitorStatus::Error) { /// ON ERROR + /// Here it is possible to handle errors with ON ERROR (as described in ISO/IEC TR 19075-6), + /// however this functionality is not implemented yet } current_element = root; } diff --git a/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h b/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h index 8a963d7fc6b..746c6211f29 100644 --- a/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h +++ b/src/Functions/JSONPath/ASTs/ASTJSONPathRange.h @@ -14,8 +14,8 @@ public: public: /// Ranges to lookup in json array ($[0, 1, 2, 4 to 9]) - /// Range is represented as - /// Single index is represented as + /// Range is represented as + /// Single index is represented as std::vector> ranges; bool is_star = false; }; diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h index 071a7ac3089..b918ceac003 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generators/GeneratorJSONPath.h @@ -73,7 +73,8 @@ public: { while (true) { - auto root = element; + /// element passed to us actually is root, so here we assign current to root + auto current = element; if (current_visitor < 0) { return VisitorStatus::Exhausted; @@ -81,13 +82,13 @@ public: for (int i = 0; i < current_visitor; ++i) { - visitors[i]->apply(root); + visitors[i]->apply(current); } VisitorStatus status = VisitorStatus::Error; for (size_t i = current_visitor; i < visitors.size(); ++i) { - status = visitors[i]->visit(root); + status = visitors[i]->visit(current); current_visitor = i; if (status == VisitorStatus::Error || status == VisitorStatus::Ignore) { @@ -98,7 +99,7 @@ public: if (status != VisitorStatus::Ignore) { - element = root; + element = current; return status; } } From 7892e44467294d6f02b0bd73c8b8a6b450f17694 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Fri, 25 Jun 2021 21:14:08 +0300 Subject: [PATCH 26/42] =?UTF-8?q?Fix=20style=20yet=20again=20=F0=9F=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Functions/FunctionSQLJSON.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index 9e13d447b7d..a6024a27e95 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -28,7 +28,6 @@ namespace DB { namespace ErrorCodes { -extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; extern const int BAD_ARGUMENTS; From 520c4a8f8a0ed65649fce49d9eea99af657a584c Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 12:10:16 +0300 Subject: [PATCH 27/42] Fix according to review --- src/Functions/FunctionSQLJSON.h | 4 +++- src/Functions/JSONPath/CMakeLists.txt | 4 ++-- src/Functions/JSONPath/Generator/CMakeLists.txt | 8 ++++++++ .../{Generators => Generator}/GeneratorJSONPath.h | 12 ++++++------ .../JSONPath/{Generators => Generator}/IGenerator.h | 4 ++-- .../{Generators => Generator}/IGenerator_fwd.h | 2 +- .../JSONPath/{Generators => Generator}/IVisitor.h | 2 +- .../VisitorJSONPathMemberAccess.h | 8 +++----- .../{Generators => Generator}/VisitorJSONPathRange.h | 4 ++-- .../{Generators => Generator}/VisitorJSONPathRoot.h | 4 ++-- .../{Generators => Generator}/VisitorJSONPathStar.h | 4 ++-- .../{Generators => Generator}/VisitorStatus.h | 0 src/Functions/JSONPath/Generators/CMakeLists.txt | 8 -------- .../JSONPath/Parsers/ParserJSONPathMemberAccess.cpp | 6 +----- src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h | 5 +---- .../0_stateless/01889_sql_json_functions.reference | 1 + .../queries/0_stateless/01889_sql_json_functions.sql | 1 + 17 files changed, 36 insertions(+), 41 deletions(-) create mode 100644 src/Functions/JSONPath/Generator/CMakeLists.txt rename src/Functions/JSONPath/{Generators => Generator}/GeneratorJSONPath.h (90%) rename src/Functions/JSONPath/{Generators => Generator}/IGenerator.h (81%) rename src/Functions/JSONPath/{Generators => Generator}/IGenerator_fwd.h (83%) rename src/Functions/JSONPath/{Generators => Generator}/IVisitor.h (94%) rename src/Functions/JSONPath/{Generators => Generator}/VisitorJSONPathMemberAccess.h (87%) rename src/Functions/JSONPath/{Generators => Generator}/VisitorJSONPathRange.h (94%) rename src/Functions/JSONPath/{Generators => Generator}/VisitorJSONPathRoot.h (87%) rename src/Functions/JSONPath/{Generators => Generator}/VisitorJSONPathStar.h (92%) rename src/Functions/JSONPath/{Generators => Generator}/VisitorStatus.h (100%) delete mode 100644 src/Functions/JSONPath/Generators/CMakeLists.txt diff --git a/src/Functions/FunctionSQLJSON.h b/src/Functions/FunctionSQLJSON.h index a6024a27e95..9e469c4ebac 100644 --- a/src/Functions/FunctionSQLJSON.h +++ b/src/Functions/FunctionSQLJSON.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include #include @@ -257,6 +257,8 @@ public: else if (status == VisitorStatus::Error) { /// ON ERROR + /// Here it is possible to handle errors with ON ERROR (as described in ISO/IEC TR 19075-6), + /// however this functionality is not implemented yet } current_element = root; } diff --git a/src/Functions/JSONPath/CMakeLists.txt b/src/Functions/JSONPath/CMakeLists.txt index 8e65f7c8c6d..a1f5bf9bf2c 100644 --- a/src/Functions/JSONPath/CMakeLists.txt +++ b/src/Functions/JSONPath/CMakeLists.txt @@ -1,8 +1,8 @@ add_subdirectory(ASTs) target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_asts) -add_subdirectory(Generators) -target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_generators) +add_subdirectory(Generator) +target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_generator) add_subdirectory(Parsers) target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_parsers) diff --git a/src/Functions/JSONPath/Generator/CMakeLists.txt b/src/Functions/JSONPath/Generator/CMakeLists.txt new file mode 100644 index 00000000000..11215ba1078 --- /dev/null +++ b/src/Functions/JSONPath/Generator/CMakeLists.txt @@ -0,0 +1,8 @@ +include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") +add_headers_and_sources(clickhouse_functions_jsonpath_generator .) +add_library(clickhouse_functions_jsonpath_generator ${clickhouse_functions_jsonpath_generator_sources} ${clickhouse_functions_jsonpath_generator_headers}) +target_link_libraries(clickhouse_functions_jsonpath_generator PRIVATE dbms) + +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + target_compile_options(clickhouse_functions_jsonpath_generator PRIVATE "-g0") +endif() diff --git a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h b/src/Functions/JSONPath/Generator/GeneratorJSONPath.h similarity index 90% rename from src/Functions/JSONPath/Generators/GeneratorJSONPath.h rename to src/Functions/JSONPath/Generator/GeneratorJSONPath.h index b918ceac003..291150f6df4 100644 --- a/src/Functions/JSONPath/Generators/GeneratorJSONPath.h +++ b/src/Functions/JSONPath/Generator/GeneratorJSONPath.h @@ -1,11 +1,11 @@ #pragma once -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include diff --git a/src/Functions/JSONPath/Generators/IGenerator.h b/src/Functions/JSONPath/Generator/IGenerator.h similarity index 81% rename from src/Functions/JSONPath/Generators/IGenerator.h rename to src/Functions/JSONPath/Generator/IGenerator.h index d2cef9fe27b..323145e07e1 100644 --- a/src/Functions/JSONPath/Generators/IGenerator.h +++ b/src/Functions/JSONPath/Generator/IGenerator.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include #include namespace DB diff --git a/src/Functions/JSONPath/Generators/IGenerator_fwd.h b/src/Functions/JSONPath/Generator/IGenerator_fwd.h similarity index 83% rename from src/Functions/JSONPath/Generators/IGenerator_fwd.h rename to src/Functions/JSONPath/Generator/IGenerator_fwd.h index 57ed04d0f6f..bb5f64cd6f9 100644 --- a/src/Functions/JSONPath/Generators/IGenerator_fwd.h +++ b/src/Functions/JSONPath/Generator/IGenerator_fwd.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace DB { diff --git a/src/Functions/JSONPath/Generators/IVisitor.h b/src/Functions/JSONPath/Generator/IVisitor.h similarity index 94% rename from src/Functions/JSONPath/Generators/IVisitor.h rename to src/Functions/JSONPath/Generator/IVisitor.h index 1461b842829..1a94106a435 100644 --- a/src/Functions/JSONPath/Generators/IVisitor.h +++ b/src/Functions/JSONPath/Generator/IVisitor.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace DB { diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h b/src/Functions/JSONPath/Generator/VisitorJSONPathMemberAccess.h similarity index 87% rename from src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h rename to src/Functions/JSONPath/Generator/VisitorJSONPathMemberAccess.h index b0c601458b6..5fe35e75a84 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathMemberAccess.h +++ b/src/Functions/JSONPath/Generator/VisitorJSONPathMemberAccess.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include namespace DB { @@ -25,19 +25,17 @@ public: VisitorStatus visit(typename JSONParser::Element & element) override { + this->setExhausted(true); if (!element.isObject()) { - this->setExhausted(true); return VisitorStatus::Error; } typename JSONParser::Element result; if (!element.getObject().find(std::string_view(member_access_ptr->member_name), result)) { - this->setExhausted(true); return VisitorStatus::Error; } apply(element); - this->setExhausted(true); return VisitorStatus::Ok; } diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h b/src/Functions/JSONPath/Generator/VisitorJSONPathRange.h similarity index 94% rename from src/Functions/JSONPath/Generators/VisitorJSONPathRange.h rename to src/Functions/JSONPath/Generator/VisitorJSONPathRange.h index 57e208271d0..40d4f6ad95e 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathRange.h +++ b/src/Functions/JSONPath/Generator/VisitorJSONPathRange.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include namespace DB { diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathRoot.h b/src/Functions/JSONPath/Generator/VisitorJSONPathRoot.h similarity index 87% rename from src/Functions/JSONPath/Generators/VisitorJSONPathRoot.h rename to src/Functions/JSONPath/Generator/VisitorJSONPathRoot.h index d8b88ce0255..5c48c12782f 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathRoot.h +++ b/src/Functions/JSONPath/Generator/VisitorJSONPathRoot.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include namespace DB { diff --git a/src/Functions/JSONPath/Generators/VisitorJSONPathStar.h b/src/Functions/JSONPath/Generator/VisitorJSONPathStar.h similarity index 92% rename from src/Functions/JSONPath/Generators/VisitorJSONPathStar.h rename to src/Functions/JSONPath/Generator/VisitorJSONPathStar.h index bc840597f2a..4a54a76c199 100644 --- a/src/Functions/JSONPath/Generators/VisitorJSONPathStar.h +++ b/src/Functions/JSONPath/Generator/VisitorJSONPathStar.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include namespace DB { diff --git a/src/Functions/JSONPath/Generators/VisitorStatus.h b/src/Functions/JSONPath/Generator/VisitorStatus.h similarity index 100% rename from src/Functions/JSONPath/Generators/VisitorStatus.h rename to src/Functions/JSONPath/Generator/VisitorStatus.h diff --git a/src/Functions/JSONPath/Generators/CMakeLists.txt b/src/Functions/JSONPath/Generators/CMakeLists.txt deleted file mode 100644 index 76a116132fd..00000000000 --- a/src/Functions/JSONPath/Generators/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") -add_headers_and_sources(clickhouse_functions_jsonpath_generators .) -add_library(clickhouse_functions_jsonpath_generators ${clickhouse_functions_jsonpath_generators_sources} ${clickhouse_functions_jsonpath_generators_headers}) -target_link_libraries(clickhouse_functions_jsonpath_generators PRIVATE dbms) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - target_compile_options(clickhouse_functions_jsonpath_generators PRIVATE "-g0") -endif() diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp index 85b43217867..c7f047eb8fb 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp @@ -36,11 +36,7 @@ bool ParserJSONPathMemberAccess::parseImpl(Pos & pos, ASTPtr & node, Expected & auto member_access = std::make_shared(); node = member_access; - if (!tryGetIdentifierNameInto(member_name, member_access->member_name)) - { - return false; - } - return true; + return tryGetIdentifierNameInto(member_name, member_access->member_name); } } diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h index cffec125c70..fbe7321562e 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathQuery.h @@ -9,9 +9,6 @@ class ParserJSONPathQuery : public IParserBase { protected: const char * getName() const override { return "ParserJSONPathQuery"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; - -private: - /// backlog: strict or lax mode + bool parseImpl(Pos & pos, ASTPtr & query, Expected & expected) override; }; } diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index e38058ffc50..8ab94781237 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -19,6 +19,7 @@ null [["world","world2"]] [{"world":"!"}] +[0, 1, 4, 0, -1, -4] --JSON_EXISTS-- 1 diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index a1749b3be24..98378a0090c 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -21,6 +21,7 @@ SELECT JSON_QUERY('$.hello', '{"hello":["world","world2"]}'); SELECT JSON_QUERY('$.hello', '{"hello":{"world":"!"}}'); SELECT JSON_QUERY('$.hello', '{hello:{"world":"!"}}}'); -- invalid json => default value (empty string) SELECT JSON_QUERY('$.hello', ''); +SELECT JSON_QUERY('$.array[*][0 to 2, 4]', '{"array":[[0, 1, 2, 3, 4, 5], [0, -1, -2, -3, -4, -5]]}'); SELECT '--JSON_EXISTS--'; SELECT JSON_EXISTS('$', '{"hello":1}'); From ae5cb2c8bfb12e42536f3832cc5c99f72331b17e Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 13:43:22 +0300 Subject: [PATCH 28/42] Fix test (empty string) --- tests/queries/0_stateless/01889_sql_json_functions.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index 8ab94781237..1ae1fccdd56 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -19,8 +19,8 @@ null [["world","world2"]] [{"world":"!"}] -[0, 1, 4, 0, -1, -4] +[0, 1, 4, 0, -1, -4] --JSON_EXISTS-- 1 0 From 7bcb57afe1f8f011d3a2e1bf6e3b6526ae958ff4 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:10:01 +0300 Subject: [PATCH 29/42] Fix failing special builds (probably) --- src/Functions/JSONPath/Parsers/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/JSONPath/Parsers/CMakeLists.txt b/src/Functions/JSONPath/Parsers/CMakeLists.txt index ecabe5cc13b..e9cdbbfea32 100644 --- a/src/Functions/JSONPath/Parsers/CMakeLists.txt +++ b/src/Functions/JSONPath/Parsers/CMakeLists.txt @@ -2,6 +2,7 @@ include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") add_headers_and_sources(clickhouse_functions_jsonpath_parsers .) add_library(clickhouse_functions_jsonpath_parsers ${clickhouse_functions_jsonpath_parsers_sources} ${clickhouse_functions_jsonpath_parsers_headers}) target_link_libraries(clickhouse_functions_jsonpath_parsers PRIVATE dbms) +target_link_libraries(clickhouse_functions_jsonpath_parsers PRIVATE clickhouse_parsers) if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) target_compile_options(clickhouse_functions_jsonpath_parsers PRIVATE "-g0") From a48f500956b769e3b6fd70f7d658e6484d927d94 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:28:05 +0300 Subject: [PATCH 30/42] Add tests with multiple rows --- .../0_stateless/01889_sql_json_functions.reference | 4 ++++ .../queries/0_stateless/01889_sql_json_functions.sql | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index 1ae1fccdd56..fae7418b818 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -37,3 +37,7 @@ null 1 0 1 +--MANY ROWS-- +["Vasily", "Kostya"] +["Katya", "Anatoliy"] +["Tihon", "Ernest"] diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 98378a0090c..9ee4ee1b95c 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -38,4 +38,13 @@ SELECT JSON_EXISTS('$.hello[0]', '{"hello":["world"]}'); SELECT JSON_EXISTS('$.hello[1]', '{"hello":["world"]}'); SELECT JSON_EXISTS('$.a[*].b', '{"a":[{"b":1},{"c":2}]}'); SELECT JSON_EXISTS('$.a[*].f', '{"a":[{"b":1},{"c":2}]}'); -SELECT JSON_EXISTS('$.a[*][0].h', '{"a":[[{"b":1}, {"g":1}],[{"h":1},{"y":1}]]}'); \ No newline at end of file +SELECT JSON_EXISTS('$.a[*][0].h', '{"a":[[{"b":1}, {"g":1}],[{"h":1},{"y":1}]]}'); + +SELECT '--MANY ROWS--' +DROP TABLE IF EXISTS 01889_sql_json; +CREATE TABLE 01889_sql_json (json String) ENGINE = MergeTree ORDER BY json; +INSERT INTO 01889_sql_json(json) VALUES('{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}') +INSERT INTO 01889_sql_json(json) VALUES('{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}') +INSERT INTO 01889_sql_json(json) VALUES('{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}') +SELECT JSON_QUERY('$.friends[0 to 2]', json) +DROP TABLE 01889_sql_json From 9f05b387e541f03301fb240728a75064bbbf8747 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:33:13 +0300 Subject: [PATCH 31/42] Add colons --- .../queries/0_stateless/01889_sql_json_functions.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 9ee4ee1b95c..9b978c2223d 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -40,11 +40,11 @@ SELECT JSON_EXISTS('$.a[*].b', '{"a":[{"b":1},{"c":2}]}'); SELECT JSON_EXISTS('$.a[*].f', '{"a":[{"b":1},{"c":2}]}'); SELECT JSON_EXISTS('$.a[*][0].h', '{"a":[[{"b":1}, {"g":1}],[{"h":1},{"y":1}]]}'); -SELECT '--MANY ROWS--' +SELECT '--MANY ROWS--'; DROP TABLE IF EXISTS 01889_sql_json; CREATE TABLE 01889_sql_json (json String) ENGINE = MergeTree ORDER BY json; -INSERT INTO 01889_sql_json(json) VALUES('{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}') -INSERT INTO 01889_sql_json(json) VALUES('{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}') -INSERT INTO 01889_sql_json(json) VALUES('{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}') -SELECT JSON_QUERY('$.friends[0 to 2]', json) -DROP TABLE 01889_sql_json +INSERT INTO 01889_sql_json(json) VALUES('{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); +INSERT INTO 01889_sql_json(json) VALUES('{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); +INSERT INTO 01889_sql_json(json) VALUES('{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); +SELECT JSON_QUERY('$.friends[0 to 2]', json); +DROP TABLE 01889_sql_json; From 3f8a22c35d23e4c1f7b7b1eea6fc95b8d797a16f Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:34:34 +0300 Subject: [PATCH 32/42] Fix syntax --- tests/queries/0_stateless/01889_sql_json_functions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 9b978c2223d..0589a330280 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -46,5 +46,5 @@ CREATE TABLE 01889_sql_json (json String) ENGINE = MergeTree ORDER BY json; INSERT INTO 01889_sql_json(json) VALUES('{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); INSERT INTO 01889_sql_json(json) VALUES('{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); INSERT INTO 01889_sql_json(json) VALUES('{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); -SELECT JSON_QUERY('$.friends[0 to 2]', json); +SELECT JSON_QUERY('$.friends[0 to 2]', json) FROM 01889_sql_json; DROP TABLE 01889_sql_json; From 4ef27cfc2d20c8ab32f6024e6e536bb51fd6cca9 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:36:35 +0300 Subject: [PATCH 33/42] Fix ans order --- tests/queries/0_stateless/01889_sql_json_functions.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index fae7418b818..d361dea6b18 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -38,6 +38,6 @@ null 0 1 --MANY ROWS-- -["Vasily", "Kostya"] ["Katya", "Anatoliy"] +["Vasily", "Kostya"] ["Tihon", "Ernest"] From 3d1e2fe55078b807f214cbea06e982dcc86a3d22 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:37:44 +0300 Subject: [PATCH 34/42] Fix order again --- tests/queries/0_stateless/01889_sql_json_functions.reference | 2 +- tests/queries/0_stateless/01889_sql_json_functions.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index d361dea6b18..bf70961b1f8 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -38,6 +38,6 @@ null 0 1 --MANY ROWS-- -["Katya", "Anatoliy"] ["Vasily", "Kostya"] ["Tihon", "Ernest"] +["Katya", "Anatoliy"] diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 0589a330280..fe37a3dc53e 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -43,8 +43,8 @@ SELECT JSON_EXISTS('$.a[*][0].h', '{"a":[[{"b":1}, {"g":1}],[{"h":1},{"y":1}]]}' SELECT '--MANY ROWS--'; DROP TABLE IF EXISTS 01889_sql_json; CREATE TABLE 01889_sql_json (json String) ENGINE = MergeTree ORDER BY json; -INSERT INTO 01889_sql_json(json) VALUES('{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); INSERT INTO 01889_sql_json(json) VALUES('{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); INSERT INTO 01889_sql_json(json) VALUES('{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); +INSERT INTO 01889_sql_json(json) VALUES('{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); SELECT JSON_QUERY('$.friends[0 to 2]', json) FROM 01889_sql_json; DROP TABLE 01889_sql_json; From a0b6281790e5e72b3ed839aa1b0e25da0a22c3a7 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:38:43 +0300 Subject: [PATCH 35/42] Final fix order --- tests/queries/0_stateless/01889_sql_json_functions.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index bf70961b1f8..8fb2f9430e0 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -38,6 +38,6 @@ null 0 1 --MANY ROWS-- -["Vasily", "Kostya"] ["Tihon", "Ernest"] ["Katya", "Anatoliy"] +["Vasily", "Kostya"] From 7965c638d2ba2d2db783a88ef63f6bb9975f825a Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:44:18 +0300 Subject: [PATCH 36/42] Another try --- .../0_stateless/01889_sql_json_functions.reference | 2 +- tests/queries/0_stateless/01889_sql_json_functions.sql | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index 8fb2f9430e0..bf70961b1f8 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -38,6 +38,6 @@ null 0 1 --MANY ROWS-- +["Vasily", "Kostya"] ["Tihon", "Ernest"] ["Katya", "Anatoliy"] -["Vasily", "Kostya"] diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index fe37a3dc53e..79af60a15a4 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -42,9 +42,9 @@ SELECT JSON_EXISTS('$.a[*][0].h', '{"a":[[{"b":1}, {"g":1}],[{"h":1},{"y":1}]]}' SELECT '--MANY ROWS--'; DROP TABLE IF EXISTS 01889_sql_json; -CREATE TABLE 01889_sql_json (json String) ENGINE = MergeTree ORDER BY json; -INSERT INTO 01889_sql_json(json) VALUES('{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); -INSERT INTO 01889_sql_json(json) VALUES('{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); -INSERT INTO 01889_sql_json(json) VALUES('{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); +CREATE TABLE 01889_sql_json (id UInt8, json String) ENGINE = MergeTree ORDER BY id; +INSERT INTO 01889_sql_json(json) VALUES(0, '{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); +INSERT INTO 01889_sql_json(json) VALUES(1, '{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); +INSERT INTO 01889_sql_json(json) VALUES(2, '{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); SELECT JSON_QUERY('$.friends[0 to 2]', json) FROM 01889_sql_json; DROP TABLE 01889_sql_json; From fd07dbe1f7379f7e7511576d8c339005b551e523 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:46:45 +0300 Subject: [PATCH 37/42] Please just work --- tests/queries/0_stateless/01889_sql_json_functions.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 79af60a15a4..77cb32352cf 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -43,8 +43,8 @@ SELECT JSON_EXISTS('$.a[*][0].h', '{"a":[[{"b":1}, {"g":1}],[{"h":1},{"y":1}]]}' SELECT '--MANY ROWS--'; DROP TABLE IF EXISTS 01889_sql_json; CREATE TABLE 01889_sql_json (id UInt8, json String) ENGINE = MergeTree ORDER BY id; -INSERT INTO 01889_sql_json(json) VALUES(0, '{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); -INSERT INTO 01889_sql_json(json) VALUES(1, '{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); -INSERT INTO 01889_sql_json(json) VALUES(2, '{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); +INSERT INTO 01889_sql_json(id, json) VALUES(0, '{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); +INSERT INTO 01889_sql_json(id, json) VALUES(1, '{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); +INSERT INTO 01889_sql_json(id, json) VALUES(2, '{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); SELECT JSON_QUERY('$.friends[0 to 2]', json) FROM 01889_sql_json; DROP TABLE 01889_sql_json; From 57972410c794a504fe407a5ec293a8484007ee6e Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:54:43 +0300 Subject: [PATCH 38/42] =?UTF-8?q?=F0=9F=98=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../queries/0_stateless/01889_sql_json_functions.reference | 6 +++--- tests/queries/0_stateless/01889_sql_json_functions.sql | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index bf70961b1f8..7d400b4f6e7 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -38,6 +38,6 @@ null 0 1 --MANY ROWS-- -["Vasily", "Kostya"] -["Tihon", "Ernest"] -["Katya", "Anatoliy"] +0 ["Vasily", "Kostya"] +1 ["Tihon", "Ernest"] +2 ["Katya", "Anatoliy"] diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 77cb32352cf..1c5069ccfde 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -46,5 +46,5 @@ CREATE TABLE 01889_sql_json (id UInt8, json String) ENGINE = MergeTree ORDER BY INSERT INTO 01889_sql_json(id, json) VALUES(0, '{"name":"Ivan","surname":"Ivanov","friends":["Vasily","Kostya","Artyom"]}'); INSERT INTO 01889_sql_json(id, json) VALUES(1, '{"name":"Katya","surname":"Baltica","friends":["Tihon","Ernest","Innokentiy"]}'); INSERT INTO 01889_sql_json(id, json) VALUES(2, '{"name":"Vitali","surname":"Brown","friends":["Katya","Anatoliy","Ivan","Oleg"]}'); -SELECT JSON_QUERY('$.friends[0 to 2]', json) FROM 01889_sql_json; +SELECT id, JSON_QUERY('$.friends[0 to 2]', json) FROM 01889_sql_json ORDER BY id; DROP TABLE 01889_sql_json; From 7d1c561a7b37634f70ba36c97d14dde51d79d0f4 Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Sun, 4 Jul 2021 22:57:16 +0300 Subject: [PATCH 39/42] =?UTF-8?q?=F0=9F=98=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../queries/0_stateless/01889_sql_json_functions.reference | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index 7d400b4f6e7..593f2fb2d20 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -38,6 +38,6 @@ null 0 1 --MANY ROWS-- -0 ["Vasily", "Kostya"] -1 ["Tihon", "Ernest"] -2 ["Katya", "Anatoliy"] +0 ["Vasily", "Kostya"] +1 ["Tihon", "Ernest"] +2 ["Katya", "Anatoliy"] From 3b186d03f5d15527183803c5f9ffd3fb110bf25a Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Mon, 5 Jul 2021 11:56:09 +0300 Subject: [PATCH 40/42] Change cmake lists --- src/Functions/JSONPath/ASTs/CMakeLists.txt | 8 -------- src/Functions/JSONPath/CMakeLists.txt | 16 ++++++++++------ src/Functions/JSONPath/Generator/CMakeLists.txt | 8 -------- src/Functions/JSONPath/Parsers/CMakeLists.txt | 9 --------- 4 files changed, 10 insertions(+), 31 deletions(-) delete mode 100644 src/Functions/JSONPath/ASTs/CMakeLists.txt delete mode 100644 src/Functions/JSONPath/Generator/CMakeLists.txt delete mode 100644 src/Functions/JSONPath/Parsers/CMakeLists.txt diff --git a/src/Functions/JSONPath/ASTs/CMakeLists.txt b/src/Functions/JSONPath/ASTs/CMakeLists.txt deleted file mode 100644 index ef56e3b0072..00000000000 --- a/src/Functions/JSONPath/ASTs/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") -add_headers_and_sources(clickhouse_functions_jsonpath_asts .) -add_library(clickhouse_functions_jsonpath_asts ${clickhouse_functions_jsonpath_asts_sources} ${clickhouse_functions_jsonpath_asts_headers}) -target_link_libraries(clickhouse_functions_jsonpath_asts PRIVATE dbms) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - target_compile_options(clickhouse_functions_jsonpath_asts PRIVATE "-g0") -endif() diff --git a/src/Functions/JSONPath/CMakeLists.txt b/src/Functions/JSONPath/CMakeLists.txt index a1f5bf9bf2c..f8f4069b1a2 100644 --- a/src/Functions/JSONPath/CMakeLists.txt +++ b/src/Functions/JSONPath/CMakeLists.txt @@ -1,8 +1,12 @@ -add_subdirectory(ASTs) -target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_asts) +include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") +add_headers_and_sources(clickhouse_functions_jsonpath Parsers) +add_headers_and_sources(clickhouse_functions_jsonpath ASTs) +add_headers_and_sources(clickhouse_functions_jsonpath Generator) +add_library(clickhouse_functions_jsonpath ${clickhouse_functions_jsonpath_sources} ${clickhouse_functions_jsonpath_headers}) -add_subdirectory(Generator) -target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_generator) +target_link_libraries(clickhouse_functions_jsonpath PRIVATE dbms) +target_link_libraries(clickhouse_functions_jsonpath PRIVATE clickhouse_parsers) -add_subdirectory(Parsers) -target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath_parsers) +if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) + target_compile_options(clickhouse_functions_jsonpath PRIVATE "-g0") +endif() diff --git a/src/Functions/JSONPath/Generator/CMakeLists.txt b/src/Functions/JSONPath/Generator/CMakeLists.txt deleted file mode 100644 index 11215ba1078..00000000000 --- a/src/Functions/JSONPath/Generator/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") -add_headers_and_sources(clickhouse_functions_jsonpath_generator .) -add_library(clickhouse_functions_jsonpath_generator ${clickhouse_functions_jsonpath_generator_sources} ${clickhouse_functions_jsonpath_generator_headers}) -target_link_libraries(clickhouse_functions_jsonpath_generator PRIVATE dbms) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - target_compile_options(clickhouse_functions_jsonpath_generator PRIVATE "-g0") -endif() diff --git a/src/Functions/JSONPath/Parsers/CMakeLists.txt b/src/Functions/JSONPath/Parsers/CMakeLists.txt deleted file mode 100644 index e9cdbbfea32..00000000000 --- a/src/Functions/JSONPath/Parsers/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") -add_headers_and_sources(clickhouse_functions_jsonpath_parsers .) -add_library(clickhouse_functions_jsonpath_parsers ${clickhouse_functions_jsonpath_parsers_sources} ${clickhouse_functions_jsonpath_parsers_headers}) -target_link_libraries(clickhouse_functions_jsonpath_parsers PRIVATE dbms) -target_link_libraries(clickhouse_functions_jsonpath_parsers PRIVATE clickhouse_parsers) - -if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) - target_compile_options(clickhouse_functions_jsonpath_parsers PRIVATE "-g0") -endif() From 5b50fbde87fdd1155cba08a4c2f05270a183c3cf Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Mon, 5 Jul 2021 12:04:45 +0300 Subject: [PATCH 41/42] =?UTF-8?q?Also=20link=20jsonpath=20to=20functions?= =?UTF-8?q?=20=F0=9F=98=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Functions/JSONPath/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/JSONPath/CMakeLists.txt b/src/Functions/JSONPath/CMakeLists.txt index f8f4069b1a2..2531a0f84e6 100644 --- a/src/Functions/JSONPath/CMakeLists.txt +++ b/src/Functions/JSONPath/CMakeLists.txt @@ -6,6 +6,7 @@ add_library(clickhouse_functions_jsonpath ${clickhouse_functions_jsonpath_source target_link_libraries(clickhouse_functions_jsonpath PRIVATE dbms) target_link_libraries(clickhouse_functions_jsonpath PRIVATE clickhouse_parsers) +target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath) if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) target_compile_options(clickhouse_functions_jsonpath PRIVATE "-g0") From 1bbfbff00358e77a7934cb3d23cbe0f02b7592ad Mon Sep 17 00:00:00 2001 From: l1tsolaiki Date: Tue, 6 Jul 2021 01:04:09 +0300 Subject: [PATCH 42/42] Regenerate ya.make --- src/Functions/ya.make | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 039c166c1a5..d4636c6c1e2 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -79,6 +79,8 @@ SRCS( JSONPath/Parsers/ParserJSONPathMemberAccess.cpp JSONPath/Parsers/ParserJSONPathQuery.cpp JSONPath/Parsers/ParserJSONPathRange.cpp + JSONPath/Parsers/ParserJSONPathRoot.cpp + JSONPath/Parsers/ParserJSONPathStar.cpp TargetSpecific.cpp URL/URLHierarchy.cpp URL/URLPathHierarchy.cpp