diff --git a/src/Storages/MeiliSearch/MeiliSearchConnection.cpp b/src/Storages/MeiliSearch/MeiliSearchConnection.cpp index b5198a10d3c..f74ee83ce49 100644 --- a/src/Storages/MeiliSearch/MeiliSearchConnection.cpp +++ b/src/Storages/MeiliSearch/MeiliSearchConnection.cpp @@ -38,9 +38,7 @@ String MeiliSearchConnection::searchQuery(const std::unordered_map #include #include +#include #include #include #include @@ -35,7 +36,7 @@ public: String searchQuery(const std::unordered_map & query_params) const; - String updateQuery(const String & data) const; + String updateQuery(std::string_view data) const; private: MeiliConfig config; diff --git a/src/Storages/MeiliSearch/SinkMeiliSearch.cpp b/src/Storages/MeiliSearch/SinkMeiliSearch.cpp index 8c188588242..156fa592be0 100644 --- a/src/Storages/MeiliSearch/SinkMeiliSearch.cpp +++ b/src/Storages/MeiliSearch/SinkMeiliSearch.cpp @@ -1,5 +1,6 @@ #include #include "Core/Field.h" +#include "Formats/FormatFactory.h" #include "IO/WriteBufferFromString.h" #include "Processors/Formats/Impl/JSONRowOutputFormat.h" #include "base/JSON.h" @@ -22,64 +23,35 @@ SinkMeiliSearch::SinkMeiliSearch( { } -String getStringRepresentation(const ColumnWithTypeAndName & col, size_t row) -{ - Field elem; - if (col.column->size() <= row) - { - return ""; - } - col.column->get(row, elem); - if (elem.getType() == Field::Types::Int64) - { - return std::to_string(elem.get()); - } - else if (elem.getType() == Field::Types::UInt64) - { - return std::to_string(elem.get()); - } - else if (elem.getType() == Field::Types::String) - { - return doubleQuoteString(elem.get()); - } - else if (elem.getType() == Field::Types::Float64) - { - return std::to_string(elem.get()); - } - return ""; -} - -String SinkMeiliSearch::getOneElement(const Block & block, int ind) const -{ - String ans = "{"; - int id = 0; - for (const auto & col : block) - { - ans += doubleQuoteString(sample_block.getByPosition(id++).name) + ":" + getStringRepresentation(col, ind) + ","; - } - ans.back() = '}'; - return ans; +void extractData(std::string_view& view) { + int ind = view.find("\"data\":") + 9; + view.remove_prefix(ind); + int bal = ind = 1; + while (bal > 0) { + if (view[ind] == '[') ++bal; + else if (view[ind] == ']') --bal; + ++ind; + } + view.remove_suffix(view.size() - ind); } void SinkMeiliSearch::writeBlockData(const Block & block) const { - size_t max_col_size = 0; - for (const auto & col : block) - { - max_col_size = std::max(max_col_size, col.column->size()); - } - String json_array = "["; - for (size_t i = 0; i < max_col_size; ++i) - { - json_array += getOneElement(block, i) + ","; - } - json_array.back() = ']'; - auto response = connection.updateQuery(json_array); - JSON jres = JSON(response).begin(); + FormatSettings settings = getFormatSettings(local_context); + settings.json.quote_64bit_integers = false; + WriteBufferFromOwnString buf; + auto writer = FormatFactory::instance().getOutputFormat("JSON", buf, sample_block, local_context, {}, settings); + writer->write(block); + writer->flush(); + writer->finalize(); + + std::string_view vbuf(buf.str()); + extractData(vbuf); + + auto response = connection.updateQuery(vbuf); + auto jres = JSON(response).begin(); if (jres.getName() == "message") - { throw Exception(ErrorCodes::MEILISEARCH_EXCEPTION, jres.getValue().toString()); - } } Blocks SinkMeiliSearch::splitBlocks(const Block & block, const size_t & max_rows) const @@ -104,9 +76,8 @@ Blocks SinkMeiliSearch::splitBlocks(const Block & block, const size_t & max_rows if (idx == split_block_size - 1) limits = rows - offsets; for (size_t col_idx = 0; col_idx < columns; ++col_idx) - { split_blocks[idx].getByPosition(col_idx).column = block.getByPosition(col_idx).column->cut(offsets, limits); - } + offsets += max_block_size; } @@ -118,9 +89,7 @@ void SinkMeiliSearch::consume(Chunk chunk) auto block = getHeader().cloneWithColumns(chunk.detachColumns()); auto blocks = splitBlocks(block, max_block_size); for (const auto & b : blocks) - { writeBlockData(b); - } } diff --git a/src/Storages/MeiliSearch/SinkMeiliSearch.h b/src/Storages/MeiliSearch/SinkMeiliSearch.h index d02047d1b56..6df8bdfb3da 100644 --- a/src/Storages/MeiliSearch/SinkMeiliSearch.h +++ b/src/Storages/MeiliSearch/SinkMeiliSearch.h @@ -22,8 +22,6 @@ public: Blocks splitBlocks(const Block & block, const size_t & max_rows) const; private: - String getOneElement(const Block & block, int ind) const; - MeiliSearchConnection connection; ContextPtr local_context; const UInt64 max_block_size; diff --git a/src/Storages/MeiliSearch/SourceMeiliSearch.cpp b/src/Storages/MeiliSearch/SourceMeiliSearch.cpp index 9e232328d21..0986e529228 100644 --- a/src/Storages/MeiliSearch/SourceMeiliSearch.cpp +++ b/src/Storages/MeiliSearch/SourceMeiliSearch.cpp @@ -32,9 +32,8 @@ MeiliSearchSource::MeiliSearchSource( String columns_to_get = "["; for (const auto & col : description.sample_block) - { columns_to_get += doubleQuoteString(col.name) + ","; - } + columns_to_get.back() = ']'; query_params[doubleQuoteString("attributesToRetrieve")] = columns_to_get; @@ -71,9 +70,7 @@ void insertWithTypeId(MutableColumnPtr & column, JSON kv_pair, int type_id) Chunk MeiliSearchSource::generate() { if (all_read) - { return {}; - } MutableColumns columns(description.sample_block.columns()); const size_t size = columns.size(); @@ -87,9 +84,7 @@ Chunk MeiliSearchSource::generate() JSON jres = JSON(response).begin(); if (jres.getName() == "message") - { throw Exception(ErrorCodes::MEILISEARCH_EXCEPTION, jres.getValue().toString()); - } size_t cnt_match = 0; String def; diff --git a/src/Storages/MeiliSearch/StorageMeiliSearch.cpp b/src/Storages/MeiliSearch/StorageMeiliSearch.cpp index 51e693f4dfd..2b2ce9016c0 100644 --- a/src/Storages/MeiliSearch/StorageMeiliSearch.cpp +++ b/src/Storages/MeiliSearch/StorageMeiliSearch.cpp @@ -62,28 +62,21 @@ std::string convertASTtoStr(ASTPtr ptr) ASTPtr getFunctionParams(ASTPtr node, const std::string & name) { if (!node) - { return nullptr; - } + const auto * ptr = node->as(); if (ptr && ptr->name == name) { - if (node->children.size() == 1) - { + if (node->children.size() == 1) return node->children[0]; - } - else - { + else return nullptr; - } } for (const auto & next : node->children) { auto res = getFunctionParams(next, name); if (res != nullptr) - { return res; - } } return nullptr; } @@ -112,9 +105,8 @@ Pipe StorageMeiliSearch::read( auto str = el->getColumnName(); auto it = find(str.begin(), str.end(), '='); if (it == str.end()) - { throw Exception("meiliMatch function must have parameters of the form \'key=value\'", ErrorCodes::BAD_QUERY_PARAMETER); - } + String key(str.begin() + 1, it); String value(it + 1, str.end() - 1); kv_pairs_params[key] = value; @@ -126,9 +118,7 @@ Pipe StorageMeiliSearch::read( } for (const auto & el : kv_pairs_params) - { LOG_TRACE(log, "Parsed parameter: key = " + el.first + ", value = " + el.second); - } Block sample_block; for (const String & column_name : column_names)