From 697d5b07ce87ff1c6a7ed866d55d1e968205ae0a Mon Sep 17 00:00:00 2001 From: Avogar Date: Fri, 1 May 2020 19:41:14 +0300 Subject: [PATCH] Fix build and style errors. Update test. --- .../Impl/JSONAsStringRowInputFormat.cpp | 28 +++-------- .../01232_json_as_string_format.reference | 6 +-- .../01232_json_as_string_format.sh | 47 +++++++++++++++++++ .../01232_json_as_string_format.sql | 7 --- 4 files changed, 57 insertions(+), 31 deletions(-) create mode 100755 tests/queries/0_stateless/01232_json_as_string_format.sh delete mode 100644 tests/queries/0_stateless/01232_json_as_string_format.sql diff --git a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp index 9d05eba78c9..4794c19b4a4 100644 --- a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp @@ -19,7 +19,7 @@ JSONAsStringRowInputFormat::JSONAsStringRowInputFormat(const Block & header_, Re { if (header_.columns() > 1 || header_.getDataTypes()[0]->getTypeId() != TypeIndex::String) { - throw Exception("This input format is only suitable for tables with a single column of type String.", ErrorCodes::LOGICAL_ERROR); + throw Exception("This input format is only suitable for tables with a single column of type String.", ErrorCodes::LOGICAL_ERROR); } } @@ -27,14 +27,12 @@ void JSONAsStringRowInputFormat::readJSONObject(IColumn & column) { PeekableReadBufferCheckpoint checkpoint{buf}; size_t balance = 0; - size_t object_size = 0; bool quotes = false; if (*buf.position() != '{') - throw Exception("JSON object must begin with '{'.", ErrorCodes::INCORRECT_DATA); + throw Exception("JSON object must begin with '{'.", ErrorCodes::INCORRECT_DATA); ++buf.position(); - ++object_size; ++balance; char * pos; @@ -42,12 +40,11 @@ void JSONAsStringRowInputFormat::readJSONObject(IColumn & column) while (balance) { if (buf.eof()) - throw Exception("Unexpected end of file while parsing JSON object.", ErrorCodes::INCORRECT_DATA); + throw Exception("Unexpected end of file while parsing JSON object.", ErrorCodes::INCORRECT_DATA); if (quotes) { pos = find_first_symbols<'"', '\\'>(buf.position(), buf.buffer().end()); - object_size += pos - buf.position(); buf.position() = pos; if (buf.position() == buf.buffer().end()) continue; @@ -55,23 +52,19 @@ void JSONAsStringRowInputFormat::readJSONObject(IColumn & column) { quotes = false; ++buf.position(); - ++object_size; } else if (*buf.position() == '\\') { ++buf.position(); - ++object_size; if (!buf.eof()) { ++buf.position(); - ++object_size; } } } else { pos = find_first_symbols<'"', '{', '}', '\\'>(buf.position(), buf.buffer().end()); - object_size += pos - buf.position(); buf.position() = pos; if (buf.position() == buf.buffer().end()) continue; @@ -79,36 +72,32 @@ void JSONAsStringRowInputFormat::readJSONObject(IColumn & column) { ++balance; ++buf.position(); - ++object_size; } else if (*buf.position() == '}') { --balance; ++buf.position(); - ++object_size; } else if (*buf.position() == '\\') { ++buf.position(); - ++object_size; if (!buf.eof()) { ++buf.position(); - ++object_size; } } else if (*buf.position() == '"') { quotes = true; ++buf.position(); - ++object_size; } } } buf.makeContinuousMemoryFromCheckpointToPos(); + char * end = buf.position(); buf.rollbackToCheckpoint(); - column.insertData(buf.position(), object_size); - buf.position() += object_size; + column.insertData(buf.position(), end - buf.position()); + buf.position() = end; } bool JSONAsStringRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) @@ -123,10 +112,7 @@ bool JSONAsStringRowInputFormat::readRow(MutableColumns & columns, RowReadExtens ++buf.position(); skipWhitespaceIfAny(buf); - if (buf.eof()) - return false; - - return true; + return !buf.eof(); } void registerInputFormatProcessorJSONAsString(FormatFactory & factory) diff --git a/tests/queries/0_stateless/01232_json_as_string_format.reference b/tests/queries/0_stateless/01232_json_as_string_format.reference index cc046468720..19d50bde85a 100644 --- a/tests/queries/0_stateless/01232_json_as_string_format.reference +++ b/tests/queries/0_stateless/01232_json_as_string_format.reference @@ -1,3 +1,3 @@ -{"id" : 1, "date" : "01.01.2020", "string" : "123{{{\\"\\\\", "array" : [1, 2, 3], "map": {"a" : 1, "b" : 2, "c" : 3}} -{"id" : 2, "date" : "01.02.2020", "string" : "{another\\" string}}", "array" : [3, 2, 1], "map" : {"z" : 1, "y" : 2, "x" : 3}} -{"id" : 3, "date" : "01.03.2020", "string" : "one more string", "array" : [3,1,2], "map" : {"{" : 1, "}}" : 2}} +{\n "id" : 1,\n "date" : "01.01.2020",\n "string" : "123{{{\\"\\\\",\n "array" : [1, 2, 3],\n "map": {\n "a" : 1,\n "b" : 2,\n "c" : 3\n }\n} +{\n "id" : 2,\n "date" : "01.02.2020",\n "string" : "{another\\"\n string}}",\n "array" : [3, 2, 1],\n "map" : {\n "z" : 1,\n "y" : 2, \n "x" : 3\n }\n} +{\n "id" : 3, \n "date" : "01.03.2020", \n "string" : "one more string", \n "array" : [3,1,2], \n "map" : {\n "{" : 1, \n "}}" : 2\n }\n} diff --git a/tests/queries/0_stateless/01232_json_as_string_format.sh b/tests/queries/0_stateless/01232_json_as_string_format.sh new file mode 100755 index 00000000000..44b3d9dd4a4 --- /dev/null +++ b/tests/queries/0_stateless/01232_json_as_string_format.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS json_as_string"; + +$CLICKHOUSE_CLIENT --query="CREATE TABLE json_as_string (field String) ENGINE = Memory"; + +echo ' +{ + "id" : 1, + "date" : "01.01.2020", + "string" : "123{{{\"\\", + "array" : [1, 2, 3], + "map": { + "a" : 1, + "b" : 2, + "c" : 3 + } +}, +{ + "id" : 2, + "date" : "01.02.2020", + "string" : "{another\" + string}}", + "array" : [3, 2, 1], + "map" : { + "z" : 1, + "y" : 2, + "x" : 3 + } +} +{ + "id" : 3, + "date" : "01.03.2020", + "string" : "one more string", + "array" : [3,1,2], + "map" : { + "{" : 1, + "}}" : 2 + } +}' | $CLICKHOUSE_CLIENT --query="INSERT INTO json_as_string FORMAT JSONAsString"; + +$CLICKHOUSE_CLIENT --query="SELECT * FROM json_as_string"; +$CLICKHOUSE_CLIENT --query="DROP TABLE json_as_string" + diff --git a/tests/queries/0_stateless/01232_json_as_string_format.sql b/tests/queries/0_stateless/01232_json_as_string_format.sql deleted file mode 100644 index 523bb60c7c8..00000000000 --- a/tests/queries/0_stateless/01232_json_as_string_format.sql +++ /dev/null @@ -1,7 +0,0 @@ -DROP TABLE IF EXISTS json_as_string; -CREATE TABLE json_as_string (field String) ENGINE = Memory; - -INSERT INTO json_as_string FORMAT JSONAsString {"id" : 1, "date" : "01.01.2020", "string" : "123{{{\"\\", "array" : [1, 2, 3], "map": {"a" : 1, "b" : 2, "c" : 3}} , {"id" : 2, "date" : "01.02.2020", "string" : "{another\" string}}", "array" : [3, 2, 1], "map" : {"z" : 1, "y" : 2, "x" : 3}} {"id" : 3, "date" : "01.03.2020", "string" : "one more string", "array" : [3,1,2], "map" : {"{" : 1, "}}" : 2}} - -SELECT * FROM json_as_string; -DROP TABLE json_as_string;