From 704ec8dea6b4fa583689508e4362644517321005 Mon Sep 17 00:00:00 2001 From: Matt Woenker Date: Mon, 5 Aug 2024 14:55:33 -0400 Subject: [PATCH 1/6] Handle incomplete sequences at end of input --- src/IO/WriteBufferValidUTF8.cpp | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/IO/WriteBufferValidUTF8.cpp b/src/IO/WriteBufferValidUTF8.cpp index d611befac37..426f302cb02 100644 --- a/src/IO/WriteBufferValidUTF8.cpp +++ b/src/IO/WriteBufferValidUTF8.cpp @@ -149,9 +149,27 @@ void WriteBufferValidUTF8::finalizeImpl() /// Write all complete sequences from buffer. nextImpl(); - /// If unfinished sequence at end, then write replacement. - if (working_buffer.begin() != memory.data()) - putReplacement(); + /// Handle remaining bytes if we have an incomplete sequence + if (working_buffer.begin() != memory.data()) { + char * p = memory.data(); + + while (p < pos) { + UInt8 len = length_of_utf8_sequence[static_cast(*p)]; + if (p + len > pos) { + // Incomplete sequence. Skip one byte. + putReplacement(); + ++p; + } else if (Poco::UTF8Encoding::isLegal(reinterpret_cast(p), len)) { + // Valid sequence + putValid(p, len); + p += len; + } else { + // Invalid sequence, skip first byte. + putReplacement(); + ++p; + } + } + } } } From 2a1ee419b473350323c5582bcb950ab5630a07e2 Mon Sep 17 00:00:00 2001 From: Matt Woenker Date: Tue, 6 Aug 2024 15:53:49 -0400 Subject: [PATCH 2/6] Fix style issue --- src/IO/WriteBufferValidUTF8.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/IO/WriteBufferValidUTF8.cpp b/src/IO/WriteBufferValidUTF8.cpp index 426f302cb02..8441b4eafa2 100644 --- a/src/IO/WriteBufferValidUTF8.cpp +++ b/src/IO/WriteBufferValidUTF8.cpp @@ -150,20 +150,27 @@ void WriteBufferValidUTF8::finalizeImpl() nextImpl(); /// Handle remaining bytes if we have an incomplete sequence - if (working_buffer.begin() != memory.data()) { + if (working_buffer.begin() != memory.data()) + { char * p = memory.data(); - while (p < pos) { + while (p < pos) + { UInt8 len = length_of_utf8_sequence[static_cast(*p)]; - if (p + len > pos) { + if (p + len > pos) + { // Incomplete sequence. Skip one byte. putReplacement(); ++p; - } else if (Poco::UTF8Encoding::isLegal(reinterpret_cast(p), len)) { + } + else if (Poco::UTF8Encoding::isLegal(reinterpret_cast(p), len)) + { // Valid sequence putValid(p, len); p += len; - } else { + } + else + { // Invalid sequence, skip first byte. putReplacement(); ++p; From f4ed3f5c6d138dc6ac7209393c60562ddf9e8d24 Mon Sep 17 00:00:00 2001 From: Matt Woenker Date: Tue, 6 Aug 2024 16:14:26 -0400 Subject: [PATCH 3/6] Add test --- .../03221_incomplete-utf8-sequence.reference | 16 ++++++++++++++++ .../03221_incomplete-utf8-sequence.sql | 2 ++ 2 files changed, 18 insertions(+) create mode 100644 tests/queries/0_stateless/03221_incomplete-utf8-sequence.reference create mode 100644 tests/queries/0_stateless/03221_incomplete-utf8-sequence.sql diff --git a/tests/queries/0_stateless/03221_incomplete-utf8-sequence.reference b/tests/queries/0_stateless/03221_incomplete-utf8-sequence.reference new file mode 100644 index 00000000000..4577427251d --- /dev/null +++ b/tests/queries/0_stateless/03221_incomplete-utf8-sequence.reference @@ -0,0 +1,16 @@ +{ + "meta": + [ + { + "name": "unhex('f0')", + "type": "String" + } + ], + + "data": + [ + ["�"] + ], + + "rows": 1 +} diff --git a/tests/queries/0_stateless/03221_incomplete-utf8-sequence.sql b/tests/queries/0_stateless/03221_incomplete-utf8-sequence.sql new file mode 100644 index 00000000000..ee4f25f3b4a --- /dev/null +++ b/tests/queries/0_stateless/03221_incomplete-utf8-sequence.sql @@ -0,0 +1,2 @@ +SET output_format_write_statistics = 0; +SELECT unhex('f0') FORMAT JSONCompact; From cff7cbac94b1b6f39dd4585fca5bcbcb2f952747 Mon Sep 17 00:00:00 2001 From: Matt Woenker Date: Wed, 7 Aug 2024 16:21:16 -0400 Subject: [PATCH 4/6] Style fix --- src/IO/WriteBufferValidUTF8.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/IO/WriteBufferValidUTF8.cpp b/src/IO/WriteBufferValidUTF8.cpp index 8441b4eafa2..25c7b3d4820 100644 --- a/src/IO/WriteBufferValidUTF8.cpp +++ b/src/IO/WriteBufferValidUTF8.cpp @@ -159,19 +159,19 @@ void WriteBufferValidUTF8::finalizeImpl() UInt8 len = length_of_utf8_sequence[static_cast(*p)]; if (p + len > pos) { - // Incomplete sequence. Skip one byte. + /// Incomplete sequence. Skip one byte. putReplacement(); ++p; } else if (Poco::UTF8Encoding::isLegal(reinterpret_cast(p), len)) { - // Valid sequence + /// Valid sequence putValid(p, len); p += len; } else { - // Invalid sequence, skip first byte. + /// Invalid sequence, skip first byte. putReplacement(); ++p; } From d09531e48a90db637a3e5b42a2fd9569c911ed44 Mon Sep 17 00:00:00 2001 From: Matt Woenker Date: Wed, 7 Aug 2024 16:25:01 -0400 Subject: [PATCH 5/6] Rename test files --- ...equence.reference => 03221_incomplete_utf8_sequence.reference} | 0 ...plete-utf8-sequence.sql => 03221_incomplete_utf8_sequence.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{03221_incomplete-utf8-sequence.reference => 03221_incomplete_utf8_sequence.reference} (100%) rename tests/queries/0_stateless/{03221_incomplete-utf8-sequence.sql => 03221_incomplete_utf8_sequence.sql} (100%) diff --git a/tests/queries/0_stateless/03221_incomplete-utf8-sequence.reference b/tests/queries/0_stateless/03221_incomplete_utf8_sequence.reference similarity index 100% rename from tests/queries/0_stateless/03221_incomplete-utf8-sequence.reference rename to tests/queries/0_stateless/03221_incomplete_utf8_sequence.reference diff --git a/tests/queries/0_stateless/03221_incomplete-utf8-sequence.sql b/tests/queries/0_stateless/03221_incomplete_utf8_sequence.sql similarity index 100% rename from tests/queries/0_stateless/03221_incomplete-utf8-sequence.sql rename to tests/queries/0_stateless/03221_incomplete_utf8_sequence.sql From f8a14e86d8db56e8ad7046b8848c7521d235a777 Mon Sep 17 00:00:00 2001 From: Matt Woenker Date: Wed, 7 Aug 2024 16:32:13 -0400 Subject: [PATCH 6/6] Add const a few places --- src/IO/WriteBufferValidUTF8.cpp | 8 ++++---- src/IO/WriteBufferValidUTF8.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/IO/WriteBufferValidUTF8.cpp b/src/IO/WriteBufferValidUTF8.cpp index 25c7b3d4820..2a86f8c2801 100644 --- a/src/IO/WriteBufferValidUTF8.cpp +++ b/src/IO/WriteBufferValidUTF8.cpp @@ -54,7 +54,7 @@ inline void WriteBufferValidUTF8::putReplacement() } -inline void WriteBufferValidUTF8::putValid(char *data, size_t len) +inline void WriteBufferValidUTF8::putValid(const char *data, size_t len) { if (len == 0) return; @@ -152,18 +152,18 @@ void WriteBufferValidUTF8::finalizeImpl() /// Handle remaining bytes if we have an incomplete sequence if (working_buffer.begin() != memory.data()) { - char * p = memory.data(); + const char * p = memory.data(); while (p < pos) { - UInt8 len = length_of_utf8_sequence[static_cast(*p)]; + UInt8 len = length_of_utf8_sequence[static_cast(*p)]; if (p + len > pos) { /// Incomplete sequence. Skip one byte. putReplacement(); ++p; } - else if (Poco::UTF8Encoding::isLegal(reinterpret_cast(p), len)) + else if (Poco::UTF8Encoding::isLegal(reinterpret_cast(p), len)) { /// Valid sequence putValid(p, len); diff --git a/src/IO/WriteBufferValidUTF8.h b/src/IO/WriteBufferValidUTF8.h index daaf0427f88..a398b8ded01 100644 --- a/src/IO/WriteBufferValidUTF8.h +++ b/src/IO/WriteBufferValidUTF8.h @@ -26,7 +26,7 @@ public: private: void putReplacement(); - void putValid(char * data, size_t len); + void putValid(const char * data, size_t len); void nextImpl() override; void finalizeImpl() override;