From 6ee6b751be55154edd8c3132c666876933e9a75e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 9 Jun 2020 03:38:47 +0300 Subject: [PATCH 1/4] Added failing test #11539 --- .../0_stateless/01304_direct_io.reference | 1 + tests/queries/0_stateless/01304_direct_io.sh | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 tests/queries/0_stateless/01304_direct_io.reference create mode 100755 tests/queries/0_stateless/01304_direct_io.sh diff --git a/tests/queries/0_stateless/01304_direct_io.reference b/tests/queries/0_stateless/01304_direct_io.reference new file mode 100644 index 00000000000..ec7a223ddc2 --- /dev/null +++ b/tests/queries/0_stateless/01304_direct_io.reference @@ -0,0 +1 @@ +Loaded 1 queries. diff --git a/tests/queries/0_stateless/01304_direct_io.sh b/tests/queries/0_stateless/01304_direct_io.sh new file mode 100755 index 00000000000..0b6af15aa3b --- /dev/null +++ b/tests/queries/0_stateless/01304_direct_io.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --multiquery --query " + DROP TABLE IF EXISTS bug; + CREATE TABLE bug (UserID UInt64, Date Date) ENGINE = MergeTree ORDER BY Date; + INSERT INTO bug SELECT rand64(), '2020-06-07' FROM numbers(50000000); + OPTIMIZE TABLE bug FINAL;" + +$CLICKHOUSE_BENCHMARK --database $CLICKHOUSE_DATABASE --iterations 10 --max_threads 100 --min_bytes_to_use_direct_io 1 <<< "SELECT sum(UserID) FROM bug PREWHERE NOT ignore(Date)" >/dev/null 2>$CLICKHOUSE_TMP/err +cat $CLICKHOUSE_TMP/err + +$CLICKHOUSE_CLIENT --multiquery --query " + DROP TABLE bug;" From 981462db041e001573ffff35c0809aa0d8482ef2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 9 Jun 2020 18:01:34 +0300 Subject: [PATCH 2/4] Remove dump flag from ReadBufferAIO --- src/IO/ReadBufferAIO.cpp | 11 ++++------- src/IO/ReadBufferAIO.h | 2 -- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/IO/ReadBufferAIO.cpp b/src/IO/ReadBufferAIO.cpp index ffe8183f005..4a52b1825f7 100644 --- a/src/IO/ReadBufferAIO.cpp +++ b/src/IO/ReadBufferAIO.cpp @@ -95,11 +95,8 @@ bool ReadBufferAIO::nextImpl() if (profile_callback) watch.emplace(clock_type); - if (!is_aio) - { + if (!is_pending_read) synchronousRead(); - is_aio = true; - } else receive(); @@ -215,7 +212,9 @@ void ReadBufferAIO::synchronousRead() void ReadBufferAIO::receive() { if (!waitForAIOCompletion()) - return; + { + throw Exception("Trying to receive data from AIO, but nothing was queued. It's a bug", ErrorCodes::LOGICAL_ERROR); + } finalize(); } @@ -224,8 +223,6 @@ void ReadBufferAIO::skip() if (!waitForAIOCompletion()) return; - is_aio = false; - /// @todo I presume this assignment is redundant since waitForAIOCompletion() performs a similar one // bytes_read = future_bytes_read.get(); if ((bytes_read < 0) || (static_cast(bytes_read) < region_left_padding)) diff --git a/src/IO/ReadBufferAIO.h b/src/IO/ReadBufferAIO.h index 77274c47073..5b2cf247a45 100644 --- a/src/IO/ReadBufferAIO.h +++ b/src/IO/ReadBufferAIO.h @@ -100,8 +100,6 @@ private: bool is_eof = false; /// At least one read request was sent. bool is_started = false; - /// Is the operation asynchronous? - bool is_aio = false; /// Did the asynchronous operation fail? bool aio_failed = false; From e3ae0734c9bbf234ed7cb8542e49f3b3778c8069 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 9 Jun 2020 18:04:56 +0300 Subject: [PATCH 3/4] Better test --- tests/queries/0_stateless/01304_direct_io.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01304_direct_io.sh b/tests/queries/0_stateless/01304_direct_io.sh index 0b6af15aa3b..32091acd5eb 100755 --- a/tests/queries/0_stateless/01304_direct_io.sh +++ b/tests/queries/0_stateless/01304_direct_io.sh @@ -9,8 +9,9 @@ $CLICKHOUSE_CLIENT --multiquery --query " INSERT INTO bug SELECT rand64(), '2020-06-07' FROM numbers(50000000); OPTIMIZE TABLE bug FINAL;" -$CLICKHOUSE_BENCHMARK --database $CLICKHOUSE_DATABASE --iterations 10 --max_threads 100 --min_bytes_to_use_direct_io 1 <<< "SELECT sum(UserID) FROM bug PREWHERE NOT ignore(Date)" >/dev/null 2>$CLICKHOUSE_TMP/err -cat $CLICKHOUSE_TMP/err +$CLICKHOUSE_BENCHMARK --database $CLICKHOUSE_DATABASE --iterations 10 --max_threads 100 --min_bytes_to_use_direct_io 1 <<< "SELECT sum(UserID) FROM bug PREWHERE NOT ignore(Date)" 1>/dev/null 2>$CLICKHOUSE_TMP/err +cat $CLICKHOUSE_TMP/err | grep Exception +cat $CLICKHOUSE_TMP/err | grep Loaded $CLICKHOUSE_CLIENT --multiquery --query " DROP TABLE bug;" From a6feb81af194376b8d6cdcdc46798418b425e368 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 9 Jun 2020 18:11:11 +0300 Subject: [PATCH 4/4] Fix under msan --- src/IO/ReadBufferAIO.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/IO/ReadBufferAIO.cpp b/src/IO/ReadBufferAIO.cpp index 4a52b1825f7..8b01b67c0c0 100644 --- a/src/IO/ReadBufferAIO.cpp +++ b/src/IO/ReadBufferAIO.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -271,6 +272,9 @@ void ReadBufferAIO::prepare() region_aligned_size = region_aligned_end - region_aligned_begin; buffer_begin = fill_buffer.internalBuffer().begin(); + + /// Unpoison because msan doesn't instrument linux AIO + __msan_unpoison(buffer_begin, fill_buffer.internalBuffer().size()); } void ReadBufferAIO::finalize()