mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-25 17:12:03 +00:00
9c7db6711b
Occasionally, 02479_mysql_connect_to_self fails on CI [1]. [1]: https://github.com/ClickHouse/ClickHouse/issues/50911 The problem was indeed query profiler and EINTR, but not in a way you may think. For such failures you may see the following trace in trace_log: contrib/openssl/crypto/bio/bss_sock.c:127::sock_read contrib/openssl/crypto/bio/bio_meth.c:121::bread_conv contrib/openssl/crypto/bio/bio_lib.c:285::bio_read_intern contrib/openssl/crypto/bio/bio_lib.c:311::BIO_read contrib/openssl/ssl/record/methods/tls_common.c:398::tls_default_read_n contrib/openssl/ssl/record/methods/tls_common.c:575::tls_get_more_records contrib/openssl/ssl/record/methods/tls_common.c:1122::tls_read_record contrib/openssl/ssl/record/rec_layer_s3.c:645::ssl3_read_bytes contrib/openssl/ssl/s3_lib.c:4527::ssl3_read_internal contrib/openssl/ssl/s3_lib.c:4551::ssl3_read contrib/openssl/ssl/ssl_lib.c:2343::ssl_read_internal contrib/openssl/ssl/ssl_lib.c:2357::SSL_read contrib/mariadb-connector-c/libmariadb/secure/openssl.c:729::ma_tls_read contrib/mariadb-connector-c/libmariadb/ma_tls.c:90::ma_pvio_tls_read contrib/mariadb-connector-c/libmariadb/ma_pvio.c:250::ma_pvio_read contrib/mariadb-connector-c/libmariadb/ma_pvio.c:297::ma_pvio_cache_read contrib/mariadb-connector-c/libmariadb/ma_net.c:373::ma_real_read contrib/mariadb-connector-c/libmariadb/ma_net.c:427::ma_net_read contrib/mariadb-connector-c/libmariadb/mariadb_lib.c:192::ma_net_safe_read contrib/mariadb-connector-c/libmariadb/mariadb_lib.c:2138::mthd_my_read_query_result contrib/mariadb-connector-c/libmariadb/mariadb_lib.c:2212::mysql_real_query src/Common/mysqlxx/Query.cpp:56::mysqlxx::Query::executeImpl() src/Common/mysqlxx/Query.cpp:73::mysqlxx::Query::use() src/Processors/Sources/MySQLSource.cpp:50::DB::MySQLSource::Connection::Connection() After which the connection will fail with: Code: 1000. DB::Exception: Received from localhost:9000. DB::Exception: mysqlxx::ConnectionLost: Lost connection to MySQL server during query (127.0.0.1:9004). (POCO_EXCEPTION) But, if you will take a look at ma_tls_read() you will see that it has proper retries for SSL_ERROR_WANT_READ (and EINTR is just a special case of it), but still, for some reason it fails. And the reason is the units of the read/write timeout, ma_tls_read() calls poll(read_timeout) in case of SSL_ERROR_WANT_READ, but, it incorrectly assume that the timeout is in milliseconds, but that timeout was in seconds, this bug had been fixed in [2], and now it works like a charm! [2]: https://github.com/ClickHouse/mariadb-connector-c/pull/17 I've verified it with patching openssl library: diff --git a/crypto/bio/bss_sock.c b/crypto/bio/bss_sock.c index 82f7be85ae..3d2f3926a0 100644 --- a/crypto/bio/bss_sock.c +++ b/crypto/bio/bss_sock.c @@ -124,7 +124,24 @@ static int sock_read(BIO *b, char *out, int outl) ret = ktls_read_record(b->num, out, outl); else # endif - ret = readsocket(b->num, out, outl); + { + static int i = 0; + static int j = 0; + if (!(++j % 5)) + { + fprintf(stderr, "sock_read: inject EAGAIN with ret=0\n"); + ret = 0; + errno = EAGAIN; + } + else if (!(++i % 3)) + { + fprintf(stderr, "sock_read: inject EAGAIN with ret=-1\n"); + ret = -1; + errno = EAGAIN; + } + else + ret = readsocket(b->num, out, outl); + } BIO_clear_retry_flags(b); if (ret <= 0) { if (BIO_sock_should_retry(ret)) And before this patch (well, not the patch itself, but the referenced patch in mariadb-connector-c) if you will pass read_write_timeout=1 it will fail: SELECT * FROM mysql('127.0.0.1:9004', system, one, 'default', '', SETTINGS connect_timeout = 100, connection_wait_timeout = 100, read_write_timeout=1) Code: 1000. DB::Exception: Received from localhost:9000. DB::Exception: mysqlxx::ConnectionLost: Lost connection to MySQL server during query (127.0.0.1:9004). (POCO_EXCEPTION) But after, it always works: $ ch benchmark -c10 -q "SELECT * FROM mysql('127.0.0.1:9004', system, one, 'default', '', SETTINGS connection_pool_size=1, connect_timeout = 100, connection_wait_timeout = 100, read_write_timeout=1)" ^CStopping launch of queries. SIGINT received. Queries executed: 478. localhost:9000, queries: 478, QPS: 120.171, RPS: 120.171, MiB/s: 0.001, result RPS: 120.171, result MiB/s: 0.001. 0.000% 0.014 sec. 10.000% 0.058 sec. 20.000% 0.065 sec. 30.000% 0.073 sec. 40.000% 0.079 sec. 50.000% 0.087 sec. 60.000% 0.089 sec. 70.000% 0.091 sec. 80.000% 0.095 sec. 90.000% 0.100 sec. 95.000% 0.102 sec. 99.000% 0.105 sec. 99.900% 0.140 sec. 99.990% 0.140 sec. Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com> |
||
---|---|---|
.. | ||
abseil-cpp@a3c4dd3e77 | ||
abseil-cpp-cmake | ||
aklomp-base64@e77bd70bdd | ||
aklomp-base64-cmake | ||
AMQP-CPP@00f09897ce | ||
amqpcpp-cmake | ||
annoy@f2ac8e7b48 | ||
annoy-cmake | ||
arrow@5cfccd8ea6 | ||
arrow-cmake | ||
avro@d43acc84d3 | ||
avro-cmake | ||
aws@1c2946bfcb | ||
aws-c-auth@baeffa791d | ||
aws-c-cal@1586846816 | ||
aws-c-common@80f21b3cac | ||
aws-c-compression@99ec79ee29 | ||
aws-c-event-stream@08f24e384e | ||
aws-c-http@a082f8a206 | ||
aws-c-io@11ce3c750a | ||
aws-c-mqtt@6d36cd3726 | ||
aws-c-s3@de36fee8fe | ||
aws-c-sdkutils@fd8c0ba2e2 | ||
aws-checksums@321b805559 | ||
aws-cmake | ||
aws-crt-cpp@f532d6abc0 | ||
aws-s2n-tls@9a1e754540 | ||
azure@6262a76ef4 | ||
azure-cmake | ||
boost@ae94606a70 | ||
boost-cmake | ||
brotli@63be8a9940 | ||
brotli-cmake | ||
bzip2@bf905ea225 | ||
bzip2-cmake | ||
c-ares@6360e96b5c | ||
c-ares-cmake | ||
capnproto@976209a6d1 | ||
capnproto-cmake | ||
cassandra@f4a31e92a2 | ||
cassandra-cmake | ||
cctz@7918cb7afe | ||
cctz-cmake | ||
cityhash102 | ||
cld2@217ba8b880 | ||
cld2-cmake | ||
consistent-hashing | ||
corrosion@d5bdbfacb4 | ||
corrosion-cmake | ||
cppkafka@9c5ea0e332 | ||
cppkafka-cmake | ||
crc32-s390x@30980583bf | ||
crc32-s390x-cmake | ||
crc32-vpmsum@4521554393 | ||
crc32-vpmsum-cmake | ||
croaring@9b7cc0ff1c | ||
croaring-cmake | ||
curl@de7b3e8921 | ||
curl-cmake | ||
cyrus-sasl@e6466edfd6 | ||
cyrus-sasl-cmake | ||
datasketches-cpp@76edd74f5d | ||
datasketches-cpp-cmake | ||
double-conversion@4f7a25d8ce | ||
double-conversion-cmake | ||
dragonbox@923705af6f | ||
dragonbox-cmake | ||
expected@3f0ca7b192 | ||
expected-cmake | ||
fast_float@7eae925b51 | ||
fast_float-cmake | ||
fastops@1460583af7 | ||
fastops-cmake | ||
flatbuffers@eb3f827948 | ||
fmtlib@a33701196a | ||
fmtlib-cmake | ||
FP16@0a92994d72 | ||
FP16-cmake | ||
google-benchmark@2257fa4d6a | ||
google-benchmark-cmake | ||
google-protobuf@0fae801fb4 | ||
google-protobuf-cmake | ||
googletest@a7f443b80b | ||
googletest-cmake | ||
grpc@77b2737a70 | ||
grpc-cmake | ||
gwpasan-cmake | ||
h3@c7f46cfd71 | ||
h3-cmake | ||
hive-metastore@809a77d435 | ||
hive-metastore-cmake | ||
icu@a56dde820d | ||
icu-cmake | ||
icudata@c8e717892a | ||
idna@3c8be01d42 | ||
idna-cmake | ||
idxd-config@a836ce0e42 | ||
idxd-config-cmake | ||
incbin@6e576cae5a | ||
incbin-cmake | ||
isa-l@9f2b68f057 | ||
isa-l-cmake | ||
jemalloc@41a859ef73 | ||
jemalloc-cmake | ||
krb5@71b06c2276 | ||
krb5-cmake | ||
lemmagen-c@59537bdcf5 | ||
lemmagen-c-cmake | ||
libarchive@ee45796171 | ||
libarchive-cmake | ||
libbcrypt@8aa32ad94e | ||
libbcrypt-cmake | ||
libcpuid@503083acb7 | ||
libcpuid-cmake | ||
libcxx-cmake | ||
libcxxabi-cmake | ||
libdivide@3bd3438857 | ||
libdivide-cmake | ||
libfarmhash | ||
libfiu@b85edbde4c | ||
libfiu-cmake | ||
libfuzzer-cmake | ||
libgsasl@0fb79e7609 | ||
libgsasl-cmake | ||
libhdfs3@0d04201c45 | ||
libhdfs3-cmake | ||
libmetrohash | ||
libpq@2446f2c856 | ||
libpq-cmake | ||
libpqxx@c995193a3a | ||
libpqxx-cmake | ||
libprotobuf-mutator@a304ec48dc | ||
libprotobuf-mutator-cmake | ||
librdkafka@2d2aab6f5b | ||
librdkafka-cmake | ||
libssh@ed4011b918 | ||
libssh-cmake | ||
libstemmer_c@c753054304 | ||
libstemmer-c-cmake | ||
libunwind@d6a01c4632 | ||
libunwind-cmake | ||
liburing@f4e42a515c | ||
liburing-cmake | ||
libuv@4482964660 | ||
libuv-cmake | ||
libxml2@223cb03a5d | ||
libxml2-cmake | ||
llvm-project@d2142eed98 | ||
llvm-project-cmake | ||
lz4@145f3804ca | ||
lz4-cmake | ||
magic_enum@38f86e4d09 | ||
magic-enum-cmake | ||
mariadb-connector-c@d0a788c5b9 | ||
mariadb-connector-c-cmake | ||
miniselect@be0af6bd0b | ||
miniselect-cmake | ||
minizip-ng@f3d400e999 | ||
minizip-ng-cmake | ||
morton-nd@3795491a4a | ||
morton-nd-cmake | ||
msgpack-c@46684265d5 | ||
msgpack-c-cmake | ||
murmurhash | ||
nanodbc@df52a1232d | ||
nanodbc-cmake | ||
nats-io@1e2597c546 | ||
nats-io-cmake | ||
nlp-data@5591f91f5e | ||
NuRaft@cb5dc3c906 | ||
nuraft-cmake | ||
openldap@5671b80e36 | ||
openldap-cmake | ||
openssl@5d81fa7068 | ||
openssl-cmake | ||
orc@947cebaf94 | ||
pdqsort | ||
pdqsort-cmake | ||
pocketfft@9efd4da52c | ||
pocketfft-cmake | ||
QAT-ZSTD-Plugin@e5a134e12d | ||
QAT-ZSTD-Plugin-cmake | ||
qatlib@abe15d7bfc | ||
qatlib-cmake | ||
qpl@d4715e0e79 | ||
qpl-cmake | ||
rapidjson@800ca2f38f | ||
rapidjson-cmake | ||
re2@85dd7ad833 | ||
re2-cmake | ||
replxx@5d04501f93 | ||
replxx-cmake | ||
robin-map@851a59e0e3 | ||
robin-map-cmake | ||
rocksdb@3a0b80ca9d | ||
rocksdb-cmake | ||
rust_vendor@08e82ca654 | ||
s2geometry@0547c38371 | ||
s2geometry-cmake | ||
sentry-native@bc359f86cb | ||
sentry-native-cmake | ||
simdjson@6060be2fdf | ||
simdjson-cmake | ||
SimSIMD@de2cb75b9e | ||
SimSIMD-cmake | ||
snappy@6ebb5b1ab8 | ||
snappy-cmake | ||
sparse-checkout | ||
sparsehash-c11@cf0bffaa45 | ||
sparsehash-c11-cmake | ||
sqids-cpp@a471f53672 | ||
sqids-cpp-cmake | ||
sqlite-amalgamation@2059807989 | ||
sqlite-cmake | ||
sysroot@39c4713334 | ||
thrift@2a93df80f2 | ||
thrift-cmake | ||
ulid-c@c433b6783c | ||
ulid-c-cmake | ||
unixodbc@18e0ebe2a1 | ||
unixodbc-cmake | ||
usearch@955c6f9c11 | ||
usearch-cmake | ||
vectorscan@38431d1117 | ||
vectorscan-cmake | ||
wordnet-blast@1d16ac2803 | ||
wordnet-blast-cmake | ||
wyhash@991aa3dab6 | ||
wyhash-cmake | ||
xxHash@bbb27a5efb | ||
xxHash-cmake | ||
xz@869b9d1b4e | ||
xz-cmake | ||
yaml-cpp@f91e938341 | ||
yaml-cpp-cmake | ||
zlib-ng@50f0eae1a4 | ||
zlib-ng-cmake | ||
zstd@63779c7982 | ||
zstd-cmake | ||
CMakeLists.txt | ||
update-submodules.sh |