#include #include #include namespace ProfileEvents { extern const Event DistributedConnectionMissingTable; extern const Event DistributedConnectionStaleReplica; } namespace DB { namespace ErrorCodes { extern const int ATTEMPT_TO_READ_AFTER_EOF; extern const int NETWORK_ERROR; extern const int SOCKET_TIMEOUT; } ConnectionEstablisher::ConnectionEstablisher( IConnectionPool * pool_, const ConnectionTimeouts * timeouts_, const Settings * settings_, Poco::Logger * log_, const QualifiedTableName * table_to_check_) : pool(pool_), timeouts(timeouts_), settings(settings_), log(log_), table_to_check(table_to_check_), is_finished(false) { } void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std::string & fail_message) { is_finished = false; SCOPE_EXIT(is_finished = true); try { result.entry = pool->get(*timeouts, settings, /* force_connected = */ false); AsyncCallbackSetter async_setter(&*result.entry, std::move(async_callback)); UInt64 server_revision = 0; if (table_to_check) server_revision = result.entry->getServerRevision(*timeouts); if (!table_to_check || server_revision < DBMS_MIN_REVISION_WITH_TABLES_STATUS) { result.entry->forceConnected(*timeouts); result.is_usable = true; result.is_up_to_date = true; return; } /// Only status of the remote table corresponding to the Distributed table is taken into account. /// TODO: request status for joined tables also. TablesStatusRequest status_request; status_request.tables.emplace(*table_to_check); TablesStatusResponse status_response = result.entry->getTablesStatus(*timeouts, status_request); auto table_status_it = status_response.table_states_by_id.find(*table_to_check); if (table_status_it == status_response.table_states_by_id.end()) { fail_message = fmt::format("There is no table {}.{} on server: {}", backQuote(table_to_check->database), backQuote(table_to_check->table), result.entry->getDescription()); LOG_WARNING(log, fmt::runtime(fail_message)); ProfileEvents::increment(ProfileEvents::DistributedConnectionMissingTable); return; } result.is_usable = true; UInt64 max_allowed_delay = settings ? UInt64(settings->max_replica_delay_for_distributed_queries) : 0; if (!max_allowed_delay) { result.is_up_to_date = true; return; } UInt32 delay = table_status_it->second.absolute_delay; if (delay < max_allowed_delay) result.is_up_to_date = true; else { result.is_up_to_date = false; result.staleness = delay; LOG_TRACE(log, "Server {} has unacceptable replica delay for table {}.{}: {}", result.entry->getDescription(), table_to_check->database, table_to_check->table, delay); ProfileEvents::increment(ProfileEvents::DistributedConnectionStaleReplica); } } catch (const Exception & e) { if (e.code() != ErrorCodes::NETWORK_ERROR && e.code() != ErrorCodes::SOCKET_TIMEOUT && e.code() != ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF) throw; fail_message = getCurrentExceptionMessage(/* with_stacktrace = */ false); if (!result.entry.isNull()) { result.entry->disconnect(); result.reset(); } } } #if defined(OS_LINUX) ConnectionEstablisherAsync::ConnectionEstablisherAsync( IConnectionPool * pool_, const ConnectionTimeouts * timeouts_, const Settings * settings_, Poco::Logger * log_, const QualifiedTableName * table_to_check_) : connection_establisher(pool_, timeouts_, settings_, log_, table_to_check_) { epoll.add(receive_timeout.getDescriptor()); } void ConnectionEstablisherAsync::Routine::ReadCallback::operator()(int fd, Poco::Timespan timeout, const std::string &) { /// Check if it's the first time and we need to add socket fd to epoll. if (connection_establisher_async.socket_fd == -1) { connection_establisher_async.epoll.add(fd); connection_establisher_async.socket_fd = fd; } connection_establisher_async.receive_timeout.setRelative(timeout); fiber = std::move(fiber).resume(); connection_establisher_async.receive_timeout.reset(); } Fiber ConnectionEstablisherAsync::Routine::operator()(Fiber && sink) { try { connection_establisher_async.connection_establisher.setAsyncCallback(ReadCallback{connection_establisher_async, sink}); connection_establisher_async.connection_establisher.run(connection_establisher_async.result, connection_establisher_async.fail_message); } catch (const boost::context::detail::forced_unwind &) { /// This exception is thrown by fiber implementation in case if fiber is being deleted but hasn't exited /// It should not be caught or it will segfault. /// Other exceptions must be caught throw; } catch (...) { connection_establisher_async.exception = std::current_exception(); } return std::move(sink); } std::variant ConnectionEstablisherAsync::resume() { if (!fiber_created) { reset(); fiber = boost::context::fiber(std::allocator_arg_t(), fiber_stack, Routine{*this}); fiber_created = true; } else if (!checkReceiveTimeout()) return result; fiber = std::move(fiber).resume(); if (exception) std::rethrow_exception(exception); if (connection_establisher.isFinished()) { destroyFiber(); return result; } return epoll.getFileDescriptor(); } bool ConnectionEstablisherAsync::checkReceiveTimeout() { bool is_socket_ready = false; bool is_receive_timeout_alarmed = false; epoll_event events[2]; events[0].data.fd = events[1].data.fd = -1; size_t ready_count = epoll.getManyReady(2, events, false); for (size_t i = 0; i != ready_count; ++i) { if (events[i].data.fd == socket_fd) is_socket_ready = true; if (events[i].data.fd == receive_timeout.getDescriptor()) is_receive_timeout_alarmed = true; } if (is_receive_timeout_alarmed && !is_socket_ready) { destroyFiber(); /// In not async case this exception would be thrown and caught in ConnectionEstablisher::run, /// but in async case we process timeout outside and cannot throw exception. So, we just save fail message. fail_message = fmt::format( "Timeout exceeded while reading from socket ({}, receive timeout {} ms)", result.entry->getDescription(), result.entry->getSocket()->getReceiveTimeout().totalMilliseconds()); epoll.remove(socket_fd); resetResult(); return false; } return true; } void ConnectionEstablisherAsync::cancel() { destroyFiber(); reset(); } void ConnectionEstablisherAsync::reset() { resetResult(); fail_message.clear(); socket_fd = -1; } void ConnectionEstablisherAsync::resetResult() { if (!result.entry.isNull()) { result.entry->disconnect(); result.reset(); } } void ConnectionEstablisherAsync::destroyFiber() { Fiber to_destroy = std::move(fiber); fiber_created = false; } #endif }