ClickHouse/base/mysqlxx/PoolWithFailover.h
Azat Khuzhin ed70ed6f71 Introduce connection_no_wait setting for MySQL engine.
This will allow to avoid superfluous sleep during query execution, since
this not only not desired behavoiur, but also may hang the server, since
if you will execute enough queries that will use MySQL database but will
not allow enough connections (or your MySQL server is too slow) then you
may run out of threads in the global thread pool.

Also note that right now it is possible to get deadlock when the mysql
pool is full, consider the following scenario:

- you have m1 and m2 mysql tables
- you have q1 and q2 queries, bot queries join m1 and m2
- q1 allocated connection for m1 but cannot allocate connection for m2
- q2 allocated connection for m2 but cannot allocate connection for m1
- but to resolve the lock one should give up on the locking while it is not possible right now...

And then you got no free threads and this:

    # grep -h ^202 /proc/$(pgrep clickhouse-serv)/task/*/syscall  | cut -d' ' -f2 | sort | uniq -c | sort -nr | head
       1554 0x7ffb60b92fe8 # mutex in mysqlxx::PoolWithFailover::get
       1375 0x7ffb9f1c4748 # mutex in ::PoolEntryHelper::~PoolEntryHelper from DB::MultiplexedConnections::invalidateReplica
       1160 0x7ffb612918b8 # mutex in mysqlxx::PoolWithFailover::get
         42 0x7ffb9f057984 # mutex in ThreadPoolImpl<std::__1::thread>::worker

    *NOTE: 202 is a `futex` with WAIT*

(Went with `syscall` because debugging 10k+ threads is not easy, and
eventually it may TRAP)
2021-09-02 22:23:37 +03:00

134 lines
4.6 KiB
C++

#pragma once
#include "Pool.h"
#define MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS 1
#define MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS 16
#define MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES 3
namespace mysqlxx
{
/** MySQL connection pool with support of failover.
*
* For dictionary source:
* Have information about replicas and their priorities.
* Tries to connect to replica in an order of priority. When equal priority, choose replica with maximum time without connections.
*
* It could be configured without replicas, exactly as ordinary Pool:
*
* <mysql_metrica>
* <host>mtstat01c*</host>
* <port>3306</port>
* <user>metrica</user>
* <password></password>
* <db>Metrica</db>
* </mysql_metrica>
*
* Or like this:
*
* <mysql_metrica>
* <replica>
* <host>mtstat01c</host>
* <port>3306</port>
* <user>metrica</user>
* <password></password>
* <db>Metrica</db>
* <priority>0</priority>
* </replica>
* <replica>
* <host>mtstat01d</host>
* <port>3306</port>
* <user>metrica</user>
* <password></password>
* <db>Metrica</db>
* <priority>1</priority>
* </replica>
* </mysql_metrica>
*
* Or like this:
*
* <mysql_metrica>
* <port>3306</port>
* <user>metrica</user>
* <password></password>
* <db>Metrica</db>
* <replica>
* <host>mtstat01c</host>
* <priority>0</priority>
* </replica>
* <replica>
* <host>mtstat01d</host>
* <priority>1</priority>
* </replica>
* </mysql_metrica>
*/
class PoolWithFailover final
{
private:
using PoolPtr = std::shared_ptr<Pool>;
using Replicas = std::vector<PoolPtr>;
/// [priority][index] -> replica. Highest priority is 0.
using ReplicasByPriority = std::map<int, Replicas>;
ReplicasByPriority replicas_by_priority;
/// Number of connection tries.
size_t max_tries;
/// Mutex for set of replicas.
std::mutex mutex;
/// Can the Pool be shared
bool shareable;
/// Timeout for waiting free connection.
uint64_t wait_timeout = 0;
public:
using Entry = Pool::Entry;
using RemoteDescription = std::vector<std::pair<std::string, uint16_t>>;
/**
* * Mysql dictionary sourse related params:
* config_name Name of parameter in configuration file for dictionary source.
*
* * Mysql storage related parameters:
* replicas_description
*
* * Mutual parameters:
* default_connections Number of connection in pool to each replica at start.
* max_connections Maximum number of connections in pool to each replica.
* max_tries_ Max number of connection tries.
* wait_timeout_ Timeout for waiting free connection.
*/
PoolWithFailover(
const std::string & config_name_,
unsigned default_connections_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS,
unsigned max_connections_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS,
size_t max_tries_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES);
PoolWithFailover(
const Poco::Util::AbstractConfiguration & config_,
const std::string & config_name_,
unsigned default_connections_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS,
unsigned max_connections_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS,
size_t max_tries_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES);
PoolWithFailover(
const std::string & database,
const RemoteDescription & addresses,
const std::string & user,
const std::string & password,
unsigned default_connections_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS,
unsigned max_connections_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS,
size_t max_tries_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES,
uint64_t wait_timeout_ = UINT64_MAX);
PoolWithFailover(const PoolWithFailover & other);
/** Allocates a connection to use. */
Entry get();
};
using PoolWithFailoverPtr = std::shared_ptr<PoolWithFailover>;
}