Add FIRST_OR_RANDOM Load Balancing policy

For cross-replication topology setups load_balancing=in_order works best
as nodes handle equal amount of load and usually they hit only 1/n of
data (n = number of replicas), which makes page cache usage more
efficient.

The problem is when one node of the shard goes down. If one replica goes
down, the next one in config will handle twice the usual load while
remaining ones will handle usual traffic.

Closes #4820.

dbms/programs/server/users.xml

@@ -16,6 +16,7 @@
                   with minimum number of different symbols between replica's hostname and local hostname
                   (Hamming distance).
                  in_order - first live replica is chosen in specified order.
+                 first_or_random - if first replica one has higher number of errors, pick a random one from replicas with minimum number of errors.
             -->
             <load_balancing>random</load_balancing>
         </default>

dbms/src/Client/ConnectionPoolWithFailover.cpp

@@ -62,6 +62,9 @@ IConnectionPool::Entry ConnectionPoolWithFailover::get(const Settings * settings
         break;
     case LoadBalancing::RANDOM:
         break;
+    case LoadBalancing::FIRST_OR_RANDOM:
+        get_priority = [](size_t i) { return i >= 1; };
+        break;
     }
 
     return Base::get(try_get_entry, get_priority);
@@ -134,6 +137,9 @@ std::vector<ConnectionPoolWithFailover::TryResult> ConnectionPoolWithFailover::g
         break;
     case LoadBalancing::RANDOM:
         break;
+    case LoadBalancing::FIRST_OR_RANDOM:
+        get_priority = [](size_t i) { return i >= 1; };
+        break;
     }
 
     bool fallback_to_stale_replicas = settings ? bool(settings->fallback_to_stale_replicas_for_distributed_queries) : true;

dbms/src/Core/SettingsCommon.cpp

@@ -247,15 +247,16 @@ LoadBalancing SettingLoadBalancing::getLoadBalancing(const String & s)
     if (s == "random")           return LoadBalancing::RANDOM;
     if (s == "nearest_hostname") return LoadBalancing::NEAREST_HOSTNAME;
     if (s == "in_order")         return LoadBalancing::IN_ORDER;
+    if (s == "first_or_random")  return LoadBalancing::FIRST_OR_RANDOM;
 
-    throw Exception("Unknown load balancing mode: '" + s + "', must be one of 'random', 'nearest_hostname', 'in_order'",
+    throw Exception("Unknown load balancing mode: '" + s + "', must be one of 'random', 'nearest_hostname', 'in_order', 'first_or_random'",
         ErrorCodes::UNKNOWN_LOAD_BALANCING);
 }
 
 String SettingLoadBalancing::toString() const
 {
-    const char * strings[] = {"random", "nearest_hostname", "in_order"};
-    if (value < LoadBalancing::RANDOM || value > LoadBalancing::IN_ORDER)
+    const char * strings[] = {"random", "nearest_hostname", "in_order", "first_or_random"};
+    if (value < LoadBalancing::RANDOM || value > LoadBalancing::FIRST_OR_RANDOM)
         throw Exception("Unknown load balancing mode", ErrorCodes::UNKNOWN_LOAD_BALANCING);
     return strings[static_cast<size_t>(value)];
 }

dbms/src/Core/SettingsCommon.h

@@ -167,6 +167,9 @@ enum class LoadBalancing
     NEAREST_HOSTNAME,
     /// replicas are walked through strictly in order; the number of errors does not matter
     IN_ORDER,
+    /// if first replica one has higher number of errors,
+    ///   pick a random one from replicas with minimum number of errors
+    FIRST_OR_RANDOM,
 };
 
 struct SettingLoadBalancing