mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-19 16:20:50 +00:00
Create Keeper on additional node
This commit is contained in:
Antonio Andelic
parent
8a3c56f09f
commit
dd709faa14
@ -1,110 +0,0 @@
|
||||
<clickhouse>
|
||||
<!-- See also the files in users.d directory where the settings can be overridden. -->
|
||||
|
||||
<!-- Profiles of settings. -->
|
||||
<profiles>
|
||||
<!-- Default settings. -->
|
||||
<default>
|
||||
</default>
|
||||
|
||||
<!-- Profile that allows only read queries. -->
|
||||
<readonly>
|
||||
<readonly>1</readonly>
|
||||
</readonly>
|
||||
</profiles>
|
||||
|
||||
<!-- Users and ACL. -->
|
||||
<users>
|
||||
<!-- If user name was not specified, 'default' user is used. -->
|
||||
<default>
|
||||
<!-- See also the files in users.d directory where the password can be overridden.
|
||||
|
||||
Password could be specified in plaintext or in SHA256 (in hex format).
|
||||
|
||||
If you want to specify password in plaintext (not recommended), place it in 'password' element.
|
||||
Example: <password>qwerty</password>.
|
||||
Password could be empty.
|
||||
|
||||
If you want to specify SHA256, place it in 'password_sha256_hex' element.
|
||||
Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
|
||||
Restrictions of SHA256: impossibility to connect to ClickHouse using MySQL JS client (as of July 2019).
|
||||
|
||||
If you want to specify double SHA1, place it in 'password_double_sha1_hex' element.
|
||||
Example: <password_double_sha1_hex>e395796d6546b1b65db9d665cd43f0e858dd4303</password_double_sha1_hex>
|
||||
|
||||
If you want to specify a previously defined LDAP server (see 'ldap_servers' in the main config) for authentication,
|
||||
place its name in 'server' element inside 'ldap' element.
|
||||
Example: <ldap><server>my_ldap_server</server></ldap>
|
||||
|
||||
If you want to authenticate the user via Kerberos (assuming Kerberos is enabled, see 'kerberos' in the main config),
|
||||
place 'kerberos' element instead of 'password' (and similar) elements.
|
||||
The name part of the canonical principal name of the initiator must match the user name for authentication to succeed.
|
||||
You can also place 'realm' element inside 'kerberos' element to further restrict authentication to only those requests
|
||||
whose initiator's realm matches it.
|
||||
Example: <kerberos />
|
||||
Example: <kerberos><realm>EXAMPLE.COM</realm></kerberos>
|
||||
|
||||
How to generate decent password:
|
||||
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
|
||||
In first line will be password and in second - corresponding SHA256.
|
||||
|
||||
How to generate double SHA1:
|
||||
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-'
|
||||
In first line will be password and in second - corresponding double SHA1.
|
||||
-->
|
||||
<password></password>
|
||||
|
||||
<!-- List of networks with open access.
|
||||
|
||||
To open access from everywhere, specify:
|
||||
<ip>::/0</ip>
|
||||
|
||||
To open access only from localhost, specify:
|
||||
<ip>::1</ip>
|
||||
<ip>127.0.0.1</ip>
|
||||
|
||||
Each element of list has one of the following forms:
|
||||
<ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0
|
||||
2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::.
|
||||
<host> Hostname. Example: server01.clickhouse.com.
|
||||
To check access, DNS query is performed, and all received addresses compared to peer address.
|
||||
<host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.clickhouse\.com$
|
||||
To check access, DNS PTR query is performed for peer address and then regexp is applied.
|
||||
Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address.
|
||||
Strongly recommended that regexp is ends with $
|
||||
All results of DNS requests are cached till server restart.
|
||||
-->
|
||||
<networks>
|
||||
<ip>::/0</ip>
|
||||
</networks>
|
||||
|
||||
<!-- Settings profile for user. -->
|
||||
<profile>default</profile>
|
||||
|
||||
<!-- Quota for user. -->
|
||||
<quota>default</quota>
|
||||
|
||||
<!-- User can create other users and grant rights to them. -->
|
||||
<!-- <access_management>1</access_management> -->
|
||||
</default>
|
||||
</users>
|
||||
|
||||
<!-- Quotas. -->
|
||||
<quotas>
|
||||
<!-- Name of quota. -->
|
||||
<default>
|
||||
<!-- Limits for time interval. You could specify many intervals with different limits. -->
|
||||
<interval>
|
||||
<!-- Length of interval. -->
|
||||
<duration>3600</duration>
|
||||
|
||||
<!-- No limits. Just calculate resource usage for time interval. -->
|
||||
<queries>0</queries>
|
||||
<errors>0</errors>
|
||||
<result_rows>0</result_rows>
|
||||
<read_rows>0</read_rows>
|
||||
<execution_time>0</execution_time>
|
||||
</interval>
|
||||
</default>
|
||||
</quotas>
|
||||
</clickhouse>
|
||||
@ -1,23 +0,0 @@
|
||||
(ns jepsen.clickhouse.nemesis
|
||||
(:require
|
||||
[clojure.tools.logging :refer :all]
|
||||
[jepsen
|
||||
[nemesis :as nemesis]
|
||||
[generator :as gen]]))
|
||||
|
||||
(defn random-node-hammer-time-nemesis
|
||||
[]
|
||||
(nemesis/hammer-time "clickhouse"))
|
||||
|
||||
(defn all-nodes-hammer-time-nemesis
|
||||
[]
|
||||
(nemesis/hammer-time identity "clickhouse"))
|
||||
|
||||
(defn start-stop-generator
|
||||
[time-corrupt time-ok]
|
||||
(->>
|
||||
(cycle [(gen/sleep time-ok)
|
||||
{:type :info, :f :start}
|
||||
(gen/sleep time-corrupt)
|
||||
{:type :info, :f :stop}])))
|
||||
|
||||
@ -1,56 +0,0 @@
|
||||
(ns jepsen.control.scp)
|
||||
|
||||
;; We need to overwrite Jepsen's implementation of scp! because it
|
||||
;; doesn't use strict-host-key-checking
|
||||
|
||||
(defn scp!
|
||||
"Runs an SCP command by shelling out. Takes a conn-spec (used for port, key,
|
||||
etc), a seq of sources, and a single destination, all as strings."
|
||||
[conn-spec sources dest]
|
||||
(apply util/sh "scp" "-rpC"
|
||||
"-P" (str (:port conn-spec))
|
||||
(concat (when-let [k (:private-key-path conn-spec)]
|
||||
["-i" k])
|
||||
(if-not (:strict-host-key-checking conn-spec)
|
||||
["-o StrictHostKeyChecking=no"])
|
||||
sources
|
||||
[dest]))
|
||||
nil)
|
||||
|
||||
(ns jepsen.clickhouse.keeper.db)
|
||||
|
||||
(ns jepsen.clickhouse.server.db
|
||||
(:require [clojure.tools.logging :refer :all]
|
||||
[clojure.java.io :as io]
|
||||
[jepsen
|
||||
[control :as c]
|
||||
[db :as db]]
|
||||
[jepsen.clickhouse.constants :refer :all]
|
||||
[jepsen.clickhouse.server.utils :refer :all]
|
||||
[jepsen.clickhouse.utils :as chu]))
|
||||
|
||||
(defn replicated-merge-tree-config
|
||||
[test node config-template]
|
||||
(let [nodes (:nodes test)
|
||||
replacement-map {#"\{server1\}" (get nodes 0)
|
||||
#"\{server2\}" (get nodes 1)
|
||||
#"\{server3\}" (get nodes 2)
|
||||
#"\{server_id\}" (str (inc (.indexOf nodes node)))
|
||||
#"\{replica_name\}" node}]
|
||||
(reduce #(clojure.string/replace %1 (get %2 0) (get %2 1)) config-template replacement-map)))
|
||||
|
||||
(defn install-configs
|
||||
[test node]
|
||||
(c/exec :echo (slurp (io/resource "config.xml")) :> (str configs-dir "/config.xml"))
|
||||
(c/exec :echo (slurp (io/resource "users.xml")) :> (str configs-dir "/users.xml"))
|
||||
(c/exec :echo (replicated-merge-tree-config test node (slurp (io/resource "replicated_merge_tree.xml"))) :> (str sub-configs-dir "/replicated_merge_tree.xml")))
|
||||
|
||||
(defn extra-setup
|
||||
[test node]
|
||||
(do
|
||||
(info "Installing configs")
|
||||
(install-configs test node)))
|
||||
|
||||
(defn db
|
||||
[version reuse-binary]
|
||||
(chu/db version reuse-binary start-clickhouse! extra-setup))
|
||||
@ -1,8 +0,0 @@
|
||||
(ns jepsen.clickhouse.server.nemesis
|
||||
(:require [jepsen.clickhouse.nemesis :as chnem]))
|
||||
|
||||
(def custom-nemeses
|
||||
{"random-node-hammer-time" {:nemesis (chnem/random-node-hammer-time-nemesis)
|
||||
:generator (chnem/start-stop-generator 5 5)}
|
||||
"all-nodes-hammer-time" {:nemesis (chnem/all-nodes-hammer-time-nemesis)
|
||||
:generator (chnem/start-stop-generator 1 10)}})
|
||||
@ -5,8 +5,6 @@
|
||||
|
||||
<logger>
|
||||
<level>trace</level>
|
||||
<log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
|
||||
<errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
|
||||
<size>never</size>
|
||||
</logger>
|
||||
<user_directories>
|
||||
34
tests/jepsen.clickhouse/resources/keeper_config_solo.xml
Normal file
34
tests/jepsen.clickhouse/resources/keeper_config_solo.xml
Normal file
@ -0,0 +1,34 @@
|
||||
<clickhouse>
|
||||
<listen_host>0.0.0.0</listen_host>
|
||||
|
||||
<logger>
|
||||
<level>trace</level>
|
||||
<log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
|
||||
<errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
|
||||
<size>never</size>
|
||||
</logger>
|
||||
|
||||
<keeper_server>
|
||||
<tcp_port>9181</tcp_port>
|
||||
<server_id>1</server_id>
|
||||
|
||||
<coordination_settings>
|
||||
<operation_timeout_ms>10000</operation_timeout_ms>
|
||||
<session_timeout_ms>30000</session_timeout_ms>
|
||||
<force_sync>false</force_sync>
|
||||
<startup_timeout>120000</startup_timeout>
|
||||
<raft_logs_level>trace</raft_logs_level>
|
||||
<heart_beat_interval_ms>1000</heart_beat_interval_ms>
|
||||
<election_timeout_lower_bound_ms>2000</election_timeout_lower_bound_ms>
|
||||
<election_timeout_upper_bound_ms>4000</election_timeout_upper_bound_ms>
|
||||
</coordination_settings>
|
||||
|
||||
<raft_configuration>
|
||||
<server>
|
||||
<id>1</id>
|
||||
<hostname>{srv1}</hostname>
|
||||
<port>9444</port>
|
||||
</server>
|
||||
</raft_configuration>
|
||||
</keeper_server>
|
||||
</clickhouse>
|
||||
@ -13,52 +13,9 @@
|
||||
<size>never</size>
|
||||
</logger>
|
||||
|
||||
<keeper_server>
|
||||
<tcp_port>9181</tcp_port>
|
||||
<server_id>{server_id}</server_id>
|
||||
|
||||
<coordination_settings>
|
||||
<operation_timeout_ms>10000</operation_timeout_ms>
|
||||
<session_timeout_ms>30000</session_timeout_ms>
|
||||
<force_sync>false</force_sync>
|
||||
<startup_timeout>120000</startup_timeout>
|
||||
<raft_logs_level>trace</raft_logs_level>
|
||||
<heart_beat_interval_ms>1000</heart_beat_interval_ms>
|
||||
<election_timeout_lower_bound_ms>2000</election_timeout_lower_bound_ms>
|
||||
<election_timeout_upper_bound_ms>4000</election_timeout_upper_bound_ms>
|
||||
<quorum_reads>false</quorum_reads>
|
||||
</coordination_settings>
|
||||
|
||||
<raft_configuration>
|
||||
<server>
|
||||
<id>1</id>
|
||||
<hostname>{server1}</hostname>
|
||||
<port>9444</port>
|
||||
</server>
|
||||
<server>
|
||||
<id>2</id>
|
||||
<hostname>{server2}</hostname>
|
||||
<port>9444</port>
|
||||
</server>
|
||||
<server>
|
||||
<id>3</id>
|
||||
<hostname>{server3}</hostname>
|
||||
<port>9444</port>
|
||||
</server>
|
||||
</raft_configuration>
|
||||
</keeper_server>
|
||||
|
||||
<zookeeper>
|
||||
<node>
|
||||
<host>{server1}</host>
|
||||
<port>9181</port>
|
||||
</node>
|
||||
<node>
|
||||
<host>{server2}</host>
|
||||
<port>9181</port>
|
||||
</node>
|
||||
<node>
|
||||
<host>{server3}</host>
|
||||
<host>{keeper}</host>
|
||||
<port>9181</port>
|
||||
</node>
|
||||
</zookeeper>
|
||||
16
tests/jepsen.clickhouse/resources/users.xml
Normal file
16
tests/jepsen.clickhouse/resources/users.xml
Normal file
@ -0,0 +1,16 @@
|
||||
<clickhouse>
|
||||
<profiles>
|
||||
<default>
|
||||
</default>
|
||||
</profiles>
|
||||
|
||||
<users>
|
||||
<default>
|
||||
<password></password>
|
||||
<networks>
|
||||
<ip>::/0</ip>
|
||||
</networks>
|
||||
<profile>default</profile>
|
||||
</default>
|
||||
</users>
|
||||
</clickhouse>
|
||||
@ -28,10 +28,14 @@
|
||||
|
||||
(invoke! [this test op]
|
||||
(case (:f op)
|
||||
:read (chu/exec-with-retries 30 (fn []
|
||||
(assoc op
|
||||
:type :ok
|
||||
:value (count (zk-list conn root-path)))))
|
||||
:read (try
|
||||
(assoc op
|
||||
:type :ok
|
||||
:value (count (zk-list conn root-path)))
|
||||
(catch Exception _ (assoc op :type :info, :error :connect-error)))
|
||||
:final-read (chu/exec-with-retries 30 (fn [] (assoc op
|
||||
:type :ok
|
||||
:value (count (zk-list conn root-path)))))
|
||||
:add (try
|
||||
(do
|
||||
(zk-multi-create-many-seq-nodes conn (concat-path root-path "seq-") (:value op))
|
||||
@ -50,7 +54,5 @@
|
||||
:checker (checker/compose
|
||||
{:counter (checker/counter)
|
||||
:perf (checker/perf)})
|
||||
:generator (->> (range)
|
||||
(map (fn [x]
|
||||
(->> (gen/mix [r add])))))
|
||||
:final-generator (gen/once {:type :invoke, :f :read, :value nil})})
|
||||
:generator (gen/mix [r add])
|
||||
:final-generator (gen/once {:type :invoke, :f :final-read, :value nil})})
|
||||
@ -10,20 +10,6 @@
|
||||
[jepsen.clickhouse.utils :as chu]
|
||||
[jepsen.clickhouse.keeper.utils :refer :all]))
|
||||
|
||||
(defn random-node-killer-nemesis
|
||||
[]
|
||||
(nemesis/node-start-stopper
|
||||
rand-nth
|
||||
(fn start [test node] (chu/kill-clickhouse! node test))
|
||||
(fn stop [test node] (start-clickhouse! node test))))
|
||||
|
||||
(defn all-nodes-killer-nemesis
|
||||
[]
|
||||
(nemesis/node-start-stopper
|
||||
identity
|
||||
(fn start [test node] (chu/kill-clickhouse! node test))
|
||||
(fn stop [test node] (start-clickhouse! node test))))
|
||||
|
||||
(defn select-last-file
|
||||
[path]
|
||||
(last (clojure.string/split
|
||||
@ -87,28 +73,6 @@
|
||||
(corruptor-nemesis coordination-data-dir (fn [path]
|
||||
(c/exec :rm :-fr path))))
|
||||
|
||||
(defn partition-bridge-nemesis
|
||||
[]
|
||||
(nemesis/partitioner nemesis/bridge))
|
||||
|
||||
(defn blind-node
|
||||
[nodes]
|
||||
(let [[[victim] others] (nemesis/split-one nodes)]
|
||||
{victim (into #{} others)}))
|
||||
|
||||
(defn blind-node-partition-nemesis
|
||||
[]
|
||||
(nemesis/partitioner blind-node))
|
||||
|
||||
(defn blind-others
|
||||
[nodes]
|
||||
(let [[[victim] others] (nemesis/split-one nodes)]
|
||||
(into {} (map (fn [node] [node #{victim}])) others)))
|
||||
|
||||
(defn blind-others-partition-nemesis
|
||||
[]
|
||||
(nemesis/partitioner blind-others))
|
||||
|
||||
(defn network-non-symmetric-nemesis
|
||||
[]
|
||||
(nemesis/partitioner nemesis/bridge))
|
||||
@ -120,9 +84,9 @@
|
||||
{:type :info, :f :corrupt}])))
|
||||
|
||||
(def custom-nemesises
|
||||
{"random-node-killer" {:nemesis (random-node-killer-nemesis)
|
||||
{"random-node-killer" {:nemesis (chnem/random-node-killer-nemesis start-clickhouse!)
|
||||
:generator (chnem/start-stop-generator 5 5)}
|
||||
"all-nodes-killer" {:nemesis (all-nodes-killer-nemesis)
|
||||
"all-nodes-killer" {:nemesis (chnem/all-nodes-killer-nemesis start-clickhouse!)
|
||||
:generator (chnem/start-stop-generator 1 10)}
|
||||
"simple-partitioner" {:nemesis (nemesis/partition-random-halves)
|
||||
:generator (chnem/start-stop-generator 5 5)}
|
||||
@ -138,9 +102,9 @@
|
||||
:generator (corruption-generator)}
|
||||
"drop-data-corruptor" {:nemesis (drop-all-corruption-nemesis)
|
||||
:generator (corruption-generator)}
|
||||
"bridge-partitioner" {:nemesis (partition-bridge-nemesis)
|
||||
"bridge-partitioner" {:nemesis (chnem/partition-bridge-nemesis)
|
||||
:generator (chnem/start-stop-generator 5 5)}
|
||||
"blind-node-partitioner" {:nemesis (blind-node-partition-nemesis)
|
||||
"blind-node-partitioner" {:nemesis (chnem/blind-node-partition-nemesis)
|
||||
:generator (chnem/start-stop-generator 5 5)}
|
||||
"blind-others-partitioner" {:nemesis (blind-others-partition-nemesis)
|
||||
"blind-others-partitioner" {:nemesis (chnem/blind-others-partition-nemesis)
|
||||
:generator (chnem/start-stop-generator 5 5)}})
|
||||
@ -21,7 +21,8 @@
|
||||
(assoc this :conn (zk-connect node 9181 30000)))
|
||||
|
||||
(setup! [this test]
|
||||
(zk-create-range conn 300)) ; 300 nodes to be sure
|
||||
(chu/exec-with-retries 30 (fn []
|
||||
(zk-create-range conn 300))))
|
||||
|
||||
(invoke! [_ test op]
|
||||
(let [[k v] (:value op)
|
||||
@ -25,7 +25,7 @@
|
||||
|
||||
(defn zk-connect
|
||||
[host port timeout]
|
||||
(chu/exec-with-retries 30 (fn [] (zk/connect (str host ":" port) :timeout-msec timeout))))
|
||||
(zk/connect (str host ":" port) :timeout-msec timeout))
|
||||
|
||||
(defn zk-create-range
|
||||
[conn n]
|
||||
@ -129,7 +129,7 @@
|
||||
|
||||
(defn clickhouse-alive?
|
||||
[node test]
|
||||
(info "Checking server alive on" node)
|
||||
(info "Checking Keeper alive on" node)
|
||||
(try
|
||||
(zk-connect (name node) 9181 30000)
|
||||
(catch Exception _ false)))
|
||||
59
tests/jepsen.clickhouse/src/jepsen/clickhouse/nemesis.clj
Normal file
59
tests/jepsen.clickhouse/src/jepsen/clickhouse/nemesis.clj
Normal file
@ -0,0 +1,59 @@
|
||||
(ns jepsen.clickhouse.nemesis
|
||||
(:require
|
||||
[clojure.tools.logging :refer :all]
|
||||
[jepsen.clickhouse.utils :as chu]
|
||||
[jepsen
|
||||
[nemesis :as nemesis]
|
||||
[generator :as gen]]))
|
||||
|
||||
(defn random-node-hammer-time-nemesis
|
||||
[]
|
||||
(nemesis/hammer-time "clickhouse"))
|
||||
|
||||
(defn all-nodes-hammer-time-nemesis
|
||||
[]
|
||||
(nemesis/hammer-time identity "clickhouse"))
|
||||
|
||||
(defn start-stop-generator
|
||||
[time-corrupt time-ok]
|
||||
(->>
|
||||
(cycle [(gen/sleep time-ok)
|
||||
{:type :info, :f :start}
|
||||
(gen/sleep time-corrupt)
|
||||
{:type :info, :f :stop}])))
|
||||
|
||||
(defn random-node-killer-nemesis
|
||||
[start-clickhouse!]
|
||||
(nemesis/node-start-stopper
|
||||
rand-nth
|
||||
(fn start [test node] (chu/kill-clickhouse! node test))
|
||||
(fn stop [test node] (start-clickhouse! node test))))
|
||||
|
||||
(defn all-nodes-killer-nemesis
|
||||
[start-clickhouse!]
|
||||
(nemesis/node-start-stopper
|
||||
identity
|
||||
(fn start [test node] (chu/kill-clickhouse! node test))
|
||||
(fn stop [test node] (start-clickhouse! node test))))
|
||||
|
||||
(defn partition-bridge-nemesis
|
||||
[]
|
||||
(nemesis/partitioner nemesis/bridge))
|
||||
|
||||
(defn blind-node
|
||||
[nodes]
|
||||
(let [[[victim] others] (nemesis/split-one nodes)]
|
||||
{victim (into #{} others)}))
|
||||
|
||||
(defn blind-node-partition-nemesis
|
||||
[]
|
||||
(nemesis/partitioner blind-node))
|
||||
|
||||
(defn blind-others
|
||||
[nodes]
|
||||
(let [[[victim] others] (nemesis/split-one nodes)]
|
||||
(into {} (map (fn [node] [node #{victim}])) others)))
|
||||
|
||||
(defn blind-others-partition-nemesis
|
||||
[]
|
||||
(nemesis/partitioner blind-others))
|
||||
185
tests/jepsen.clickhouse/src/jepsen/clickhouse/server/db.clj
Normal file
185
tests/jepsen.clickhouse/src/jepsen/clickhouse/server/db.clj
Normal file
@ -0,0 +1,185 @@
|
||||
(ns jepsen.control.scp)
|
||||
|
||||
;; We need to overwrite Jepsen's implementation of scp! because it
|
||||
;; doesn't use strict-host-key-checking
|
||||
|
||||
(defn scp!
|
||||
"Runs an SCP command by shelling out. Takes a conn-spec (used for port, key,
|
||||
etc), a seq of sources, and a single destination, all as strings."
|
||||
[conn-spec sources dest]
|
||||
(apply util/sh "scp" "-rpC"
|
||||
"-P" (str (:port conn-spec))
|
||||
(concat (when-let [k (:private-key-path conn-spec)]
|
||||
["-i" k])
|
||||
(if-not (:strict-host-key-checking conn-spec)
|
||||
["-o StrictHostKeyChecking=no"])
|
||||
sources
|
||||
[dest]))
|
||||
nil)
|
||||
|
||||
(ns jepsen.clickhouse.keeper.db)
|
||||
|
||||
(ns jepsen.clickhouse.server.db
|
||||
(:require [clojure.tools.logging :refer :all]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.string :as str]
|
||||
[zookeeper :as zk]
|
||||
[jepsen
|
||||
[control :as c]
|
||||
[store :as store]
|
||||
[core :as core]
|
||||
[os :as os]
|
||||
[db :as db]]
|
||||
[jepsen.control.util :as cu]
|
||||
[jepsen.clickhouse.constants :refer :all]
|
||||
[jepsen.clickhouse.server.utils :refer :all]
|
||||
[jepsen.clickhouse.keeper.utils :as keeperutils]
|
||||
[jepsen.clickhouse.utils :as chu]))
|
||||
|
||||
(defn replicated-merge-tree-config
|
||||
[test node config-template]
|
||||
(let [nodes (:nodes test)
|
||||
replacement-map {#"\{server1\}" (get nodes 0)
|
||||
#"\{server2\}" (get nodes 1)
|
||||
#"\{server3\}" (get nodes 2)
|
||||
#"\{keeper\}" (:keeper test)
|
||||
#"\{replica_name\}" node}]
|
||||
(reduce #(clojure.string/replace %1 (get %2 0) (get %2 1)) config-template replacement-map)))
|
||||
|
||||
(defn install-configs
|
||||
[test node]
|
||||
(c/exec :echo (slurp (io/resource "config.xml")) :> (str configs-dir "/config.xml"))
|
||||
(c/exec :echo (slurp (io/resource "users.xml")) :> (str configs-dir "/users.xml"))
|
||||
(c/exec :echo (replicated-merge-tree-config test node (slurp (io/resource "replicated_merge_tree.xml"))) :> (str sub-configs-dir "/replicated_merge_tree.xml")))
|
||||
|
||||
(defn extra-setup
|
||||
[test node]
|
||||
(do
|
||||
(info "Installing configs")
|
||||
(install-configs test node)))
|
||||
|
||||
(defn keeper-config
|
||||
[test node config-template]
|
||||
(let [replacement-map {#"\{srv1\}" node}]
|
||||
(reduce #(clojure.string/replace %1 (get %2 0) (get %2 1)) config-template replacement-map)))
|
||||
|
||||
(defn install-keeper-configs
|
||||
[test node]
|
||||
(c/exec :echo (keeper-config test node (slurp (io/resource "keeper_config_solo.xml"))) :> (str configs-dir "/keeper_config.xml")))
|
||||
|
||||
(defn keeper
|
||||
[version reuse-binary]
|
||||
(chu/db version reuse-binary keeperutils/start-clickhouse! install-keeper-configs))
|
||||
|
||||
(defn snarf-keeper-logs!
|
||||
"Downloads Keeper logs"
|
||||
[test]
|
||||
; Download logs
|
||||
(let [keeper-node (:keeper test)]
|
||||
(info "Snarfing Keeper log files")
|
||||
(c/on keeper-node
|
||||
(doseq [[remote local] (db/log-files-map (:db test) test keeper-node)]
|
||||
(when (cu/exists? remote)
|
||||
(info "downloading" remote "to" local)
|
||||
(try
|
||||
(c/download
|
||||
remote
|
||||
(.getCanonicalPath
|
||||
(store/path! test (name keeper-node)
|
||||
; strip leading /
|
||||
(str/replace local #"^/" ""))))
|
||||
(catch java.io.IOException e
|
||||
(if (= "Pipe closed" (.getMessage e))
|
||||
(info remote "pipe closed")
|
||||
(throw e)))
|
||||
(catch java.lang.IllegalArgumentException e
|
||||
; This is a jsch bug where the file is just being
|
||||
; created
|
||||
(info remote "doesn't exist"))))))))
|
||||
|
||||
(defn is-primary
|
||||
"Is node primary"
|
||||
[test node]
|
||||
(= 0 (.indexOf (:nodes test) node)))
|
||||
|
||||
(defn zk-connect
|
||||
[host port timeout]
|
||||
(let [conn (zk/connect (str host ":" port) :timeout-msec timeout)
|
||||
sessionId (.getSessionId conn)]
|
||||
(when (= -1 sessionId)
|
||||
(throw (RuntimeException.
|
||||
(str "Connection to " host " failed"))))
|
||||
conn))
|
||||
|
||||
(defn keeper-alive?
|
||||
[node test]
|
||||
(info "Checking Keeper alive on" node)
|
||||
(try
|
||||
(zk-connect (name node) 9181 30000)
|
||||
(catch Exception _ false)))
|
||||
|
||||
(defn db
|
||||
[version reuse-binary]
|
||||
(reify db/DB
|
||||
(setup! [this test node]
|
||||
(let [keeper-node (:keeper test)]
|
||||
(when (is-primary test node)
|
||||
(info (str "Starting Keeper on " keeper-node))
|
||||
(c/on keeper-node
|
||||
(os/setup! (:os test) test keeper-node)
|
||||
(db/setup! (keeper version reuse-binary) test keeper-node)))
|
||||
(c/su
|
||||
(do
|
||||
(info "Preparing directories")
|
||||
(chu/prepare-dirs)
|
||||
(if (or (not (cu/exists? binary-path)) (not reuse-binary))
|
||||
(do (info "Downloading clickhouse")
|
||||
(let [clickhouse-path (chu/download-clickhouse version)]
|
||||
(chu/install-downloaded-clickhouse clickhouse-path)))
|
||||
(info "Binary already exsist on path" binary-path "skipping download"))
|
||||
(extra-setup test node)
|
||||
(info "Waiting for Keeper")
|
||||
(chu/wait-clickhouse-alive! keeper-node test keeper-alive?)
|
||||
(info "Starting server")
|
||||
(start-clickhouse! node test)
|
||||
(info "ClickHouse started")))))
|
||||
|
||||
(teardown! [_ test node]
|
||||
(let [keeper-node (:keeper test)]
|
||||
(when (is-primary test node)
|
||||
(info (str "Tearing down Keeper on " keeper-node))
|
||||
(c/on keeper-node
|
||||
(db/teardown! (keeper version reuse-binary) test keeper-node))
|
||||
(os/teardown! (:os test) test keeper-node)))
|
||||
(info node "Tearing down clickhouse")
|
||||
(c/su
|
||||
(chu/kill-clickhouse! node test)
|
||||
(if (not reuse-binary)
|
||||
(c/exec :rm :-rf binary-path))
|
||||
(c/exec :rm :-rf pid-file-path)
|
||||
(c/exec :rm :-rf data-dir)
|
||||
(c/exec :rm :-rf logs-dir)
|
||||
(c/exec :rm :-rf configs-dir)))
|
||||
|
||||
db/LogFiles
|
||||
(log-files [_ test node]
|
||||
(when (is-primary test node)
|
||||
(info "Downloading Keeper logs")
|
||||
(snarf-keeper-logs! test))
|
||||
(c/su
|
||||
(chu/kill-clickhouse! node test)
|
||||
(if (cu/exists? data-dir)
|
||||
(do
|
||||
(info node "Data folder exists, going to compress")
|
||||
(c/cd root-folder
|
||||
(c/exec :tar :czf "data.tar.gz" "db"))))
|
||||
(if (cu/exists? (str logs-dir))
|
||||
(do
|
||||
(info node "Logs exist, going to compress")
|
||||
(c/cd root-folder
|
||||
(c/exec :tar :czf "logs.tar.gz" "logs"))) (info node "Logs are missing")))
|
||||
(let [common-logs [(str root-folder "/logs.tar.gz") (str root-folder "/data.tar.gz")]
|
||||
gdb-log (str logs-dir "/gdb.log")]
|
||||
(if (cu/exists? (str logs-dir "/gdb.log"))
|
||||
(conj common-logs gdb-log)
|
||||
common-logs)))))
|
||||
@ -17,7 +17,9 @@
|
||||
[jepsen.control.util :as cu]
|
||||
[jepsen.os.ubuntu :as ubuntu]
|
||||
[jepsen.checker.timeline :as timeline]
|
||||
[clojure.java.io :as io]))
|
||||
[clojure.java.io :as io])
|
||||
(:import (ch.qos.logback.classic Level)
|
||||
(org.slf4j Logger LoggerFactory)))
|
||||
|
||||
(def workloads
|
||||
"A map of workload names to functions that construct workloads, given opts."
|
||||
@ -28,6 +30,9 @@
|
||||
[["-w" "--workload NAME" "What workload should we run?"
|
||||
:default "set"
|
||||
:validate [workloads (cli/one-of workloads)]]
|
||||
[nil "--keeper ADDRESS", "Address of a Keeper instance"
|
||||
:default ""
|
||||
:validate [#(not-empty %) "Address for Keeper cannot be empty"]]
|
||||
[nil "--nemesis NAME" "Which nemesis will poison our lives?"
|
||||
:default "random-node-killer"
|
||||
:validate [ch-nemesis/custom-nemeses (cli/one-of ch-nemesis/custom-nemeses)]]
|
||||
@ -99,6 +104,8 @@
|
||||
"Handles command line arguments. Can either run a test, or a web server for
|
||||
browsing results."
|
||||
[& args]
|
||||
(.setLevel
|
||||
(LoggerFactory/getLogger "org.apache.zookeeper") Level/OFF)
|
||||
(cli/run! (merge (cli/single-test-cmd {:test-fn clickhouse-server-test
|
||||
:opt-spec cli-opts})
|
||||
(cli/test-all-cmd {:tests-fn (partial all-tests clickhouse-server-test)
|
||||
@ -0,0 +1,22 @@
|
||||
(ns jepsen.clickhouse.server.nemesis
|
||||
(:require [jepsen.clickhouse.nemesis :as chnem]
|
||||
[jepsen.clickhouse.server.utils :refer :all]
|
||||
[jepsen.nemesis :as nemesis]))
|
||||
|
||||
(def custom-nemeses
|
||||
{"random-node-killer" {:nemesis (chnem/random-node-killer-nemesis start-clickhouse!)
|
||||
:generator (chnem/start-stop-generator 5 5)}
|
||||
"all-nodes-killer" {:nemesis (chnem/all-nodes-killer-nemesis start-clickhouse!)
|
||||
:generator (chnem/start-stop-generator 1 10)}
|
||||
"simple-partitioner" {:nemesis (nemesis/partition-random-halves)
|
||||
:generator (chnem/start-stop-generator 5 5)}
|
||||
"random-node-hammer-time" {:nemesis (chnem/random-node-hammer-time-nemesis)
|
||||
:generator (chnem/start-stop-generator 5 5)}
|
||||
"all-nodes-hammer-time" {:nemesis (chnem/all-nodes-hammer-time-nemesis)
|
||||
:generator (chnem/start-stop-generator 1 10)}
|
||||
"bridge-partitioner" {:nemesis (chnem/partition-bridge-nemesis)
|
||||
:generator (chnem/start-stop-generator 5 5)}
|
||||
"blind-node-partitioner" {:nemesis (chnem/blind-node-partition-nemesis)
|
||||
:generator (chnem/start-stop-generator 5 5)}
|
||||
"blind-others-partitioner" {:nemesis (chnem/blind-others-partition-nemesis)
|
||||
:generator (chnem/start-stop-generator 5 5)}})
|
||||
@ -34,10 +34,7 @@
|
||||
(mapv :value)
|
||||
(assoc op :type :ok, :value))))))
|
||||
|
||||
(teardown! [_ test]
|
||||
(util/timeout chc/operation-timeout
|
||||
(chc/with-connection [c conn] false
|
||||
(j/query c ["DROP TABLE set ON CLUSTER test_cluster"]))))
|
||||
(teardown! [_ test])
|
||||
|
||||
(close! [_ test]
|
||||
(rc/close! conn)))
|
||||
@ -42,7 +42,7 @@
|
||||
(let [encoded-url (md5 url)
|
||||
expected-file-name (.getName (io/file url))
|
||||
dest-folder (str binaries-cache-dir "/" encoded-url)
|
||||
dest-file (str root-folder "/clickhouse")
|
||||
dest-file (str dest-folder "/clickhouse")
|
||||
dest-symlink (str root-folder "/" expected-file-name)
|
||||
wget-opts (concat cu/std-wget-opts [:-O dest-file])]
|
||||
(when-not (cu/exists? dest-file)
|
||||
@ -56,7 +56,7 @@
|
||||
|
||||
(defn get-clickhouse-url
|
||||
[url]
|
||||
(non-precise-cached-wget! url root-folder binaries-cache-dir))
|
||||
(non-precise-cached-wget! url))
|
||||
|
||||
(defn get-clickhouse-scp
|
||||
[path]
|
||||
@ -66,7 +66,7 @@
|
||||
[source]
|
||||
(info "Downloading clickhouse from" source)
|
||||
(cond
|
||||
(clojure.string/starts-with? source "http") (get-clickhouse-url source root-folder binaries-cache-dir)
|
||||
(clojure.string/starts-with? source "http") (get-clickhouse-url source)
|
||||
(.exists (io/file source)) (get-clickhouse-scp source root-folder)
|
||||
:else (throw (Exception. (str "Don't know how to download clickhouse from" source)))))
|
||||
|
||||
@ -156,7 +156,7 @@
|
||||
(prepare-dirs)
|
||||
(if (or (not (cu/exists? binary-path)) (not reuse-binary))
|
||||
(do (info "Downloading clickhouse")
|
||||
(let [clickhouse-path (download-clickhouse version root-folder binaries-cache-dir)]
|
||||
(let [clickhouse-path (download-clickhouse version)]
|
||||
(install-downloaded-clickhouse clickhouse-path)))
|
||||
(info "Binary already exsist on path" binary-path "skipping download"))
|
||||
(extra-setup test node)
|
||||
Loading…
Reference in New Issue
Block a user