thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-02 04:22:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

start adding nemesis

Browse Source
This commit is contained in:
Antonio Andelic 2022-10-28 07:34:23 +00:00
parent cc65def352
commit 77a7411bd0
5 changed files with 54 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse/keeper/nemesis.clj
View File

@ -5,6 +5,7 @@
[nemesis :as nemesis]
[control :as c]
[generator :as gen]]
[jepsen.clickhouse.nemesis :as chnem]
[jepsen.clickhouse.constants :refer :all]
[jepsen.clickhouse.utils :as chu]
[jepsen.clickhouse.keeper.utils :refer :all]))
@ -23,14 +24,6 @@
(fn start [test node] (chu/kill-clickhouse! node test))
(fn stop [test node] (start-clickhouse! node test))))
(defn random-node-hammer-time-nemesis
[]
(nemesis/hammer-time "clickhouse"))
(defn all-nodes-hammer-time-nemesis
[]
(nemesis/hammer-time identity "clickhouse"))
(defn select-last-file
[path]
(last (clojure.string/split
@ -141,9 +134,9 @@
:generator (start-stop-generator 1 10)}
"simple-partitioner" {:nemesis (nemesis/partition-random-halves)
:generator (start-stop-generator 5 5)}
"random-node-hammer-time" {:nemesis (random-node-hammer-time-nemesis)
"random-node-hammer-time" {:nemesis (chnem/random-node-hammer-time-nemesis)
:generator (start-stop-generator 5 5)}
"all-nodes-hammer-time" {:nemesis (all-nodes-hammer-time-nemesis)
"all-nodes-hammer-time" {:nemesis (chnem/all-nodes-hammer-time-nemesis)
:generator (start-stop-generator 1 10)}
"logs-corruptor" {:nemesis (logs-corruption-nemesis)
:generator (corruption-generator)}

23
tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse/nemesis.clj Normal file
View File

@ -0,0 +1,23 @@
(ns jepsen.clickhouse.nemesis
(:require
[clojure.tools.logging :refer :all]
[jepsen
[nemesis :as nemesis]
[generator :as gen]]))
(defn random-node-hammer-time-nemesis
[]
(nemesis/hammer-time "clickhouse"))
(defn all-nodes-hammer-time-nemesis
[]
(nemesis/hammer-time identity "clickhouse"))
(defn start-stop-generator
[time-corrupt time-ok]
(->>
(cycle [(gen/sleep time-ok)
{:type :info, :f :start}
(gen/sleep time-corrupt)
{:type :info, :f :stop}])))

17
tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse/server/client.clj
View File

@ -1,6 +1,7 @@
(ns jepsen.clickhouse.server.client
(:require [clojure.java.jdbc :as j]
[clojure.tools.logging :refer :all]
[jepsen.util :as util]
[jepsen.reconnect :as rc]))
(def operation-timeout "Default operation timeout in ms" 10000)
@ -16,11 +17,15 @@
(defn open-connection
[node]
(let [spec (db-spec node)
connection (j/get-connection spec)
added-connection (j/add-connection spec connection)]
(assert added-connection)
added-connection))
(util/timeout 30000
(throw (RuntimeException.
(str "Connection to " node " timed out")))
(util/retry 0.1
(let [spec (db-spec node)
connection (j/get-connection spec)
added-connection (j/add-connection spec connection)]
(assert added-connection)
added-connection))))
(defn close-connection
"Close connection"
@ -35,7 +40,7 @@
(rc/open!
(rc/wrapper
{:name (name node)
:open #(open-connection node)
:open (partial open-connection node)
:close close-connection
:log? true})))

10
tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse/server/main.clj
View File

@ -12,6 +12,7 @@
[jepsen.clickhouse.server
[register :as register]
[set :as set]]
[jepsen.clickhouse.server.nemesis :as custom-nemesis]
[jepsen.control.util :as cu]
[jepsen.os.ubuntu :as ubuntu]
[jepsen.checker.timeline :as timeline]
@ -48,13 +49,15 @@
[opts]
(info "Test opts\n" (with-out-str (pprint opts)))
(let [quorum (boolean (:quorum opts))
workload ((get workloads (:workload opts)) opts)]
workload ((get workloads (:workload opts)) opts)
current-nemesis (get custom-nemesis/custom-nemesises "random-node-hammer-time")]
(merge tests/noop-test
opts
{:name (str "clickhouse-server-" (name (:workload opts)))
:os ubuntu/os
:db (get-db opts)
:pure-generators true
:nemesis (:nemesis current-nemesis)
:client (:client workload)
:checker (checker/compose
{:perf (checker/perf)
@ -62,7 +65,12 @@
:generator (gen/phases
(->> (:generator workload)
(gen/stagger (/ (:rate opts)))
(gen/nemesis (:generator current-nemesis))
(gen/time-limit (:time-limit opts)))
(gen/log "Healing cluster")
(gen/nemesis (gen/once {:type :info, :f :stop}))
(gen/log "Waiting for recovery")
(gen/sleep 10)
(gen/clients (:final-generator workload)))})))
(defn clickhouse-server-test

8
tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse/server/nemesis.clj Normal file
View File

@ -0,0 +1,8 @@
(ns jepsen.clickhouse.server.nemesis
(:require [jepsen.clickhouse.nemesis :as chnem]))
(def custom-nemesises
{"random-node-hammer-time" {:nemesis (chnem/random-node-hammer-time-nemesis)
:generator (chnem/start-stop-generator 5 5)}
"all-nodes-hammer-time" {:nemesis (chnem/all-nodes-hammer-time-nemesis)
:generator (chnem/start-stop-generator 1 10)}})