start adding nemesis

tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse/keeper/nemesis.clj

@@ -5,6 +5,7 @@
     [nemesis :as nemesis]
     [control :as c]
     [generator :as gen]]
+   [jepsen.clickhouse.nemesis :as chnem]
    [jepsen.clickhouse.constants :refer :all]
    [jepsen.clickhouse.utils :as chu]
    [jepsen.clickhouse.keeper.utils :refer :all]))
@@ -23,14 +24,6 @@
    (fn start [test node] (chu/kill-clickhouse! node test))
    (fn stop [test node] (start-clickhouse! node test))))
 
-(defn random-node-hammer-time-nemesis
-  []
-  (nemesis/hammer-time "clickhouse"))
-
-(defn all-nodes-hammer-time-nemesis
-  []
-  (nemesis/hammer-time identity "clickhouse"))
-
 (defn select-last-file
   [path]
   (last (clojure.string/split
@@ -141,9 +134,9 @@
                        :generator (start-stop-generator 1 10)}
    "simple-partitioner" {:nemesis (nemesis/partition-random-halves)
                          :generator (start-stop-generator 5 5)}
-   "random-node-hammer-time"    {:nemesis (random-node-hammer-time-nemesis)
+   "random-node-hammer-time"    {:nemesis (chnem/random-node-hammer-time-nemesis)
                                  :generator (start-stop-generator 5 5)}
-   "all-nodes-hammer-time"    {:nemesis (all-nodes-hammer-time-nemesis)
+   "all-nodes-hammer-time"    {:nemesis (chnem/all-nodes-hammer-time-nemesis)
                                :generator (start-stop-generator 1 10)}
    "logs-corruptor" {:nemesis (logs-corruption-nemesis)
                      :generator (corruption-generator)}

tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse/nemesis.clj

@@ -0,0 +1,23 @@
+(ns jepsen.clickhouse.nemesis
+  (:require
+   [clojure.tools.logging :refer :all]
+   [jepsen
+    [nemesis :as nemesis]
+    [generator :as gen]]))
+
+(defn random-node-hammer-time-nemesis
+  []
+  (nemesis/hammer-time "clickhouse"))
+
+(defn all-nodes-hammer-time-nemesis
+  []
+  (nemesis/hammer-time identity "clickhouse"))
+
+(defn start-stop-generator
+  [time-corrupt time-ok]
+  (->>
+   (cycle [(gen/sleep time-ok)
+           {:type :info, :f :start}
+           (gen/sleep time-corrupt)
+           {:type :info, :f :stop}])))
+

tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse/server/client.clj

@@ -1,6 +1,7 @@
 (ns jepsen.clickhouse.server.client
   (:require [clojure.java.jdbc :as j]
             [clojure.tools.logging :refer :all]
+            [jepsen.util :as util]
             [jepsen.reconnect :as rc]))
 
 (def operation-timeout "Default operation timeout in ms" 10000)
@@ -16,11 +17,15 @@
 
 (defn open-connection
   [node]
-  (let [spec (db-spec node)
+   (util/timeout 30000
-        connection (j/get-connection spec)
+               (throw (RuntimeException.
-        added-connection (j/add-connection spec connection)]
+                        (str "Connection to " node " timed out")))
-    (assert added-connection)
+    (util/retry 0.1
-    added-connection))
+      (let [spec (db-spec node)
+            connection (j/get-connection spec)
+            added-connection (j/add-connection spec connection)]
+        (assert added-connection)
+        added-connection))))
 
 (defn close-connection
   "Close connection"
@@ -35,7 +40,7 @@
   (rc/open!
     (rc/wrapper
       {:name (name node)
-       :open #(open-connection node)
+       :open (partial open-connection node)
        :close close-connection
        :log? true})))
 

tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse/server/main.clj

@@ -12,6 +12,7 @@
             [jepsen.clickhouse.server
              [register :as register]
              [set :as set]]
+            [jepsen.clickhouse.server.nemesis :as custom-nemesis]
             [jepsen.control.util :as cu]
             [jepsen.os.ubuntu :as ubuntu]
             [jepsen.checker.timeline :as timeline]
@@ -48,13 +49,15 @@
   [opts]
   (info "Test opts\n" (with-out-str (pprint opts)))
   (let [quorum (boolean (:quorum opts))
-        workload  ((get workloads (:workload opts)) opts)]
+        workload  ((get workloads (:workload opts)) opts)
+        current-nemesis (get custom-nemesis/custom-nemesises "random-node-hammer-time")]
     (merge tests/noop-test
            opts
            {:name (str "clickhouse-server-"  (name (:workload opts)))
             :os ubuntu/os
             :db (get-db opts)
             :pure-generators true
+            :nemesis (:nemesis current-nemesis)
             :client (:client workload)
             :checker (checker/compose
                       {:perf     (checker/perf)
@@ -62,7 +65,12 @@
             :generator (gen/phases
                         (->> (:generator workload)
                              (gen/stagger (/ (:rate opts)))
+                             (gen/nemesis (:generator current-nemesis))
                              (gen/time-limit (:time-limit opts)))
+                        (gen/log "Healing cluster")
+                        (gen/nemesis (gen/once {:type :info, :f :stop}))
+                        (gen/log "Waiting for recovery")
+                        (gen/sleep 10)
                         (gen/clients (:final-generator workload)))})))
 
 (defn clickhouse-server-test

tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse/server/nemesis.clj

@@ -0,0 +1,8 @@
+(ns jepsen.clickhouse.server.nemesis
+  (:require [jepsen.clickhouse.nemesis :as chnem]))
+
+(def custom-nemesises
+  {"random-node-hammer-time"  {:nemesis (chnem/random-node-hammer-time-nemesis)
+                                 :generator (chnem/start-stop-generator 5 5)}
+   "all-nodes-hammer-time"    {:nemesis (chnem/all-nodes-hammer-time-nemesis)
+                               :generator (chnem/start-stop-generator 1 10)}})