From ba9e1e5a8d23e7fbf6bff83e6493d46f1d49ef75 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 10 Mar 2021 19:12:32 +0300 Subject: [PATCH 01/43] Some initial code Add some java magic Allow to connect with old session id More angry nemesis and fixes Angry Fix style Split to files Better wrappers Better structure Add set test and split to separate files (I think something broken now) Better Missed files --- src/Coordination/CoordinationSettings.h | 1 + src/Coordination/NuKeeperServer.cpp | 4 +- src/Server/NuKeeperTCPHandler.cpp | 10 +- tests/jepsen.nukeeper/.gitignore | 13 + tests/jepsen.nukeeper/CHANGELOG.md | 24 ++ tests/jepsen.nukeeper/LICENSE | 280 ++++++++++++++++++ tests/jepsen.nukeeper/README.md | 22 ++ tests/jepsen.nukeeper/doc/intro.md | 3 + tests/jepsen.nukeeper/project.clj | 13 + tests/jepsen.nukeeper/resources/config.xml | 1 + tests/jepsen.nukeeper/resources/listen.xml | 3 + .../resources/test_keeper_config.xml | 33 +++ tests/jepsen.nukeeper/resources/users.xml | 1 + .../src/jepsen/nukeeper/main.clj | 143 +++++++++ .../src/jepsen/nukeeper/register.clj | 64 ++++ .../src/jepsen/nukeeper/set.clj | 43 +++ .../src/jepsen/nukeeper/utils.clj | 56 ++++ .../test/jepsen/nukeeper_test.clj | 28 ++ 18 files changed, 733 insertions(+), 9 deletions(-) create mode 100644 tests/jepsen.nukeeper/.gitignore create mode 100644 tests/jepsen.nukeeper/CHANGELOG.md create mode 100644 tests/jepsen.nukeeper/LICENSE create mode 100644 tests/jepsen.nukeeper/README.md create mode 100644 tests/jepsen.nukeeper/doc/intro.md create mode 100644 tests/jepsen.nukeeper/project.clj create mode 120000 tests/jepsen.nukeeper/resources/config.xml create mode 100644 tests/jepsen.nukeeper/resources/listen.xml create mode 100644 tests/jepsen.nukeeper/resources/test_keeper_config.xml create mode 120000 tests/jepsen.nukeeper/resources/users.xml create mode 100644 tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj create mode 100644 tests/jepsen.nukeeper/src/jepsen/nukeeper/register.clj create mode 100644 tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj create mode 100644 tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj create mode 100644 tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj diff --git a/src/Coordination/CoordinationSettings.h b/src/Coordination/CoordinationSettings.h index dcfb13c359e..c816f8089d5 100644 --- a/src/Coordination/CoordinationSettings.h +++ b/src/Coordination/CoordinationSettings.h @@ -31,6 +31,7 @@ struct Settings; M(UInt64, rotate_log_storage_interval, 10000, "How many records will be stored in one log storage file", 0) \ M(UInt64, snapshots_to_keep, 3, "How many compressed snapshots to keep on disk", 0) \ M(UInt64, stale_log_gap, 10000, "When node became stale and should receive snapshots from leader", 0) \ + M(Bool, quorum_reads, false, "Execute read requests as writes through whole RAFT consesus with similar speed", 0) \ M(Bool, force_sync, true, " Call fsync on each change in RAFT changelog", 0) DECLARE_SETTINGS_TRAITS(CoordinationSettingsTraits, LIST_OF_COORDINATION_SETTINGS) diff --git a/src/Coordination/NuKeeperServer.cpp b/src/Coordination/NuKeeperServer.cpp index edda26613dd..2081c969523 100644 --- a/src/Coordination/NuKeeperServer.cpp +++ b/src/Coordination/NuKeeperServer.cpp @@ -30,6 +30,8 @@ NuKeeperServer::NuKeeperServer( , state_manager(nuraft::cs_new(server_id, "test_keeper_server", config, coordination_settings)) , responses_queue(responses_queue_) { + if (coordination_settings->quorum_reads) + LOG_WARNING(&Poco::Logger::get("NuKeeperServer"), "Quorum reads enabled, NuKeeper will work slower."); } void NuKeeperServer::startup() @@ -106,7 +108,7 @@ nuraft::ptr getZooKeeperLogEntry(int64_t session_id, const Coord void NuKeeperServer::putRequest(const NuKeeperStorage::RequestForSession & request_for_session) { auto [session_id, request] = request_for_session; - if (isLeaderAlive() && request->isReadRequest()) + if (!coordination_settings->quorum_reads && isLeaderAlive() && request->isReadRequest()) { state_machine->processReadRequest(request_for_session); } diff --git a/src/Server/NuKeeperTCPHandler.cpp b/src/Server/NuKeeperTCPHandler.cpp index b283356d27d..b676331f6c0 100644 --- a/src/Server/NuKeeperTCPHandler.cpp +++ b/src/Server/NuKeeperTCPHandler.cpp @@ -240,16 +240,10 @@ Poco::Timespan NuKeeperTCPHandler::receiveHandshake() throw Exception("Unexpected protocol version: " + toString(protocol_version), ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); Coordination::read(last_zxid_seen, *in); - - if (last_zxid_seen != 0) - throw Exception("Non zero last_zxid_seen is not supported", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); - Coordination::read(timeout_ms, *in); + + /// TODO Stop ignoring this value Coordination::read(previous_session_id, *in); - - if (previous_session_id != 0) - throw Exception("Non zero previous session id is not supported", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); - Coordination::read(passwd, *in); int8_t readonly; diff --git a/tests/jepsen.nukeeper/.gitignore b/tests/jepsen.nukeeper/.gitignore new file mode 100644 index 00000000000..d956ab0a125 --- /dev/null +++ b/tests/jepsen.nukeeper/.gitignore @@ -0,0 +1,13 @@ +/target +/classes +/checkouts +profiles.clj +pom.xml +pom.xml.asc +*.jar +*.class +/.lein-* +/.nrepl-port +/.prepl-port +.hgignore +.hg/ diff --git a/tests/jepsen.nukeeper/CHANGELOG.md b/tests/jepsen.nukeeper/CHANGELOG.md new file mode 100644 index 00000000000..6c7cb4f7c8a --- /dev/null +++ b/tests/jepsen.nukeeper/CHANGELOG.md @@ -0,0 +1,24 @@ +# Change Log +All notable changes to this project will be documented in this file. This change log follows the conventions of [keepachangelog.com](http://keepachangelog.com/). + +## [Unreleased] +### Changed +- Add a new arity to `make-widget-async` to provide a different widget shape. + +## [0.1.1] - 2021-03-10 +### Changed +- Documentation on how to make the widgets. + +### Removed +- `make-widget-sync` - we're all async, all the time. + +### Fixed +- Fixed widget maker to keep working when daylight savings switches over. + +## 0.1.0 - 2021-03-10 +### Added +- Files from the new template. +- Widget maker public API - `make-widget-sync`. + +[Unreleased]: https://github.com/your-name/jepsen.nukeeper/compare/0.1.1...HEAD +[0.1.1]: https://github.com/your-name/jepsen.nukeeper/compare/0.1.0...0.1.1 diff --git a/tests/jepsen.nukeeper/LICENSE b/tests/jepsen.nukeeper/LICENSE new file mode 100644 index 00000000000..231512650b9 --- /dev/null +++ b/tests/jepsen.nukeeper/LICENSE @@ -0,0 +1,280 @@ +Eclipse Public License - v 2.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + + a) in the case of the initial Contributor, the initial content + Distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + i) changes to the Program, and + ii) additions to the Program; + where such changes and/or additions to the Program originate from + and are Distributed by that particular Contributor. A Contribution + "originates" from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include changes or additions to the Program that + are not Modified Works. + +"Contributor" means any person or entity that Distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which +are necessarily infringed by the use or sale of its Contribution alone +or when combined with the Program. + +"Program" means the Contributions Distributed in accordance with this +Agreement. + +"Recipient" means anyone who receives the Program under this Agreement +or any Secondary License (as applicable), including Contributors. + +"Derivative Works" shall mean any work, whether in Source Code or other +form, that is based on (or derived from) the Program and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. + +"Modified Works" shall mean any work in Source Code or other form that +results from an addition to, deletion from, or modification of the +contents of the Program, including, for purposes of clarity any new file +in Source Code form that contains any contents of the Program. Modified +Works shall not include works that contain only declarations, +interfaces, types, classes, structures, or files of the Program solely +in each case in order to link to, bind by name, or subclass the Program +or Modified Works thereof. + +"Distribute" means the acts of a) distributing or b) making available +in any manner that enables the transfer of a copy. + +"Source Code" means the form of a Program preferred for making +modifications, including but not limited to software source code, +documentation source, and configuration files. + +"Secondary License" means either the GNU General Public License, +Version 2.0, or any later versions of that license, including any +exceptions or additional permissions as identified by the initial +Contributor. + +2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free copyright + license to reproduce, prepare Derivative Works of, publicly display, + publicly perform, Distribute and sublicense the Contribution of such + Contributor, if any, and such Derivative Works. + + b) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free patent + license under Licensed Patents to make, use, sell, offer to sell, + import and otherwise transfer the Contribution of such Contributor, + if any, in Source Code or other form. This patent license shall + apply to the combination of the Contribution and the Program if, at + the time the Contribution is added by the Contributor, such addition + of the Contribution causes such combination to be covered by the + Licensed Patents. The patent license shall not apply to any other + combinations which include the Contribution. No hardware per se is + licensed hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the + rights and licenses granted hereunder, each Recipient hereby + assumes sole responsibility to secure any other intellectual + property rights needed, if any. For example, if a third party + patent license is required to allow Recipient to Distribute the + Program, it is Recipient's responsibility to acquire that license + before distributing the Program. + + d) Each Contributor represents that to its knowledge it has + sufficient copyright rights in its Contribution, if any, to grant + the copyright license set forth in this Agreement. + + e) Notwithstanding the terms of any Secondary License, no + Contributor makes additional grants to any Recipient (other than + those set forth in this Agreement) as a result of such Recipient's + receipt of the Program under the terms of a Secondary License + (if permitted under the terms of Section 3). + +3. REQUIREMENTS + +3.1 If a Contributor Distributes the Program in any form, then: + + a) the Program must also be made available as Source Code, in + accordance with section 3.2, and the Contributor must accompany + the Program with a statement that the Source Code for the Program + is available under this Agreement, and informs Recipients how to + obtain it in a reasonable manner on or through a medium customarily + used for software exchange; and + + b) the Contributor may Distribute the Program under a license + different than this Agreement, provided that such license: + i) effectively disclaims on behalf of all other Contributors all + warranties and conditions, express and implied, including + warranties or conditions of title and non-infringement, and + implied warranties or conditions of merchantability and fitness + for a particular purpose; + + ii) effectively excludes on behalf of all other Contributors all + liability for damages, including direct, indirect, special, + incidental and consequential damages, such as lost profits; + + iii) does not attempt to limit or alter the recipients' rights + in the Source Code under section 3.2; and + + iv) requires any subsequent distribution of the Program by any + party to be under a license that satisfies the requirements + of this section 3. + +3.2 When the Program is Distributed as Source Code: + + a) it must be made available under this Agreement, or if the + Program (i) is combined with other material in a separate file or + files made available under a Secondary License, and (ii) the initial + Contributor attached to the Source Code the notice described in + Exhibit A of this Agreement, then the Program may be made available + under the terms of such Secondary Licenses, and + + b) a copy of this Agreement must be included with each copy of + the Program. + +3.3 Contributors may not remove or alter any copyright, patent, +trademark, attribution notices, disclaimers of warranty, or limitations +of liability ("notices") contained within the Program from any copy of +the Program which they Distribute, provided that Contributors may add +their own appropriate notices. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities +with respect to end users, business partners and the like. While this +license is intended to facilitate the commercial use of the Program, +the Contributor who includes the Program in a commercial product +offering should do so in a manner which does not create potential +liability for other Contributors. Therefore, if a Contributor includes +the Program in a commercial product offering, such Contributor +("Commercial Contributor") hereby agrees to defend and indemnify every +other Contributor ("Indemnified Contributor") against any losses, +damages and costs (collectively "Losses") arising from claims, lawsuits +and other legal actions brought by a third party against the Indemnified +Contributor to the extent caused by the acts or omissions of such +Commercial Contributor in connection with its distribution of the Program +in a commercial product offering. The obligations in this section do not +apply to any claims or Losses relating to any actual or alleged +intellectual property infringement. In order to qualify, an Indemnified +Contributor must: a) promptly notify the Commercial Contributor in +writing of such claim, and b) allow the Commercial Contributor to control, +and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may +participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial +product offering, Product X. That Contributor is then a Commercial +Contributor. If that Commercial Contributor then makes performance +claims, or offers warranties related to Product X, those performance +claims and warranties are such Commercial Contributor's responsibility +alone. Under this section, the Commercial Contributor would have to +defend claims against the other Contributors related to those performance +claims and warranties, and if a court requires any other Contributor to +pay any damages as a result, the Commercial Contributor must pay +those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS" +BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF +TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR +PURPOSE. Each Recipient is solely responsible for determining the +appropriateness of using and distributing the Program and assumes all +risks associated with its exercise of rights under this Agreement, +including but not limited to the risks and costs of program errors, +compliance with applicable laws, damage to or loss of data, programs +or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS +SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE +EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under +applicable law, it shall not affect the validity or enforceability of +the remainder of the terms of this Agreement, and without further +action by the parties hereto, such provision shall be reformed to the +minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity +(including a cross-claim or counterclaim in a lawsuit) alleging that the +Program itself (excluding combinations of the Program with other software +or hardware) infringes such Recipient's patent(s), then such Recipient's +rights granted under Section 2(b) shall terminate as of the date such +litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it +fails to comply with any of the material terms or conditions of this +Agreement and does not cure such failure in a reasonable period of +time after becoming aware of such noncompliance. If all Recipient's +rights under this Agreement terminate, Recipient agrees to cease use +and distribution of the Program as soon as reasonably practicable. +However, Recipient's obligations under this Agreement and any licenses +granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, +but in order to avoid inconsistency the Agreement is copyrighted and +may only be modified in the following manner. The Agreement Steward +reserves the right to publish new versions (including revisions) of +this Agreement from time to time. No one other than the Agreement +Steward has the right to modify this Agreement. The Eclipse Foundation +is the initial Agreement Steward. The Eclipse Foundation may assign the +responsibility to serve as the Agreement Steward to a suitable separate +entity. Each new version of the Agreement will be given a distinguishing +version number. The Program (including Contributions) may always be +Distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to Distribute the Program (including its +Contributions) under the new version. + +Except as expressly stated in Sections 2(a) and 2(b) above, Recipient +receives no rights or licenses to the intellectual property of any +Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted +under this Agreement are reserved. Nothing in this Agreement is intended +to be enforceable by any entity that is not a Contributor or Recipient. +No third-party beneficiary rights are created under this Agreement. + +Exhibit A - Form of Secondary Licenses Notice + +"This Source Code may also be made available under the following +Secondary Licenses when the conditions for such availability set forth +in the Eclipse Public License, v. 2.0 are satisfied: GNU General Public +License as published by the Free Software Foundation, either version 2 +of the License, or (at your option) any later version, with the GNU +Classpath Exception which is available at +https://www.gnu.org/software/classpath/license.html." + + Simply including a copy of this Agreement, including this Exhibit A + is not sufficient to license the Source Code under Secondary Licenses. + + If it is not possible or desirable to put the notice in a particular + file, then You may include the notice in a location (such as a LICENSE + file in a relevant directory) where a recipient would be likely to + look for such a notice. + + You may add additional accurate notices of copyright ownership. diff --git a/tests/jepsen.nukeeper/README.md b/tests/jepsen.nukeeper/README.md new file mode 100644 index 00000000000..f72409e080f --- /dev/null +++ b/tests/jepsen.nukeeper/README.md @@ -0,0 +1,22 @@ +# jepsen.nukeeper + +A Clojure library designed to ... well, that part is up to you. + +## Usage + +FIXME + +## License + +Copyright © 2021 FIXME + +This program and the accompanying materials are made available under the +terms of the Eclipse Public License 2.0 which is available at +http://www.eclipse.org/legal/epl-2.0. + +This Source Code may also be made available under the following Secondary +Licenses when the conditions for such availability set forth in the Eclipse +Public License, v. 2.0 are satisfied: GNU General Public License as published by +the Free Software Foundation, either version 2 of the License, or (at your +option) any later version, with the GNU Classpath Exception which is available +at https://www.gnu.org/software/classpath/license.html. diff --git a/tests/jepsen.nukeeper/doc/intro.md b/tests/jepsen.nukeeper/doc/intro.md new file mode 100644 index 00000000000..c6e5ccbd04a --- /dev/null +++ b/tests/jepsen.nukeeper/doc/intro.md @@ -0,0 +1,3 @@ +# Introduction to jepsen.nukeeper + +TODO: write [great documentation](http://jacobian.org/writing/what-to-write/) diff --git a/tests/jepsen.nukeeper/project.clj b/tests/jepsen.nukeeper/project.clj new file mode 100644 index 00000000000..e7150c9e5d4 --- /dev/null +++ b/tests/jepsen.nukeeper/project.clj @@ -0,0 +1,13 @@ +(defproject jepsen.nukeeper "0.1.0-SNAPSHOT" + :injections [(.. System (setProperty "zookeeper.request.timeout" "10000"))] + :description "A jepsen tests for ClickHouse NuKeeper" + :url "https://clickhouse.tech/" + :license {:name "EPL-2.0" + :url "https://www.eclipse.org/legal/epl-2.0/"} + :main jepsen.nukeeper.main + :plugins [[lein-cljfmt "0.7.0"]] + :dependencies [[org.clojure/clojure "1.10.1"] + [jepsen "0.2.3"] + [zookeeper-clj "0.9.4"] + [org.apache.zookeeper/zookeeper "3.6.1" :exclusions [org.slf4j/slf4j-log4j12]]] + :repl-options {:init-ns jepsen.nukeeper.main}) diff --git a/tests/jepsen.nukeeper/resources/config.xml b/tests/jepsen.nukeeper/resources/config.xml new file mode 120000 index 00000000000..c7596baa075 --- /dev/null +++ b/tests/jepsen.nukeeper/resources/config.xml @@ -0,0 +1 @@ +../../../programs/server/config.xml \ No newline at end of file diff --git a/tests/jepsen.nukeeper/resources/listen.xml b/tests/jepsen.nukeeper/resources/listen.xml new file mode 100644 index 00000000000..de8c737ff75 --- /dev/null +++ b/tests/jepsen.nukeeper/resources/listen.xml @@ -0,0 +1,3 @@ + + :: + diff --git a/tests/jepsen.nukeeper/resources/test_keeper_config.xml b/tests/jepsen.nukeeper/resources/test_keeper_config.xml new file mode 100644 index 00000000000..0e2a688ea0b --- /dev/null +++ b/tests/jepsen.nukeeper/resources/test_keeper_config.xml @@ -0,0 +1,33 @@ + + + 9181 + {id} + + + 10000 + 30000 + false + 60000 + trace + {quorum_reads} + + + + + 1 + {srv1} + 9444 + + + 2 + {srv2} + 9444 + + + 3 + {srv3} + 9444 + + + + diff --git a/tests/jepsen.nukeeper/resources/users.xml b/tests/jepsen.nukeeper/resources/users.xml new file mode 120000 index 00000000000..41b137a130f --- /dev/null +++ b/tests/jepsen.nukeeper/resources/users.xml @@ -0,0 +1 @@ +../../../programs/server/users.xml \ No newline at end of file diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj new file mode 100644 index 00000000000..8aa157bc16e --- /dev/null +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -0,0 +1,143 @@ +(ns jepsen.nukeeper.main + (:require [clojure.tools.logging :refer :all] + [jepsen.nukeeper.utils :refer :all] + [jepsen.nukeeper.set :as set] + [jepsen.nukeeper.register :as register] + [clojure.string :as str] + [jepsen + [checker :as checker] + [cli :as cli] + [client :as client] + [control :as c] + [db :as db] + [nemesis :as nemesis] + [generator :as gen] + [independent :as independent] + [tests :as tests]] + [jepsen.control.util :as cu] + [jepsen.os.ubuntu :as ubuntu] + [jepsen.checker.timeline :as timeline] + [clojure.java.io :as io] + [knossos.model :as model] + [zookeeper.data :as data] + [zookeeper :as zk]) + (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) + +(def dir "/var/lib/clickhouse") +(def binary "clickhouse") +(def logdir "/var/log/clickhouse-server") +(def logfile "/var/log/clickhouse-server/stderr.log") +(def serverlog "/var/log/clickhouse-server/clickhouse-server.log") +(def pidfile (str dir "/clickhouse.pid")) +(def binary-path "/tmp") + +(defn cluster-config + [test node config-template] + (let [nodes (:nodes test)] + (clojure.string/replace + (clojure.string/replace + (clojure.string/replace + (clojure.string/replace + (clojure.string/replace config-template #"\{quorum_reads\}" (str (boolean (:quorum test)))) + #"\{srv1\}" (get nodes 0)) + #"\{srv2\}" (get nodes 1)) + #"\{srv3\}" (get nodes 2)) + #"\{id\}" (str (inc (.indexOf nodes node)))))) + +(defn db + [version] + (reify db/DB + (setup! [_ test node] + (info node "installing clickhouse" version) + (c/su + (if-not (cu/exists? (str binary-path "/clickhouse")) + (c/exec :sky :get :-d binary-path :-N :Backbone version)) + (c/exec :mkdir :-p logdir) + (c/exec :touch logfile) + (c/exec (str binary-path "/clickhouse") :install) + (c/exec :chown :-R :root dir) + (c/exec :chown :-R :root logdir) + (c/exec :echo (slurp (io/resource "listen.xml")) :> "/etc/clickhouse-server/config.d/listen.xml") + (c/exec :echo (cluster-config test node (slurp (io/resource "test_keeper_config.xml"))) :> "/etc/clickhouse-server/config.d/test_keeper_config.xml") + (cu/start-daemon! + {:pidfile pidfile + :logfile logfile + :chdir dir} + (str binary-path "/clickhouse") + :server + :--config "/etc/clickhouse-server/config.xml") + (Thread/sleep 10000))) + + (teardown! [_ test node] + (info node "tearing down clickhouse") + (cu/stop-daemon! (str binary-path "/clickhouse") pidfile) + (c/su + (c/exec :rm :-f (str binary-path "/clickhouse")) + (c/exec :rm :-rf dir) + (c/exec :rm :-rf logdir) + (c/exec :rm :-rf "/etc/clickhouse-server"))) + + db/LogFiles + (log-files [_ test node] + [logfile serverlog]))) + +(def workloads + "A map of workload names to functions that construct workloads, given opts." + {"set" set/workload + "register" register/workload}) + +(def cli-opts + "Additional command line options." + [["-w" "--workload NAME" "What workload should we run?" + :missing (str "--workload " (cli/one-of workloads)) + :validate [workloads (cli/one-of workloads)]] + ["-q" "--quorum" "Use quorum reads, instead of reading from any primary."] + ["-r" "--rate HZ" "Approximate number of requests per second, per thread." + :default 10 + :parse-fn read-string + :validate [#(and (number? %) (pos? %)) "Must be a positive number"]] + [nil "--ops-per-key NUM" "Maximum number of operations on any given key." + :default 100 + :parse-fn parse-long + :validate [pos? "Must be a positive integer."]]]) + +(defn nukeeper-test + "Given an options map from the command line runner (e.g. :nodes, :ssh, + :concurrency, ...), constructs a test map." + [opts] + (let [quorum (boolean (:quorum opts)) + workload ((get workloads (:workload opts)) opts)] + (merge tests/noop-test + opts + {:name (str "clickhouse-keeper quorum=" quorum " " (name (:workload opts))) + :os ubuntu/os + :db (db "rbtorrent:8831b5baa571abc28340cf66a9279a4ce45fac64") + :pure-generators true + :client (:client workload) + :nemesis (nemesis/partition-random-halves) + :checker (checker/compose + {:perf (checker/perf) + :workload (:checker workload)}) + :generator (gen/phases + (->> (:generator workload) + (gen/stagger (/ (:rate opts))) + (gen/nemesis + (cycle [(gen/sleep 5) + {:type :info, :f :start} + (gen/sleep 5) + {:type :info, :f :stop}])) + (gen/time-limit (:time-limit opts))) + (gen/log "Healing cluster") + (gen/nemesis (gen/once {:type :info, :f :stop})) + (gen/log "Waiting for recovery") + (gen/sleep 10) + (gen/clients (:final-generator workload)))}))) + +(defn -main + "Handles command line arguments. Can either run a test, or a web server for + browsing results." + [& args] + (cli/run! (merge (cli/single-test-cmd {:test-fn nukeeper-test + :opt-spec cli-opts}) + (cli/serve-cmd)) + args)) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/register.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/register.clj new file mode 100644 index 00000000000..98322845346 --- /dev/null +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/register.clj @@ -0,0 +1,64 @@ +(ns jepsen.nukeeper.register + (:require [jepsen + [checker :as checker] + [client :as client] + [independent :as independent] + [generator :as gen]] + [jepsen.checker.timeline :as timeline] + [knossos.model :as model] + [jepsen.nukeeper.utils :refer :all] + [zookeeper :as zk]) + (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) + +(defn r [_ _] {:type :invoke, :f :read, :value nil}) +(defn w [_ _] {:type :invoke, :f :write, :value (rand-int 5)}) +(defn cas [_ _] {:type :invoke, :f :cas, :value [(rand-int 5) (rand-int 5)]}) + +(defrecord RegisterClient [conn] + client/Client + (open! [this test node] + (assoc this :conn (zk-connect node 9181 30000))) + + (setup! [this test] + (zk-create-range conn 300)) ; 300 nodes to be sure + + (invoke! [_ test op] + (let [[k v] (:value op) + zk-k (zk-path k)] + (case (:f op) + :read (try + (assoc op :type :ok, :value (independent/tuple k (parse-long (:data (zk-get-str conn zk-k))))) + (catch Exception _ (assoc op :type :fail, :error :connect-error))) + :write (try + (do (zk-set conn zk-k v) + (assoc op :type :ok)) + (catch Exception _ (assoc op :type :info, :error :connect-error))) + :cas (try + (let [[old new] v] + (assoc op :type (if (zk-cas conn zk-k old new) + :ok + :fail))) + (catch KeeperException$BadVersionException _ (assoc op :type :fail, :error :bad-version)) + (catch Exception _ (assoc op :type :info, :error :connect-error)))))) + + (teardown! [this test]) + + (close! [_ test] + (zk/close conn))) + +(defn workload + "Tests linearizable reads, writes, and compare-and-set operations on + independent keys." + [opts] + {:client (RegisterClient. nil) + :checker (independent/checker + (checker/compose + {:linear (checker/linearizable {:model (model/cas-register) + :algorithm :linear}) + :timeline (timeline/html)})) + :generator (independent/concurrent-generator + 10 + (range) + (fn [k] + (->> (gen/mix [r w cas]) + (gen/limit (:ops-per-key opts)))))}) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj new file mode 100644 index 00000000000..7e196fab4c7 --- /dev/null +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj @@ -0,0 +1,43 @@ +(ns jepsen.nukeeper.set + (:require [jepsen + [checker :as checker] + [client :as client] + [generator :as gen]] + [jepsen.nukeeper.utils :refer :all] + [zookeeper :as zk]) + (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) + +(defrecord SetClient [k conn] + client/Client + (open! [this test node] + (assoc this :conn (zk-connect node 9181 30000))) + + (setup! [this test] + (zk-create-if-not-exists conn k "#{}")) + + (invoke! [_ test op] + (case (:f op) + :read ;(try + (assoc op + :type :ok + :value (read-string (:data (zk-get-str conn k)))) + ;(catch Exception _ (assoc op :type :fail, :error :connect-error))) + :add (try + (do + (zk-add-to-set conn k (:value op)) + (assoc op :type :ok)) + (catch KeeperException$BadVersionException _ (assoc op :type :fail, :error :bad-version)) + (catch Exception _ (assoc op :type :info, :error :connect-error))))) + + (teardown! [_ test]) + + (close! [_ test])) + +(defn workload + "A generator, client, and checker for a set test." + [opts] + {:client (SetClient. "/a-set" nil) + :checker (checker/set) + :generator (->> (range) + (map (fn [x] {:type :invoke, :f :add, :value x}))) + :final-generator (gen/once {:type :invoke, :f :read, :value nil})}) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj new file mode 100644 index 00000000000..3caec8e5f62 --- /dev/null +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj @@ -0,0 +1,56 @@ +(ns jepsen.nukeeper.utils + (:require [clojure.string :as str] + [zookeeper.data :as data] + [zookeeper :as zk])) + +(defn parse-long + "Parses a string to a Long. Passes through `nil` and empty strings." + [s] + (if (and s (> (count s) 0)) + (Long/parseLong s))) + +(defn zk-range + [] + (map (fn [v] (str "/" v)) (range))) + +(defn zk-path + [n] + (str "/" n)) + +(defn zk-connect + [host port timeout] + (zk/connect (str host ":" port) :timeout-msec timeout)) + +(defn zk-create-range + [conn n] + (dorun (map (fn [v] (zk/create-all conn v :persistent? true)) (take n (zk-range))))) + +(defn zk-set + ([conn path value] + (zk/set-data conn path (data/to-bytes (str value)) -1)) + ([conn path value version] + (zk/set-data conn path (data/to-bytes (str value)) version))) + +(defn zk-get-str + [conn path] + (let [zk-result (zk/data conn path)] + {:data (data/to-string (:data zk-result)) + :stat (:stat zk-result)})) + +(defn zk-cas + [conn path old-value new-value] + (let [current-value (zk-get-str conn path)] + (if (= (parse-long (:data current-value)) old-value) + (do (zk-set conn path new-value (:version (:stat current-value))) + true)))) + +(defn zk-add-to-set + [conn path elem] + (let [current-value (zk-get-str conn path) + current-set (read-string (:data current-value)) + new-set (conj current-set elem)] + (zk-set conn path (pr-str new-set) (:version (:stat current-value))))) + +(defn zk-create-if-not-exists + [conn path data] + (zk/create conn path :data (data/to-bytes (str data)))) diff --git a/tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj b/tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj new file mode 100644 index 00000000000..824aa40d2c8 --- /dev/null +++ b/tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj @@ -0,0 +1,28 @@ +(ns jepsen.nukeeper-test + (:require [clojure.test :refer :all] + [jepsen.nukeeper.utils :refer :all] + [zookeeper :as zk] + [zookeeper.data :as data])) + +(defn multicreate + [conn] + (dorun (map (fn [v] (zk/create conn v :persistent? true)) (take 10 (zk-range))))) + +(defn multidelete + [conn] + (dorun (map (fn [v] (zk/delete conn v)) (take 10 (zk-range))))) + +(deftest a-test + (testing "nukeeper connection" + (let [conn (zk/connect "localhost:9181" :timeout-msec 5000)] + (println (take 10 (zk-range))) + (multidelete conn) + (multicreate conn) + (zk/create-all conn "/0") + (zk/create conn "/0") + (println (zk/children conn "/")) + (zk/set-data conn "/0" (data/to-bytes "777") -1) + (Thread/sleep 5000) + (println "VALUE" (data/to-string (:data (zk/data conn "/0")))) + (is (= (data/to-string (:data (zk/data conn "/0"))) "777")) + (zk/close conn)))) From f49d6404f39807850ef8fca116dd180261cf3be2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 16 Mar 2021 11:03:47 +0300 Subject: [PATCH 02/43] Trying to add new nemesis --- .../src/jepsen/nukeeper/constants.clj | 9 ++++ .../src/jepsen/nukeeper/main.clj | 18 +++----- .../src/jepsen/nukeeper/nemesis.clj | 13 ++++++ .../src/jepsen/nukeeper/set.clj | 10 +++-- .../src/jepsen/nukeeper/utils.clj | 44 ++++++++++++++++++- 5 files changed, 78 insertions(+), 16 deletions(-) create mode 100644 tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj create mode 100644 tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj new file mode 100644 index 00000000000..0a20adea086 --- /dev/null +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj @@ -0,0 +1,9 @@ +(ns jepsen.nukeeper.constants) + +(def dir "/var/lib/clickhouse") +(def binary "clickhouse") +(def logdir "/var/log/clickhouse-server") +(def logfile "/var/log/clickhouse-server/stderr.log") +(def serverlog "/var/log/clickhouse-server/clickhouse-server.log") +(def pidfile (str dir "/clickhouse.pid")) +(def binary-path "/tmp") diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index 8aa157bc16e..2b244c924bd 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -2,7 +2,9 @@ (:require [clojure.tools.logging :refer :all] [jepsen.nukeeper.utils :refer :all] [jepsen.nukeeper.set :as set] + [jepsen.nukeeper.nemesis :as custom-nemesis] [jepsen.nukeeper.register :as register] + [jepsen.nukeeper.constants :refer :all] [clojure.string :as str] [jepsen [checker :as checker] @@ -23,14 +25,6 @@ [zookeeper :as zk]) (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) -(def dir "/var/lib/clickhouse") -(def binary "clickhouse") -(def logdir "/var/log/clickhouse-server") -(def logfile "/var/log/clickhouse-server/stderr.log") -(def serverlog "/var/log/clickhouse-server/clickhouse-server.log") -(def pidfile (str dir "/clickhouse.pid")) -(def binary-path "/tmp") - (defn cluster-config [test node config-template] (let [nodes (:nodes test)] @@ -66,13 +60,13 @@ (str binary-path "/clickhouse") :server :--config "/etc/clickhouse-server/config.xml") - (Thread/sleep 10000))) + (wait-clickhouse-alive! node test))) (teardown! [_ test node] (info node "tearing down clickhouse") (cu/stop-daemon! (str binary-path "/clickhouse") pidfile) (c/su - (c/exec :rm :-f (str binary-path "/clickhouse")) + ;(c/exec :rm :-f (str binary-path "/clickhouse")) (c/exec :rm :-rf dir) (c/exec :rm :-rf logdir) (c/exec :rm :-rf "/etc/clickhouse-server"))) @@ -111,10 +105,10 @@ opts {:name (str "clickhouse-keeper quorum=" quorum " " (name (:workload opts))) :os ubuntu/os - :db (db "rbtorrent:8831b5baa571abc28340cf66a9279a4ce45fac64") + :db (db "rbtorrent:46832e8fa975b094a5591184b3c854700ed770f4") :pure-generators true :client (:client workload) - :nemesis (nemesis/partition-random-halves) + :nemesis (custom-nemesis/random-single-node-killer-nemesis) :checker (checker/compose {:perf (checker/perf) :workload (:checker workload)}) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj new file mode 100644 index 00000000000..2f359bc5cba --- /dev/null +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj @@ -0,0 +1,13 @@ +(ns jepsen.nukeeper.nemesis + (:require [jepsen + [nemesis :as nemesis]] + [jepsen.nukeeper.utils :refer :all])) + + + +(defn random-single-node-killer-nemesis + [] + (nemesis/node-start-stopper + rand-nth + (fn start [test node] (kill-clickhouse! node test)) + (fn stop [test node] (start-clickhouse! node test)))) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj index 7e196fab4c7..6a33350673d 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj @@ -1,5 +1,7 @@ (ns jepsen.nukeeper.set - (:require [jepsen + (:require + [clojure.tools.logging :refer :all] + [jepsen [checker :as checker] [client :as client] [generator :as gen]] @@ -18,9 +20,11 @@ (invoke! [_ test op] (case (:f op) :read ;(try - (assoc op + (do (info "LIST ON NODE" (zk-list conn "/")) + (info "EXISTS NODE" (zk/exists conn "/a-set")) + (assoc op :type :ok - :value (read-string (:data (zk-get-str conn k)))) + :value (read-string (:data (zk-get-str conn k))))) ;(catch Exception _ (assoc op :type :fail, :error :connect-error))) :add (try (do diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj index 3caec8e5f62..e398039a329 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj @@ -1,7 +1,11 @@ (ns jepsen.nukeeper.utils (:require [clojure.string :as str] [zookeeper.data :as data] - [zookeeper :as zk])) + [zookeeper :as zk] + [jepsen.control.util :as cu] + [jepsen.nukeeper.constants :refer :all] + [jepsen.control :as c] + [clojure.tools.logging :refer :all])) (defn parse-long "Parses a string to a Long. Passes through `nil` and empty strings." @@ -37,6 +41,10 @@ {:data (data/to-string (:data zk-result)) :stat (:stat zk-result)})) +(defn zk-list + [conn path] + (zk/children conn path)) + (defn zk-cas [conn path old-value new-value] (let [current-value (zk-get-str conn path)] @@ -54,3 +62,37 @@ (defn zk-create-if-not-exists [conn path data] (zk/create conn path :data (data/to-bytes (str data)))) + + +(defn clickhouse-alive? + [node test] + (info "Checking server alive on" node) + (try + (c/exec (str binary-path "/clickhouse") :client :--query "SELECT 1") + (catch Exception _ false))) + +(defn wait-clickhouse-alive! + [node test & {:keys [maxtries] :or {maxtries 30}}] + (loop [i 0] + (cond (> i maxtries) false + (clickhouse-alive? node test) true + :else (do (Thread/sleep 1000) (recur (inc i)))))) + +(defn kill-clickhouse! + [node test] + (info "Killing server on node" node) + (c/su + (cu/stop-daemon! (str binary-path "/clickhouse") pidfile))) + +(defn start-clickhouse! + [node test] + (info "Starting server on node" node) + (c/su + (cu/start-daemon! + {:pidfile pidfile + :logfile logfile + :chdir dir} + (str binary-path "/clickhouse") + :server + :--config "/etc/clickhouse-server/config.xml")) + (wait-clickhouse-alive! node test)) From 6454479edda94ed7df6b00e25f77388139ae0fb8 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 16 Mar 2021 14:44:43 +0300 Subject: [PATCH 03/43] Add useful util for state dump --- .../src/jepsen/nukeeper/constants.clj | 3 + .../src/jepsen/nukeeper/main.clj | 9 +- .../src/jepsen/nukeeper/nemesis.clj | 1 - .../src/jepsen/nukeeper/set.clj | 13 +-- utils/CMakeLists.txt | 1 + utils/nukeeper-data-dumper/CMakeLists.txt | 2 + utils/nukeeper-data-dumper/main.cpp | 87 +++++++++++++++++++ 7 files changed, 108 insertions(+), 8 deletions(-) create mode 100644 utils/nukeeper-data-dumper/CMakeLists.txt create mode 100644 utils/nukeeper-data-dumper/main.cpp diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj index 0a20adea086..511ff8e3bf3 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj @@ -5,5 +5,8 @@ (def logdir "/var/log/clickhouse-server") (def logfile "/var/log/clickhouse-server/stderr.log") (def serverlog "/var/log/clickhouse-server/clickhouse-server.log") +(def snapshotsdir "/var/lib/clickhouse/coordination/snapshots") +(def coordinationdir "/var/lib/clickhouse/coordination") +(def logsdir "/var/lib/clickhouse/coordination/logs") (def pidfile (str dir "/clickhouse.pid")) (def binary-path "/tmp") diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index 2b244c924bd..1153f6f1389 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -15,7 +15,8 @@ [nemesis :as nemesis] [generator :as gen] [independent :as independent] - [tests :as tests]] + [tests :as tests] + [util :as util :refer [meh]]] [jepsen.control.util :as cu] [jepsen.os.ubuntu :as ubuntu] [jepsen.checker.timeline :as timeline] @@ -73,7 +74,11 @@ db/LogFiles (log-files [_ test node] - [logfile serverlog]))) + (c/su + (cu/stop-daemon! (str binary-path "/clickhouse") pidfile) + (c/cd dir + (c/exec :tar :czf "coordination.tar.gz" "coordination"))) + [logfile serverlog (str dir "/coordination.tar.gz")]))) (def workloads "A map of workload names to functions that construct workloads, given opts." diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj index 2f359bc5cba..84253dd6d42 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj @@ -4,7 +4,6 @@ [jepsen.nukeeper.utils :refer :all])) - (defn random-single-node-killer-nemesis [] (nemesis/node-start-stopper diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj index 6a33350673d..fcdfa138c4c 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj @@ -9,18 +9,21 @@ [zookeeper :as zk]) (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) -(defrecord SetClient [k conn] +(defrecord SetClient [k conn nodename] client/Client (open! [this test node] - (assoc this :conn (zk-connect node 9181 30000))) + (assoc + (assoc this + :conn (zk-connect node 9181 30000)) + :nodename node)) (setup! [this test] (zk-create-if-not-exists conn k "#{}")) - (invoke! [_ test op] + (invoke! [this test op] (case (:f op) :read ;(try - (do (info "LIST ON NODE" (zk-list conn "/")) + (do (info "LIST ON NODE" nodename (zk-list conn "/")) (info "EXISTS NODE" (zk/exists conn "/a-set")) (assoc op :type :ok @@ -40,7 +43,7 @@ (defn workload "A generator, client, and checker for a set test." [opts] - {:client (SetClient. "/a-set" nil) + {:client (SetClient. "/a-set" nil nil) :checker (checker/set) :generator (->> (range) (map (fn [x] {:type :invoke, :f :add, :value x}))) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index d38b34f3419..dc077f0e49a 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -21,6 +21,7 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS) add_subdirectory (corrector_utf8) add_subdirectory (zookeeper-cli) add_subdirectory (zookeeper-test) + add_subdirectory (nukeeper-data-dumper) add_subdirectory (zookeeper-dump-tree) add_subdirectory (zookeeper-remove-by-list) add_subdirectory (zookeeper-create-entry-to-download-part) diff --git a/utils/nukeeper-data-dumper/CMakeLists.txt b/utils/nukeeper-data-dumper/CMakeLists.txt new file mode 100644 index 00000000000..bab1137bf4d --- /dev/null +++ b/utils/nukeeper-data-dumper/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(nukeeper-data-dumper main.cpp) +target_link_libraries(nukeeper-data-dumper PRIVATE dbms) diff --git a/utils/nukeeper-data-dumper/main.cpp b/utils/nukeeper-data-dumper/main.cpp new file mode 100644 index 00000000000..20682bdb366 --- /dev/null +++ b/utils/nukeeper-data-dumper/main.cpp @@ -0,0 +1,87 @@ +#include +#include +#include +#include +#include +#include +#include // Y_IGNORE +#include +#include +#include + +using namespace Coordination; +using namespace DB; + +void dumpMachine(std::shared_ptr machine) +{ + auto & storage = machine->getStorage(); + std::queue keys; + keys.push("/"); + + while (!keys.empty()) + { + auto key = keys.front(); + keys.pop(); + auto value = storage.container.getValue(key); + std::cout << key << "\n"; + std::cout << "\tStat: {version: " << value.stat.version << + ", mtime: " << value.stat.mtime << + ", emphemeralOwner: " << value.stat.ephemeralOwner << + ", czxid: " << value.stat.czxid << + ", mzxid: " << value.stat.mzxid << + ", numChildren: " << value.stat.numChildren << + ", dataLength: " << value.stat.dataLength << + "}" << std::endl; + std::cout << "\tData: " << storage.container.getValue(key).data << std::endl; + + for (const auto & child : value.children) + { + if (key == "/") + keys.push(key + child); + else + keys.push(key + "/" + child); + } + } + std::cout << std::flush; +} + +int main(int argc, char *argv[]) +{ + if (argc != 3) + { + std::cerr << "usage: " << argv[0] << " snapshotpath logpath" << std::endl; + return 3; + } + else + { + Poco::AutoPtr channel(new Poco::ConsoleChannel(std::cerr)); + Poco::Logger::root().setChannel(channel); + Poco::Logger::root().setLevel("trace"); + } + auto * logger = &Poco::Logger::get("nukeeper-dumper"); + ResponsesQueue queue; + SnapshotsQueue snapshots_queue{1}; + CoordinationSettingsPtr settings = std::make_shared(); + auto state_machine = std::make_shared(queue, snapshots_queue, argv[1], settings); + state_machine->init(); + size_t last_commited_index = state_machine->last_commit_index(); + + LOG_INFO(logger, "Last commited index: {}", last_commited_index); + + DB::NuKeeperLogStore changelog(argv[2], 10000000, true); + changelog.init(last_commited_index, 10000000000UL); /// collect all logs + if (changelog.size() == 0) + LOG_INFO(logger, "Changelog empty"); + else + LOG_INFO(logger, "Last changelog entry {}", changelog.next_slot() - 1); + + for (size_t i = last_commited_index + 1; i < changelog.next_slot(); ++i) + { + if (changelog.entry_at(i)->get_val_type() == nuraft::log_val_type::app_log) + state_machine->commit(i, changelog.entry_at(i)->get_buf()); + } + + dumpMachine(state_machine); + + return 0; +} From 077a2019b6e577b530c7edd116b16dbe35168692 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 16 Mar 2021 15:36:54 +0300 Subject: [PATCH 04/43] Found first real bug with jepsen --- src/Coordination/NuKeeperStorage.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Coordination/NuKeeperStorage.cpp b/src/Coordination/NuKeeperStorage.cpp index fff44163b71..2440d6f6613 100644 --- a/src/Coordination/NuKeeperStorage.cpp +++ b/src/Coordination/NuKeeperStorage.cpp @@ -641,6 +641,13 @@ NuKeeperStorage::ResponsesForSessions NuKeeperStorage::processRequest(const Coor for (const auto & ephemeral_path : it->second) { container.erase(ephemeral_path); + container.updateValue(parentPath(ephemeral_path), [&ephemeral_path] (NuKeeperStorage::Node & parent) + { + --parent.stat.numChildren; + ++parent.stat.cversion; + parent.children.erase(getBaseName(ephemeral_path)); + }); + auto responses = processWatchesImpl(ephemeral_path, watches, list_watches, Coordination::Event::DELETED); results.insert(results.end(), responses.begin(), responses.end()); } From 8cf8265d474b038c60ffdb5a855451cadb24520c Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 16 Mar 2021 15:37:46 +0300 Subject: [PATCH 05/43] Style --- .../src/jepsen/nukeeper/main.clj | 9 ++++--- .../src/jepsen/nukeeper/nemesis.clj | 26 +++++++++++++++---- .../src/jepsen/nukeeper/set.clj | 18 ++++++------- .../src/jepsen/nukeeper/utils.clj | 21 +++++++-------- utils/nukeeper-data-dumper/main.cpp | 2 +- 5 files changed, 46 insertions(+), 30 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index 1153f6f1389..dd40b7e399b 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -75,9 +75,9 @@ db/LogFiles (log-files [_ test node] (c/su - (cu/stop-daemon! (str binary-path "/clickhouse") pidfile) - (c/cd dir - (c/exec :tar :czf "coordination.tar.gz" "coordination"))) + (cu/stop-daemon! (str binary-path "/clickhouse") pidfile) + (c/cd dir + (c/exec :tar :czf "coordination.tar.gz" "coordination"))) [logfile serverlog (str dir "/coordination.tar.gz")]))) (def workloads @@ -105,7 +105,8 @@ :concurrency, ...), constructs a test map." [opts] (let [quorum (boolean (:quorum opts)) - workload ((get workloads (:workload opts)) opts)] + workload ((get workloads (:workload opts)) opts) + current-nemesis (get custom-nemesis/custom-nemesises "killer")] (merge tests/noop-test opts {:name (str "clickhouse-keeper quorum=" quorum " " (name (:workload opts))) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj index 84253dd6d42..620ad1bd3d3 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj @@ -1,12 +1,28 @@ (ns jepsen.nukeeper.nemesis (:require [jepsen - [nemesis :as nemesis]] + [nemesis :as nemesis] + [generator :as gen]] [jepsen.nukeeper.utils :refer :all])) - (defn random-single-node-killer-nemesis [] (nemesis/node-start-stopper - rand-nth - (fn start [test node] (kill-clickhouse! node test)) - (fn stop [test node] (start-clickhouse! node test)))) + rand-nth + (fn start [test node] (kill-clickhouse! node test)) + (fn stop [test node] (start-clickhouse! node test)))) + +(def custom-nemesises + {"killer" {:nemesis (random-single-node-killer-nemesis) + :generator + (gen/nemesis + (cycle [(gen/sleep 5) + {:type :info, :f :start} + (gen/sleep 5) + {:type :info, :f :stop}]))} + "simple-partitioner" {:nemesis (nemesis/partition-random-halves) + :generator + (gen/nemesis + (cycle [(gen/sleep 5) + {:type :info, :f :start} + (gen/sleep 5) + {:type :info, :f :stop}]))}}) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj index fcdfa138c4c..f2f614b2d17 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj @@ -1,12 +1,12 @@ (ns jepsen.nukeeper.set (:require - [clojure.tools.logging :refer :all] - [jepsen - [checker :as checker] - [client :as client] - [generator :as gen]] - [jepsen.nukeeper.utils :refer :all] - [zookeeper :as zk]) + [clojure.tools.logging :refer :all] + [jepsen + [checker :as checker] + [client :as client] + [generator :as gen]] + [jepsen.nukeeper.utils :refer :all] + [zookeeper :as zk]) (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) (defrecord SetClient [k conn nodename] @@ -26,8 +26,8 @@ (do (info "LIST ON NODE" nodename (zk-list conn "/")) (info "EXISTS NODE" (zk/exists conn "/a-set")) (assoc op - :type :ok - :value (read-string (:data (zk-get-str conn k))))) + :type :ok + :value (read-string (:data (zk-get-str conn k))))) ;(catch Exception _ (assoc op :type :fail, :error :connect-error))) :add (try (do diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj index e398039a329..19b4959d742 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj @@ -63,13 +63,12 @@ [conn path data] (zk/create conn path :data (data/to-bytes (str data)))) - (defn clickhouse-alive? [node test] (info "Checking server alive on" node) (try - (c/exec (str binary-path "/clickhouse") :client :--query "SELECT 1") - (catch Exception _ false))) + (c/exec (str binary-path "/clickhouse") :client :--query "SELECT 1") + (catch Exception _ false))) (defn wait-clickhouse-alive! [node test & {:keys [maxtries] :or {maxtries 30}}] @@ -82,17 +81,17 @@ [node test] (info "Killing server on node" node) (c/su - (cu/stop-daemon! (str binary-path "/clickhouse") pidfile))) + (cu/stop-daemon! (str binary-path "/clickhouse") pidfile))) (defn start-clickhouse! [node test] (info "Starting server on node" node) (c/su - (cu/start-daemon! - {:pidfile pidfile - :logfile logfile - :chdir dir} - (str binary-path "/clickhouse") - :server - :--config "/etc/clickhouse-server/config.xml")) + (cu/start-daemon! + {:pidfile pidfile + :logfile logfile + :chdir dir} + (str binary-path "/clickhouse") + :server + :--config "/etc/clickhouse-server/config.xml")) (wait-clickhouse-alive! node test)) diff --git a/utils/nukeeper-data-dumper/main.cpp b/utils/nukeeper-data-dumper/main.cpp index 20682bdb366..0340c94c5a0 100644 --- a/utils/nukeeper-data-dumper/main.cpp +++ b/utils/nukeeper-data-dumper/main.cpp @@ -22,8 +22,8 @@ void dumpMachine(std::shared_ptr machine) { auto key = keys.front(); keys.pop(); - auto value = storage.container.getValue(key); std::cout << key << "\n"; + auto value = storage.container.getValue(key); std::cout << "\tStat: {version: " << value.stat.version << ", mtime: " << value.stat.mtime << ", emphemeralOwner: " << value.stat.ephemeralOwner << From 63873f46bbb791cc9f5f094af3a7b35a64e4f04a Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 16 Mar 2021 15:40:28 +0300 Subject: [PATCH 06/43] Create persistent nodes in tests --- tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj | 6 ++---- tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj index f2f614b2d17..deb69c3ced4 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj @@ -22,13 +22,11 @@ (invoke! [this test op] (case (:f op) - :read ;(try - (do (info "LIST ON NODE" nodename (zk-list conn "/")) - (info "EXISTS NODE" (zk/exists conn "/a-set")) + :read + (do (assoc op :type :ok :value (read-string (:data (zk-get-str conn k))))) - ;(catch Exception _ (assoc op :type :fail, :error :connect-error))) :add (try (do (zk-add-to-set conn k (:value op)) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj index 19b4959d742..9912b34cd46 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj @@ -61,7 +61,7 @@ (defn zk-create-if-not-exists [conn path data] - (zk/create conn path :data (data/to-bytes (str data)))) + (zk/create conn path :data (data/to-bytes (str data)) :persistent? true)) (defn clickhouse-alive? [node test] From 54fbea68a194cccfd286cc76b9224684667ec5f8 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 16 Mar 2021 17:53:49 +0300 Subject: [PATCH 07/43] Add hammer-time nemesis --- tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj | 4 ++-- tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index dd40b7e399b..6e3777d3141 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -111,10 +111,10 @@ opts {:name (str "clickhouse-keeper quorum=" quorum " " (name (:workload opts))) :os ubuntu/os - :db (db "rbtorrent:46832e8fa975b094a5591184b3c854700ed770f4") + :db (db "rbtorrent:a122093aee0bdcb70ca42d5e5fb4ba5544372f5f") :pure-generators true :client (:client workload) - :nemesis (custom-nemesis/random-single-node-killer-nemesis) + :nemesis (custom-nemesis/hammer-time-nemesis) :checker (checker/compose {:perf (checker/perf) :workload (:checker workload)}) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj index 620ad1bd3d3..f3e01714128 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj @@ -11,6 +11,10 @@ (fn start [test node] (kill-clickhouse! node test)) (fn stop [test node] (start-clickhouse! node test)))) +(defn hammer-time-nemesis + [] + (nemesis/hammer-time "clickhouse")) + (def custom-nemesises {"killer" {:nemesis (random-single-node-killer-nemesis) :generator From 82b2c34c4029ab0dd80ba0bf97974d2ffb1285d2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 16 Mar 2021 23:27:09 +0300 Subject: [PATCH 08/43] Remove strange file --- tests/jepsen.nukeeper/CHANGELOG.md | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 tests/jepsen.nukeeper/CHANGELOG.md diff --git a/tests/jepsen.nukeeper/CHANGELOG.md b/tests/jepsen.nukeeper/CHANGELOG.md deleted file mode 100644 index 6c7cb4f7c8a..00000000000 --- a/tests/jepsen.nukeeper/CHANGELOG.md +++ /dev/null @@ -1,24 +0,0 @@ -# Change Log -All notable changes to this project will be documented in this file. This change log follows the conventions of [keepachangelog.com](http://keepachangelog.com/). - -## [Unreleased] -### Changed -- Add a new arity to `make-widget-async` to provide a different widget shape. - -## [0.1.1] - 2021-03-10 -### Changed -- Documentation on how to make the widgets. - -### Removed -- `make-widget-sync` - we're all async, all the time. - -### Fixed -- Fixed widget maker to keep working when daylight savings switches over. - -## 0.1.0 - 2021-03-10 -### Added -- Files from the new template. -- Widget maker public API - `make-widget-sync`. - -[Unreleased]: https://github.com/your-name/jepsen.nukeeper/compare/0.1.1...HEAD -[0.1.1]: https://github.com/your-name/jepsen.nukeeper/compare/0.1.0...0.1.1 From 46af999f3aea5db1963ed241062ed3048af8f103 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 17 Mar 2021 10:11:55 +0300 Subject: [PATCH 09/43] Trying to add corruption nemesis --- .../src/jepsen/nukeeper/main.clj | 6 +-- .../src/jepsen/nukeeper/nemesis.clj | 44 ++++++++++++++++++- .../src/jepsen/nukeeper/set.clj | 6 +-- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index 6e3777d3141..d62cbabd56f 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -114,7 +114,7 @@ :db (db "rbtorrent:a122093aee0bdcb70ca42d5e5fb4ba5544372f5f") :pure-generators true :client (:client workload) - :nemesis (custom-nemesis/hammer-time-nemesis) + :nemesis (custom-nemesis/logs-corruption-nemesis) :checker (checker/compose {:perf (checker/perf) :workload (:checker workload)}) @@ -123,9 +123,7 @@ (gen/stagger (/ (:rate opts))) (gen/nemesis (cycle [(gen/sleep 5) - {:type :info, :f :start} - (gen/sleep 5) - {:type :info, :f :stop}])) + {:type :info, :f :corrupt}])) (gen/time-limit (:time-limit opts))) (gen/log "Healing cluster") (gen/nemesis (gen/once {:type :info, :f :stop})) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj index f3e01714128..6b0497cd0af 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj @@ -1,7 +1,11 @@ (ns jepsen.nukeeper.nemesis - (:require [jepsen + (:require + [clojure.tools.logging :refer :all] + [jepsen [nemesis :as nemesis] + [control :as c] [generator :as gen]] + [jepsen.nukeeper.constants :refer :all] [jepsen.nukeeper.utils :refer :all])) (defn random-single-node-killer-nemesis @@ -15,6 +19,44 @@ [] (nemesis/hammer-time "clickhouse")) +(defn select-last-file + [path] + (info "EXECUTE ON PATH" path) + (last (clojure.string/split (c/exec :find path :-type :f :-printf "%T+ $PWD%p\n" :| :sort :| :awk "'{print $2}'")) #"\n")) + +(defn corrupt-file + [fname] + (c/exec :dd "if=/dev/zero" ("str of=" fname) "bs=1" "count=1" "seek=N" "conv=notrunc")) + +(defn corruptor-nemesis + [path corruption-op] + (reify nemesis/Nemesis + (setup! [this test] this) + + (invoke! [this test op] + (let [nodes (list (rand-nth (:nodes test)))] + (info "Corruption on node" nodes) + (c/on-nodes test nodes + (fn [node] + (let [file-to-corrupt (select-last-file path)] + (info "Corrupting file" file-to-corrupt) + (c/su + (corruption-op (select-last-file path)) + (kill-clickhouse! node test) + (start-clickhouse! node test))))) + {:f (:f op) + :value :corrupted})) + + (teardown! [this test]))) + +(defn logs-corruption-nemesis + [] + (corruptor-nemesis logsdir corrupt-file)) + +(defn snapshots-corruption-nemesis + [] + (corruptor-nemesis snapshotsdir corrupt-file)) + (def custom-nemesises {"killer" {:nemesis (random-single-node-killer-nemesis) :generator diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj index deb69c3ced4..d50253aa174 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj @@ -24,9 +24,9 @@ (case (:f op) :read (do - (assoc op - :type :ok - :value (read-string (:data (zk-get-str conn k))))) + (assoc op + :type :ok + :value (read-string (:data (zk-get-str conn k))))) :add (try (do (zk-add-to-set conn k (:value op)) From d9f835a242743116332604fde39db3c74aa0afc9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 17 Mar 2021 11:13:52 +0300 Subject: [PATCH 10/43] Finally corrupted logs --- .../src/jepsen/nukeeper/nemesis.clj | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj index 6b0497cd0af..bf2348f1860 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj @@ -22,30 +22,40 @@ (defn select-last-file [path] (info "EXECUTE ON PATH" path) - (last (clojure.string/split (c/exec :find path :-type :f :-printf "%T+ $PWD%p\n" :| :sort :| :awk "'{print $2}'")) #"\n")) + (last (clojure.string/split + (c/exec :find path :-type :f :-printf "%T+ %p\n" :| :sort :| :awk "{print $2}") + #"\n"))) + +(defn random-file-pos + [fname] + (let [fsize (Integer/parseInt (c/exec :du :-b fname :| :cut :-f1))] + (rand-int fsize))) (defn corrupt-file [fname] - (c/exec :dd "if=/dev/zero" ("str of=" fname) "bs=1" "count=1" "seek=N" "conv=notrunc")) + (info "Corrupting" fname) + (c/exec :dd "if=/dev/zero" (str "of=" fname) "bs=1" "count=1" (str "seek=" (random-file-pos fname)) "conv=notrunc")) (defn corruptor-nemesis [path corruption-op] (reify nemesis/Nemesis + (setup! [this test] this) (invoke! [this test op] - (let [nodes (list (rand-nth (:nodes test)))] - (info "Corruption on node" nodes) - (c/on-nodes test nodes - (fn [node] - (let [file-to-corrupt (select-last-file path)] - (info "Corrupting file" file-to-corrupt) - (c/su - (corruption-op (select-last-file path)) - (kill-clickhouse! node test) - (start-clickhouse! node test))))) - {:f (:f op) - :value :corrupted})) + (cond (= (:f op) :corrupt) + (let [nodes (list (rand-nth (:nodes test)))] + (info "Corruption on node" nodes) + (c/on-nodes test nodes + (fn [test node] + (let [file-to-corrupt (select-last-file path)] + (info "Corrupting file" file-to-corrupt) + (c/su + (corruption-op (select-last-file path)) + (kill-clickhouse! node test) + (start-clickhouse! node test))))) + (assoc op :type :info, :value :corrupted)) + :else (assoc op :type :info, :value :not-started))) (teardown! [this test]))) From 341e22341944a405af306e5dd75631f81228c8e6 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 17 Mar 2021 14:35:37 +0300 Subject: [PATCH 11/43] Better corruption nemesises, options --- .../resources/test_keeper_config.xml | 3 + .../src/jepsen/nukeeper/main.clj | 45 +++++--- .../src/jepsen/nukeeper/nemesis.clj | 103 +++++++++++------- .../src/jepsen/nukeeper/utils.clj | 3 +- 4 files changed, 99 insertions(+), 55 deletions(-) diff --git a/tests/jepsen.nukeeper/resources/test_keeper_config.xml b/tests/jepsen.nukeeper/resources/test_keeper_config.xml index 0e2a688ea0b..7ef34d4bea1 100644 --- a/tests/jepsen.nukeeper/resources/test_keeper_config.xml +++ b/tests/jepsen.nukeeper/resources/test_keeper_config.xml @@ -10,6 +10,9 @@ 60000 trace {quorum_reads} + {snapshot_distance} + {stale_log_gap} + {reserved_log_items} diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index d62cbabd56f..a5ceae5d5ae 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -28,16 +28,16 @@ (defn cluster-config [test node config-template] - (let [nodes (:nodes test)] - (clojure.string/replace - (clojure.string/replace - (clojure.string/replace - (clojure.string/replace - (clojure.string/replace config-template #"\{quorum_reads\}" (str (boolean (:quorum test)))) - #"\{srv1\}" (get nodes 0)) - #"\{srv2\}" (get nodes 1)) - #"\{srv3\}" (get nodes 2)) - #"\{id\}" (str (inc (.indexOf nodes node)))))) + (let [nodes (:nodes test) + replacement-map {#"\{srv1\}" (get nodes 0) + #"\{srv2\}" (get nodes 1) + #"\{srv3\}" (get nodes 2) + #"\{id\}" (str (inc (.indexOf nodes node))) + #"\{quorum_reads\}" (str (boolean (:quorum test))) + #"\{snapshot_distance\}" (str (:snapshot-distance test)) + #"\{stale_log_gap\}" (str (:stale-log-gap test)) + #"\{reserved_log_items\}" (str (:reserved-log-items test))}] + (reduce #(clojure.string/replace %1 (get %2 0) (get %2 1)) config-template replacement-map))) (defn db [version] @@ -90,11 +90,26 @@ [["-w" "--workload NAME" "What workload should we run?" :missing (str "--workload " (cli/one-of workloads)) :validate [workloads (cli/one-of workloads)]] + [nil "--nemesis NAME" "Which nemesis will poison our lives?" + :missing (str "--nemesis " (cli/one-of custom-nemesis/custom-nemesises)) + :validate [custom-nemesis/custom-nemesises (cli/one-of custom-nemesis/custom-nemesises)]] ["-q" "--quorum" "Use quorum reads, instead of reading from any primary."] ["-r" "--rate HZ" "Approximate number of requests per second, per thread." :default 10 :parse-fn read-string :validate [#(and (number? %) (pos? %)) "Must be a positive number"]] + ["-s" "--snapshot-distance NUM" "Number of log entries to create snapshot" + :default 10000 + :parse-fn read-string + :validate [#(and (number? %) (pos? %)) "Must be a positive number"]] + [nil "--stale-log-gap NUM" "Number of log entries to send snapshot instead of separate logs" + :default 1000 + :parse-fn read-string + :validate [#(and (number? %) (pos? %)) "Must be a positive number"]] + [nil "--reserved-log-items NUM" "Number of log entries to keep after snapshot" + :default 1000 + :parse-fn read-string + :validate [#(and (number? %) (pos? %)) "Must be a positive number"]] [nil "--ops-per-key NUM" "Maximum number of operations on any given key." :default 100 :parse-fn parse-long @@ -106,24 +121,22 @@ [opts] (let [quorum (boolean (:quorum opts)) workload ((get workloads (:workload opts)) opts) - current-nemesis (get custom-nemesis/custom-nemesises "killer")] + current-nemesis (get custom-nemesis/custom-nemesises (:nemesis opts))] (merge tests/noop-test opts - {:name (str "clickhouse-keeper quorum=" quorum " " (name (:workload opts))) + {:name (str "clickhouse-keeper quorum=" quorum " " (name (:workload opts)) (name (:nemesis opts))) :os ubuntu/os :db (db "rbtorrent:a122093aee0bdcb70ca42d5e5fb4ba5544372f5f") :pure-generators true :client (:client workload) - :nemesis (custom-nemesis/logs-corruption-nemesis) + :nemesis (:nemesis current-nemesis) :checker (checker/compose {:perf (checker/perf) :workload (:checker workload)}) :generator (gen/phases (->> (:generator workload) (gen/stagger (/ (:rate opts))) - (gen/nemesis - (cycle [(gen/sleep 5) - {:type :info, :f :corrupt}])) + (gen/nemesis (:generator current-nemesis)) (gen/time-limit (:time-limit opts))) (gen/log "Healing cluster") (gen/nemesis (gen/once {:type :info, :f :stop})) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj index bf2348f1860..93026a7d64c 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj @@ -1,12 +1,12 @@ (ns jepsen.nukeeper.nemesis (:require - [clojure.tools.logging :refer :all] - [jepsen - [nemesis :as nemesis] - [control :as c] - [generator :as gen]] - [jepsen.nukeeper.constants :refer :all] - [jepsen.nukeeper.utils :refer :all])) + [clojure.tools.logging :refer :all] + [jepsen + [nemesis :as nemesis] + [control :as c] + [generator :as gen]] + [jepsen.nukeeper.constants :refer :all] + [jepsen.nukeeper.utils :refer :all])) (defn random-single-node-killer-nemesis [] @@ -21,9 +21,8 @@ (defn select-last-file [path] - (info "EXECUTE ON PATH" path) (last (clojure.string/split - (c/exec :find path :-type :f :-printf "%T+ %p\n" :| :sort :| :awk "{print $2}") + (c/exec :find path :-type :f :-printf "%T+ %p\n" :| :grep :-v :tmp_ :| :sort :| :awk "{print $2}") #"\n"))) (defn random-file-pos @@ -33,8 +32,11 @@ (defn corrupt-file [fname] - (info "Corrupting" fname) - (c/exec :dd "if=/dev/zero" (str "of=" fname) "bs=1" "count=1" (str "seek=" (random-file-pos fname)) "conv=notrunc")) + (if (not (empty? fname)) + (do + (info "Corrupting" fname) + (c/exec :dd "if=/dev/zero" (str "of=" fname) "bs=1" "count=1" (str "seek=" (random-file-pos fname)) "conv=notrunc")) + (info "Nothing to corrupt"))) (defn corruptor-nemesis [path corruption-op] @@ -44,41 +46,66 @@ (invoke! [this test op] (cond (= (:f op) :corrupt) - (let [nodes (list (rand-nth (:nodes test)))] - (info "Corruption on node" nodes) - (c/on-nodes test nodes - (fn [test node] - (let [file-to-corrupt (select-last-file path)] - (info "Corrupting file" file-to-corrupt) - (c/su - (corruption-op (select-last-file path)) - (kill-clickhouse! node test) - (start-clickhouse! node test))))) - (assoc op :type :info, :value :corrupted)) - :else (assoc op :type :info, :value :not-started))) + (let [nodes (list (rand-nth (:nodes test)))] + (info "Corruption on node" nodes) + (c/on-nodes test nodes + (fn [test node] + (c/su + (kill-clickhouse! node test) + (corruption-op path) + (start-clickhouse! node test)))) + (assoc op :type :info, :value :corrupted)) + :else (do (c/on-nodes test (:nodes test) + (fn [test node] + (c/su + (start-clickhouse! node test)))) + (assoc op :type :info, :value :done)))) (teardown! [this test]))) (defn logs-corruption-nemesis [] - (corruptor-nemesis logsdir corrupt-file)) + (corruptor-nemesis logsdir #(corrupt-file (select-last-file %1)))) (defn snapshots-corruption-nemesis [] - (corruptor-nemesis snapshotsdir corrupt-file)) + (corruptor-nemesis snapshotsdir #(corrupt-file (select-last-file %1)))) + +(defn logs-and-snapshots-corruption-nemesis + [] + (corruptor-nemesis coordinationdir (fn [path] + (do + (corrupt-file (select-last-file (str path "/snapshots"))) + (corrupt-file (select-last-file (str path "/logs"))))))) +(defn drop-all-corruption-nemesis + [] + (corruptor-nemesis coordinationdir (fn [path] + (c/exec :rm :-fr path)))) + +(defn start-stop-generator + [] + (->> + (cycle [(gen/sleep 5) + {:type :info, :f :start} + (gen/sleep 5) + {:type :info, :f :stop}]))) + +(defn corruption-generator + [] + (->> + (cycle [(gen/sleep 5) + {:type :info, :f :corrupt}]))) (def custom-nemesises - {"killer" {:nemesis (random-single-node-killer-nemesis) - :generator - (gen/nemesis - (cycle [(gen/sleep 5) - {:type :info, :f :start} - (gen/sleep 5) - {:type :info, :f :stop}]))} + {"single-node-killer" {:nemesis (random-single-node-killer-nemesis) + :generator (start-stop-generator)} "simple-partitioner" {:nemesis (nemesis/partition-random-halves) - :generator - (gen/nemesis - (cycle [(gen/sleep 5) - {:type :info, :f :start} - (gen/sleep 5) - {:type :info, :f :stop}]))}}) + :generator (start-stop-generator)} + "logs-corruptor" {:nemesis (logs-corruption-nemesis) + :generator (corruption-generator)} + "snapshots-corruptor" {:nemesis (snapshots-corruption-nemesis) + :generator (corruption-generator)} + "logs-and-snapshots-corruptor" {:nemesis (logs-and-snapshots-corruption-nemesis) + :generator (corruption-generator)} + "drop-data-corruptor" {:nemesis (drop-all-corruption-nemesis) + :generator (corruption-generator)}}) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj index 9912b34cd46..e9658e9d6d5 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj @@ -81,7 +81,8 @@ [node test] (info "Killing server on node" node) (c/su - (cu/stop-daemon! (str binary-path "/clickhouse") pidfile))) + (cu/stop-daemon! (str binary-path "/clickhouse") pidfile) + (c/exec :rm :-fr (str dir "/status")))) (defn start-clickhouse! [node test] From ecd081144c6a1db08b4952b0be19548b54d0f873 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 17 Mar 2021 14:54:26 +0300 Subject: [PATCH 12/43] Add missing hammer-time nemesis --- tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj index 93026a7d64c..d1dc0d55e5f 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj @@ -101,6 +101,8 @@ :generator (start-stop-generator)} "simple-partitioner" {:nemesis (nemesis/partition-random-halves) :generator (start-stop-generator)} + "hammer-time" {:nemesis (hammer-time-nemesis) + :generator (start-stop-generator)} "logs-corruptor" {:nemesis (logs-corruption-nemesis) :generator (corruption-generator)} "snapshots-corruptor" {:nemesis (snapshots-corruption-nemesis) From 7c4fdd79cfa0461156c6dae6015d64ff5e8d66ca Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 17 Mar 2021 15:58:01 +0300 Subject: [PATCH 13/43] Add unique-ids workload --- .../src/jepsen/nukeeper/main.clj | 4 +- .../src/jepsen/nukeeper/unique.clj | 45 +++++++++++++++++++ .../src/jepsen/nukeeper/utils.clj | 4 ++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 tests/jepsen.nukeeper/src/jepsen/nukeeper/unique.clj diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index a5ceae5d5ae..8b7c1a6caac 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -4,6 +4,7 @@ [jepsen.nukeeper.set :as set] [jepsen.nukeeper.nemesis :as custom-nemesis] [jepsen.nukeeper.register :as register] + [jepsen.nukeeper.unique :as unique] [jepsen.nukeeper.constants :refer :all] [clojure.string :as str] [jepsen @@ -83,7 +84,8 @@ (def workloads "A map of workload names to functions that construct workloads, given opts." {"set" set/workload - "register" register/workload}) + "register" register/workload + "unique-ids" unique/workload}) (def cli-opts "Additional command line options." diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/unique.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/unique.clj new file mode 100644 index 00000000000..fc8370005aa --- /dev/null +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/unique.clj @@ -0,0 +1,45 @@ +(ns jepsen.nukeeper.unique + (:require + [clojure.tools.logging :refer :all] + [jepsen + [checker :as checker] + [client :as client] + [generator :as gen]] + [jepsen.nukeeper.utils :refer :all] + [zookeeper :as zk]) + (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) + +(defn parse-and-get-counter + [path] + (Integer/parseInt (apply str (take-last 10 (seq (str path)))))) + +(defrecord UniqueClient [conn nodename] + client/Client + (open! [this test node] + (assoc + (assoc this + :conn (zk-connect node 9181 30000)) + :nodename node)) + + (setup! [this test]) + + (invoke! [this test op] + (case + :generate + (try + (let [result-path (zk-create-sequential conn "/seq-" "")] + (assoc op :type :ok :value (parse-and-get-counter result-path))) + (catch Exception _ (assoc op :type :info, :error :connect-error))))) + + (teardown! [_ test]) + + (close! [_ test])) + +(defn workload + "A generator, client, and checker for a set test." + [opts] + {:client (UniqueClient. nil nil) + :checker (checker/unique-ids) + :generator (->> + (range) + (map (fn [_] {:type :invoke, :f :generate})))}) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj index e9658e9d6d5..10851a2adc7 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj @@ -63,6 +63,10 @@ [conn path data] (zk/create conn path :data (data/to-bytes (str data)) :persistent? true)) +(defn zk-create-sequential + [conn path-prefix data] + (zk/create conn path-prefix :data (data/to-bytes (str data)) :persistent? true :sequential? true)) + (defn clickhouse-alive? [node test] (info "Checking server alive on" node) From 2ee58ed82fc6fe98d67b5f5cf9469c17e60602d6 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 17 Mar 2021 16:00:08 +0300 Subject: [PATCH 14/43] Fix style --- tests/jepsen.nukeeper/src/jepsen/nukeeper/unique.clj | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/unique.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/unique.clj index fc8370005aa..9c753dfe0ab 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/unique.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/unique.clj @@ -25,11 +25,11 @@ (invoke! [this test op] (case - :generate - (try - (let [result-path (zk-create-sequential conn "/seq-" "")] - (assoc op :type :ok :value (parse-and-get-counter result-path))) - (catch Exception _ (assoc op :type :info, :error :connect-error))))) + :generate + (try + (let [result-path (zk-create-sequential conn "/seq-" "")] + (assoc op :type :ok :value (parse-and-get-counter result-path))) + (catch Exception _ (assoc op :type :info, :error :connect-error))))) (teardown! [_ test]) From f3ff437a3997e399b013f7634b4f7dd7a5184e96 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 18 Mar 2021 14:32:45 +0300 Subject: [PATCH 15/43] Add all nodes killer/stop, one multitransaction request and counter test --- .../src/jepsen/nukeeper/counter.clj | 52 +++++++++++++++++++ .../src/jepsen/nukeeper/main.clj | 6 ++- .../src/jepsen/nukeeper/nemesis.clj | 35 +++++++++---- .../src/jepsen/nukeeper/set.clj | 6 +-- .../src/jepsen/nukeeper/utils.clj | 16 +++++- .../test/jepsen/nukeeper_test.clj | 2 + 6 files changed, 100 insertions(+), 17 deletions(-) create mode 100644 tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj new file mode 100644 index 00000000000..1bdf3f89186 --- /dev/null +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj @@ -0,0 +1,52 @@ +(ns jepsen.nukeeper.counter + (:require + [clojure.tools.logging :refer :all] + [jepsen + [checker :as checker] + [client :as client] + [generator :as gen]] + [jepsen.nukeeper.utils :refer :all] + [zookeeper :as zk]) + (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) + + +(defn r [_ _] {:type :invoke, :f :read}) +(defn add [_ _] {:type :invoke, :f :add, :value (rand-int 5)}) + + +(defrecord CounterClient [conn nodename] + client/Client + (open! [this test node] + (assoc + (assoc this + :conn (zk-connect node 9181 30000)) + :nodename node)) + + (setup! [this test]) + + (invoke! [this test op] + (case (:f op) + :read (try + (assoc op + :type :ok + :value (count (zk-list conn "/"))) + (catch Exception _ (assoc op :type :fail, :error :connect-error))) + :add (try + (do + (zk-multi-create-many-seq-nodes conn "/seq-" (:value op)) + (assoc op :type :ok)) + (catch Exception _ (assoc op :type :info, :error :connect-error))))) + + (teardown! [_ test]) + + (close! [_ test])) + +(defn workload + "A generator, client, and checker for a set test." + [opts] + {:client (CounterClient. nil nil) + :checker (checker/counter) + :generator (->> (range) + (map (fn [x] + (->> (gen/mix [r add]))))) + :final-generator (gen/once {:type :invoke, :f :read, :value nil})}) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index 8b7c1a6caac..0f9619a7653 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -5,6 +5,7 @@ [jepsen.nukeeper.nemesis :as custom-nemesis] [jepsen.nukeeper.register :as register] [jepsen.nukeeper.unique :as unique] + [jepsen.nukeeper.counter :as counter] [jepsen.nukeeper.constants :refer :all] [clojure.string :as str] [jepsen @@ -85,7 +86,8 @@ "A map of workload names to functions that construct workloads, given opts." {"set" set/workload "register" register/workload - "unique-ids" unique/workload}) + "unique-ids" unique/workload + "counter" counter/workload}) (def cli-opts "Additional command line options." @@ -126,7 +128,7 @@ current-nemesis (get custom-nemesis/custom-nemesises (:nemesis opts))] (merge tests/noop-test opts - {:name (str "clickhouse-keeper quorum=" quorum " " (name (:workload opts)) (name (:nemesis opts))) + {:name (str "clickhouse-keeper quorum=" quorum " " (name (:workload opts)) " " (name (:nemesis opts))) :os ubuntu/os :db (db "rbtorrent:a122093aee0bdcb70ca42d5e5fb4ba5544372f5f") :pure-generators true diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj index d1dc0d55e5f..bf22f9ad1f6 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj @@ -8,17 +8,28 @@ [jepsen.nukeeper.constants :refer :all] [jepsen.nukeeper.utils :refer :all])) -(defn random-single-node-killer-nemesis +(defn random-node-killer-nemesis [] (nemesis/node-start-stopper rand-nth (fn start [test node] (kill-clickhouse! node test)) (fn stop [test node] (start-clickhouse! node test)))) -(defn hammer-time-nemesis +(defn all-nodes-killer-nemesis + [] + (nemesis/node-start-stopper + identity + (fn start [test node] (kill-clickhouse! node test)) + (fn stop [test node] (start-clickhouse! node test)))) + +(defn random-node-hammer-time-nemesis [] (nemesis/hammer-time "clickhouse")) +(defn all-nodes-hammer-time-nemesis + [] + (nemesis/hammer-time identity "clickhouse")) + (defn select-last-file [path] (last (clojure.string/split @@ -83,11 +94,11 @@ (c/exec :rm :-fr path)))) (defn start-stop-generator - [] + [time-corrupt time-ok] (->> - (cycle [(gen/sleep 5) + (cycle [(gen/sleep time-ok) {:type :info, :f :start} - (gen/sleep 5) + (gen/sleep time-corrupt) {:type :info, :f :stop}]))) (defn corruption-generator @@ -97,12 +108,16 @@ {:type :info, :f :corrupt}]))) (def custom-nemesises - {"single-node-killer" {:nemesis (random-single-node-killer-nemesis) - :generator (start-stop-generator)} + {"random-node-killer" {:nemesis (random-node-killer-nemesis) + :generator (start-stop-generator 5 5)} + "all-nodes-killer" {:nemesis (all-nodes-killer-nemesis) + :generator (start-stop-generator 1 10)} "simple-partitioner" {:nemesis (nemesis/partition-random-halves) - :generator (start-stop-generator)} - "hammer-time" {:nemesis (hammer-time-nemesis) - :generator (start-stop-generator)} + :generator (start-stop-generator 5 5)} + "random-node-hammer-time" {:nemesis (random-node-hammer-time-nemesis) + :generator (start-stop-generator 5 5)} + "all-nodes-hammer-time" {:nemesis (all-nodes-hammer-time-nemesis) + :generator (start-stop-generator 1 10)} "logs-corruptor" {:nemesis (logs-corruption-nemesis) :generator (corruption-generator)} "snapshots-corruptor" {:nemesis (snapshots-corruption-nemesis) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj index d50253aa174..c30ec9635a1 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj @@ -22,11 +22,9 @@ (invoke! [this test op] (case (:f op) - :read - (do - (assoc op + :read (assoc op :type :ok - :value (read-string (:data (zk-get-str conn k))))) + :value (read-string (:data (zk-get-str conn k)))) :add (try (do (zk-add-to-set conn k (:value op)) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj index 10851a2adc7..6fd2f3c87f4 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj @@ -2,10 +2,13 @@ (:require [clojure.string :as str] [zookeeper.data :as data] [zookeeper :as zk] + [zookeeper.internal :as zi] [jepsen.control.util :as cu] [jepsen.nukeeper.constants :refer :all] [jepsen.control :as c] - [clojure.tools.logging :refer :all])) + [clojure.tools.logging :refer :all]) + (:import (org.apache.zookeeper CreateMode + ZooKeeper))) (defn parse-long "Parses a string to a Long. Passes through `nil` and empty strings." @@ -67,6 +70,17 @@ [conn path-prefix data] (zk/create conn path-prefix :data (data/to-bytes (str data)) :persistent? true :sequential? true)) +(defn zk-multi-create-many-seq-nodes + [conn path-prefix num] + (let [txn (.transaction conn)] + (loop [i 0] + (cond (>= i num) (.commit txn) + :else (do (.create txn path-prefix + (data/to-bytes "") + (zi/acls :open-acl-unsafe) + CreateMode/PERSISTENT_SEQUENTIAL) + (recur (inc i))))))) + (defn clickhouse-alive? [node test] (info "Checking server alive on" node) diff --git a/tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj b/tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj index 824aa40d2c8..1a3e8646574 100644 --- a/tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj +++ b/tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj @@ -22,6 +22,8 @@ (zk/create conn "/0") (println (zk/children conn "/")) (zk/set-data conn "/0" (data/to-bytes "777") -1) + (zk-multi-create-many-seq-nodes conn "/seq-" 5) + (println (zk/children conn "/")) (Thread/sleep 5000) (println "VALUE" (data/to-string (:data (zk/data conn "/0")))) (is (= (data/to-string (:data (zk/data conn "/0"))) "777")) From 0137a6baac723f94c3ee5401bbde98b2e0c51379 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 18 Mar 2021 23:55:11 +0300 Subject: [PATCH 16/43] Add test founding bug --- src/Coordination/NuKeeperSnapshotManager.cpp | 1 + src/Coordination/NuKeeperStorage.cpp | 11 +++ src/Coordination/NuKeeperStorage.h | 4 ++ .../src/jepsen/nukeeper/counter.clj | 7 +- .../src/jepsen/nukeeper/main.clj | 9 +-- .../src/jepsen/nukeeper/nemesis.clj | 6 +- .../src/jepsen/nukeeper/queue.clj | 67 +++++++++++++++++++ .../src/jepsen/nukeeper/set.clj | 7 +- .../src/jepsen/nukeeper/unique.clj | 7 +- .../src/jepsen/nukeeper/utils.clj | 36 +++++++++- .../test/jepsen/nukeeper_test.clj | 27 +++++--- 11 files changed, 150 insertions(+), 32 deletions(-) create mode 100644 tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj diff --git a/src/Coordination/NuKeeperSnapshotManager.cpp b/src/Coordination/NuKeeperSnapshotManager.cpp index f5a97619976..5cc7bc356be 100644 --- a/src/Coordination/NuKeeperSnapshotManager.cpp +++ b/src/Coordination/NuKeeperSnapshotManager.cpp @@ -161,6 +161,7 @@ void NuKeeperStorageSnapshot::serialize(const NuKeeperStorageSnapshot & snapshot SnapshotMetadataPtr NuKeeperStorageSnapshot::deserialize(NuKeeperStorage & storage, ReadBuffer & in) { + storage.clearData(); uint8_t version; readBinary(version, in); if (static_cast(version) > SnapshotVersion::V0) diff --git a/src/Coordination/NuKeeperStorage.cpp b/src/Coordination/NuKeeperStorage.cpp index 2440d6f6613..0b773aeaafd 100644 --- a/src/Coordination/NuKeeperStorage.cpp +++ b/src/Coordination/NuKeeperStorage.cpp @@ -752,4 +752,15 @@ void NuKeeperStorage::clearDeadWatches(int64_t session_id) } } +void NuKeeperStorage::clearData() +{ + container.clear(); + ephemerals.clear(); + sessions_and_watchers.clear(); + session_expiry_queue.clear(); + session_and_timeout.clear(); + session_id_counter = 1; + zxid = 0; +} + } diff --git a/src/Coordination/NuKeeperStorage.h b/src/Coordination/NuKeeperStorage.h index c49df88159f..b44a077c277 100644 --- a/src/Coordination/NuKeeperStorage.h +++ b/src/Coordination/NuKeeperStorage.h @@ -82,6 +82,8 @@ public: public: NuKeeperStorage(int64_t tick_time_ms); + void clearData(); + int64_t getSessionID(int64_t session_timeout_ms) { auto result = session_id_counter++; @@ -131,4 +133,6 @@ public: } }; +using NuKeeperStoragePtr = std::unique_ptr; + } diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj index 1bdf3f89186..48b270517a4 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj @@ -1,5 +1,5 @@ (ns jepsen.nukeeper.counter - (:require + (:require [clojure.tools.logging :refer :all] [jepsen [checker :as checker] @@ -9,11 +9,9 @@ [zookeeper :as zk]) (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) - (defn r [_ _] {:type :invoke, :f :read}) (defn add [_ _] {:type :invoke, :f :add, :value (rand-int 5)}) - (defrecord CounterClient [conn nodename] client/Client (open! [this test node] @@ -39,7 +37,8 @@ (teardown! [_ test]) - (close! [_ test])) + (close! [_ test] + (zk/close conn))) (defn workload "A generator, client, and checker for a set test." diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index 0f9619a7653..b8854638ed0 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -5,6 +5,7 @@ [jepsen.nukeeper.nemesis :as custom-nemesis] [jepsen.nukeeper.register :as register] [jepsen.nukeeper.unique :as unique] + [jepsen.nukeeper.queue :as queue] [jepsen.nukeeper.counter :as counter] [jepsen.nukeeper.constants :refer :all] [clojure.string :as str] @@ -23,7 +24,6 @@ [jepsen.os.ubuntu :as ubuntu] [jepsen.checker.timeline :as timeline] [clojure.java.io :as io] - [knossos.model :as model] [zookeeper.data :as data] [zookeeper :as zk]) (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) @@ -69,7 +69,7 @@ (info node "tearing down clickhouse") (cu/stop-daemon! (str binary-path "/clickhouse") pidfile) (c/su - ;(c/exec :rm :-f (str binary-path "/clickhouse")) + (c/exec :rm :-f (str binary-path "/clickhouse")) (c/exec :rm :-rf dir) (c/exec :rm :-rf logdir) (c/exec :rm :-rf "/etc/clickhouse-server"))) @@ -87,7 +87,8 @@ {"set" set/workload "register" register/workload "unique-ids" unique/workload - "counter" counter/workload}) + "counter" counter/workload + "queue" queue/workload}) (def cli-opts "Additional command line options." @@ -130,7 +131,7 @@ opts {:name (str "clickhouse-keeper quorum=" quorum " " (name (:workload opts)) " " (name (:nemesis opts))) :os ubuntu/os - :db (db "rbtorrent:a122093aee0bdcb70ca42d5e5fb4ba5544372f5f") + :db (db "rbtorrent:711cf0ff9281804eb53875d0c12499df1c2a0adc") :pure-generators true :client (:client workload) :nemesis (:nemesis current-nemesis) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj index bf22f9ad1f6..59f3cb52dae 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj @@ -111,13 +111,13 @@ {"random-node-killer" {:nemesis (random-node-killer-nemesis) :generator (start-stop-generator 5 5)} "all-nodes-killer" {:nemesis (all-nodes-killer-nemesis) - :generator (start-stop-generator 1 10)} + :generator (start-stop-generator 1 10)} "simple-partitioner" {:nemesis (nemesis/partition-random-halves) :generator (start-stop-generator 5 5)} "random-node-hammer-time" {:nemesis (random-node-hammer-time-nemesis) - :generator (start-stop-generator 5 5)} + :generator (start-stop-generator 5 5)} "all-nodes-hammer-time" {:nemesis (all-nodes-hammer-time-nemesis) - :generator (start-stop-generator 1 10)} + :generator (start-stop-generator 1 10)} "logs-corruptor" {:nemesis (logs-corruption-nemesis) :generator (corruption-generator)} "snapshots-corruptor" {:nemesis (snapshots-corruption-nemesis) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj new file mode 100644 index 00000000000..f6f7abb51b6 --- /dev/null +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj @@ -0,0 +1,67 @@ +(ns jepsen.nukeeper.queue + (:require + [clojure.tools.logging :refer :all] + [jepsen + [checker :as checker] + [client :as client] + [generator :as gen]] + [jepsen.nukeeper.utils :refer :all] + [zookeeper :as zk]) + (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) + +(defn enqueue [val _ _] {:type :invoke, :f :enqueue :value val}) +(defn dequeue [_ _] {:type :invoke, :f :dequeue}) + +(defrecord QueueClient [conn nodename] + client/Client + (open! [this test node] + (assoc + (assoc this + :conn (zk-connect node 9181 30000)) + :nodename node)) + + (setup! [this test]) + + (invoke! [this test op] + (case (:f op) + :enqueue (try + (do + (zk-create-if-not-exists conn (str "/" (:value op)) "") + (assoc op :type :ok)) + (catch Exception _ (assoc op :type :info, :error :connect-error))) + :dequeue + (try + (let [result (zk-multi-delete-first-child conn "/")] + (if (not (nil? result)) + (assoc op :type :ok :value result) + (assoc op :type :fail :value result))) + (catch KeeperException$BadVersionException _ (assoc op :type :fail, :error :bad-version)) + (catch Exception _ (assoc op :type :info, :error :connect-error))) + :drain + (try + (loop [result '()] + (let [deleted-child (zk-multi-delete-first-child conn "/")] + (if (not (nil? deleted-child)) + (recur (concat result [deleted-child])) + (assoc op :type :ok :value result)))) + (catch Exception _ (assoc op :type :info, :error :connect-error))))) + + (teardown! [_ test]) + + (close! [_ test] + (zk/close conn))) + +(defn sorted-str-range + [n] + (sort (map (fn [v] (str v)) (take n (range))))) + +(defn workload + "A generator, client, and checker for a set test." + [opts] + {:client (QueueClient. nil nil) + :checker (checker/total-queue) + :generator (->> (sorted-str-range 10000) + (map (fn [x] + (rand-nth [{:type :invoke, :f :enqueue :value x} + {:type :invoke, :f :dequeue}])))) + :final-generator (gen/once {:type :invoke, :f :drain, :value nil})}) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj index c30ec9635a1..3213042a3cc 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj @@ -23,8 +23,8 @@ (invoke! [this test op] (case (:f op) :read (assoc op - :type :ok - :value (read-string (:data (zk-get-str conn k)))) + :type :ok + :value (read-string (:data (zk-get-str conn k)))) :add (try (do (zk-add-to-set conn k (:value op)) @@ -34,7 +34,8 @@ (teardown! [_ test]) - (close! [_ test])) + (close! [_ test] + (zk/close conn))) (defn workload "A generator, client, and checker for a set test." diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/unique.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/unique.clj index 9c753dfe0ab..9dfb906bc17 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/unique.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/unique.clj @@ -9,10 +9,6 @@ [zookeeper :as zk]) (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) -(defn parse-and-get-counter - [path] - (Integer/parseInt (apply str (take-last 10 (seq (str path)))))) - (defrecord UniqueClient [conn nodename] client/Client (open! [this test node] @@ -33,7 +29,8 @@ (teardown! [_ test]) - (close! [_ test])) + (close! [_ test] + (zk/close conn))) (defn workload "A generator, client, and checker for a set test." diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj index 6fd2f3c87f4..fd2b2b5acb3 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj @@ -7,8 +7,9 @@ [jepsen.nukeeper.constants :refer :all] [jepsen.control :as c] [clojure.tools.logging :refer :all]) - (:import (org.apache.zookeeper CreateMode - ZooKeeper))) + (:import (org.apache.zookeeper.data Stat) + (org.apache.zookeeper CreateMode + ZooKeeper))) (defn parse-long "Parses a string to a Long. Passes through `nil` and empty strings." @@ -16,6 +17,10 @@ (if (and s (> (count s) 0)) (Long/parseLong s))) +(defn parse-and-get-counter + [path] + (Integer/parseInt (apply str (take-last 10 (seq (str path)))))) + (defn zk-range [] (map (fn [v] (str "/" v)) (range))) @@ -48,6 +53,13 @@ [conn path] (zk/children conn path)) +(defn zk-list-with-stat + [conn path] + (let [stat (new Stat) + children (seq (.getChildren conn path false stat))] + {:children children + :stat (zi/stat-to-map stat)})) + (defn zk-cas [conn path old-value new-value] (let [current-value (zk-get-str conn path)] @@ -81,6 +93,26 @@ CreateMode/PERSISTENT_SEQUENTIAL) (recur (inc i))))))) +(defn zk-parent-path + [path] + (let [rslash_pos (str/last-index-of path "/")] + (if (> rslash_pos 0) + (subs path 0 rslash_pos) + "/"))) + +(defn zk-multi-delete-first-child + [conn path] + (let [{children :children stat :stat} (zk-list-with-stat conn path) + txn (.transaction conn) + first-child (first (sort children))] + (if (not (nil? first-child)) + (do (.check txn path (:version stat)) + (.setData txn path (data/to-bytes "") -1) ; I'm just checking multitransactions + (.delete txn (str path first-child) -1) + (.commit txn) + first-child) + nil))) + (defn clickhouse-alive? [node test] (info "Checking server alive on" node) diff --git a/tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj b/tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj index 1a3e8646574..1981e01ebcb 100644 --- a/tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj +++ b/tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj @@ -15,16 +15,21 @@ (deftest a-test (testing "nukeeper connection" (let [conn (zk/connect "localhost:9181" :timeout-msec 5000)] - (println (take 10 (zk-range))) - (multidelete conn) - (multicreate conn) - (zk/create-all conn "/0") - (zk/create conn "/0") + ;(println (take 10 (zk-range))) + ;(multidelete conn) + ;(multicreate conn) + ;(zk/create-all conn "/0") + ;(zk/create conn "/0") + ;(println (zk/children conn "/")) + ;(zk/set-data conn "/0" (data/to-bytes "777") -1) + (println (zk-parent-path "/sasds/dasda/das")) + (println (zk-parent-path "/sasds")) + (zk-multi-create-many-seq-nodes conn "/a-" 5) (println (zk/children conn "/")) - (zk/set-data conn "/0" (data/to-bytes "777") -1) - (zk-multi-create-many-seq-nodes conn "/seq-" 5) - (println (zk/children conn "/")) - (Thread/sleep 5000) - (println "VALUE" (data/to-string (:data (zk/data conn "/0")))) - (is (= (data/to-string (:data (zk/data conn "/0"))) "777")) + (println (zk-list-with-stat conn "/")) + (println (zk-multi-delete-first-child conn "/")) + (println (zk-list-with-stat conn "/")) + ;(Thread/sleep 5000) + ;(println "VALUE" (data/to-string (:data (zk/data conn "/0")))) + ;(is (= (data/to-string (:data (zk/data conn "/0"))) "777")) (zk/close conn)))) From 26541471137806f701d7e8c24a9b00c298844cf2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 19 Mar 2021 00:14:43 +0300 Subject: [PATCH 17/43] Fix on fix --- src/Coordination/NuKeeperStorage.cpp | 2 ++ tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Coordination/NuKeeperStorage.cpp b/src/Coordination/NuKeeperStorage.cpp index 0b773aeaafd..62f998761ea 100644 --- a/src/Coordination/NuKeeperStorage.cpp +++ b/src/Coordination/NuKeeperStorage.cpp @@ -761,6 +761,8 @@ void NuKeeperStorage::clearData() session_and_timeout.clear(); session_id_counter = 1; zxid = 0; + + container.insert("/", Node()); } } diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index b8854638ed0..e852c7c4720 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -131,7 +131,7 @@ opts {:name (str "clickhouse-keeper quorum=" quorum " " (name (:workload opts)) " " (name (:nemesis opts))) :os ubuntu/os - :db (db "rbtorrent:711cf0ff9281804eb53875d0c12499df1c2a0adc") + :db (db "rbtorrent:af3f7a797953f7f359bd3550fe3fd4a68fd27345") :pure-generators true :client (:client workload) :nemesis (:nemesis current-nemesis) From 81c408cb7f8bd11a121906be33cec4b6e5770553 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 19 Mar 2021 11:08:43 +0300 Subject: [PATCH 18/43] Return meta and storage from snapshot --- src/Coordination/NuKeeperSnapshotManager.cpp | 18 ++++--- src/Coordination/NuKeeperSnapshotManager.h | 10 ++-- src/Coordination/NuKeeperStateMachine.cpp | 26 +++++----- src/Coordination/NuKeeperStateMachine.h | 4 +- src/Coordination/NuKeeperStorage.cpp | 13 ----- src/Coordination/NuKeeperStorage.h | 2 - src/Coordination/tests/gtest_for_build.cpp | 52 +++++++++---------- .../src/jepsen/nukeeper/main.clj | 4 +- 8 files changed, 59 insertions(+), 70 deletions(-) diff --git a/src/Coordination/NuKeeperSnapshotManager.cpp b/src/Coordination/NuKeeperSnapshotManager.cpp index 5cc7bc356be..1caa1ea94b8 100644 --- a/src/Coordination/NuKeeperSnapshotManager.cpp +++ b/src/Coordination/NuKeeperSnapshotManager.cpp @@ -161,7 +161,6 @@ void NuKeeperStorageSnapshot::serialize(const NuKeeperStorageSnapshot & snapshot SnapshotMetadataPtr NuKeeperStorageSnapshot::deserialize(NuKeeperStorage & storage, ReadBuffer & in) { - storage.clearData(); uint8_t version; readBinary(version, in); if (static_cast(version) > SnapshotVersion::V0) @@ -242,9 +241,10 @@ NuKeeperStorageSnapshot::~NuKeeperStorageSnapshot() storage->disableSnapshotMode(); } -NuKeeperSnapshotManager::NuKeeperSnapshotManager(const std::string & snapshots_path_, size_t snapshots_to_keep_) +NuKeeperSnapshotManager::NuKeeperSnapshotManager(const std::string & snapshots_path_, size_t snapshots_to_keep_, size_t storage_tick_time_) : snapshots_path(snapshots_path_) , snapshots_to_keep(snapshots_to_keep_) + , storage_tick_time(storage_tick_time_) { namespace fs = std::filesystem; @@ -326,22 +326,24 @@ nuraft::ptr NuKeeperSnapshotManager::serializeSnapshotToBuffer(c return writer.getBuffer(); } -SnapshotMetadataPtr NuKeeperSnapshotManager::deserializeSnapshotFromBuffer(NuKeeperStorage * storage, nuraft::ptr buffer) +SnapshotMetaAndStorage NuKeeperSnapshotManager::deserializeSnapshotFromBuffer(nuraft::ptr buffer) const { ReadBufferFromNuraftBuffer reader(buffer); CompressedReadBuffer compressed_reader(reader); - return NuKeeperStorageSnapshot::deserialize(*storage, compressed_reader); + auto storage = std::make_unique(storage_tick_time); + auto snapshot_metadata = NuKeeperStorageSnapshot::deserialize(*storage, compressed_reader); + return std::make_pair(snapshot_metadata, std::move(storage)); } -SnapshotMetadataPtr NuKeeperSnapshotManager::restoreFromLatestSnapshot(NuKeeperStorage * storage) +SnapshotMetaAndStorage NuKeeperSnapshotManager::restoreFromLatestSnapshot() { if (existing_snapshots.empty()) - return nullptr; + return {}; auto buffer = deserializeLatestSnapshotBufferFromDisk(); if (!buffer) - return nullptr; - return deserializeSnapshotFromBuffer(storage, buffer); + return {}; + return deserializeSnapshotFromBuffer(buffer); } void NuKeeperSnapshotManager::removeOutdatedSnapshotsIfNeeded() diff --git a/src/Coordination/NuKeeperSnapshotManager.h b/src/Coordination/NuKeeperSnapshotManager.h index 422baf11a65..d844a52eaf4 100644 --- a/src/Coordination/NuKeeperSnapshotManager.h +++ b/src/Coordination/NuKeeperSnapshotManager.h @@ -40,17 +40,20 @@ public: using NuKeeperStorageSnapshotPtr = std::shared_ptr; using CreateSnapshotCallback = std::function; + +using SnapshotMetaAndStorage = std::pair; + class NuKeeperSnapshotManager { public: - NuKeeperSnapshotManager(const std::string & snapshots_path_, size_t snapshots_to_keep_); + NuKeeperSnapshotManager(const std::string & snapshots_path_, size_t snapshots_to_keep_, size_t storage_tick_time_ = 500); - SnapshotMetadataPtr restoreFromLatestSnapshot(NuKeeperStorage * storage); + SnapshotMetaAndStorage restoreFromLatestSnapshot(); static nuraft::ptr serializeSnapshotToBuffer(const NuKeeperStorageSnapshot & snapshot); std::string serializeSnapshotBufferToDisk(nuraft::buffer & buffer, size_t up_to_log_idx); - static SnapshotMetadataPtr deserializeSnapshotFromBuffer(NuKeeperStorage * storage, nuraft::ptr buffer); + SnapshotMetaAndStorage deserializeSnapshotFromBuffer(nuraft::ptr buffer) const; nuraft::ptr deserializeSnapshotBufferFromDisk(size_t up_to_log_idx) const; nuraft::ptr deserializeLatestSnapshotBufferFromDisk(); @@ -74,6 +77,7 @@ private: const std::string snapshots_path; const size_t snapshots_to_keep; std::map existing_snapshots; + size_t storage_tick_time; }; struct CreateSnapshotTask diff --git a/src/Coordination/NuKeeperStateMachine.cpp b/src/Coordination/NuKeeperStateMachine.cpp index 58a7ca3d5bc..32bb4269f20 100644 --- a/src/Coordination/NuKeeperStateMachine.cpp +++ b/src/Coordination/NuKeeperStateMachine.cpp @@ -37,8 +37,7 @@ NuKeeperStorage::RequestForSession parseRequest(nuraft::buffer & data) NuKeeperStateMachine::NuKeeperStateMachine(ResponsesQueue & responses_queue_, SnapshotsQueue & snapshots_queue_, const std::string & snapshots_path_, const CoordinationSettingsPtr & coordination_settings_) : coordination_settings(coordination_settings_) - , storage(coordination_settings->dead_session_check_period_ms.totalMilliseconds()) - , snapshot_manager(snapshots_path_, coordination_settings->snapshots_to_keep) + , snapshot_manager(snapshots_path_, coordination_settings->snapshots_to_keep, coordination_settings->dead_session_check_period_ms.totalMicroseconds()) , responses_queue(responses_queue_) , snapshots_queue(snapshots_queue_) , last_committed_idx(0) @@ -60,7 +59,7 @@ void NuKeeperStateMachine::init() try { latest_snapshot_buf = snapshot_manager.deserializeSnapshotBufferFromDisk(latest_log_index); - latest_snapshot_meta = snapshot_manager.deserializeSnapshotFromBuffer(&storage, latest_snapshot_buf); + std::tie(latest_snapshot_meta, storage) = snapshot_manager.deserializeSnapshotFromBuffer(latest_snapshot_buf); last_committed_idx = latest_snapshot_meta->get_last_log_idx(); loaded = true; break; @@ -83,6 +82,9 @@ void NuKeeperStateMachine::init() { LOG_DEBUG(log, "No existing snapshots, last committed log index {}", last_committed_idx); } + + if (!storage) + storage = std::make_unique(coordination_settings->dead_session_check_period_ms.totalMilliseconds()); } nuraft::ptr NuKeeperStateMachine::commit(const size_t log_idx, nuraft::buffer & data) @@ -96,7 +98,7 @@ nuraft::ptr NuKeeperStateMachine::commit(const size_t log_idx, n nuraft::buffer_serializer bs(response); { std::lock_guard lock(storage_lock); - session_id = storage.getSessionID(session_timeout_ms); + session_id = storage->getSessionID(session_timeout_ms); bs.put_i64(session_id); } LOG_DEBUG(log, "Session ID response {} with timeout {}", session_id, session_timeout_ms); @@ -109,7 +111,7 @@ nuraft::ptr NuKeeperStateMachine::commit(const size_t log_idx, n NuKeeperStorage::ResponsesForSessions responses_for_sessions; { std::lock_guard lock(storage_lock); - responses_for_sessions = storage.processRequest(request_for_session.request, request_for_session.session_id, log_idx); + responses_for_sessions = storage->processRequest(request_for_session.request, request_for_session.session_id, log_idx); for (auto & response_for_session : responses_for_sessions) responses_queue.push(response_for_session); } @@ -133,7 +135,7 @@ bool NuKeeperStateMachine::apply_snapshot(nuraft::snapshot & s) { std::lock_guard lock(storage_lock); - snapshot_manager.deserializeSnapshotFromBuffer(&storage, latest_snapshot_ptr); + std::tie(latest_snapshot_meta, storage) = snapshot_manager.deserializeSnapshotFromBuffer(latest_snapshot_ptr); } last_committed_idx = s.get_last_log_idx(); return true; @@ -157,7 +159,7 @@ void NuKeeperStateMachine::create_snapshot( CreateSnapshotTask snapshot_task; { std::lock_guard lock(storage_lock); - snapshot_task.snapshot = std::make_shared(&storage, snapshot_meta_copy); + snapshot_task.snapshot = std::make_shared(storage.get(), snapshot_meta_copy); } snapshot_task.create_snapshot = [this, when_done] (NuKeeperStorageSnapshotPtr && snapshot) @@ -179,7 +181,7 @@ void NuKeeperStateMachine::create_snapshot( { /// Must do it with lock (clearing elements from list) std::lock_guard lock(storage_lock); - storage.clearGarbageAfterSnapshot(); + storage->clearGarbageAfterSnapshot(); /// Destroy snapshot with lock snapshot.reset(); LOG_TRACE(log, "Cleared garbage after snapshot"); @@ -214,7 +216,7 @@ void NuKeeperStateMachine::save_logical_snp_obj( if (obj_id == 0) { std::lock_guard lock(storage_lock); - NuKeeperStorageSnapshot snapshot(&storage, s.get_last_log_idx()); + NuKeeperStorageSnapshot snapshot(storage.get(), s.get_last_log_idx()); cloned_buffer = snapshot_manager.serializeSnapshotToBuffer(snapshot); } else @@ -271,7 +273,7 @@ void NuKeeperStateMachine::processReadRequest(const NuKeeperStorage::RequestForS NuKeeperStorage::ResponsesForSessions responses; { std::lock_guard lock(storage_lock); - responses = storage.processRequest(request_for_session.request, request_for_session.session_id, std::nullopt); + responses = storage->processRequest(request_for_session.request, request_for_session.session_id, std::nullopt); } for (const auto & response : responses) responses_queue.push(response); @@ -280,13 +282,13 @@ void NuKeeperStateMachine::processReadRequest(const NuKeeperStorage::RequestForS std::unordered_set NuKeeperStateMachine::getDeadSessions() { std::lock_guard lock(storage_lock); - return storage.getDeadSessions(); + return storage->getDeadSessions(); } void NuKeeperStateMachine::shutdownStorage() { std::lock_guard lock(storage_lock); - storage.finalize(); + storage->finalize(); } } diff --git a/src/Coordination/NuKeeperStateMachine.h b/src/Coordination/NuKeeperStateMachine.h index 905f3448c1a..af9ad6de4d2 100644 --- a/src/Coordination/NuKeeperStateMachine.h +++ b/src/Coordination/NuKeeperStateMachine.h @@ -52,7 +52,7 @@ public: NuKeeperStorage & getStorage() { - return storage; + return *storage; } void processReadRequest(const NuKeeperStorage::RequestForSession & request_for_session); @@ -68,7 +68,7 @@ private: CoordinationSettingsPtr coordination_settings; - NuKeeperStorage storage; + NuKeeperStoragePtr storage; NuKeeperSnapshotManager snapshot_manager; diff --git a/src/Coordination/NuKeeperStorage.cpp b/src/Coordination/NuKeeperStorage.cpp index 62f998761ea..2440d6f6613 100644 --- a/src/Coordination/NuKeeperStorage.cpp +++ b/src/Coordination/NuKeeperStorage.cpp @@ -752,17 +752,4 @@ void NuKeeperStorage::clearDeadWatches(int64_t session_id) } } -void NuKeeperStorage::clearData() -{ - container.clear(); - ephemerals.clear(); - sessions_and_watchers.clear(); - session_expiry_queue.clear(); - session_and_timeout.clear(); - session_id_counter = 1; - zxid = 0; - - container.insert("/", Node()); -} - } diff --git a/src/Coordination/NuKeeperStorage.h b/src/Coordination/NuKeeperStorage.h index b44a077c277..058eed55cab 100644 --- a/src/Coordination/NuKeeperStorage.h +++ b/src/Coordination/NuKeeperStorage.h @@ -82,8 +82,6 @@ public: public: NuKeeperStorage(int64_t tick_time_ms); - void clearData(); - int64_t getSessionID(int64_t session_timeout_ms) { auto result = session_id_counter++; diff --git a/src/Coordination/tests/gtest_for_build.cpp b/src/Coordination/tests/gtest_for_build.cpp index 01146248f63..d90b711498e 100644 --- a/src/Coordination/tests/gtest_for_build.cpp +++ b/src/Coordination/tests/gtest_for_build.cpp @@ -897,25 +897,25 @@ TEST(CoordinationTest, TestStorageSnapshotSimple) manager.serializeSnapshotBufferToDisk(*buf, 2); EXPECT_TRUE(fs::exists("./snapshots/snapshot_2.bin")); - DB::NuKeeperStorage restored_storage(500); auto debuf = manager.deserializeSnapshotBufferFromDisk(2); - manager.deserializeSnapshotFromBuffer(&restored_storage, debuf); - EXPECT_EQ(restored_storage.container.size(), 3); - EXPECT_EQ(restored_storage.container.getValue("/").children.size(), 1); - EXPECT_EQ(restored_storage.container.getValue("/hello").children.size(), 1); - EXPECT_EQ(restored_storage.container.getValue("/hello/somepath").children.size(), 0); + auto [snapshot_meta, restored_storage] = manager.deserializeSnapshotFromBuffer(debuf); - EXPECT_EQ(restored_storage.container.getValue("/").data, ""); - EXPECT_EQ(restored_storage.container.getValue("/hello").data, "world"); - EXPECT_EQ(restored_storage.container.getValue("/hello/somepath").data, "somedata"); - EXPECT_EQ(restored_storage.session_id_counter, 7); - EXPECT_EQ(restored_storage.zxid, 2); - EXPECT_EQ(restored_storage.ephemerals.size(), 2); - EXPECT_EQ(restored_storage.ephemerals[3].size(), 1); - EXPECT_EQ(restored_storage.ephemerals[1].size(), 1); - EXPECT_EQ(restored_storage.session_and_timeout.size(), 2); + EXPECT_EQ(restored_storage->container.size(), 3); + EXPECT_EQ(restored_storage->container.getValue("/").children.size(), 1); + EXPECT_EQ(restored_storage->container.getValue("/hello").children.size(), 1); + EXPECT_EQ(restored_storage->container.getValue("/hello/somepath").children.size(), 0); + + EXPECT_EQ(restored_storage->container.getValue("/").data, ""); + EXPECT_EQ(restored_storage->container.getValue("/hello").data, "world"); + EXPECT_EQ(restored_storage->container.getValue("/hello/somepath").data, "somedata"); + EXPECT_EQ(restored_storage->session_id_counter, 7); + EXPECT_EQ(restored_storage->zxid, 2); + EXPECT_EQ(restored_storage->ephemerals.size(), 2); + EXPECT_EQ(restored_storage->ephemerals[3].size(), 1); + EXPECT_EQ(restored_storage->ephemerals[1].size(), 1); + EXPECT_EQ(restored_storage->session_and_timeout.size(), 2); } TEST(CoordinationTest, TestStorageSnapshotMoreWrites) @@ -946,15 +946,14 @@ TEST(CoordinationTest, TestStorageSnapshotMoreWrites) manager.serializeSnapshotBufferToDisk(*buf, 50); EXPECT_TRUE(fs::exists("./snapshots/snapshot_50.bin")); - DB::NuKeeperStorage restored_storage(500); auto debuf = manager.deserializeSnapshotBufferFromDisk(50); - manager.deserializeSnapshotFromBuffer(&restored_storage, debuf); + auto [meta, restored_storage] = manager.deserializeSnapshotFromBuffer(debuf); - EXPECT_EQ(restored_storage.container.size(), 51); + EXPECT_EQ(restored_storage->container.size(), 51); for (size_t i = 0; i < 50; ++i) { - EXPECT_EQ(restored_storage.container.getValue("/hello_" + std::to_string(i)).data, "world_" + std::to_string(i)); + EXPECT_EQ(restored_storage->container.getValue("/hello_" + std::to_string(i)).data, "world_" + std::to_string(i)); } } @@ -987,14 +986,13 @@ TEST(CoordinationTest, TestStorageSnapshotManySnapshots) EXPECT_TRUE(fs::exists("./snapshots/snapshot_250.bin")); - DB::NuKeeperStorage restored_storage(500); - manager.restoreFromLatestSnapshot(&restored_storage); + auto [meta, restored_storage] = manager.restoreFromLatestSnapshot(); - EXPECT_EQ(restored_storage.container.size(), 251); + EXPECT_EQ(restored_storage->container.size(), 251); for (size_t i = 0; i < 250; ++i) { - EXPECT_EQ(restored_storage.container.getValue("/hello_" + std::to_string(i)).data, "world_" + std::to_string(i)); + EXPECT_EQ(restored_storage->container.getValue("/hello_" + std::to_string(i)).data, "world_" + std::to_string(i)); } } @@ -1040,12 +1038,11 @@ TEST(CoordinationTest, TestStorageSnapshotMode) EXPECT_FALSE(storage.container.contains("/hello_" + std::to_string(i))); } - DB::NuKeeperStorage restored_storage(500); - manager.restoreFromLatestSnapshot(&restored_storage); + auto [meta, restored_storage] = manager.restoreFromLatestSnapshot(); for (size_t i = 0; i < 50; ++i) { - EXPECT_EQ(restored_storage.container.getValue("/hello_" + std::to_string(i)).data, "world_" + std::to_string(i)); + EXPECT_EQ(restored_storage->container.getValue("/hello_" + std::to_string(i)).data, "world_" + std::to_string(i)); } } @@ -1071,8 +1068,7 @@ TEST(CoordinationTest, TestStorageSnapshotBroken) plain_buf.truncate(34); plain_buf.sync(); - DB::NuKeeperStorage restored_storage(500); - EXPECT_THROW(manager.restoreFromLatestSnapshot(&restored_storage), DB::Exception); + EXPECT_THROW(manager.restoreFromLatestSnapshot(), DB::Exception); } nuraft::ptr getBufferFromZKRequest(int64_t session_id, const Coordination::ZooKeeperRequestPtr & request) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index e852c7c4720..9ef3ab4ca2d 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -69,7 +69,7 @@ (info node "tearing down clickhouse") (cu/stop-daemon! (str binary-path "/clickhouse") pidfile) (c/su - (c/exec :rm :-f (str binary-path "/clickhouse")) + ;(c/exec :rm :-f (str binary-path "/clickhouse")) (c/exec :rm :-rf dir) (c/exec :rm :-rf logdir) (c/exec :rm :-rf "/etc/clickhouse-server"))) @@ -131,7 +131,7 @@ opts {:name (str "clickhouse-keeper quorum=" quorum " " (name (:workload opts)) " " (name (:nemesis opts))) :os ubuntu/os - :db (db "rbtorrent:af3f7a797953f7f359bd3550fe3fd4a68fd27345") + :db (db "rbtorrent:71c60699aa56568ded73c4a48cecd2fd5e0956cb") :pure-generators true :client (:client workload) :nemesis (:nemesis current-nemesis) From 58eac8a8b4d15699e2bc8d6784d66076fcb4c2d1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 19 Mar 2021 12:40:59 +0300 Subject: [PATCH 19/43] Add non-symmetric network partitioners --- .../src/jepsen/nukeeper/nemesis.clj | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj index 59f3cb52dae..9e5841ad8e4 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj @@ -93,6 +93,33 @@ (corruptor-nemesis coordinationdir (fn [path] (c/exec :rm :-fr path)))) +(defn partition-bridge-nemesis + [] + (nemesis/partitioner nemesis/bridge)) + +(defn blind-node + [nodes] + (let [[[victim] others] (nemesis/split-one nodes)] + {victim (into #{} others)})) + + +(defn blind-node-partition-nemesis + [] + (nemesis/partitioner blind-node)) + +(defn blind-others + [nodes] + (let [[[victim] others] (nemesis/split-one nodes)] + (into {} (map (fn [node] [node #{victim}])) others))) + +(defn blind-others-partition-nemesis + [] + (nemesis/partitioner blind-others)) + +(defn network-non-symmetric-nemesis + [] + (nemesis/partitioner nemesis/bridge)) + (defn start-stop-generator [time-corrupt time-ok] (->> @@ -125,4 +152,10 @@ "logs-and-snapshots-corruptor" {:nemesis (logs-and-snapshots-corruption-nemesis) :generator (corruption-generator)} "drop-data-corruptor" {:nemesis (drop-all-corruption-nemesis) - :generator (corruption-generator)}}) + :generator (corruption-generator)} + "bridge-partitioner" {:nemesis (partition-bridge-nemesis) + :generator (start-stop-generator 5 5)} + "blind-node-partitioner" {:nemesis (blind-node-partition-nemesis) + :generator (start-stop-generator 5 5)} + "blind-others-partitioner" {:nemesis (blind-others-partition-nemesis) + :generator (start-stop-generator 5 5)}}) From 260a978636cc4273a49739b8a786a0665652706b Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 19 Mar 2021 13:46:14 +0300 Subject: [PATCH 20/43] Check linearizeability for queue workload --- .../src/jepsen/nukeeper/main.clj | 3 ++- .../src/jepsen/nukeeper/queue.clj | 20 +++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index 9ef3ab4ca2d..0d93368595b 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -88,7 +88,8 @@ "register" register/workload "unique-ids" unique/workload "counter" counter/workload - "queue" queue/workload}) + "total-queue" queue/total-workload + "linear-queue" queue/linear-workload}) (def cli-opts "Additional command line options." diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj index f6f7abb51b6..fa6b96944b2 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj @@ -5,6 +5,8 @@ [checker :as checker] [client :as client] [generator :as gen]] + [knossos.model :as model] + [jepsen.checker.timeline :as timeline] [jepsen.nukeeper.utils :refer :all] [zookeeper :as zk]) (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) @@ -55,13 +57,27 @@ [n] (sort (map (fn [v] (str v)) (take n (range))))) -(defn workload +(defn total-workload "A generator, client, and checker for a set test." [opts] {:client (QueueClient. nil nil) - :checker (checker/total-queue) + :checker (checker/compose + {:total-queue (checker/total-queue) + :timeline (timeline/html)}) :generator (->> (sorted-str-range 10000) (map (fn [x] (rand-nth [{:type :invoke, :f :enqueue :value x} {:type :invoke, :f :dequeue}])))) :final-generator (gen/once {:type :invoke, :f :drain, :value nil})}) + +(defn linear-workload + [opts] + {:client (QueueClient. nil nil) + :checker (checker/compose + {:linear (checker/linearizable {:model (model/unordered-queue) + :algorithm :linear}) + :timeline (timeline/html)}) + :generator (->> (sorted-str-range 10000) + (map (fn [x] + (rand-nth [{:type :invoke, :f :enqueue :value x} + {:type :invoke, :f :dequeue}]))))}) From 2c00b48f858763bc0efef83489b12f3dea8f9841 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 19 Mar 2021 15:10:18 +0300 Subject: [PATCH 21/43] Add an ability to run N random tests --- .../src/jepsen/nukeeper/main.clj | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index 0d93368595b..86297473180 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -1,6 +1,7 @@ (ns jepsen.nukeeper.main (:require [clojure.tools.logging :refer :all] [jepsen.nukeeper.utils :refer :all] + [clojure.pprint :refer [pprint]] [jepsen.nukeeper.set :as set] [jepsen.nukeeper.nemesis :as custom-nemesis] [jepsen.nukeeper.register :as register] @@ -94,10 +95,10 @@ (def cli-opts "Additional command line options." [["-w" "--workload NAME" "What workload should we run?" - :missing (str "--workload " (cli/one-of workloads)) + :default "set" :validate [workloads (cli/one-of workloads)]] [nil "--nemesis NAME" "Which nemesis will poison our lives?" - :missing (str "--nemesis " (cli/one-of custom-nemesis/custom-nemesises)) + :default "random-node-killer" :validate [custom-nemesis/custom-nemesises (cli/one-of custom-nemesis/custom-nemesises)]] ["-q" "--quorum" "Use quorum reads, instead of reading from any primary."] ["-r" "--rate HZ" "Approximate number of requests per second, per thread." @@ -125,6 +126,7 @@ "Given an options map from the command line runner (e.g. :nodes, :ssh, :concurrency, ...), constructs a test map." [opts] + (info "Test opts\n" (with-out-str (pprint opts))) (let [quorum (boolean (:quorum opts)) workload ((get workloads (:workload opts)) opts) current-nemesis (get custom-nemesis/custom-nemesises (:nemesis opts))] @@ -150,11 +152,32 @@ (gen/sleep 10) (gen/clients (:final-generator workload)))}))) +(def all-nemesises (keys custom-nemesis/custom-nemesises)) + +(def all-workloads (keys workloads)) + +(defn all-test-options + "Takes base cli options, a collection of nemeses, workloads, and a test count, + and constructs a sequence of test options." + [cli nemeses workloads] + (take (:test-count cli) (shuffle (for [n nemeses, w workloads] + (assoc cli + :nemesis n + :workload w + :test-count 1))))) + +(defn all-tests + "Turns CLI options into a sequence of tests." + [test-fn cli] + (map test-fn (all-test-options cli all-nemesises all-workloads))) + (defn -main "Handles command line arguments. Can either run a test, or a web server for browsing results." [& args] (cli/run! (merge (cli/single-test-cmd {:test-fn nukeeper-test :opt-spec cli-opts}) + (cli/test-all-cmd {:tests-fn (partial all-tests nukeeper-test) + :opt-spec cli-opts}) (cli/serve-cmd)) args)) From 95cf05b0ad346b18f10b426723b58269e038c226 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 19 Mar 2021 15:25:44 +0300 Subject: [PATCH 22/43] Fix style and add sync --- .../jepsen.nukeeper/src/jepsen/nukeeper/counter.clj | 8 +++++--- tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj | 8 ++++---- .../jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj | 5 ++--- tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj | 12 +++++++----- tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj | 8 +++++--- tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj | 5 +++++ 6 files changed, 28 insertions(+), 18 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj index 48b270517a4..6f0cee113c6 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj @@ -25,9 +25,11 @@ (invoke! [this test op] (case (:f op) :read (try - (assoc op - :type :ok - :value (count (zk-list conn "/"))) + (do + (zk-sync conn) + (assoc op + :type :ok + :value (count (zk-list conn "/")))) (catch Exception _ (assoc op :type :fail, :error :connect-error))) :add (try (do diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index 86297473180..feca05d8190 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -161,10 +161,10 @@ and constructs a sequence of test options." [cli nemeses workloads] (take (:test-count cli) (shuffle (for [n nemeses, w workloads] - (assoc cli - :nemesis n - :workload w - :test-count 1))))) + (assoc cli + :nemesis n + :workload w + :test-count 1))))) (defn all-tests "Turns CLI options into a sequence of tests." diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj index 9e5841ad8e4..ec39c2b3e35 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj @@ -102,7 +102,6 @@ (let [[[victim] others] (nemesis/split-one nodes)] {victim (into #{} others)})) - (defn blind-node-partition-nemesis [] (nemesis/partitioner blind-node)) @@ -156,6 +155,6 @@ "bridge-partitioner" {:nemesis (partition-bridge-nemesis) :generator (start-stop-generator 5 5)} "blind-node-partitioner" {:nemesis (blind-node-partition-nemesis) - :generator (start-stop-generator 5 5)} + :generator (start-stop-generator 5 5)} "blind-others-partitioner" {:nemesis (blind-others-partition-nemesis) - :generator (start-stop-generator 5 5)}}) + :generator (start-stop-generator 5 5)}}) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj index fa6b96944b2..323d74acd67 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj @@ -41,11 +41,13 @@ (catch Exception _ (assoc op :type :info, :error :connect-error))) :drain (try - (loop [result '()] - (let [deleted-child (zk-multi-delete-first-child conn "/")] - (if (not (nil? deleted-child)) - (recur (concat result [deleted-child])) - (assoc op :type :ok :value result)))) + (do + (zk-sync conn) + (loop [result '()] + (let [deleted-child (zk-multi-delete-first-child conn "/")] + (if (not (nil? deleted-child)) + (recur (concat result [deleted-child])) + (assoc op :type :ok :value result))))) (catch Exception _ (assoc op :type :info, :error :connect-error))))) (teardown! [_ test]) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj index 3213042a3cc..23461591eaf 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj @@ -22,9 +22,11 @@ (invoke! [this test op] (case (:f op) - :read (assoc op - :type :ok - :value (read-string (:data (zk-get-str conn k)))) + :read (do + (zk-sync conn) + (assoc op + :type :ok + :value (read-string (:data (zk-get-str conn k))))) :add (try (do (zk-add-to-set conn k (:value op)) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj index fd2b2b5acb3..c7e46a75d5f 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj @@ -93,6 +93,11 @@ CreateMode/PERSISTENT_SEQUENTIAL) (recur (inc i))))))) +; sync call not implemented in zookeeper-clj and don't have sync version in java API +(defn zk-sync + [conn] + (zk-set conn "/" "" -1)) + (defn zk-parent-path [path] (let [rslash_pos (str/last-index-of path "/")] From 0bf897993236d68d979a37d59d2d1fa81c6ef394 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 19 Mar 2021 15:27:49 +0300 Subject: [PATCH 23/43] Remove redundant code from counter --- tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj index 6f0cee113c6..48b270517a4 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj @@ -25,11 +25,9 @@ (invoke! [this test op] (case (:f op) :read (try - (do - (zk-sync conn) - (assoc op - :type :ok - :value (count (zk-list conn "/")))) + (assoc op + :type :ok + :value (count (zk-list conn "/"))) (catch Exception _ (assoc op :type :fail, :error :connect-error))) :add (try (do From 3159b9dacf545134b1f85829d059123d5d71474a Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 19 Mar 2021 21:53:09 +0300 Subject: [PATCH 24/43] Disable zookeeper logger Better --- tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj | 6 +++++- tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index feca05d8190..b7f2bb0b98b 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -27,7 +27,9 @@ [clojure.java.io :as io] [zookeeper.data :as data] [zookeeper :as zk]) - (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) + (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException) + (ch.qos.logback.classic Level) + (org.slf4j Logger LoggerFactory))) (defn cluster-config [test node config-template] @@ -175,6 +177,8 @@ "Handles command line arguments. Can either run a test, or a web server for browsing results." [& args] + (.setLevel + (LoggerFactory/getLogger "org.apache.zookeeper") Level/OFF) (cli/run! (merge (cli/single-test-cmd {:test-fn nukeeper-test :opt-spec cli-opts}) (cli/test-all-cmd {:tests-fn (partial all-tests nukeeper-test) diff --git a/tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj b/tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj index 1981e01ebcb..db84ff33ee3 100644 --- a/tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj +++ b/tests/jepsen.nukeeper/test/jepsen/nukeeper_test.clj @@ -2,7 +2,9 @@ (:require [clojure.test :refer :all] [jepsen.nukeeper.utils :refer :all] [zookeeper :as zk] - [zookeeper.data :as data])) + [zookeeper.data :as data]) + (:import (ch.qos.logback.classic Level) + (org.slf4j Logger LoggerFactory))) (defn multicreate [conn] @@ -14,6 +16,8 @@ (deftest a-test (testing "nukeeper connection" + (.setLevel + (LoggerFactory/getLogger "org.apache.zookeeper") Level/OFF) (let [conn (zk/connect "localhost:9181" :timeout-msec 5000)] ;(println (take 10 (zk-range))) ;(multidelete conn) From 5ec7dbbdad1aa37d626bcc57ba5d8b324849650d Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 22 Mar 2021 13:06:09 +0300 Subject: [PATCH 25/43] Add lightweight run and fix queue workload --- .../resources/test_keeper_config.xml | 2 +- .../src/jepsen/nukeeper/main.clj | 36 ++++++++++++++----- .../src/jepsen/nukeeper/queue.clj | 10 ++---- .../src/jepsen/nukeeper/utils.clj | 20 +++++++---- 4 files changed, 46 insertions(+), 22 deletions(-) diff --git a/tests/jepsen.nukeeper/resources/test_keeper_config.xml b/tests/jepsen.nukeeper/resources/test_keeper_config.xml index 7ef34d4bea1..c69fb0f228c 100644 --- a/tests/jepsen.nukeeper/resources/test_keeper_config.xml +++ b/tests/jepsen.nukeeper/resources/test_keeper_config.xml @@ -7,7 +7,7 @@ 10000 30000 false - 60000 + 120000 trace {quorum_reads} {snapshot_distance} diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index b7f2bb0b98b..4e7c16930d4 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -122,7 +122,8 @@ [nil "--ops-per-key NUM" "Maximum number of operations on any given key." :default 100 :parse-fn parse-long - :validate [pos? "Must be a positive integer."]]]) + :validate [pos? "Must be a positive integer."]] + [nil, "--lightweight-run", "Subset of workloads/nemesises which is simple to validate"]]) (defn nukeeper-test "Given an options map from the command line runner (e.g. :nodes, :ssh, @@ -136,7 +137,7 @@ opts {:name (str "clickhouse-keeper quorum=" quorum " " (name (:workload opts)) " " (name (:nemesis opts))) :os ubuntu/os - :db (db "rbtorrent:71c60699aa56568ded73c4a48cecd2fd5e0956cb") + :db (db "rbtorrent:5fecc75309f38e302c95b4a226b2de60dfbb5681") :pure-generators true :client (:client workload) :nemesis (:nemesis current-nemesis) @@ -158,20 +159,39 @@ (def all-workloads (keys workloads)) +(def lightweight-workloads ["set" "unique-ids" "counter" "total-queue"]) + +(def useful-nemesises ["random-node-killer" + "simple-partitioner" + "logs-and-snapshots-corruptor" + "drop-data-corruptor" + "bridge-partitioner" + "blind-node-partitioner" + "blind-others-partitioner"]) + +(defn cart [colls] + (if (empty? colls) + '(()) + (for [more (cart (rest colls)) + x (first colls)] + (cons x more)))) + (defn all-test-options "Takes base cli options, a collection of nemeses, workloads, and a test count, and constructs a sequence of test options." - [cli nemeses workloads] - (take (:test-count cli) (shuffle (for [n nemeses, w workloads] + [cli worload-nemeseis-collection] + (take (:test-count cli) + (shuffle (for [[workload nemesis] worload-nemeseis-collection] (assoc cli - :nemesis n - :workload w + :nemesis nemesis + :workload workload :test-count 1))))) - (defn all-tests "Turns CLI options into a sequence of tests." [test-fn cli] - (map test-fn (all-test-options cli all-nemesises all-workloads))) + (if (boolean (:lightweight-run cli)) + (map test-fn (all-test-options cli (cart [all-workloads all-nemesises]))) + (map test-fn (all-test-options cli (cart [lightweight-workloads useful-nemesises]))))) (defn -main "Handles command line arguments. Can either run a test, or a web server for diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj index 323d74acd67..951c0822ad2 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj @@ -37,17 +37,13 @@ (if (not (nil? result)) (assoc op :type :ok :value result) (assoc op :type :fail :value result))) - (catch KeeperException$BadVersionException _ (assoc op :type :fail, :error :bad-version)) (catch Exception _ (assoc op :type :info, :error :connect-error))) :drain + ; drain via delete is to long, just list all nodes (try (do (zk-sync conn) - (loop [result '()] - (let [deleted-child (zk-multi-delete-first-child conn "/")] - (if (not (nil? deleted-child)) - (recur (concat result [deleted-child])) - (assoc op :type :ok :value result))))) + (assoc op :type :ok :value (into #{} (map #(str %1) (zk-list conn "/"))))) (catch Exception _ (assoc op :type :info, :error :connect-error))))) (teardown! [_ test]) @@ -66,7 +62,7 @@ :checker (checker/compose {:total-queue (checker/total-queue) :timeline (timeline/html)}) - :generator (->> (sorted-str-range 10000) + :generator (->> (sorted-str-range 50000) (map (fn [x] (rand-nth [{:type :invoke, :f :enqueue :value x} {:type :invoke, :f :dequeue}])))) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj index c7e46a75d5f..fe415ff9e51 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj @@ -9,7 +9,8 @@ [clojure.tools.logging :refer :all]) (:import (org.apache.zookeeper.data Stat) (org.apache.zookeeper CreateMode - ZooKeeper))) + ZooKeeper) + (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) (defn parse-long "Parses a string to a Long. Passes through `nil` and empty strings." @@ -111,11 +112,18 @@ txn (.transaction conn) first-child (first (sort children))] (if (not (nil? first-child)) - (do (.check txn path (:version stat)) - (.setData txn path (data/to-bytes "") -1) ; I'm just checking multitransactions - (.delete txn (str path first-child) -1) - (.commit txn) - first-child) + (try + (do (.check txn path (:version stat)) + (.setData txn path (data/to-bytes "") -1) ; I'm just checking multitransactions + (.delete txn (str path first-child) -1) + (.commit txn) + first-child) + (catch KeeperException$BadVersionException _ nil) + ; Even if we got connection loss, delete may actually be executed. + ; This function is used for queue model, which strictly require + ; all enqueued elements to be dequeued, but allow duplicates. + ; So even in case when we not sure about delete we return first-child. + (catch Exception _ first-child)) nil))) (defn clickhouse-alive? From 043b3cc7b589ec29ff03f3c3e005fd6c2718e05c Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 22 Mar 2021 13:45:22 +0300 Subject: [PATCH 26/43] Fix startup when leadership changed --- src/Coordination/CoordinationSettings.h | 1 + src/Coordination/NuKeeperServer.cpp | 13 +++++++++++++ tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/Coordination/CoordinationSettings.h b/src/Coordination/CoordinationSettings.h index c816f8089d5..45eb1348ac6 100644 --- a/src/Coordination/CoordinationSettings.h +++ b/src/Coordination/CoordinationSettings.h @@ -31,6 +31,7 @@ struct Settings; M(UInt64, rotate_log_storage_interval, 10000, "How many records will be stored in one log storage file", 0) \ M(UInt64, snapshots_to_keep, 3, "How many compressed snapshots to keep on disk", 0) \ M(UInt64, stale_log_gap, 10000, "When node became stale and should receive snapshots from leader", 0) \ + M(UInt64, fresh_log_gap, 200, "When node became fresh", 0) \ M(Bool, quorum_reads, false, "Execute read requests as writes through whole RAFT consesus with similar speed", 0) \ M(Bool, force_sync, true, " Call fsync on each change in RAFT changelog", 0) diff --git a/src/Coordination/NuKeeperServer.cpp b/src/Coordination/NuKeeperServer.cpp index 2081c969523..bfff7bf8f69 100644 --- a/src/Coordination/NuKeeperServer.cpp +++ b/src/Coordination/NuKeeperServer.cpp @@ -61,6 +61,7 @@ void NuKeeperServer::startup() params.reserved_log_items_ = coordination_settings->reserved_log_items; params.snapshot_distance_ = coordination_settings->snapshot_distance; params.stale_log_gap_ = coordination_settings->stale_log_gap; + params.fresh_log_gap_ = coordination_settings->fresh_log_gap; params.client_req_timeout_ = coordination_settings->operation_timeout_ms.totalMilliseconds(); params.auto_forwarding_ = coordination_settings->auto_forwarding; params.auto_forwarding_req_timeout_ = coordination_settings->operation_timeout_ms.totalMilliseconds() * 2; @@ -202,6 +203,18 @@ nuraft::cb_func::ReturnCode NuKeeperServer::callbackFunc(nuraft::cb_func::Type t set_initialized(); return nuraft::cb_func::ReturnCode::Ok; } + case nuraft::cb_func::BecomeFollower: + { + auto leader_index = raft_instance->get_leader_committed_log_idx(); + auto our_index = raft_instance->get_committed_log_idx(); + /// This may happen when we start RAFT claster from scratch. + /// Node first became leader, and after that some other node became leader. + /// BecameFresh for this node will not be called because it was already fresh + /// when it was leader. + if (isLeaderAlive() && leader_index < our_index + coordination_settings->fresh_log_gap) + set_initialized(); + return nuraft::cb_func::ReturnCode::Ok; + } case nuraft::cb_func::BecomeFresh: { set_initialized(); /// We are fresh follower, ready to serve requests. diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index 4e7c16930d4..5167da96c59 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -137,7 +137,7 @@ opts {:name (str "clickhouse-keeper quorum=" quorum " " (name (:workload opts)) " " (name (:nemesis opts))) :os ubuntu/os - :db (db "rbtorrent:5fecc75309f38e302c95b4a226b2de60dfbb5681") + :db (db "rbtorrent:156b85947eac9c85ef5d0ef15757a9f9e7c9e430") :pure-generators true :client (:client workload) :nemesis (:nemesis current-nemesis) From bf3a4361caaaa60696ed4fcc3a9d9978b4818503 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 22 Mar 2021 13:49:47 +0300 Subject: [PATCH 27/43] Followup fix --- tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index 5167da96c59..dfa1cfd913e 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -190,8 +190,8 @@ "Turns CLI options into a sequence of tests." [test-fn cli] (if (boolean (:lightweight-run cli)) - (map test-fn (all-test-options cli (cart [all-workloads all-nemesises]))) - (map test-fn (all-test-options cli (cart [lightweight-workloads useful-nemesises]))))) + (map test-fn (all-test-options cli (cart [lightweight-workloads useful-nemesises]))) + (map test-fn (all-test-options cli (cart [all-workloads all-nemesises]))))) (defn -main "Handles command line arguments. Can either run a test, or a web server for From 9845ff6694cea2786be7b88f9c90db307d399e7e Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 22 Mar 2021 23:03:51 +0300 Subject: [PATCH 28/43] Move db to separate file --- .../src/jepsen/nukeeper/constants.clj | 26 +++-- .../src/jepsen/nukeeper/db.clj | 99 +++++++++++++++++++ .../src/jepsen/nukeeper/main.clj | 57 +---------- .../src/jepsen/nukeeper/nemesis.clj | 8 +- .../src/jepsen/nukeeper/utils.clj | 26 +++-- 5 files changed, 138 insertions(+), 78 deletions(-) create mode 100644 tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj index 511ff8e3bf3..95b142e43f9 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj @@ -1,12 +1,18 @@ (ns jepsen.nukeeper.constants) -(def dir "/var/lib/clickhouse") -(def binary "clickhouse") -(def logdir "/var/log/clickhouse-server") -(def logfile "/var/log/clickhouse-server/stderr.log") -(def serverlog "/var/log/clickhouse-server/clickhouse-server.log") -(def snapshotsdir "/var/lib/clickhouse/coordination/snapshots") -(def coordinationdir "/var/lib/clickhouse/coordination") -(def logsdir "/var/lib/clickhouse/coordination/logs") -(def pidfile (str dir "/clickhouse.pid")) -(def binary-path "/tmp") +(def common-prefix "/tmp/clickhouse") + +(def binary-name "clickhouse") + +(def binary-path (str common-prefix "/" binary-name)) +(def pid-file-path (str common-prefix "/clickhouse.pid")) + +(def data-dir (str common-prefix "/db")) +(def logs-dir (str common-prefix "/logs")) +(def configs-dir (str common-prefix "/config")) +(def sub-configs-dir (str configs-dir "/config.d")) +(def coordination-data-dir (str data-dir "/coordination")) +(def coordination-snapshots-dir (str coordination-data-dir "/snapshots")) +(def coordination-logs-dir (str coordination-data-dir "/logs")) + +(def stderr-file (str logs-dir "/stderr.log")) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj new file mode 100644 index 00000000000..b4bcd363740 --- /dev/null +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj @@ -0,0 +1,99 @@ +(ns jepsen.nukeeper.db + (:require [clojure.tools.logging :refer :all] + [jepsen + [control :as c] + [db :as db] + [util :as util :refer [meh]]] + [jepsen.nukeeper.constants :refer :all] + [jepsen.nukeeper.utils :refer :all] + [clojure.java.io :as io] + [jepsen.control.util :as cu] + [jepsen.os.ubuntu :as ubuntu])) + +(defn get-clickhouse-sky + [version] + (c/exec :sky :get :-d common-prefix :-N :Backbone version)) + +(defn get-clickhouse-url + [url] + (let [download-result (cu/wget! url)] + (do (c/exec :mv download-result common-prefix) + (str common-prefix "/" download-result)))) + +(defn unpack-deb + [path] + (do + (c/exec :dpkg :-x path :.) + (c/exec :mv "usr/bin/clickhouse" common-prefix))) + +(defn unpack-tgz + [path] + (do + (c/exec :tar :-zxvf path :.) + (c/exec :mv "usr/bin/clickhouse" common-prefix))) + +(defn prepare-dirs + [] + (do + (c/exec :rm :-rf common-prefix) + (c/exec :mkdir :-p common-prefix) + (c/exec :mkdir :-p data-dir) + (c/exec :mkdir :-p logs-dir) + (c/exec :mkdir :-p configs-dir) + (c/exec :mkdir :-p sub-configs-dir) + (c/exec :touch stderr-file) + (c/exec :chown :-R :root common-prefix))) + +(defn cluster-config + [test node config-template] + (let [nodes (:nodes test) + replacement-map {#"\{srv1\}" (get nodes 0) + #"\{srv2\}" (get nodes 1) + #"\{srv3\}" (get nodes 2) + #"\{id\}" (str (inc (.indexOf nodes node))) + #"\{quorum_reads\}" (str (boolean (:quorum test))) + #"\{snapshot_distance\}" (str (:snapshot-distance test)) + #"\{stale_log_gap\}" (str (:stale-log-gap test)) + #"\{reserved_log_items\}" (str (:reserved-log-items test))}] + (reduce #(clojure.string/replace %1 (get %2 0) (get %2 1)) config-template replacement-map))) + +(defn install-configs + [test node] + (c/exec :echo (slurp (io/resource "config.xml")) :> (str configs-dir "/config.xml")) + (c/exec :echo (slurp (io/resource "users.xml")) :> (str configs-dir "/users.xml")) + (c/exec :echo (slurp (io/resource "listen.xml")) :> (str sub-configs-dir "/listen.xml")) + (c/exec :echo (cluster-config test node (slurp (io/resource "test_keeper_config.xml"))) :> (str sub-configs-dir "/test_keeper_config.xml"))) + +(defn db + [version] + (reify db/DB + (setup! [_ test node] + (c/su + (do + (info "Preparing directories") + (prepare-dirs) + (info "Downloading clickhouse") + (get-clickhouse-sky version) + (info "Installing configs") + (install-configs test node) + (info "Starting server") + (start-clickhouse! node test) + (info "ClickHouse started")))) + + + (teardown! [_ test node] + (info node "Tearing down clickhouse") + (kill-clickhouse! node test) + (c/su + ;(c/exec :rm :-f binary-path) + (c/exec :rm :-rf data-dir) + (c/exec :rm :-rf logs-dir) + (c/exec :rm :-rf configs-dir))) + + db/LogFiles + (log-files [_ test node] + (c/su + (kill-clickhouse! node test) + (c/cd data-dir + (c/exec :tar :czf "coordination.tar.gz" "coordination"))) + [stderr-file (str logs-dir "/clickhouse-server.log") (str data-dir "/coordination.tar.gz")]))) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index dfa1cfd913e..e027b956937 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -3,6 +3,7 @@ [jepsen.nukeeper.utils :refer :all] [clojure.pprint :refer [pprint]] [jepsen.nukeeper.set :as set] + [jepsen.nukeeper.db :refer :all] [jepsen.nukeeper.nemesis :as custom-nemesis] [jepsen.nukeeper.register :as register] [jepsen.nukeeper.unique :as unique] @@ -31,60 +32,6 @@ (ch.qos.logback.classic Level) (org.slf4j Logger LoggerFactory))) -(defn cluster-config - [test node config-template] - (let [nodes (:nodes test) - replacement-map {#"\{srv1\}" (get nodes 0) - #"\{srv2\}" (get nodes 1) - #"\{srv3\}" (get nodes 2) - #"\{id\}" (str (inc (.indexOf nodes node))) - #"\{quorum_reads\}" (str (boolean (:quorum test))) - #"\{snapshot_distance\}" (str (:snapshot-distance test)) - #"\{stale_log_gap\}" (str (:stale-log-gap test)) - #"\{reserved_log_items\}" (str (:reserved-log-items test))}] - (reduce #(clojure.string/replace %1 (get %2 0) (get %2 1)) config-template replacement-map))) - -(defn db - [version] - (reify db/DB - (setup! [_ test node] - (info node "installing clickhouse" version) - (c/su - (if-not (cu/exists? (str binary-path "/clickhouse")) - (c/exec :sky :get :-d binary-path :-N :Backbone version)) - (c/exec :mkdir :-p logdir) - (c/exec :touch logfile) - (c/exec (str binary-path "/clickhouse") :install) - (c/exec :chown :-R :root dir) - (c/exec :chown :-R :root logdir) - (c/exec :echo (slurp (io/resource "listen.xml")) :> "/etc/clickhouse-server/config.d/listen.xml") - (c/exec :echo (cluster-config test node (slurp (io/resource "test_keeper_config.xml"))) :> "/etc/clickhouse-server/config.d/test_keeper_config.xml") - (cu/start-daemon! - {:pidfile pidfile - :logfile logfile - :chdir dir} - (str binary-path "/clickhouse") - :server - :--config "/etc/clickhouse-server/config.xml") - (wait-clickhouse-alive! node test))) - - (teardown! [_ test node] - (info node "tearing down clickhouse") - (cu/stop-daemon! (str binary-path "/clickhouse") pidfile) - (c/su - ;(c/exec :rm :-f (str binary-path "/clickhouse")) - (c/exec :rm :-rf dir) - (c/exec :rm :-rf logdir) - (c/exec :rm :-rf "/etc/clickhouse-server"))) - - db/LogFiles - (log-files [_ test node] - (c/su - (cu/stop-daemon! (str binary-path "/clickhouse") pidfile) - (c/cd dir - (c/exec :tar :czf "coordination.tar.gz" "coordination"))) - [logfile serverlog (str dir "/coordination.tar.gz")]))) - (def workloads "A map of workload names to functions that construct workloads, given opts." {"set" set/workload @@ -137,7 +84,7 @@ opts {:name (str "clickhouse-keeper quorum=" quorum " " (name (:workload opts)) " " (name (:nemesis opts))) :os ubuntu/os - :db (db "rbtorrent:156b85947eac9c85ef5d0ef15757a9f9e7c9e430") + :db (db "rbtorrent:a284492c715974b69f73add62b4ff590110369af") :pure-generators true :client (:client workload) :nemesis (:nemesis current-nemesis) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj index ec39c2b3e35..8314d29f575 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj @@ -76,21 +76,21 @@ (defn logs-corruption-nemesis [] - (corruptor-nemesis logsdir #(corrupt-file (select-last-file %1)))) + (corruptor-nemesis coordination-logs-dir #(corrupt-file (select-last-file %1)))) (defn snapshots-corruption-nemesis [] - (corruptor-nemesis snapshotsdir #(corrupt-file (select-last-file %1)))) + (corruptor-nemesis coordination-snapshots-dir #(corrupt-file (select-last-file %1)))) (defn logs-and-snapshots-corruption-nemesis [] - (corruptor-nemesis coordinationdir (fn [path] + (corruptor-nemesis coordination-data-dir (fn [path] (do (corrupt-file (select-last-file (str path "/snapshots"))) (corrupt-file (select-last-file (str path "/logs"))))))) (defn drop-all-corruption-nemesis [] - (corruptor-nemesis coordinationdir (fn [path] + (corruptor-nemesis coordination-data-dir (fn [path] (c/exec :rm :-fr path)))) (defn partition-bridge-nemesis diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj index fe415ff9e51..30774c24dae 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj @@ -130,7 +130,7 @@ [node test] (info "Checking server alive on" node) (try - (c/exec (str binary-path "/clickhouse") :client :--query "SELECT 1") + (c/exec binary-path :client :--query "SELECT 1") (catch Exception _ false))) (defn wait-clickhouse-alive! @@ -144,18 +144,26 @@ [node test] (info "Killing server on node" node) (c/su - (cu/stop-daemon! (str binary-path "/clickhouse") pidfile) - (c/exec :rm :-fr (str dir "/status")))) + (cu/stop-daemon! binary-path pid-file-path) + (c/exec :rm :-fr (str data-dir "/status")))) (defn start-clickhouse! [node test] (info "Starting server on node" node) (c/su (cu/start-daemon! - {:pidfile pidfile - :logfile logfile - :chdir dir} - (str binary-path "/clickhouse") + {:pidfile pid-file-path + :logfile stderr-file + :chdir data-dir} + binary-path :server - :--config "/etc/clickhouse-server/config.xml")) - (wait-clickhouse-alive! node test)) + :--config (str configs-dir "/config.xml") + :-- + :--path data-dir + :--user_files_path (str data-dir "/user_files") + :--top_level_domains_path (str data-dir "/top_level_domains") + :--logger.log (str logs-dir "/clickhouse-server.log") + :--logger.errorlog (str logs-dir "/clickhouse-server.err.log") + :--test_keeper_server.snapshot_storage_path coordination-snapshots-dir + :--test_keeper_server.logs_storage_path coordination-logs-dir) + (wait-clickhouse-alive! node test))) From cce2e0acaffaaf5c26b452d26455851b21acaab2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 23 Mar 2021 10:28:14 +0300 Subject: [PATCH 29/43] Fix typo --- utils/nukeeper-data-dumper/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/nukeeper-data-dumper/main.cpp b/utils/nukeeper-data-dumper/main.cpp index 0340c94c5a0..c80aeb473e2 100644 --- a/utils/nukeeper-data-dumper/main.cpp +++ b/utils/nukeeper-data-dumper/main.cpp @@ -66,7 +66,7 @@ int main(int argc, char *argv[]) state_machine->init(); size_t last_commited_index = state_machine->last_commit_index(); - LOG_INFO(logger, "Last commited index: {}", last_commited_index); + LOG_INFO(logger, "Last committed index: {}", last_commited_index); DB::NuKeeperLogStore changelog(argv[2], 10000000, true); changelog.init(last_commited_index, 10000000000UL); /// collect all logs From 0c525b4ec4fcd3f6225333f7c9a9f7ecea5abdbb Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 23 Mar 2021 15:07:21 +0300 Subject: [PATCH 30/43] Add an ability to run from .deb and .tgz package --- src/Coordination/NuKeeperServer.cpp | 4 +- src/Coordination/NuKeeperServer.h | 1 + .../src/jepsen/nukeeper/constants.clj | 2 +- .../src/jepsen/nukeeper/db.clj | 47 +++++++++++++++---- .../src/jepsen/nukeeper/main.clj | 8 ++-- .../src/jepsen/nukeeper/utils.clj | 2 +- 6 files changed, 49 insertions(+), 15 deletions(-) diff --git a/src/Coordination/NuKeeperServer.cpp b/src/Coordination/NuKeeperServer.cpp index bfff7bf8f69..62af9656fb9 100644 --- a/src/Coordination/NuKeeperServer.cpp +++ b/src/Coordination/NuKeeperServer.cpp @@ -199,7 +199,8 @@ nuraft::cb_func::ReturnCode NuKeeperServer::callbackFunc(nuraft::cb_func::Type t { case nuraft::cb_func::BecomeLeader: { - if (commited_store) /// We become leader and store is empty, ready to serve requests + /// We become leader and store is empty or we already committed it + if (commited_store || initial_batch_committed) set_initialized(); return nuraft::cb_func::ReturnCode::Ok; } @@ -224,6 +225,7 @@ nuraft::cb_func::ReturnCode NuKeeperServer::callbackFunc(nuraft::cb_func::Type t { if (isLeader()) /// We have committed our log store and we are leader, ready to serve requests. set_initialized(); + initial_batch_committed = true; return nuraft::cb_func::ReturnCode::Ok; } default: /// ignore other events diff --git a/src/Coordination/NuKeeperServer.h b/src/Coordination/NuKeeperServer.h index 17099045640..ba25d5c181b 100644 --- a/src/Coordination/NuKeeperServer.h +++ b/src/Coordination/NuKeeperServer.h @@ -33,6 +33,7 @@ private: std::mutex initialized_mutex; bool initialized_flag = false; std::condition_variable initialized_cv; + std::atomic initial_batch_committed = false; nuraft::cb_func::ReturnCode callbackFunc(nuraft::cb_func::Type type, nuraft::cb_func::Param * param); diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj index 95b142e43f9..d6245d450f5 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/constants.clj @@ -1,6 +1,6 @@ (ns jepsen.nukeeper.constants) -(def common-prefix "/tmp/clickhouse") +(def common-prefix "/home/robot-clickhouse") (def binary-name "clickhouse") diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj index b4bcd363740..106af25be17 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj @@ -12,7 +12,8 @@ (defn get-clickhouse-sky [version] - (c/exec :sky :get :-d common-prefix :-N :Backbone version)) + (c/exec :sky :get :-d common-prefix :-N :Backbone version) + (str common-prefix "/clickhouse")) (defn get-clickhouse-url [url] @@ -20,22 +21,47 @@ (do (c/exec :mv download-result common-prefix) (str common-prefix "/" download-result)))) +(defn download-clickhouse + [source] + (info "Downloading clickhouse from" source) + (cond + (clojure.string/starts-with? source "rbtorrent:") (get-clickhouse-sky source) + (clojure.string/starts-with? source "http") (get-clickhouse-url source) + :else (throw (Exception. (str "Don't know how to download clickhouse from" source))))) + (defn unpack-deb [path] (do - (c/exec :dpkg :-x path :.) - (c/exec :mv "usr/bin/clickhouse" common-prefix))) + (c/exec :dpkg :-x path common-prefix) + (c/exec :rm :-f path) + (c/exec :mv (str common-prefix "/usr/bin/clickhouse") common-prefix) + (c/exec :rm :-rf (str common-prefix "/usr") (str common-prefix "/etc")))) (defn unpack-tgz [path] (do - (c/exec :tar :-zxvf path :.) - (c/exec :mv "usr/bin/clickhouse" common-prefix))) + (c/exec :mkdir :-p (str common-prefix "/unpacked")) + (c/exec :tar :-zxvf path :-C (str common-prefix "/unpacked")) + (c/exec :rm :-f path) + (let [subdir (c/exec :ls (str common-prefix "/unpacked"))] + (c/exec :mv (str common-prefix "/unpacked/" subdir "/usr/bin/clickhouse") common-prefix) + (c/exec :rm :-fr (str common-prefix "/unpacked"))))) + +(defn chmod-binary + [path] + (c/exec :chmod :+x path)) + +(defn install-downloaded-clickhouse + [path] + (cond + (clojure.string/ends-with? path ".deb") (unpack-deb path) + (clojure.string/ends-with? path ".tgz") (unpack-tgz path) + (clojure.string/ends-with? path "clickhouse") (chmod-binary path) + :else (throw (Exception. (str "Don't know how to install clickhouse from path" path))))) (defn prepare-dirs [] (do - (c/exec :rm :-rf common-prefix) (c/exec :mkdir :-p common-prefix) (c/exec :mkdir :-p data-dir) (c/exec :mkdir :-p logs-dir) @@ -72,8 +98,10 @@ (do (info "Preparing directories") (prepare-dirs) - (info "Downloading clickhouse") - (get-clickhouse-sky version) + (if (not (cu/exists? binary-path)) + (do (info "Downloading clickhouse") + (install-downloaded-clickhouse (download-clickhouse version))) + (info "Binary already exsist on path" binary-path "skipping download")) (info "Installing configs") (install-configs test node) (info "Starting server") @@ -85,7 +113,8 @@ (info node "Tearing down clickhouse") (kill-clickhouse! node test) (c/su - ;(c/exec :rm :-f binary-path) + (c/exec :rm :-rf binary-path) + (c/exec :rm :-rf pid-file-path) (c/exec :rm :-rf data-dir) (c/exec :rm :-rf logs-dir) (c/exec :rm :-rf configs-dir))) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index e027b956937..f3db61c6d53 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -70,7 +70,9 @@ :default 100 :parse-fn parse-long :validate [pos? "Must be a positive integer."]] - [nil, "--lightweight-run", "Subset of workloads/nemesises which is simple to validate"]]) + [nil, "--lightweight-run" "Subset of workloads/nemesises which is simple to validate"] + ["-c" "--clickhouse-source URL" "URL for clickhouse deb or tgz package" + :default "https://clickhouse-builds.s3.yandex.net/21677/ef82333089156907a0979669d9374c2e18daabe5/clickhouse_build_check/clang-11_relwithdebuginfo_none_bundled_unsplitted_disable_False_deb/clickhouse-common-static_21.4.1.6313_amd64.deb"]]) (defn nukeeper-test "Given an options map from the command line runner (e.g. :nodes, :ssh, @@ -82,9 +84,9 @@ current-nemesis (get custom-nemesis/custom-nemesises (:nemesis opts))] (merge tests/noop-test opts - {:name (str "clickhouse-keeper quorum=" quorum " " (name (:workload opts)) " " (name (:nemesis opts))) + {:name (str "clickhouse-keeper-quorum=" quorum "-" (name (:workload opts)) "-" (name (:nemesis opts))) :os ubuntu/os - :db (db "rbtorrent:a284492c715974b69f73add62b4ff590110369af") + :db (db (:clickhouse-source opts)) :pure-generators true :client (:client workload) :nemesis (:nemesis current-nemesis) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj index 30774c24dae..0e0db2d3a6d 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj @@ -159,7 +159,7 @@ :server :--config (str configs-dir "/config.xml") :-- - :--path data-dir + :--path (str data-dir "/") :--user_files_path (str data-dir "/user_files") :--top_level_domains_path (str data-dir "/top_level_domains") :--logger.log (str logs-dir "/clickhouse-server.log") From 83255cbd64305910fd2a72b0f1d00fcb63ed5ea6 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 23 Mar 2021 15:19:37 +0300 Subject: [PATCH 31/43] Add option to reuse same binary --- tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj | 7 ++++--- tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj index 106af25be17..7bc2b9c6cea 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj @@ -91,14 +91,14 @@ (c/exec :echo (cluster-config test node (slurp (io/resource "test_keeper_config.xml"))) :> (str sub-configs-dir "/test_keeper_config.xml"))) (defn db - [version] + [version reuse-binary] (reify db/DB (setup! [_ test node] (c/su (do (info "Preparing directories") (prepare-dirs) - (if (not (cu/exists? binary-path)) + (if (or (not (cu/exists? binary-path)) (not reuse-binary)) (do (info "Downloading clickhouse") (install-downloaded-clickhouse (download-clickhouse version))) (info "Binary already exsist on path" binary-path "skipping download")) @@ -113,7 +113,8 @@ (info node "Tearing down clickhouse") (kill-clickhouse! node test) (c/su - (c/exec :rm :-rf binary-path) + (if (not reuse-binary) + (c/exec :rm :-rf binary-path)) (c/exec :rm :-rf pid-file-path) (c/exec :rm :-rf data-dir) (c/exec :rm :-rf logs-dir) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index f3db61c6d53..45a1f442d24 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -71,6 +71,7 @@ :parse-fn parse-long :validate [pos? "Must be a positive integer."]] [nil, "--lightweight-run" "Subset of workloads/nemesises which is simple to validate"] + [nil, "--reuse-binary" "Use already downloaded binary if it exists, don't remove it on shutdown"] ["-c" "--clickhouse-source URL" "URL for clickhouse deb or tgz package" :default "https://clickhouse-builds.s3.yandex.net/21677/ef82333089156907a0979669d9374c2e18daabe5/clickhouse_build_check/clang-11_relwithdebuginfo_none_bundled_unsplitted_disable_False_deb/clickhouse-common-static_21.4.1.6313_amd64.deb"]]) @@ -86,7 +87,7 @@ opts {:name (str "clickhouse-keeper-quorum=" quorum "-" (name (:workload opts)) "-" (name (:nemesis opts))) :os ubuntu/os - :db (db (:clickhouse-source opts)) + :db (db (:clickhouse-source opts) (boolean (:reuse-binary opts))) :pure-generators true :client (:client workload) :nemesis (:nemesis current-nemesis) From 4716791a1abf6522c9628d429497229479fba568 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 23 Mar 2021 19:06:13 +0300 Subject: [PATCH 32/43] Better README.md --- tests/jepsen.nukeeper/README.md | 137 +++++++++++++++++++++++++++++++- 1 file changed, 135 insertions(+), 2 deletions(-) diff --git a/tests/jepsen.nukeeper/README.md b/tests/jepsen.nukeeper/README.md index f72409e080f..6bcd7a37069 100644 --- a/tests/jepsen.nukeeper/README.md +++ b/tests/jepsen.nukeeper/README.md @@ -1,10 +1,143 @@ # jepsen.nukeeper -A Clojure library designed to ... well, that part is up to you. +A Clojure library designed to test ZooKeeper-like implementation inside ClickHouse. + +## Test scenarios (workloads) + +### CAS register + +CAS Register has three operations: read number, write number, compare-and-swap number. This register is simulated as a single ZooKeeper node. Read transforms to ZooKeeper's `getData` request. Write transforms to the `set` request. Compare-and-swap implemented via `getData` + compare in code + `set` new value with `version` from `getData`. + +In this test, we use a linearizable checker, so Jepsen validates that history was linearizable. One of the heaviest workloads. + +Strictly requires `quorum_reads` to be true. + +### Set + +Set has two operations: add a number to set and read all values from set. This workload is simulated on a single ZooKeeper node with a string value that represents Clojure set data structure. Add operation very similar to compare-and-swap. We read string value from ZooKeeper node with `getData`, parse it to Clojure's set, add new value to the set and try to write it with the received version. + +In this test, Jepsen validates that all successfully added values can be read. Generator for this workload performs only add operations until a timeout and after that tries to read set once. + +### Unique IDs + +In the Unique IDs workload we have only one operation: generate a new unique number. It's implemented using ZooKeeper's sequential nodes. For each generates request client just creates a new sequential node in ZooKeeper with a fixed prefix. After that cuts the prefix off from the returned path and parses the number from the rest part. + +Jepsen checks that all returned IDs were unique. + +### Counter + +Counter workload has two operations: read counter value and add some number to the counter. Its implementation is quite weird. We add number `N` to the counter creating `N` sequential nodes in a single ZooKeeper transaction. Counter read implemented as `getChildren` ZooKeeper request and count of all returned nodes. + +Jepsen checks that counter value lies in the interval of possible value. Strictly requires `quorum_reads` to be true. + +### Total queue + +Simulates an unordered queue with three operations: enqueue number, dequeue, and drain. Enqueue operation uses `create` request with node name equals to number. `Dequeue` operation is more interesting. We list (`getChildren`) all nodes and remember the parent node version. After that we choose the smallest one and prepare the transaction: `check` parent node version + set an empty value to parent node + delete smalled child node. Drain operation is just `getChildren` on the parent path. + +Jepsen checks that all enqueued values were dequeued or drained. Duplicates are allowed because Jepsen doesn't know the value of the unknown-status (`:info`) dequeue operation. So when we try to `dequeue` some element we should return it even if our delete transaction failed with `Connection loss` error. + +### Linear queue + +Same with the total queue, but without drain operation. Checks linearizability between enqueue and dequeue. Sometimes consume more than 10GB during validation even for very short histories. + + +## Nemesis + +We use almost all standard nemeses with small changes for our storage. + +### Random node killer (random-node-killer) + +Sleep 5 seconds, kills random node, sleep for 5 seconds, and starts it back. + +### All nodes killer (all-nodes-killer) + +Kill all nodes at once, sleep for 5 seconds, and starts them back. + +### Simple partitioner (simple-partitioner) + +Partition one node from others using iptables. No one can see the victim and the victim cannot see anybody. + +### Random node stop (random-node-hammer-time) + +Send `SIGSTOP` to the random node. Sleep 5 seconds. Send `SIGCONT`. + +### All nodes stop (all-nodes-hammer-time) + +Send `SIGSTOP` to all nodes. Sleep 5 seconds. Send `SIGCONT`. + +### Logs corruptor (logs-corruptor) + +Corrupts latest log (change one random byte) in `clickhouse_path/coordination/logs`. Restarts nodes. + +### Snapshots corruptor (snapshots-corruptor) + +Corrupts latest snapshot (change one random byte) in `clickhouse_path/coordination/snapshots`. Restarts nodes. + +### Logs and snapshots corruptor (logs-and-snapshots-corruptor) + +Corrupts both the latest log and snapshot. Restarts node. + +### Drop data corruptor (drop-data-corruptor) + +Drop all data from `clickhouse_path/coordinator`. Restarts node. + +### Bridge partitioner (bridge-partitioner) + +Two nodes don't see each other but can see another node. The last node can see both. + +### Blind node partitioner (blind-node-partitioner) + +One of the nodes cannot see another, but they can see it. + +### Blind others partitioner (blind-others-partitioner) + +Two nodes don't see one node but it can see both. ## Usage -FIXME +### Dependencies + +- leiningen (https://leiningen.org/) +- clojure (https://clojure.org/) +- jvm + +### Options for `lein run` + +- `test` Run a single test. +- `test-all` Run all available tests from tests-set. +- `-w (--workload)` One of the workloads. Option for a single `test`. +- `--nemesis` One of nemeses. Option for a single `test`. +- `-q (--quorum)` Run test with quorum reads. +- `-r (--rate)` How many operations per second Jepsen will generate in a single thread. +- `-s (--snapshot-distance)` ClickHouse Keeper setting. How often we will create a new snapshot. +- `--stale-log-gap` ClickHosue Keeper setting. A leader will send a snapshot instead of a log to this node if it's committed index less than leaders - this setting value. +- `--reserved-log-items` ClickHouse Keeper setting. How many log items to keep after the snapshot. +- `--ops-per-key` Option for CAS register workload. Total ops that will be generated for a single register. +- `--lightweight-run` Run some lightweight tests without linearizability checks. Option for `tests-all` run. +- `--reuse-binary` Don't download clickhouse binary if it already exists on the node. +- `--clickhouse-source` URL to clickhouse `.deb`, `.tgz` or binary. +- `--time-limit` (in seconds) How long Jepsen will generate new operations. +- `--nodes-file` File with nodes for SSH. Newline separated. +- `--username` SSH username for nodes. +- `--password` SSH password for nodes. +- `--concurrency` How many threads Jepsen will use for concurrent requests. +- `--test-count` How many times to run a single test or how many tests to run from the tests set. + + +### Examples: + +1. Run `Set` workload with `logs-and-snapshots-corruptor` ten times: + +```sh +$ lein run test --nodes-file nodes.txt --username root --password '' --time-limit 30 --concurrency 50 -r 50 --workload set --nemesis logs-and-snapshots-corruptor --clickhouse-source 'https://clickhouse-builds.s3.yandex.net/someurl/clickhouse-common-static_21.4.1.6321_amd64.deb' -q --test-count 10 --reuse-binary +``` + +2. Run ten random tests from `lightweight-run` with some custom Keeper settings: + +``` sh +$ lein run test-all --nodes-file nodes.txt --username root --password '' --time-limit 30 --concurrency 50 -r 50 --snapshot-distance 100 --stale-log-gap 100 --reserved-log-items 10 --lightweight-run --clickhouse-source 'someurl' -q --reuse-binary --test-count 10 +``` + ## License From ba6ccbab42fd19da1322f8443de737fc8ef08edc Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 23 Mar 2021 19:07:41 +0300 Subject: [PATCH 33/43] Fix header --- tests/jepsen.nukeeper/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/jepsen.nukeeper/README.md b/tests/jepsen.nukeeper/README.md index 6bcd7a37069..8f3754b8f7b 100644 --- a/tests/jepsen.nukeeper/README.md +++ b/tests/jepsen.nukeeper/README.md @@ -1,4 +1,4 @@ -# jepsen.nukeeper +# Jepsen tests ClickHouse Keeper A Clojure library designed to test ZooKeeper-like implementation inside ClickHouse. From 9d8b21a04dbca5d27fdb504a9ad667de5b540acb Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 24 Mar 2021 11:12:37 +0300 Subject: [PATCH 34/43] Fix ephemeral node removal --- src/Coordination/NuKeeperStorage.cpp | 14 ++++++---- src/Coordination/tests/gtest_for_build.cpp | 31 ++++++++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/Coordination/NuKeeperStorage.cpp b/src/Coordination/NuKeeperStorage.cpp index 2440d6f6613..c1a8ebdfb44 100644 --- a/src/Coordination/NuKeeperStorage.cpp +++ b/src/Coordination/NuKeeperStorage.cpp @@ -233,7 +233,7 @@ struct NuKeeperStorageGetRequest final : public NuKeeperStorageRequest struct NuKeeperStorageRemoveRequest final : public NuKeeperStorageRequest { using NuKeeperStorageRequest::NuKeeperStorageRequest; - std::pair process(NuKeeperStorage::Container & container, NuKeeperStorage::Ephemerals & ephemerals, int64_t /*zxid*/, int64_t session_id) const override + std::pair process(NuKeeperStorage::Container & container, NuKeeperStorage::Ephemerals & ephemerals, int64_t /*zxid*/, int64_t /*session_id*/) const override { Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse(); Coordination::ZooKeeperRemoveResponse & response = dynamic_cast(*response_ptr); @@ -257,7 +257,12 @@ struct NuKeeperStorageRemoveRequest final : public NuKeeperStorageRequest { auto prev_node = it->value; if (prev_node.stat.ephemeralOwner != 0) - ephemerals[session_id].erase(request.path); + { + auto ephemerals_it = ephemerals.find(prev_node.stat.ephemeralOwner); + ephemerals_it->second.erase(request.path); + if (ephemerals_it->second.empty()) + ephemerals.erase(ephemerals_it); + } auto child_basename = getBaseName(it->key); container.updateValue(parentPath(request.path), [&child_basename] (NuKeeperStorage::Node & parent) @@ -271,10 +276,10 @@ struct NuKeeperStorageRemoveRequest final : public NuKeeperStorageRequest container.erase(request.path); - undo = [prev_node, &container, &ephemerals, session_id, path = request.path, child_basename] + undo = [prev_node, &container, &ephemerals, path = request.path, child_basename] { if (prev_node.stat.ephemeralOwner != 0) - ephemerals[session_id].emplace(path); + ephemerals[prev_node.stat.ephemeralOwner].emplace(path); container.insert(path, prev_node); container.updateValue(parentPath(path), [&child_basename] (NuKeeperStorage::Node & parent) @@ -377,7 +382,6 @@ struct NuKeeperStorageSetRequest final : public NuKeeperStorageRequest { return processWatchesImpl(zk_request->getPath(), watches, list_watches, Coordination::Event::CHANGED); } - }; struct NuKeeperStorageListRequest final : public NuKeeperStorageRequest diff --git a/src/Coordination/tests/gtest_for_build.cpp b/src/Coordination/tests/gtest_for_build.cpp index d90b711498e..cc3dcc04e53 100644 --- a/src/Coordination/tests/gtest_for_build.cpp +++ b/src/Coordination/tests/gtest_for_build.cpp @@ -1232,6 +1232,37 @@ TEST(CoordinationTest, TestStateMachineAndLogStore) } } +TEST(CoordinationTest, TestEphemeralNodeRemove) +{ + using namespace Coordination; + using namespace DB; + + ChangelogDirTest snapshots("./snapshots"); + CoordinationSettingsPtr settings = std::make_shared(); + + ResponsesQueue queue; + SnapshotsQueue snapshots_queue{1}; + auto state_machine = std::make_shared(queue, snapshots_queue, "./snapshots", settings); + state_machine->init(); + + std::shared_ptr request_c = std::make_shared(); + request_c->path = "/hello"; + request_c->is_ephemeral = true; + auto entry_c = getLogEntryFromZKRequest(0, 1, request_c); + state_machine->commit(1, entry_c->get_buf()); + const auto & storage = state_machine->getStorage(); + + EXPECT_EQ(storage.ephemerals.size(), 1); + std::shared_ptr request_d = std::make_shared(); + request_d->path = "/hello"; + /// Delete from other session + auto entry_d = getLogEntryFromZKRequest(0, 2, request_d); + state_machine->commit(2, entry_d->get_buf()); + + EXPECT_EQ(storage.ephemerals.size(), 0); +} + + int main(int argc, char ** argv) { Poco::AutoPtr channel(new Poco::ConsoleChannel(std::cerr)); From d36d3f036dfda47bf0212e7810f799377b14aaff Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 25 Mar 2021 13:04:16 +0300 Subject: [PATCH 35/43] Fix several races in NuRaft --- contrib/NuRaft | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/NuRaft b/contrib/NuRaft index 3d3683e7775..70468326ad5 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit 3d3683e77753cfe015a05fae95ddf418e19f59e1 +Subproject commit 70468326ad5d72e9497944838484c591dae054ea From 640ba7928880d8bc42b4efc494e2dbd21203fbf5 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 25 Mar 2021 13:23:25 +0300 Subject: [PATCH 36/43] Remove data corruption from lightweight run --- tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index 45a1f442d24..7380a9d9cbb 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -113,8 +113,10 @@ (def useful-nemesises ["random-node-killer" "simple-partitioner" - "logs-and-snapshots-corruptor" - "drop-data-corruptor" + "all-nodes-hammer-time" + ; can lead to a very rare data loss https://github.com/eBay/NuRaft/issues/185 + ;"logs-and-snapshots-corruptor" + ;"drop-data-corruptor" "bridge-partitioner" "blind-node-partitioner" "blind-others-partitioner"]) From ba5c15103720b12ab4adeffb55035ee7c438e3e0 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 26 Mar 2021 13:20:07 +0300 Subject: [PATCH 37/43] Fix race condition on snapshots --- src/Coordination/NuKeeperStateMachine.cpp | 25 +++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/Coordination/NuKeeperStateMachine.cpp b/src/Coordination/NuKeeperStateMachine.cpp index 32bb4269f20..23485cb8b5b 100644 --- a/src/Coordination/NuKeeperStateMachine.cpp +++ b/src/Coordination/NuKeeperStateMachine.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { @@ -227,7 +228,28 @@ void NuKeeperStateMachine::save_logical_snp_obj( nuraft::ptr snp_buf = s.serialize(); cloned_meta = nuraft::snapshot::deserialize(*snp_buf); - auto result_path = snapshot_manager.serializeSnapshotBufferToDisk(*cloned_buffer, s.get_last_log_idx()); + /// Sometimes NuRaft can call save and create snapshots from different threads + /// at onces. To avoid race conditions we serialize snapshots through snapshots_queue + /// TODO: make something better + CreateSnapshotTask snapshot_task; + std::shared_ptr> waiter = std::make_shared>(); + auto future = waiter->get_future(); + snapshot_task.snapshot = nullptr; + snapshot_task.create_snapshot = [this, waiter, cloned_buffer, log_idx = s.get_last_log_idx()] (NuKeeperStorageSnapshotPtr &&) + { + try + { + auto result_path = snapshot_manager.serializeSnapshotBufferToDisk(*cloned_buffer, log_idx); + LOG_DEBUG(log, "Saved snapshot {} to path {}", log_idx, result_path); + } + catch (...) + { + tryLogCurrentException(log); + } + waiter->set_value(); + }; + snapshots_queue.push(std::move(snapshot_task)); + future.wait(); { std::lock_guard lock(snapshots_lock); @@ -235,7 +257,6 @@ void NuKeeperStateMachine::save_logical_snp_obj( latest_snapshot_meta = cloned_meta; } - LOG_DEBUG(log, "Created snapshot {} with path {}", s.get_last_log_idx(), result_path); obj_id++; } From 331c5b66365d96541ea1f5a913b0b4beae747416 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 26 Mar 2021 13:55:39 +0300 Subject: [PATCH 38/43] Fix startup one more time --- src/Coordination/NuKeeperServer.cpp | 25 ++++++++++++++++--------- src/Coordination/NuKeeperServer.h | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/Coordination/NuKeeperServer.cpp b/src/Coordination/NuKeeperServer.cpp index 62af9656fb9..7e6c10ca125 100644 --- a/src/Coordination/NuKeeperServer.cpp +++ b/src/Coordination/NuKeeperServer.cpp @@ -188,6 +188,9 @@ nuraft::cb_func::ReturnCode NuKeeperServer::callbackFunc(nuraft::cb_func::Type t if (next_index < last_commited || next_index - last_commited <= 1) commited_store = true; + if (initialized_flag) + return nuraft::cb_func::ReturnCode::Ok; + auto set_initialized = [this] () { std::unique_lock lock(initialized_mutex); @@ -205,15 +208,19 @@ nuraft::cb_func::ReturnCode NuKeeperServer::callbackFunc(nuraft::cb_func::Type t return nuraft::cb_func::ReturnCode::Ok; } case nuraft::cb_func::BecomeFollower: + case nuraft::cb_func::GotAppendEntryReqFromLeader: { - auto leader_index = raft_instance->get_leader_committed_log_idx(); - auto our_index = raft_instance->get_committed_log_idx(); - /// This may happen when we start RAFT claster from scratch. - /// Node first became leader, and after that some other node became leader. - /// BecameFresh for this node will not be called because it was already fresh - /// when it was leader. - if (isLeaderAlive() && leader_index < our_index + coordination_settings->fresh_log_gap) - set_initialized(); + if (isLeaderAlive()) + { + auto leader_index = raft_instance->get_leader_committed_log_idx(); + auto our_index = raft_instance->get_committed_log_idx(); + /// This may happen when we start RAFT cluster from scratch. + /// Node first became leader, and after that some other node became leader. + /// BecameFresh for this node will not be called because it was already fresh + /// when it was leader. + if (leader_index < our_index + coordination_settings->fresh_log_gap) + set_initialized(); + } return nuraft::cb_func::ReturnCode::Ok; } case nuraft::cb_func::BecomeFresh: @@ -237,7 +244,7 @@ void NuKeeperServer::waitInit() { std::unique_lock lock(initialized_mutex); int64_t timeout = coordination_settings->startup_timeout.totalMilliseconds(); - if (!initialized_cv.wait_for(lock, std::chrono::milliseconds(timeout), [&] { return initialized_flag; })) + if (!initialized_cv.wait_for(lock, std::chrono::milliseconds(timeout), [&] { return initialized_flag.load(); })) throw Exception(ErrorCodes::RAFT_ERROR, "Failed to wait RAFT initialization"); } diff --git a/src/Coordination/NuKeeperServer.h b/src/Coordination/NuKeeperServer.h index ba25d5c181b..b5c13e62212 100644 --- a/src/Coordination/NuKeeperServer.h +++ b/src/Coordination/NuKeeperServer.h @@ -31,7 +31,7 @@ private: ResponsesQueue & responses_queue; std::mutex initialized_mutex; - bool initialized_flag = false; + std::atomic initialized_flag = false; std::condition_variable initialized_cv; std::atomic initial_batch_committed = false; From 2db57f0f1669ded0768a00becf7747249f99d930 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 26 Mar 2021 14:18:31 +0300 Subject: [PATCH 39/43] Followup fix --- src/Coordination/NuKeeperStorageDispatcher.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Coordination/NuKeeperStorageDispatcher.cpp b/src/Coordination/NuKeeperStorageDispatcher.cpp index 3aed0d99568..5b35b9c4829 100644 --- a/src/Coordination/NuKeeperStorageDispatcher.cpp +++ b/src/Coordination/NuKeeperStorageDispatcher.cpp @@ -132,6 +132,10 @@ void NuKeeperStorageDispatcher::initialize(const Poco::Util::AbstractConfigurati coordination_settings->loadFromConfig("test_keeper_server.coordination_settings", config); + request_thread = ThreadFromGlobalPool([this] { requestThread(); }); + responses_thread = ThreadFromGlobalPool([this] { responseThread(); }); + snapshot_thread = ThreadFromGlobalPool([this] { snapshotThread(); }); + server = std::make_unique(myid, coordination_settings, config, responses_queue, snapshots_queue); try { @@ -148,10 +152,8 @@ void NuKeeperStorageDispatcher::initialize(const Poco::Util::AbstractConfigurati throw; } - request_thread = ThreadFromGlobalPool([this] { requestThread(); }); - responses_thread = ThreadFromGlobalPool([this] { responseThread(); }); + session_cleaner_thread = ThreadFromGlobalPool([this] { sessionCleanerTask(); }); - snapshot_thread = ThreadFromGlobalPool([this] { snapshotThread(); }); LOG_DEBUG(log, "Dispatcher initialized"); } From 9bdeb436c2671d92d3462374777bdf88b9e06d12 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 26 Mar 2021 15:06:36 +0300 Subject: [PATCH 40/43] Fix typo --- src/Coordination/NuKeeperStateMachine.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Coordination/NuKeeperStateMachine.cpp b/src/Coordination/NuKeeperStateMachine.cpp index 23485cb8b5b..a7037b8d644 100644 --- a/src/Coordination/NuKeeperStateMachine.cpp +++ b/src/Coordination/NuKeeperStateMachine.cpp @@ -229,7 +229,7 @@ void NuKeeperStateMachine::save_logical_snp_obj( cloned_meta = nuraft::snapshot::deserialize(*snp_buf); /// Sometimes NuRaft can call save and create snapshots from different threads - /// at onces. To avoid race conditions we serialize snapshots through snapshots_queue + /// at once. To avoid race conditions we serialize snapshots through snapshots_queue /// TODO: make something better CreateSnapshotTask snapshot_task; std::shared_ptr> waiter = std::make_shared>(); From 2f07056ef6ff60444ab333d3357431f88fa4f0d5 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 26 Mar 2021 16:39:09 +0300 Subject: [PATCH 41/43] More stable last get --- tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index 7380a9d9cbb..f0b4998dad0 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -103,7 +103,7 @@ (gen/nemesis (gen/once {:type :info, :f :stop})) (gen/log "Waiting for recovery") (gen/sleep 10) - (gen/clients (:final-generator workload)))}))) + (gen/clients (gen/until-ok (:final-generator workload))))}))) (def all-nemesises (keys custom-nemesis/custom-nemesises)) From f32704101b5e415b341e28e653ab7239e7c440b1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 26 Mar 2021 19:56:08 +0300 Subject: [PATCH 42/43] Add retries to final operations --- tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj | 5 ++--- tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj | 2 +- tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj | 8 +++----- tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj | 10 +++++----- tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj | 11 +++++++++++ 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj index 48b270517a4..7e2cd00736f 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj @@ -24,11 +24,10 @@ (invoke! [this test op] (case (:f op) - :read (try + :read (exec-with-retries 30 (fn [] (assoc op :type :ok - :value (count (zk-list conn "/"))) - (catch Exception _ (assoc op :type :fail, :error :connect-error))) + :value (count (zk-list conn "/"))))) :add (try (do (zk-multi-create-many-seq-nodes conn "/seq-" (:value op)) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index f0b4998dad0..7380a9d9cbb 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -103,7 +103,7 @@ (gen/nemesis (gen/once {:type :info, :f :stop})) (gen/log "Waiting for recovery") (gen/sleep 10) - (gen/clients (gen/until-ok (:final-generator workload))))}))) + (gen/clients (:final-generator workload)))}))) (def all-nemesises (keys custom-nemesis/custom-nemesises)) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj index 951c0822ad2..494e0357bc1 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj @@ -40,11 +40,9 @@ (catch Exception _ (assoc op :type :info, :error :connect-error))) :drain ; drain via delete is to long, just list all nodes - (try - (do - (zk-sync conn) - (assoc op :type :ok :value (into #{} (map #(str %1) (zk-list conn "/"))))) - (catch Exception _ (assoc op :type :info, :error :connect-error))))) + (exec-with-retries 30 (fn [] + (zk-sync conn) + (assoc op :type :ok :value (into #{} (map #(str %1) (zk-list conn "/")))))))) (teardown! [_ test]) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj index 23461591eaf..01cc10e9a0f 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj @@ -22,11 +22,11 @@ (invoke! [this test op] (case (:f op) - :read (do - (zk-sync conn) - (assoc op - :type :ok - :value (read-string (:data (zk-get-str conn k))))) + :read (exec-with-retries 30 (fn [] + (zk-sync conn) + (assoc op + :type :ok + :value (read-string (:data (zk-get-str conn k)))))) :add (try (do (zk-add-to-set conn k (:value op)) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj index 0e0db2d3a6d..032a8829514 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj @@ -167,3 +167,14 @@ :--test_keeper_server.snapshot_storage_path coordination-snapshots-dir :--test_keeper_server.logs_storage_path coordination-logs-dir) (wait-clickhouse-alive! node test))) + +(defn exec-with-retries + [retries f & args] + (let [res (try {:value (apply f args)} + (catch Exception e + (if (zero? retries) + (throw e) + {:exception e})))] + (if (:exception res) + (do (Thread/sleep 1000) (recur (dec retries) f args)) + (:value res)))) From e101fbab53ec7c984bd0e6cc8ec5e6d9fd1a897a Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 26 Mar 2021 19:57:23 +0300 Subject: [PATCH 43/43] Fix style --- .../src/jepsen/nukeeper/counter.clj | 6 +-- .../src/jepsen/nukeeper/db.clj | 51 +++++++++---------- .../src/jepsen/nukeeper/main.clj | 8 +-- .../src/jepsen/nukeeper/nemesis.clj | 8 +-- .../src/jepsen/nukeeper/queue.clj | 4 +- .../src/jepsen/nukeeper/set.clj | 8 +-- .../src/jepsen/nukeeper/utils.clj | 12 ++--- 7 files changed, 48 insertions(+), 49 deletions(-) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj index 7e2cd00736f..b426a8ea90d 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/counter.clj @@ -25,9 +25,9 @@ (invoke! [this test op] (case (:f op) :read (exec-with-retries 30 (fn [] - (assoc op - :type :ok - :value (count (zk-list conn "/"))))) + (assoc op + :type :ok + :value (count (zk-list conn "/"))))) :add (try (do (zk-multi-create-many-seq-nodes conn "/seq-" (:value op)) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj index 7bc2b9c6cea..d82d628cc95 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/db.clj @@ -32,20 +32,20 @@ (defn unpack-deb [path] (do - (c/exec :dpkg :-x path common-prefix) - (c/exec :rm :-f path) - (c/exec :mv (str common-prefix "/usr/bin/clickhouse") common-prefix) - (c/exec :rm :-rf (str common-prefix "/usr") (str common-prefix "/etc")))) + (c/exec :dpkg :-x path common-prefix) + (c/exec :rm :-f path) + (c/exec :mv (str common-prefix "/usr/bin/clickhouse") common-prefix) + (c/exec :rm :-rf (str common-prefix "/usr") (str common-prefix "/etc")))) (defn unpack-tgz [path] (do - (c/exec :mkdir :-p (str common-prefix "/unpacked")) - (c/exec :tar :-zxvf path :-C (str common-prefix "/unpacked")) - (c/exec :rm :-f path) - (let [subdir (c/exec :ls (str common-prefix "/unpacked"))] - (c/exec :mv (str common-prefix "/unpacked/" subdir "/usr/bin/clickhouse") common-prefix) - (c/exec :rm :-fr (str common-prefix "/unpacked"))))) + (c/exec :mkdir :-p (str common-prefix "/unpacked")) + (c/exec :tar :-zxvf path :-C (str common-prefix "/unpacked")) + (c/exec :rm :-f path) + (let [subdir (c/exec :ls (str common-prefix "/unpacked"))] + (c/exec :mv (str common-prefix "/unpacked/" subdir "/usr/bin/clickhouse") common-prefix) + (c/exec :rm :-fr (str common-prefix "/unpacked"))))) (defn chmod-binary [path] @@ -85,10 +85,10 @@ (defn install-configs [test node] - (c/exec :echo (slurp (io/resource "config.xml")) :> (str configs-dir "/config.xml")) - (c/exec :echo (slurp (io/resource "users.xml")) :> (str configs-dir "/users.xml")) - (c/exec :echo (slurp (io/resource "listen.xml")) :> (str sub-configs-dir "/listen.xml")) - (c/exec :echo (cluster-config test node (slurp (io/resource "test_keeper_config.xml"))) :> (str sub-configs-dir "/test_keeper_config.xml"))) + (c/exec :echo (slurp (io/resource "config.xml")) :> (str configs-dir "/config.xml")) + (c/exec :echo (slurp (io/resource "users.xml")) :> (str configs-dir "/users.xml")) + (c/exec :echo (slurp (io/resource "listen.xml")) :> (str sub-configs-dir "/listen.xml")) + (c/exec :echo (cluster-config test node (slurp (io/resource "test_keeper_config.xml"))) :> (str sub-configs-dir "/test_keeper_config.xml"))) (defn db [version reuse-binary] @@ -96,25 +96,24 @@ (setup! [_ test node] (c/su (do - (info "Preparing directories") - (prepare-dirs) - (if (or (not (cu/exists? binary-path)) (not reuse-binary)) + (info "Preparing directories") + (prepare-dirs) + (if (or (not (cu/exists? binary-path)) (not reuse-binary)) (do (info "Downloading clickhouse") - (install-downloaded-clickhouse (download-clickhouse version))) + (install-downloaded-clickhouse (download-clickhouse version))) (info "Binary already exsist on path" binary-path "skipping download")) - (info "Installing configs") - (install-configs test node) - (info "Starting server") - (start-clickhouse! node test) - (info "ClickHouse started")))) - + (info "Installing configs") + (install-configs test node) + (info "Starting server") + (start-clickhouse! node test) + (info "ClickHouse started")))) (teardown! [_ test node] (info node "Tearing down clickhouse") (kill-clickhouse! node test) (c/su (if (not reuse-binary) - (c/exec :rm :-rf binary-path)) + (c/exec :rm :-rf binary-path)) (c/exec :rm :-rf pid-file-path) (c/exec :rm :-rf data-dir) (c/exec :rm :-rf logs-dir) @@ -125,5 +124,5 @@ (c/su (kill-clickhouse! node test) (c/cd data-dir - (c/exec :tar :czf "coordination.tar.gz" "coordination"))) + (c/exec :tar :czf "coordination.tar.gz" "coordination"))) [stderr-file (str logs-dir "/clickhouse-server.log") (str data-dir "/coordination.tar.gz")]))) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj index 7380a9d9cbb..b9439097e85 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/main.clj @@ -134,10 +134,10 @@ [cli worload-nemeseis-collection] (take (:test-count cli) (shuffle (for [[workload nemesis] worload-nemeseis-collection] - (assoc cli - :nemesis nemesis - :workload workload - :test-count 1))))) + (assoc cli + :nemesis nemesis + :workload workload + :test-count 1))))) (defn all-tests "Turns CLI options into a sequence of tests." [test-fn cli] diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj index 8314d29f575..7d4941cdc8e 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/nemesis.clj @@ -85,13 +85,13 @@ (defn logs-and-snapshots-corruption-nemesis [] (corruptor-nemesis coordination-data-dir (fn [path] - (do - (corrupt-file (select-last-file (str path "/snapshots"))) - (corrupt-file (select-last-file (str path "/logs"))))))) + (do + (corrupt-file (select-last-file (str path "/snapshots"))) + (corrupt-file (select-last-file (str path "/logs"))))))) (defn drop-all-corruption-nemesis [] (corruptor-nemesis coordination-data-dir (fn [path] - (c/exec :rm :-fr path)))) + (c/exec :rm :-fr path)))) (defn partition-bridge-nemesis [] diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj index 494e0357bc1..308778983aa 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/queue.clj @@ -41,8 +41,8 @@ :drain ; drain via delete is to long, just list all nodes (exec-with-retries 30 (fn [] - (zk-sync conn) - (assoc op :type :ok :value (into #{} (map #(str %1) (zk-list conn "/")))))))) + (zk-sync conn) + (assoc op :type :ok :value (into #{} (map #(str %1) (zk-list conn "/")))))))) (teardown! [_ test]) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj index 01cc10e9a0f..f9d21a8dc62 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/set.clj @@ -23,10 +23,10 @@ (invoke! [this test op] (case (:f op) :read (exec-with-retries 30 (fn [] - (zk-sync conn) - (assoc op - :type :ok - :value (read-string (:data (zk-get-str conn k)))))) + (zk-sync conn) + (assoc op + :type :ok + :value (read-string (:data (zk-get-str conn k)))))) :add (try (do (zk-add-to-set conn k (:value op)) diff --git a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj index 032a8829514..cfe9add238b 100644 --- a/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj +++ b/tests/jepsen.nukeeper/src/jepsen/nukeeper/utils.clj @@ -113,11 +113,11 @@ first-child (first (sort children))] (if (not (nil? first-child)) (try - (do (.check txn path (:version stat)) - (.setData txn path (data/to-bytes "") -1) ; I'm just checking multitransactions - (.delete txn (str path first-child) -1) - (.commit txn) - first-child) + (do (.check txn path (:version stat)) + (.setData txn path (data/to-bytes "") -1) ; I'm just checking multitransactions + (.delete txn (str path first-child) -1) + (.commit txn) + first-child) (catch KeeperException$BadVersionException _ nil) ; Even if we got connection loss, delete may actually be executed. ; This function is used for queue model, which strictly require @@ -166,7 +166,7 @@ :--logger.errorlog (str logs-dir "/clickhouse-server.err.log") :--test_keeper_server.snapshot_storage_path coordination-snapshots-dir :--test_keeper_server.logs_storage_path coordination-logs-dir) - (wait-clickhouse-alive! node test))) + (wait-clickhouse-alive! node test))) (defn exec-with-retries [retries f & args]