ClickHouse/utils/keeper-bench/Generator.cpp

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

575 lines
17 KiB
C++
Raw Normal View History

2021-04-12 08:10:23 +00:00
#include "Generator.h"
2023-04-06 15:10:58 +00:00
#include "Common/Exception.h"
2023-04-07 13:02:42 +00:00
#include "Common/ZooKeeper/ZooKeeperCommon.h"
2023-04-06 10:25:37 +00:00
#include <Common/Config/ConfigProcessor.h>
2021-04-12 08:10:23 +00:00
#include <random>
#include <filesystem>
2023-04-06 19:17:35 +00:00
#include <Poco/Util/AbstractConfiguration.h>
2021-04-12 08:10:23 +00:00
2021-04-12 15:40:42 +00:00
using namespace Coordination;
using namespace zkutil;
2021-04-12 08:10:23 +00:00
2023-04-06 15:10:58 +00:00
namespace DB::ErrorCodes
2021-04-13 11:55:08 +00:00
{
extern const int LOGICAL_ERROR;
2023-04-14 13:32:08 +00:00
extern const int BAD_ARGUMENTS;
2021-04-13 11:55:08 +00:00
}
2021-04-12 08:10:23 +00:00
namespace
{
std::string generateRandomString(size_t length)
{
2021-04-13 11:55:08 +00:00
if (length == 0)
return "";
2021-04-13 13:26:55 +00:00
static const auto & chars = "0123456789"
2021-04-12 08:10:23 +00:00
"abcdefghijklmnopqrstuvwxyz"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ";
2021-04-14 09:01:33 +00:00
static pcg64 rng(randomSeed());
static std::uniform_int_distribution<size_t> pick(0, sizeof(chars) - 2);
2021-04-12 08:10:23 +00:00
std::string s;
s.reserve(length);
2021-04-13 13:26:55 +00:00
while (length--)
s += chars[pick(rng)];
2021-04-12 08:10:23 +00:00
return s;
}
}
2023-04-06 15:10:58 +00:00
NumberGetter
NumberGetter::fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config, std::optional<uint64_t> default_value)
{
NumberGetter number_getter;
if (!config.has(key) && default_value.has_value())
2021-04-13 11:55:08 +00:00
{
2023-04-06 15:10:58 +00:00
number_getter.value = *default_value;
}
else if (config.has(key + ".min_value") && config.has(key + ".max_value"))
{
NumberRange range{.min_value = config.getUInt64(key + ".min_value"), .max_value = config.getUInt64(key + ".max_value")};
if (range.max_value <= range.min_value)
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Range is invalid for key {}: [{}, {}]", key, range.min_value, range.max_value);
number_getter.value = range;
}
else
{
number_getter.value = config.getUInt64(key);
}
return number_getter;
2021-04-13 11:55:08 +00:00
}
2023-04-06 15:10:58 +00:00
std::string NumberGetter::description() const
2021-04-12 08:10:23 +00:00
{
2023-04-06 15:10:58 +00:00
if (const auto * number = std::get_if<uint64_t>(&value))
return std::to_string(*number);
2021-04-12 08:10:23 +00:00
2023-04-06 15:10:58 +00:00
const auto & range = std::get<NumberRange>(value);
return fmt::format("random value from range [{}, {}]", range.min_value, range.max_value);
}
2021-04-12 08:10:23 +00:00
2023-04-06 15:10:58 +00:00
uint64_t NumberGetter::getNumber() const
{
if (const auto * number = std::get_if<uint64_t>(&value))
return *number;
2021-04-12 08:10:23 +00:00
2023-04-06 15:10:58 +00:00
const auto & range = std::get<NumberRange>(value);
static pcg64 rng(randomSeed());
return std::uniform_int_distribution<uint64_t>(range.min_value, range.max_value)(rng);
}
2021-04-12 08:10:23 +00:00
2023-04-06 15:10:58 +00:00
StringGetter StringGetter::fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config)
{
StringGetter string_getter;
if (config.has(key + ".random_string"))
string_getter.value
= NumberGetter::fromConfig(key + ".random_string.size", config);
else
string_getter.value = config.getString(key);
return string_getter;
2021-04-12 08:10:23 +00:00
}
2021-04-13 11:55:08 +00:00
2023-04-06 15:10:58 +00:00
void StringGetter::setString(std::string name)
{
value = std::move(name);
}
2021-04-13 11:55:08 +00:00
2023-04-06 15:10:58 +00:00
std::string StringGetter::getString() const
2022-01-20 17:06:34 +00:00
{
2023-04-06 15:10:58 +00:00
if (const auto * string = std::get_if<std::string>(&value))
return *string;
2022-01-20 17:06:34 +00:00
2023-04-06 15:10:58 +00:00
const auto number_getter = std::get<NumberGetter>(value);
return generateRandomString(number_getter.getNumber());
2022-01-20 17:06:34 +00:00
}
2023-04-06 15:10:58 +00:00
std::string StringGetter::description() const
2022-01-20 17:06:34 +00:00
{
2023-04-06 15:10:58 +00:00
if (const auto * string = std::get_if<std::string>(&value))
return *string;
2022-01-20 17:06:34 +00:00
2023-04-06 15:10:58 +00:00
const auto number_getter = std::get<NumberGetter>(value);
return fmt::format("random string with size of {}", number_getter.description());
2022-01-20 17:06:34 +00:00
}
2023-04-06 15:10:58 +00:00
bool StringGetter::isRandom() const
2023-03-24 14:37:51 +00:00
{
2023-04-06 15:10:58 +00:00
return std::holds_alternative<NumberGetter>(value);
2023-03-24 14:37:51 +00:00
}
2023-04-06 19:17:35 +00:00
PathGetter PathGetter::fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config)
{
static constexpr std::string_view path_key_string = "path";
PathGetter path_getter;
Poco::Util::AbstractConfiguration::Keys path_keys;
config.keys(key, path_keys);
for (const auto & path_key : path_keys)
{
if (!path_key.starts_with(path_key_string))
continue;
const auto current_path_key_string = key + "." + path_key;
const auto children_of_key = current_path_key_string + ".children_of";
if (config.has(children_of_key))
{
auto parent_node = config.getString(children_of_key);
if (parent_node.empty() || parent_node[0] != '/')
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid path for request generator: '{}'", parent_node);
path_getter.parent_paths.push_back(std::move(parent_node));
}
else
{
auto path = config.getString(key + "." + path_key);
if (path.empty() || path[0] != '/')
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid path for request generator: '{}'", path);
path_getter.paths.push_back(std::move(path));
}
}
path_getter.path_picker = std::uniform_int_distribution<size_t>(0, path_getter.paths.size() - 1);
return path_getter;
}
void PathGetter::initialize(Coordination::ZooKeeper & zookeeper)
{
for (const auto & parent_path : parent_paths)
{
auto list_promise = std::make_shared<std::promise<ListResponse>>();
auto list_future = list_promise->get_future();
auto callback = [list_promise] (const ListResponse & response)
{
if (response.error != Coordination::Error::ZOK)
list_promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error)));
else
list_promise->set_value(response);
};
zookeeper.list(parent_path, ListRequestType::ALL, std::move(callback), {});
auto list_response = list_future.get();
for (const auto & child : list_response.names)
paths.push_back(std::filesystem::path(parent_path) / child);
}
path_picker = std::uniform_int_distribution<size_t>(0, paths.size() - 1);
initialized = true;
}
std::string PathGetter::getPath() const
{
if (!initialized)
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "PathGetter is not initialized");
if (paths.size() == 1)
return paths[0];
static pcg64 rng(randomSeed());
return paths[path_picker(rng)];
}
std::string PathGetter::description() const
{
std::string description;
for (const auto & path : parent_paths)
{
if (!description.empty())
description += ", ";
description += fmt::format("children of {}", path);
}
for (const auto & path : paths)
{
if (!description.empty())
description += ", ";
description += path;
}
return description;
}
2023-04-07 15:20:36 +00:00
RequestGetter::RequestGetter(std::vector<RequestGeneratorPtr> request_generators_)
: request_generators(std::move(request_generators_))
{}
2023-04-07 13:02:42 +00:00
RequestGetter RequestGetter::fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config, bool for_multi)
{
RequestGetter request_getter;
Poco::Util::AbstractConfiguration::Keys generator_keys;
config.keys(key, generator_keys);
bool use_weights = false;
size_t weight_sum = 0;
auto & generators = request_getter.request_generators;
for (const auto & generator_key : generator_keys)
{
RequestGeneratorPtr request_generator;
if (generator_key.starts_with("create"))
request_generator = std::make_unique<CreateRequestGenerator>();
else if (generator_key.starts_with("set"))
request_generator = std::make_unique<SetRequestGenerator>();
else if (generator_key.starts_with("get"))
request_generator = std::make_unique<GetRequestGenerator>();
else if (generator_key.starts_with("list"))
request_generator = std::make_unique<ListRequestGenerator>();
else if (generator_key.starts_with("multi"))
{
if (for_multi)
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Nested multi requests are not allowed");
request_generator = std::make_unique<MultiRequestGenerator>();
}
else
{
if (for_multi)
continue;
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Unknown generator {}", key + "." + generator_key);
}
request_generator->getFromConfig(key + "." + generator_key, config);
auto weight = request_generator->getWeight();
use_weights |= weight != 1;
weight_sum += weight;
2023-04-14 13:32:08 +00:00
2023-04-07 13:02:42 +00:00
generators.push_back(std::move(request_generator));
}
if (generators.empty())
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "No request generators found in config for key '{}'", key);
size_t max_value = use_weights ? weight_sum - 1 : generators.size() - 1;
request_getter.request_generator_picker = std::uniform_int_distribution<size_t>(0, max_value);
/// construct weight vector
if (use_weights)
{
auto & weights = request_getter.weights;
weights.reserve(generators.size());
weights.push_back(generators[0]->getWeight() - 1);
for (size_t i = 1; i < generators.size(); ++i)
weights.push_back(weights.back() + generators[i]->getWeight());
}
return request_getter;
}
RequestGeneratorPtr RequestGetter::getRequestGenerator() const
{
static pcg64 rng(randomSeed());
auto random_number = request_generator_picker(rng);
if (weights.empty())
return request_generators[random_number];
for (size_t i = 0; i < request_generators.size(); ++i)
{
if (random_number <= weights[i])
return request_generators[i];
}
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Invalid number generated: {}", random_number);
}
std::string RequestGetter::description() const
{
std::string guard(30, '-');
std::string description = guard;
for (const auto & request_generator : request_generators)
description += fmt::format("\n{}\n", request_generator->description());
return description + guard;
}
void RequestGetter::startup(Coordination::ZooKeeper & zookeeper)
{
for (const auto & request_generator : request_generators)
request_generator->startup(zookeeper);
}
const std::vector<RequestGeneratorPtr> & RequestGetter::requestGenerators() const
{
return request_generators;
}
2023-04-06 15:10:58 +00:00
void RequestGenerator::getFromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config)
2023-03-24 14:37:51 +00:00
{
2023-04-07 13:02:42 +00:00
if (config.has(key + ".weight"))
weight = config.getUInt64(key + ".weight");
2023-04-06 15:10:58 +00:00
getFromConfigImpl(key, config);
}
2023-03-24 14:37:51 +00:00
2023-04-06 15:10:58 +00:00
std::string RequestGenerator::description()
{
2023-04-07 13:02:42 +00:00
std::string weight_string = weight == 1 ? "" : fmt::format("\n- weight: {}", weight);
return fmt::format("{}{}", descriptionImpl(), weight_string);
2023-03-24 14:37:51 +00:00
}
2022-01-20 17:06:34 +00:00
2023-04-06 15:10:58 +00:00
Coordination::ZooKeeperRequestPtr RequestGenerator::generate(const Coordination::ACLs & acls)
2021-04-13 11:55:08 +00:00
{
2023-04-06 15:10:58 +00:00
return generateImpl(acls);
}
2023-04-06 19:17:35 +00:00
void RequestGenerator::startup(Coordination::ZooKeeper & zookeeper)
{
startupImpl(zookeeper);
}
2023-04-07 13:02:42 +00:00
size_t RequestGenerator::getWeight() const
{
return weight;
}
2023-04-06 15:10:58 +00:00
CreateRequestGenerator::CreateRequestGenerator()
: rng(randomSeed())
, remove_picker(0, 1.0)
{}
void CreateRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config)
{
2023-04-06 19:17:35 +00:00
parent_path = PathGetter::fromConfig(key, config);
2023-04-06 15:10:58 +00:00
2023-04-06 19:17:35 +00:00
name = StringGetter(NumberGetter::fromConfig(key + ".name_length", config, 5));
2023-04-06 15:10:58 +00:00
if (config.has(key + ".data"))
data = StringGetter::fromConfig(key + ".data", config);
2023-04-07 14:51:25 +00:00
if (config.has(key + ".remove_factor"))
remove_factor = config.getDouble(key + ".remove_factor");
2023-04-06 15:10:58 +00:00
}
std::string CreateRequestGenerator::descriptionImpl()
{
std::string data_string
= data.has_value() ? fmt::format("data for created nodes: {}", data->description()) : "no data for created nodes";
2023-04-07 14:51:25 +00:00
std::string remove_factor_string
= remove_factor.has_value() ? fmt::format("- remove factor: {}", *remove_factor) : "- without removes";
2023-04-06 15:10:58 +00:00
return fmt::format(
"Create Request Generator\n"
2023-04-06 19:17:35 +00:00
"- parent path(s) for created nodes: {}\n"
2023-04-06 15:10:58 +00:00
"- name for created nodes: {}\n"
"- {}\n"
2023-04-07 14:51:25 +00:00
"{}",
2023-04-06 19:17:35 +00:00
parent_path.description(),
2023-04-06 15:10:58 +00:00
name.description(),
data_string,
2023-04-07 14:51:25 +00:00
remove_factor_string);
2023-04-06 15:10:58 +00:00
}
2023-04-06 19:17:35 +00:00
void CreateRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper)
{
parent_path.initialize(zookeeper);
}
2023-04-06 15:10:58 +00:00
Coordination::ZooKeeperRequestPtr CreateRequestGenerator::generateImpl(const Coordination::ACLs & acls)
{
2023-04-07 14:51:25 +00:00
if (remove_factor.has_value() && !paths_created.empty() && remove_picker(rng) < *remove_factor)
2021-04-13 11:55:08 +00:00
{
2023-04-06 15:10:58 +00:00
auto request = std::make_shared<ZooKeeperRemoveRequest>();
auto it = paths_created.begin();
request->path = *it;
paths_created.erase(it);
return request;
}
auto request = std::make_shared<ZooKeeperCreateRequest>();
request->acls = acls;
2023-04-06 19:17:35 +00:00
std::string path_candidate = std::filesystem::path(parent_path.getPath()) / name.getString();
2023-04-06 15:10:58 +00:00
while (paths_created.contains(path_candidate))
2023-04-06 19:17:35 +00:00
path_candidate = std::filesystem::path(parent_path.getPath()) / name.getString();
2023-04-06 15:10:58 +00:00
paths_created.insert(path_candidate);
request->path = std::move(path_candidate);
if (data)
request->data = data->getString();
return request;
}
2023-04-06 19:17:35 +00:00
void SetRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config)
{
path = PathGetter::fromConfig(key, config);
data = StringGetter::fromConfig(key + ".data", config);
}
std::string SetRequestGenerator::descriptionImpl()
{
return fmt::format(
"Set Request Generator\n"
"- path(s) to set: {}\n"
"- data to set: {}",
path.description(),
data.description());
}
Coordination::ZooKeeperRequestPtr SetRequestGenerator::generateImpl(const Coordination::ACLs & /*acls*/)
{
auto request = std::make_shared<ZooKeeperSetRequest>();
request->path = path.getPath();
request->data = data.getString();
return request;
}
void SetRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper)
{
path.initialize(zookeeper);
}
void GetRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config)
{
path = PathGetter::fromConfig(key, config);
}
std::string GetRequestGenerator::descriptionImpl()
{
return fmt::format(
"Get Request Generator\n"
"- path(s) to get: {}",
path.description());
}
Coordination::ZooKeeperRequestPtr GetRequestGenerator::generateImpl(const Coordination::ACLs & /*acls*/)
{
auto request = std::make_shared<ZooKeeperGetRequest>();
request->path = path.getPath();
return request;
}
void GetRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper)
{
path.initialize(zookeeper);
}
void ListRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config)
{
path = PathGetter::fromConfig(key, config);
}
std::string ListRequestGenerator::descriptionImpl()
{
return fmt::format(
"List Request Generator\n"
"- path(s) to get: {}",
path.description());
}
Coordination::ZooKeeperRequestPtr ListRequestGenerator::generateImpl(const Coordination::ACLs & /*acls*/)
{
2023-04-07 14:14:39 +00:00
auto request = std::make_shared<ZooKeeperFilteredListRequest>();
2023-04-06 19:17:35 +00:00
request->path = path.getPath();
return request;
}
void ListRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper)
{
path.initialize(zookeeper);
}
2023-04-07 13:02:42 +00:00
void MultiRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config)
{
if (config.has(key + ".size"))
size = NumberGetter::fromConfig(key + ".size", config);
request_getter = RequestGetter::fromConfig(key, config, /*for_multi*/ true);
};
std::string MultiRequestGenerator::descriptionImpl()
{
std::string size_string = size.has_value() ? fmt::format("- number of requests: {}\n", size->description()) : "";
return fmt::format(
"Multi Request Generator\n"
"{}"
"- requests:\n{}",
size_string,
request_getter.description());
}
Coordination::ZooKeeperRequestPtr MultiRequestGenerator::generateImpl(const Coordination::ACLs & acls)
{
Coordination::Requests ops;
if (size)
{
2023-04-14 13:32:08 +00:00
auto request_count = size->getNumber();
2023-04-07 13:02:42 +00:00
for (size_t i = 0; i < request_count; ++i)
ops.push_back(request_getter.getRequestGenerator()->generate(acls));
}
else
{
for (const auto & request_generator : request_getter.requestGenerators())
ops.push_back(request_generator->generate(acls));
}
return std::make_shared<ZooKeeperMultiRequest>(ops, acls);
}
void MultiRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper)
{
request_getter.startup(zookeeper);
}
2023-04-06 15:10:58 +00:00
Generator::Generator(const Poco::Util::AbstractConfiguration & config)
{
Coordination::ACL acl;
acl.permissions = Coordination::ACL::All;
acl.scheme = "world";
acl.id = "anyone";
default_acls.emplace_back(std::move(acl));
2021-04-13 11:55:08 +00:00
2023-04-14 13:32:08 +00:00
std::cerr << "---- Collecting request generators ----" << std::endl;
2024-04-10 10:56:29 +00:00
static const std::string requests_key = "generator.requests";
2023-04-07 13:02:42 +00:00
request_getter = RequestGetter::fromConfig(requests_key, config);
2023-04-14 13:32:08 +00:00
std::cerr << request_getter.description() << std::endl;
std::cerr << "---- Done collecting request generators ----\n" << std::endl;
2021-04-13 11:55:08 +00:00
}
2023-04-06 15:10:58 +00:00
void Generator::startup(Coordination::ZooKeeper & zookeeper)
2021-04-13 11:55:08 +00:00
{
2023-04-14 13:32:08 +00:00
std::cerr << "---- Initializing generators ----" << std::endl;
2023-04-07 13:02:42 +00:00
request_getter.startup(zookeeper);
2021-04-13 11:55:08 +00:00
}
2023-04-06 10:25:37 +00:00
2023-04-06 15:10:58 +00:00
Coordination::ZooKeeperRequestPtr Generator::generate()
2023-04-06 10:25:37 +00:00
{
2023-04-07 13:02:42 +00:00
return request_getter.getRequestGenerator()->generate(default_acls);
2023-04-06 10:25:37 +00:00
}