thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 01:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into benchmark-hetzner

Browse Source
This commit is contained in:
Alexey Milovidov 2022-07-08 00:23:50 +03:00 committed by GitHub
commit b033d34c87
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 127 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/Columns/ColumnObject.cpp
View File

@ -21,7 +21,7 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
extern const int ILLEGAL_COLUMN;
extern const int DUPLICATE_COLUMN;
extern const int NUMBER_OF_DIMENSIONS_MISMATHED;
extern const int NUMBER_OF_DIMENSIONS_MISMATCHED;
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
extern const int ARGUMENT_OUT_OF_BOUND;
}
@ -298,7 +298,7 @@ void ColumnObject::Subcolumn::insert(Field field, FieldInfo info)
value_dim = column_dim;
if (value_dim != column_dim)
throw Exception(ErrorCodes::NUMBER_OF_DIMENSIONS_MISMATHED,
throw Exception(ErrorCodes::NUMBER_OF_DIMENSIONS_MISMATCHED,
"Dimension of types mismatched between inserted value and column. "
"Dimension of value: {}. Dimension of column: {}",
value_dim, column_dim);

2
src/Common/ErrorCodes.cpp
View File

@ -613,7 +613,7 @@
M(642, CANNOT_PACK_ARCHIVE) \
M(643, CANNOT_UNPACK_ARCHIVE) \
M(644, REMOTE_FS_OBJECT_CACHE_ERROR) \
M(645, NUMBER_OF_DIMENSIONS_MISMATHED) \
M(645, NUMBER_OF_DIMENSIONS_MISMATCHED) \
M(646, CANNOT_BACKUP_DATABASE) \
M(647, CANNOT_BACKUP_TABLE) \
M(648, WRONG_DDL_RENAMING_SETTINGS) \

4
src/Common/ZooKeeper/KeeperException.h
View File

@ -24,9 +24,7 @@ public:
static void check(Coordination::Error code, const Coordination::Requests & requests, const Coordination::Responses & responses);
KeeperMultiException(Coordination::Error code, const Coordination::Requests & requests, const Coordination::Responses & responses);
private:
static size_t getFailedOpIndex(Coordination::Error code, const Coordination::Responses & responses);
};
size_t getFailedOpIndex(Coordination::Error code, const Coordination::Responses & responses);
}

2
src/Common/ZooKeeper/ZooKeeper.cpp
View File

@ -1227,7 +1227,7 @@ void ZooKeeper::setZooKeeperLog(std::shared_ptr<DB::ZooKeeperLog> zk_log_)
}
size_t KeeperMultiException::getFailedOpIndex(Coordination::Error exception_code, const Coordination::Responses & responses)
size_t getFailedOpIndex(Coordination::Error exception_code, const Coordination::Responses & responses)
{
if (responses.empty())
throw DB::Exception("Responses for multi transaction is empty", DB::ErrorCodes::LOGICAL_ERROR);

49
src/Storages/MergeTree/EphemeralLockInZooKeeper.cpp
View File

@ -12,30 +12,53 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
}
EphemeralLockInZooKeeper::EphemeralLockInZooKeeper(
const String & path_prefix_, const String & temp_path, zkutil::ZooKeeper & zookeeper_, Coordination::Requests * precheck_ops)
: zookeeper(&zookeeper_), path_prefix(path_prefix_)
EphemeralLockInZooKeeper::EphemeralLockInZooKeeper(const String & path_prefix_, zkutil::ZooKeeper & zookeeper_, const String & holder_path_)
: zookeeper(&zookeeper_), path_prefix(path_prefix_), holder_path(holder_path_)
{
/// Write the path to the secondary node in the main node.
path = zookeeper->create(path_prefix, holder_path, zkutil::CreateMode::EphemeralSequential);
if (path.size() <= path_prefix.size())
throw Exception("Logical error: name of the main node is shorter than prefix.", ErrorCodes::LOGICAL_ERROR);
}
std::optional<EphemeralLockInZooKeeper> createEphemeralLockInZooKeeper(
const String & path_prefix_, const String & temp_path, zkutil::ZooKeeper & zookeeper_, const String & deduplication_path)
{
/// The /abandonable_lock- name is for backward compatibility.
String holder_path_prefix = temp_path + "/abandonable_lock-";
String holder_path;
/// Let's create an secondary ephemeral node.
if (!precheck_ops || precheck_ops->empty())
if (deduplication_path.empty())
{
holder_path = zookeeper->create(holder_path_prefix, "", zkutil::CreateMode::EphemeralSequential);
holder_path = zookeeper_.create(holder_path_prefix, "", zkutil::CreateMode::EphemeralSequential);
}
else
{
precheck_ops->emplace_back(zkutil::makeCreateRequest(holder_path_prefix, "", zkutil::CreateMode::EphemeralSequential));
Coordination::Responses op_results = zookeeper->multi(*precheck_ops);
holder_path = dynamic_cast<const Coordination::CreateResponse &>(*op_results.back()).path_created;
/// Check for duplicates in advance, to avoid superfluous block numbers allocation
Coordination::Requests ops;
ops.emplace_back(zkutil::makeCreateRequest(deduplication_path, "", zkutil::CreateMode::Persistent));
ops.emplace_back(zkutil::makeRemoveRequest(deduplication_path, -1));
ops.emplace_back(zkutil::makeCreateRequest(holder_path_prefix, "", zkutil::CreateMode::EphemeralSequential));
Coordination::Responses responses;
Coordination::Error e = zookeeper_.tryMulti(ops, responses);
if (e != Coordination::Error::ZOK)
{
if (responses[0]->error == Coordination::Error::ZNODEEXISTS)
{
return {};
}
else
{
zkutil::KeeperMultiException::check(e, ops, responses); // This should always throw the proper exception
throw Exception("Unable to handle error {} when acquiring ephemeral lock in ZK", ErrorCodes::LOGICAL_ERROR);
}
}
holder_path = dynamic_cast<const Coordination::CreateResponse *>(responses.back().get())->path_created;
}
/// Write the path to the secondary node in the main node.
path = zookeeper->create(path_prefix, holder_path, zkutil::CreateMode::EphemeralSequential);
if (path.size() <= path_prefix.size())
throw Exception("Logical error: name of the main node is shorter than prefix.", ErrorCodes::LOGICAL_ERROR);
return EphemeralLockInZooKeeper{path_prefix_, zookeeper_, holder_path};
}
void EphemeralLockInZooKeeper::unlock()

15
src/Storages/MergeTree/EphemeralLockInZooKeeper.h
View File

@ -24,12 +24,14 @@ namespace ErrorCodes
/// it would be simpler to allocate block numbers for all partitions in one ZK directory).
class EphemeralLockInZooKeeper : public boost::noncopyable
{
friend std::optional<EphemeralLockInZooKeeper> createEphemeralLockInZooKeeper(
const String & path_prefix_, const String & temp_path, zkutil::ZooKeeper & zookeeper_, const String & deduplication_path);
protected:
EphemeralLockInZooKeeper() = delete;
EphemeralLockInZooKeeper(const String & path_prefix_, zkutil::ZooKeeper & zookeeper_, const String & holder_path_);
public:
EphemeralLockInZooKeeper(
const String & path_prefix_, const String & temp_path, zkutil::ZooKeeper & zookeeper_, Coordination::Requests * precheck_ops = nullptr);
EphemeralLockInZooKeeper() = default;
EphemeralLockInZooKeeper(EphemeralLockInZooKeeper && rhs) noexcept
{
*this = std::move(rhs);
@ -90,6 +92,9 @@ private:
String holder_path;
};
std::optional<EphemeralLockInZooKeeper> createEphemeralLockInZooKeeper(
const String & path_prefix_, const String & temp_path, zkutil::ZooKeeper & zookeeper_, const String & deduplication_path);
/// Acquires block number locks in all partitions.
class EphemeralLocksInAllPartitions : public boost::noncopyable

64
src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -146,7 +146,6 @@ namespace ErrorCodes
extern const int RECEIVED_ERROR_TOO_MANY_REQUESTS;
extern const int PART_IS_TEMPORARILY_LOCKED;
extern const int CANNOT_ASSIGN_OPTIMIZE;
extern const int KEEPER_EXCEPTION;
extern const int ALL_REPLICAS_LOST;
extern const int REPLICA_STATUS_CHANGED;
extern const int CANNOT_ASSIGN_ALTER;
@ -1435,23 +1434,28 @@ MergeTreeData::DataPartsVector StorageReplicatedMergeTree::checkPartChecksumsAnd
try
{
zookeeper->multi(ops);
return transaction.commit();
}
catch (const zkutil::KeeperMultiException & e)
{
size_t num_check_ops = 2 * absent_part_paths_on_replicas.size();
size_t failed_op_index = e.failed_op_index;
Coordination::Responses responses;
Coordination::Error e = zookeeper->tryMulti(ops, responses);
if (e == Coordination::Error::ZOK)
return transaction.commit();
if (failed_op_index < num_check_ops && e.code == Coordination::Error::ZNODEEXISTS)
if (e == Coordination::Error::ZNODEEXISTS)
{
LOG_INFO(log, "The part {} on a replica suddenly appeared, will recheck checksums", e.getPathForFirstFailedOp());
}
else
{
unlockSharedData(*part);
throw;
size_t num_check_ops = 2 * absent_part_paths_on_replicas.size();
size_t failed_op_index = zkutil::getFailedOpIndex(e, responses);
if (failed_op_index < num_check_ops)
{
LOG_INFO(log, "The part {} on a replica suddenly appeared, will recheck checksums", ops[failed_op_index]->getPath());
continue;
}
}
throw zkutil::KeeperException(e);
}
catch (const std::exception &)
{
unlockSharedData(*part);
throw;
}
}
}
@ -5199,14 +5203,6 @@ StorageReplicatedMergeTree::allocateBlockNumber(
else
zookeeper_table_path = zookeeper_path_prefix;
/// Lets check for duplicates in advance, to avoid superfluous block numbers allocation
Coordination::Requests deduplication_check_ops;
if (!zookeeper_block_id_path.empty())
{
deduplication_check_ops.emplace_back(zkutil::makeCreateRequest(zookeeper_block_id_path, "", zkutil::CreateMode::Persistent));
deduplication_check_ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_block_id_path, -1));
}
String block_numbers_path = fs::path(zookeeper_table_path) / "block_numbers";
String partition_path = fs::path(block_numbers_path) / partition_id;
@ -5225,26 +5221,8 @@ StorageReplicatedMergeTree::allocateBlockNumber(
zkutil::KeeperMultiException::check(code, ops, responses);
}
EphemeralLockInZooKeeper lock;
/// 2 RTT
try
{
lock = EphemeralLockInZooKeeper(
fs::path(partition_path) / "block-", fs::path(zookeeper_table_path) / "temp", *zookeeper, &deduplication_check_ops);
}
catch (const zkutil::KeeperMultiException & e)
{
if (e.code == Coordination::Error::ZNODEEXISTS && e.getPathForFirstFailedOp() == zookeeper_block_id_path)
return {};
throw Exception("Cannot allocate block number in ZooKeeper: " + e.displayText(), ErrorCodes::KEEPER_EXCEPTION);
}
catch (const Coordination::Exception & e)
{
throw Exception("Cannot allocate block number in ZooKeeper: " + e.displayText(), ErrorCodes::KEEPER_EXCEPTION);
}
return {std::move(lock)};
return createEphemeralLockInZooKeeper(
fs::path(partition_path) / "block-", fs::path(zookeeper_table_path) / "temp", *zookeeper, zookeeper_block_id_path);
}

1
website/benchmark/hardware/index.html
View File

@ -98,6 +98,7 @@ Results for 2x EPYC 7702 on ZFS mirror NVME are from <b>Alibek A</b>.<br/>
Results for Intel 11th Gen Core i9-11900KF are from <b>Tim Xian</b>.<br/>
Results for AWS instance type m5a.4xlarge are from <b>Daniel Chimeno</b>.<br/>
Results for Hetzner EX62-NVME are from <b>Talles Airan</b>.<br/>
Results for AMD Ryzen 9 5950X are from <b>Stefan</b>.<br/>
</p>
</div>
</div>

54
website/benchmark/hardware/results/amd_ryzen_9_5950x.json Normal file
View File

@ -0,0 +1,54 @@
[
{
"system": "AMD Ryzen 9 (2022)",
"system_full": " AMD Ryzen 9 5950X 16-Core Processor, 125Gi RAM",
"time": "2022-05-03 00:00:00",
"kind": "desktop",
"result":
[
[0.001, 0.001, 0.010],
[0.008, 0.006, 0.006],
[0.017, 0.014, 0.014],
[0.039, 0.022, 0.028],
[0.074, 0.064, 0.074],
[0.150, 0.128, 0.133],
[0.001, 0.001, 0.001],
[0.006, 0.007, 0.006],
[0.215, 0.195, 0.197],
[0.247, 0.221, 0.221],
[0.100, 0.089, 0.098],
[0.112, 0.111, 0.111],
[0.467, 0.445, 0.444],
[0.588, 0.563, 0.554],
[0.521, 0.482, 0.485],
[0.503, 0.616, 0.619],
[1.465, 1.636, 1.630],
[0.907, 0.881, 0.893],
[3.401, 2.832, 2.822],
[0.040, 0.025, 0.022],
[0.465, 0.322, 0.309],
[0.501, 0.356, 0.358],
[1.207, 0.944, 0.938],
[0.869, 0.415, 0.401],
[0.143, 0.101, 0.099],
[0.101, 0.090, 0.090],
[0.139, 0.096, 0.096],
[0.467, 0.332, 0.317],
[0.558, 0.468, 0.464],
[2.288, 2.128, 2.058],
[0.322, 0.285, 0.283],
[0.768, 0.545, 0.537],
[4.126, 4.078, 4.155],
[2.730, 2.511, 2.510],
[2.658, 2.536, 2.566],
[0.877, 0.732, 0.747],
[0.096, 0.085, 0.082],
[0.038, 0.035, 0.036],
[0.038, 0.034, 0.034],
[0.228, 0.218, 0.219],
[0.014, 0.014, 0.016],
[0.016, 0.013, 0.010],
[0.003, 0.006, 0.003]
]
}
]