thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into imbloo

Browse Source
This commit is contained in:
Michael Kolupaev 2024-10-23 11:03:57 +00:00
commit 6af560a32d
10 changed files with 445 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

345
docs/en/getting-started/example-datasets/tpch.md
View File

@ -155,10 +155,10 @@ The queries are generated by `./qgen -s <scaling_factor>`. Example queries for `
## Queries
::::warning
TPC-H makes heavy use of correlated subqueries which are at the time of writing (October 2024) not supported by ClickHouse ([issue #6697](https://github.com/ClickHouse/ClickHouse/issues/6697)).
As a result, many of below benchmark queries will fail with errors.
::::
**Correctness**
The result of the queries agrees with the official results unless mentioned otherwise. To verify, generate a TPC-H database with scale
factor = 1 (`dbgen`, see above) and compare with the [expected results in tpch-kit](https://github.com/gregrahn/tpch-kit/tree/master/dbgen/answers).
**Q1**
@ -177,7 +177,7 @@ SELECT
FROM
lineitem
WHERE
l_shipdate <= date '1998-12-01' - INTERVAL '90' DAY
l_shipdate <= DATE '1998-12-01' - INTERVAL '90' DAY
GROUP BY
l_returnflag,
l_linestatus
@ -234,6 +234,62 @@ ORDER BY
p_partkey;
```
::::note
As of October 2024, the query does not work out-of-the box due to correlated subqueries. Corresponding issue: https://github.com/ClickHouse/ClickHouse/issues/6697
This alternative formulation works and was verified to return the reference results.
```sql
WITH MinSupplyCost AS (
SELECT
ps_partkey,
MIN(ps_supplycost) AS min_supplycost
FROM
partsupp ps
JOIN
supplier s ON ps.ps_suppkey = s.s_suppkey
JOIN
nation n ON s.s_nationkey = n.n_nationkey
JOIN
region r ON n.n_regionkey = r.r_regionkey
WHERE
r.r_name = 'EUROPE'
GROUP BY
ps_partkey
)
SELECT
s.s_acctbal,
s.s_name,
n.n_name,
p.p_partkey,
p.p_mfgr,
s.s_address,
s.s_phone,
s.s_comment
FROM
part p
JOIN
partsupp ps ON p.p_partkey = ps.ps_partkey
JOIN
supplier s ON s.s_suppkey = ps.ps_suppkey
JOIN
nation n ON s.s_nationkey = n.n_nationkey
JOIN
region r ON n.n_regionkey = r.r_regionkey
JOIN
MinSupplyCost msc ON ps.ps_partkey = msc.ps_partkey AND ps.ps_supplycost = msc.min_supplycost
WHERE
p.p_size = 15
AND p.p_type LIKE '%BRASS'
AND r.r_name = 'EUROPE'
ORDER BY
s.s_acctbal DESC,
n.n_name,
s.s_name,
p.p_partkey;
```
::::
**Q3**
```sql
@ -250,8 +306,8 @@ WHERE
c_mktsegment = 'BUILDING'
AND c_custkey = o_custkey
AND l_orderkey = o_orderkey
AND o_orderdate < date '1995-03-15'
AND l_shipdate > date '1995-03-15'
AND o_orderdate < DATE '1995-03-15'
AND l_shipdate > DATE '1995-03-15'
GROUP BY
l_orderkey,
o_orderdate,
@ -270,8 +326,8 @@ SELECT
FROM
orders
WHERE
o_orderdate >= date '1993-07-01'
AND o_orderdate < date '1993-07-01' + INTERVAL '3' MONTH
o_orderdate >= DATE '1993-07-01'
AND o_orderdate < DATE '1993-07-01' + INTERVAL '3' MONTH
AND EXISTS (
SELECT
*
@ -287,6 +343,39 @@ ORDER BY
o_orderpriority;
```
::::note
As of October 2024, the query does not work out-of-the box due to correlated subqueries. Corresponding issue: https://github.com/ClickHouse/ClickHouse/issues/6697
This alternative formulation works and was verified to return the reference results.
```sql
WITH ValidLineItems AS (
SELECT
l_orderkey
FROM
lineitem
WHERE
l_commitdate < l_receiptdate
GROUP BY
l_orderkey
)
SELECT
o.o_orderpriority,
COUNT(*) AS order_count
FROM
orders o
JOIN
ValidLineItems vli ON o.o_orderkey = vli.l_orderkey
WHERE
o.o_orderdate >= DATE '1993-07-01'
AND o.o_orderdate < DATE '1993-07-01' + INTERVAL '3' MONTH
GROUP BY
o.o_orderpriority
ORDER BY
o.o_orderpriority;
```
::::
**Q5**
```sql
@ -308,8 +397,8 @@ WHERE
AND s_nationkey = n_nationkey
AND n_regionkey = r_regionkey
AND r_name = 'ASIA'
AND o_orderdate >= date '1994-01-01'
AND o_orderdate < date '1994-01-01' + INTERVAL '1' year
AND o_orderdate >= DATE '1994-01-01'
AND o_orderdate < DATE '1994-01-01' + INTERVAL '1' year
GROUP BY
n_name
ORDER BY
@ -324,12 +413,30 @@ SELECT
FROM
lineitem
WHERE
l_shipdate >= date '1994-01-01'
AND l_shipdate < date '1994-01-01' + INTERVAL '1' year
l_shipdate >= DATE '1994-01-01'
AND l_shipdate < DATE '1994-01-01' + INTERVAL '1' year
AND l_discount BETWEEN 0.06 - 0.01 AND 0.06 + 0.01
AND l_quantity < 24;
```
::::note
As of October 2024, the query does not work out-of-the box due to a bug with Decimal addition. Corresponding issue: https://github.com/ClickHouse/ClickHouse/issues/70136
This alternative formulation works and was verified to return the reference results.
```sql
SELECT
sum(l_extendedprice * l_discount) AS revenue
FROM
lineitem
WHERE
l_shipdate >= DATE '1994-01-01'
AND l_shipdate < DATE '1994-01-01' + INTERVAL '1' year
AND l_discount BETWEEN 0.05 AND 0.07
AND l_quantity < 24;
```
::::
**Q7**
```sql
@ -361,7 +468,7 @@ FROM (
(n1.n_name = 'FRANCE' AND n2.n_name = 'GERMANY')
OR (n1.n_name = 'GERMANY' AND n2.n_name = 'FRANCE')
)
AND l_shipdate BETWEEN date '1995-01-01' AND date '1996-12-31'
AND l_shipdate BETWEEN DATE '1995-01-01' AND DATE '1996-12-31'
) AS shipping
GROUP BY
supp_nation,
@ -406,7 +513,7 @@ FROM (
AND n1.n_regionkey = r_regionkey
AND r_name = 'AMERICA'
AND s_nationkey = n2.n_nationkey
AND o_orderdate BETWEEN date '1995-01-01' AND date '1996-12-31'
AND o_orderdate BETWEEN DATE '1995-01-01' AND DATE '1996-12-31'
AND p_type = 'ECONOMY ANODIZED STEEL'
) AS all_nations
GROUP BY
@ -471,8 +578,8 @@ FROM
WHERE
c_custkey = o_custkey
AND l_orderkey = o_orderkey
AND o_orderdate >= date '1993-10-01'
AND o_orderdate < date '1993-10-01' + INTERVAL '3' MONTH
AND o_orderdate >= DATE '1993-10-01'
AND o_orderdate < DATE '1993-10-01' + INTERVAL '3' MONTH
AND l_returnflag = 'R'
AND c_nationkey = n_nationkey
GROUP BY
@ -544,8 +651,8 @@ WHERE
AND l_shipmode in ('MAIL', 'SHIP')
AND l_commitdate < l_receiptdate
AND l_shipdate < l_commitdate
AND l_receiptdate >= date '1994-01-01'
AND l_receiptdate < date '1994-01-01' + INTERVAL '1' year
AND l_receiptdate >= DATE '1994-01-01'
AND l_receiptdate < DATE '1994-01-01' + INTERVAL '1' year
GROUP BY
l_shipmode
ORDER BY
@ -576,6 +683,37 @@ ORDER BY
c_count DESC;
```
::::note
As of October 2024, the query does not work out-of-the box due to correlated subqueries. Corresponding issue: https://github.com/ClickHouse/ClickHouse/issues/6697
This alternative formulation works and was verified to return the reference results.
```sql
WITH CustomerOrderCounts AS (
SELECT
c.c_custkey,
count(o.o_orderkey) AS order_count
FROM
customer c
LEFT OUTER JOIN
orders o ON c.c_custkey = o.o_custkey
AND o.o_comment NOT LIKE '%special%requests%'
GROUP BY
c.c_custkey
)
SELECT
order_count AS c_count,
count(*) AS custdist
FROM
CustomerOrderCounts
GROUP BY
order_count
ORDER BY
custdist DESC,
c_count DESC;
```
::::
**Q14**
```sql
@ -590,8 +728,8 @@ FROM
part
WHERE
l_partkey = p_partkey
AND l_shipdate >= date '1995-09-01'
AND l_shipdate < date '1995-09-01' + INTERVAL '1' MONTH;
AND l_shipdate >= DATE '1995-09-01'
AND l_shipdate < DATE '1995-09-01' + INTERVAL '1' MONTH;
```
**Q15**
@ -604,8 +742,8 @@ CREATE VIEW revenue0 (supplier_no, total_revenue) AS
FROM
lineitem
WHERE
l_shipdate >= date '1996-01-01'
AND l_shipdate < date '1996-01-01' + INTERVAL '3' MONTH
l_shipdate >= DATE '1996-01-01'
AND l_shipdate < DATE '1996-01-01' + INTERVAL '3' MONTH
GROUP BY
l_suppkey;
@ -632,6 +770,26 @@ ORDER BY
DROP VIEW revenue0;
```
::::note
As of October 2024, the view definition does not work out-of-the box. Corresponding issue: https://github.com/ClickHouse/ClickHouse/issues/70139
This alternative view definition does work:
```sql
CREATE VIEW revenue0 AS
SELECT
l_suppkey AS supplier_no,
sum(l_extendedprice * (1 - l_discount)) AS total_revenue
FROM
lineitem
WHERE
l_shipdate >= DATE '1996-01-01'
AND l_shipdate < DATE '1996-01-01' + INTERVAL '3' MONTH
GROUP BY
l_suppkey;
```
::::
**Q16**
```sql
@ -689,6 +847,37 @@ WHERE
);
```
::::note
As of October 2024, the query does not work out-of-the box due to correlated subqueries. Corresponding issue: https://github.com/ClickHouse/ClickHouse/issues/6697
This alternative formulation works and was verified to return the reference results.
```sql
WITH AvgQuantity AS (
SELECT
l_partkey,
AVG(l_quantity) * 0.2 AS avg_quantity
FROM
lineitem
GROUP BY
l_partkey
)
SELECT
SUM(l.l_extendedprice) / 7.0 AS avg_yearly
FROM
lineitem l
JOIN
part p ON p.p_partkey = l.l_partkey
JOIN
AvgQuantity aq ON l.l_partkey = aq.l_partkey
WHERE
p.p_brand = 'Brand#23'
AND p.p_container = 'MED BOX'
AND l.l_quantity < aq.avg_quantity;
```
::::
**Q18**
```sql
@ -731,7 +920,7 @@ ORDER BY
```sql
SELECT
sum(l_extendedprice* (1 - l_discount)) AS revenue
sum(l_extendedprice * (1 - l_discount)) AS revenue
FROM
lineitem,
part
@ -767,6 +956,46 @@ WHERE
);
```
::::note
As of October 2024, the query is extremely slow due to missing join predicate pushdown. Corresponding issue: https://github.com/ClickHouse/ClickHouse/issues/70802
This alternative formulation works and was verified to return the reference results.
```sql
SELECT
sum(l_extendedprice * (1 - l_discount)) AS revenue
FROM
lineitem,
part
WHERE
p_partkey = l_partkey
AND l_shipinstruct = 'DELIVER IN PERSON'
AND l_shipmode IN ('AIR', 'AIR REG')
AND (
(
p_brand = 'Brand#12'
AND p_container IN ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG')
AND l_quantity >= 1 AND l_quantity <= 1 + 10
AND p_size BETWEEN 1 AND 5
)
OR
(
p_brand = 'Brand#23'
AND p_container IN ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK')
AND l_quantity >= 10 AND l_quantity <= 10 + 10
AND p_size BETWEEN 1 AND 10
)
OR
(
p_brand = 'Brand#34'
AND p_container IN ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG')
AND l_quantity >= 20 AND l_quantity <= 20 + 10
AND p_size BETWEEN 1 AND 15
)
)
```
::::
**Q20**
```sql
@ -789,7 +1018,7 @@ WHERE
FROM
part
WHERE
p_name LIKE 'forrest%'
p_name LIKE 'forest%'
)
AND ps_availqty > (
SELECT
@ -799,8 +1028,8 @@ WHERE
WHERE
l_partkey = ps_partkey
AND l_suppkey = ps_suppkey
AND l_shipdate >= date '1994-01-01'
AND l_shipdate < date '1994-01-01' + INTERVAL '1' year
AND l_shipdate >= DATE '1994-01-01'
AND l_shipdate < DATE '1994-01-01' + INTERVAL '1' year
)
)
AND s_nationkey = n_nationkey
@ -809,6 +1038,10 @@ ORDER BY
s_name;
```
::::note
As of October 2024, the query does not work out-of-the box due to correlated subqueries. Corresponding issue: https://github.com/ClickHouse/ClickHouse/issues/6697
::::
**Q21**
```sql
@ -852,6 +1085,9 @@ ORDER BY
numwait DESC,
s_name;
```
::::note
As of October 2024, the query does not work out-of-the box due to correlated subqueries. Corresponding issue: https://github.com/ClickHouse/ClickHouse/issues/6697
::::
**Q22**
@ -860,34 +1096,33 @@ SELECT
cntrycode,
count(*) AS numcust,
sum(c_acctbal) AS totacctbal
FROM
(
SELECT
substring(c_phone FROM 1 for 2) AS cntrycode,
c_acctbal
FROM
customer
WHERE
substring(c_phone FROM 1 for 2) in
('13', '31', '23', '29', '30', '18', '17')
AND c_acctbal > (
SELECT
avg(c_acctbal)
FROM
customer
WHERE
c_acctbal > 0.00
AND substring(c_phone FROM 1 for 2) in
('13', '31', '23', '29', '30', '18', '17')
)
AND NOT EXISTS (
SELECT
*
FROM
orders
WHERE
o_custkey = c_custkey
)
FROM (
SELECT
substring(c_phone FROM 1 for 2) AS cntrycode,
c_acctbal
FROM
customer
WHERE
substring(c_phone FROM 1 for 2) in
('13', '31', '23', '29', '30', '18', '17')
AND c_acctbal > (
SELECT
avg(c_acctbal)
FROM
customer
WHERE
c_acctbal > 0.00
AND substring(c_phone FROM 1 for 2) in
('13', '31', '23', '29', '30', '18', '17')
)
AND NOT EXISTS (
SELECT
*
FROM
orders
WHERE
o_custkey = c_custkey
)
) AS custsale
GROUP BY
cntrycode

21
src/Common/Config/ConfigProcessor.cpp
View File

@ -671,9 +671,11 @@ XMLDocumentPtr ConfigProcessor::parseConfig(const std::string & config_path)
XMLDocumentPtr ConfigProcessor::processConfig(
bool * has_zk_includes,
zkutil::ZooKeeperNodeCache * zk_node_cache,
const zkutil::EventPtr & zk_changed_event)
const zkutil::EventPtr & zk_changed_event,
bool is_config_changed)
{
LOG_DEBUG(log, "Processing configuration file '{}'.", path);
if (is_config_changed)
LOG_DEBUG(log, "Processing configuration file '{}'.", path);
XMLDocumentPtr config;
@ -686,7 +688,8 @@ XMLDocumentPtr ConfigProcessor::processConfig(
/// When we can use a config embedded in the binary.
if (auto it = embedded_configs.find(path); it != embedded_configs.end())
{
LOG_DEBUG(log, "There is no file '{}', will use embedded config.", path);
if (is_config_changed)
LOG_DEBUG(log, "There is no file '{}', will use embedded config.", path);
config = dom_parser.parseMemory(it->second.data(), it->second.size());
}
else
@ -700,7 +703,8 @@ XMLDocumentPtr ConfigProcessor::processConfig(
{
try
{
LOG_DEBUG(log, "Merging configuration file '{}'.", merge_file);
if (is_config_changed)
LOG_DEBUG(log, "Merging configuration file '{}'.", merge_file);
XMLDocumentPtr with;
with = parseConfig(merge_file);
@ -790,10 +794,10 @@ XMLDocumentPtr ConfigProcessor::processConfig(
return config;
}
ConfigProcessor::LoadedConfig ConfigProcessor::loadConfig(bool allow_zk_includes)
ConfigProcessor::LoadedConfig ConfigProcessor::loadConfig(bool allow_zk_includes, bool is_config_changed)
{
bool has_zk_includes;
XMLDocumentPtr config_xml = processConfig(&has_zk_includes);
XMLDocumentPtr config_xml = processConfig(&has_zk_includes, nullptr, nullptr, is_config_changed);
if (has_zk_includes && !allow_zk_includes)
throw Poco::Exception("Error while loading config '" + path + "': from_zk includes are not allowed!");
@ -806,14 +810,15 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfig(bool allow_zk_includes
ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes(
zkutil::ZooKeeperNodeCache & zk_node_cache,
const zkutil::EventPtr & zk_changed_event,
bool fallback_to_preprocessed)
bool fallback_to_preprocessed,
bool is_config_changed)
{
XMLDocumentPtr config_xml;
bool has_zk_includes;
bool processed_successfully = false;
try
{
config_xml = processConfig(&has_zk_includes, &zk_node_cache, zk_changed_event);
config_xml = processConfig(&has_zk_includes, &zk_node_cache, zk_changed_event, is_config_changed);
processed_successfully = true;
}
catch (const Poco::Exception & ex)

8
src/Common/Config/ConfigProcessor.h
View File

@ -63,7 +63,8 @@ public:
XMLDocumentPtr processConfig(
bool * has_zk_includes = nullptr,
zkutil::ZooKeeperNodeCache * zk_node_cache = nullptr,
const zkutil::EventPtr & zk_changed_event = nullptr);
const zkutil::EventPtr & zk_changed_event = nullptr,
bool is_config_changed = true);
XMLDocumentPtr parseConfig(const std::string & config_path);
@ -88,14 +89,15 @@ public:
/// If allow_zk_includes is true, expect that the configuration XML can contain from_zk nodes.
/// If it is the case, set has_zk_includes to true and don't write config-preprocessed.xml,
/// expecting that config would be reloaded with zookeeper later.
LoadedConfig loadConfig(bool allow_zk_includes = false);
LoadedConfig loadConfig(bool allow_zk_includes = false, bool is_config_changed = true);
/// If fallback_to_preprocessed is true, then if KeeperException is thrown during config
/// processing, load the configuration from the preprocessed file.
LoadedConfig loadConfigWithZooKeeperIncludes(
zkutil::ZooKeeperNodeCache & zk_node_cache,
const zkutil::EventPtr & zk_changed_event,
bool fallback_to_preprocessed = false);
bool fallback_to_preprocessed = false,
bool is_config_changed = true);
/// Save preprocessed config to specified directory.
/// If preprocessed_dir is empty - calculate from loaded_config.path + /preprocessed_configs/

7
src/Common/Config/ConfigReloader.cpp
View File

@ -111,7 +111,8 @@ std::optional<ConfigProcessor::LoadedConfig> ConfigReloader::reloadIfNewer(bool
std::lock_guard lock(reload_mutex);
FilesChangesTracker new_files = getNewFileList();
if (force || need_reload_from_zk || new_files.isDifferOrNewerThan(files))
const bool is_config_changed = new_files.isDifferOrNewerThan(files);
if (force || need_reload_from_zk || is_config_changed)
{
ConfigProcessor config_processor(config_path);
ConfigProcessor::LoadedConfig loaded_config;
@ -120,10 +121,10 @@ std::optional<ConfigProcessor::LoadedConfig> ConfigReloader::reloadIfNewer(bool
try
{
loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true);
loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true, is_config_changed);
if (loaded_config.has_zk_includes)
loaded_config = config_processor.loadConfigWithZooKeeperIncludes(
zk_node_cache, zk_changed_event, fallback_to_preprocessed);
zk_node_cache, zk_changed_event, fallback_to_preprocessed, is_config_changed);
}
catch (const Coordination::Exception & e)
{

45
src/Disks/StoragePolicy.cpp
View File

@ -342,24 +342,45 @@ VolumePtr StoragePolicy::tryGetVolumeByName(const String & volume_name) const
void StoragePolicy::checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const
{
std::unordered_set<String> new_volume_names;
for (const auto & volume : new_storage_policy->getVolumes())
new_volume_names.insert(volume->getName());
/// Do not check volumes for temporary policy because their names are automatically generated
bool skip_volume_check = this->getName().starts_with(StoragePolicySelector::TMP_STORAGE_POLICY_PREFIX)
|| new_storage_policy->getName().starts_with(StoragePolicySelector::TMP_STORAGE_POLICY_PREFIX);
for (const auto & volume : getVolumes())
if (!skip_volume_check)
{
if (!new_volume_names.contains(volume->getName()))
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"New storage policy {} shall contain volumes of the old storage policy {}",
backQuote(new_storage_policy->getName()),
backQuote(name));
std::unordered_set<String> new_volume_names;
for (const auto & volume : new_storage_policy->getVolumes())
new_volume_names.insert(volume->getName());
for (const auto & volume : getVolumes())
{
if (!new_volume_names.contains(volume->getName()))
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"New storage policy {} shall contain volumes of the old storage policy {}",
backQuote(new_storage_policy->getName()),
backQuote(name));
std::unordered_set<String> new_disk_names;
for (const auto & disk : new_storage_policy->getVolumeByName(volume->getName())->getDisks())
new_disk_names.insert(disk->getName());
for (const auto & disk : volume->getDisks())
if (!new_disk_names.contains(disk->getName()))
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"New storage policy {} shall contain disks of the old storage policy {}",
backQuote(new_storage_policy->getName()),
backQuote(name));
}
}
else
{
std::unordered_set<String> new_disk_names;
for (const auto & disk : new_storage_policy->getVolumeByName(volume->getName())->getDisks())
for (const auto & disk : new_storage_policy->getDisks())
new_disk_names.insert(disk->getName());
for (const auto & disk : volume->getDisks())
for (const auto & disk : this->getDisks())
if (!new_disk_names.contains(disk->getName()))
throw Exception(
ErrorCodes::BAD_ARGUMENTS,

24
src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -3643,6 +3643,9 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
const auto & new_changes = new_metadata.settings_changes->as<const ASTSetQuery &>().changes;
local_context->checkMergeTreeSettingsConstraints(*settings_from_storage, new_changes);
bool found_disk_setting = false;
bool found_storage_policy_setting = false;
for (const auto & changed_setting : new_changes)
{
const auto & setting_name = changed_setting.name;
@ -3666,9 +3669,22 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
}
if (setting_name == "storage_policy")
{
checkStoragePolicy(local_context->getStoragePolicy(new_value.safeGet<String>()));
found_storage_policy_setting = true;
}
else if (setting_name == "disk")
{
checkStoragePolicy(local_context->getStoragePolicyFromDisk(new_value.safeGet<String>()));
found_disk_setting = true;
}
}
if (found_storage_policy_setting && found_disk_setting)
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"MergeTree settings `storage_policy` and `disk` cannot be specified at the same time");
/// Check if it is safe to reset the settings
for (const auto & current_setting : current_changes)
{
@ -3767,12 +3783,16 @@ void MergeTreeData::changeSettings(
bool has_storage_policy_changed = false;
const auto & new_changes = new_settings->as<const ASTSetQuery &>().changes;
StoragePolicyPtr new_storage_policy = nullptr;
for (const auto & change : new_changes)
{
if (change.name == "storage_policy")
if (change.name == "disk" || change.name == "storage_policy")
{
StoragePolicyPtr new_storage_policy = getContext()->getStoragePolicy(change.value.safeGet<String>());
if (change.name == "disk")
new_storage_policy = getContext()->getStoragePolicyFromDisk(change.value.safeGet<String>());
else
new_storage_policy = getContext()->getStoragePolicy(change.value.safeGet<String>());
StoragePolicyPtr old_storage_policy = getStoragePolicy();
/// StoragePolicy of different version or name is guaranteed to have different pointer

2
src/Storages/MergeTree/MergeTreeSettings.cpp
View File

@ -331,7 +331,7 @@ void MergeTreeSettingsImpl::loadFromQuery(ASTStorage & storage_def, ContextPtr c
else if (name == "storage_policy")
found_storage_policy_setting = true;
if (found_disk_setting && found_storage_policy_setting)
if (!is_attach && found_disk_setting && found_storage_policy_setting)
{
throw Exception(
ErrorCodes::BAD_ARGUMENTS,

2
tests/integration/test_disk_configuration/configs/config.d/mergetree_settings.xml
View File

@ -1,5 +1,5 @@
<clickhouse>
<merge_tree>
<storage_policy>unknown</storage_policy>
<storage_policy>hybrid</storage_policy>
</merge_tree>
</clickhouse>

18
tests/integration/test_disk_configuration/configs/config.d/storage_configuration.xml
View File

@ -20,6 +20,24 @@
</main>
</volumes>
</local>
<s3>
<volumes>
<s3>
<disk>s3</disk>
</s3>
</volumes>
</s3>
<hybrid>
<volumes>
<disk_local>
<disk>disk_local</disk>
</disk_local>
<s3>
<disk>s3</disk>
</s3>
</volumes>
</hybrid>
</policies>
</storage_configuration>
</clickhouse>

59
tests/integration/test_disk_configuration/test.py
View File

@ -381,12 +381,69 @@ def test_merge_tree_setting_override(start_cluster):
)
)
assert (
"MergeTree settings `storage_policy` and `disk` cannot be specified at the same time"
in node.query_and_get_error(
f"""
DROP TABLE IF EXISTS {TABLE_NAME} SYNC;
CREATE TABLE {TABLE_NAME} (a Int32)
ENGINE = MergeTree()
ORDER BY tuple()
SETTINGS storage_policy = 's3';
ALTER TABLE {TABLE_NAME} MODIFY SETTING disk = 's3';
"""
)
)
assert (
"MergeTree settings `storage_policy` and `disk` cannot be specified at the same time"
in node.query_and_get_error(
f"""
DROP TABLE IF EXISTS {TABLE_NAME} SYNC;
CREATE TABLE {TABLE_NAME} (a Int32)
ENGINE = MergeTree()
ORDER BY tuple()
SETTINGS disk = 's3';
ALTER TABLE {TABLE_NAME} MODIFY SETTING storage_policy = 's3';
"""
)
)
assert (
"New storage policy `local` shall contain volumes of the old storage policy `s3`"
in node.query_and_get_error(
f"""
DROP TABLE IF EXISTS {TABLE_NAME};
CREATE TABLE {TABLE_NAME} (a Int32)
ENGINE = MergeTree()
ORDER BY tuple()
SETTINGS storage_policy = 's3';
ALTER TABLE {TABLE_NAME} MODIFY SETTING storage_policy = 'local';
"""
)
)
# Using default policy so storage_policy and disk are not set at the same time
assert (
"New storage policy `__disk_local` shall contain disks of the old storage policy `hybrid`"
in node.query_and_get_error(
f"""
DROP TABLE IF EXISTS {TABLE_NAME};
CREATE TABLE {TABLE_NAME} (a Int32)
ENGINE = MergeTree()
ORDER BY tuple();
ALTER TABLE {TABLE_NAME} MODIFY SETTING disk = 'disk_local';
"""
)
)
assert "Unknown storage policy" in node.query_and_get_error(
f"""
DROP TABLE IF EXISTS {TABLE_NAME};
CREATE TABLE {TABLE_NAME} (a Int32)
ENGINE = MergeTree()
ORDER BY tuple();
ORDER BY tuple()
SETTINGS storage_policy = 'kek';
"""
)