mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-30 03:22:14 +00:00
Merge with master
This commit is contained in:
alesapin
commit
10312a1d88
@ -217,9 +217,18 @@ const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(si
|
||||
|
||||
void SettingsConstraints::setProfile(const String & profile_name, const Poco::Util::AbstractConfiguration & config)
|
||||
{
|
||||
String parent_profile = "profiles." + profile_name + ".profile";
|
||||
if (config.has(parent_profile))
|
||||
setProfile(parent_profile, config); // Inheritance of one profile from another.
|
||||
String elem = "profiles." + profile_name;
|
||||
|
||||
Poco::Util::AbstractConfiguration::Keys config_keys;
|
||||
config.keys(elem, config_keys);
|
||||
|
||||
for (const std::string & key : config_keys)
|
||||
{
|
||||
if (key == "profile" || 0 == key.compare(0, strlen("profile["), "profile[")) /// Inheritance of profiles from the current one.
|
||||
setProfile(config.getString(elem + "." + key), config);
|
||||
else
|
||||
continue;
|
||||
}
|
||||
|
||||
String path_to_constraints = "profiles." + profile_name + ".constraints";
|
||||
if (config.has(path_to_constraints))
|
||||
|
||||
@ -37,7 +37,7 @@ void Settings::setProfile(const String & profile_name, const Poco::Util::Abstrac
|
||||
{
|
||||
if (key == "constraints")
|
||||
continue;
|
||||
if (key == "profile") /// Inheritance of one profile from another.
|
||||
if (key == "profile" || 0 == key.compare(0, strlen("profile["), "profile[")) /// Inheritance of profiles from the current one.
|
||||
setProfile(config.getString(elem + "." + key), config);
|
||||
else
|
||||
set(key, config.getString(elem + "." + key));
|
||||
|
||||
@ -941,22 +941,19 @@ void DDLWorker::runMainThread()
|
||||
{
|
||||
try
|
||||
{
|
||||
try
|
||||
{
|
||||
auto zookeeper = getAndSetZooKeeper();
|
||||
zookeeper->createAncestors(queue_dir + "/");
|
||||
initialized = true;
|
||||
}
|
||||
catch (const Coordination::Exception & e)
|
||||
{
|
||||
if (!Coordination::isHardwareError(e.code))
|
||||
throw; /// A logical error.
|
||||
auto zookeeper = getAndSetZooKeeper();
|
||||
zookeeper->createAncestors(queue_dir + "/");
|
||||
initialized = true;
|
||||
}
|
||||
catch (const Coordination::Exception & e)
|
||||
{
|
||||
if (!Coordination::isHardwareError(e.code))
|
||||
throw; /// A logical error.
|
||||
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
|
||||
/// Avoid busy loop when ZooKeeper is not available.
|
||||
sleepForSeconds(1);
|
||||
}
|
||||
/// Avoid busy loop when ZooKeeper is not available.
|
||||
sleepForSeconds(1);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
|
||||
@ -0,0 +1,59 @@
|
||||
<yandex>
|
||||
<profiles>
|
||||
<profile_1>
|
||||
<max_parallel_replicas>2</max_parallel_replicas>
|
||||
<!-- max_query_size is inherited one by one and final should be 400000000 -->
|
||||
<max_query_size>200000000</max_query_size>
|
||||
<constraints>
|
||||
<max_memory_usage>
|
||||
<min>100000000</min>
|
||||
</max_memory_usage>
|
||||
<max_parallel_replicas>
|
||||
<readonly/>
|
||||
</max_parallel_replicas>
|
||||
</constraints>
|
||||
</profile_1>
|
||||
<profile_2>
|
||||
<max_network_bytes>1234567890</max_network_bytes>
|
||||
<max_query_size>300000000</max_query_size>
|
||||
<constraints>
|
||||
<max_memory_usage>
|
||||
<min>200000000</min>
|
||||
</max_memory_usage>
|
||||
<max_network_bytes>
|
||||
<min>1234567889</min>
|
||||
<max>1234567891</max>
|
||||
</max_network_bytes>
|
||||
</constraints>
|
||||
</profile_2>
|
||||
<profile_3>
|
||||
<max_insert_block_size>654321</max_insert_block_size>
|
||||
<max_query_size>400000000</max_query_size>
|
||||
<constraints>
|
||||
<max_memory_usage>
|
||||
<min>300000000</min>
|
||||
</max_memory_usage>
|
||||
<max_insert_block_size>
|
||||
<min>654320</min>
|
||||
<max>654322</max>
|
||||
</max_insert_block_size>
|
||||
</constraints>
|
||||
</profile_3>
|
||||
<combined_profile>
|
||||
<profile>profile_1</profile>
|
||||
<profile>profile_2</profile>
|
||||
<profile>profile_3</profile>
|
||||
<readonly>2</readonly>
|
||||
</combined_profile>
|
||||
</profiles>
|
||||
<users>
|
||||
<test_combined_profile>
|
||||
<password></password>
|
||||
<networks>
|
||||
<ip>::/0</ip>
|
||||
</networks>
|
||||
<quota>default</quota>
|
||||
<profile>combined_profile</profile>
|
||||
</test_combined_profile>
|
||||
</users>
|
||||
</yandex>
|
||||
@ -0,0 +1,74 @@
|
||||
import pytest
|
||||
|
||||
from helpers.client import QueryRuntimeException
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.test_tools import TSV
|
||||
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
instance = cluster.add_instance('instance',
|
||||
user_configs=['configs/combined_profile.xml'])
|
||||
q = instance.query
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
|
||||
yield cluster
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def test_combined_profile(started_cluster):
|
||||
settings = q('''
|
||||
SELECT name, value FROM system.settings
|
||||
WHERE name IN
|
||||
('max_insert_block_size', 'max_network_bytes', 'max_query_size',
|
||||
'max_parallel_replicas', 'readonly')
|
||||
AND changed
|
||||
ORDER BY name
|
||||
''', user='test_combined_profile')
|
||||
|
||||
expected1 = '''\
|
||||
max_insert_block_size 654321
|
||||
max_network_bytes 1234567890
|
||||
max_parallel_replicas 2
|
||||
max_query_size 400000000
|
||||
readonly 2'''
|
||||
|
||||
assert TSV(settings) == TSV(expected1)
|
||||
|
||||
with pytest.raises(QueryRuntimeException) as exc:
|
||||
q('''
|
||||
SET max_insert_block_size = 1000;
|
||||
''', user='test_combined_profile')
|
||||
|
||||
assert ("max_insert_block_size shouldn't be less than 654320." in
|
||||
str(exc.value))
|
||||
|
||||
with pytest.raises(QueryRuntimeException) as exc:
|
||||
q('''
|
||||
SET max_network_bytes = 2000000000;
|
||||
''', user='test_combined_profile')
|
||||
|
||||
assert ("max_network_bytes shouldn't be greater than 1234567891." in
|
||||
str(exc.value))
|
||||
|
||||
with pytest.raises(QueryRuntimeException) as exc:
|
||||
q('''
|
||||
SET max_parallel_replicas = 1000;
|
||||
''', user='test_combined_profile')
|
||||
|
||||
assert ('max_parallel_replicas should not be changed.' in
|
||||
str(exc.value))
|
||||
|
||||
with pytest.raises(QueryRuntimeException) as exc:
|
||||
q('''
|
||||
SET max_memory_usage = 1000;
|
||||
''', user='test_combined_profile')
|
||||
|
||||
assert ("max_memory_usage shouldn't be less than 300000000." in
|
||||
str(exc.value))
|
||||
@ -538,3 +538,85 @@ def test_ttls_do_not_work_after_alter(started_cluster, name, engine, positive):
|
||||
|
||||
finally:
|
||||
node1.query("DROP TABLE IF EXISTS {}".format(name))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name,engine,positive", [
|
||||
("mt_test_alter_multiple_ttls_positive", "MergeTree()", True),
|
||||
("mt_replicated_test_alter_multiple_ttls_positive", "ReplicatedMergeTree('/clickhouse/replicated_test_alter_multiple_ttls_positive', '1')", True),
|
||||
("mt_test_alter_multiple_ttls_negative", "MergeTree()", False),
|
||||
("mt_replicated_test_alter_multiple_ttls_negative", "ReplicatedMergeTree('/clickhouse/replicated_test_alter_multiple_ttls_negative', '1')", False),
|
||||
])
|
||||
def test_alter_multiple_ttls(started_cluster, name, engine, positive):
|
||||
"""Copyright 2019, Altinity LTD
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License."""
|
||||
"""Check that when multiple TTL expressions are set
|
||||
and before any parts are inserted the TTL expressions
|
||||
are changed with ALTER command then all old
|
||||
TTL expressions are removed and the
|
||||
the parts are moved to the specified disk or volume or
|
||||
deleted if the new TTL expression is triggered
|
||||
and are not moved or deleted when it is not.
|
||||
"""
|
||||
now = time.time()
|
||||
try:
|
||||
node1.query("""
|
||||
CREATE TABLE {name} (
|
||||
p1 Int64,
|
||||
s1 String,
|
||||
d1 DateTime
|
||||
) ENGINE = {engine}
|
||||
ORDER BY tuple()
|
||||
PARTITION BY p1
|
||||
TTL d1 + INTERVAL 30 SECOND TO DISK 'jbod2',
|
||||
d1 + INTERVAL 60 SECOND TO VOLUME 'external'
|
||||
SETTINGS storage_policy='jbods_with_external', merge_with_ttl_timeout=0
|
||||
""".format(name=name, engine=engine))
|
||||
|
||||
node1.query("""
|
||||
ALTER TABLE {name} MODIFY
|
||||
TTL d1 + INTERVAL 0 SECOND TO DISK 'jbod2',
|
||||
d1 + INTERVAL 5 SECOND TO VOLUME 'external',
|
||||
d1 + INTERVAL 10 SECOND DELETE
|
||||
""".format(name=name))
|
||||
|
||||
for p in range(3):
|
||||
data = [] # 6MB in total
|
||||
now = time.time()
|
||||
for i in range(2):
|
||||
p1 = p
|
||||
s1 = get_random_string(1024 * 1024) # 1MB
|
||||
d1 = now - 1 if i > 0 or positive else now + 300
|
||||
data.append("({}, '{}', toDateTime({}))".format(p1, s1, d1))
|
||||
node1.query("INSERT INTO {name} (p1, s1, d1) VALUES {values}".format(name=name, values=",".join(data)))
|
||||
|
||||
used_disks = get_used_disks_for_table(node1, name)
|
||||
assert set(used_disks) == {"jbod2"} if positive else {"jbod1", "jbod2"}
|
||||
|
||||
assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["6"]
|
||||
|
||||
time.sleep(5)
|
||||
|
||||
used_disks = get_used_disks_for_table(node1, name)
|
||||
assert set(used_disks) == {"external"} if positive else {"jbod1", "jbod2"}
|
||||
|
||||
assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["6"]
|
||||
|
||||
time.sleep(5)
|
||||
|
||||
node1.query("OPTIMIZE TABLE {name} FINAL".format(name=name))
|
||||
|
||||
assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["0"] if positive else ["3"]
|
||||
|
||||
finally:
|
||||
node1.query("DROP TABLE IF EXISTS {name}".format(name=name))
|
||||
|
||||
@ -60,7 +60,7 @@ Example:
|
||||
|
||||
The example specifies two profiles: `default` and `web`. The `default` profile has a special purpose: it must always be present and is applied when starting the server. In other words, the `default` profile contains default settings. The `web` profile is a regular profile that can be set using the `SET` query or using a URL parameter in an HTTP query.
|
||||
|
||||
Settings profiles can inherit from each other. To use inheritance, indicate the `profile` setting before the other settings that are listed in the profile.
|
||||
Settings profiles can inherit from each other. To use inheritance, indicate one or multiple `profile` settings before the other settings that are listed in the profile. In case when one setting is defined in different profiles, the latest defined is used.
|
||||
|
||||
|
||||
[Original article](https://clickhouse.yandex/docs/en/operations/settings/settings_profiles/) <!--hide-->
|
||||
|
||||
@ -60,6 +60,6 @@ SET profile = 'web'
|
||||
|
||||
В примере задано два профиля: `default` и `web`. Профиль `default` имеет специальное значение - он всегда обязан присутствовать и применяется при запуске сервера. То есть, профиль `default` содержит настройки по умолчанию. Профиль `web` - обычный профиль, который может быть установлен с помощью запроса `SET` или с помощью параметра URL при запросе по HTTP.
|
||||
|
||||
Профили настроек могут наследоваться от друг-друга - это реализуется указанием настройки `profile` перед остальными настройками, перечисленными в профиле.
|
||||
Профили настроек могут наследоваться от друг-друга - это реализуется указанием одной или нескольких настроек `profile` перед остальными настройками, перечисленными в профиле. Если одна настройка указана в нескольких профилях, используется последнее из значений.
|
||||
|
||||
[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/settings/settings_profiles/) <!--hide-->
|
||||
|
||||
@ -1,66 +0,0 @@
|
||||
|
||||
# Settings profiles
|
||||
|
||||
A settings profile is a collection of settings grouped under the same name. Each ClickHouse user has a profile.
|
||||
To apply all the settings in a profile, set the `profile` setting.
|
||||
|
||||
Example:
|
||||
|
||||
Install the `web` profile.
|
||||
|
||||
``` sql
|
||||
SET profile = 'web'
|
||||
```
|
||||
|
||||
Settings profiles are declared in the user config file. This is usually `users.xml`.
|
||||
|
||||
Example:
|
||||
|
||||
```xml
|
||||
<!-- Settings profiles -->
|
||||
<profiles>
|
||||
<!-- Default settings -->
|
||||
<default>
|
||||
<!-- The maximum number of threads when running a single query. -->
|
||||
<max_threads>8</max_threads>
|
||||
</default>
|
||||
|
||||
<!-- Settings for quries from the user interface -->
|
||||
<web>
|
||||
<max_rows_to_read>1000000000</max_rows_to_read>
|
||||
<max_bytes_to_read>100000000000</max_bytes_to_read>
|
||||
|
||||
<max_rows_to_group_by>1000000</max_rows_to_group_by>
|
||||
<group_by_overflow_mode>any</group_by_overflow_mode>
|
||||
|
||||
<max_rows_to_sort>1000000</max_rows_to_sort>
|
||||
<max_bytes_to_sort>1000000000</max_bytes_to_sort>
|
||||
|
||||
<max_result_rows>100000</max_result_rows>
|
||||
<max_result_bytes>100000000</max_result_bytes>
|
||||
<result_overflow_mode>break</result_overflow_mode>
|
||||
|
||||
<max_execution_time>600</max_execution_time>
|
||||
<min_execution_speed>1000000</min_execution_speed>
|
||||
<timeout_before_checking_execution_speed>15</timeout_before_checking_execution_speed>
|
||||
|
||||
<max_columns_to_read>25</max_columns_to_read>
|
||||
<max_temporary_columns>100</max_temporary_columns>
|
||||
<max_temporary_non_const_columns>50</max_temporary_non_const_columns>
|
||||
|
||||
<max_subquery_depth>2</max_subquery_depth>
|
||||
<max_pipeline_depth>25</max_pipeline_depth>
|
||||
<max_ast_depth>50</max_ast_depth>
|
||||
<max_ast_elements>100</max_ast_elements>
|
||||
|
||||
<readonly>1</readonly>
|
||||
</web>
|
||||
</profiles>
|
||||
```
|
||||
|
||||
The example specifies two profiles: `default` and `web`. The `default` profile has a special purpose: it must always be present and is applied when starting the server. In other words, the `default` profile contains default settings. The `web` profile is a regular profile that can be set using the `SET` query or using a URL parameter in an HTTP query.
|
||||
|
||||
Settings profiles can inherit from each other. To use inheritance, indicate the `profile` setting before the other settings that are listed in the profile.
|
||||
|
||||
|
||||
[Original article](https://clickhouse.yandex/docs/en/operations/settings/settings_profiles/) <!--hide-->
|
||||
1
docs/zh/operations/settings/settings_profiles.md
Symbolic link
1
docs/zh/operations/settings/settings_profiles.md
Symbolic link
@ -0,0 +1 @@
|
||||
../../../en/operations/settings/settings_profiles.md
|
||||
Loading…
Reference in New Issue
Block a user