Merge remote-tracking branch 'upstream/master' into fix27

dbms/src/Access/SettingsConstraints.cpp

@@ -217,9 +217,18 @@ const SettingsConstraints::Constraint * SettingsConstraints::tryGetConstraint(si
 
 void SettingsConstraints::setProfile(const String & profile_name, const Poco::Util::AbstractConfiguration & config)
 {
-    String parent_profile = "profiles." + profile_name + ".profile";
+    String elem = "profiles." + profile_name;
-    if (config.has(parent_profile))
+
-        setProfile(parent_profile, config); // Inheritance of one profile from another.
+    Poco::Util::AbstractConfiguration::Keys config_keys;
+    config.keys(elem, config_keys);
+
+    for (const std::string & key : config_keys)
+    {
+        if (key == "profile" || 0 == key.compare(0, strlen("profile["), "profile["))   /// Inheritance of profiles from the current one.
+            setProfile(config.getString(elem + "." + key), config);
+        else
+            continue;
+    }
 
     String path_to_constraints = "profiles." + profile_name + ".constraints";
     if (config.has(path_to_constraints))

dbms/src/Core/Settings.cpp

@@ -37,7 +37,7 @@ void Settings::setProfile(const String & profile_name, const Poco::Util::Abstrac
     {
         if (key == "constraints")
             continue;
-        if (key == "profile")   /// Inheritance of one profile from another.
+        if (key == "profile" || 0 == key.compare(0, strlen("profile["), "profile["))   /// Inheritance of profiles from the current one.
             setProfile(config.getString(elem + "." + key), config);
         else
             set(key, config.getString(elem + "." + key));

dbms/src/Storages/StorageReplicatedMergeTree.cpp

@@ -534,7 +534,7 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column
 
         if (metadata_diff.ttl_table_changed)
         {
-            ParserExpression parser;
+            ParserTTLExpressionList parser;
             new_ttl_table_ast = parseQuery(parser, metadata_diff.new_ttl_table, 0);
         }
 

dbms/tests/integration/test_inherit_multiple_profiles/configs/combined_profile.xml

@@ -0,0 +1,59 @@
+<yandex>
+  <profiles>
+    <profile_1>
+      <max_parallel_replicas>2</max_parallel_replicas>
+      <!-- max_query_size is inherited one by one and final should be 400000000 -->
+      <max_query_size>200000000</max_query_size>
+      <constraints>
+        <max_memory_usage>
+          <min>100000000</min>
+        </max_memory_usage>
+        <max_parallel_replicas>
+          <readonly/>
+        </max_parallel_replicas>
+      </constraints>
+    </profile_1>
+    <profile_2>
+      <max_network_bytes>1234567890</max_network_bytes>
+      <max_query_size>300000000</max_query_size>
+      <constraints>
+        <max_memory_usage>
+          <min>200000000</min>
+        </max_memory_usage>
+        <max_network_bytes>
+          <min>1234567889</min>
+          <max>1234567891</max>
+        </max_network_bytes>
+      </constraints>
+    </profile_2>
+    <profile_3>
+      <max_insert_block_size>654321</max_insert_block_size>
+      <max_query_size>400000000</max_query_size>
+      <constraints>
+        <max_memory_usage>
+          <min>300000000</min>
+        </max_memory_usage>
+        <max_insert_block_size>
+          <min>654320</min>
+          <max>654322</max>
+        </max_insert_block_size>
+      </constraints>
+    </profile_3>
+    <combined_profile>
+      <profile>profile_1</profile>
+      <profile>profile_2</profile>
+      <profile>profile_3</profile>
+      <readonly>2</readonly>
+    </combined_profile>
+  </profiles>
+  <users>
+    <test_combined_profile>
+      <password></password>
+      <networks>
+        <ip>::/0</ip>
+      </networks>
+      <quota>default</quota>
+      <profile>combined_profile</profile>
+    </test_combined_profile>
+  </users>
+</yandex>

dbms/tests/integration/test_inherit_multiple_profiles/test.py

@@ -0,0 +1,74 @@
+import pytest
+
+from helpers.client import QueryRuntimeException
+from helpers.cluster import ClickHouseCluster
+from helpers.test_tools import TSV
+
+
+cluster = ClickHouseCluster(__file__)
+instance = cluster.add_instance('instance',
+                                user_configs=['configs/combined_profile.xml'])
+q = instance.query
+
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    try:
+        cluster.start()
+
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+def test_combined_profile(started_cluster):
+    settings = q('''
+SELECT name, value FROM system.settings
+    WHERE name IN
+      ('max_insert_block_size', 'max_network_bytes', 'max_query_size',
+       'max_parallel_replicas', 'readonly')
+    AND changed
+ORDER BY name
+''', user='test_combined_profile')
+
+    expected1 = '''\
+max_insert_block_size	654321
+max_network_bytes	1234567890
+max_parallel_replicas	2
+max_query_size	400000000
+readonly	2'''
+
+    assert TSV(settings) == TSV(expected1)
+
+    with pytest.raises(QueryRuntimeException) as exc:
+        q('''
+          SET max_insert_block_size = 1000;
+          ''', user='test_combined_profile')
+
+    assert ("max_insert_block_size shouldn't be less than 654320." in
+            str(exc.value))
+
+    with pytest.raises(QueryRuntimeException) as exc:
+        q('''
+          SET max_network_bytes = 2000000000;
+          ''', user='test_combined_profile')
+
+    assert ("max_network_bytes shouldn't be greater than 1234567891." in
+            str(exc.value))
+
+    with pytest.raises(QueryRuntimeException) as exc:
+        q('''
+          SET max_parallel_replicas = 1000;
+          ''', user='test_combined_profile')
+
+    assert ('max_parallel_replicas should not be changed.' in
+            str(exc.value))
+
+    with pytest.raises(QueryRuntimeException) as exc:
+        q('''
+          SET max_memory_usage = 1000;
+          ''', user='test_combined_profile')
+
+    assert ("max_memory_usage shouldn't be less than 300000000." in
+            str(exc.value))

dbms/tests/integration/test_ttl_move/test.py

@@ -538,3 +538,85 @@ def test_ttls_do_not_work_after_alter(started_cluster, name, engine, positive):
 
     finally:
         node1.query("DROP TABLE IF EXISTS {}".format(name))
+
+
+@pytest.mark.parametrize("name,engine,positive", [
+    ("mt_test_alter_multiple_ttls_positive", "MergeTree()", True),
+    ("mt_replicated_test_alter_multiple_ttls_positive", "ReplicatedMergeTree('/clickhouse/replicated_test_alter_multiple_ttls_positive', '1')", True),
+    ("mt_test_alter_multiple_ttls_negative", "MergeTree()", False),
+    ("mt_replicated_test_alter_multiple_ttls_negative", "ReplicatedMergeTree('/clickhouse/replicated_test_alter_multiple_ttls_negative', '1')", False),
+])
+def test_alter_multiple_ttls(started_cluster, name, engine, positive):
+    """Copyright 2019, Altinity LTD
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License."""
+    """Check that when multiple TTL expressions are set
+    and before any parts are inserted the TTL expressions
+    are changed with ALTER command then all old
+    TTL expressions are removed and the
+    the parts are moved to the specified disk or volume or
+    deleted if the new TTL expression is triggered
+    and are not moved or deleted when it is not.
+    """
+    now = time.time()
+    try:
+        node1.query("""
+            CREATE TABLE {name} (
+                p1 Int64,
+                s1 String,
+                d1 DateTime
+            ) ENGINE = {engine}
+            ORDER BY tuple()
+            PARTITION BY p1
+            TTL d1 + INTERVAL 30 SECOND TO DISK 'jbod2', 
+                d1 + INTERVAL 60 SECOND TO VOLUME 'external'
+            SETTINGS storage_policy='jbods_with_external', merge_with_ttl_timeout=0
+        """.format(name=name, engine=engine))
+
+        node1.query("""
+            ALTER TABLE {name} MODIFY
+            TTL d1 + INTERVAL 0 SECOND TO DISK 'jbod2', 
+                d1 + INTERVAL 5 SECOND TO VOLUME 'external',
+                d1 + INTERVAL 10 SECOND DELETE
+        """.format(name=name))
+
+        for p in range(3):
+            data = [] # 6MB in total
+            now = time.time()
+            for i in range(2):
+                p1 = p
+                s1 = get_random_string(1024 * 1024) # 1MB
+                d1 = now - 1 if i > 0 or positive else now + 300
+                data.append("({}, '{}', toDateTime({}))".format(p1, s1, d1))
+            node1.query("INSERT INTO {name} (p1, s1, d1) VALUES {values}".format(name=name, values=",".join(data)))
+
+        used_disks = get_used_disks_for_table(node1, name)
+        assert set(used_disks) == {"jbod2"} if positive else {"jbod1", "jbod2"}
+
+        assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["6"]
+
+        time.sleep(5)
+
+        used_disks = get_used_disks_for_table(node1, name)
+        assert set(used_disks) == {"external"} if positive else {"jbod1", "jbod2"}
+
+        assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["6"]
+
+        time.sleep(5)
+
+        node1.query("OPTIMIZE TABLE {name} FINAL".format(name=name))
+
+        assert node1.query("SELECT count() FROM {name}".format(name=name)).splitlines() == ["0"] if positive else ["3"]
+
+    finally:
+        node1.query("DROP TABLE IF EXISTS {name}".format(name=name))

docs/en/operations/settings/settings_profiles.md

@@ -60,7 +60,7 @@ Example:
 
 The example specifies two profiles: `default` and `web`. The `default` profile has a special purpose: it must always be present and is applied when starting the server. In other words, the `default` profile contains default settings. The `web` profile is a regular profile that can be set using the `SET` query or using a URL parameter in an HTTP query.
 
-Settings profiles can inherit from each other. To use inheritance, indicate the `profile` setting before the other settings that are listed in the profile.
+Settings profiles can inherit from each other. To use inheritance, indicate one or multiple `profile` settings before the other settings that are listed in the profile. In case when one setting is defined in different profiles, the latest defined is used.
 
 
 [Original article](https://clickhouse.yandex/docs/en/operations/settings/settings_profiles/) <!--hide-->

docs/ru/operations/settings/settings_profiles.md

@@ -60,6 +60,6 @@ SET profile = 'web'
 
 В примере задано два профиля: `default` и `web`. Профиль `default` имеет специальное значение - он всегда обязан присутствовать и применяется при запуске сервера. То есть, профиль `default` содержит настройки по умолчанию. Профиль `web` - обычный профиль, который может быть установлен с помощью запроса `SET` или с помощью параметра URL при запросе по HTTP.
 
-Профили настроек могут наследоваться от друг-друга - это реализуется указанием настройки `profile` перед остальными настройками, перечисленными в профиле.
+Профили настроек могут наследоваться от друг-друга - это реализуется указанием одной или нескольких настроек `profile` перед остальными настройками, перечисленными в профиле. Если одна настройка указана в нескольких профилях, используется последнее из значений.
 
 [Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/settings/settings_profiles/) <!--hide-->

docs/zh/operations/settings/settings_profiles.md

@@ -1,66 +0,0 @@
-
-# Settings profiles
-
-A settings profile is a collection of settings grouped under the same name. Each ClickHouse user has a profile.
-To apply all the settings in a profile, set the `profile` setting.
-
-Example:
-
-Install the `web` profile.
-
-``` sql
-SET profile = 'web'
-```
-
-Settings profiles are declared in the user config file. This is usually `users.xml`.
-
-Example:
-
-```xml
-<!-- Settings profiles -->
-<profiles>
-    <!-- Default settings -->
-    <default>
-        <!-- The maximum number of threads when running a single query. -->
-        <max_threads>8</max_threads>
-    </default>
-
-    <!-- Settings for quries from the user interface -->
-    <web>
-        <max_rows_to_read>1000000000</max_rows_to_read>
-        <max_bytes_to_read>100000000000</max_bytes_to_read>
-
-        <max_rows_to_group_by>1000000</max_rows_to_group_by>
-        <group_by_overflow_mode>any</group_by_overflow_mode>
-
-        <max_rows_to_sort>1000000</max_rows_to_sort>
-        <max_bytes_to_sort>1000000000</max_bytes_to_sort>
-
-        <max_result_rows>100000</max_result_rows>
-        <max_result_bytes>100000000</max_result_bytes>
-        <result_overflow_mode>break</result_overflow_mode>
-
-        <max_execution_time>600</max_execution_time>
-        <min_execution_speed>1000000</min_execution_speed>
-        <timeout_before_checking_execution_speed>15</timeout_before_checking_execution_speed>
-
-        <max_columns_to_read>25</max_columns_to_read>
-        <max_temporary_columns>100</max_temporary_columns>
-        <max_temporary_non_const_columns>50</max_temporary_non_const_columns>
-
-        <max_subquery_depth>2</max_subquery_depth>
-        <max_pipeline_depth>25</max_pipeline_depth>
-        <max_ast_depth>50</max_ast_depth>
-        <max_ast_elements>100</max_ast_elements>
-
-        <readonly>1</readonly>
-    </web>
-</profiles>
-```
-
-The example specifies two profiles: `default` and `web`. The `default` profile has a special purpose: it must always be present and is applied when starting the server. In other words, the `default` profile contains default settings. The `web` profile is a regular profile that can be set using the `SET` query or using a URL parameter in an HTTP query.
-
-Settings profiles can inherit from each other. To use inheritance, indicate the `profile` setting before the other settings that are listed in the profile.
-
-
-[Original article](https://clickhouse.yandex/docs/en/operations/settings/settings_profiles/) <!--hide-->

docs/zh/operations/settings/settings_profiles.md

@@ -0,0 +1 @@
+../../../en/operations/settings/settings_profiles.md